Skip to content

Commit ea758df

Browse files
Copilotkobenguyent
Copilot
and
kobenguyent
committed
Add comprehensive pool mode acceptance tests - statistics accuracy and edge cases coverage
Co-authored-by: kobenguyent <[email protected]>
1 parent 7a5ece5 commit ea758df

File tree

2 files changed

+254
-19
lines changed

2 files changed

+254
-19
lines changed

‎lib/command/workers/runTests.js‎

Lines changed: 72 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ if (poolMode) {
5757
}
5858
})()
5959

60+
let globalStats = { passes: 0, failures: 0, tests: 0, pending: 0, failedHooks: 0 }
61+
6062
async function runTests() {
6163
try {
6264
await codecept.bootstrap()
@@ -87,6 +89,7 @@ async function runPoolTests() {
8789
let consolidatedStats = { passes: 0, failures: 0, tests: 0, pending: 0, failedHooks: 0 }
8890
let allTests = []
8991
let allFailures = []
92+
let previousStats = { passes: 0, failures: 0, tests: 0, pending: 0, failedHooks: 0 }
9093

9194
// Keep requesting tests until no more available
9295
while (true) {
@@ -103,30 +106,38 @@ async function runPoolTests() {
103106
// In pool mode, we need to create a fresh Mocha instance for each test
104107
// because Mocha instances become disposed after running tests
105108
container.createMocha() // Create fresh Mocha instance
106-
const mocha = container.mocha()
107-
108-
// Filter to run only the assigned test
109109
filterTestById(testUid)
110+
const mocha = container.mocha()
110111

111112
if (mocha.suite.total() > 0) {
112113
// Run the test and complete
113114
await codecept.run()
114115

115-
// Accumulate the results from this test run
116+
// Get the results from this specific test run
116117
const result = container.result()
117-
118-
consolidatedStats.passes += result.stats.passes || 0
119-
consolidatedStats.failures += result.stats.failures || 0
120-
consolidatedStats.tests += result.stats.tests || 0
121-
consolidatedStats.pending += result.stats.pending || 0
122-
consolidatedStats.failedHooks += result.stats.failedHooks || 0
123-
124-
// Add tests and failures to consolidated collections
125-
if (result.tests) {
126-
allTests.push(...result.tests)
127-
}
128-
if (result.failures) {
129-
allFailures.push(...result.failures)
118+
const currentStats = result.stats || {}
119+
120+
// Calculate the difference from previous accumulated stats
121+
const newPasses = Math.max(0, (currentStats.passes || 0) - previousStats.passes)
122+
const newFailures = Math.max(0, (currentStats.failures || 0) - previousStats.failures)
123+
const newTests = Math.max(0, (currentStats.tests || 0) - previousStats.tests)
124+
const newPending = Math.max(0, (currentStats.pending || 0) - previousStats.pending)
125+
const newFailedHooks = Math.max(0, (currentStats.failedHooks || 0) - previousStats.failedHooks)
126+
127+
// Add only the new results
128+
consolidatedStats.passes += newPasses
129+
consolidatedStats.failures += newFailures
130+
consolidatedStats.tests += newTests
131+
consolidatedStats.pending += newPending
132+
consolidatedStats.failedHooks += newFailedHooks
133+
134+
// Update previous stats for next comparison
135+
previousStats = { ...currentStats }
136+
137+
// Add new failures to consolidated collections
138+
if (result.failures && result.failures.length > allFailures.length) {
139+
const newFailures = result.failures.slice(allFailures.length)
140+
allFailures.push(...newFailures)
130141
}
131142
}
132143

@@ -202,9 +213,51 @@ function filterTestById(testUid) {
202213
mocha.files = files
203214
mocha.loadFiles()
204215

205-
// Now filter to only the target test
216+
// Now filter to only the target test - use a more robust approach
217+
let foundTest = false
206218
for (const suite of mocha.suite.suites) {
207-
suite.tests = suite.tests.filter(test => test.uid === testUid)
219+
const originalTests = [...suite.tests]
220+
suite.tests = []
221+
222+
for (const test of originalTests) {
223+
if (test.uid === testUid) {
224+
suite.tests.push(test)
225+
foundTest = true
226+
break // Only add one matching test
227+
}
228+
}
229+
230+
// If no tests found in this suite, remove it
231+
if (suite.tests.length === 0) {
232+
suite.parent.suites = suite.parent.suites.filter(s => s !== suite)
233+
}
234+
}
235+
236+
// Filter out empty suites from the root
237+
mocha.suite.suites = mocha.suite.suites.filter(suite => suite.tests.length > 0)
238+
239+
if (!foundTest) {
240+
// If testUid doesn't match, maybe it's a simple test name - try fallback
241+
mocha.suite.suites = []
242+
mocha.suite.tests = []
243+
mocha.loadFiles()
244+
245+
// Try matching by title
246+
for (const suite of mocha.suite.suites) {
247+
const originalTests = [...suite.tests]
248+
suite.tests = []
249+
250+
for (const test of originalTests) {
251+
if (test.title === testUid || test.fullTitle() === testUid || test.uid === testUid) {
252+
suite.tests.push(test)
253+
foundTest = true
254+
break
255+
}
256+
}
257+
}
258+
259+
// Clean up empty suites again
260+
mocha.suite.suites = mocha.suite.suites.filter(suite => suite.tests.length > 0)
208261
}
209262
}
210263

‎test/runner/run_workers_test.js‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,4 +330,186 @@ describe('CodeceptJS Workers Runner', function () {
330330
done()
331331
})
332332
})
333+
334+
it('should report accurate test statistics in pool mode', function (done) {
335+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
336+
// Run regular workers mode first to get baseline counts
337+
exec(`${codecept_run} 2`, (err, stdout) => {
338+
const regularStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/)
339+
if (!regularStats) return done(new Error('Could not parse regular mode statistics'))
340+
341+
const expectedPassed = parseInt(regularStats[2])
342+
const expectedFailed = parseInt(regularStats[3] || '0')
343+
const expectedFailedHooks = parseInt(regularStats[4] || '0')
344+
345+
// Now run pool mode and compare
346+
exec(`${codecept_run} 2 --by pool`, (err2, stdout2) => {
347+
expect(stdout2).toContain('CodeceptJS')
348+
expect(stdout2).toContain('Running tests in 2 workers')
349+
350+
// Extract pool mode statistics
351+
const poolStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/)
352+
expect(poolStats).toBeTruthy()
353+
354+
const actualPassed = parseInt(poolStats[2])
355+
const actualFailed = parseInt(poolStats[3] || '0')
356+
const actualFailedHooks = parseInt(poolStats[4] || '0')
357+
358+
// Pool mode should report exactly the same statistics as regular mode
359+
expect(actualPassed).toEqual(expectedPassed)
360+
expect(actualFailed).toEqual(expectedFailed)
361+
expect(actualFailedHooks).toEqual(expectedFailedHooks)
362+
363+
// Both should have same exit code
364+
expect(err?.code || 0).toEqual(err2?.code || 0)
365+
done()
366+
})
367+
})
368+
})
369+
370+
it('should report correct test counts with grep filtering in pool mode', function (done) {
371+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
372+
// Run regular workers mode with grep first
373+
exec(`${codecept_run} 2 --grep "grep"`, (err, stdout) => {
374+
const regularStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/)
375+
if (!regularStats) return done(new Error('Could not parse regular mode grep statistics'))
376+
377+
const expectedPassed = parseInt(regularStats[2])
378+
const expectedFailed = parseInt(regularStats[3] || '0')
379+
380+
// Now run pool mode with grep and compare
381+
exec(`${codecept_run} 2 --by pool --grep "grep"`, (err2, stdout2) => {
382+
const poolStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/)
383+
expect(poolStats).toBeTruthy()
384+
385+
const actualPassed = parseInt(poolStats[2])
386+
const actualFailed = parseInt(poolStats[3] || '0')
387+
388+
// Should match exactly
389+
expect(actualPassed).toEqual(expectedPassed)
390+
expect(actualFailed).toEqual(expectedFailed)
391+
expect(err?.code || 0).toEqual(err2?.code || 0)
392+
done()
393+
})
394+
})
395+
})
396+
397+
it('should handle single vs multiple workers statistics consistently in pool mode', function (done) {
398+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
399+
// Run pool mode with 1 worker
400+
exec(`${codecept_run} 1 --by pool --grep "grep"`, (err, stdout) => {
401+
const singleStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/)
402+
if (!singleStats) return done(new Error('Could not parse single worker statistics'))
403+
404+
const singlePassed = parseInt(singleStats[2])
405+
const singleFailed = parseInt(singleStats[3] || '0')
406+
407+
// Run pool mode with multiple workers
408+
exec(`${codecept_run} 3 --by pool --grep "grep"`, (err2, stdout2) => {
409+
const multiStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/)
410+
expect(multiStats).toBeTruthy()
411+
412+
const multiPassed = parseInt(multiStats[2])
413+
const multiFailed = parseInt(multiStats[3] || '0')
414+
415+
// Statistics should be identical regardless of worker count
416+
expect(multiPassed).toEqual(singlePassed)
417+
expect(multiFailed).toEqual(singleFailed)
418+
expect(err?.code || 0).toEqual(err2?.code || 0)
419+
done()
420+
})
421+
})
422+
})
423+
424+
it('should exit with correct code in pool mode for failing tests', function (done) {
425+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
426+
exec(`${codecept_run} 2 --by pool --grep "Workers Failing"`, (err, stdout) => {
427+
expect(stdout).toContain('CodeceptJS')
428+
expect(stdout).toContain('Running tests in 2 workers')
429+
expect(stdout).toContain('FAILURES')
430+
expect(stdout).toContain('worker has failed')
431+
expect(stdout).toContain('FAIL | 0 passed, 1 failed')
432+
expect(err.code).toEqual(1) // Should exit with failure code
433+
done()
434+
})
435+
})
436+
437+
it('should exit with correct code in pool mode for passing tests', function (done) {
438+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
439+
exec(`${codecept_run} 2 --by pool --grep "grep"`, (err, stdout) => {
440+
expect(stdout).toContain('CodeceptJS')
441+
expect(stdout).toContain('Running tests in 2 workers')
442+
expect(stdout).toContain('OK | 2 passed')
443+
expect(err).toEqual(null) // Should exit with success code (0)
444+
done()
445+
})
446+
})
447+
448+
it('should accurately count tests with mixed results in pool mode', function (done) {
449+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
450+
// Use a specific test that has mixed results
451+
exec(`${codecept_run} 2 --by pool --grep "Workers|retry"`, (err, stdout) => {
452+
expect(stdout).toContain('CodeceptJS')
453+
expect(stdout).toContain('Running tests in 2 workers')
454+
455+
// Should have some passing and some failing tests
456+
const stats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/)
457+
expect(stats).toBeTruthy()
458+
459+
const passed = parseInt(stats[2])
460+
const failed = parseInt(stats[3] || '0')
461+
const failedHooks = parseInt(stats[4] || '0')
462+
463+
// Should have at least some passing and failing
464+
expect(passed).toBeGreaterThan(0)
465+
expect(failed + failedHooks).toBeGreaterThan(0)
466+
expect(err.code).toEqual(1) // Should fail due to failures
467+
done()
468+
})
469+
})
470+
471+
it('should maintain consistency across multiple pool mode runs', function (done) {
472+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
473+
// Run pool mode first time
474+
exec(`${codecept_run} 2 --by pool --grep "grep"`, (err, stdout) => {
475+
const firstStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/)
476+
if (!firstStats) return done(new Error('Could not parse first run statistics'))
477+
478+
const firstPassed = parseInt(firstStats[2])
479+
const firstFailed = parseInt(firstStats[3] || '0')
480+
481+
// Run pool mode second time
482+
exec(`${codecept_run} 2 --by pool --grep "grep"`, (err2, stdout2) => {
483+
const secondStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/)
484+
expect(secondStats).toBeTruthy()
485+
486+
const secondPassed = parseInt(secondStats[2])
487+
const secondFailed = parseInt(secondStats[3] || '0')
488+
489+
// Results should be consistent across runs
490+
expect(secondPassed).toEqual(firstPassed)
491+
expect(secondFailed).toEqual(firstFailed)
492+
expect(err?.code || 0).toEqual(err2?.code || 0)
493+
done()
494+
})
495+
})
496+
})
497+
498+
it('should handle large worker count without inflating statistics', function (done) {
499+
if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
500+
// Test with more workers than tests to ensure no inflation
501+
exec(`${codecept_run} 8 --by pool --grep "grep"`, (err, stdout) => {
502+
expect(stdout).toContain('CodeceptJS')
503+
expect(stdout).toContain('Running tests in 8 workers')
504+
505+
const stats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/)
506+
expect(stats).toBeTruthy()
507+
508+
const passed = parseInt(stats[2])
509+
// Should only be 2 tests matching "grep", not more due to worker count
510+
expect(passed).toEqual(2)
511+
expect(err).toEqual(null)
512+
done()
513+
})
514+
})
333515
})

0 commit comments

Comments
 (0)