Changes before error encountered

Co-authored-by: kobenguyent <[email protected]>

Author: Copilot

bin/codecept.js

@@ -165,6 +165,7 @@ program
   .option('--no-timeouts', 'disable all timeouts')
   .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
   .option('--shuffle', 'Shuffle the order in which test files run')
+  .option('--shard <index/total>', 'run only a fraction of tests (e.g., --shard 1/4)')
 
   // mocha options
   .option('--colors', 'force enabling of colors')

docs/parallel.md

@@ -5,13 +5,71 @@ title: Parallel Execution
 
 # Parallel Execution
 
-CodeceptJS has two engines for running tests in parallel:
+CodeceptJS has multiple approaches for running tests in parallel:
 
-* `run-workers` - which spawns [NodeJS Worker](https://nodejs.org/api/worker_threads.html) in a thread. Tests are split by scenarios, scenarios are mixed between groups, each worker runs tests from its own group.
-* `run-multiple` - which spawns a subprocess with CodeceptJS. Tests are split by files and configured in `codecept.conf.js`.
+- **Test Sharding** - distributes tests across multiple machines for CI matrix execution
+- `run-workers` - which spawns [NodeJS Worker](https://nodejs.org/api/worker_threads.html) in a thread. Tests are split by scenarios, scenarios are mixed between groups, each worker runs tests from its own group.
+- `run-multiple` - which spawns a subprocess with CodeceptJS. Tests are split by files and configured in `codecept.conf.js`.
 
 Workers are faster and simpler to start, while `run-multiple` requires additional configuration and can be used to run tests in different browsers at once.
 
+## Test Sharding for CI Matrix
+
+Test sharding allows you to split your test suite across multiple machines or CI workers without manual configuration. This is particularly useful for CI/CD pipelines where you want to run tests in parallel across different machines.
+
+Use the `--shard` option with the `run` command to execute only a portion of your tests:
+
+```bash
+# Run the first quarter of tests
+npx codeceptjs run --shard 1/4
+
+# Run the second quarter of tests
+npx codeceptjs run --shard 2/4
+
+# Run the third quarter of tests
+npx codeceptjs run --shard 3/4
+
+# Run the fourth quarter of tests
+npx codeceptjs run --shard 4/4
+```
+
+### CI Matrix Example
+
+Here's how you can use test sharding with GitHub Actions matrix strategy:
+
+```yaml
+name: Tests
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shard: [1/4, 2/4, 3/4, 4/4]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - run: npx codeceptjs run --shard ${{ matrix.shard }}
+```
+
+This approach ensures:
+
+- Each CI job runs only its assigned portion of tests
+- Tests are distributed evenly across shards
+- No manual configuration or maintenance of test lists
+- Automatic load balancing as you add or remove tests
+
+### Shard Distribution
+
+Tests are distributed evenly across shards using a round-robin approach:
+
+- If you have 100 tests and 4 shards, each shard runs approximately 25 tests
+- The first shard gets tests 1-25, second gets 26-50, third gets 51-75, fourth gets 76-100
+- If tests don't divide evenly, earlier shards may get one extra test
+
 ## Parallel Execution by Workers
 
 It is easy to run tests in parallel if you have a lots of tests and free CPU cores. Just execute your tests using `run-workers` command specifying the number of workers to spawn:
@@ -128,27 +186,27 @@ FAIL  | 7 passed, 1 failed, 1 skipped   // 2s
 CodeceptJS also exposes the env var `process.env.RUNS_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers
 
 ```js
-const { event } = require('codeceptjs');
+const { event } = require('codeceptjs')
 
-module.exports = function() {
-    // this event would trigger the  `_publishResultsToTestrail` when running `run-workers` command
+module.exports = function () {
+  // this event would trigger the  `_publishResultsToTestrail` when running `run-workers` command
   event.dispatcher.on(event.workers.result, async () => {
-    await _publishResultsToTestrail();
-  });
-  
+    await _publishResultsToTestrail()
+  })
+
   // this event would not trigger the  `_publishResultsToTestrail` multiple times when running `run-workers` command
   event.dispatcher.on(event.all.result, async () => {
-      // when running `run` command, this env var is undefined
-    if (!process.env.RUNS_WITH_WORKERS) await _publishResultsToTestrail();
-  });
+    // when running `run` command, this env var is undefined
+    if (!process.env.RUNS_WITH_WORKERS) await _publishResultsToTestrail()
+  })
 }
 ```
 
 ## Parallel Execution by Workers on Multiple Browsers
 
-To run tests in parallel across multiple browsers, modify your `codecept.conf.js` file to configure multiple browsers on which you want to run your tests and your tests will run across multiple browsers. 
+To run tests in parallel across multiple browsers, modify your `codecept.conf.js` file to configure multiple browsers on which you want to run your tests and your tests will run across multiple browsers.
 
-Start with modifying the `codecept.conf.js` file. Add multiple key inside the config which will be used to configure multiple profiles. 
+Start with modifying the `codecept.conf.js` file. Add multiple key inside the config which will be used to configure multiple profiles.
 
 ```
 exports.config = {
@@ -174,7 +232,7 @@ exports.config = {
           }
         }
       ]
-    }, 
+    },
     profile2: {
       browsers: [
         {
@@ -188,16 +246,21 @@ exports.config = {
   }
 };
 ```
-To trigger tests on all the profiles configured, you can use the following command: 
+
+To trigger tests on all the profiles configured, you can use the following command:
+
 ```
 npx codeceptjs run-workers 3 all -c codecept.conf.js
 ```
+
 This will run your tests across all browsers configured from profile1 & profile2 on 3 workers.
 
-To trigger tests on specific profile, you can use the following command: 
+To trigger tests on specific profile, you can use the following command:
+
 ```
 npx codeceptjs run-workers 2 profile1 -c codecept.conf.js
 ```
+
 This will run your tests across 2 browsers from profile1 on 2 workers.
 
 ## Custom Parallel Execution
@@ -221,7 +284,7 @@ Create a placeholder in file:
 
 ```js
 #!/usr/bin/env node
-const { Workers, event } = require('codeceptjs');
+const { Workers, event } = require('codeceptjs')
 // here will go magic
 ```
 
@@ -232,59 +295,59 @@ Now let's see how to update this file for different parallelization modes:
 ```js
 const workerConfig = {
   testConfig: './test/data/sandbox/codecept.customworker.js',
-};
+}
 
 // don't initialize workers in constructor
-const workers = new Workers(null, workerConfig);
+const workers = new Workers(null, workerConfig)
 // split tests by suites in 2 groups
-const testGroups = workers.createGroupsOfSuites(2);
+const testGroups = workers.createGroupsOfSuites(2)
 
-const browsers = ['firefox', 'chrome'];
+const browsers = ['firefox', 'chrome']
 
 const configs = browsers.map(browser => {
   return {
     helpers: {
-      WebDriver: { browser }
-    }
-  };
-});
+      WebDriver: { browser },
+    },
+  }
+})
 
 for (const config of configs) {
   for (group of testGroups) {
-    const worker = workers.spawn();
-    worker.addTests(group);
-    worker.addConfig(config);
+    const worker = workers.spawn()
+    worker.addTests(group)
+    worker.addConfig(config)
   }
 }
 
 // Listen events for failed test
-workers.on(event.test.failed, (failedTest) => {
-  console.log('Failed : ', failedTest.title);
-});
+workers.on(event.test.failed, failedTest => {
+  console.log('Failed : ', failedTest.title)
+})
 
 // Listen events for passed test
-workers.on(event.test.passed, (successTest) => {
-  console.log('Passed : ', successTest.title);
-});
+workers.on(event.test.passed, successTest => {
+  console.log('Passed : ', successTest.title)
+})
 
 // test run status will also be available in event
 workers.on(event.all.result, () => {
   // Use printResults() to display result with standard style
-  workers.printResults();
-});
+  workers.printResults()
+})
 
 // run workers as async function
-runWorkers();
+runWorkers()
 
 async function runWorkers() {
   try {
     // run bootstrapAll
-    await workers.bootstrapAll();
+    await workers.bootstrapAll()
     // run tests
-    await workers.run();
+    await workers.run()
   } finally {
     // run teardown All
-    await workers.teardownAll();
+    await workers.teardownAll()
   }
 }
 ```
@@ -313,7 +376,6 @@ workers.on(event.all.result, (status, completedTests, workerStats) => {
 If you want your tests to split according to your need this method is suited for you. For example: If you have 4 long running test files and 4 normal test files there chance all 4 tests end up in same worker thread. For these cases custom function will be helpful.
 
 ```js
-
 /*
  Define a function to split your tests.
 
@@ -322,28 +384,25 @@ If you want your tests to split according to your need this method is suited for
  where file1 and file2 will run in a worker thread and file3 will run in a worker thread
 */
 const splitTests = () => {
-  const files = [
-    ['./test/data/sandbox/guthub_test.js', './test/data/sandbox/devto_test.js'],
-    ['./test/data/sandbox/longrunnig_test.js']
-  ];
+  const files = [['./test/data/sandbox/guthub_test.js', './test/data/sandbox/devto_test.js'], ['./test/data/sandbox/longrunnig_test.js']]
 
-  return files;
+  return files
 }
 
 const workerConfig = {
   testConfig: './test/data/sandbox/codecept.customworker.js',
-  by: splitTests
-};
+  by: splitTests,
+}
 
 // don't initialize workers in constructor
-const customWorkers = new Workers(null,  workerConfig);
+const customWorkers = new Workers(null, workerConfig)
 
-customWorkers.run();
+customWorkers.run()
 
 // You can use event listeners similar to above example.
 customWorkers.on(event.all.result, () => {
-  workers.printResults();
-});
+  workers.printResults()
+})
 ```
 
 ### Emitting messages to the parent worker
@@ -353,13 +412,13 @@ Child workers can send non-test events to the main process. This is useful if yo
 ```js
 // inside main process
 // listen for any non test related events
-workers.on('message', (data) => {
+workers.on('message', data => {
   console.log(data)
-});
+})
 
 workers.on(event.all.result, (status, completedTests, workerStats) => {
   // logic
-});
+})
 ```
 
 ## Sharing Data Between Workers
@@ -372,12 +431,12 @@ You can share data directly using the `share()` function and access it using `in
 
 ```js
 // In one test or worker
-share({ userData: { name: 'user', password: '123456' } });
+share({ userData: { name: 'user', password: '123456' } })
 
 // In another test or worker
-const testData = inject();
-console.log(testData.userData.name); // 'user'
-console.log(testData.userData.password); // '123456'
+const testData = inject()
+console.log(testData.userData.name) // 'user'
+console.log(testData.userData.password) // '123456'
 ```
 
 ### Initializing Data in Bootstrap
@@ -389,20 +448,20 @@ For complex scenarios where you need to initialize shared data before tests run,
 exports.config = {
   bootstrap() {
     // Initialize shared data container
-    share({ userData: null, config: { retries: 3 } });
-  }
+    share({ userData: null, config: { retries: 3 } })
+  },
 }
 ```
 
 Then in your tests, you can check and update the shared data:
 
 ```js
-const testData = inject();
+const testData = inject()
 if (!testData.userData) {
   // Update shared data - both approaches work:
-  share({ userData: { name: 'user', password: '123456' } });
+  share({ userData: { name: 'user', password: '123456' } })
   // or mutate the injected object:
-  testData.userData = { name: 'user', password: '123456' };
+  testData.userData = { name: 'user', password: '123456' }
 }
 ```
 
@@ -412,30 +471,30 @@ Since CodeceptJS 3.7.0+, shared data uses Proxy objects for synchronization betw
 
 ```js
 // ✅ All of these work correctly:
-const data = inject();
-console.log(data.userData.name);           // Access nested properties
-console.log(Object.keys(data));            // Enumerate shared keys  
-data.newProperty = 'value';                // Add new properties
-Object.assign(data, { more: 'data' });     // Merge objects
+const data = inject()
+console.log(data.userData.name) // Access nested properties
+console.log(Object.keys(data)) // Enumerate shared keys
+data.newProperty = 'value' // Add new properties
+Object.assign(data, { more: 'data' }) // Merge objects
 ```
 
 **Important Note:** Avoid reassigning the entire injected object:
 
 ```js
 // ❌ AVOID: This breaks the proxy reference
-let testData = inject();
-testData = someOtherObject; // This will NOT work as expected!
+let testData = inject()
+testData = someOtherObject // This will NOT work as expected!
 
 // ✅ PREFERRED: Use share() to replace data or mutate properties
-share({ userData: someOtherObject }); // This works!
+share({ userData: someOtherObject }) // This works!
 // or
-Object.assign(inject(), someOtherObject); // This works!
+Object.assign(inject(), someOtherObject) // This works!
 ```
 
 ### Local Data (Worker-Specific)
 
 If you want to share data only within the same worker (not across all workers), use the `local` option:
 
 ```js
-share({ localData: 'worker-specific' }, { local: true });
+share({ localData: 'worker-specific' }, { local: true })
 ```