rerun failed

Author: EugeniyKiyashko

.github/workflows/testcafe_tests.yml

@@ -54,19 +54,21 @@ jobs:
       shell: bash
       run: |
         echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+        pnpm config set network-concurrency 16
 
     - uses: actions/cache@v4
       name: Setup pnpm cache
       with:
         path: |
           ${{ env.STORE_PATH }}
           .nx/cache
+          node_modules/.cache
         key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
         restore-keys: |
           ${{ runner.os }}-pnpm-store
 
     - name: Install dependencies
-      run: pnpm install --frozen-lockfile
+      run: pnpm install --frozen-lockfile --prefer-offline
 
     - name: Build
       shell: bash
@@ -157,19 +159,21 @@ jobs:
       shell: bash
       run: |
         echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+        pnpm config set network-concurrency 16
 
     - uses: actions/cache@v4
       name: Setup pnpm cache
       with:
         path: |
           ${{ env.STORE_PATH }}
           .nx/cache
+          node_modules/.cache
         key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
         restore-keys: |
           ${{ runner.os }}-pnpm-store
 
     - name: Install dependencies
-      run: pnpm install --frozen-lockfile
+      run: pnpm install --frozen-lockfile --prefer-offline
 
     - name: Run TestCafe tests
       working-directory: ./e2e/testcafe-devextreme

e2e/testcafe-devextreme/.testcaferc.json

@@ -2,6 +2,7 @@
   "screenshots": {
     "path": "./",
     "takeOnFails": true,
+    "thumbnails": false,
     "pathPattern": "/artifacts/failedtests/${TEST}"
   },
   "screenshots-comparer": {

e2e/testcafe-devextreme/package.json

@@ -20,6 +20,7 @@
     "mockdate": "3.0.5",
     "nconf": "0.12.1",
     "testcafe": "3.7.2",
+    "testcafe-reporter-spec-time": "4.0.0",
     "ts-node": "10.9.2",
     "eslint": "catalog:",
     "@eslint/eslintrc": "catalog:",

e2e/testcafe-devextreme/runner.ts

@@ -15,6 +15,7 @@ import { getCurrentTheme } from './helpers/themeUtils';
 
 const LAUNCH_RETRY_ATTEMPTS = 5;
 const LAUNCH_RETRY_TIMEOUT = 10000;
+const FAILED_TESTS_RETRY_ATTEMPTS = 3;
 
 const wait = async (
   timeout: number,
@@ -55,6 +56,7 @@ interface ParsedArgs {
   shadowDom: boolean;
   skipUnstable: boolean;
   disableScreenshots: boolean;
+  retryFailed: boolean;
 }
 
 const TESTCAFE_CONFIG: Partial<TestCafeConfigurationOptions> = {
@@ -101,7 +103,7 @@ function getArgs(): ParsedArgs {
       concurrency: 0,
       browsers: 'chrome',
       test: '',
-      reporter: process.env.CI === 'true' ? 'list' : 'spec',
+      reporter: 'spec-time',
       componentFolder: '',
       file: '*',
       cache: true,
@@ -112,6 +114,7 @@ function getArgs(): ParsedArgs {
       shadowDom: false,
       skipUnstable: true,
       disableScreenshots: false,
+      retryFailed: true,
     },
   }) as ParsedArgs;
 }
@@ -157,77 +160,89 @@ async function main() {
     // eslint-disable-next-line no-console
     console.info('Browsers:', browsers);
 
-    const runner: Runner = testCafe.createRunner()
-      .browsers(browsers)
-      .reporter(reporter)
-      .src([`./tests/${componentFolder}/${file}.ts`]);
+    const failedTests: Set<string> = new Set();
 
-    runner.compilerOptions({
-      typescript: {
-        customCompilerModulePath: '../../node_modules/typescript',
-      },
-    });
-
-    runner.concurrency(args.concurrency || 4);
+    const createRunner = (filterByFailedTests = false) => {
+      const runner: Runner = testCafe!.createRunner()
+        .browsers(browsers)
+        .reporter(reporter)
+        .src([`./tests/${componentFolder}/${file}.ts`]);
 
-    const filters: FilterFunction[] = [];
+      runner.compilerOptions({
+        typescript: {
+          customCompilerModulePath: '../../node_modules/typescript',
+        },
+      });
 
-    if (indices) {
-      const [current, total] = indices.split(/_|of|\\|\//ig).map((x) => +x);
-      const fixtures = globSync([`./tests/${componentFolder}/*.ts`]);
-      const fixtureChunks = split(fixtures, total);
-      const targetFixtureChunk = fixtureChunks[current - 1] ?? [];
-      const targetFixtureChunkSet = new Set(targetFixtureChunk);
+      runner.concurrency(filterByFailedTests ? 1 : (args.concurrency || 4));
+
+      const filters: FilterFunction[] = [];
+
+      if (indices) {
+        const [current, total] = indices.split(/_|of|\\|\//ig).map((x) => +x);
+        const fixtures = globSync([`./tests/${componentFolder}/*.ts`]);
+        const fixtureChunks = split(fixtures, total);
+        const targetFixtureChunk = fixtureChunks[current - 1] ?? [];
+        const targetFixtureChunkSet = new Set(targetFixtureChunk);
+
+        if (!filterByFailedTests) {
+          /* eslint-disable no-console */
+          console.info(' === test run config ===');
+          console.info(` > indices: current = ${current} | total = ${total}`);
+          console.info(' > glob: ', [`./tests/${componentFolder}/*.ts`]);
+          console.info(' > all fixtures: ', fixtureChunks);
+          console.info(' > fixtures: ', targetFixtureChunk, '\n');
+          /* eslint-enable no-console */
+        }
 
-      /* eslint-disable no-console */
-      console.info(' === test run config ===');
-      console.info(` > indices: current = ${current} | total = ${total}`);
-      console.info(' > glob: ', [`./tests/${componentFolder}/*.ts`]);
-      console.info(' > all fixtures: ', fixtureChunks);
-      console.info(' > fixtures: ', targetFixtureChunk, '\n');
-      /* eslint-enable no-console */
+        filters.push((
+          _testName: string,
+          _fixtureName: string,
+          fixturePath: string,
+        ) => {
+          const testPath = fixturePath.split('/testcafe-devextreme/')[1];
+          return targetFixtureChunkSet.has(testPath);
+        });
+      }
 
-      filters.push((
-        _testName: string,
-        _fixtureName: string,
-        fixturePath: string,
-      ) => {
-        const testPath = fixturePath.split('/testcafe-devextreme/')[1];
-        return targetFixtureChunkSet.has(testPath);
-      });
-    }
+      if (testName) {
+        filters.push((name: string) => name === testName);
+      }
 
-    if (testName) {
-      filters.push((name: string) => name === testName);
-    }
+      if (filterByFailedTests && failedTests.size > 0) {
+        filters.push((name: string) => failedTests.has(name));
+      }
 
-    if (args.skipUnstable) {
-      filters.push((
-        _testName: string,
-        _fixtureName: string,
-        _fixturePath: string,
-        testMeta?: any,
-      ) => !(testMeta)?.unstable);
-    }
+      if (args.skipUnstable) {
+        filters.push((
+          _testName: string,
+          _fixtureName: string,
+          _fixturePath: string,
+          testMeta?: any,
+        ) => !(testMeta)?.unstable);
+      }
 
-    if (filters.length) {
-      runner.filter((...filterArgs: Parameters<FilterFunction>) => {
-        // eslint-disable-next-line @typescript-eslint/prefer-for-of
-        for (let i = 0; i < filters.length; i += 1) {
-          if (!filters[i](...filterArgs)) {
-            return false;
+      if (filters.length) {
+        runner.filter((...filterArgs: Parameters<FilterFunction>) => {
+          // eslint-disable-next-line @typescript-eslint/prefer-for-of
+          for (let i = 0; i < filters.length; i += 1) {
+            if (!filters[i](...filterArgs)) {
+              return false;
+            }
           }
-        }
-        return true;
-      });
-    }
+          return true;
+        });
+      }
 
-    if (args.cache) {
-      (runner as any).cache = args.cache;
-    }
+      if (args.cache) {
+        (runner as any).cache = args.cache;
+      }
+
+      return runner;
+    };
 
     const runOptions: RunOptions = {
-      quarantineMode: { successThreshold: 1, attemptLimit: 2 },
+      quarantineMode: false,
       // @ts-expect-error ts-error
       hooks: {
         test: {
@@ -266,6 +281,14 @@ async function main() {
           },
           after: async (t: TestController) => {
             await clearTestPage(t);
+
+            if (args.retryFailed) {
+              // @ts-expect-error ts-errors
+              const { test, errs } = t.testRun;
+              if (errs && errs.length > 0) {
+                failedTests.add(test.name);
+              }
+            }
           },
         },
       },
@@ -275,7 +298,75 @@ async function main() {
       runOptions.disableScreenshots = true;
     }
 
-    const failedCount = await retry(() => runner.run(runOptions), LAUNCH_RETRY_ATTEMPTS);
+    // First run - all tests
+    const runner = createRunner(false);
+    let failedCount = await retry(() => runner.run(runOptions), LAUNCH_RETRY_ATTEMPTS);
+
+    // Retry failed tests multiple times if enabled and there are failures
+    if (args.retryFailed && failedTests.size > 0 && failedCount > 0) {
+      const initialFailedCount = failedTests.size;
+      let attemptsLeft = FAILED_TESTS_RETRY_ATTEMPTS;
+
+      while (attemptsLeft > 0 && failedTests.size > 0 && failedCount > 0) {
+        const attemptNumber = FAILED_TESTS_RETRY_ATTEMPTS - attemptsLeft + 1;
+
+        /* eslint-disable no-console */
+        console.info('\n');
+        console.info('='.repeat(60));
+        console.info(`RETRY ATTEMPT ${attemptNumber}/${FAILED_TESTS_RETRY_ATTEMPTS}`);
+        console.info(`Retrying ${failedTests.size} failed test(s)`);
+        console.info('='.repeat(60));
+        console.info('Failed tests:');
+        failedTests.forEach((failedTestName) => console.info(`  - ${failedTestName}`));
+        console.info('='.repeat(60));
+        console.info('\n');
+        /* eslint-enable no-console */
+
+        const testsToRetry = new Set(failedTests);
+        failedTests.clear();
+
+        const retryRunner = createRunner(true);
+        failedCount = await retry(
+          () => retryRunner.run(runOptions),
+          LAUNCH_RETRY_ATTEMPTS,
+        );
+
+        attemptsLeft -= 1;
+
+        /* eslint-disable no-console */
+        console.info('\n');
+        console.info('='.repeat(60));
+        console.info(`ATTEMPT ${attemptNumber} RESULTS`);
+        console.info('='.repeat(60));
+        console.info(`Tests retried: ${testsToRetry.size}`);
+        console.info(`Still failing: ${failedCount}`);
+        console.info(`Passed on this attempt: ${testsToRetry.size - failedCount}`);
+        console.info('='.repeat(60));
+        console.info('\n');
+        /* eslint-enable no-console */
+
+        if (failedCount === 0) {
+          /* eslint-disable no-console */
+          console.info('✅ All previously failed tests now pass!');
+          console.info('\n');
+          /* eslint-enable no-console */
+          break;
+        }
+      }
+
+      /* eslint-disable no-console */
+      console.info('\n');
+      console.info('='.repeat(60));
+      console.info('FINAL RETRY RESULTS');
+      console.info('='.repeat(60));
+      console.info(`Initially failed: ${initialFailedCount}`);
+      console.info(`Total retry attempts used: ${FAILED_TESTS_RETRY_ATTEMPTS - attemptsLeft}`);
+      console.info(`Final failing count: ${failedCount}`);
+      console.info(`Successfully recovered: ${initialFailedCount - failedCount}`);
+      console.info('='.repeat(60));
+      console.info('\n');
+      /* eslint-enable no-console */
+    }
 
     await testCafe.close();
 

e2e/testcafe-devextreme/tests/dataGrid/common/keyboardNavigation/keyboardNavigation.functional.ts

@@ -23,7 +23,7 @@ const CLASS = ClassNames;
 
 const getOnKeyDownCallCount = ClientFunction(() => (window as any).onKeyDownCallCount);
 
-fixture`Keyboard Navigation - common`
+fixture.disablePageReloads`Keyboard Navigation - common`
   .page(url(__dirname, '../../../container.html'));
 
 test('Changing keyboardNavigation options should not invalidate the entire content (T1197829)', async (t) => {

e2e/testcafe-devextreme/tests/dataGrid/common/rowDragging.ts

@@ -369,7 +369,7 @@ test('The cross-component drag and drop rows should not block rows', async (t) =
 
 test('Virtual rendering during auto scrolling should not cause errors in onDragChange', async (t) => {
   const dataGrid = new DataGrid('#container');
-  await t.drag(dataGrid.getDataRow(0).getDragCommand(), 0, 100, { speed: 0.01 });
+  await t.drag(dataGrid.getDataRow(0).getDragCommand(), 0, 100, { speed: 0.1 });
 
   const lastRow = dataGrid.getDataRow(9);
 
@@ -414,7 +414,7 @@ test('Virtual rendering during auto scrolling should not cause errors in onDragC
 // T1078513
 test('Headers should not be hidden during auto scrolling when virtual scrollling is specified', async (t) => {
   const dataGrid = new DataGrid('#container');
-  await t.drag(dataGrid.getDataRow(0).getDragCommand(), 0, 90, { speed: 0.01 });
+  await t.drag(dataGrid.getDataRow(0).getDragCommand(), 0, 90, { speed: 0.1 });
 
   const headerRow = dataGrid.getHeaders().getHeaderRow(0).element;
 
@@ -470,7 +470,7 @@ test('Headers should not be hidden during auto scrolling when virtual scrollling
 // T1078513
 test('Footer should not be hidden during auto scrolling when virtual scrollling is specified', async (t) => {
   const dataGrid = new DataGrid('#container');
-  await t.drag(dataGrid.getDataRow(0).getDragCommand(), 0, 90, { speed: 0.01 });
+  await t.drag(dataGrid.getDataRow(0).getDragCommand(), 0, 90, { speed: 0.1 });
 
   const footerRow = dataGrid.getFooterRow();
 
@@ -578,7 +578,7 @@ test.meta({ unstable: true })('Dragging with scrolling should be prevented by e.
 
   await MouseUpEvents.disable(MouseAction.dragToOffset);
 
-  await t.drag(dataGrid.getDataRow(98).getDragCommand(), 0, -180, { speed: 0.01 });
+  await t.drag(dataGrid.getDataRow(98).getDragCommand(), 0, -180, { speed: 0.1 });
 
   await t.expect(Selector('.dx-sortable-placeholder').visible).notOk();
 

e2e/testcafe-devextreme/tests/editors/loadIndIcator/common.ts

@@ -1,5 +1,4 @@
 import { createScreenshotsComparer } from 'devextreme-screenshot-comparer';
-import LoadIndicator from 'devextreme-testcafe-models/loadindicator';
 import url from '../../../helpers/getPageUrl';
 import { createWidget } from '../../../helpers/createWidget';
 import { testScreenshot } from '../../../helpers/themeUtils';