Skip to content

Commit f068326

Browse files
EugeniyKiyashkoEugeniyKiyashko
committed
rerun failed
1 parent 5bcdd96 commit f068326

File tree

7 files changed

+205
-132
lines changed

7 files changed

+205
-132
lines changed

‎e2e/testcafe-devextreme/.testcaferc.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"screenshots": {
33
"path": "./",
44
"takeOnFails": true,
5+
"thumbnails": false,
56
"pathPattern": "/artifacts/failedtests/${TEST}"
67
},
78
"screenshots-comparer": {

‎e2e/testcafe-devextreme/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"mockdate": "3.0.5",
2121
"nconf": "0.12.1",
2222
"testcafe": "3.7.2",
23+
"testcafe-reporter-spec-time": "4.0.0",
2324
"ts-node": "10.9.2",
2425
"eslint": "catalog:",
2526
"@eslint/eslintrc": "catalog:",

‎e2e/testcafe-devextreme/runner.ts‎

Lines changed: 119 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import { getCurrentTheme } from './helpers/themeUtils';
1515

1616
const LAUNCH_RETRY_ATTEMPTS = 5;
1717
const LAUNCH_RETRY_TIMEOUT = 10000;
18+
const FAILED_TESTS_RETRY_ATTEMPTS = 3;
1819

1920
const wait = async (
2021
timeout: number,
@@ -55,6 +56,7 @@ interface ParsedArgs {
5556
shadowDom: boolean;
5657
skipUnstable: boolean;
5758
disableScreenshots: boolean;
59+
retryFailed: boolean;
5860
}
5961

6062
const TESTCAFE_CONFIG: Partial<TestCafeConfigurationOptions> = {
@@ -101,7 +103,7 @@ function getArgs(): ParsedArgs {
101103
concurrency: 0,
102104
browsers: 'chrome',
103105
test: '',
104-
reporter: process.env.CI === 'true' ? 'list' : 'spec',
106+
reporter: 'spec-time',
105107
componentFolder: '',
106108
file: '*',
107109
cache: true,
@@ -112,6 +114,7 @@ function getArgs(): ParsedArgs {
112114
shadowDom: false,
113115
skipUnstable: true,
114116
disableScreenshots: false,
117+
retryFailed: true,
115118
},
116119
}) as ParsedArgs;
117120
}
@@ -157,77 +160,87 @@ async function main() {
157160
// eslint-disable-next-line no-console
158161
console.info('Browsers:', browsers);
159162

160-
const runner: Runner = testCafe.createRunner()
161-
.browsers(browsers)
162-
.reporter(reporter)
163-
.src([`./tests/${componentFolder}/${file}.ts`]);
163+
const failedTests: Set<string> = new Set();
164164

165-
runner.compilerOptions({
166-
typescript: {
167-
customCompilerModulePath: '../../node_modules/typescript',
168-
},
169-
});
170-
171-
runner.concurrency(args.concurrency || 4);
172-
173-
const filters: FilterFunction[] = [];
165+
const createRunner = (filterByFailedTests = false) => {
166+
const runner: Runner = testCafe!.createRunner()
167+
.browsers(browsers)
168+
.reporter(reporter)
169+
.src([`./tests/${componentFolder}/${file}.ts`]);
174170

175-
if (indices) {
176-
const [current, total] = indices.split(/_|of|\\|\//ig).map((x) => +x);
177-
const fixtures = globSync([`./tests/${componentFolder}/*.ts`]);
178-
const fixtureChunks = split(fixtures, total);
179-
const targetFixtureChunk = fixtureChunks[current - 1] ?? [];
180-
const targetFixtureChunkSet = new Set(targetFixtureChunk);
171+
runner.compilerOptions({
172+
typescript: {
173+
customCompilerModulePath: '../../node_modules/typescript',
174+
},
175+
});
181176

182-
/* eslint-disable no-console */
183-
console.info(' === test run config ===');
184-
console.info(` > indices: current = ${current} | total = ${total}`);
185-
console.info(' > glob: ', [`./tests/${componentFolder}/*.ts`]);
186-
console.info(' > all fixtures: ', fixtureChunks);
187-
console.info(' > fixtures: ', targetFixtureChunk, '\n');
188-
/* eslint-enable no-console */
177+
runner.concurrency(args.concurrency || 2);
178+
179+
const filters: FilterFunction[] = [];
180+
181+
if (indices) {
182+
const [current, total] = indices.split(/_|of|\\|\//ig).map((x) => +x);
183+
const fixtures = globSync([`./tests/${componentFolder}/*.ts`]);
184+
const fixtureChunks = split(fixtures, total);
185+
const targetFixtureChunk = fixtureChunks[current - 1] ?? [];
186+
const targetFixtureChunkSet = new Set(targetFixtureChunk);
187+
188+
/* eslint-disable no-console */
189+
console.info(' === test run config ===');
190+
console.info(` > indices: current = ${current} | total = ${total}`);
191+
console.info(' > glob: ', [`./tests/${componentFolder}/*.ts`]);
192+
console.info(' > all fixtures: ', fixtureChunks);
193+
console.info(' > fixtures: ', targetFixtureChunk, '\n');
194+
/* eslint-enable no-console */
195+
196+
filters.push((
197+
_testName: string,
198+
_fixtureName: string,
199+
fixturePath: string,
200+
) => {
201+
const testPath = fixturePath.split('/testcafe-devextreme/')[1];
202+
return targetFixtureChunkSet.has(testPath);
203+
});
204+
}
189205

190-
filters.push((
191-
_testName: string,
192-
_fixtureName: string,
193-
fixturePath: string,
194-
) => {
195-
const testPath = fixturePath.split('/testcafe-devextreme/')[1];
196-
return targetFixtureChunkSet.has(testPath);
197-
});
198-
}
206+
if (testName) {
207+
filters.push((name: string) => name === testName);
208+
}
199209

200-
if (testName) {
201-
filters.push((name: string) => name === testName);
202-
}
210+
if (filterByFailedTests && failedTests.size > 0) {
211+
filters.push((name: string) => failedTests.has(name));
212+
}
203213

204-
if (args.skipUnstable) {
205-
filters.push((
206-
_testName: string,
207-
_fixtureName: string,
208-
_fixturePath: string,
209-
testMeta?: any,
210-
) => !(testMeta)?.unstable);
211-
}
214+
if (args.skipUnstable) {
215+
filters.push((
216+
_testName: string,
217+
_fixtureName: string,
218+
_fixturePath: string,
219+
testMeta?: any,
220+
) => !(testMeta)?.unstable);
221+
}
212222

213-
if (filters.length) {
214-
runner.filter((...filterArgs: Parameters<FilterFunction>) => {
215-
// eslint-disable-next-line @typescript-eslint/prefer-for-of
216-
for (let i = 0; i < filters.length; i += 1) {
217-
if (!filters[i](...filterArgs)) {
218-
return false;
223+
if (filters.length) {
224+
runner.filter((...filterArgs: Parameters<FilterFunction>) => {
225+
// eslint-disable-next-line @typescript-eslint/prefer-for-of
226+
for (let i = 0; i < filters.length; i += 1) {
227+
if (!filters[i](...filterArgs)) {
228+
return false;
229+
}
219230
}
220-
}
221-
return true;
222-
});
223-
}
231+
return true;
232+
});
233+
}
224234

225-
if (args.cache) {
226-
(runner as any).cache = args.cache;
227-
}
235+
if (args.cache) {
236+
(runner as any).cache = args.cache;
237+
}
238+
239+
return runner;
240+
};
228241

229242
const runOptions: RunOptions = {
230-
quarantineMode: { successThreshold: 1, attemptLimit: 2 },
243+
quarantineMode: false,
231244
// @ts-expect-error ts-error
232245
hooks: {
233246
test: {
@@ -266,6 +279,14 @@ async function main() {
266279
},
267280
after: async (t: TestController) => {
268281
await clearTestPage(t);
282+
283+
if (args.retryFailed) {
284+
// @ts-expect-error ts-errors
285+
const { test, errs } = t.testRun;
286+
if (errs && errs.length > 0) {
287+
failedTests.add(test.name);
288+
}
289+
}
269290
},
270291
},
271292
},
@@ -275,7 +296,43 @@ async function main() {
275296
runOptions.disableScreenshots = true;
276297
}
277298

278-
const failedCount = await retry(() => runner.run(runOptions), LAUNCH_RETRY_ATTEMPTS);
299+
// First run - all tests
300+
const runner = createRunner(false);
301+
let failedCount = await retry(() => runner.run(runOptions), LAUNCH_RETRY_ATTEMPTS);
302+
303+
// Retry failed tests if enabled and there are failures
304+
if (args.retryFailed && failedTests.size > 0 && failedCount > 0) {
305+
/* eslint-disable no-console */
306+
console.info('\n');
307+
console.info('='.repeat(60));
308+
console.info(`RETRYING ${failedTests.size} FAILED TEST(S)`);
309+
console.info('='.repeat(60));
310+
console.info('Failed tests:');
311+
failedTests.forEach((failedTestName) => console.info(` - ${failedTestName}`));
312+
console.info('='.repeat(60));
313+
console.info('\n');
314+
/* eslint-enable no-console */
315+
316+
const retryRunner = createRunner(true);
317+
const retryFailedCount = await retry(
318+
() => retryRunner.run(runOptions),
319+
FAILED_TESTS_RETRY_ATTEMPTS,
320+
);
321+
322+
/* eslint-disable no-console */
323+
console.info('\n');
324+
console.info('='.repeat(60));
325+
console.info('RETRY RESULTS');
326+
console.info('='.repeat(60));
327+
console.info(`Initially failed: ${failedTests.size}`);
328+
console.info(`Still failing after retry: ${retryFailedCount}`);
329+
console.info(`Successfully passed on retry: ${failedTests.size - retryFailedCount}`);
330+
console.info('='.repeat(60));
331+
console.info('\n');
332+
/* eslint-enable no-console */
333+
334+
failedCount = retryFailedCount;
335+
}
279336

280337
await testCafe.close();
281338

‎e2e/testcafe-devextreme/tests/dataGrid/common/keyboardNavigation/keyboardNavigation.functional.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ const CLASS = ClassNames;
2323

2424
const getOnKeyDownCallCount = ClientFunction(() => (window as any).onKeyDownCallCount);
2525

26-
fixture`Keyboard Navigation - common`
26+
fixture.disablePageReloads`Keyboard Navigation - common`
2727
.page(url(__dirname, '../../../container.html'));
2828

2929
test('Changing keyboardNavigation options should not invalidate the entire content (T1197829)', async (t) => {

‎e2e/testcafe-devextreme/tests/editors/loadIndIcator/common.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import { createScreenshotsComparer } from 'devextreme-screenshot-comparer';
2-
import LoadIndicator from 'devextreme-testcafe-models/loadindicator';
32
import url from '../../../helpers/getPageUrl';
43
import { createWidget } from '../../../helpers/createWidget';
54
import { testScreenshot } from '../../../helpers/themeUtils';

‎e2e/testcafe-devextreme/tests/editors/textBox/validationMessage.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ test.meta({ browserSize: [300, 200] })('Validation Message position should be co
2121
.click(Selector(`.${TEXTEDITOR_INPUT_CLASS}`))
2222
.pressKey('backspace')
2323
.pressKey('enter')
24-
.click('focusable-start');
24+
.click(Selector('#focusable-start'));
2525

2626
await setAttribute('#container', 'hidden', 'true');
2727
await removeAttribute('#container', 'hidden');

0 commit comments

Comments
 (0)