diff --git a/.gitignore b/.gitignore index a4542ab1403..351e1c08440 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ yarn-error.log testem.log /typings TESTS-**.xml +polluter-runner.spec.ts # System Files .DS_Store diff --git a/package.json b/package.json index 169ead2725f..10a27530733 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,8 @@ "postinstall": "gulp copyGitHooks", "cypress:open": "cypress open --config-file=cypress.config.ts", "cypress:run": "cypress run --config-file=cypress.config.ts", - "serve:ssr:ssr-test": "node dist/ssr-test/server/server.mjs" + "serve:ssr:ssr-test": "node dist/ssr-test/server/server.mjs", + "polluter:bisect": "node scripts/test-polluter-bisect.js" }, "private": true, "dependencies": { diff --git a/scripts/test-polluter-bisect/README.md b/scripts/test-polluter-bisect/README.md new file mode 100644 index 00000000000..97dac9eabc8 --- /dev/null +++ b/scripts/test-polluter-bisect/README.md @@ -0,0 +1,35 @@ +## Description +Sometimes tests from the `igniteui-angular` project fail when run as part of the full suite, even though they pass when executed in isolation. +The reason is that some tests pollute the testing environment (e.g. by leaving behind global state or async handles), which causes later tests to behave incorrectly. + +With a suite of ~200 test files and more than 5000 tests, it’s not feasible to manually track down the culprit file. + +This PR introduces a polluter-bisect script that streamlines the process of identifying polluting tests by: +- Allowing you to specify a sentinel test file (the test known to fail in polluted environments). +- Running a binary search over the rest of the suite to narrow down the minimal set of files that trigger the sentinel failure. +- Supporting two modes: + - `before` - only considers tests that run before the sentinel. (default) + - `all` - considers all tests in the suite and runs the sentinel last. +- Providing a flag to optionally skip the initial full-set scan if you already know the sentinel fails in the suite. +- Generating a temporary polluter-runner spec file to enforce deterministic execution order (bypassing Karma’s automatic sorting). + +This makes it possible to isolate polluting test files much faster than running the entire suite repeatedly. +The script is not intended to run in CI; it’s a developer tool to aid in diagnosing flaky tests. + +## Usage +From the root of the repo, run: + +```bash +# Default: search only in files before the sentinel +npm run polluter:bisect -- sentinel-file.spec.ts before + +# Search across all test files, with sentinel always last +npm run polluter:bisect -- sentinel-file.spec.ts all + +# Skip the initial full-set scan (faster if you already know the sentinel fails in the suite) +npm run polluter:bisect -- sentinel-file.spec.ts before --skip-initial +``` +The script will iteratively run subsets of tests until it identifies the polluting test file that causes the sentinel to fail. + +>NOTE: +> In order for the script to work correctly you should set only a single test executor in `projects/igniteui-angular/karma.conf.js` under `parallelOptions`. diff --git a/scripts/test-polluter-bisect/test-polluter-bisect.js b/scripts/test-polluter-bisect/test-polluter-bisect.js new file mode 100644 index 00000000000..f869a88f193 --- /dev/null +++ b/scripts/test-polluter-bisect/test-polluter-bisect.js @@ -0,0 +1,154 @@ +import { spawn } from 'child_process'; +import fs from "fs"; +import path from "path"; + +main().catch((err) => { + console.error(err); + process.exit(1); +}) + +async function main() { + const allFiles = getAllSpecFiles("projects/igniteui-angular/src/lib"); + + const sentinelArg = process.argv[2]; + const mode = process.argv[3] || "before"; + const skipInitial = process.argv.includes("--skip-initial"); + + if (!sentinelArg) { + console.error("Usage: node test-polluter-bisect.js [before|all]"); + process.exit(1); + } + + const sentinelFile = allFiles.find(f => f.includes(sentinelArg)); + + if (!sentinelFile) { + console.error(`Sentinel file '${sentinelArg}' not found in the test set.`); + process.exit(1); + } + + console.log(`Running polluter search with sentinel: ${sentinelArg}, mode: ${mode}`); + const culprit = await findPolluter(allFiles, sentinelFile, mode, !skipInitial); + + if (culprit) { + console.log(`Polluter file is: ${culprit}`); + } else { + console.log("No polluter found in the set."); + } +} + +async function findPolluter(allFiles, sentinelFile, mode = "before", doInitialScan = true) { + let suspects; + + if (mode === "before") { + suspects = allFiles.slice(0, allFiles.indexOf(sentinelFile)); + } else if (mode === "all") { + suspects = allFiles.filter(f => f !== sentinelFile); + } else { + throw new Error(`Unknown mode: ${mode}`); + } + + if (doInitialScan) { + console.log("Initial run with full set..."); + const initialPass = await runTests([...suspects, sentinelFile], sentinelFile); + + if (initialPass) { + console.log("Sentinel passed even after full set — no polluter detected."); + return null; + } + } else { + console.log("Skipping initial full-set scan."); + } + + while (suspects.length > 1) { + const mid = Math.floor(suspects.length / 2); + const left = suspects.slice(0, mid); + const right = suspects.slice(mid); + + if (await runTests([...left, sentinelFile], sentinelFile)) { + suspects = right; + } else { + suspects = left; + } + } + return suspects[0]; +} + +function runTests(files, sentinelFile) { + return new Promise((resolve) => { + const sentinelNorm = normalizeForNg(sentinelFile); + const runnerFile = createPolluterRunner(files); + + const args = [ + "test", + "igniteui-angular", + "--watch=false", + "--include", + runnerFile + ]; + + let output = ""; + let finished = false; + + const finish = (reason) => { + if (finished) return; + finished = true; + + const sentinelFailed = path.basename(sentinelNorm); + const failed = output.includes("FAILED") && output.includes(sentinelFailed); + console.log(`Sentinel ${sentinelFailed} ${failed ? "FAILED" : "PASSED"} [via ${reason}]`); + resolve(!failed); + + if (!proc.killed) proc.kill(); + }; + + const proc = spawn("npx", ["ng", ...args], { shell: true }); + + proc.stdout.on("data", (data) => { + const text = data.toString(); + output += text; + process.stdout.write(text); + + if (text.includes("TOTAL:")) { + finish("stdout"); + } + }); + proc.stderr.on("data", (data) => { + const text = data.toString(); + output += text; + process.stdout.write(text); + }); + + proc.on("exit", () => { + finish("exit"); + }); + }) +} + +function getAllSpecFiles(dir) { + let files = []; + fs.readdirSync(dir).forEach((file) => { + const full = path.join(dir, file); + if (fs.statSync(full).isDirectory()) { + files = files.concat(getAllSpecFiles(full)); + } else if (file.endsWith(".spec.ts")) { + files.push(full); + } + }); + return files.sort(); +} + +function normalizeForNg(file) { + const rel = path.relative(process.cwd(), file); + return rel.split(path.sep).join("/"); +} + +function createPolluterRunner(files) { + const imports = files.map(f => + `require('${normalizeForNg(f).replace(/\.ts$/, "")}');` + ).join("\n"); + + const runnerPath = path.join(process.cwd(), "projects/igniteui-angular/src/polluter-runner.spec.ts"); + fs.mkdirSync(path.dirname(runnerPath), { recursive: true}); + fs.writeFileSync(runnerPath, imports, "utf8"); + return runnerPath; +} \ No newline at end of file