Merge pull request microsoft#31945 from microsoft/skip-costly-tests

Skip costly tests

Author: sandersn

Gulpfile.js

@@ -429,6 +429,7 @@ task("runtests-parallel").flags = {
     "   --workers=<number>": "The number of parallel workers to use.",
     "   --timeout=<ms>": "Overrides the default test timeout.",
     "   --built": "Compile using the built version of the compiler.",
+    "   --skipPercent=<number>": "Skip expensive tests with <percent> chance to miss an edit. Default 5%.",
 };
 
 task("diff", () => exec(getDiffTool(), [refBaseline, localBaseline], { ignoreExitCode: true }));

package.json

@@ -82,6 +82,7 @@
         "prex": "^0.4.3",
         "q": "latest",
         "remove-internal": "^2.9.2",
+        "simple-git": "^1.113.0",
         "source-map-support": "latest",
         "through2": "latest",
         "travis-fold": "latest",
@@ -102,7 +103,8 @@
         "gulp": "gulp",
         "jake": "gulp",
         "lint": "gulp lint",
-        "setup-hooks": "node scripts/link-hooks.js"
+        "setup-hooks": "node scripts/link-hooks.js",
+        "update-costly-tests": "node scripts/costly-tests.js"
     },
     "browser": {
         "fs": false,

scripts/build/options.js

@@ -14,6 +14,7 @@ module.exports = minimist(process.argv.slice(2), {
         "ru": "runners", "runner": "runners",
         "r": "reporter",
         "c": "colors", "color": "colors",
+        "skip-percent": "skipPercent",
         "w": "workers",
         "f": "fix"
     },
@@ -69,4 +70,4 @@ if (module.exports.built) {
  *
  * @typedef {import("minimist").ParsedArgs & TypedOptions} CommandLineOptions
  */
-void 0;
+void 0;

scripts/build/tests.js

@@ -31,6 +31,7 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
     const inspect = cmdLineOptions.inspect;
     const runners = cmdLineOptions.runners;
     const light = cmdLineOptions.light;
+    const skipPercent = process.env.CI === "true" ? 0 : cmdLineOptions.skipPercent;
     const stackTraceLimit = cmdLineOptions.stackTraceLimit;
     const testConfigFile = "test.config";
     const failed = cmdLineOptions.failed;
@@ -62,8 +63,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
         testTimeout = 400000;
     }
 
-    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
-        writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
+    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined) {
+        writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
     }
 
     const colors = cmdLineOptions.colors;
@@ -158,17 +159,19 @@ exports.cleanTestDirs = cleanTestDirs;
  * @param {string} tests
  * @param {string} runners
  * @param {boolean} light
+ * @param {string} skipPercent
  * @param {string} [taskConfigsFolder]
  * @param {string | number} [workerCount]
  * @param {string} [stackTraceLimit]
  * @param {string | number} [timeout]
  * @param {boolean} [keepFailed]
  */
-function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
+function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
     const testConfigContents = JSON.stringify({
         test: tests ? [tests] : undefined,
         runners: runners ? runners.split(",") : undefined,
         light,
+        skipPercent,
         workerCount,
         stackTraceLimit,
         taskConfigsFolder,

scripts/costly-tests.js

@@ -0,0 +1,103 @@
+// @ts-check
+const fs = require("fs");
+const git = require('simple-git/promise')('.')
+const readline = require('readline')
+
+/** @typedef {{ [s: string]: number}} Histogram */
+
+async function main() {
+    /** @type {Histogram} */
+    const edits = Object.create(null)
+    /** @type {Histogram} */
+    const perf = JSON.parse(fs.readFileSync('.parallelperf.json', 'utf8'))
+
+    await collectCommits(git, "release-2.3", "master", /*author*/ undefined, files => fillMap(files, edits))
+
+    const totalTime = Object.values(perf).reduce((n,m) => n + m, 0)
+    const untouched = Object.values(perf).length - Object.values(edits).length
+    const totalEdits = Object.values(edits).reduce((n,m) => n + m, 0) + untouched + Object.values(edits).length
+
+    let i = 0
+    /** @type {{ name: string, time: number, edits: number, cost: number }[]} */
+    let data = []
+    for (const k in perf) {
+        const otherk = k.replace(/tsrunner-[a-z-]+?:\/\//, '')
+        const percentTime = perf[k] / totalTime
+        const percentHits = (1 + (edits[otherk] || 0)) / totalEdits
+        const cost = 5 + Math.log(percentTime / percentHits)
+        data.push({ name: otherk, time: perf[k], edits: 1 + (edits[otherk] || 0), cost})
+        if (edits[otherk])
+            i++
+    }
+    const output = {
+        totalTime,
+        totalEdits,
+        data: data.sort((x,y) => y.cost - x.cost).map(x => ({ ...x, cost: x.cost.toFixed(2) }))
+    }
+
+    fs.writeFileSync('tests/.test-cost.json', JSON.stringify(output), 'utf8')
+}
+
+main().catch(e => {
+    console.log(e);
+    process.exit(1);
+})
+
+/**
+ * @param {string[]} files
+ * @param {Histogram} histogram
+ */
+function fillMap(files, histogram) {
+    // keep edits to test cases (but not /users), and not file moves
+    const tests = files.filter(f => f.startsWith('tests/cases/') && !f.startsWith('tests/cases/user') && !/=>/.test(f))
+    for (const test of tests) {
+        histogram[test] = (histogram[test] || 0) + 1
+    }
+}
+
+/**
+ * @param {string} s
+ */
+function isSquashMergeMessage(s) {
+    return /\(#[0-9]+\)$/.test(s)
+}
+
+/**
+ * @param {string} s
+ */
+function isMergeCommit(s) {
+    return /Merge pull request #[0-9]+/.test(s)
+}
+
+/**
+ * @param {string} s
+ */
+function parseFiles(s) {
+    const lines = s.split('\n')
+    // Note that slice(2) only works for merge commits, which have an empty newline after the title
+    return lines.slice(2, lines.length - 2).map(line => line.split("|")[0].trim())
+}
+
+/**
+ * @param {import('simple-git/promise').SimpleGit} git
+ * @param {string} from
+ * @param {string} to
+ * @param {string | undefined} author - only include commits from this author
+ * @param {(files: string[]) => void} update
+ */
+ async function collectCommits(git, from, to, author, update) {
+    let i = 0
+    for (const commit of (await git.log({ from, to })).all) {
+        i++
+        if ((!author || commit.author_name === author) && isMergeCommit(commit.message) || isSquashMergeMessage(commit.message)) {
+            readline.clearLine(process.stdout, /*left*/ -1)
+            readline.cursorTo(process.stdout, 0)
+            process.stdout.write(i + ": " + commit.date)
+            const files = parseFiles(await git.show([commit.hash, "--stat=1000,960,40", "--pretty=oneline"]))
+            update(files)
+        }
+    }
+}
+
+
+

src/testRunner/parallel/host.ts

@@ -14,7 +14,7 @@ namespace Harness.Parallel.Host {
         const isatty = tty.isatty(1) && tty.isatty(2);
         const path = require("path") as typeof import("path");
         const { fork } = require("child_process") as typeof import("child_process");
-        const { statSync } = require("fs") as typeof import("fs");
+        const { statSync, readFileSync } = require("fs") as typeof import("fs");
 
         // NOTE: paths for module and types for FailedTestReporter _do not_ line up due to our use of --outFile for run.js
         // tslint:disable-next-line:variable-name
@@ -192,6 +192,31 @@ namespace Harness.Parallel.Host {
             return `tsrunner-${runner}://${test}`;
         }
 
+        function skipCostlyTests(tasks: Task[]) {
+            if (statSync("tests/.test-cost.json")) {
+                const costs = JSON.parse(readFileSync("tests/.test-cost.json", "utf8")) as {
+                    totalTime: number,
+                    totalEdits: number,
+                    data: { name: string, time: number, edits: number, costs: number }[]
+                };
+                let skippedEdits = 0;
+                let skippedTime = 0;
+                const skippedTests = new Set<string>();
+                let i = 0;
+                for (; i < costs.data.length && (skippedEdits / costs.totalEdits) < (skipPercent / 100); i++) {
+                    skippedEdits += costs.data[i].edits;
+                    skippedTime += costs.data[i].time;
+                    skippedTests.add(costs.data[i].name);
+                }
+                console.log(`Skipped ${i} expensive tests; estimated time savings of ${(skippedTime / costs.totalTime * 100).toFixed(2)}% with --skipPercent=${skipPercent.toFixed(2)} chance of missing a test.`);
+                return tasks.filter(t => !skippedTests.has(t.file));
+            }
+            else {
+                console.log("No cost analysis discovered.");
+                return tasks;
+            }
+        }
+
         function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number) {
             console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : "."));
             console.log("Discovering runner-based tests...");
@@ -231,6 +256,7 @@ namespace Harness.Parallel.Host {
             }
             tasks.sort((a, b) => a.size - b.size);
             tasks = tasks.concat(newTasks);
+            tasks = skipCostlyTests(tasks);
             const batchCount = workerCount;
             const packfraction = 0.9;
             const chunkSize = 1000; // ~1KB or 1s for sending batches near the end of a test

src/testRunner/runner.ts

@@ -65,6 +65,7 @@ let runUnitTests: boolean | undefined;
 let stackTraceLimit: number | "full" | undefined;
 let noColors = false;
 let keepFailed = false;
+let skipPercent = 5;
 
 interface TestConfig {
     light?: boolean;
@@ -78,6 +79,7 @@ interface TestConfig {
     noColors?: boolean;
     timeout?: number;
     keepFailed?: boolean;
+    skipPercent?: number;
 }
 
 interface TaskSet {
@@ -109,6 +111,9 @@ function handleTestConfig() {
         if (testConfig.keepFailed) {
             keepFailed = true;
         }
+        if (testConfig.skipPercent !== undefined) {
+            skipPercent = testConfig.skipPercent;
+        }
 
         if (testConfig.stackTraceLimit === "full") {
             (<any>Error).stackTraceLimit = Infinity;