move subset regular expressions to a separate file (#1690)

josharian · pokey · AndreasArvidsson · web-flow · commit c2e9c0024180 · 2023-08-14T13:52:43.000Z
This enables us to ask git not to [track changes to that file. And while we're here, support multiple regular expressions. ## Checklist - [/] I have added [tests](https://www.cursorless.org/docs/contributing/test-case-recorder/) - [x] I have updated the [docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and [cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet) - [/] I have not broken the cheatsheet --------- Co-authored-by: Pokey Rule <755842+pokey@users.noreply.github.com> Co-authored-by: Andreas Arvidsson <andreas.arvidsson87@gmail.com>
diff --git a/.gitignore b/.gitignore
@@ -39,3 +39,6 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# test subset config
+packages/test-harness/testSubsetGrep.properties
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -44,29 +44,33 @@
       ]
     },
     {
-      "name": "Talon tests",
-      "type": "node",
+      "name": "Extension tests subset",
+      "type": "extensionHost",
       "request": "launch",
-      "program": "${workspaceFolder}/packages/test-harness/out/scripts/runTalonTests",
       "env": {
         "CURSORLESS_TEST": "true",
+        "CURSORLESS_RUN_TEST_SUBSET": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
+      "args": [
+        "--profile=cursorlessDevelopment",
+        "--extensionDevelopmentPath=${workspaceFolder}/packages/cursorless-vscode/dist",
+        "--extensionTestsPath=${workspaceFolder}/packages/test-harness/out/runners/extensionTests"
+      ],
       "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
-      "preLaunchTask": "${defaultBuildTask}",
+      "preLaunchTask": "Prepare test subset",
       "resolveSourceMapLocations": [
         "${workspaceFolder}/**",
         "!**/node_modules/**"
       ]
     },
     {
-      "name": "Talon tests subset",
+      "name": "Talon tests",
       "type": "node",
       "request": "launch",
       "program": "${workspaceFolder}/packages/test-harness/out/scripts/runTalonTests",
       "env": {
         "CURSORLESS_TEST": "true",
-        "CURSORLESS_RUN_TEST_SUBSET": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
       "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
@@ -77,12 +81,13 @@
       ]
     },
     {
-      "name": "Unit tests only",
+      "name": "Talon tests subset",
       "type": "node",
       "request": "launch",
-      "program": "${workspaceFolder}/packages/test-harness/out/scripts/runUnitTestsOnly",
+      "program": "${workspaceFolder}/packages/test-harness/out/scripts/runTalonTests",
       "env": {
         "CURSORLESS_TEST": "true",
+        "CURSORLESS_RUN_TEST_SUBSET": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
       "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
@@ -93,19 +98,14 @@
       ]
     },
     {
-      "name": "Run test subset",
-      "type": "extensionHost",
+      "name": "Unit tests only",
+      "type": "node",
       "request": "launch",
+      "program": "${workspaceFolder}/packages/test-harness/out/scripts/runUnitTestsOnly",
       "env": {
         "CURSORLESS_TEST": "true",
-        "CURSORLESS_RUN_TEST_SUBSET": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
-      "args": [
-        "--profile=cursorlessDevelopment",
-        "--extensionDevelopmentPath=${workspaceFolder}/packages/cursorless-vscode/dist",
-        "--extensionTestsPath=${workspaceFolder}/packages/test-harness/out/runners/extensionTests"
-      ],
       "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
       "preLaunchTask": "${defaultBuildTask}",
       "resolveSourceMapLocations": [
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -46,6 +46,33 @@
       },
       "group": "build"
     },
+    {
+      "label": "Ensure test subset file exists",
+      "type": "npm",
+      "script": "generate-test-subset-file-strict",
+      "path": "packages/test-harness",
+      "presentation": {
+        "reveal": "silent"
+      },
+      "group": "test"
+    },
+    {
+      "label": "Generate test subset file",
+      "type": "npm",
+      "script": "generate-test-subset-file",
+      "path": "packages/test-harness",
+      "presentation": {
+        "reveal": "silent"
+      },
+      "group": "none",
+      "problemMatcher": []
+    },
+    {
+      "label": "Prepare test subset",
+      "dependsOn": ["Ensure test subset file exists", "Build"],
+      "dependsOrder": "sequence",
+      "group": "test"
+    },
     {
       "type": "npm",
       "script": "watch",
diff --git a/cursorless-talon-dev/src/cursorless_dev_vscode.talon b/cursorless-talon-dev/src/cursorless_dev_vscode.talon
@@ -11,3 +11,8 @@ app: vscode
     user.vscode("workbench.action.tasks.terminate")
     insert("Serve")
     key(enter)
+
+^debug generate subset$:
+    user.vscode_with_plugin("workbench.action.tasks.runTask", "Generate test subset file")
+^debug edit subset$:
+    user.vscode_with_plugin("commands.openFolder", "./packages/test-harness/testSubsetGrep.properties")
diff --git a/docs/contributing/CONTRIBUTING.md b/docs/contributing/CONTRIBUTING.md
@@ -56,7 +56,13 @@ Extension" or "Extension Tests".
 
 ### Running a subset of tests
 
-The entire test suite takes a little while to run (1-2 mins), so if you'd like to run just a subset of the tests, you can edit the constant in [`runTestSubset`](../../packages/common/src/testUtil/runTestSubset.ts) to a string supported by [mocha grep](https://mochajs.org/#-grep-regexp-g-regexp) and use the "Run Test Subset" launch config instead of the usual "Extension Tests".
+The entire test suite takes a little while to run (1-2 mins), so you may want to run just a subset of the tests.
+
+To specify the set of tests to run, say `"debug generate subset"`, or if you haven't installed the cursorless-talon-dev files, run the VSCode task "Generate test subset file". To do this, choose "Tasks: Run Task" from the command pallete.
+
+To run the specified subset of tests, use the "Run test subset" launch config instead of the usual "Extension Tests".
+
+If you need to edit the subset of tests to run in the future, you can say `"debug edit subset"`.
 
 ## Code formatting
 
diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts
@@ -1,10 +1,6 @@
 export * from "./cursorlessCommandIds";
 export * from "./testUtil/extractTargetedMarks";
 export { default as FakeIDE } from "./ide/fake/FakeIDE";
-export {
-  runTestSubset,
-  TEST_SUBSET_GREP_STRING,
-} from "./testUtil/runTestSubset";
 export { Message } from "./ide/spy/SpyMessages";
 export { SpyIDERecordedValues } from "./ide/spy/SpyIDE";
 export { default as SpyIDE } from "./ide/spy/SpyIDE";
diff --git a/packages/common/src/testUtil/runTestSubset.ts b/packages/common/src/testUtil/runTestSubset.ts
diff --git a/packages/test-harness/package.json b/packages/test-harness/package.json
@@ -8,7 +8,9 @@
     "test": "env CURSORLESS_TEST=true tsx --conditions=cursorless:bundler src/scripts/runTestsCI.ts",
     "compile": "tsc --build",
     "watch": "tsc --build --watch",
-    "clean": "rm -rf ./out tsconfig.tsbuildinfo"
+    "clean": "rm -rf ./out tsconfig.tsbuildinfo",
+    "generate-test-subset-file": "tsx --conditions=cursorless:bundler ./src/scripts/generateTestSubsetFile.ts --always-open",
+    "generate-test-subset-file-strict": "tsx --conditions=cursorless:bundler ./src/scripts/generateTestSubsetFile.ts --fail-if-not-exists"
   },
   "keywords": [],
   "author": "",
diff --git a/packages/test-harness/src/launchVscodeAndRunTests.ts b/packages/test-harness/src/launchVscodeAndRunTests.ts
diff --git a/packages/test-harness/src/runAllTests.ts b/packages/test-harness/src/runAllTests.ts
@@ -1,12 +1,9 @@
 import * as globRaw from "glob";
 import * as Mocha from "mocha";
 import * as path from "path";
-import {
-  getCursorlessRepoRoot,
-  runTestSubset,
-  TEST_SUBSET_GREP_STRING,
-} from "@cursorless/common";
+import { getCursorlessRepoRoot } from "@cursorless/common";
 import { promisify } from "util";
+import { runTestSubset, testSubsetGrepString } from "./testSubset";
 
 const glob = promisify(globRaw);
 
@@ -50,7 +47,7 @@ async function runTestsInDir(
   const mocha = new Mocha({
     ui: "tdd",
     color: true,
-    grep: runTestSubset() ? TEST_SUBSET_GREP_STRING : undefined, // Only run a subset of tests
+    grep: runTestSubset() ? testSubsetGrepString() : undefined, // Only run a subset of tests
   });
 
   const files = filterFiles(await glob("**/**.test.js", { cwd: testRoot }));
diff --git a/packages/test-harness/src/runners/extensionTests.ts b/packages/test-harness/src/runners/extensionTests.ts
@@ -1,4 +1,4 @@
-import { TestType, runAllTests } from "../util/runAllTests";
+import { TestType, runAllTests } from "../runAllTests";
 
 /**
  * Runs all extension tests.  This function should only be called via the
diff --git a/packages/test-harness/src/scripts/generateTestSubsetFile.ts b/packages/test-harness/src/scripts/generateTestSubsetFile.ts
@@ -0,0 +1,40 @@
+import * as child from "child_process";
+import * as fs from "fs";
+import { testSubsetFilePath } from "../testSubset";
+
+const TEMPLATE = `# This file contains the grep strings to pass to Mocha when running a subset of tests.
+# These grep strings will be used with the "Run test subset" launch configuration.
+# See https://mochajs.org/#-grep-regexp-g-regexp for supported syntax.
+#
+# One regular expression per line.
+# Tests matching any of the regular expressions will be run.
+
+snippets
+languages/go/
+`;
+
+/**
+ * Generates the testSubsetGrep.properties file if it doesn't exist. If the
+ * --fail-if-not-exists flag is passed, the script will fail if the file doesn't
+ * exist.
+ */
+function run() {
+  const testSubsetGrepPath = testSubsetFilePath();
+
+  const exists = fs.existsSync(testSubsetGrepPath);
+
+  if (!exists) {
+    fs.writeFileSync(testSubsetGrepPath, TEMPLATE);
+  }
+
+  if (!exists || process.argv.includes("--always-open")) {
+    child.execSync(`code ${testSubsetGrepPath}`);
+  }
+
+  if (!exists && process.argv.includes("--fail-if-not-exists")) {
+    console.log(`Please edit ${testSubsetGrepPath} and re-run.`);
+    process.exit(1);
+  }
+}
+
+run();
diff --git a/packages/test-harness/src/scripts/runTalonTests.ts b/packages/test-harness/src/scripts/runTalonTests.ts
@@ -1,6 +1,6 @@
 /**
  * Runs all Talon tests.
  */
-import { TestType, runAllTests } from "../util/runAllTests";
+import { TestType, runAllTests } from "../runAllTests";
 
 runAllTests(TestType.talon);
diff --git a/packages/test-harness/src/scripts/runTestsCI.ts b/packages/test-harness/src/scripts/runTestsCI.ts
@@ -4,7 +4,7 @@
 
 import { getCursorlessRepoRoot } from "@cursorless/common";
 import * as path from "path";
-import { launchVscodeAndRunTests } from "../util/launchVscodeAndRunTests";
+import { launchVscodeAndRunTests } from "../launchVscodeAndRunTests";
 
 (async () => {
   // Note that we run all extension tests, including unit tests, in VSCode, even though
diff --git a/packages/test-harness/src/scripts/runUnitTestsOnly.ts b/packages/test-harness/src/scripts/runUnitTestsOnly.ts
@@ -1,6 +1,6 @@
 /**
  * Runs all tests that don't have to be run within VSCode.
  */
-import { TestType, runAllTests } from "../util/runAllTests";
+import { TestType, runAllTests } from "../runAllTests";
 
 runAllTests(TestType.unit);
diff --git a/packages/test-harness/src/testSubset.ts b/packages/test-harness/src/testSubset.ts
@@ -0,0 +1,39 @@
+import * as fs from "fs";
+import * as path from "path";
+import { getCursorlessRepoRoot } from "@cursorless/common";
+
+/**
+ * Returns the grep string to pass to Mocha when running a subset of tests.
+ * @returns the grep string to pass to Mocha:
+ */
+export function testSubsetGrepString(): string {
+  const inFile = testSubsetFilePath();
+  return fs
+    .readFileSync(inFile, "utf-8")
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0 && !line.startsWith("#"))
+    .join("|");
+}
+
+/**
+ * Returns the path of the test subset file.
+ * @returns the path of the test subset file
+ */
+export function testSubsetFilePath() {
+  return path.join(
+    getCursorlessRepoRoot(),
+    "packages",
+    "test-harness",
+    "testSubsetGrep.properties",
+  );
+}
+
+/**
+ * Determine whether we should run just the subset of the tests specified by
+ * {@link TEST_SUBSET_GREP_STRING}.
+ * @returns `true` if we are using the run test subset launch config
+ */
+export function runTestSubset() {
+  return process.env.CURSORLESS_RUN_TEST_SUBSET === "true";
+}
