Log fail test to file went running tests locally (#2809)

The issue suggested showing a pop-up, but this wasn't optimal. It was
either blocking the tests from terminating or it closed immediately by
itself. I also don't want to clobber the clipboard all the time.

Let's log to a file because it seems like the best of the options.

Fixes #2805

---------

Co-authored-by: Phil Cohen <[email protected]>

Author: AndreasArvidsson

.gitignore

@@ -45,6 +45,8 @@ next-env.d.ts
 
 # test subset config
 packages/test-harness/testSubsetGrep.properties
+packages/test-harness/failedTests.properties
+
 
 # cursorless-neovim
 cursorless.nvim/node/cursorless-neovim

.vscode/launch.json

@@ -31,6 +31,7 @@
       "request": "launch",
       "env": {
         "CURSORLESS_MODE": "test",
+        "CURSORLESS_LOG_FAILED": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
       "args": [
@@ -52,6 +53,7 @@
       "env": {
         "CURSORLESS_MODE": "test",
         "CURSORLESS_RUN_TEST_SUBSET": "true",
+        "CURSORLESS_LOG_FAILED": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
       "args": [
@@ -136,6 +138,7 @@
       "program": "${workspaceFolder}/packages/test-harness/dist/runTalonTests.cjs",
       "env": {
         "CURSORLESS_MODE": "test",
+        "CURSORLESS_LOG_FAILED": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
       "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
@@ -171,6 +174,7 @@
       "program": "${workspaceFolder}/packages/test-harness/dist/runTalonJsTests.cjs",
       "env": {
         "CURSORLESS_MODE": "test",
+        "CURSORLESS_LOG_FAILED": "true",
         "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
       },
       "outFiles": ["${workspaceFolder}/**/out/**/*.js"],

packages/test-harness/src/runAllTests.ts

@@ -1,8 +1,13 @@
-import Mocha from "mocha";
-import * as path from "node:path";
 import { getCursorlessRepoRoot } from "@cursorless/node-common";
-import { runTestSubset, testSubsetGrepString } from "./testSubset";
 import { glob } from "glob";
+import Mocha from "mocha";
+import * as path from "node:path";
+import {
+  logFailedTests,
+  runTestSubset,
+  shouldLogFailedTests,
+  testSubsetGrepString,
+} from "./testSubset";
 
 /**
  * Type of test to run, eg unit, vscode, talon
@@ -24,7 +29,7 @@ export enum TestType {
   neovim,
 }
 
-export function runAllTests(...types: TestType[]) {
+export function runAllTests(...types: TestType[]): Promise<void> {
   return runTestsInDir(
     path.join(getCursorlessRepoRoot(), "packages"),
     (files) =>
@@ -68,14 +73,23 @@ async function runTestsInDir(
 
   try {
     // Run the mocha test
-    await new Promise<void>((c, e) => {
-      mocha.run((failures) => {
+    await new Promise<void>((resolve, reject) => {
+      const failedTests: string[] = [];
+
+      const runner = mocha.run((failures) => {
         if (failures > 0) {
-          e(new Error(`${failures} tests failed.`));
+          if (shouldLogFailedTests()) {
+            logFailedTests(failedTests);
+          }
+          reject(`${failures} tests failed.`);
         } else {
-          c();
+          resolve();
         }
       });
+
+      if (shouldLogFailedTests()) {
+        runner.on("fail", (test) => failedTests.push(test.fullTitle()));
+      }
     });
   } catch (err) {
     console.error(err);

packages/test-harness/src/testSubset.ts

@@ -29,6 +29,20 @@ export function testSubsetFilePath() {
   );
 }
 
+function testFailedFilePath() {
+  return path.join(
+    getCursorlessRepoRoot(),
+    "packages",
+    "test-harness",
+    "failedTests.properties",
+  );
+}
+
+export function logFailedTests(testNames: string[]) {
+  const lines = [`${testNames.length} failed tests`, "", ...testNames];
+  fs.writeFileSync(testFailedFilePath(), lines.join("\n"));
+}
+
 /**
  * Determine whether we should run just the subset of the tests specified by
  * {@link TEST_SUBSET_GREP_STRING}.
@@ -37,3 +51,11 @@ export function testSubsetFilePath() {
 export function runTestSubset() {
   return process.env.CURSORLESS_RUN_TEST_SUBSET === "true";
 }
+
+/**
+ * Determine whether we should log the failed tests to a file. This makes it easier to put them in `testSubsetGrep.properties` for faster iterating.
+ * @returns `true` if we should log failed tests to `packages/test-harness/failedTests.properties`
+ */
+export function shouldLogFailedTests() {
+  return process.env.CURSORLESS_LOG_FAILED === "true";
+}