Refactor command history to remove node dependencies from engine (#2479)

## Checklist

- [/] I have added
[tests](https://www.cursorless.org/docs/contributing/test-case-recorder/)
- [/] I have updated the
[docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and
[cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet)
- [/] I have not broken the cheatsheet

---------

Co-authored-by: Pokey Rule <[email protected]>

Author: AndreasArvidsson

packages/common/src/ide/types/CommandHistoryStorage.ts

@@ -0,0 +1,20 @@
+import type { CommandHistoryEntry } from "../../types/commandHistory";
+
+/**
+ * Used by command history machinery to store entries.
+ */
+export interface CommandHistoryStorage {
+  /**
+   * Append an entry to a command history file.
+   *
+   * @param fileName The name of the file to append the entry to. We usually use
+   * the a name derived from the current month to do a form of log rotation.
+   * @param entry The entry to append to the file
+   */
+  appendEntry(fileName: string, entry: CommandHistoryEntry): Promise<void>;
+
+  /**
+   * Get all entries from all command history files.
+   */
+  getEntries(): Promise<CommandHistoryEntry[]>;
+}

packages/common/src/index.ts

@@ -30,6 +30,7 @@ export * from "./ide/types/Events";
 export * from "./ide/types/QuickPickOptions";
 export * from "./ide/types/events.types";
 export * from "./ide/types/Paths";
+export * from "./ide/types/CommandHistoryStorage";
 export * from "./ide/types/FileSystem.types";
 export * from "./types/RangeExpansionBehavior";
 export * from "./types/InputBoxOptions";

packages/cursorless-engine/src/CommandHistory.ts

@@ -3,17 +3,15 @@ import {
   CommandComplete,
   CommandHistoryEntry,
   CommandServerApi,
-  FileSystem,
   IDE,
   ReadOnlyHatMap,
+  type CommandHistoryStorage,
 } from "@cursorless/common";
 import type {
   CommandRunner,
   CommandRunnerDecorator,
 } from "@cursorless/cursorless-engine";
 import { produce } from "immer";
-import * as fs from "node:fs/promises";
-import * as path from "node:path";
 import { v4 as uuid } from "uuid";
 
 const filePrefix = "cursorlessCommandHistory";
@@ -23,17 +21,14 @@ const filePrefix = "cursorlessCommandHistory";
  * to a local log file in `.cursorless/commandHistory` dir.
  */
 export class CommandHistory implements CommandRunnerDecorator {
-  private readonly dirPath: string;
   private currentPhraseSignal = "";
   private currentPhraseId = "";
 
   constructor(
     private ide: IDE,
+    private storage: CommandHistoryStorage,
     private commandServerApi: CommandServerApi | null,
-    fileSystem: FileSystem,
-  ) {
-    this.dirPath = fileSystem.cursorlessCommandHistoryDirPath;
-  }
+  ) {}
 
   wrapCommandRunner(
     _readableHatMap: ReadOnlyHatMap,
@@ -65,7 +60,6 @@ export class CommandHistory implements CommandRunnerDecorator {
   ): Promise<void> {
     const date = new Date();
     const fileName = `${filePrefix}_${getMonthDate(date)}.jsonl`;
-    const file = path.join(this.dirPath, fileName);
 
     const historyItem: CommandHistoryEntry = {
       id: uuid(),
@@ -75,10 +69,8 @@ export class CommandHistory implements CommandRunnerDecorator {
       phraseId: await this.getPhraseId(),
       command: produce(command, sanitizeCommandInPlace),
     };
-    const data = JSON.stringify(historyItem) + "\n";
 
-    await fs.mkdir(this.dirPath, { recursive: true });
-    await fs.appendFile(file, data, "utf8");
+    await this.storage.appendEntry(fileName, historyItem);
   }
 
   private async getPhraseId(): Promise<string | undefined> {

packages/cursorless-engine/src/CommandHistoryAnalyzer.ts

@@ -4,11 +4,10 @@ import {
   PartialPrimitiveTargetDescriptor,
   ScopeType,
   showWarning,
+  type CommandHistoryStorage,
 } from "@cursorless/common";
 import { groupBy, map, sum } from "lodash-es";
-import { asyncIteratorToList } from "./asyncIteratorToList";
 import { canonicalizeAndValidateCommand } from "./core/commandVersionUpgrades/canonicalizeAndValidateCommand";
-import { generateCommandHistoryEntries } from "./generateCommandHistoryEntries";
 import { ide } from "./singletons/ide.singleton";
 import { getPartialTargetDescriptors } from "./util/getPartialTargetDescriptors";
 import { getPartialPrimitiveTargets } from "./util/getPrimitiveTargets";
@@ -97,8 +96,10 @@ function getMonth(entry: CommandHistoryEntry): string {
   return entry.date.slice(0, 7);
 }
 
-export async function analyzeCommandHistory(dir: string) {
-  const entries = await asyncIteratorToList(generateCommandHistoryEntries(dir));
+export async function analyzeCommandHistory(
+  commandHistoryStorage: CommandHistoryStorage,
+) {
+  const entries = await commandHistoryStorage.getEntries();
 
   if (entries.length === 0) {
     const TAKE_ME_THERE = "Show me";

packages/cursorless-engine/src/asyncIteratorToList.ts

@@ -51,6 +51,7 @@ import {
 import { StatusBarItem } from "./StatusBarItem";
 import { storedTargetHighlighter } from "./storedTargetHighlighter";
 import { vscodeApi } from "./vscodeApi";
+import { FileSystemCommandHistoryStorage } from "@cursorless/file-system-common";
 
 /**
  * Extension entrypoint called by VSCode on Cursorless startup.
@@ -101,8 +102,12 @@ export async function activate(
     fileSystem,
   );
 
+  const commandHistoryStorage = new FileSystemCommandHistoryStorage(
+    fileSystem.cursorlessCommandHistoryDirPath,
+  );
+
   addCommandRunnerDecorator(
-    new CommandHistory(normalizedIde, commandServerApi, fileSystem),
+    new CommandHistory(normalizedIde, commandHistoryStorage, commandServerApi),
   );
 
   const testCaseRecorder = new TestCaseRecorder(
@@ -140,7 +145,7 @@ export async function activate(
     context,
     vscodeIDE,
     commandApi,
-    fileSystem,
+    commandHistoryStorage,
     testCaseRecorder,
     scopeTestRecorder,
     scopeVisualizer,

packages/cursorless-engine/src/generateCommandHistoryEntries.ts

@@ -1,8 +1,8 @@
 import {
   CURSORLESS_COMMAND_ID,
   CursorlessCommandId,
-  FileSystem,
   isTesting,
+  type CommandHistoryStorage,
 } from "@cursorless/common";
 import {
   CommandApi,
@@ -25,7 +25,7 @@ export function registerCommands(
   extensionContext: vscode.ExtensionContext,
   vscodeIde: VscodeIDE,
   commandApi: CommandApi,
-  fileSystem: FileSystem,
+  commandHistoryStorage: CommandHistoryStorage,
   testCaseRecorder: TestCaseRecorder,
   scopeTestRecorder: ScopeTestRecorder,
   scopeVisualizer: ScopeVisualizer,
@@ -90,7 +90,7 @@ export function registerCommands(
 
     // Command history
     ["cursorless.analyzeCommandHistory"]: () =>
-      analyzeCommandHistory(fileSystem.cursorlessCommandHistoryDirPath),
+      analyzeCommandHistory(commandHistoryStorage),
 
     // General keyboard commands
     ["cursorless.keyboard.escape"]:

packages/cursorless-vscode/src/extension.ts

@@ -24,6 +24,10 @@
     }
   },
   "dependencies": {
-    "@cursorless/common": "workspace:*"
+    "@cursorless/common": "workspace:*",
+    "glob": "^10.3.10"
+  },
+  "devDependencies": {
+    "@types/glob": "^8.1.0"
   }
 }

packages/cursorless-vscode/src/registerCommands.ts

@@ -0,0 +1,45 @@
+import type {
+  CommandHistoryEntry,
+  CommandHistoryStorage,
+} from "@cursorless/common";
+import { glob } from "glob";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export class FileSystemCommandHistoryStorage implements CommandHistoryStorage {
+  constructor(private dir: string) {}
+
+  async appendEntry(
+    fileName: string,
+    entry: CommandHistoryEntry,
+  ): Promise<void> {
+    const data = JSON.stringify(entry) + "\n";
+    const file = path.join(this.dir, fileName);
+    await fs.mkdir(this.dir, { recursive: true });
+    await fs.appendFile(file, data, "utf8");
+  }
+
+  async getEntries(): Promise<CommandHistoryEntry[]> {
+    const files = await glob("*.jsonl", {
+      cwd: this.dir,
+    });
+
+    const entries: CommandHistoryEntry[] = [];
+
+    for (const file of files) {
+      const filePath = path.join(this.dir, file);
+      const content = await fs.readFile(filePath, "utf8");
+      const lines = content.split("\n");
+
+      for (const line of lines) {
+        if (line.length === 0) {
+          continue;
+        }
+
+        entries.push(JSON.parse(line));
+      }
+    }
+
+    return entries;
+  }
+}