Change LanguageDefinitions to use VscodeFilesystem. (#2199)

Remove some uses of `fs`, `fs/promises`, and `path` for the web port.

---------

Co-authored-by: Barry Jaspan <[email protected]>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: Pokey Rule <[email protected]>

Author: 4 people

packages/common/src/ide/types/FileSystem.types.ts

@@ -3,6 +3,19 @@ import { Disposable } from "./ide.types";
 export type PathChangeListener = () => void;
 
 export interface FileSystem {
+  /**
+   * Reads a file that comes bundled with Cursorless, with the utf-8 encoding.
+   * {@link path} is expected to be relative to the root of the extension
+   * bundle. If the file doesn't exist, returns `undefined`.
+   *
+   * Note that in development mode, it is possible to supply an absolute path to
+   * a file on the local filesystem, for things like hot-reloading.
+   *
+   * @param path The path of the file to read
+   * @returns The contents of path, decoded as UTF-8
+   */
+  readBundledFile(path: string): Promise<string | undefined>;
+
   /**
    * Recursively watch a directory for changes.
    * @param path The path of the directory to watch
@@ -11,12 +24,6 @@ export interface FileSystem {
    */
   watchDir(path: string, onDidChange: PathChangeListener): Disposable;
 
-  /**
-   * The path to the directory that Cursorless talon uses to share its state
-   * with the Cursorless engine.
-   */
-  readonly cursorlessDir: string;
-
   /**
    * The path to the Cursorless talon state JSON file.
    */

packages/cursorless-engine/src/languages/LanguageDefinition.ts

@@ -1,5 +1,9 @@
-import { ScopeType, SimpleScopeType, showError } from "@cursorless/common";
-import { existsSync, readFileSync } from "fs";
+import {
+  FileSystem,
+  ScopeType,
+  SimpleScopeType,
+  showError,
+} from "@cursorless/common";
 import { dirname, join } from "path";
 import { TreeSitterScopeHandler } from "../processTargets/modifiers/scopeHandlers";
 import { TreeSitterTextFragmentScopeHandler } from "../processTargets/modifiers/scopeHandlers/TreeSitterScopeHandler/TreeSitterTextFragmentScopeHandler";
@@ -33,19 +37,23 @@ export class LanguageDefinition {
    * @returns A language definition for the given language id, or undefined if the given language
    * id doesn't have a new-style query definition
    */
-  static create(
+  static async create(
     treeSitter: TreeSitter,
+    fileSystem: FileSystem,
     queryDir: string,
     languageId: string,
-  ): LanguageDefinition | undefined {
+  ): Promise<LanguageDefinition | undefined> {
     const languageQueryPath = join(queryDir, `${languageId}.scm`);
 
-    if (!existsSync(languageQueryPath)) {
+    const rawLanguageQueryString = await readQueryFileAndImports(
+      fileSystem,
+      languageQueryPath,
+    );
+
+    if (rawLanguageQueryString == null) {
       return undefined;
     }
 
-    const rawLanguageQueryString = readQueryFileAndImports(languageQueryPath);
-
     const rawQuery = treeSitter
       .getLanguage(languageId)!
       .query(rawLanguageQueryString);
@@ -88,7 +96,10 @@ export class LanguageDefinition {
  * @param languageQueryPath The path to the query file to read
  * @returns The text of the query file, with all imports inlined
  */
-function readQueryFileAndImports(languageQueryPath: string) {
+async function readQueryFileAndImports(
+  fileSystem: FileSystem,
+  languageQueryPath: string,
+) {
   // Seed the map with the query file itself
   const rawQueryStrings: Record<string, string | null> = {
     [languageQueryPath]: null,
@@ -103,7 +114,29 @@ function readQueryFileAndImports(languageQueryPath: string) {
         continue;
       }
 
-      const rawQuery = readFileSync(queryPath, "utf8");
+      let rawQuery = await fileSystem.readBundledFile(queryPath);
+
+      if (rawQuery == null) {
+        if (queryPath === languageQueryPath) {
+          // If this is the main query file, then we know that this language
+          // just isn't defined using new-style queries
+          return undefined;
+        }
+
+        showError(
+          ide().messages,
+          "LanguageDefinition.readQueryFileAndImports.queryNotFound",
+          `Could not find imported query file ${queryPath}`,
+        );
+
+        if (ide().runMode === "test") {
+          throw new Error("Invalid import statement");
+        }
+
+        // If we're not in test mode, we just ignore the import and continue
+        rawQuery = "";
+      }
+
       rawQueryStrings[queryPath] = rawQuery;
       matchAll(
         rawQuery,

packages/cursorless-engine/src/languages/LanguageDefinitions.ts

@@ -43,28 +43,61 @@ export class LanguageDefinitions {
   private disposables: Disposable[] = [];
 
   constructor(
-    fileSystem: FileSystem,
+    private fileSystem: FileSystem,
     private treeSitter: TreeSitter,
   ) {
+    ide().onDidOpenTextDocument((document) => {
+      this.loadLanguage(document.languageId);
+    });
+    ide().onDidChangeVisibleTextEditors((editors) => {
+      editors.forEach(({ document }) => this.loadLanguage(document.languageId));
+    });
+
     // Use the repo root as the root for development mode, so that we can
     // we can make hot-reloading work for the queries
-    this.queryDir = join(
+    this.queryDir =
       ide().runMode === "development"
-        ? getCursorlessRepoRoot()
-        : ide().assetsRoot,
-      "queries",
+        ? join(getCursorlessRepoRoot(), "queries")
+        : "queries";
+
+    ide().visibleTextEditors.forEach(({ document }) =>
+      this.loadLanguage(document.languageId),
     );
 
     if (ide().runMode === "development") {
       this.disposables.push(
         fileSystem.watchDir(this.queryDir, () => {
-          this.languageDefinitions.clear();
-          this.notifier.notifyListeners();
+          this.reloadLanguageDefinitions();
         }),
       );
     }
   }
 
+  public async loadLanguage(languageId: string): Promise<void> {
+    if (this.languageDefinitions.has(languageId)) {
+      return;
+    }
+
+    const definition =
+      (await LanguageDefinition.create(
+        this.treeSitter,
+        this.fileSystem,
+        this.queryDir,
+        languageId,
+      )) ?? LANGUAGE_UNDEFINED;
+
+    this.languageDefinitions.set(languageId, definition);
+  }
+
+  private async reloadLanguageDefinitions(): Promise<void> {
+    const languageIds = Array.from(this.languageDefinitions.keys());
+    this.languageDefinitions.clear();
+    await Promise.all(
+      languageIds.map((languageId) => this.loadLanguage(languageId)),
+    );
+    this.notifier.notifyListeners();
+  }
+
   /**
    * Get a language definition for the given language id, if the language
    * has a new-style query definition, or return undefined if the language doesn't
@@ -74,14 +107,13 @@ export class LanguageDefinitions {
    * the given language id doesn't have a new-style query definition
    */
   get(languageId: string): LanguageDefinition | undefined {
-    let definition = this.languageDefinitions.get(languageId);
+    const definition = this.languageDefinitions.get(languageId);
 
     if (definition == null) {
-      definition =
-        LanguageDefinition.create(this.treeSitter, this.queryDir, languageId) ??
-        LANGUAGE_UNDEFINED;
-
-      this.languageDefinitions.set(languageId, definition);
+      throw new Error(
+        "Expected language definition entry missing for languageId " +
+          languageId,
+      );
     }
 
     return definition === LANGUAGE_UNDEFINED ? undefined : definition;

packages/cursorless-engine/src/runIntegrationTests.ts

@@ -27,6 +27,7 @@ async function assertNoScopesBothLegacyAndNew(
   const errors: string[] = [];
   for (const languageId of legacyLanguageIds) {
     await treeSitter.loadLanguage(languageId);
+    await languageDefinitions.loadLanguage(languageId);
 
     unsafeKeys(languageMatchers[languageId] ?? {}).map((scopeTypeType) => {
       if (

packages/cursorless-vscode/src/extension.ts

@@ -24,7 +24,6 @@ import {
   toVscodeRange,
 } from "@cursorless/vscode-common";
 import * as crypto from "crypto";
-import { mkdir } from "fs/promises";
 import * as os from "os";
 import * as path from "path";
 import * as vscode from "vscode";
@@ -183,9 +182,15 @@ async function createVscodeIde(context: vscode.ExtensionContext) {
   const cursorlessDir = isTesting()
     ? path.join(os.tmpdir(), crypto.randomBytes(16).toString("hex"))
     : path.join(os.homedir(), ".cursorless");
-  await mkdir(cursorlessDir, { recursive: true });
 
-  return { vscodeIDE, hats, fileSystem: new VscodeFileSystem(cursorlessDir) };
+  const fileSystem = new VscodeFileSystem(
+    context,
+    vscodeIDE.runMode,
+    cursorlessDir,
+  );
+  await fileSystem.initialize();
+
+  return { vscodeIDE, hats, fileSystem };
 }
 
 function createTreeSitter(parseTreeApi: ParseTreeApi): TreeSitter {

packages/cursorless-vscode/src/ide/vscode/VscodeFileSystem.ts

@@ -1,26 +1,87 @@
-import { Disposable, FileSystem, PathChangeListener } from "@cursorless/common";
-import { RelativePattern, workspace } from "vscode";
-import * as path from "path";
+import {
+  Disposable,
+  FileSystem,
+  PathChangeListener,
+  RunMode,
+} from "@cursorless/common";
+import { isAbsolute, join } from "path";
+import * as vscode from "vscode";
 
 export class VscodeFileSystem implements FileSystem {
   public readonly cursorlessTalonStateJsonPath: string;
   public readonly cursorlessCommandHistoryDirPath: string;
 
-  constructor(public readonly cursorlessDir: string) {
-    this.cursorlessTalonStateJsonPath = path.join(
-      this.cursorlessDir,
-      "state.json",
-    );
-    this.cursorlessCommandHistoryDirPath = path.join(
+  private decoder = new TextDecoder("utf-8");
+
+  constructor(
+    private readonly extensionContext: vscode.ExtensionContext,
+    private readonly runMode: RunMode,
+    private readonly cursorlessDir: string,
+  ) {
+    this.cursorlessTalonStateJsonPath = join(this.cursorlessDir, "state.json");
+    this.cursorlessCommandHistoryDirPath = join(
       this.cursorlessDir,
       "commandHistory",
     );
   }
 
-  watchDir(path: string, onDidChange: PathChangeListener): Disposable {
+  public async initialize(): Promise<void> {
+    try {
+      await vscode.workspace.fs.createDirectory(
+        vscode.Uri.file(this.cursorlessDir),
+      );
+    } catch (err) {
+      console.log("Cannot create cursorlessDir", this.cursorlessDir, err);
+    }
+  }
+
+  /**
+   * Reads a file that comes bundled with Cursorless, with the utf-8 encoding.
+   * {@link path} is expected to be relative to the root of the extension
+   * bundle. If the file doesn't exist, returns `undefined`.
+   *
+   * Note that in development mode, it is possible to supply an absolute path to
+   * a file on the local filesystem, for things like hot-reloading.
+   *
+   * @param path The path of the file to read
+   * @returns The contents of path, decoded as UTF-8
+   */
+  public async readBundledFile(path: string): Promise<string | undefined> {
+    try {
+      return this.decoder.decode(
+        await vscode.workspace.fs.readFile(this.resolveBundledPath(path)),
+      );
+    } catch (err) {
+      if (
+        err instanceof Error &&
+        "code" in err &&
+        err.code === "FileNotFound"
+      ) {
+        return undefined;
+      }
+      throw err;
+    }
+  }
+
+  private resolveBundledPath(path: string) {
+    if (isAbsolute(path)) {
+      if (this.runMode !== "development") {
+        throw new Error(
+          "Absolute paths are not supported outside of development mode",
+        );
+      }
+
+      return vscode.Uri.file(path);
+    }
+
+    return vscode.Uri.joinPath(this.extensionContext.extensionUri, path);
+  }
+
+  public watchDir(path: string, onDidChange: PathChangeListener): Disposable {
+    // return { dispose: () => {} };
     // FIXME: Support globs?
-    const watcher = workspace.createFileSystemWatcher(
-      new RelativePattern(path, "**"),
+    const watcher = vscode.workspace.createFileSystemWatcher(
+      new vscode.RelativePattern(path, "**"),
     );
 
     watcher.onDidChange(onDidChange);