Add back indexing to fix performance issues

Author: PhilippeChab

package.json

@@ -8,7 +8,7 @@
     "url": "https://github.com/PhilippeChab/nwscript-ee-language-server"
   },
   "license": "MIT",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "author": {
     "name": "Philippe Chabot"
   },
@@ -161,7 +161,7 @@
     "vscode:prepublish": "yarn build",
     "build": "yarn clean && yarn buildClient && yarn buildServer",
     "buildClient": "esbuild extension=./client/src/extension.ts --sourcemap --bundle --external:vscode --platform=node --outdir=./client/out/",
-    "buildServer": "esbuild server=./server/src/server.ts --sourcemap --bundle --external:vscode --platform=node --outdir=./server/out/",
+    "buildServer": "esbuild server=./server/src/server.ts indexer=./server/src/Documents/DocumentsIndexer.ts --sourcemap --bundle --external:vscode --platform=node --outdir=./server/out/",
     "compile": "tsc -b",
     "compile:client": "tsc -b ./client/tsconfig.json",
     "compile:server": "tsc -b ./server/tsconfig.json",

server/src/Documents/DocumentsCollection.ts

@@ -53,6 +53,8 @@ export default class DocumentsCollection extends Dictionnary<string, Document> {
       if (!filePath) return;
 
       const uri = pathToFileURL(filePath).href;
+      if (this.get(this.getKey(uri, false))) return;
+
       const fileContent = readFileSync(filePath).toString();
       this.createDocuments(uri, fileContent, tokenizer, workespaceFilesSystem);
     });

server/src/Documents/DocumentsIndexer.ts

@@ -0,0 +1,26 @@
+import { readFileSync } from "fs";
+import { exit } from "process";
+
+import { Tokenizer } from "../Tokenizer";
+import { TokenizedScope } from "../Tokenizer/Tokenizer";
+
+const generateTokens = async (filesPath: string[]) => {
+  if (filesPath.length === 1 && !Boolean(filesPath[0])) {
+    exit(0);
+  }
+
+  const tokenizer = await new Tokenizer().loadGrammar();
+  for (let i = 0; i < filesPath.length; i++) {
+    const filePath = filesPath[i];
+    const fileContent = readFileSync(filePath).toString();
+    const globalScope = tokenizer.tokenizeContent(fileContent, TokenizedScope.global);
+
+    process?.send!(JSON.stringify({ filePath, globalScope }));
+  }
+
+  exit(0);
+};
+
+process.on("message", (filesPath: string) => {
+  generateTokens(filesPath.split(","));
+});

server/src/ServerManager/ServerManager.ts

@@ -1,3 +1,7 @@
+import { cpus } from "os";
+import { join } from "path";
+import { pathToFileURL } from "url";
+import * as clustering from "cluster";
 import type { Connection, InitializeParams } from "vscode-languageserver";
 
 import {
@@ -67,8 +71,42 @@ export default class ServerManger {
     }
 
     await this.loadConfig();
-    this.configLoaded = true;
-    this.diagnosticsProvider?.processDocumentsWaitingForPublish();
+
+    const numCPUs = cpus().length;
+    const cluster = clustering.default;
+    if (cluster.isPrimary) {
+      cluster.setupPrimary({
+        exec: join(__dirname, "indexer.js"),
+      });
+    }
+
+    let filesIndexedCount = 0;
+    const filesPath = this.workspaceFilesSystem.getAllFilesPath();
+    const progressReporter = await this.connection.window.createWorkDoneProgress();
+    const filesCount = filesPath.length;
+    this.logger.info(`Indexing files ...`);
+
+    progressReporter.begin("Indexing files for NWScript: EE Language Server ...", 0);
+    const partCount = Math.ceil(filesCount / numCPUs);
+    for (let i = 0; i < Math.min(numCPUs, filesCount); i++) {
+      const worker = cluster.fork();
+      worker.send(filesPath.slice(i * partCount, Math.min((i + 1) * partCount, filesCount - 1)).join(","));
+      worker.on("message", (message: string) => {
+        const { filePath, globalScope } = JSON.parse(message);
+        this.documentsCollection?.createDocument(pathToFileURL(filePath).href, globalScope);
+        filesIndexedCount++;
+        progressReporter?.report(filesIndexedCount / filesCount);
+      });
+    }
+
+    cluster.on("exit", () => {
+      if (Object.keys(cluster.workers || {}).length === 0) {
+        progressReporter?.done();
+        this.logger.info(`Indexed ${filesIndexedCount} files.`);
+        this.configLoaded = true;
+        this.diagnosticsProvider?.processDocumentsWaitingForPublish();
+      }
+    });
   }
 
   public down() {}

server/src/WorkspaceFilesSystem/WorkspaceFilesSystem.ts

@@ -11,6 +11,10 @@ export default class WorkspaceFilesSystem {
     return normalize(join(this.rootPath, ...parts));
   }
 
+  public getAllFilesPath() {
+    return new GlobSync(`**/*${FILES_EXTENSION}`).found.map((filename) => this.normalizedAbsolutePath(filename));
+  }
+
   public getFilePath(filename: string) {
     const path = new GlobSync(`**/${filename}${FILES_EXTENSION}`).found[0];
     if (path) {