wip: worker forks for indexers

Author: raideer

esbuild.js

@@ -77,6 +77,24 @@ async function main() {
     plugins: [esbuildProblemMatcherPlugin, copyAssetsPlugin],
   });
 
+  // Worker build context
+  const workerCtx = await esbuild.context({
+    entryPoints: ['src/indexer/IndexWorker.ts'],
+    bundle: true,
+    format: 'cjs',
+    minify: production,
+    sourcemap: !production,
+    sourcesContent: false,
+    platform: 'node',
+    outdir: 'dist/indexer',
+    define: {
+      'process.env.NODE_ENV': JSON.stringify(production ? 'production' : 'development'),
+    },
+    external: ['vscode'],
+    logLevel: 'silent',
+    plugins: [esbuildProblemMatcherPlugin],
+  });
+
   // Webview build context
   const webviewCtx = await esbuild.context({
     entryPoints: ['src/webview/index.tsx'],
@@ -98,10 +116,10 @@ async function main() {
   });
 
   if (watch) {
-    await Promise.all([extensionCtx.watch(), webviewCtx.watch()]);
+    await Promise.all([extensionCtx.watch(), workerCtx.watch(), webviewCtx.watch()]);
   } else {
-    await Promise.all([extensionCtx.rebuild(), webviewCtx.rebuild()]);
-    await Promise.all([extensionCtx.dispose(), webviewCtx.dispose()]);
+    await Promise.all([extensionCtx.rebuild(), workerCtx.rebuild(), webviewCtx.rebuild()]);
+    await Promise.all([extensionCtx.dispose(), workerCtx.dispose(), webviewCtx.dispose()]);
   }
 }
 

indexer.log

@@ -0,0 +1,39 @@
+[INDEXER] Indexing di.xml
+[INDEXER] Discovering files to index
+[INDEXER] Found 271 files to index in 255 ms
+[INDEXER] Indexed 271 files of 271 in 935ms
+[INDEXER] Indexing di.xml done in 1198 ms
+[INDEXER] Indexing module.xml
+[INDEXER] Discovering files to index
+[INDEXER] Found 435 files to index in 1228 ms
+[INDEXER] Indexed 435 files of 435 in 1789ms
+[INDEXER] Indexing module.xml done in 3018 ms
+[INDEXER] Indexing namespaces
+[INDEXER] Discovering files to index
+[INDEXER] Found 716 files to index in 1149 ms
+[Indexed] 500 files of 716
+[INDEXER] Indexed 716 files of 716 in 11581ms
+[INDEXER] Indexing namespaces done in 12755 ms
+[INDEXER] Indexing events.xml
+[INDEXER] Discovering files to index
+[INDEXER] Found 45 files to index in 1140 ms
+[INDEXER] Indexed 45 files of 45 in 121ms
+[INDEXER] Indexing events.xml done in 1262 ms
+[INDEXER] Indexing acl.xml
+[INDEXER] Discovering files to index
+[INDEXER] Found 81 files to index in 2298 ms
+[INDEXER] Indexed 81 files of 81 in 697ms
+[INDEXER] Indexing acl.xml done in 2996 ms
+[INDEXER] Indexing templates
+[INDEXER] Discovering files to index
+[INDEXER] Found 1020 files to index in 1258 ms
+[Indexed] 500 files of 1020
+[Indexed] 1000 files of 1020
+[INDEXER] Indexed 1020 files of 1020 in 443ms
+[INDEXER] Indexing templates done in 1704 ms
+[INDEXER] Indexing crontab.xml
+[INDEXER] Discovering files to index
+[INDEXER] Found 29 files to index in 1261 ms
+[INDEXER] Indexed 29 files of 29 in 60ms
+[INDEXER] Indexing crontab.xml done in 1323 ms
+[INDEXER] Finished indexing workspace magento-demo

package-lock.json

@@ -431,6 +431,7 @@
     "@xml-tools/validation": "^1.0.16",
     "cronstrue": "^2.59.0",
     "fast-xml-parser": "^4.5.1",
+    "fdir": "^6.4.6",
     "formik": "^2.4.6",
     "glob": "^11.0.1",
     "handlebars": "^4.7.8",
@@ -444,6 +445,7 @@
     "slugify": "^1.6.6",
     "typescript-json-serializer": "^6.0.1",
     "typescript-memoize": "^1.1.1",
-    "validatorjs": "^3.22.1"
+    "validatorjs": "^3.22.1",
+    "vscode-uri": "^3.1.0"
   }
 }

package.json

@@ -1,4 +1,4 @@
-import { Progress, Uri, workspace, WorkspaceFolder } from 'vscode';
+import { Progress, RelativePattern, Uri, workspace, WorkspaceFolder } from 'vscode';
 import { Indexer } from './Indexer';
 import Common from 'util/Common';
 import { minimatch } from 'minimatch';
@@ -20,7 +20,7 @@ import TemplateIndexer from './template/TemplateIndexer';
 import { TemplateIndexData } from './template/TemplateIndexData';
 import CronIndexer from './cron/CronIndexer';
 import { CronIndexData } from './cron/CronIndexData';
-import Config from 'common/Config';
+import { IndexWorkerManager } from './IndexWorkerManager';
 
 type IndexerInstance =
   | DiIndexer
@@ -42,11 +42,12 @@ type IndexerDataMap = {
 };
 
 class IndexManager {
-  private static readonly INDEXING_BATCH_SIZE = 50;
+  private static readonly INDEXING_BATCH_SIZE = 200;
   private static readonly REPORTING_BATCH_SIZE = 500;
 
   protected indexers: IndexerInstance[] = [];
   protected indexStorage: IndexStorage;
+  protected workerManager: IndexWorkerManager;
 
   public constructor() {
     this.indexers = [
@@ -59,6 +60,7 @@ class IndexManager {
       new CronIndexer(),
     ];
     this.indexStorage = new IndexStorage();
+    this.workerManager = new IndexWorkerManager();
   }
 
   public getIndexers(): IndexerInstance[] {
@@ -99,11 +101,13 @@ class IndexManager {
       const startDiscover = Date.now();
       Logger.log('INDEXER', 'Discovering files to index');
 
-      const files = await workspace.findFiles(
-        indexer.getPattern(workspaceUri),
-        indexer.getExcludePattern(workspaceUri)
+      const indexPattern = new RelativePattern(workspaceUri, indexer.getPattern());
+      const indexExcludePattern = new RelativePattern(
+        workspaceUri,
+        indexer.getExcludePattern() ?? ''
       );
 
+      const files = await workspace.findFiles(indexPattern, indexExcludePattern);
       const discoverDuration = Date.now() - startDiscover;
       Logger.log('INDEXER', 'Found', files.length, 'files to index in', discoverDuration, 'ms');
 
@@ -114,21 +118,12 @@ class IndexManager {
 
       for (let i = 0; i < files.length; i += IndexManager.INDEXING_BATCH_SIZE) {
         const batch = files.slice(i, i + IndexManager.INDEXING_BATCH_SIZE);
+        const batchResult = await this.workerManager.processBatch(indexer, batch, workspaceUri);
 
-        const promises = batch.map(async file => {
-          try {
-            const data = await indexer.indexFile(file);
-
-            if (data !== undefined) {
-              indexData.set(file.fsPath, data);
-            }
-          } catch (error) {
-            Logger.error('Error indexing file', file.fsPath, String(error));
-          }
+        batchResult.forEach((data, filePath) => {
+          indexData.set(filePath, data);
         });
 
-        await Promise.all(promises);
-
         doneCount += batch.length;
 
         if (doneCount % IndexManager.REPORTING_BATCH_SIZE === 0) {
@@ -235,11 +230,10 @@ class IndexManager {
     indexer: Indexer
   ): Promise<void> {
     const indexData = this.getIndexStorageData(indexer.getId()) || new Map();
-    const pattern = indexer.getPattern(workspaceFolder.uri);
-    const patternString = typeof pattern === 'string' ? pattern : pattern.pattern;
+    const pattern = indexer.getPattern();
 
-    if (minimatch(file.fsPath, patternString, { matchBase: true })) {
-      const data = await indexer.indexFile(file);
+    if (minimatch(file.fsPath, pattern, { matchBase: true })) {
+      const data = await indexer.indexFile(file.fsPath);
 
       if (data !== undefined) {
         indexData.set(file.fsPath, data);

src/indexer/IndexManager.ts

@@ -0,0 +1,82 @@
+import { Indexer } from './Indexer';
+import DiIndexer from './di/DiIndexer';
+import ModuleIndexer from './module/ModuleIndexer';
+import AutoloadNamespaceIndexer from './autoload-namespace/AutoloadNamespaceIndexer';
+import EventsIndexer from './events/EventsIndexer';
+import AclIndexer from './acl/AclIndexer';
+import TemplateIndexer from './template/TemplateIndexer';
+import CronIndexer from './cron/CronIndexer';
+
+export interface WorkerMessage {
+  type: 'batch';
+  indexerId: string;
+  files: string[];
+  workspaceUri: string;
+}
+
+export interface WorkerResult {
+  success: boolean;
+  indexData: { [filePath: string]: any };
+  error?: string;
+}
+
+// Map of indexer IDs to their classes
+const indexerMap = {
+  [DiIndexer.KEY]: DiIndexer,
+  [ModuleIndexer.KEY]: ModuleIndexer,
+  [AutoloadNamespaceIndexer.KEY]: AutoloadNamespaceIndexer,
+  [EventsIndexer.KEY]: EventsIndexer,
+  [AclIndexer.KEY]: AclIndexer,
+  [TemplateIndexer.KEY]: TemplateIndexer,
+  [CronIndexer.KEY]: CronIndexer,
+};
+
+async function processBatch(message: WorkerMessage): Promise<WorkerResult> {
+  try {
+    const IndexerClass = indexerMap[message.indexerId as keyof typeof indexerMap];
+
+    if (!IndexerClass) {
+      throw new Error(`Unknown indexer: ${message.indexerId}`);
+    }
+
+    const indexer: Indexer = new IndexerClass();
+    const indexData: { [filePath: string]: any } = {};
+
+    console.log(
+      `WORKER: Processing batch of ${message.files.length} files for ${indexer.getName()}`
+    );
+
+    for (const filePath of message.files) {
+      try {
+        const data = await indexer.indexFile(filePath);
+
+        if (data !== undefined) {
+          indexData[filePath] = data;
+        }
+      } catch (error) {
+        console.error('WORKER: Error indexing file', filePath, String(error));
+      }
+    }
+
+    return {
+      success: true,
+      indexData,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      indexData: {},
+      error: String(error),
+    };
+  }
+}
+
+// Handle messages from the main thread (child process)
+if (process.send) {
+  process.on('message', async (message: WorkerMessage) => {
+    if (message.type === 'batch') {
+      const result = await processBatch(message);
+      process.send!(result);
+    }
+  });
+}