Run indexer on main thread for smaller projects. Repair symbol loc.file.package when copied from worker.

Author: ShukantPal

example/package.json

@@ -27,7 +27,7 @@
     "build-next": "cd .. && webdoc && cd example",
     "build-pixi-api": "cd ../../pixi-api && webdoc --verbose && cd ../webdoc/example",
     "build-pixi-api-prod": "cd ../../pixi-api && webdoc --site-root pixi-api && cd ../webdoc/example",
-    "build-pixi-api-gcp": "cd ../../pixi-api && webdoc --tutorials ./projects/guides --verbose && cd ../webdoc/example",
+    "build-pixi-api-gcp": "cd ../../pixi-api && node --inspect-brk ../webdoc/example/node_modules/@webdoc/cli/cli --tutorials ./projects/guides --verbose && cd ../webdoc/example",
     "build-pixi-guides": "cd ../../guides && webdoc --tutorials docs && cd ../webdoc/example",
     "build-mongodb": "cd ../../node-mongodb-native && webdoc --verbose && cd ../webdoc/example"
   },

packages/webdoc-parser/src/indexer/IndexerWorkerPool.js

@@ -1,9 +1,9 @@
 // @flow
 
+import {type PackageDoc, type SourceFile} from "@webdoc/types";
 import {parserLogger, tag} from "../Logger";
 import EventEmitter from "events";
 import type {LanguageConfig} from "../types/LanguageIntegration";
-import {type SourceFile} from "@webdoc/types";
 import {type Symbol} from "../types/Symbol";
 import {Worker} from "worker_threads";
 import os from "os";
@@ -81,15 +81,37 @@ export class IndexerWorkerPool {
     this.workers = workers;
   }
 
-  index(langIntegrationModule: string, file: SourceFile, config: LanguageConfig): Promise<Symbol> {
+  repair(symbol: Symbol, packages: { [id: string]: PackageDoc }): Symbol {
+    const file = symbol.loc.file;
+    const pkgId = file?.package.id;
+
+    if (file && pkgId && packages[pkgId]) {
+      file.package = packages[pkgId];
+    }
+    for (const child of symbol.members) {
+      this.repair(child, packages);
+    }
+
+    return symbol;
+  }
+
+  index(
+    langIntegrationModule: string,
+    file: SourceFile,
+    config: LanguageConfig,
+    packages: { [id: string]: PackageDoc },
+  ): Promise<Symbol> {
     return new Promise<Symbol>((resolve, reject) => {
       const jobId = this.ptr++;
       const worker = this.workers[jobId % this.workers.length];
 
       worker.send(jobId, langIntegrationModule, file, config);
       worker.on(`response-${jobId}`, (jobResult: Symbol, jobError: any): void => {
-        if (jobResult) resolve(jobResult);
-        else reject(jobError);
+        if (jobResult) {
+          resolve(this.repair(jobResult, packages));
+        } else {
+          reject(jobError);
+        }
       });
     });
   }

packages/webdoc-parser/src/indexer/index.js

@@ -5,6 +5,9 @@ import {parserLogger, tag} from "../Logger";
 import {IndexerWorkerPool} from "./IndexerWorkerPool";
 import {type SourceFile} from "@webdoc/types";
 import {type Symbol} from "../types/Symbol";
+import _ from "lodash";
+import fs from "fs";
+import os from "os";
 import path from "path";
 
 declare var globalThis: any;
@@ -27,37 +30,52 @@ export function register(lang: LanguageIntegration): void {
 
 export async function run(files: SourceFile[], config: LanguageConfig): Promise<Symbol[]> {
   const startTime = Date.now();
-  const symbolTrees: Array<Symbol> = new Array(files.length);
-  const symbolIndexingOperations: Array<Promise<void>> = new Array(files.length);
-  const workerPool = new IndexerWorkerPool(Math.floor(files.length / 125));
+  const maxThreads = Math.min(os.cpus().length, 1 + Math.floor(files.length / 125));
 
   parserLogger.info(tag.Indexer, "Indexing " + files.length + " files");
 
-  for (let i = 0; i < files.length; i++) {
-    const fileName = files[i].path;
-    const extension = fileName.substring(fileName.lastIndexOf(".") + 1, fileName.length);
+  const symbolTrees: Array<Symbol> = new Array(files.length);
 
-    if (!(extension in languages)) {
-      throw new Error(`.${extension} language is not registered with the Indexer!`);
+  if (maxThreads > 1) {
+    const packages = _.keyBy(
+      files.map((file) => file.package),
+      (pkg) => pkg.id,
+    );
+    const symbolIndexingOperations: Array<Promise<void>> = new Array(files.length);
+    const workerPool = new IndexerWorkerPool(maxThreads);
+
+    for (let i = 0; i < files.length; i++) {
+      const fileName = files[i].path;
+      const extension = fileName.substring(fileName.lastIndexOf(".") + 1, fileName.length);
+
+      if (!(extension in languages)) {
+        throw new Error(`.${extension} language is not registered with the Indexer!`);
+      }
+
+      const file = {
+        ...files[i],
+        path: path.resolve(globalThis.process.cwd(), files[i].path),
+      };
+      const lang = languages[extension].module;
+
+      symbolIndexingOperations[i] = workerPool.index(lang, file, config, packages).then(
+        (symbolTree: Symbol): void => {
+          symbolTrees[i] = symbolTree;
+        },
+      );
     }
 
-    const file = {
-      ...files[i],
-      path: path.resolve(globalThis.process.cwd(), files[i].path),
-    };
-    const lang = languages[extension].module;
+    await Promise.all(symbolIndexingOperations);
+    workerPool.destroy();
+  } else {
+    for (let i = 0; i < files.length; i++) {
+      const filePath = path.resolve(globalThis.process.cwd(), files[i].path);
+      const fileContent = files[i].content || fs.readFileSync(filePath, "utf8");
 
-    symbolIndexingOperations[i] = workerPool.index(lang, file, config).then(
-      (symbolTree: Symbol): void => {
-        symbolTrees[i] = symbolTree;
-      },
-    );
+      symbolTrees[i] = process(fileContent, files[i], config);
+    }
   }
 
-  await Promise.all(symbolIndexingOperations);
-
-  workerPool.destroy();
-
   const endTime = Date.now();
 
   parserLogger.info(tag.Indexer, "Indexing took " + (endTime - startTime) + "ms");