Add logging for worker performance

Author: ShukantPal

example/package.json

@@ -25,7 +25,7 @@
     "unit-test": "",
     "build": "webdoc --tutorials ./tutorials --site-root example-documentation --site-domain https://webdoc-labs.github.io --verbose",
     "build-next": "cd .. && webdoc && cd example",
-    "build-pixi-api": "cd ../../pixi-api && webdoc && cd ../webdoc/example",
+    "build-pixi-api": "cd ../../pixi-api && webdoc --verbose && cd ../webdoc/example",
     "build-pixi-api-prod": "cd ../../pixi-api && webdoc --site-root pixi-api && cd ../webdoc/example",
     "build-pixi-api-gcp": "cd ../../pixi-api && webdoc --tutorials ./projects/guides --verbose && cd ../webdoc/example",
     "build-pixi-guides": "cd ../../guides && webdoc --tutorials docs && cd ../webdoc/example",

packages/webdoc-parser/src/Logger.js

@@ -35,6 +35,7 @@ export function initLogger(defaultLevel: string = "INFO") {
       PartialParser: defaultLevel,
       DocParser: defaultLevel,
       DocumentTreeModifier: defaultLevel,
+      Indexer: defaultLevel,
     },
     (level, tag, msg, params) => {
       let tagPrefix = `[${tag}]:`;

packages/webdoc-parser/src/indexer/IndexerWorkerPool.js

@@ -1,5 +1,6 @@
 // @flow
 
+import {parserLogger, tag} from "../Logger";
 import EventEmitter from "events";
 import type {LanguageConfig} from "../types/LanguageIntegration";
 import {type SourceFile} from "@webdoc/types";
@@ -67,10 +68,12 @@ export class IndexerWorkerPool {
   ptr: number = 0;
 
   constructor(limit?: number) {
-    const workerPoolSize = Math.min(os.cpus().length, limit || Infinity);
+    const workerPoolSize = Math.min(os.cpus().length, limit || 4);
     const workers = new Array<Worker>(workerPoolSize);
     const workerPath = path.resolve(__dirname, "./worker.js");
 
+    parserLogger.info(tag.Indexer, "Using " + workerPoolSize + " worker threads for indexing");
+
     for (let i = 0; i < workerPoolSize; i++) {
       workers[i] = new IndexerWorker(new Worker(workerPath));
     }

packages/webdoc-parser/src/indexer/index.js

@@ -1,10 +1,10 @@
 // @flow
 
 import type {LanguageConfig, LanguageIntegration} from "../types/LanguageIntegration";
+import {parserLogger, tag} from "../Logger";
 import {IndexerWorkerPool} from "./IndexerWorkerPool";
 import {type SourceFile} from "@webdoc/types";
 import {type Symbol} from "../types/Symbol";
-import {parserLogger} from "../Logger";
 import path from "path";
 
 declare var globalThis: any;
@@ -26,9 +26,12 @@ export function register(lang: LanguageIntegration): void {
 }
 
 export async function run(files: SourceFile[], config: LanguageConfig): Promise<Symbol[]> {
+  const startTime = Date.now();
   const symbolTrees: Array<Symbol> = new Array(files.length);
   const symbolIndexingOperations: Array<Promise<void>> = new Array(files.length);
-  const workerPool = new IndexerWorkerPool();
+  const workerPool = new IndexerWorkerPool(Math.floor(files.length / 125));
+
+  parserLogger.info(tag.Indexer, "Indexing " + files.length + " files");
 
   for (let i = 0; i < files.length; i++) {
     const fileName = files[i].path;
@@ -55,6 +58,10 @@ export async function run(files: SourceFile[], config: LanguageConfig): Promise<
 
   workerPool.destroy();
 
+  const endTime = Date.now();
+
+  parserLogger.info(tag.Indexer, "Indexing took " + (endTime - startTime) + "ms");
+
   return symbolTrees;
 }