Add --no-workers flag in @webdoc/cli to disable worker threads.

Author: ShukantPal

example/package.json

@@ -27,7 +27,7 @@
     "build-next": "cd .. && webdoc && cd example",
     "build-pixi-api": "cd ../../pixi-api && webdoc --verbose && cd ../webdoc/example",
     "build-pixi-api-prod": "cd ../../pixi-api && webdoc --site-root pixi-api && cd ../webdoc/example",
-    "build-pixi-api-gcp": "cd ../../pixi-api && node --inspect-brk ../webdoc/example/node_modules/@webdoc/cli/cli --tutorials ./projects/guides --verbose && cd ../webdoc/example",
+    "build-pixi-api-gcp": "cd ../../pixi-api && webdoc --tutorials ./projects/guides --verbose && cd ../webdoc/example",
     "build-pixi-guides": "cd ../../guides && webdoc --tutorials docs && cd ../webdoc/example",
     "build-mongodb": "cd ../../node-mongodb-native && webdoc --verbose && cd ../webdoc/example"
   },

packages/webdoc-cli/README.md

@@ -19,6 +19,7 @@ generate the `sitemap.xml`. This is useful if you want to integrate with [Algoli
 * `--site-root <path>`: If using absolute links in a template, this will set the basepath. The basepath should the directory in which the documentation is being stored relative to where the server is running. The site root is "/" by default - which means that you'll need to serve the documentation directory as top-level. Note that @webdoc/default-template uses absolute links.
 * `-c <config-path>`: This sets the path of the configuration file webdoc uses.
 * `-u <tutorials-directory>` -  (optional) This should point to a directory containing tutorials written in Markdown (".md") or HTML ".html, ".htm". JSON files can be used to configure the hierarchy and naming of tutorials (see the Tutorial Configuration section).
+* `--no-workers` - Disables usage of worker threads to parallelize parsing.
 
 ### Configuration
 

packages/webdoc-cli/src/index.js

@@ -63,7 +63,6 @@ async function main(argv: yargs.Argv) {
   const config = loadConfig(argv.config);
   const tutorials = loadTutorials(argv.tutorials, config.template.routes.tutorials);
 
-
   if (argv.siteRoot) {
     config.template.siteRoot = argv.siteRoot;
   }
@@ -107,7 +106,9 @@ async function main(argv: yargs.Argv) {
   }
 
   try {
-    await parse(sourceFiles, documentTree);
+    await parse(sourceFiles, documentTree, {
+      mainThread: !argv.workers,
+    });
   } catch (e) {
     // Make sure we get that API structure out so the user can debug the problem!
     if (config.opts && config.opts.export) {
@@ -163,7 +164,9 @@ async function main(argv: yargs.Argv) {
 const argv = yargs.scriptName("@webdoc/cli")
   .usage("$0 -c <configFile> -u <tutorialDir> --verbose " +
     "--site-root <siteRoot> " +
-    "--site-domain <siteDomain>")
+    "--site-domain <siteDomain>" +
+    "--no-workers")
+  .default("workers", true)
   .default("config", path.join(process.cwd(), "webdoc.conf.json"), "webdoc config file")
   .alias("c", "config")
   .alias("u", "tutorials")

packages/webdoc-parser/index.flow.js

@@ -1,13 +1,9 @@
 declare module "@webdoc/parser" {
-  import type {Doc} from "@webdoc/types";
+  import type {RootDoc} from "@webdoc/types";
 
-  /**
-   * The parser will accept:
-   * + a file's contents
-   * + an array of file contents
-   + + a map mapping file-names to file contents
-   */
-  declare type ParserInput = string | string[] | Map<string, string>;
-
-  declare function parse(input: ParserInput): Doc;
+  declare function parse(
+    target: string | SourceFile[],
+    root?: RootDoc,
+    options?: $Shape<{ mainThread: boolean }>,
+  ): Promise<RootDoc>;
 }

packages/webdoc-parser/src/indexer/IndexerWorkerPool.js

@@ -5,6 +5,7 @@ import {parserLogger, tag} from "../Logger";
 import EventEmitter from "events";
 import type {LanguageConfig} from "../types/LanguageIntegration";
 import {type Symbol} from "../types/Symbol";
+// $FlowFixMe
 import {Worker} from "worker_threads";
 import os from "os";
 import path from "path";
@@ -83,7 +84,7 @@ export class IndexerWorkerPool {
 
   repair(symbol: Symbol, packages: { [id: string]: PackageDoc }): Symbol {
     const file = symbol.loc.file;
-    const pkgId = file?.package.id;
+    const pkgId = file && file.package.id;
 
     if (file && pkgId && packages[pkgId]) {
       file.package = packages[pkgId];

packages/webdoc-parser/src/indexer/index.js

@@ -28,15 +28,21 @@ export function register(lang: LanguageIntegration): void {
   }
 }
 
-export async function run(files: SourceFile[], config: LanguageConfig): Promise<Symbol[]> {
+export async function run(
+  files: SourceFile[],
+  config: LanguageConfig,
+  options?: $Shape<{
+    mainThread: boolean,
+  }> = {},
+): Promise<Symbol[]> {
   const startTime = Date.now();
   const maxThreads = Math.min(os.cpus().length, 1 + Math.floor(files.length / 125));
 
   parserLogger.info(tag.Indexer, "Indexing " + files.length + " files");
 
   const symbolTrees: Array<Symbol> = new Array(files.length);
 
-  if (maxThreads > 1) {
+  if (maxThreads > 1 && !options.mainThread) {
     const packages = _.keyBy(
       files.map((file) => file.package),
       (pkg) => pkg.id,

packages/webdoc-parser/src/indexer/worker.js

@@ -2,6 +2,7 @@
 
 import * as Indexer from "./";
 import type {JobData, JobResult} from "./IndexerWorkerPool";
+// $FlowFixMe
 import {isMainThread, parentPort} from "worker_threads";
 import fs from "fs";
 

packages/webdoc-parser/src/parse.js

@@ -80,11 +80,16 @@ export function buildSymbolTree(
  *
  * @param {string | SourceFile[]} target
  * @param {RootDoc} root
+ * @param {object}[options]
+ * @param {boolean}[options.mainThread] - Force the indexer to run on the main thread.
  * @return {RootDoc}
  */
 export async function parse(
   target: string | SourceFile[],
   root?: RootDoc = createRootDoc(),
+  options?: $Shape<{
+    mainThread: boolean
+  }>,
 ): Promise<RootDoc> {
   if (typeof target === "string") {
     target = [{
@@ -94,7 +99,7 @@ export async function parse(
     }];
   }
 
-  const perFileSymbolTrees = await Indexer.run(target, Webdoc.DEFAULT_LANG_CONFIG);
+  const perFileSymbolTrees = await Indexer.run(target, Webdoc.DEFAULT_LANG_CONFIG, options);
   const fullSymbolTree = assemble(perFileSymbolTrees);
 
   root.children = root.members;