11'use strict' ;
22
3+ import { Worker , isMainThread , parentPort , workerData } from 'worker_threads' ;
4+ import os from 'os' ;
5+
36import publicGenerators from './generators/index.mjs' ;
47import astJs from './generators/ast-js/index.mjs' ;
58import oramaDb from './generators/orama-db/index.mjs' ;
@@ -12,6 +15,25 @@ const availableGenerators = {
1215 'orama-db' : oramaDb ,
1316} ;
1417
18+ // Thread pool max limit
19+ const MAX_THREADS = Math . max ( 1 , os . cpus ( ) . length - 1 ) ;
20+
21+ // If inside a worker thread, perform the generator logic here
22+ if ( ! isMainThread ) {
23+ const { name, dependencyOutput, extra } = workerData ;
24+ const generator = availableGenerators [ name ] ;
25+
26+ // Execute the generator and send the result back to the parent thread
27+ generator
28+ . generate ( dependencyOutput , extra )
29+ . then ( result => {
30+ parentPort . postMessage ( result ) ;
31+ } )
32+ . catch ( error => {
33+ parentPort . postMessage ( { error } ) ;
34+ } ) ;
35+ }
36+
1537/**
1638 * @typedef {{ ast: GeneratorMetadata<ApiDocMetadataEntry, ApiDocMetadataEntry>} } AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator
1739 * @typedef {AvailableGenerators & AstGenerator } AllGenerators A complete set of the available generators, including the AST one
@@ -43,30 +65,103 @@ const createGenerator = markdownInput => {
4365 */
4466 const cachedGenerators = { ast : Promise . resolve ( markdownInput ) } ;
4567
68+ // Keep track of how many threads are currently running
69+ let activeThreads = 0 ;
70+ const threadQueue = [ ] ;
71+
72+ /**
73+ *
74+ * @param name
75+ * @param dependencyOutput
76+ * @param extra
77+ */
78+ const runInWorker = ( name , dependencyOutput , extra ) => {
79+ return new Promise ( ( resolve , reject ) => {
80+ /**
81+ *
82+ */
83+ const run = ( ) => {
84+ activeThreads ++ ;
85+
86+ const worker = new Worker ( new URL ( import . meta. url ) , {
87+ workerData : { name, dependencyOutput, extra } ,
88+ } ) ;
89+
90+ worker . on ( 'message' , result => {
91+ activeThreads -- ;
92+ processQueue ( ) ;
93+
94+ if ( result && result . error ) {
95+ reject ( result . error ) ;
96+ } else {
97+ resolve ( result ) ;
98+ }
99+ } ) ;
100+
101+ worker . on ( 'error' , err => {
102+ activeThreads -- ;
103+ processQueue ( ) ;
104+ reject ( err ) ;
105+ } ) ;
106+ } ;
107+
108+ if ( activeThreads >= MAX_THREADS ) {
109+ threadQueue . push ( run ) ;
110+ } else {
111+ run ( ) ;
112+ }
113+ } ) ;
114+ } ;
115+
116+ /**
117+ *
118+ */
119+ const processQueue = ( ) => {
120+ if ( threadQueue . length > 0 && activeThreads < MAX_THREADS ) {
121+ const next = threadQueue . shift ( ) ;
122+ next ( ) ;
123+ }
124+ } ;
125+
46126 /**
47127 * Runs the Generator engine with the provided top-level input and the given generator options
48128 *
49129 * @param {GeneratorOptions } options The options for the generator runtime
50130 */
51- const runGenerators = async ( { generators, ...extra } ) => {
131+ const runGenerators = async ( {
132+ generators,
133+ disableParallelism = false ,
134+ ...extra
135+ } ) => {
52136 // Note that this method is blocking, and will only execute one generator per-time
53137 // but it ensures all dependencies are resolved, and that multiple bottom-level generators
54138 // can reuse the already parsed content from the top-level/dependency generators
55139 for ( const generatorName of generators ) {
56- const { dependsOn, generate } = availableGenerators [ generatorName ] ;
140+ const {
141+ dependsOn,
142+ generate,
143+ parallizable = true ,
144+ } = availableGenerators [ generatorName ] ;
57145
58146 // If the generator dependency has not yet been resolved, we resolve
59147 // the dependency first before running the current generator
60- if ( dependsOn && dependsOn in cachedGenerators === false ) {
61- await runGenerators ( { ...extra , generators : [ dependsOn ] } ) ;
148+ if ( dependsOn && ! ( dependsOn in cachedGenerators ) ) {
149+ await runGenerators ( {
150+ ...extra ,
151+ disableParallelism,
152+ generators : [ dependsOn ] ,
153+ } ) ;
62154 }
63155
64156 // Ensures that the dependency output gets resolved before we run the current
65157 // generator with its dependency output as the input
66158 const dependencyOutput = await cachedGenerators [ dependsOn ] ;
67159
68160 // Adds the current generator execution Promise to the cache
69- cachedGenerators [ generatorName ] = generate ( dependencyOutput , extra ) ;
161+ cachedGenerators [ generatorName ] =
162+ disableParallelism || ! parallizable
163+ ? generate ( dependencyOutput , extra ) // Run in main thread
164+ : runInWorker ( generatorName , dependencyOutput , extra ) ; // Offload to worker thread
70165 }
71166
72167 // Returns the value of the last generator of the current pipeline
0 commit comments