Skip to content

Commit ac9af00

Browse files
ShukantPalShukantPal
committed
Add workers for highlighting sources.
1 parent 6298033 commit ac9af00

File tree

10 files changed

+162
-36
lines changed

10 files changed

+162
-36
lines changed

‎example/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
"build-pixi-api": "cd ../../pixi-api && webdoc --verbose && cd ../webdoc/example",
2929
"build-pixi-api-prod": "cd ../../pixi-api && webdoc --site-root pixi-api && cd ../webdoc/example",
3030
"build-pixi-api-gcp": "cd ../../pixi-api && webdoc --tutorials ./projects/guides --verbose && cd ../webdoc/example",
31+
"build-pixi-api-gcp-inspect": "cd ../../pixi-api && node --inspect-brk --trace-warnings ../webdoc/example/node_modules/@webdoc/cli/cli.js --tutorials ./projects/guides --verbose && cd ../webdoc/example",
3132
"build-pixi-guides": "cd ../../guides && webdoc --tutorials docs && cd ../webdoc/example",
3233
"build-mongodb": "cd ../../node-mongodb-native && webdoc --verbose && cd ../webdoc/example"
3334
},

‎packages/webdoc-cli/src/index.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,9 @@ async function main(argv: yargs.Argv) {
143143
tutorials,
144144
source: sourceFiles,
145145
verbose: !!argv.verbose,
146+
cmdLine: {
147+
mainThread: argv.mainThread,
148+
},
146149
};
147150

148151
if (template.publish && typeof template.publish === "function") {

‎packages/webdoc-default-template/helper/workers/hl.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// @flow
2+
3+
const fs = require("fs");
4+
const hljs = require("highlight.js");
5+
// $FlowFixMe
6+
const {isMainThread, parentPort} = require("worker_threads");
7+
8+
function onMessage(data /*: { id: number, file: string } */) {
9+
const {id, file} = data;
10+
11+
fs.readFile(file, "utf8", function onFileRead(err, raw) {
12+
if (err) {
13+
return parentPort.postMessage({
14+
id,
15+
file,
16+
error: true,
17+
result: null,
18+
});
19+
}
20+
21+
const languageSubset = file.endsWith(".js") ? ["javascript"] : undefined;
22+
const result = hljs.highlightAuto(raw, languageSubset).value;
23+
24+
parentPort.postMessage({
25+
id,
26+
file,
27+
error: false,
28+
result,
29+
});
30+
});
31+
}
32+
33+
if (!isMainThread) {
34+
parentPort.on("message", onMessage);
35+
}

‎packages/webdoc-default-template/publish.js‎

Lines changed: 111 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
// @flow
22

3+
// $FlowFixMe
4+
const {Worker} = require("worker_threads");
35
const {crawl} = require("./helper/crawl");
46
const fs = require("fs");
57
const fse = require("fs-extra");
68
const hljs = require("highlight.js");
9+
const os = require("os");
710
const path = require("path");
811
const {traverse} = require("@webdoc/model");
912
const {
@@ -150,12 +153,14 @@ exports.publish = async function publish(options /*: PublishOptions */) {
150153
});
151154

152155
await outStaticFiles(outDir, config);
153-
outSource(outDir, pipeline, options.config, source);
154-
outExplorerData(outDir, crawlData);
155-
outMainPage(indexRelative ? path.join(outDir, indexRelative) : null, pipeline, options.config);
156-
outIndexes(outDir, pipeline, options.config, crawlData.index);
157-
outReference(outDir, pipeline, options.config, docTree, crawlData.reference);
158-
outTutorials(outDir, pipeline, options.config, docTree, crawlData.tutorials);
156+
await Promise.all([
157+
outSource(outDir, pipeline, options.config, source, options.cmdLine.mainThread || false),
158+
outExplorerData(outDir, crawlData),
159+
outMainPage(indexRelative ? path.join(outDir, indexRelative) : null, pipeline, options.config),
160+
outIndexes(outDir, pipeline, options.config, crawlData.index),
161+
outReference(outDir, pipeline, options.config, docTree, crawlData.reference),
162+
outTutorials(outDir, pipeline, options.config, docTree, crawlData.tutorials),
163+
]);
159164

160165
pipeline.close();
161166
};
@@ -209,10 +214,13 @@ async function outStaticFiles(
209214
}
210215

211216
// Write the explorer JSON data in the output directory
212-
function outExplorerData(outDir /*: string */, crawlData /*: CrawlData */) {
217+
async function outExplorerData(
218+
outDir /*: string */,
219+
crawlData /*: CrawlData */,
220+
) /*: Promise<void> */ {
213221
const explorerDir = path.join(outDir, "./explorer");
214222

215-
fse.ensureDir(explorerDir).then(() => {
223+
return fse.ensureDir(explorerDir).then(() => new Promise((resolve) => {
216224
fse.writeFile(
217225
path.join(explorerDir, "./reference.json"),
218226
JSON.stringify(crawlData.reference),
@@ -228,18 +236,21 @@ function outExplorerData(outDir /*: string */, crawlData /*: CrawlData */) {
228236
"utf8",
229237
(err) => {
230238
if (err) throw err;
239+
resolve();
231240
},
232241
);
242+
} else {
243+
resolve();
233244
}
234-
});
245+
}));
235246
}
236247

237248
// Render the main-page into index.tmpl (outputFile)
238249
async function outMainPage(
239250
outputFile /*: ?string */,
240251
pipeline /*: TemplatePipeline */,
241252
config /*: WebdocConfig */,
242-
) {
253+
)/*: Promise<void> */ {
243254
if (outputFile && config.template.readme) {
244255
const readmeFile = path.join(process.cwd(), config.template.readme);
245256

@@ -252,28 +263,97 @@ async function outMainPage(
252263
}
253264
}
254265

255-
function outSource(
266+
async function outSource(
256267
outDir /*: string */,
257268
pipeline /*: TemplatePipeline */,
258269
config /*: ConfigSchema */,
259270
source /*: ?$ReadOnlyArray<SourceFile> */,
260-
) {
261-
if (source) {
271+
mainThread /*:: ?: boolean */,
272+
)/*: Promise<void> */ {
273+
if (!source) return;
274+
275+
function renderSource(file /*: SourceFile */, raw /*: string */) {
276+
const pkgName = file.package.name || "";
277+
const pkgRelativePath = path.relative(file.package.location || "", file.path);
278+
const outFile = path.join(pkgName, pkgRelativePath + ".html");
279+
280+
pipeline.render("source.tmpl", {
281+
appBar: {current: "sources"},
282+
env: config,
283+
raw,
284+
title: path.basename(file.path),
285+
}, {
286+
outputFile: path.join(outDir, outFile),
287+
});
288+
}
289+
290+
const workerCount = Math.min(os.cpus().length, 1 + Math.floor(source.length / 32));
291+
292+
if (workerCount > 1 && !mainThread) {
293+
const workers = new Array(workerCount);
294+
const renderJobs/*: Array<{
295+
resolve: Function,
296+
reject: Function,
297+
promise: Promise<void>,
298+
}> */ = new Array(source.length);// eslint-disable-line operator-linebreak
299+
300+
const onMessage = function onMessage(
301+
{
302+
id,
303+
file,
304+
error,
305+
result,
306+
} /*: {
307+
id: number,
308+
file: string,
309+
error: boolean,
310+
result: ?string }
311+
*/,
312+
) {
313+
if (error || typeof result !== "string") {
314+
renderJobs[id].reject("Error in highlighting worker");
315+
} else {
316+
const raw = result;
317+
const sourceFile = source[id];
318+
319+
renderSource(sourceFile, raw);
320+
renderJobs[id].resolve();
321+
}
322+
};
323+
324+
const startTime = Date.now();
325+
console.log("Creating " + workerCount + " workers for highlighting source code.");
326+
327+
for (let i = 0; i < workers.length; i++) {
328+
workers[i] = new Worker(path.resolve(__dirname, "./helper/workers/hl.js"));
329+
workers[i].on("message", onMessage);
330+
}
331+
332+
for (let i = 0; i < source.length; i++) {
333+
let resolveFn/*: () => void */;
334+
let rejectFn/*: () => void */;
335+
const promise = new Promise((resolve, reject) => {
336+
resolveFn = resolve;
337+
rejectFn = reject;
338+
});
339+
renderJobs[i] = {resolve: resolveFn, reject: rejectFn, promise};
340+
341+
workers[i % workers.length].postMessage({
342+
id: i,
343+
file: source[i].path,
344+
});
345+
}
346+
347+
await Promise.all(renderJobs.map((job) => job.promise));
348+
await Promise.all(workers.map((worker) => worker.terminate()));
349+
350+
console.log("Rendering sources took " + (Date.now() - startTime) + "ms time!");
351+
} else {
262352
for (const file of source) {
263353
const raw = hljs.highlightAuto(
264354
fs.readFileSync(path.resolve(process.cwd(), file.path), "utf8"),
265355
).value;
266-
const pkgName = file.package.name || "";
267-
const outFile = path.join(pkgName, file.path + ".html");
268-
269-
pipeline.render("source.tmpl", {
270-
appBar: {current: "sources"},
271-
env: config,
272-
raw,
273-
title: path.basename(file.path),
274-
}, {
275-
outputFile: path.join(outDir, outFile),
276-
});
356+
renderSource(file, raw);
277357
}
278358
}
279359
}
@@ -283,7 +363,7 @@ async function outReadme(
283363
pipeline /*: TemplatePipeline */,
284364
config /*: WebdocConfig */,
285365
readmeFile /*: string */,
286-
) {
366+
)/*: Promise<void> */ {
287367
if (!(await fse.pathExists(readmeFile))) {
288368
return;
289369
}
@@ -310,12 +390,12 @@ async function outReadme(
310390
}, {outputFile});
311391
}
312392

313-
function outIndexes(
393+
async function outIndexes(
314394
outDir /*: string */,
315395
pipeline /*: TemplatePipeline */,
316396
config /*: WebdocConfig */,
317397
index /*: Index */,
318-
) {
398+
)/*: Promise<void> */ {
319399
const KEY_TO_TITLE = {
320400
"classes": "Class Index",
321401
};
@@ -341,13 +421,13 @@ function outIndexes(
341421
}
342422
}
343423

344-
function outReference(
424+
async function outReference(
345425
outDir /*: string */,
346426
pipeline /*: TemplatePipeline */,
347427
config /*: WebdocConfig */,
348428
docTree /*: RootDoc */,
349429
explorerData /* any */,
350-
) {
430+
)/*: Promise<void> */ {
351431
// Don't output if nothing's there
352432
if (!docTree.members.length) {
353433
return;
@@ -397,13 +477,13 @@ function outReference(
397477
}
398478
}
399479

400-
function outTutorials(
480+
async function outTutorials(
401481
outDir /*: string */,
402482
pipeline /*: TemplatePipeline */,
403483
config /*: WebdocConfig */,
404484
docTree /*: RootDoc */,
405485
explorerData /* any */,
406-
) {
486+
)/*: Promise<void> */ {
407487
function out(parent /*: { members: any[] } */) {
408488
return function renderRecursive(tutorial /*: TutorialDoc */, i /*: number */) {
409489
const uri = linker.getURI(tutorial, true);

‎packages/webdoc-default-template/src/styles/global.scss‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,7 @@ a {
121121
}
122122
}
123123
}
124+
125+
.search-container {
126+
margin-left: 12px;
127+
}

‎packages/webdoc-default-template/src/styles/source.scss‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131

3232
code {
3333
background-color: white !important;
34-
font-size: 14px;
34+
font-family: "Source Code Pro", "Roboto Mono", monospace;
35+
font-size: 12px;
3536
line-height: 24px;
3637
}
3738
}

‎packages/webdoc-default-template/static/styles/index.css‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/webdoc-default-template/tmpl/components/member/index.tmpl‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55
const { doc, require } = obj;
66
const linkSvg = this.getPlugin("linker").getResourceURI("icons/link.svg");
77
const pkgName = doc.loc && doc.loc.file && doc.loc.file.package && doc.loc.file.package.name || "";
8+
const pkgRelativePath = (doc.loc && doc.loc.file && doc.loc.file.package
9+
&& require("path").relative(doc.loc.file.package.location || "", doc.loc.file.path));
810
const source = doc.loc && doc.loc.file && doc.loc.file.path
9-
&& this.getPlugin("linker").getResourceURI(require("path").join(pkgName, doc.loc.file.path + ".html"));
11+
&& this.getPlugin("linker").getResourceURI(require("path").join(pkgName, pkgRelativePath + ".html"));
1012
const sourceName = source && require("path").basename(doc.loc.file.path);
1113
const sourceStart = source && doc.loc.start ? ":" + doc.loc.start.line : "";
1214
const sourceLinkFragment = source && doc.loc.start ? "#" + doc.loc.start.line : "";

‎packages/webdoc-default-template/tmpl/layout.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ const cssPageKind =
2828
<?js } ?>
2929

3030
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap" />
31-
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono&family=Roboto+Mono&display=swap" />
31+
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono&family=Roboto+Mono&family=Source+Code+Pro&display=swap" />
3232
<link type="text/css" rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/10.0.3/styles/googlecode.min.css">
3333

3434
<?js Object.keys(config.meta).forEach((key) => { ?>

‎packages/webdoc-default-template/tmpl/source.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<div class="source">
88
<div class="source__split">
99
<ul>
10-
<?js lines.forEach((line, i) => { ?>
10+
<?js lines.forEach((line, i) => { i++; ?>
1111
<li id="<?js= i ?>">
1212
<a href="#<?js= i ?>"><?js= i ?></a>
1313
</li>

0 commit comments

Comments
 (0)