[POC] Introduce Task Workers

Implementing UI5/cli#897

Based on #955

Author: RandomByte

lib/processors/minifier.js

@@ -2,11 +2,26 @@ import {fileURLToPath} from "node:url";
 import posixPath from "node:path/posix";
 import {promisify} from "node:util";
 import os from "node:os";
-import workerpool from "workerpool";
-import Resource from "@ui5/fs/Resource";
-import {getLogger} from "@ui5/logger";
-const log = getLogger("builder:processors:minifier");
 import {setTimeout as setTimeoutPromise} from "node:timers/promises";
+import {minify} from "terser";
+
+/**
+ * @private
+ * @module @ui5/builder/tasks/minifyWorker
+ */
+
+/**
+ * Preserve comments which contain:
+ * <ul>
+ *  <li>copyright notice</li>
+ *  <li>license terms</li>
+ *  <li>"@ui5-bundle"</li>
+ *  <li>"@ui5-bundle-raw-include"</li>
+ * </ul>
+ *
+ * @type {RegExp}
+ */
+const copyrightCommentsAndBundleCommentPattern = /copyright|\(c\)(?:[0-9]+|\s+[0-9A-Za-z])|released under|license|\u00a9|^@ui5-bundle-raw-include |^@ui5-bundle /i;
 
 const debugFileRegex = /((?:\.view|\.fragment|\.controller|\.designtime|\.support)?\.js)$/;
 
@@ -21,40 +36,6 @@ const httpPattern = /^https?:\/\//i;
 // Shared workerpool across all executions until the taskUtil cleanup is triggered
 let pool;
 
-function getPool(taskUtil) {
-	if (!pool) {
-		log.verbose(`Creating workerpool with up to ${maxWorkers} workers (available CPU cores: ${osCpus})`);
-		const workerPath = fileURLToPath(new URL("./minifierWorker.js", import.meta.url));
-		pool = workerpool.pool(workerPath, {
-			workerType: "auto",
-			maxWorkers
-		});
-		taskUtil.registerCleanupTask((force) => {
-			const attemptPoolTermination = async () => {
-				log.verbose(`Attempt to terminate the workerpool...`);
-
-				if (!pool) {
-					return;
-				}
-
-				// There are many stats that could be used, but these ones seem the most
-				// convenient. When all the (available) workers are idle, then it's safe to terminate.
-				let {idleWorkers, totalWorkers} = pool.stats();
-				while (idleWorkers !== totalWorkers && !force) {
-					await setTimeoutPromise(100); // Wait a bit workers to finish and try again
-					({idleWorkers, totalWorkers} = pool.stats());
-				}
-
-				const poolToBeTerminated = pool;
-				pool = null;
-				return poolToBeTerminated.terminate(force);
-			};
-
-			return attemptPoolTermination();
-		});
-	}
-	return pool;
-}
 
 async function minifyInWorker(options, taskUtil) {
 	return getPool(taskUtil).exec("execMinification", [options]);
@@ -159,24 +140,12 @@ async function getSourceMapFromUrl({sourceMappingUrl, resourcePath, readFile}) {
  *   Promise resolving with object of resource, dbgResource and sourceMap
  */
 export default async function({
-	resources, fs, taskUtil, options: {readSourceMappingUrl = false, addSourceMappingUrl = true, useWorkers = false
-	} = {}}) {
-	let minify;
+	resources, fs, options: {readSourceMappingUrl = false, addSourceMappingUrl = true, useWorkers = false
+	} = {}, log, resourceFactory}) {
 	if (readSourceMappingUrl && !fs) {
 		throw new Error(`Option 'readSourceMappingUrl' requires parameter 'fs' to be provided`);
 	}
 
-	if (useWorkers) {
-		if (!taskUtil) {
-			// TaskUtil is required for worker support
-			throw new Error(`Minifier: Option 'useWorkers' requires a taskUtil instance to be provided`);
-		}
-		minify = minifyInWorker;
-	} else {
-		// Do not use workerpool
-		minify = (await import("./minifierWorker.js")).default;
-	}
-
 	return Promise.all(resources.map(async (resource) => {
 		const resourcePath = resource.getPath();
 		const dbgPath = resourcePath.replace(debugFileRegex, "-dbg$1");
@@ -248,7 +217,7 @@ export default async function({
 					sourceMapJson.file = dbgFilename;
 
 					// Then create a new resource
-					dbgSourceMapResource = new Resource({
+					dbgSourceMapResource = resourceFactory.createResource({
 						string: JSON.stringify(sourceMapJson),
 						path: dbgPath + ".map"
 					});
@@ -265,19 +234,39 @@ export default async function({
 				}
 			}
 		}
-
-		const result = await minify({
-			filename,
-			dbgFilename,
-			code,
-			sourceMapOptions
-		}, taskUtil);
-		resource.setString(result.code);
-		const sourceMapResource = new Resource({
-			path: resource.getPath() + ".map",
-			string: result.map
-		});
-		return {resource, dbgResource, sourceMapResource, dbgSourceMapResource};
+		try {
+			const result = await minify({
+				// Use debug-name since this will be referenced in the source map "sources"
+				[dbgFilename]: code
+			}, {
+				output: {
+					comments: copyrightCommentsAndBundleCommentPattern,
+					wrap_func_args: false
+				},
+				compress: false,
+				mangle: {
+					reserved: [
+						"jQuery",
+						"jquery",
+						"sap",
+					]
+				},
+				sourceMap: sourceMapOptions
+			});
+			resource.setString(result.code);
+			const sourceMapResource = resourceFactory.createResource({
+				path: resource.getPath() + ".map",
+				string: result.map
+			});
+			return {resource, dbgResource, sourceMapResource, dbgSourceMapResource};
+		} catch (err) {
+			// Note: err.filename contains the debug-name
+			throw new Error(
+				`Minification failed with error: ${err.message} in file ${filename} ` +
+				`(line ${err.line}, col ${err.col}, pos ${err.pos})`, {
+					cause: err
+				});
+		}
 	}));
 }
 

lib/tasks/minify.js

@@ -1,6 +1,3 @@
-import minifier from "../processors/minifier.js";
-import fsInterface from "@ui5/fs/fsInterface";
-
 /**
  * @public
  * @module @ui5/builder/tasks/minify
@@ -16,6 +13,7 @@ import fsInterface from "@ui5/fs/fsInterface";
  * @param {object} parameters Parameters
  * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files
  * @param {@ui5/project/build/helpers/TaskUtil|object} [parameters.taskUtil] TaskUtil
+ * @param {object} parameters.processors
  * @param {object} parameters.options Options
  * @param {string} parameters.options.pattern Pattern to locate the files to be processed
  * @param {boolean} [parameters.options.omitSourceMapResources=false] Whether source map resources shall
@@ -26,13 +24,12 @@ import fsInterface from "@ui5/fs/fsInterface";
  * @returns {Promise<undefined>} Promise resolving with <code>undefined</code> once data has been written
  */
 export default async function({
-	workspace, taskUtil, options: {pattern, omitSourceMapResources = false, useInputSourceMaps = true
-	}}) {
+	workspace, taskUtil, processors, options: {pattern, omitSourceMapResources = false, useInputSourceMaps = true}
+}) {
 	const resources = await workspace.byGlob(pattern);
-	const processedResources = await minifier({
+	const processedResources = await processors.execute("minifier", {
 		resources,
-		fs: fsInterface(workspace),
-		taskUtil,
+		reader: workspace,
 		options: {
 			addSourceMappingUrl: !omitSourceMapResources,
 			readSourceMappingUrl: !!useInputSourceMaps,

lib/tasks/taskRepository.js

@@ -1,4 +1,5 @@
 import {createRequire} from "node:module";
+import {fileURLToPath} from "node:url";
 
 /**
  * Repository providing access to all UI5 Builder tasks and various metadata required by the build process.
@@ -21,7 +22,14 @@ const taskInfos = {
 	executeJsdocSdkTransformation: {path: "./jsdoc/executeJsdocSdkTransformation.js"},
 	generateApiIndex: {path: "./jsdoc/generateApiIndex.js"},
 	generateJsdoc: {path: "./jsdoc/generateJsdoc.js"},
-	minify: {path: "./minify.js"},
+	minify: {
+		path: "./minify.js",
+		processors: {
+			minifier: {
+				path: "../processors/minifier.js"
+			}
+		}
+	},
 	buildThemes: {path: "./buildThemes.js"},
 	transformBootstrapHtml: {path: "./transformBootstrapHtml.js"},
 	generateLibraryManifest: {path: "./generateLibraryManifest.js"},
@@ -67,8 +75,20 @@ export async function getTask(taskName) {
 	}
 	try {
 		const {default: task} = await import(taskInfo.path);
+		let processors = null;
+		if (taskInfo.processors) {
+			processors = Object.create(null);
+			for (const processorName in taskInfo.processors) {
+				if (Object.hasOwn(taskInfo.processors, processorName)) {
+					processors[processorName] = {
+						path: fileURLToPath(new URL(taskInfo.processors[processorName].path, import.meta.url))
+					};
+				}
+			}
+		}
 		return {
-			task
+			task,
+			processors
 		};
 	} catch (err) {
 		throw new Error(`taskRepository: Failed to require task module for ${taskName}: ${err.message}`);