refactor(project): API refactoring

Author: RandomByte

packages/project/lib/build/ProjectBuilder.js

@@ -5,6 +5,7 @@ import composeProjectList from "./helpers/composeProjectList.js";
 import BuildContext from "./helpers/BuildContext.js";
 import prettyHrtime from "pretty-hrtime";
 import OutputStyleEnum from "./helpers/ProjectBuilderOutputStyle.js";
+import createBuildManifest from "./helpers/createBuildManifest.js";
 
 /**
  * @public
@@ -140,7 +141,6 @@ class ProjectBuilder {
 		destPath, cleanDest = false,
 		includedDependencies = [], excludedDependencies = [],
 		dependencyIncludes,
-		cacheDir,
 		watch,
 	}) {
 		if (!destPath && !watch) {
@@ -179,7 +179,7 @@ class ProjectBuilder {
 			}
 		}
 
-		const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects, cacheDir);
+		const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects);
 		const cleanupSigHooks = this._registerCleanupSigHooks();
 		let fsTarget;
 		if (destPath) {
@@ -274,9 +274,13 @@ class ProjectBuilder {
 					pWrites.push(this._writeResults(projectBuildContext, fsTarget));
 				}
 
-				if (cacheDir && !alreadyBuilt.includes(projectName)) {
-					this.#log.verbose(`Serializing cache...`);
-					pWrites.push(projectBuildContext.getBuildCache().serializeToDisk());
+				if (!alreadyBuilt.includes(projectName)) {
+					this.#log.verbose(`Saving cache...`);
+					const metadata = await createBuildManifest(
+						project,
+						this._graph, this._buildContext.getBuildConfig(), this._buildContext.getTaskRepository(),
+						projectBuildContext.getBuildSignature());
+					pWrites.push(projectBuildContext.getBuildCache().saveToDisk(metadata));
 				}
 			}
 			await Promise.all(pWrites);
@@ -294,7 +298,7 @@ class ProjectBuilder {
 				return projectBuildContext.getProject();
 			});
 			const watchHandler = this._buildContext.initWatchHandler(relevantProjects, async () => {
-				await this.#update(projectBuildContexts, requestedProjects, fsTarget, cacheDir);
+				await this.#update(projectBuildContexts, requestedProjects, fsTarget);
 			});
 			return watchHandler;
 
@@ -315,7 +319,7 @@ class ProjectBuilder {
 		}
 	}
 
-	async #update(projectBuildContexts, requestedProjects, fsTarget, cacheDir) {
+	async #update(projectBuildContexts, requestedProjects, fsTarget) {
 		const queue = [];
 		await this._graph.traverseDepthFirst(async ({project}) => {
 			const projectName = project.getName();
@@ -362,17 +366,14 @@ class ProjectBuilder {
 				pWrites.push(this._writeResults(projectBuildContext, fsTarget));
 			}
 
-			if (cacheDir) {
-				this.#log.verbose(`Updating cache...`);
-				// TODO: Only serialize if cache has changed
-				// TODO: Serialize lazily, or based on memory pressure
-				pWrites.push(projectBuildContext.getBuildCache().serializeToDisk());
-			}
+			this.#log.verbose(`Updating cache...`);
+			// TODO: Serialize lazily, or based on memory pressure
+			pWrites.push(projectBuildContext.getBuildCache().saveToDisk());
 		}
 		await Promise.all(pWrites);
 	}
 
-	async _createRequiredBuildContexts(requestedProjects, cacheDir) {
+	async _createRequiredBuildContexts(requestedProjects) {
 		const requiredProjects = new Set(this._graph.getProjectNames().filter((projectName) => {
 			return requestedProjects.includes(projectName);
 		}));
@@ -382,8 +383,7 @@ class ProjectBuilder {
 		for (const projectName of requiredProjects) {
 			this.#log.verbose(`Creating build context for project ${projectName}...`);
 			const projectBuildContext = await this._buildContext.createProjectContext({
-				project: this._graph.getProject(projectName),
-				cacheDir,
+				project: this._graph.getProject(projectName)
 			});
 
 			projectBuildContexts.set(projectName, projectBuildContext);
@@ -488,12 +488,9 @@ class ProjectBuilder {
 
 		if (createBuildManifest) {
 			// Create and write a build manifest metadata file
-			const {
-				default: createBuildManifest
-			} = await import("./helpers/createBuildManifest.js");
 			const metadata = await createBuildManifest(
 				project, this._graph, buildConfig, this._buildContext.getTaskRepository(),
-				projectBuildContext.getBuildCache());
+				projectBuildContext.getBuildSignature());
 			await target.write(resourceFactory.createResource({
 				path: `/.ui5/build-manifest.json`,
 				string: JSON.stringify(metadata, null, "\t")

packages/project/lib/build/TaskRunner.js

@@ -16,13 +16,14 @@ class TaskRunner {
 	 * @param {object} parameters.graph
 	 * @param {object} parameters.project
 	 * @param {@ui5/logger/loggers/ProjectBuild} parameters.log Logger to use
+	 * @param {@ui5/project/build/cache/ProjectBuildCache} parameters.buildCache Build cache instance
 	 * @param {@ui5/project/build/helpers/TaskUtil} parameters.taskUtil TaskUtil instance
 	 * @param {@ui5/builder/tasks/taskRepository} parameters.taskRepository Task repository
 	 * @param {@ui5/project/build/ProjectBuilder~BuildConfiguration} parameters.buildConfig
 	 * 			Build configuration
 	 */
-	constructor({graph, project, log, cache, taskUtil, taskRepository, buildConfig}) {
-		if (!graph || !project || !log || !cache || !taskUtil || !taskRepository || !buildConfig) {
+	constructor({graph, project, log, buildCache, taskUtil, taskRepository, buildConfig}) {
+		if (!graph || !project || !log || !buildCache || !taskUtil || !taskRepository || !buildConfig) {
 			throw new Error("TaskRunner: One or more mandatory parameters not provided");
 		}
 		this._project = project;
@@ -31,7 +32,7 @@ class TaskRunner {
 		this._taskRepository = taskRepository;
 		this._buildConfig = buildConfig;
 		this._log = log;
-		this._cache = cache;
+		this._buildCache = buildCache;
 
 		this._directDependencies = new Set(this._taskUtil.getDependencies());
 	}
@@ -192,38 +193,35 @@ class TaskRunner {
 				options.projectNamespace = this._project.getNamespace();
 
 				// TODO: Apply cache and stage handling for custom tasks as well
-				this._project.useStage(taskName);
-
-				// Check whether any of the relevant resources have changed
-				if (this._cache.hasCacheForTask(taskName)) {
-					await this._cache.validateChangedProjectResources(
-						taskName, this._project.getReader(), this._allDependenciesReader);
-					if (this._cache.hasValidCacheForTask(taskName)) {
-						this._log.skipTask(taskName);
-						return;
-					}
+				const requiresRun = await this._buildCache.prepareTaskExecution(taskName, this._allDependenciesReader);
+				if (!requiresRun) {
+					this._log.skipTask(taskName);
+					return;
 				}
+
+				const expectedOutput = new Set(); // TODO: Determine expected output properly
+
 				this._log.info(
 					`Executing task ${taskName} for project ${this._project.getName()}`);
 				const workspace = createTracker(this._project.getWorkspace());
 				const params = {
 					workspace,
 					taskUtil: this._taskUtil,
-					options,
-					buildCache: {
+					cacheUtil: {
 						// TODO: Create a proper interface for this
 						hasCache: () => {
-							return this._cache.hasCacheForTask(taskName);
+							return this._buildCache.hasTaskCache(taskName);
 						},
 						getChangedProjectResourcePaths: () => {
-							return this._cache.getChangedProjectResourcePaths(taskName);
+							return this._buildCache.getChangedProjectResourcePaths(taskName);
 						},
 						getChangedDependencyResourcePaths: () => {
-							return this._cache.getChangedDependencyResourcePaths(taskName);
+							return this._buildCache.getChangedDependencyResourcePaths(taskName);
 						},
-					}
+					},
+					options,
 				};
-				// const invalidatedResources = this._cache.getDepsOfInvalidatedResourcesForTask(taskName);
+				// const invalidatedResources = this._buildCache.getDepsOfInvalidatedResourcesForTask(taskName);
 				// if (invalidatedResources) {
 				// 	params.invalidatedResources = invalidatedResources;
 				// }
@@ -246,7 +244,7 @@ class TaskRunner {
 						`Task ${taskName} finished in ${Math.round((performance.now() - this._taskStart))} ms`);
 				}
 				this._log.endTask(taskName);
-				await this._cache.updateTaskResult(taskName, workspace, dependencies);
+				await this._buildCache.recordTaskResult(taskName, expectedOutput, workspace, dependencies);
 			};
 		}
 		this._tasks[taskName] = {
@@ -319,6 +317,8 @@ class TaskRunner {
 
 		// Tasks can provide an optional callback to tell build process which dependencies they require
 		const requiredDependenciesCallback = await task.getRequiredDependenciesCallback();
+		const getBuildSignatureCallback = await task.getBuildSignatureCallback();
+		const getExpectedOutputCallback = await task.getExpectedOutputCallback();
 		const specVersion = task.getSpecVersion();
 		let requiredDependencies;
 
@@ -390,6 +390,8 @@ class TaskRunner {
 				taskName: newTaskName,
 				taskConfiguration: taskDef.configuration,
 				provideDependenciesReader,
+				getBuildSignatureCallback,
+				getExpectedOutputCallback,
 				getDependenciesReader: () => {
 					// Create the dependencies reader on-demand
 					return this._createDependenciesReader(requiredDependencies);
@@ -488,7 +490,7 @@ class TaskRunner {
 	 * @returns {Promise} Resolves when task has finished
 	 */
 	async _executeTask(taskName, taskFunction, taskParams) {
-		if (this._cache.hasValidCacheForTask(taskName)) {
+		if (this._buildCache.isTaskCacheValid(taskName)) {
 			// Immediately skip task if cache is valid
 			// Continue if cache is (potentially) invalid, in which case taskFunction will
 			// validate the cache thoroughly

packages/project/lib/build/cache/BuildTaskCache.js

@@ -1,5 +1,6 @@
 import micromatch from "micromatch";
 import {getLogger} from "@ui5/logger";
+import {createResourceIndex} from "./utils.js";
 const log = getLogger("build:cache:BuildTaskCache");
 
 /**
@@ -37,23 +38,23 @@ function unionObject(target, obj) {
 	}
 }
 
-async function createMetadataForResources(resourceMap) {
-	const metadata = Object.create(null);
-	await Promise.all(Object.keys(resourceMap).map(async (resourcePath) => {
-		const resource = resourceMap[resourcePath];
-		if (resource.hash) {
-			// Metadata object
-			metadata[resourcePath] = resource;
-			return;
-		}
-		// Resource instance
-		metadata[resourcePath] = {
-			hash: await resource.getHash(),
-			lastModified: resource.getStatInfo()?.mtimeMs,
-		};
-	}));
-	return metadata;
-}
+// async function createMetadataForResources(resourceMap) {
+// 	const metadata = Object.create(null);
+// 	await Promise.all(Object.keys(resourceMap).map(async (resourcePath) => {
+// 		const resource = resourceMap[resourcePath];
+// 		if (resource.hash) {
+// 			// Metadata object
+// 			metadata[resourcePath] = resource;
+// 			return;
+// 		}
+// 		// Resource instance
+// 		metadata[resourcePath] = {
+// 			integrity: await resource.getIntegrity(),
+// 			lastModified: resource.getLastModified(),
+// 		};
+// 	}));
+// 	return metadata;
+// }
 
 /**
  * Manages the build cache for a single task
@@ -89,7 +90,7 @@ export default class BuildTaskCache {
 	 * @param {string} taskName - Name of the task
 	 * @param {TaskCacheMetadata} metadata - Task cache metadata
 	 */
-	constructor(projectName, taskName, {projectRequests, dependencyRequests, resourcesRead, resourcesWritten}) {
+	constructor(projectName, taskName, {projectRequests, dependencyRequests, input, output}) {
 		this.#projectName = projectName;
 		this.#taskName = taskName;
 
@@ -102,8 +103,8 @@ export default class BuildTaskCache {
 			pathsRead: [],
 			patterns: [],
 		};
-		this.#resourcesRead = resourcesRead ?? Object.create(null);
-		this.#resourcesWritten = resourcesWritten ?? Object.create(null);
+		this.#resourcesRead = input ?? Object.create(null);
+		this.#resourcesWritten = output ?? Object.create(null);
 	}
 
 	// ===== METADATA ACCESS =====
@@ -122,8 +123,8 @@ export default class BuildTaskCache {
 	 *
 	 * @param {RequestMetadata} projectRequests - Project resource requests
 	 * @param {RequestMetadata} [dependencyRequests] - Dependency resource requests
-	 * @param {Object.<string, object>} resourcesRead - Resources read by task
-	 * @param {Object.<string, object>} resourcesWritten - Resources written by task
+	 * @param {Object<string, object>} resourcesRead - Resources read by task
+	 * @param {Object<string, object>} resourcesWritten - Resources written by task
 	 * @returns {void}
 	 */
 	updateMetadata(projectRequests, dependencyRequests, resourcesRead, resourcesWritten) {
@@ -144,15 +145,12 @@ export default class BuildTaskCache {
 	 *
 	 * @returns {Promise<object>} Serialized task cache data
 	 */
-	async toJSON() {
+	async createMetadata() {
 		return {
-			taskName: this.#taskName,
-			resourceMetadata: {
-				projectRequests: this.#projectRequests,
-				dependencyRequests: this.#dependencyRequests,
-				resourcesRead: await createMetadataForResources(this.#resourcesRead),
-				resourcesWritten: await createMetadataForResources(this.#resourcesWritten)
-			}
+			projectRequests: this.#projectRequests,
+			dependencyRequests: this.#dependencyRequests,
+			taskIndex: await createResourceIndex(Object.values(this.#resourcesRead)),
+			// resourcesWritten: await createMetadataForResources(this.#resourcesWritten)
 		};
 	}