diff --git a/lib/build/ProjectBuilder.js b/lib/build/ProjectBuilder.js index 63f10aa1c..71bb99f2a 100644 --- a/lib/build/ProjectBuilder.js +++ b/lib/build/ProjectBuilder.js @@ -121,6 +121,7 @@ class ProjectBuilder { * @param {object} parameters Parameters * @param {string} parameters.destPath Target path * @param {boolean} [parameters.cleanDest=false] Decides whether project should clean the target path before build + * @param {boolean} [parameters.concurrentBuild=false] Whether to build projects concurrently if possible * @param {Array.} [parameters.includedDependencies=[]] * List of names of projects to include in the build result * If the wildcard '*' is provided, all dependencies will be included in the build result. @@ -133,7 +134,7 @@ class ProjectBuilder { * @returns {Promise} Promise resolving once the build has finished */ async build({ - destPath, cleanDest = false, + destPath, cleanDest = false, concurrentBuild = !!process.env.UI5_BUILD_CONCURRENT, includedDependencies = [], excludedDependencies = [], dependencyIncludes }) { @@ -170,45 +171,46 @@ class ProjectBuilder { `while including any dependencies into the build result`); } - const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects); + const projectContexts = await this._createRequiredBuildContexts(requestedProjects); const cleanupSigHooks = this._registerCleanupSigHooks(); const fsTarget = resourceFactory.createAdapter({ fsBasePath: destPath, virBasePath: "/" }); - const queue = []; const alreadyBuilt = []; - // Create build queue based on graph depth-first search to ensure correct build order - await this._graph.traverseDepthFirst(async ({project}) => { + const pendingDependencies = new Map(); + const pendingProjects = new Set(); + + // Start with a deep graph traversal to get a deterministic (and somewhat reasonable) build order + await this._graph.traverseDepthFirst(({project}) => { const projectName = project.getName(); - const projectBuildContext = projectBuildContexts.get(projectName); + const {projectBuildContext, requiredDependencies} = projectContexts.get(projectName); if (projectBuildContext) { // Build context exists // => This project needs to be built or, in case it has already // been built, it's build result needs to be written out (if requested) - queue.push(projectBuildContext); + pendingProjects.add(projectName); + pendingDependencies.set(projectName, new Set(requiredDependencies)); + if (!projectBuildContext.requiresBuild()) { alreadyBuilt.push(projectName); } } }); - log.setProjects(queue.map((projectBuildContext) => { - return projectBuildContext.getProject().getName(); - })); - if (queue.length > 1) { // Do not log if only the root project is being built - log.info(`Processing ${queue.length} projects`); + log.setProjects(Array.from(pendingProjects.keys())); + if (pendingProjects.size > 1) { // Do not log if only the root project is being built + log.info(`Processing ${pendingProjects.size} projects`); if (alreadyBuilt.length) { log.info(` Reusing build results of ${alreadyBuilt.length} projects`); - log.info(` Building ${queue.length - alreadyBuilt.length} projects`); + log.info(` Building ${pendingProjects.size - alreadyBuilt.length} projects`); } if (log.isLevelEnabled("verbose")) { - log.verbose(` Required projects:`); - log.verbose(` ${queue - .map((projectBuildContext) => { + log.verbose(` Required projects:\n ${Array.from(projectContexts.values()) + .map(({projectBuildContext}) => { const projectName = projectBuildContext.getProject().getName(); let msg; if (alreadyBuilt.includes(projectName)) { @@ -224,35 +226,78 @@ class ProjectBuilder { } } + const projectsInProcess = new Set(); + function hasPendingDependencies(projectName) { + const pendingDeps = pendingDependencies.get(projectName); + for (const depName of pendingDeps) { + if (!pendingProjects.has(depName) && !projectsInProcess.has(depName)) { + pendingDeps.delete(depName); + } + } + + if (log.isLevelEnabled("verbose")) { + log.verbose(`${projectName} is waiting for: ${Array.from(pendingDeps.keys()).join(", ")}`); + } + return !!pendingDeps.size; + } + if (cleanDest) { log.info(`Cleaning target directory...`); await rimraf(destPath); } const startTime = process.hrtime(); try { + const pBuilds = new Map(); const pWrites = []; - for (const projectBuildContext of queue) { - const projectName = projectBuildContext.getProject().getName(); - const projectType = projectBuildContext.getProject().getType(); - log.verbose(`Processing project ${projectName}...`); - - // Only build projects that are not already build (i.e. provide a matching build manifest) - if (alreadyBuilt.includes(projectName)) { - log.skipProjectBuild(projectName, projectType); - } else { - log.startProjectBuild(projectName, projectType); - await projectBuildContext.getTaskRunner().runTasks(); - log.endProjectBuild(projectName, projectType); + + while (pendingProjects.size) { + const buildCount = pBuilds.length; + for (const projectName of pendingProjects) { + if (!hasPendingDependencies(projectName)) { + projectsInProcess.add(projectName); + pendingProjects.delete(projectName); + // => Build the project + pBuilds.set(projectName, (async ({projectBuildContext}) => { + const projectName = projectBuildContext.getProject().getName(); + const projectType = projectBuildContext.getProject().getType(); + + // Only build projects that are not already build (i.e. provide a matching build manifest) + if (alreadyBuilt.includes(projectName)) { + log.skipProjectBuild(projectName, projectType); + } else { + log.startProjectBuild(projectName, projectType); + await projectBuildContext.getTaskRunner().runTasks(); + log.endProjectBuild(projectName, projectType); + } + projectsInProcess.delete(projectName); + if (!requestedProjects.includes(projectName)) { + // Project has not been requested + // => Its resources shall not be part of the build result + return; + } + + log.verbose(`Writing out files...`); + pWrites.push(this._writeResults(projectBuildContext, fsTarget)); + return projectName; + })(projectContexts.get(projectName))); + + if (!concurrentBuild) { + // Do not add more builds + break; + } + } } - if (!requestedProjects.includes(projectName)) { - // Project has not been requested - // => Its resources shall not be part of the build result - continue; + if (buildCount === pBuilds.length || !concurrentBuild) { + log.verbose(`Waiting for a build to finish`); + // Either schedule a build or wait till any promise resolved + const finishedProjectName = await Promise.any(pBuilds.values()); + pBuilds.delete(finishedProjectName); + log.verbose(`${finishedProjectName} finished building. Builds in process: ${pBuilds.size}`); } - - log.verbose(`Writing out files...`); - pWrites.push(this._writeResults(projectBuildContext, fsTarget)); } + // Wait for any remaining builds + await Promise.all(pBuilds); + // Wait for the deferred writes await Promise.all(pWrites); log.info(`Build succeeded in ${this._getElapsedTime(startTime)}`); } catch (err) { @@ -269,7 +314,7 @@ class ProjectBuilder { return requestedProjects.includes(projectName); })); - const projectBuildContexts = new Map(); + const buildContexts = new Map(); for (const projectName of requiredProjects) { log.verbose(`Creating build context for project ${projectName}...`); @@ -277,18 +322,23 @@ class ProjectBuilder { project: this._graph.getProject(projectName) }); - projectBuildContexts.set(projectName, projectBuildContext); + const projectInfo = { + projectBuildContext, + requiredDependencies: new Set() + }; + buildContexts.set(projectName, projectInfo); if (projectBuildContext.requiresBuild()) { const taskRunner = projectBuildContext.getTaskRunner(); const requiredDependencies = await taskRunner.getRequiredDependencies(); + projectInfo.requiredDependencies = requiredDependencies; if (requiredDependencies.size === 0) { continue; } // This project needs to be built and required dependencies to be built as well this._graph.getDependencies(projectName).forEach((depName) => { - if (projectBuildContexts.has(depName)) { + if (buildContexts.has(depName)) { // Build context already exists // => Dependency will be built return; @@ -302,7 +352,7 @@ class ProjectBuilder { } } - return projectBuildContexts; + return buildContexts; } async _getProjectFilter({