Limit number of projects that are built concurrently

internal/compiler/projectreferenceparser.go

@@ -18,11 +18,7 @@ func (t *projectReferenceParseTask) parse(projectReferenceParser *projectReferen
 	if t.resolved == nil {
 		return
 	}
-	if t.resolved.SourceToProjectReference() == nil {
-		projectReferenceParser.wg.Queue(func() {
-			t.resolved.ParseInputOutputNames()
-		})
-	}
+	t.resolved.ParseInputOutputNames()
 	if subReferences := t.resolved.ResolvedProjectReferencePaths(); len(subReferences) > 0 {
 		t.subTasks = createProjectReferenceParseTasks(subReferences)
 	}

internal/core/buildoptions.go

@@ -3,10 +3,11 @@ package core
 type BuildOptions struct {
 	_ noCopy
 
-	Dry               Tristate `json:"dry,omitzero"`
-	Force             Tristate `json:"force,omitzero"`
-	Verbose           Tristate `json:"verbose,omitzero"`
-	StopBuildOnErrors Tristate `json:"stopBuildOnErrors,omitzero"`
+	Dry                   Tristate `json:"dry,omitzero"`
+	Force                 Tristate `json:"force,omitzero"`
+	Verbose               Tristate `json:"verbose,omitzero"`
+	MaxConcurrentProjects *int     `json:"maxConcurrentProjects,omitzero"`
+	StopBuildOnErrors     Tristate `json:"stopBuildOnErrors,omitzero"`
 
 	// CompilerOptions are not parsed here and will be available on ParsedBuildCommandLine
 

internal/diagnostics/extraDiagnosticMessages.json

@@ -38,5 +38,13 @@
     "Failed to update timestamp of file '{0}'.": {
         "category": "Message",
         "code": 5074
+    },
+    "Option '{0}' requires value to be greater than 0.": {
+        "category": "Error",
+        "code": 5002
+    },
+    "Specify the maximum number of projects that can be built concurrently.": {
+        "category": "Message",
+        "code": 6730
     }
 }

internal/execute/build/buildtask.go

@@ -36,13 +36,10 @@ type buildTask struct {
 	reportDone         chan struct{}
 }
 
-func (t *buildTask) waitOnUpstream() []*upToDateStatus {
-	upStreamStatus := make([]*upToDateStatus, len(t.upStream))
-	for i, upstream := range t.upStream {
+func (t *buildTask) waitOnUpstream() {
+	for _, upstream := range t.upStream {
 		<-upstream.done
-		upStreamStatus[i] = upstream.status
 	}
-	return upStreamStatus
 }
 
 func (t *buildTask) unblockDownstream(status *upToDateStatus) {
@@ -85,59 +82,11 @@ func (t *buildTask) report(orchestrator *Orchestrator, configPath tspath.Path, b
 
 func (t *buildTask) buildProject(orchestrator *Orchestrator, path tspath.Path) {
 	// Wait on upstream tasks to complete
-	upStreamStatus := t.waitOnUpstream()
-	status := t.getUpToDateStatus(orchestrator, path, upStreamStatus)
+	t.waitOnUpstream()
+	status := t.getUpToDateStatus(orchestrator, path)
 	t.reportUpToDateStatus(orchestrator, status)
 	if handled := t.handleStatusThatDoesntRequireBuild(orchestrator, status); handled == nil {
-		if orchestrator.opts.Command.BuildOptions.Verbose.IsTrue() {
-			t.reportStatus(ast.NewCompilerDiagnostic(diagnostics.Building_project_0, orchestrator.relativeFileName(t.config)))
-		}
-
-		// Real build
-		var compileTimes tsc.CompileTimes
-		configAndTime, _ := orchestrator.host.resolvedReferences.Load(path)
-		compileTimes.ConfigTime = configAndTime.time
-		buildInfoReadStart := orchestrator.opts.Sys.Now()
-		oldProgram := incremental.ReadBuildInfoProgram(t.resolved, orchestrator.host, orchestrator.host)
-		compileTimes.BuildInfoReadTime = orchestrator.opts.Sys.Now().Sub(buildInfoReadStart)
-		parseStart := orchestrator.opts.Sys.Now()
-		program := compiler.NewProgram(compiler.ProgramOptions{
-			Config: t.resolved,
-			Host: &compilerHost{
-				host:  orchestrator.host,
-				trace: tsc.GetTraceWithWriterFromSys(&t.builder, orchestrator.opts.Testing),
-			},
-			JSDocParsingMode: ast.JSDocParsingModeParseForTypeErrors,
-		})
-		compileTimes.ParseTime = orchestrator.opts.Sys.Now().Sub(parseStart)
-		changesComputeStart := orchestrator.opts.Sys.Now()
-		t.program = incremental.NewProgram(program, oldProgram, orchestrator.host, orchestrator.opts.Testing != nil)
-		compileTimes.ChangesComputeTime = orchestrator.opts.Sys.Now().Sub(changesComputeStart)
-
-		result, statistics := tsc.EmitAndReportStatistics(
-			orchestrator.opts.Sys,
-			t.program,
-			program,
-			t.resolved,
-			t.reportDiagnostic,
-			tsc.QuietDiagnosticsReporter,
-			&t.builder,
-			&compileTimes,
-			orchestrator.opts.Testing,
-		)
-		t.exitStatus = result.Status
-		t.statistics = statistics
-		if (!program.Options().NoEmitOnError.IsTrue() || len(result.Diagnostics) == 0) &&
-			(len(result.EmitResult.EmittedFiles) > 0 || status.kind != upToDateStatusTypeOutOfDateBuildInfoWithErrors) {
-			// Update time stamps for rest of the outputs
-			t.updateTimeStamps(orchestrator, result.EmitResult.EmittedFiles, diagnostics.Updating_unchanged_output_timestamps_of_project_0)
-		}
-
-		if result.Status == tsc.ExitStatusDiagnosticsPresent_OutputsSkipped || result.Status == tsc.ExitStatusDiagnosticsPresent_OutputsGenerated {
-			status = &upToDateStatus{kind: upToDateStatusTypeBuildErrors}
-		} else {
-			status = &upToDateStatus{kind: upToDateStatusTypeUpToDate}
-		}
+		status = t.compileAndEmit(orchestrator, path, status)
 	} else {
 		status = handled
 		if t.resolved != nil {
@@ -152,6 +101,60 @@ func (t *buildTask) buildProject(orchestrator *Orchestrator, path tspath.Path) {
 	t.unblockDownstream(status)
 }
 
+func (t *buildTask) compileAndEmit(orchestrator *Orchestrator, path tspath.Path, status *upToDateStatus) *upToDateStatus {
+	orchestrator.buildSemaphore <- struct{}{}        // Acquire a slot
+	defer func() { <-orchestrator.buildSemaphore }() // Release the slot
+	if orchestrator.opts.Command.BuildOptions.Verbose.IsTrue() {
+		t.reportStatus(ast.NewCompilerDiagnostic(diagnostics.Building_project_0, orchestrator.relativeFileName(t.config)))
+	}
+
+	// Real build
+	var compileTimes tsc.CompileTimes
+	configAndTime, _ := orchestrator.host.resolvedReferences.Load(path)
+	compileTimes.ConfigTime = configAndTime.time
+	buildInfoReadStart := orchestrator.opts.Sys.Now()
+	oldProgram := incremental.ReadBuildInfoProgram(t.resolved, orchestrator.host, orchestrator.host)
+	compileTimes.BuildInfoReadTime = orchestrator.opts.Sys.Now().Sub(buildInfoReadStart)
+	parseStart := orchestrator.opts.Sys.Now()
+	program := compiler.NewProgram(compiler.ProgramOptions{
+		Config: t.resolved,
+		Host: &compilerHost{
+			host:  orchestrator.host,
+			trace: tsc.GetTraceWithWriterFromSys(&t.builder, orchestrator.opts.Testing),
+		},
+		JSDocParsingMode: ast.JSDocParsingModeParseForTypeErrors,
+	})
+	compileTimes.ParseTime = orchestrator.opts.Sys.Now().Sub(parseStart)
+	changesComputeStart := orchestrator.opts.Sys.Now()
+	t.program = incremental.NewProgram(program, oldProgram, orchestrator.host, orchestrator.opts.Testing != nil)
+	compileTimes.ChangesComputeTime = orchestrator.opts.Sys.Now().Sub(changesComputeStart)
+
+	result, statistics := tsc.EmitAndReportStatistics(
+		orchestrator.opts.Sys,
+		t.program,
+		program,
+		t.resolved,
+		t.reportDiagnostic,
+		tsc.QuietDiagnosticsReporter,
+		&t.builder,
+		&compileTimes,
+		orchestrator.opts.Testing,
+	)
+	t.exitStatus = result.Status
+	t.statistics = statistics
+	if (!program.Options().NoEmitOnError.IsTrue() || len(result.Diagnostics) == 0) &&
+		(len(result.EmitResult.EmittedFiles) > 0 || status.kind != upToDateStatusTypeOutOfDateBuildInfoWithErrors) {
+		// Update time stamps for rest of the outputs
+		t.updateTimeStamps(orchestrator, result.EmitResult.EmittedFiles, diagnostics.Updating_unchanged_output_timestamps_of_project_0)
+	}
+
+	if result.Status == tsc.ExitStatusDiagnosticsPresent_OutputsSkipped || result.Status == tsc.ExitStatusDiagnosticsPresent_OutputsGenerated {
+		return &upToDateStatus{kind: upToDateStatusTypeBuildErrors}
+	} else {
+		return &upToDateStatus{kind: upToDateStatusTypeUpToDate}
+	}
+}
+
 func (t *buildTask) handleStatusThatDoesntRequireBuild(orchestrator *Orchestrator, status *upToDateStatus) *upToDateStatus {
 	switch status.kind {
 	case upToDateStatusTypeUpToDate:
@@ -202,7 +205,7 @@ func (t *buildTask) handleStatusThatDoesntRequireBuild(orchestrator *Orchestrato
 	return nil
 }
 
-func (t *buildTask) getUpToDateStatus(orchestrator *Orchestrator, configPath tspath.Path, upStreamStatus []*upToDateStatus) *upToDateStatus {
+func (t *buildTask) getUpToDateStatus(orchestrator *Orchestrator, configPath tspath.Path) *upToDateStatus {
 	// Config file not found
 	if t.resolved == nil {
 		return &upToDateStatus{kind: upToDateStatusTypeConfigFileNotFound}
@@ -213,15 +216,10 @@ func (t *buildTask) getUpToDateStatus(orchestrator *Orchestrator, configPath tsp
 		return &upToDateStatus{kind: upToDateStatusTypeSolution}
 	}
 
-	for index, upstreamStatus := range upStreamStatus {
-		if upstreamStatus == nil {
-			// Not dependent on this upstream project (expected cycle was detected and hence skipped)
-			continue
-		}
-
-		if orchestrator.opts.Command.BuildOptions.StopBuildOnErrors.IsTrue() && upstreamStatus.isError() {
+	for index, upstream := range t.upStream {
+		if orchestrator.opts.Command.BuildOptions.StopBuildOnErrors.IsTrue() && upstream.status.isError() {
 			// Upstream project has errors, so we cannot build this project
-			return &upToDateStatus{kind: upToDateStatusTypeUpstreamErrors, data: &upstreamErrors{t.resolved.ProjectReferences()[index].Path, upstreamStatus.kind == upToDateStatusTypeUpstreamErrors}}
+			return &upToDateStatus{kind: upToDateStatusTypeUpstreamErrors, data: &upstreamErrors{t.resolved.ProjectReferences()[index].Path, upstream.status.kind == upToDateStatusTypeUpstreamErrors}}
 		}
 	}
 
@@ -342,8 +340,8 @@ func (t *buildTask) getUpToDateStatus(orchestrator *Orchestrator, configPath tsp
 	}
 
 	var refDtsUnchanged bool
-	for index, upstreamStatus := range upStreamStatus {
-		if upstreamStatus == nil || upstreamStatus.kind == upToDateStatusTypeSolution {
+	for index, upstream := range t.upStream {
+		if upstream.status.kind == upToDateStatusTypeSolution {
 			// Not dependent on the status or this upstream project
 			// (eg: expected cycle was detected and hence skipped, or is solution)
 			continue
@@ -353,7 +351,7 @@ func (t *buildTask) getUpToDateStatus(orchestrator *Orchestrator, configPath tsp
 		// we can't be out of date because of it
 		// inputTime will not be present if we just built this project or updated timestamps
 		// - in that case we do want to either build or update timestamps
-		refInputOutputFileAndTime := upstreamStatus.inputOutputFileAndTime()
+		refInputOutputFileAndTime := upstream.status.inputOutputFileAndTime()
 		if refInputOutputFileAndTime != nil && !refInputOutputFileAndTime.input.time.IsZero() && refInputOutputFileAndTime.input.time.Before(oldestOutputFileAndTime.time) {
 			continue
 		}

internal/execute/build/orchestrator.go

@@ -55,9 +55,10 @@ type Orchestrator struct {
 	host                *host
 
 	// order generation result
-	tasks  collections.SyncMap[tspath.Path, *buildTask]
-	order  []string
-	errors []*ast.Diagnostic
+	tasks          collections.SyncMap[tspath.Path, *buildTask]
+	order          []string
+	errors         []*ast.Diagnostic
+	buildSemaphore chan struct{}
 }
 
 func (o *Orchestrator) relativeFileName(fileName string) string {
@@ -226,11 +227,17 @@ func (o *Orchestrator) createDiagnosticReporter(task *buildTask) tsc.DiagnosticR
 }
 
 func NewOrchestrator(opts Options) *Orchestrator {
-	return &Orchestrator{
+	orchestrator := &Orchestrator{
 		opts: opts,
 		comparePathsOptions: tspath.ComparePathsOptions{
 			CurrentDirectory:          opts.Sys.GetCurrentDirectory(),
 			UseCaseSensitiveFileNames: opts.Sys.FS().UseCaseSensitiveFileNames(),
 		},
 	}
+	maxConcurrentProjects := tsoptions.TscMaxConcurrentProjectsOption.DefaultValueDescription.(int)
+	if opts.Command.BuildOptions.MaxConcurrentProjects != nil {
+		maxConcurrentProjects = *opts.Command.BuildOptions.MaxConcurrentProjects
+	}
+	orchestrator.buildSemaphore = make(chan struct{}, maxConcurrentProjects)
+	return orchestrator
 }

internal/execute/tsctests/sys.go

@@ -308,9 +308,6 @@ func (s *testSys) baselinePrograms(baseline *strings.Builder) {
 	s.programBaselines.Reset()
 }
 
-func (s *testSys) baselineProgram(program *incremental.Program) {
-}
-
 func (s *testSys) serializeState(baseline *strings.Builder) {
 	s.baselineOutput(baseline)
 	s.baselineFSwithDiff(baseline)

internal/execute/tsctests/tscbuild_test.go

@@ -3,6 +3,7 @@ package tsctests
 import (
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -1527,6 +1528,89 @@ func TestBuildOutputPaths(t *testing.T) {
 	}
 }
 
+func TestProjectsBuilding(t *testing.T) {
+	t.Parallel()
+	addPackageFiles := func(files FileMap, index int) {
+		files[fmt.Sprintf(`/user/username/projects/myproject/pkg%d/index.ts`, index)] = fmt.Sprintf(`export const pkg%d = %d;`, index, index)
+		var references string
+		if index > 0 {
+			references = `"references": [{ "path": "../pkg0" }],`
+		}
+		files[fmt.Sprintf(`/user/username/projects/myproject/pkg%d/tsconfig.json`, index)] = stringtestutil.Dedent(fmt.Sprintf(`
+		{
+			"compilerOptions": { "composite": true },
+			%s
+		}`, references))
+	}
+	addSolution := func(files FileMap, count int) {
+		var pkgReferences []string
+		for i := range count {
+			pkgReferences = append(pkgReferences, fmt.Sprintf(`{ "path": "./pkg%d" }`, i))
+		}
+		files[`/user/username/projects/myproject/tsconfig.json`] = stringtestutil.Dedent(fmt.Sprintf(`
+		{
+			"compilerOptions": { "composite": true },
+			"references": [
+				%s
+			]
+		}`, strings.Join(pkgReferences, ",\n\t\t\t\t")))
+	}
+	files := func(count int) FileMap {
+		files := FileMap{}
+		for i := range count {
+			addPackageFiles(files, i)
+		}
+		addSolution(files, count)
+		return files
+	}
+
+	getTestCases := func(pkgCount int, maxBuilding int) []*tscInput {
+		edits := []*tscEdit{
+			{
+				caption: "dts doesn't change",
+				edit: func(sys *testSys) {
+					sys.appendFile(`/user/username/projects/myproject/pkg0/index.ts`, `const someConst2 = 10;`)
+				},
+			},
+			noChange,
+			{
+				caption: "dts change",
+				edit: func(sys *testSys) {
+					sys.appendFile(`/user/username/projects/myproject/pkg0/index.ts`, `export const someConst = 10;`)
+				},
+			},
+			noChange,
+		}
+		return []*tscInput{
+			{
+				subScenario:     fmt.Sprintf(`when there are %d projects in a solution`, pkgCount),
+				files:           files(pkgCount),
+				cwd:             "/user/username/projects/myproject",
+				commandLineArgs: []string{"-b", "-v"},
+				edits:           edits,
+			},
+			{
+				subScenario:     fmt.Sprintf(`when there are %d projects in a solution with maxConcurrentProjects %d`, pkgCount, maxBuilding),
+				files:           files(pkgCount),
+				cwd:             "/user/username/projects/myproject",
+				commandLineArgs: []string{"-b", "-v", "--maxConcurrentProjects", strconv.Itoa(maxBuilding)},
+				edits:           edits,
+			},
+		}
+	}
+
+	testCases := slices.Concat(
+		getTestCases(3, 2),
+		getTestCases(5, 2),
+		getTestCases(8, 3),
+		getTestCases(23, 3),
+	)
+
+	for _, test := range testCases {
+		test.run(t, "projectsBuilding")
+	}
+}
+
 func TestBuildProjectReferenceWithRootDirInParent(t *testing.T) {
 	t.Parallel()
 	getBuildProjectReferenceWithRootDirInParentFileMap := func(modify func(files FileMap)) FileMap {