diff --git a/cli/azd/.vscode/cspell.yaml b/cli/azd/.vscode/cspell.yaml index e63e5a8e145..25630d0a53d 100644 --- a/cli/azd/.vscode/cspell.yaml +++ b/cli/azd/.vscode/cspell.yaml @@ -45,6 +45,7 @@ words: - jsonschema - rustc - figspec + - vbauerster languageSettings: - languageId: go ignoreRegExpList: @@ -237,6 +238,9 @@ overrides: - filename: pkg/project/service_target_dotnet_containerapp.go words: - IMAGENAME + - filename: internal/cmd/deploy.go + words: + - gctx ignorePaths: - "**/*_test.go" - "**/mock*.go" diff --git a/cli/azd/extensions/azure.coding-agent/go.sum b/cli/azd/extensions/azure.coding-agent/go.sum index 2132125684d..1659ab4acd7 100644 --- a/cli/azd/extensions/azure.coding-agent/go.sum +++ b/cli/azd/extensions/azure.coding-agent/go.sum @@ -83,6 +83,7 @@ github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= @@ -169,6 +170,7 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= diff --git a/cli/azd/internal/cmd/deploy.go b/cli/azd/internal/cmd/deploy.go index 9669efc9197..2c16cae6ad7 100644 --- a/cli/azd/internal/cmd/deploy.go +++ b/cli/azd/internal/cmd/deploy.go @@ -238,105 +238,90 @@ func (da *DeployAction) Run(ctx context.Context) (*actions.ActionResult, error) deployResults := map[string]*project.ServiceDeployResult{} err = da.projectConfig.Invoke(ctx, project.ProjectEventDeploy, projectEventArgs, func() error { + // Filter services based on target service name + var servicesToDeploy []*project.ServiceConfig for _, svc := range stableServices { - stepMessage := fmt.Sprintf("Deploying service %s", svc.Name) - da.console.ShowSpinner(ctx, stepMessage, input.Step) - // Skip this service if both cases are true: // 1. The user specified a service name // 2. This service is not the one the user specified if targetServiceName != "" && targetServiceName != svc.Name { - da.console.StopSpinner(ctx, stepMessage, input.StepSkipped) continue } + servicesToDeploy = append(servicesToDeploy, svc) + } - if alphaFeatureId, isAlphaFeature := alpha.IsFeatureKey(string(svc.Host)); isAlphaFeature { - // alpha feature on/off detection for host is done during initialization. - // This is just for displaying the warning during deployment. - da.console.WarnForFeature(ctx, alphaFeatureId) - } - - // Initialize service context for tracking artifacts across operations - serviceContext := project.NewServiceContext() + // If no services to deploy, nothing to do + if len(servicesToDeploy) == 0 { + return nil + } - if da.flags.fromPackage != "" { - // --from-package set, skip packaging and create package artifact - err = serviceContext.Package.Add(&project.Artifact{ - Kind: determineArtifactKind(da.flags.fromPackage), - Location: da.flags.fromPackage, - LocationKind: project.LocationKindLocal, - }) + // Build a map of service names to services for dependency lookup + serviceMap := make(map[string]*project.ServiceConfig) + for _, svc := range servicesToDeploy { + serviceMap[svc.Name] = svc + } - if err != nil { - da.console.StopSpinner(ctx, stepMessage, input.StepFailed) - return err + // Check if any service has dependencies (uses other services) + hasDependencies := false + for _, svc := range servicesToDeploy { + if len(svc.Uses) > 0 { + // Check if any dependency is another service (not a resource) + for _, dep := range svc.Uses { + if _, isService := serviceMap[dep]; isService { + hasDependencies = true + break + } } - } else { - // --from-package not set, automatically package the application - _, err := async.RunWithProgress( - func(packageProgress project.ServiceProgress) { - progressMessage := fmt.Sprintf("Packaging service %s (%s)", svc.Name, packageProgress.Message) - da.console.ShowSpinner(ctx, progressMessage, input.Step) - }, - func(progress *async.Progress[project.ServiceProgress]) (*project.ServicePackageResult, error) { - return da.serviceManager.Package(ctx, svc, serviceContext, progress, nil) - }, - ) - - // do not stop progress here as next step is to publish - if err != nil { - da.console.StopSpinner(ctx, stepMessage, input.StepFailed) + } + if hasDependencies { + break + } + } + + // Check if parallel deployment is enabled (default: enabled) + useParallelDeploy := os.Getenv("AZD_DISABLE_PARALLEL_DEPLOY") != "1" + + if len(servicesToDeploy) == 1 || !useParallelDeploy { + // Single service or parallel disabled - deploy sequentially + for _, svc := range servicesToDeploy { + if err := da.deployService(ctx, svc, targetServiceName, deployResults); err != nil { return err } } - - _, err := async.RunWithProgress( - func(publishProgress project.ServiceProgress) { - progressMessage := fmt.Sprintf("Publishing service %s (%s)", svc.Name, publishProgress.Message) - da.console.ShowSpinner(ctx, progressMessage, input.Step) - }, - func(progress *async.Progress[project.ServiceProgress]) (*project.ServicePublishResult, error) { - return da.serviceManager.Publish(ctx, svc, serviceContext, progress, nil) - }, - ) - - // do not stop progress here as next step is to deploy + } else if hasDependencies { + // Services have dependencies - use dependency-aware deployment + da.console.Message(ctx, fmt.Sprintf( + "Deploying %d services with dependency awareness", + len(servicesToDeploy))) + + parallelManager := NewParallelDeploymentManager(&da.serviceManager, 0) + serviceResults, err := parallelManager.DeployServicesWithDependencies(ctx, servicesToDeploy, serviceMap) if err != nil { - da.console.StopSpinner(ctx, stepMessage, input.StepFailed) - return err + return fmt.Errorf("parallel deployment failed: %w", err) } - deployResult, err := async.RunWithProgress( - func(deployProgress project.ServiceProgress) { - progressMessage := fmt.Sprintf("Deploying service %s (%s)", svc.Name, deployProgress.Message) - da.console.ShowSpinner(ctx, progressMessage, input.Step) - }, - func(progress *async.Progress[project.ServiceProgress]) (*project.ServiceDeployResult, error) { - return da.serviceManager.Deploy(ctx, svc, serviceContext, progress) - }, - ) - + // Store results and report artifacts + for serviceName, serviceResult := range serviceResults { + deployResults[serviceName] = serviceResult + da.console.MessageUxItem(ctx, serviceResult.Artifacts) + } + } else { + // No dependencies - deploy all services in parallel + da.console.Message(ctx, fmt.Sprintf( + "Deploying %d services in parallel", + len(servicesToDeploy))) + + parallelManager := NewParallelDeploymentManager(&da.serviceManager, 0) + serviceResults, err := parallelManager.DeployServices(ctx, servicesToDeploy) if err != nil { - da.console.StopSpinner(ctx, stepMessage, input.StepFailed) - return err + return fmt.Errorf("parallel deployment failed: %w", err) } - // clean up for packages automatically created in temp dir - if da.flags.fromPackage == "" { - for _, artifact := range serviceContext.Package { - if strings.HasPrefix(artifact.Location, os.TempDir()) { - if err := os.RemoveAll(artifact.Location); err != nil { - log.Printf("failed to remove temporary package: %s : %s", artifact.Location, err) - } - } - } + // Store results and report artifacts + for serviceName, serviceResult := range serviceResults { + deployResults[serviceName] = serviceResult + da.console.MessageUxItem(ctx, serviceResult.Artifacts) } - - da.console.StopSpinner(ctx, stepMessage, input.GetStepResultFormat(err)) - deployResults[svc.Name] = deployResult - - // report deploy outputs - da.console.MessageUxItem(ctx, deployResult.Artifacts) } return nil @@ -405,3 +390,104 @@ func GetCmdDeployHelpFooter(*cobra.Command) string { ), }) } + +// deployService deploys a single service sequentially +func (da *DeployAction) deployService( + ctx context.Context, + svc *project.ServiceConfig, + targetServiceName string, + deployResults map[string]*project.ServiceDeployResult, +) error { + stepMessage := fmt.Sprintf("Deploying service %s", svc.Name) + da.console.ShowSpinner(ctx, stepMessage, input.Step) + + if alphaFeatureId, isAlphaFeature := alpha.IsFeatureKey(string(svc.Host)); isAlphaFeature { + // alpha feature on/off detection for host is done during initialization. + // This is just for displaying the warning during deployment. + da.console.WarnForFeature(ctx, alphaFeatureId) + } + + // Initialize service context for tracking artifacts across operations + serviceContext := project.NewServiceContext() + + if da.flags.fromPackage != "" { + // --from-package set, skip packaging and create package artifact + err := serviceContext.Package.Add(&project.Artifact{ + Kind: determineArtifactKind(da.flags.fromPackage), + Location: da.flags.fromPackage, + LocationKind: project.LocationKindLocal, + }) + + if err != nil { + da.console.StopSpinner(ctx, stepMessage, input.StepFailed) + return err + } + } else { + // --from-package not set, automatically package the application + _, err := async.RunWithProgress( + func(packageProgress project.ServiceProgress) { + progressMessage := fmt.Sprintf("Packaging service %s (%s)", svc.Name, packageProgress.Message) + da.console.ShowSpinner(ctx, progressMessage, input.Step) + }, + func(progress *async.Progress[project.ServiceProgress]) (*project.ServicePackageResult, error) { + return da.serviceManager.Package(ctx, svc, serviceContext, progress, nil) + }, + ) + + // do not stop progress here as next step is to publish + if err != nil { + da.console.StopSpinner(ctx, stepMessage, input.StepFailed) + return err + } + } + + _, err := async.RunWithProgress( + func(publishProgress project.ServiceProgress) { + progressMessage := fmt.Sprintf("Publishing service %s (%s)", svc.Name, publishProgress.Message) + da.console.ShowSpinner(ctx, progressMessage, input.Step) + }, + func(progress *async.Progress[project.ServiceProgress]) (*project.ServicePublishResult, error) { + return da.serviceManager.Publish(ctx, svc, serviceContext, progress, nil) + }, + ) + + // do not stop progress here as next step is to deploy + if err != nil { + da.console.StopSpinner(ctx, stepMessage, input.StepFailed) + return err + } + + deployResult, err := async.RunWithProgress( + func(deployProgress project.ServiceProgress) { + progressMessage := fmt.Sprintf("Deploying service %s (%s)", svc.Name, deployProgress.Message) + da.console.ShowSpinner(ctx, progressMessage, input.Step) + }, + func(progress *async.Progress[project.ServiceProgress]) (*project.ServiceDeployResult, error) { + return da.serviceManager.Deploy(ctx, svc, serviceContext, progress) + }, + ) + + if err != nil { + da.console.StopSpinner(ctx, stepMessage, input.StepFailed) + return err + } + + // clean up for packages automatically created in temp dir + if da.flags.fromPackage == "" { + for _, artifact := range serviceContext.Package { + if strings.HasPrefix(artifact.Location, os.TempDir()) { + if err := os.RemoveAll(artifact.Location); err != nil { + log.Printf("failed to remove temporary package: %s : %s", artifact.Location, err) + } + } + } + } + + da.console.StopSpinner(ctx, stepMessage, input.GetStepResultFormat(err)) + deployResults[svc.Name] = deployResult + + // report deploy outputs + da.console.MessageUxItem(ctx, deployResult.Artifacts) + + return nil +} diff --git a/cli/azd/internal/cmd/deploy_test.go b/cli/azd/internal/cmd/deploy_test.go new file mode 100644 index 00000000000..f9527c3b2c6 --- /dev/null +++ b/cli/azd/internal/cmd/deploy_test.go @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/project" + "github.com/stretchr/testify/require" +) + +func TestServiceFiltering(t *testing.T) { + tests := []struct { + name string + services []*project.ServiceConfig + targetServiceName string + expectedServices int + }{ + { + name: "AllServicesNoFilter", + services: []*project.ServiceConfig{ + {Name: "api", Host: project.ContainerAppTarget}, + {Name: "web", Host: project.ContainerAppTarget}, + }, + targetServiceName: "", + expectedServices: 2, + }, + { + name: "MixedServicesNoFilter", + services: []*project.ServiceConfig{ + {Name: "api", Host: project.ContainerAppTarget}, + {Name: "web", Host: project.AppServiceTarget}, + {Name: "worker", Host: project.DotNetContainerAppTarget}, + }, + targetServiceName: "", + expectedServices: 3, + }, + { + name: "FilterByTargetService", + services: []*project.ServiceConfig{ + {Name: "api", Host: project.ContainerAppTarget}, + {Name: "web", Host: project.AppServiceTarget}, + {Name: "worker", Host: project.ContainerAppTarget}, + }, + targetServiceName: "api", + expectedServices: 1, + }, + { + name: "EmptyServiceList", + services: []*project.ServiceConfig{}, + targetServiceName: "", + expectedServices: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var servicesToDeploy []*project.ServiceConfig + + for _, svc := range tt.services { + // Skip this service if both cases are true: + // 1. The user specified a service name + // 2. This service is not the one the user specified + if tt.targetServiceName != "" && tt.targetServiceName != svc.Name { + continue + } + servicesToDeploy = append(servicesToDeploy, svc) + } + + require.Equal(t, tt.expectedServices, len(servicesToDeploy), + "Expected %d services, got %d", tt.expectedServices, len(servicesToDeploy)) + }) + } +} + +func TestServiceDependencyDetection(t *testing.T) { + tests := []struct { + name string + services []*project.ServiceConfig + hasDependencies bool + }{ + { + name: "NoDependencies", + services: []*project.ServiceConfig{ + {Name: "api", Host: project.ContainerAppTarget, Uses: []string{}}, + {Name: "web", Host: project.ContainerAppTarget, Uses: []string{}}, + }, + hasDependencies: false, + }, + { + name: "WithServiceDependencies", + services: []*project.ServiceConfig{ + {Name: "api", Host: project.ContainerAppTarget, Uses: []string{}}, + {Name: "web", Host: project.ContainerAppTarget, Uses: []string{"api"}}, + }, + hasDependencies: true, + }, + { + name: "OnlyResourceDependencies", + services: []*project.ServiceConfig{ + {Name: "api", Host: project.ContainerAppTarget, Uses: []string{"postgresdb"}}, + {Name: "web", Host: project.ContainerAppTarget, Uses: []string{"redis"}}, + }, + hasDependencies: false, // postgresdb and redis are not services + }, + { + name: "MixedDependencies", + services: []*project.ServiceConfig{ + {Name: "api", Host: project.ContainerAppTarget, Uses: []string{"postgresdb"}}, + {Name: "web", Host: project.ContainerAppTarget, Uses: []string{"api"}}, + }, + hasDependencies: true, // web depends on api service + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Build service map + serviceMap := make(map[string]*project.ServiceConfig) + for _, svc := range tt.services { + serviceMap[svc.Name] = svc + } + + // Check for dependencies + hasDependencies := false + for _, svc := range tt.services { + if len(svc.Uses) > 0 { + for _, dep := range svc.Uses { + if _, isService := serviceMap[dep]; isService { + hasDependencies = true + break + } + } + } + if hasDependencies { + break + } + } + + require.Equal(t, tt.hasDependencies, hasDependencies, + "Expected hasDependencies=%v, got %v", tt.hasDependencies, hasDependencies) + }) + } +} + +func TestCyclicDependencyDetection(t *testing.T) { + tests := []struct { + name string + services []*project.ServiceConfig + hasCycle bool + }{ + { + name: "NoCycle", + services: []*project.ServiceConfig{ + {Name: "api", Uses: []string{}}, + {Name: "web", Uses: []string{"api"}}, + }, + hasCycle: false, + }, + { + name: "SimpleCycle", + services: []*project.ServiceConfig{ + {Name: "api", Uses: []string{"web"}}, + {Name: "web", Uses: []string{"api"}}, + }, + hasCycle: true, + }, + { + name: "IndirectCycle", + services: []*project.ServiceConfig{ + {Name: "a", Uses: []string{"b"}}, + {Name: "b", Uses: []string{"c"}}, + {Name: "c", Uses: []string{"a"}}, + }, + hasCycle: true, + }, + { + name: "SelfCycle", + services: []*project.ServiceConfig{ + {Name: "api", Uses: []string{"api"}}, + }, + hasCycle: true, + }, + { + name: "NoCycleWithExternalDeps", + services: []*project.ServiceConfig{ + {Name: "api", Uses: []string{"postgresdb"}}, + {Name: "web", Uses: []string{"api", "redis"}}, + }, + hasCycle: false, // postgresdb and redis are not services + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Build service map + serviceMap := make(map[string]*project.ServiceConfig) + for _, svc := range tt.services { + serviceMap[svc.Name] = svc + } + + // Test cycle detection using the function from parallel_deploy.go + hasCycle := hasCyclicDependencies(tt.services, serviceMap) + + require.Equal(t, tt.hasCycle, hasCycle, + "Expected hasCycle=%v, got %v", tt.hasCycle, hasCycle) + }) + } +} diff --git a/cli/azd/internal/cmd/parallel_deploy.go b/cli/azd/internal/cmd/parallel_deploy.go new file mode 100644 index 00000000000..dc19c51d384 --- /dev/null +++ b/cli/azd/internal/cmd/parallel_deploy.go @@ -0,0 +1,428 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "runtime" + "sync" + + "github.com/azure/azure-dev/cli/azd/pkg/async" + "github.com/azure/azure-dev/cli/azd/pkg/project" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" + "golang.org/x/sync/errgroup" +) + +// TaskState represents the current state of a service deployment task +type TaskState int + +const ( + StatePending TaskState = iota + StateWaiting + StatePackaging + StatePublishing + StateDeploying + StateComplete + StateError +) + +func (s TaskState) String() string { + switch s { + case StatePending: + return "Pending" + case StateWaiting: + return "Waiting" + case StatePackaging: + return "Packaging" + case StatePublishing: + return "Publishing" + case StateDeploying: + return "Deploying" + case StateComplete: + return "Complete" + case StateError: + return "Error" + default: + return "Unknown" + } +} + +// ServiceTask represents a single service deployment task with progress tracking +type ServiceTask struct { + ServiceName string + ProgressBar *mpb.Bar + State TaskState + Error error + mu sync.Mutex +} + +// UpdateState updates the task state and progress bar label +func (t *ServiceTask) UpdateState(state TaskState, message string) { + t.mu.Lock() + defer t.mu.Unlock() + t.State = state + // Progress bar decorators are updated automatically via the Any decorator +} + +// SetError marks the task as errored +func (t *ServiceTask) SetError(err error) { + t.mu.Lock() + defer t.mu.Unlock() + t.State = StateError + t.Error = err +} + +// GetState returns the current state (thread-safe) +func (t *ServiceTask) GetState() TaskState { + t.mu.Lock() + defer t.mu.Unlock() + return t.State +} + +// ParallelDeploymentManager manages parallel deployment of services with progress tracking +type ParallelDeploymentManager struct { + serviceManager *project.ServiceManager + maxParallel int +} + +// NewParallelDeploymentManager creates a new parallel deployment manager +func NewParallelDeploymentManager(serviceManager *project.ServiceManager, maxParallel int) *ParallelDeploymentManager { + if maxParallel <= 0 { + maxParallel = runtime.NumCPU() + } + return &ParallelDeploymentManager{ + serviceManager: serviceManager, + maxParallel: maxParallel, + } +} + +// DeployServices deploys multiple services in parallel with progress tracking +func (m *ParallelDeploymentManager) DeployServices( + ctx context.Context, + serviceConfigs []*project.ServiceConfig, +) (map[string]*project.ServiceDeployResult, error) { + if len(serviceConfigs) == 0 { + return make(map[string]*project.ServiceDeployResult), nil + } + + // Create progress container + p := mpb.NewWithContext(ctx, + mpb.WithWidth(80), + mpb.WithAutoRefresh(), + ) + + // Create tasks and progress bars for each service + tasks := make([]*ServiceTask, len(serviceConfigs)) + for i, svc := range serviceConfigs { + task := &ServiceTask{ + ServiceName: svc.Name, + State: StatePending, + } + + // Create progress bar with state decorator + bar := p.AddBar(100, + mpb.PrependDecorators( + // Service name with fixed width for alignment + decor.Name(svc.Name, decor.WC{W: 15}), + // Dynamic state display + decor.Any(func(decor.Statistics) string { + state := task.GetState() + if state == StateError { + return fmt.Sprintf("[%s ✗]", state.String()) + } else if state == StateComplete { + return fmt.Sprintf("[%s ✓]", state.String()) + } + return fmt.Sprintf("[%s]", state.String()) + }, decor.WC{W: 15}), + ), + mpb.AppendDecorators( + decor.Percentage(decor.WC{W: 5}), + ), + ) + + task.ProgressBar = bar + tasks[i] = task + } + + // Deploy services in parallel with controlled concurrency + eg, ctx := errgroup.WithContext(ctx) + sem := make(chan struct{}, m.maxParallel) + + resultsMu := sync.Mutex{} + results := make(map[string]*project.ServiceDeployResult) + + for i, svc := range serviceConfigs { + svc := svc // capture loop variable + task := tasks[i] // capture loop variable + + eg.Go(func() error { + // Acquire semaphore + select { + case sem <- struct{}{}: + defer func() { <-sem }() // Release on exit + case <-ctx.Done(): + return ctx.Err() + } + + // Deploy the service with progress updates + result, err := m.deployServiceWithProgress(ctx, svc, task) + + if err != nil { + task.SetError(err) + task.ProgressBar.Abort(false) + return fmt.Errorf("deploying service %s: %w", svc.Name, err) + } + + // Store result + resultsMu.Lock() + results[svc.Name] = result + resultsMu.Unlock() + + task.UpdateState(StateComplete, "Complete") + task.ProgressBar.SetCurrent(100) + + return nil + }) + } + + // Wait for all deployments to complete + deployErr := eg.Wait() + + // Wait for all progress bars to finish rendering + p.Wait() + + return results, deployErr +} + +// deployServiceWithProgress deploys a single service with progress updates +func (m *ParallelDeploymentManager) deployServiceWithProgress( + ctx context.Context, + svc *project.ServiceConfig, + task *ServiceTask, +) (*project.ServiceDeployResult, error) { + // Create service context for tracking artifacts + serviceContext := project.NewServiceContext() + + // Use noop progress since we're tracking with MPB progress bars instead + noopProgress := async.NewNoopProgress[project.ServiceProgress]() + + // Package phase (0-33%) + task.UpdateState(StatePackaging, "Packaging") + task.ProgressBar.SetCurrent(5) + + _, err := (*m.serviceManager).Package(ctx, svc, serviceContext, noopProgress, nil) + if err != nil { + return nil, fmt.Errorf("packaging: %w", err) + } + + task.ProgressBar.SetCurrent(33) + + // Publish phase (33-66%) + task.UpdateState(StatePublishing, "Publishing") + + _, err = (*m.serviceManager).Publish(ctx, svc, serviceContext, noopProgress, nil) + if err != nil { + return nil, fmt.Errorf("publishing: %w", err) + } + + task.ProgressBar.SetCurrent(66) + + // Deploy phase (66-100%) + task.UpdateState(StateDeploying, "Deploying") + + deployResult, err := (*m.serviceManager).Deploy(ctx, svc, serviceContext, noopProgress) + if err != nil { + return nil, fmt.Errorf("deploying: %w", err) + } + + task.ProgressBar.SetCurrent(95) + + return deployResult, nil +} + +// detectCycle checks if there's a circular dependency in the service graph +func detectCycle(serviceName string, serviceMap map[string]*project.ServiceConfig, visited, recStack map[string]bool) bool { + visited[serviceName] = true + recStack[serviceName] = true + + svc, exists := serviceMap[serviceName] + if !exists { + return false + } + + for _, dep := range svc.Uses { + // Only consider dependencies that are actual services + if _, isService := serviceMap[dep]; !isService { + continue + } + if !visited[dep] { + if detectCycle(dep, serviceMap, visited, recStack) { + return true + } + } else if recStack[dep] { + return true + } + } + + recStack[serviceName] = false + return false +} + +// hasCyclicDependencies checks if any service has circular dependencies +func hasCyclicDependencies(serviceConfigs []*project.ServiceConfig, serviceMap map[string]*project.ServiceConfig) bool { + visited := make(map[string]bool) + recStack := make(map[string]bool) + + for _, svc := range serviceConfigs { + if !visited[svc.Name] { + if detectCycle(svc.Name, serviceMap, visited, recStack) { + return true + } + } + } + return false +} + +// DeployServicesWithDependencies deploys services respecting their dependencies +// Services without dependencies (or whose dependencies are complete) deploy in parallel +// Services with dependencies wait for their dependencies to complete first +func (m *ParallelDeploymentManager) DeployServicesWithDependencies( + ctx context.Context, + serviceConfigs []*project.ServiceConfig, + serviceMap map[string]*project.ServiceConfig, +) (map[string]*project.ServiceDeployResult, error) { + if len(serviceConfigs) == 0 { + return make(map[string]*project.ServiceDeployResult), nil + } + + // Check for circular dependencies to prevent deadlock + if hasCyclicDependencies(serviceConfigs, serviceMap) { + return nil, fmt.Errorf("circular dependency detected between services") + } + + // Track completed services and their results + resultsMu := sync.Mutex{} + results := make(map[string]*project.ServiceDeployResult) + + // Track completed services for dependency checking + completedMu := sync.Mutex{} + completed := make(map[string]bool) + + // Create channels to signal completion for each service being deployed + completionChans := make(map[string]chan struct{}) + for _, svc := range serviceConfigs { + completionChans[svc.Name] = make(chan struct{}) + } + + // Create progress container + p := mpb.NewWithContext(ctx, + mpb.WithWidth(80), + mpb.WithAutoRefresh(), + ) + + // Create tasks and progress bars for each service + tasks := make(map[string]*ServiceTask) + for _, svc := range serviceConfigs { + task := &ServiceTask{ + ServiceName: svc.Name, + State: StatePending, + } + + // Create progress bar with state decorator + bar := p.AddBar(100, + mpb.PrependDecorators( + decor.Name(svc.Name, decor.WC{W: 15}), + decor.Any(func(decor.Statistics) string { + state := task.GetState() + if state == StateError { + return fmt.Sprintf("[%s ✗]", state.String()) + } else if state == StateComplete { + return fmt.Sprintf("[%s ✓]", state.String()) + } else if state == StateWaiting { + return fmt.Sprintf("[%s]", state.String()) + } + return fmt.Sprintf("[%s]", state.String()) + }, decor.WC{W: 15}), + ), + mpb.AppendDecorators( + decor.Percentage(decor.WC{W: 5}), + ), + ) + + task.ProgressBar = bar + tasks[svc.Name] = task + } + + // Deploy services with dependency awareness + eg, ctx := errgroup.WithContext(ctx) + sem := make(chan struct{}, m.maxParallel) + + for _, svc := range serviceConfigs { + svc := svc + task := tasks[svc.Name] + + eg.Go(func() error { + // Wait for all service dependencies to complete + for _, dep := range svc.Uses { + // Only wait if the dependency is another service being deployed + if depChan, exists := completionChans[dep]; exists { + task.UpdateState(StateWaiting, fmt.Sprintf("Waiting for %s", dep)) + select { + case <-depChan: + // Dependency completed + case <-ctx.Done(): + return ctx.Err() + } + } + } + + // Acquire semaphore for actual work + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + return ctx.Err() + } + + // Deploy the service with progress updates + result, err := m.deployServiceWithProgress(ctx, svc, task) + + if err != nil { + task.SetError(err) + task.ProgressBar.Abort(false) + // Signal completion even on error so dependents don't hang + close(completionChans[svc.Name]) + return fmt.Errorf("deploying service %s: %w", svc.Name, err) + } + + // Mark as completed + completedMu.Lock() + completed[svc.Name] = true + completedMu.Unlock() + + // Store result + resultsMu.Lock() + results[svc.Name] = result + resultsMu.Unlock() + + task.UpdateState(StateComplete, "Complete") + task.ProgressBar.SetCurrent(100) + + // Signal completion + close(completionChans[svc.Name]) + + return nil + }) + } + + // Wait for all deployments to complete + deployErr := eg.Wait() + + // Wait for all progress bars to finish rendering + p.Wait() + + return results, deployErr +} diff --git a/cli/azd/pkg/project/container_helper.go b/cli/azd/pkg/project/container_helper.go index ea3c5d20746..0dc9525f3e5 100644 --- a/cli/azd/pkg/project/container_helper.go +++ b/cli/azd/pkg/project/container_helper.go @@ -7,11 +7,13 @@ import ( "context" "errors" "fmt" + "io" "log" "os" "path" "path/filepath" "strings" + "sync" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -59,6 +61,10 @@ type ContainerHelper struct { clock clock.Clock console input.Console cloud *cloud.Cloud + // loginMutexes serializes docker login operations per registry to prevent + // concurrent keychain access errors on macOS during parallel deployments + loginMutexes map[string]*sync.Mutex + loginMu sync.Mutex // protects loginMutexes map access } func NewContainerHelper( @@ -84,6 +90,7 @@ func NewContainerHelper( clock: clock, console: console, cloud: cloud, + loginMutexes: make(map[string]*sync.Mutex), } } @@ -307,6 +314,19 @@ func (ch *ContainerHelper) Login( if err != nil { return "", err } + + // Serialize docker login operations per registry to prevent concurrent + // keychain access errors on macOS during parallel deployments + ch.loginMu.Lock() + if ch.loginMutexes[registryName] == nil { + ch.loginMutexes[registryName] = &sync.Mutex{} + } + registryMutex := ch.loginMutexes[registryName] + ch.loginMu.Unlock() + + registryMutex.Lock() + defer registryMutex.Unlock() + return registryName, ch.containerRegistryService.Login(ctx, env.GetSubscriptionId(), registryName) } @@ -467,12 +487,18 @@ func (ch *ContainerHelper) Build( // Build the container progress.SetProgress(NewServiceProgress("Building Docker image")) - previewerWriter := ch.console.ShowPreviewer(ctx, - &input.ShowPreviewerOptions{ - Prefix: " ", - MaxLineCount: 8, - Title: "Docker Output", - }) + // Suppress previewer during parallel builds (when AZD_DISABLE_PARALLEL_DEPLOY != 1) + var previewerWriter io.Writer + if os.Getenv("AZD_DISABLE_PARALLEL_DEPLOY") == "1" { + previewerWriter = ch.console.ShowPreviewer(ctx, + &input.ShowPreviewerOptions{ + Prefix: " ", + MaxLineCount: 8, + Title: "Docker Output", + }) + } else { + previewerWriter = io.Discard + } dockerFilePath := dockerOptions.Path if dockerOptions.InMemDockerfile != nil { @@ -515,7 +541,9 @@ func (ch *ContainerHelper) Build( dockerEnv, previewerWriter, ) - ch.console.StopPreviewer(ctx, false) + if os.Getenv("AZD_DISABLE_PARALLEL_DEPLOY") == "1" { + ch.console.StopPreviewer(ctx, false) + } if err != nil { return nil, fmt.Errorf("building container: %s at %s: %w", serviceConfig.Name, dockerOptions.Context, err) } @@ -996,12 +1024,18 @@ func (ch *ContainerHelper) packBuild( } } - previewer := ch.console.ShowPreviewer(ctx, - &input.ShowPreviewerOptions{ - Prefix: " ", - MaxLineCount: 8, - Title: "Docker (pack) Output", - }) + // Suppress previewer during parallel builds + var previewer io.Writer + if os.Getenv("AZD_DISABLE_PARALLEL_DEPLOY") == "1" { + previewer = ch.console.ShowPreviewer(ctx, + &input.ShowPreviewerOptions{ + Prefix: " ", + MaxLineCount: 8, + Title: "Docker (pack) Output", + }) + } else { + previewer = io.Discard + } ctx, span := tracing.Start( ctx, @@ -1028,7 +1062,9 @@ func (ch *ContainerHelper) packBuild( imageName, environ, previewer) - ch.console.StopPreviewer(ctx, false) + if os.Getenv("AZD_DISABLE_PARALLEL_DEPLOY") == "1" { + ch.console.StopPreviewer(ctx, false) + } if err != nil { span.EndWithStatus(err) diff --git a/go.mod b/go.mod index d329cf27f49..7dcd18bb1fa 100644 --- a/go.mod +++ b/go.mod @@ -72,6 +72,7 @@ require ( github.com/theckman/yacspin v0.13.12 github.com/tidwall/gjson v1.18.0 github.com/tmc/langchaingo v0.1.13 + github.com/vbauerster/mpb/v8 v8.11.2 go.lsp.dev/jsonrpc2 v0.10.0 go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 @@ -81,7 +82,8 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.yaml.in/yaml/v3 v3.0.4 - golang.org/x/sys v0.37.0 + golang.org/x/sync v0.17.0 + golang.org/x/sys v0.38.0 google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.10 gopkg.in/dnaeon/go-vcr.v3 v3.2.0 @@ -93,6 +95,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/alecthomas/chroma/v2 v2.20.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect @@ -105,7 +109,8 @@ require ( github.com/charmbracelet/x/cellbuf v0.0.13 // indirect github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect - github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2 v1.11.5 // indirect @@ -163,7 +168,6 @@ require ( golang.org/x/crypto v0.43.0 // indirect golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect golang.org/x/net v0.46.0 // indirect - golang.org/x/sync v0.17.0 // indirect golang.org/x/term v0.36.0 // indirect golang.org/x/text v0.30.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250922171735-9219d122eba9 // indirect diff --git a/go.sum b/go.sum index 9a9a668bb40..a8ecb60ca42 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,10 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b h1:g9SuFmxM/WucQFKTMSP+irxyf5m0RiUJreBDhGI6jSA= github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b/go.mod h1:XjvqMUpGd3Xn9Jtzk/4GEBCSoBX0eB2RyriXgne0IdM= github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= @@ -166,8 +170,10 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/cli/browser v1.3.0 h1:LejqCrpWr+1pRqmEPDGnTZOjsMe7sehifLynZJuqJpo= github.com/cli/browser v1.3.0/go.mod h1:HH8s+fOAxjhQoBUAsKuPCbqUuxZDhQ2/aD+SzsEfBTk= -github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= -github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= +github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -414,6 +420,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tmc/langchaingo v0.1.13 h1:rcpMWBIi2y3B90XxfE4Ao8dhCQPVDMaNPnN5cGB1CaA= github.com/tmc/langchaingo v0.1.13/go.mod h1:vpQ5NOIhpzxDfTZK9B6tf2GM/MoaHewPWM5KXXGh7hg= +github.com/vbauerster/mpb/v8 v8.11.2 h1:OqLoHznUVU7SKS/WV+1dB5/hm20YLheYupiHhL5+M1Y= +github.com/vbauerster/mpb/v8 v8.11.2/go.mod h1:mEB/M353al1a7wMUNtiymmPsEkGlJgeJmtlbY5adCJ8= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= @@ -515,8 +523,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=