Add hidden "bundle plan" command (#3329)

## Changes
- Add "bundle plan" command that shows resource-level plan. It works for
both TF and direct backend.
- Make plan in direct deployment sort groups and resources so that
output of "bundle plan" is consistent.

Depends on #3328

## Why
Going to use this in acceptance tests to verify direct deployment.

It'll be extended with an option to show field-level changes as well
(most likely for direct deployment only).

## Tests
Modified different acceptance tests to show different actions.

Author: denik

acceptance/bundle/deploy/pipeline/auto-approve/output.txt

@@ -35,6 +35,10 @@ Deployment complete!
 === Remove resources from configuration.
 >>> rm resources.yml
 
+>>> [CLI] bundle plan
+delete jobs.foo
+delete pipelines.bar
+
 === Try to redeploy the bundle - should fail without --auto-approve
 >>> errcode [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...

acceptance/bundle/deploy/pipeline/auto-approve/script

@@ -19,5 +19,7 @@ $CLI jobs get "${JOB_ID}" | jq '{name: .settings.name}'
 title "Remove resources from configuration."
 trace rm resources.yml
 
+trace $CLI bundle plan
+
 title "Try to redeploy the bundle - should fail without --auto-approve"
 trace errcode $CLI bundle deploy

acceptance/bundle/deploy/pipeline/recreate/output.txt

@@ -30,6 +30,10 @@ Deployment complete!
   }
 }
 
+>>> [CLI] bundle plan --var=catalog=another_catalog
+recreate pipelines.foo
+recreate schemas.bar
+
 === Try to redeploy the bundle, pointing the DLT pipeline to a different UC catalog
 >>> errcode [CLI] bundle deploy --force-lock --var=catalog=another_catalog
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...

acceptance/bundle/deploy/pipeline/recreate/script

@@ -11,5 +11,7 @@ title "Assert the pipeline is created"
 PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id')
 trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}"
 
+trace $CLI bundle plan --var="catalog=another_catalog"
+
 title "Try to redeploy the bundle, pointing the DLT pipeline to a different UC catalog"
 trace errcode $CLI bundle deploy --force-lock --var="catalog=another_catalog"

acceptance/bundle/resources/jobs/update/output.txt

@@ -1,10 +1,15 @@
 
+>>> [CLI] bundle plan
+create jobs.foo
+
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 
+>>> [CLI] bundle plan
+
 >>> print_requests
 {
   "body": {
@@ -44,12 +49,17 @@ jobs foo id='[JOB_ID]' name='foo'
 === Update trigger.periodic.unit and re-deploy
 >>> update_file.py databricks.yml DAYS HOURS
 
+>>> [CLI] bundle plan
+update jobs.foo
+
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 
+>>> [CLI] bundle plan
+
 >>> print_requests
 {
   "body": {

acceptance/bundle/resources/jobs/update/script

@@ -1,5 +1,7 @@
 echo "*" > .gitignore
+trace $CLI bundle plan
 trace $CLI bundle deploy
+trace $CLI bundle plan
 
 print_requests() {
     jq --sort-keys 'select(.method != "GET" and (.path | contains("/jobs")))' < out.requests.txt
@@ -11,7 +13,9 @@ trace print_requests
 
 title "Update trigger.periodic.unit and re-deploy"
 trace update_file.py databricks.yml DAYS HOURS
+trace $CLI bundle plan
 trace $CLI bundle deploy
+trace $CLI bundle plan
 trace print_requests
 
 title "Fetch job ID and verify remote state"

bundle/phases/deploy.go

@@ -28,26 +28,27 @@ import (
 	"github.com/databricks/cli/libs/sync"
 )
 
-func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) {
+func getActions(ctx context.Context, b *bundle.Bundle) ([]deployplan.Action, error) {
 	var actions []deployplan.Action
 	var err error
 
 	if b.DirectDeployment {
-		actions, err = terranova.CalculateDeployActions(ctx, b)
-		if err != nil {
-			return false, err
-		}
+		return terranova.CalculateDeployActions(ctx, b)
 	} else {
 		tf := b.Terraform
 		if tf == nil {
-			return false, errors.New("terraform not initialized")
+			return nil, errors.New("terraform not initialized")
 		}
 		actions, err = terraform.ShowPlanFile(ctx, tf, b.Plan.TerraformPlanPath)
-		if err != nil {
-			return false, err
-		}
+		return actions, err
 	}
+}
 
+func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) {
+	actions, err := getActions(ctx, b)
+	if err != nil {
+		return false, err
+	}
 	b.Plan.Actions = actions
 
 	types := []deployplan.ActionType{deployplan.ActionTypeRecreate, deployplan.ActionTypeDelete}
@@ -258,6 +259,32 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 	bundle.ApplyContext(ctx, b, scripts.Execute(config.ScriptPostDeploy))
 }
 
+func Diff(ctx context.Context, b *bundle.Bundle) []deployplan.Action {
+	deployPrepare(ctx, b)
+	if logdiag.HasError(ctx) {
+		return nil
+	}
+
+	if !b.DirectDeployment {
+		bundle.ApplySeqContext(ctx, b,
+			terraform.Interpolate(),
+			terraform.Write(),
+			terraform.Plan(terraform.PlanGoal("deploy")),
+		)
+	}
+
+	if logdiag.HasError(ctx) {
+		return nil
+	}
+
+	actions, err := getActions(ctx, b)
+	if err != nil {
+		logdiag.LogError(ctx, err)
+	}
+
+	return actions
+}
+
 // If there are more than 1 thousand of a resource type, do not
 // include more resources.
 // Since we have a timeout of 3 seconds, we cap the maximum number of IDs

bundle/terranova/plan.go

@@ -117,8 +117,9 @@ func CalculateDeployActions(ctx context.Context, b *bundle.Bundle) ([]deployplan
 	)
 
 	// Remained in state are resources that no longer present in the config
-	for group, groupState := range state {
-		for name := range groupState {
+	for _, group := range utils.SortedKeys(state) {
+		groupData := state[group]
+		for _, name := range utils.SortedKeys(groupData) {
 			actions = append(actions, deployplan.Action{
 				Group:      group,
 				Name:       name,

cmd/bundle/bundle.go

@@ -28,5 +28,6 @@ func New() *cobra.Command {
 	cmd.AddCommand(newDebugCommand())
 	cmd.AddCommand(deployment.NewDeploymentCommand())
 	cmd.AddCommand(newOpenCommand())
+	cmd.AddCommand(newPlanCommand())
 	return cmd
 }

cmd/bundle/plan.go

@@ -0,0 +1,89 @@
+package bundle
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/validate"
+	"github.com/databricks/cli/bundle/phases"
+	"github.com/databricks/cli/cmd/bundle/utils"
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/logdiag"
+	"github.com/spf13/cobra"
+)
+
+func newPlanCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "plan",
+		Short: "Show deployment plan",
+		Args:  root.NoArgs,
+
+		// Output format may change without notice; main use case is in acceptance tests.
+		// Today, this command also uploads libraries, which is not the intent here. We need to refactor
+		// libraries.Upload() mutator to separate config mutation with actual upload.
+		Hidden: true,
+	}
+
+	var force bool
+	var clusterId string
+	cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.")
+	cmd.Flags().StringVar(&clusterId, "compute-id", "", "Override cluster in the deployment with the given compute ID.")
+	cmd.Flags().StringVarP(&clusterId, "cluster-id", "c", "", "Override cluster in the deployment with the given cluster ID.")
+	cmd.Flags().MarkDeprecated("compute-id", "use --cluster-id instead")
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		ctx := logdiag.InitContext(cmd.Context())
+		cmd.SetContext(ctx)
+
+		b := utils.ConfigureBundleWithVariables(cmd)
+		if b == nil || logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) {
+			b.Config.Bundle.Force = force
+
+			if cmd.Flag("compute-id").Changed {
+				b.Config.Bundle.ClusterId = clusterId
+			}
+
+			if cmd.Flag("cluster-id").Changed {
+				b.Config.Bundle.ClusterId = clusterId
+			}
+		})
+
+		phases.Initialize(ctx, b)
+
+		if logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		bundle.ApplyContext(ctx, b, validate.FastValidate())
+
+		if logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		phases.Build(ctx, b)
+
+		if logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		changes := phases.Diff(ctx, b)
+
+		for _, change := range changes {
+			cmdio.LogString(ctx, fmt.Sprintf("%s %s.%s", change.ActionType, change.Group, change.Name))
+		}
+
+		if logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		return nil
+	}
+
+	return cmd
+}