internal: remove DirectDeployment bundle attribute (#3848)

## Changes
Instead of having global *bool for directDeployment, make it explicit
parameter that is passed around.

## Why
Enables migration: we need first run deploy on terraform then in direct,
having parameter helps.

Prevents panic, make it explicit which parts depend on engine.

## Tests
Existing tests.

Author: denik

bundle/bundle.go

@@ -141,9 +141,6 @@ type Bundle struct {
 	Tagging tags.Cloud
 
 	Metrics Metrics
-
-	// If true, don't use terraform. Set by DATABRICKS_BUNDLE_ENGINE=direct
-	DirectDeployment *bool
 }
 
 func Load(ctx context.Context, path string) (*Bundle, error) {

bundle/deploy/terraform/check_dashboards_modified_remotely.go

@@ -34,7 +34,8 @@ func collectDashboardsFromState(ctx context.Context, b *bundle.Bundle) ([]dashbo
 }
 
 type checkDashboardsModifiedRemotely struct {
-	isPlan bool
+	isPlan           bool
+	directDeployment bool
 }
 
 func (l *checkDashboardsModifiedRemotely) Name() string {
@@ -47,7 +48,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B
 		return nil
 	}
 
-	if *b.DirectDeployment {
+	if l.directDeployment {
 		// TODO: not implemented yet
 		return nil
 	}
@@ -115,6 +116,6 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B
 	return diags
 }
 
-func CheckDashboardsModifiedRemotely(isPlan bool) *checkDashboardsModifiedRemotely {
-	return &checkDashboardsModifiedRemotely{isPlan: isPlan}
+func CheckDashboardsModifiedRemotely(isPlan, directDeployment bool) *checkDashboardsModifiedRemotely {
+	return &checkDashboardsModifiedRemotely{isPlan: isPlan, directDeployment: directDeployment}
 }

bundle/deploy/terraform/check_dashboards_modified_remotely_test.go

@@ -17,11 +17,9 @@ import (
 )
 
 func mockDashboardBundle(t *testing.T) *bundle.Bundle {
-	falseBool := false
 	dir := t.TempDir()
 	b := &bundle.Bundle{
-		DirectDeployment: &falseBool,
-		BundleRootPath:   dir,
+		BundleRootPath: dir,
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "test",
@@ -54,13 +52,13 @@ func TestCheckDashboardsModifiedRemotely_NoDashboards(t *testing.T) {
 		},
 	}
 
-	diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely(false))
+	diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely(false, false))
 	assert.Empty(t, diags)
 }
 
 func TestCheckDashboardsModifiedRemotely_FirstDeployment(t *testing.T) {
 	b := mockDashboardBundle(t)
-	diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely(false))
+	diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely(false, false))
 	assert.Empty(t, diags)
 }
 
@@ -83,7 +81,7 @@ func TestCheckDashboardsModifiedRemotely_ExistingStateNoChange(t *testing.T) {
 	b.SetWorkpaceClient(m.WorkspaceClient)
 
 	// No changes, so no diags.
-	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(false))
+	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(false, false))
 	assert.Empty(t, diags)
 }
 
@@ -106,7 +104,7 @@ func TestCheckDashboardsModifiedRemotely_ExistingStateChange(t *testing.T) {
 	b.SetWorkpaceClient(m.WorkspaceClient)
 
 	// The dashboard has changed, so expect an error.
-	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(false))
+	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(false, false))
 	if assert.Len(t, diags, 1) {
 		assert.Equal(t, diag.Error, diags[0].Severity)
 		assert.Equal(t, `dashboard "dash1" has been modified remotely`, diags[0].Summary)
@@ -129,7 +127,7 @@ func TestCheckDashboardsModifiedRemotely_ExistingStateFailureToGet(t *testing.T)
 	b.SetWorkpaceClient(m.WorkspaceClient)
 
 	// Unable to get the dashboard, so expect an error.
-	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(false))
+	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(false, false))
 	if assert.Len(t, diags, 1) {
 		assert.Equal(t, diag.Error, diags[0].Severity)
 		assert.Equal(t, `failed to get dashboard "dash1"`, diags[0].Summary)
@@ -155,7 +153,7 @@ func TestCheckDashboardsModifiedRemotely_ExistingStateChangePlanMode(t *testing.
 	b.SetWorkpaceClient(m.WorkspaceClient)
 
 	// The dashboard has changed, but in plan mode expect a warning instead of an error.
-	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(true))
+	diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely(true, false))
 	if assert.Len(t, diags, 1) {
 		assert.Equal(t, diag.Warning, diags[0].Severity)
 		assert.Equal(t, `dashboard "dash1" has been modified remotely`, diags[0].Summary)

bundle/deploy/terraform/util_test.go

@@ -12,10 +12,8 @@ import (
 )
 
 func TestParseResourcesStateWithNoFile(t *testing.T) {
-	falseBool := false
 	b := &bundle.Bundle{
-		DirectDeployment: &falseBool,
-		BundleRootPath:   t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -31,11 +29,9 @@ func TestParseResourcesStateWithNoFile(t *testing.T) {
 }
 
 func TestParseResourcesStateWithExistingStateFile(t *testing.T) {
-	falseBool := false
 	ctx := context.Background()
 	b := &bundle.Bundle{
-		DirectDeployment: &falseBool,
-		BundleRootPath:   t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",

bundle/phases/bind.go

@@ -27,7 +27,7 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions) {
 		terraform.Interpolate(),
 		terraform.Write(),
 		terraform.Import(opts),
-		statemgmt.StatePush(),
+		statemgmt.StatePush(false),
 	)
 }
 
@@ -47,6 +47,6 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r
 		terraform.Interpolate(),
 		terraform.Write(),
 		terraform.Unbind(bundleType, tfResourceType, resourceKey),
-		statemgmt.StatePush(),
+		statemgmt.StatePush(false),
 	)
 }

bundle/phases/deploy.go

@@ -92,27 +92,27 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P
 	return approved, nil
 }
 
-func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) {
+func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, directDeployment bool) {
 	// Core mutators that CRUD resources and modify deployment state. These
 	// mutators need informed consent if they are potentially destructive.
 	cmdio.LogString(ctx, "Deploying resources...")
 
-	if *b.DirectDeployment {
+	if directDeployment {
 		b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(), &b.Config, plan)
 	} else {
 		bundle.ApplyContext(ctx, b, terraform.Apply())
 	}
 
 	// Even if deployment failed, there might be updates in states that we need to upload
 	bundle.ApplyContext(ctx, b,
-		statemgmt.StatePush(),
+		statemgmt.StatePush(directDeployment),
 	)
 	if logdiag.HasError(ctx) {
 		return
 	}
 
 	bundle.ApplySeqContext(ctx, b,
-		statemgmt.Load(),
+		statemgmt.Load(directDeployment),
 		metadata.Compute(),
 		metadata.Upload(),
 	)
@@ -133,7 +133,7 @@ func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]li
 }
 
 // The deploy phase deploys artifacts and resources.
-func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler) {
+func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, directDeployment bool) {
 	log.Info(ctx, "Phase: deploy")
 
 	// Core mutators that CRUD resources and modify deployment state. These
@@ -153,7 +153,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 		bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy))
 	}()
 
-	libs := deployPrepare(ctx, b, false)
+	libs := deployPrepare(ctx, b, false, directDeployment)
 	if logdiag.HasError(ctx) {
 		return
 	}
@@ -176,7 +176,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 		return
 	}
 
-	plan := planWithoutPrepare(ctx, b)
+	plan := planWithoutPrepare(ctx, b, directDeployment)
 	if logdiag.HasError(ctx) {
 		return
 	}
@@ -187,7 +187,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 		return
 	}
 	if haveApproval {
-		deployCore(ctx, b, plan)
+		deployCore(ctx, b, plan, directDeployment)
 	} else {
 		cmdio.LogString(ctx, "Deployment cancelled!")
 		return
@@ -203,8 +203,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 
 // planWithoutPrepare builds a deployment plan without running deployPrepare.
 // This is used when deployPrepare has already been called.
-func planWithoutPrepare(ctx context.Context, b *bundle.Bundle) *deployplan.Plan {
-	if *b.DirectDeployment {
+func planWithoutPrepare(ctx context.Context, b *bundle.Bundle, directDeployment bool) *deployplan.Plan {
+	if directDeployment {
 		_, localPath := b.StateFilenameDirect(ctx)
 		plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, localPath)
 		if err != nil {
@@ -236,16 +236,27 @@ func planWithoutPrepare(ctx context.Context, b *bundle.Bundle) *deployplan.Plan
 		return nil
 	}
 
+	for _, group := range b.Config.Resources.AllResources() {
+		for rKey := range group.Resources {
+			resourceKey := "resources." + group.Description.PluralName + "." + rKey
+			if _, ok := plan.Plan[resourceKey]; !ok {
+				plan.Plan[resourceKey] = &deployplan.PlanEntry{
+					Action: deployplan.ActionTypeSkip.String(),
+				}
+			}
+		}
+	}
+
 	return plan
 }
 
-func Plan(ctx context.Context, b *bundle.Bundle) *deployplan.Plan {
-	deployPrepare(ctx, b, true)
+func Plan(ctx context.Context, b *bundle.Bundle, directDeployment bool) *deployplan.Plan {
+	deployPrepare(ctx, b, true, directDeployment)
 	if logdiag.HasError(ctx) {
 		return nil
 	}
 
-	return planWithoutPrepare(ctx, b)
+	return planWithoutPrepare(ctx, b, directDeployment)
 }
 
 // If there are more than 1 thousand of a resource type, do not

bundle/phases/destroy.go

@@ -89,8 +89,8 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan.
 	return approved, nil
 }
 
-func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) {
-	if *b.DirectDeployment {
+func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, directDeployment bool) {
+	if directDeployment {
 		b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(), &b.Config, plan)
 	} else {
 		// Core destructive mutators for destroy. These require informed user consent.
@@ -109,7 +109,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) {
 }
 
 // The destroy phase deletes artifacts and resources.
-func Destroy(ctx context.Context, b *bundle.Bundle) {
+func Destroy(ctx context.Context, b *bundle.Bundle, directDeployment bool) {
 	log.Info(ctx, "Phase: destroy")
 
 	ok, err := assertRootPathExists(ctx, b)
@@ -132,7 +132,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle) {
 		bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy))
 	}()
 
-	if !*b.DirectDeployment {
+	if !directDeployment {
 		bundle.ApplySeqContext(ctx, b,
 			// We need to resolve artifact variable (how we do it in build phase)
 			// because some of the to-be-destroyed resource might use this variable.
@@ -151,7 +151,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle) {
 	}
 
 	var plan *deployplan.Plan
-	if *b.DirectDeployment {
+	if directDeployment {
 		_, localPath := b.StateFilenameDirect(ctx)
 		plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), nil, localPath)
 		if err != nil {
@@ -179,7 +179,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle) {
 	}
 
 	if hasApproval {
-		destroyCore(ctx, b, plan)
+		destroyCore(ctx, b, plan, directDeployment)
 	} else {
 		cmdio.LogString(ctx, "Destroy cancelled!")
 	}

bundle/phases/plan.go

@@ -18,9 +18,9 @@ import (
 
 // deployPrepare is common set of mutators between "bundle plan" and "bundle deploy".
 // This function does not make any mutations in the workspace remotely, only in-memory bundle config mutations
-func deployPrepare(ctx context.Context, b *bundle.Bundle, isPlan bool) map[string][]libraries.LocationToUpdate {
+func deployPrepare(ctx context.Context, b *bundle.Bundle, isPlan, directDeployment bool) map[string][]libraries.LocationToUpdate {
 	bundle.ApplySeqContext(ctx, b,
-		terraform.CheckDashboardsModifiedRemotely(isPlan),
+		terraform.CheckDashboardsModifiedRemotely(isPlan, directDeployment),
 		deploy.StatePull(),
 		mutator.ValidateGitDetails(),
 		terraform.CheckRunningResource(),

bundle/statemgmt/state_load.go

@@ -24,7 +24,8 @@ type (
 const ErrorOnEmptyState LoadMode = 0
 
 type load struct {
-	modes []LoadMode
+	modes            []LoadMode
+	directDeployment bool
 }
 
 func (l *load) Name() string {
@@ -35,11 +36,7 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	var err error
 	var state ExportedResourcesMap
 
-	if b.DirectDeployment == nil {
-		return diag.Errorf("internal error: statemgmt.Load() called without statemgmt.PullResourcesState()")
-	}
-
-	if *b.DirectDeployment {
+	if l.directDeployment {
 		_, fullPathDirect := b.StateFilenameDirect(ctx)
 		state, err = b.DeploymentBundle.ExportState(ctx, fullPathDirect)
 		if err != nil {
@@ -137,6 +134,6 @@ func (l *load) validateState(state ExportedResourcesMap) error {
 	return nil
 }
 
-func Load(modes ...LoadMode) bundle.Mutator {
-	return &load{modes: modes}
+func Load(directDeployment bool, modes ...LoadMode) bundle.Mutator {
+	return &load{modes: modes, directDeployment: directDeployment}
 }