Fix "migrate" to propagate --profile to "bundle plan" (#4117)

Follow up to #4088, we need to
pass "-p profile" as well.

Author: denik

acceptance/bundle/migrate/profile_arg/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+  name: migrate-basic-test
+
+resources:
+  jobs:
+    test_job:
+      name: "Test Migration Job"
+      tasks:
+        - task_key: "main"
+          notebook_task:
+            notebook_path: "./notebook.py"
+
+targets:
+  dev:
+    default: true
+  prod:
+    resources:
+      schemas:
+        test_schema:
+          catalog_name: mycat
+          name: myschema

acceptance/bundle/migrate/profile_arg/notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("Hello from test migration job")

acceptance/bundle/migrate/profile_arg/out.test.toml

@@ -0,0 +1,36 @@
+
+>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle deployment migrate -p non_existent321
+Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json
+
+Validate the migration by running "databricks bundle plan -p non_existent321", there should be no actions planned.
+
+The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy -p non_existent321".
+
+To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/dev/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/dev/terraform/terraform.tfstate
+
+
+>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy -t prod
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/prod/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle deployment migrate -p non_existent321 -t prod
+Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/prod/resources.json
+
+Validate the migration by running "databricks bundle plan -t prod -p non_existent321", there should be no actions planned.
+
+The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy -t prod -p non_existent321".
+
+To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/prod/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/prod/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/prod/terraform/terraform.tfstate
+

acceptance/bundle/migrate/profile_arg/output.txt

@@ -0,0 +1,2 @@
+# Databricks notebook source
+# This is a test pipeline notebook

acceptance/bundle/migrate/profile_arg/pipeline.py

@@ -0,0 +1,7 @@
+trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
+
+# only interested in output here; for some reason in test env -p is not validated
+trace $CLI bundle deployment migrate -p non_existent321
+
+trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t prod
+trace $CLI bundle deployment migrate -p non_existent321 -t prod

acceptance/bundle/migrate/profile_arg/script

@@ -0,0 +1 @@
+RecordRequests = false

acceptance/bundle/migrate/profile_arg/test.toml

@@ -24,7 +24,7 @@ const backupSuffix = ".backup"
 
 // runPlanCheck runs bundle plan and checks if there are any actions planned.
 // Returns error if plan fails or if there are actions planned.
-func runPlanCheck(cmd *cobra.Command, target string) error {
+func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error {
 	ctx := cmd.Context()
 
 	executable, err := os.Executable()
@@ -33,11 +33,7 @@ func runPlanCheck(cmd *cobra.Command, target string) error {
 	}
 
 	args := []string{"bundle", "plan"}
-	targetArgs := ""
-	if target != "" {
-		targetArgs = " -t " + target
-		args = append(args, "-t", target)
-	}
+	args = append(args, extraArgs...)
 
 	planCmd := exec.CommandContext(ctx, executable, args...)
 	var stdout bytes.Buffer
@@ -65,16 +61,42 @@ func runPlanCheck(cmd *cobra.Command, target string) error {
 	}
 
 	if !strings.Contains(output, "Plan:") {
-		return fmt.Errorf("cannot parse 'databricks bundle plan%s' output, aborting migration. Skip plan check with --noplancheck option", targetArgs)
+		return fmt.Errorf("cannot parse 'databricks bundle plan%s' output, aborting migration. Skip plan check with --noplancheck option", extraArgsStr)
 	}
 
 	if !strings.Contains(output, "Plan: 0 to add, 0 to change, 0 to delete") {
-		return fmt.Errorf("'databricks bundle plan%s' shows actions planned, aborting migration. Please run 'databricks bundle deploy%s' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option", targetArgs, targetArgs)
+		return fmt.Errorf("'databricks bundle plan%s' shows actions planned, aborting migration. Please run 'databricks bundle deploy%s' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option", extraArgsStr, extraArgsStr)
 	}
 
 	return nil
 }
 
+func getCommonArgs(cmd *cobra.Command) ([]string, string) {
+	var args []string
+	if flag := cmd.Flag("target"); flag != nil && flag.Changed {
+		target := flag.Value.String()
+		if target != "" {
+			args = append(args, "-t")
+			args = append(args, target)
+		}
+	}
+	if flag := cmd.Flag("profile"); flag != nil && flag.Changed {
+		profile := flag.Value.String()
+		if profile != "" {
+			args = append(args, "-p")
+			args = append(args, profile)
+		}
+	}
+
+	argsStr := ""
+
+	if len(args) > 0 {
+		argsStr = " " + strings.Join(args, " ")
+	}
+
+	return args, argsStr
+}
+
 func newMigrateCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "migrate",
@@ -96,15 +118,7 @@ WARNING: Both direct deployment engine and this command are experimental and not
 	cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		// Get the target if it was explicitly passed
-		target := ""
-		if flag := cmd.Flag("target"); flag != nil && flag.Changed {
-			target = flag.Value.String()
-		}
-		targetArgs := ""
-		if target != "" {
-			targetArgs += " -t " + target
-		}
+		extraArgs, extraArgsStr := getCommonArgs(cmd)
 
 		opts := utils.ProcessOptions{
 			SkipEngineEnvVar: true,
@@ -154,7 +168,7 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va
 		// Run plan check unless --noplancheck is set
 		if !noPlanCheck {
 			fmt.Fprintf(cmd.OutOrStdout(), "Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:\n")
-			if err = runPlanCheck(cmd, target); err != nil {
+			if err = runPlanCheck(cmd, extraArgs, extraArgsStr); err != nil {
 				return err
 			}
 		}
@@ -237,7 +251,7 @@ Validate the migration by running "databricks bundle plan%s", there should be no
 The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy%s".
 
 To undo the migration, remove %s and rename %s to %s
-`, len(deploymentBundle.StateDB.Data.State), localPath, targetArgs, targetArgs, localPath, localTerraformBackupPath, localTerraformPath))
+`, len(deploymentBundle.StateDB.Data.State), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath))
 		return nil
 	}