wip migrate command & test

Author: denik

acceptance/bundle/deployment/migrate/basic/databricks.yml

@@ -12,10 +12,29 @@ resources:
             #permissions:
             #  - level: CAN_VIEW
             #    user_name: [email protected]
+  volumes:
+    test_volume:
+      catalog_name: "mycat"
+      schema_name: "myschema"
+      name: "myvol"
 
   pipelines:
     test_pipeline:
       name: "Test Migration Pipeline"
+      tags:
+        # ids
+        myjob_id: ${resources.jobs.test_job.id}
+        myvolume_id: ${resources.volumes.test_volume.id}
+
+        # local field, string:
+        myjob_name: ${resources.jobs.test_job.name}
+        volume_catalog_name: ${resources.volumes.test_volume.catalog_name}
+
+        # remote field, int, null
+        myjob_timeout: ${resources.jobs.test_job.timeout_seconds}
+
+        # remote field, string:
+        volume_storage_location: ${resources.volumes.test_volume.storage_location}
       libraries:
         - notebook:
             path: "./pipeline.py"

acceptance/bundle/deployment/migrate/basic/output.txt

@@ -1,6 +1,6 @@
 
 >>> musterr [CLI] bundle deployment migrate
-This command migrate existing terraform state file to direct deployment engine state. However, no existing local or remote state found.
+This command migrates existing terraform state file (terraform.tfstate) to direct deployment engine state file (resources.json). However, no existing local or remote state found.
 
 To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env var or bundle.engine="direct" in databricks.yml.
 
@@ -10,78 +10,40 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> head -n 5 .databricks/bundle/default/terraform/terraform.tfstate
-{
-  "version": 4,
-  "terraform_version": "1.5.5",
-  "serial": 3,
-  "lineage": "[UUID]",
-
 >>> [CLI] bundle deployment migrate
-Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
+Migrated 3 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
+To finalize deployment, run "bundle deploy".
+
+>>> ls -1 .databricks/bundle/default/
 deployment.json
 resources.json
 sync-snapshots
 terraform
 
->>> cat .databricks/bundle/default/resources.json
+>>> [CLI] bundle plan
+update pipelines.test_pipeline
+
+Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle debug plan
+
+>>> print_requests.py --get //jobs/get
 {
- "lineage": "[UUID]",
- "serial": 4,
- "state": {
-  "resources.jobs.test_job": {
-   "__id__": "[NUMID]",
-   "state": {
-    "deployment": {
-     "kind": "BUNDLE",
-     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
-    },
-    "edit_mode": "UI_LOCKED",
-    "format": "MULTI_TASK",
-    "max_concurrent_runs": 1,
-    "name": "Test Migration Job",
-    "queue": {
-     "enabled": true
-    },
-    "tasks": [
-     {
-      "notebook_task": {
-       "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/notebook"
-      },
-      "task_key": "main"
-     }
+  "headers": {
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat"
     ]
-   }
   },
-  "resources.pipelines.test_pipeline": {
-   "__id__": "[UUID]",
-   "state": {
-    "channel": "CURRENT",
-    "deployment": {
-     "kind": "BUNDLE",
-     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
-    },
-    "edition": "ADVANCED",
-    "libraries": [
-     {
-      "notebook": {
-       "path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline"
-      }
-     }
-    ],
-    "name": "Test Migration Pipeline"
-   }
+  "method": "GET",
+  "path": "/api/2.2/jobs/get",
+  "q": {
+    "job_id": "[NUMID]"
   }
- }
 }
->>> [CLI] bundle plan
-Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
-
->>> print_requests.py --get //jobs/get
 {
   "headers": {
     "User-Agent": [
-      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat"
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_debug_plan cmd-exec-id/[UUID] engine/direct auth/pat"
     ]
   },
   "method": "GET",
@@ -99,8 +61,9 @@ Deployment complete!
 
 >>> print_requests.py --get //jobs/get
 "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat"
+"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat"
 
 === Should show that it's already migrated
 >>> musterr [CLI] bundle deployment migrate
 Error: already using direct engine
-Details: <local direct state serial=4 lineage="[UUID]">
+Details: <local direct state serial=5 lineage="[UUID]">

acceptance/bundle/deployment/migrate/basic/script

@@ -2,15 +2,15 @@
 trace musterr $CLI bundle deployment migrate
 
 trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
-trace head -n 5 .databricks/bundle/default/terraform/terraform.tfstate
+print_state.py > out.original_state.json
 
 trace $CLI bundle deployment migrate
-
-ls -1 .databricks/bundle/default/ | contains.py "resources.json"
-trace cat .databricks/bundle/default/resources.json
+print_state.py > out.new_state.json
+trace ls -1 .databricks/bundle/default/ | contains.py "resources.json"
 
 rm out.requests.txt
-trace $CLI bundle plan
+trace $CLI bundle plan | contains.py "2 unchanged"
+trace $CLI bundle debug plan > out.plan_after_migrate.json
 trace print_requests.py --get //jobs/get | contains.py 'engine/direct'
 
 trace $CLI bundle deploy

cmd/bundle/deployment/migrate.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/cmd/bundle/utils"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/logdiag"
 	"github.com/spf13/cobra"
 )
 
@@ -55,7 +56,7 @@ proper state management as it may cause resource conflicts.`,
 		ctx := cmd.Context()
 
 		if stateDesc.Lineage == "" {
-			cmdio.LogString(ctx, `This command migrate existing terraform state file to direct deployment engine state. However, no existing local or remote state found.
+			cmdio.LogString(ctx, `This command migrates existing terraform state file (terraform.tfstate) to direct deployment engine state file (resources.json). However, no existing local or remote state found.
 
 To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env var or bundle.engine="direct" in databricks.yml.`)
 			return root.ErrAlreadyPrinted
@@ -65,6 +66,11 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va
 			return fmt.Errorf("already using direct engine\nDetails: %s", stateDesc.String())
 		}
 
+		_, localTerraformPath := b.StateFilenameTerraform(ctx)
+		if _, err = os.Stat(localTerraformPath); err != nil {
+			return fmt.Errorf("reading %s: %w", localTerraformPath, err)
+		}
+
 		terraformResources, err := terraform.ParseResourcesState(ctx, b)
 		if err != nil {
 			return fmt.Errorf("failed to parse terraform state: %w", err)
@@ -109,7 +115,14 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va
 		if err := os.Rename(tempStatePath, localPath); err != nil {
 			return fmt.Errorf("renaming %s to %s: %w", tempStatePath, localPath, err)
 		}
-		cmdio.LogString(ctx, fmt.Sprintf("Migrated %d resources to direct engine state file: %s", len(deploymentBundle.StateDB.Data.State), localPath))
+		err = os.Remove(localTerraformPath)
+		if err != nil {
+			// not fatal, since we've increased serial
+			logdiag.LogError(ctx, err)
+		}
+
+		cmdio.LogString(ctx, fmt.Sprintf(`Migrated %d resources to direct engine state file: %s
+To finalize deployment, run "bundle deploy".`, len(deploymentBundle.StateDB.Data.State), localPath))
 		return nil
 	}