Add "bundle deployment migrate" command (#3847)

## Changes

New migrate command to convert terraform state to direct engine state.

Depends on #3875

## Tests

Acceptance tests.

Extended print_state.py utility to accept -t/--target and --backup args.

---------

Co-authored-by: Pieter Noordhuis <[email protected]>

Author: denik

acceptance/bin/print_state.py

@@ -7,6 +7,7 @@
 """
 
 import os
+import argparse
 
 
 def write(filename):
@@ -16,10 +17,30 @@ def write(filename):
         print()
 
 
-filename = ".databricks/bundle/default/terraform/terraform.tfstate"
-if os.path.exists(filename):
-    write(filename)
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-t", "--target", default="default")
+    parser.add_argument("--backup", action="store_true")
+    args = parser.parse_args()
 
-filename = ".databricks/bundle/default/resources.json"
-if os.path.exists(filename):
-    write(filename)
+    if args.target:
+        target_dir = f".databricks/bundle/{args.target}"
+        if not os.path.exists(target_dir):
+            raise SystemExit(f"Invalid target {args.target!r}: {target_dir} does not exist")
+
+    if args.backup:
+        filename = f".databricks/bundle/{args.target}/terraform/terraform.tfstate.backup"
+        if os.path.exists(filename):
+            write(filename)
+    else:
+        filename = f".databricks/bundle/{args.target}/terraform/terraform.tfstate"
+        if os.path.exists(filename):
+            write(filename)
+
+        filename = f".databricks/bundle/{args.target}/resources.json"
+        if os.path.exists(filename):
+            write(filename)
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/help/bundle-deployment-migrate/out.test.toml

@@ -0,0 +1,24 @@
+
+>>> [CLI] bundle deployment migrate --help
+This command converts your bundle from using Terraform for deployment to using
+the Direct deployment engine. It reads resource IDs from the existing Terraform
+state and creates a Direct deployment state file (resources.json) with the same
+lineage and incremented serial number.
+
+Note, the migration is performed locally only. To finalize it, run 'bundle deploy'. This will synchronize the state file
+to the workspace so that subsequent deploys of this bundle use direct deployment engine as well.
+
+WARNING: Both direct deployment engine and this command are experimental and not recommended for production targets yet.
+
+Usage:
+  databricks bundle deployment migrate [flags]
+
+Flags:
+  -h, --help   help for migrate
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"

acceptance/bundle/help/bundle-deployment-migrate/output.txt

@@ -0,0 +1 @@
+trace $CLI bundle deployment migrate --help

acceptance/bundle/help/bundle-deployment-migrate/script

@@ -21,6 +21,7 @@ Usage:
 
 Available Commands:
   bind        Bind bundle-defined resources to existing resources
+  migrate     Migrate from Terraform to Direct deployment engine
   unbind      Unbind bundle-defined resources from its managed remote resource
 
 Flags:

acceptance/bundle/help/bundle-deployment/output.txt

@@ -0,0 +1,47 @@
+bundle:
+  name: migrate-basic-test
+
+resources:
+  jobs:
+    test_job:
+      name: "Test Migration Job"
+      tasks:
+        - task_key: "main"
+          notebook_task:
+            notebook_path: "./notebook.py"
+  volumes:
+    test_volume:
+      catalog_name: "mycat"
+      schema_name: "myschema"
+      name: "myvol"
+
+  pipelines:
+    test_pipeline:
+      name: "Test Migration Pipeline"
+      tags:
+        # ids
+        myjob_id: ${resources.jobs.test_job.id}
+        myvolume_id: ${resources.volumes.test_volume.id}
+
+        # local field, string:
+        myjob_name: ${resources.jobs.test_job.name}
+        volume_catalog_name: ${resources.volumes.test_volume.catalog_name}
+
+        # remote field, int, null
+        myjob_timeout: ${resources.jobs.test_job.timeout_seconds}
+
+        # remote field, string:
+        volume_storage_location: ${resources.volumes.test_volume.storage_location}
+      libraries:
+        - notebook:
+            path: "./pipeline.py"
+
+targets:
+  dev:
+    default: true
+  prod:
+    resources:
+      schemas:
+        test_schema:
+          catalog_name: mycat
+          name: myschema

acceptance/bundle/migrate/basic/databricks.yml

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("Hello from test migration job")

acceptance/bundle/migrate/basic/notebook.py

@@ -0,0 +1,66 @@
+{
+ "lineage": "[UUID]",
+ "serial": 5,
+ "state": {
+  "resources.jobs.test_job": {
+   "__id__": "[NUMID]",
+   "state": {
+    "deployment": {
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "Test Migration Job",
+    "queue": {
+     "enabled": true
+    },
+    "tasks": [
+     {
+      "notebook_task": {
+       "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/notebook"
+      },
+      "task_key": "main"
+     }
+    ]
+   }
+  },
+  "resources.pipelines.test_pipeline": {
+   "__id__": "[UUID]",
+   "state": {
+    "channel": "CURRENT",
+    "deployment": {
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
+    },
+    "edition": "ADVANCED",
+    "libraries": [
+     {
+      "notebook": {
+       "path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/pipeline"
+      }
+     }
+    ],
+    "name": "Test Migration Pipeline",
+    "tags": {
+     "myjob_id": "[NUMID]",
+     "myjob_name": "Test Migration Job",
+     "myjob_timeout": "",
+     "myvolume_id": "mycat.myschema.myvol",
+     "volume_catalog_name": "mycat",
+     "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+    }
+   }
+  },
+  "resources.volumes.test_volume": {
+   "__id__": "mycat.myschema.myvol",
+   "state": {
+    "catalog_name": "mycat",
+    "name": "myvol",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+   }
+  }
+ }
+}