Use EngineType; enforce env var engine matching state

Author: denik

acceptance/bundle/deployment/migrate/basic/out.plan_after_migrate.json

@@ -23,12 +23,31 @@ terraform
 >>> jq .state."resources.pipelines.test_pipeline".state.tags.volume_storage_location out.new_state.json
 "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
 
->>> [CLI] bundle plan
+>>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle plan
+Error: Required engine "terraform" does not match present state files. Set required engine via "DATABRICKS_BUNDLE_ENGINE" env var.
+
+Available state files:
+- terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
+
+
+>>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
+Error: Required engine "terraform" does not match present state files. Set required engine via "DATABRICKS_BUNDLE_ENGINE" env var.
+
+Available state files:
+- terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
+
+
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
 update pipelines.test_pipeline
 
 Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
 
->>> [CLI] bundle debug plan
+>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan
+update pipelines.test_pipeline
+
+Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
 
 >>> print_requests.py --get //jobs/get
 {
@@ -46,7 +65,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
 {
   "headers": {
     "User-Agent": [
-      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_debug_plan cmd-exec-id/[UUID] engine/direct auth/pat"
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat"
     ]
   },
   "method": "GET",
@@ -56,7 +75,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
   }
 }
 
->>> [CLI] bundle deploy
+>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files...
 Deploying resources...
 Updating deployment state...
@@ -70,3 +89,11 @@ Deployment complete!
 >>> musterr [CLI] bundle deployment migrate
 Error: already using direct engine
 Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
+
+>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
+
+>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull
+terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
+resources.json: remote direct state serial=5 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"

acceptance/bundle/deployment/migrate/basic/out.test.toml

@@ -4,20 +4,28 @@ trace musterr $CLI bundle deployment migrate
 trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
 print_state.py > out.original_state.json
 
-trace $CLI bundle deployment migrate
+trace $CLI bundle deployment migrate 2>&1 | contains.py 'Migrated 3 resources'
 print_state.py > out.new_state.json
 trace ls -1 .databricks/bundle/default/ | contains.py "resources.json"
 # check that this is stored as resolved value, not as reference:
 trace jq '.state."resources.pipelines.test_pipeline".state.tags.volume_storage_location' out.new_state.json | contains.py 's3://'
 
+# this is expected to fail because state does not match env var
+trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle plan
+trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle deploy
+
 rm out.requests.txt
-trace $CLI bundle plan | contains.py "2 unchanged"  # should be "3 unchanged" when permanent drift is fixed
-trace $CLI bundle debug plan > out.plan_after_migrate.json
+# Both DATABRICKS_BUNDLE_ENGINE=direct and DATABRICKS_BUNDLE_ENGINE= work
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "2 unchanged"  # should be "3 unchanged" once permanent drift is fixed
+trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan | contains.py "2 unchanged"
 trace print_requests.py --get //jobs/get | contains.py 'engine/direct'
 
-trace $CLI bundle deploy
+trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy
 trace print_requests.py --get //jobs/get | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct'
 
 title "Should show that it's already migrated"
 trace musterr $CLI bundle deployment migrate
+trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states
+trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states --force-pull
+
 rm out.requests.txt