update test

Author: denik

acceptance/bundle/deployment/migrate/basic/databricks.yml

@@ -30,12 +30,12 @@ resources:
         myjob_name: ${resources.jobs.test_job.name}
         volume_catalog_name: ${resources.volumes.test_volume.catalog_name}
 
-        # Remote field cause permanent drift (unrelated to migration)
+        # Remote fields cause permanent drift (unrelated to migration)
         # remote field, int, null
-        #myjob_timeout: ${resources.jobs.test_job.timeout_seconds}
+        myjob_timeout: ${resources.jobs.test_job.timeout_seconds}
 
         # remote field, string:
-        #volume_storage_location: ${resources.volumes.test_volume.storage_location}
+        volume_storage_location: ${resources.volumes.test_volume.storage_location}
       libraries:
         - notebook:
             path: "./pipeline.py"

acceptance/bundle/deployment/migrate/basic/out.new_state.json

@@ -46,8 +46,10 @@
     "tags": {
      "myjob_id": "[NUMID]",
      "myjob_name": "Test Migration Job",
+     "myjob_timeout": "",
      "myvolume_id": "mycat.myschema.myvol",
-     "volume_catalog_name": "mycat"
+     "volume_catalog_name": "mycat",
+     "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
     }
    }
   },

acceptance/bundle/deployment/migrate/basic/out.original_state.json

@@ -184,8 +184,10 @@
             "tags": {
               "myjob_id": "[NUMID]",
               "myjob_name": "Test Migration Job",
+              "myjob_timeout": "0",
               "myvolume_id": "mycat.myschema.myvol",
-              "volume_catalog_name": "mycat"
+              "volume_catalog_name": "mycat",
+              "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
             },
             "target": null,
             "timeouts": null,

acceptance/bundle/deployment/migrate/basic/out.plan_after_migrate.json

@@ -47,16 +47,53 @@
           "node": "resources.jobs.test_job",
           "label": "${resources.jobs.test_job.name}"
         },
+        {
+          "node": "resources.jobs.test_job",
+          "label": "${resources.jobs.test_job.timeout_seconds}"
+        },
         {
           "node": "resources.volumes.test_volume",
           "label": "${resources.volumes.test_volume.catalog_name}"
         },
         {
           "node": "resources.volumes.test_volume",
           "label": "${resources.volumes.test_volume.id}"
+        },
+        {
+          "node": "resources.volumes.test_volume",
+          "label": "${resources.volumes.test_volume.storage_location}"
         }
       ],
-      "action": "skip",
+      "action": "update",
+      "new_state": {
+        "config": {
+          "channel": "CURRENT",
+          "deployment": {
+            "kind": "BUNDLE",
+            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
+          },
+          "edition": "ADVANCED",
+          "libraries": [
+            {
+              "notebook": {
+                "path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline"
+              }
+            }
+          ],
+          "name": "Test Migration Pipeline",
+          "tags": {
+            "myjob_id": "[NUMID]",
+            "myjob_name": "Test Migration Job",
+            "myjob_timeout": "",
+            "myvolume_id": "mycat.myschema.myvol",
+            "volume_catalog_name": "mycat",
+            "volume_storage_location": "${resources.volumes.test_volume.storage_location}"
+          }
+        },
+        "vars": {
+          "tags.volume_storage_location": "${resources.volumes.test_volume.storage_location}"
+        }
+      },
       "remote_state": {
         "creator_user_name": "[USERNAME]",
         "last_modified": [UNIX_TIME_MILLIS],
@@ -83,17 +120,27 @@
           "tags": {
             "myjob_id": "[NUMID]",
             "myjob_name": "Test Migration Job",
+            "myjob_timeout": "0",
             "myvolume_id": "mycat.myschema.myvol",
-            "volume_catalog_name": "mycat"
+            "volume_catalog_name": "mycat",
+            "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
           }
         },
         "state": "IDLE"
       },
       "changes": {
+        "local": {
+          "tags.volume_storage_location": {
+            "action": "update"
+          }
+        },
         "remote": {
           "storage": {
             "action": "skip",
             "reason": "server_side_default"
+          },
+          "tags.myjob_timeout": {
+            "action": "update"
           }
         }
       }

acceptance/bundle/deployment/migrate/basic/output.txt

@@ -20,8 +20,13 @@ resources.json
 sync-snapshots
 terraform
 
+>>> jq .state."resources.pipelines.test_pipeline".state.tags.volume_storage_location out.new_state.json
+"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+
 >>> [CLI] bundle plan
-Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
+update pipelines.test_pipeline
+
+Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
 
 >>> [CLI] bundle debug plan
 

acceptance/bundle/deployment/migrate/basic/script

@@ -7,9 +7,11 @@ print_state.py > out.original_state.json
 trace $CLI bundle deployment migrate
 print_state.py > out.new_state.json
 trace ls -1 .databricks/bundle/default/ | contains.py "resources.json"
+# check that this is stored as resolved value, not as reference:
+trace jq '.state."resources.pipelines.test_pipeline".state.tags.volume_storage_location' out.new_state.json | contains.py 's3://'
 
 rm out.requests.txt
-trace $CLI bundle plan | contains.py "3 unchanged"
+trace $CLI bundle plan | contains.py "2 unchanged"  # should be "3 unchanged" when permanent drift is fixed
 trace $CLI bundle debug plan > out.plan_after_migrate.json
 trace print_requests.py --get //jobs/get | contains.py 'engine/direct'