Fix migrate to store expanded globs in the new state

acceptance/bundle/migrate/default-python/databricks.yml

@@ -0,0 +1,47 @@
+bundle:
+  name: migrate-basic-test
+
+resources:
+  jobs:
+    test_job:
+      name: "Test Migration Job"
+      tasks:
+        - task_key: "main"
+          notebook_task:
+            notebook_path: "./notebook.py"
+  volumes:
+    test_volume:
+      catalog_name: "mycat"
+      schema_name: "myschema"
+      name: "myvol"
+
+  pipelines:
+    test_pipeline:
+      name: "Test Migration Pipeline"
+      tags:
+        # ids
+        myjob_id: ${resources.jobs.test_job.id}
+        myvolume_id: ${resources.volumes.test_volume.id}
+
+        # local field, string:
+        myjob_name: ${resources.jobs.test_job.name}
+        volume_catalog_name: ${resources.volumes.test_volume.catalog_name}
+
+        # remote field, int, null
+        myjob_timeout: ${resources.jobs.test_job.timeout_seconds}
+
+        # remote field, string:
+        volume_storage_location: ${resources.volumes.test_volume.storage_location}
+      libraries:
+        - notebook:
+            path: "./pipeline.py"
+
+targets:
+  dev:
+    default: true
+  prod:
+    resources:
+      schemas:
+        test_schema:
+          catalog_name: mycat
+          name: myschema

acceptance/bundle/migrate/default-python/input.json

@@ -0,0 +1,7 @@
+{
+    "project_name": "my_default_python",
+    "include_notebook": "yes",
+    "include_dlt": "yes",
+    "include_python": "yes",
+    "serverless": "no"
+}

acceptance/bundle/migrate/default-python/notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("Hello from test migration job")

acceptance/bundle/migrate/default-python/out.plan_after_deploy.json

@@ -0,0 +1,299 @@
+{
+  "plan": {
+    "resources.jobs.sample_job": {
+      "depends_on": [
+        {
+          "node": "resources.pipelines.my_default_python_etl",
+          "label": "${resources.pipelines.my_default_python_etl.id}"
+        }
+      ],
+      "action": "update",
+      "new_state": {
+        "value": {
+          "deployment": {
+            "kind": "BUNDLE",
+            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
+          },
+          "edit_mode": "UI_LOCKED",
+          "format": "MULTI_TASK",
+          "job_clusters": [
+            {
+              "job_cluster_key": "job_cluster",
+              "new_cluster": {
+                "autoscale": {
+                  "max_workers": 4,
+                  "min_workers": 1
+                },
+                "data_security_mode": "SINGLE_USER",
+                "node_type_id": "[NODE_TYPE_ID]",
+                "num_workers": 0,
+                "spark_version": "16.4.x-scala2.12"
+              }
+            }
+          ],
+          "max_concurrent_runs": 4,
+          "name": "[dev [USERNAME]] sample_job",
+          "parameters": [
+            {
+              "default": "hive_metastore",
+              "name": "catalog"
+            },
+            {
+              "default": "[USERNAME]",
+              "name": "schema"
+            }
+          ],
+          "queue": {
+            "enabled": true
+          },
+          "tags": {
+            "dev": "[USERNAME]"
+          },
+          "tasks": [
+            {
+              "job_cluster_key": "job_cluster",
+              "libraries": [
+                {
+                  "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
+                }
+              ],
+              "notebook_task": {
+                "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook"
+              },
+              "task_key": "notebook_task"
+            },
+            {
+              "depends_on": [
+                {
+                  "task_key": "notebook_task"
+                }
+              ],
+              "job_cluster_key": "job_cluster",
+              "libraries": [
+                {
+                  "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
+                }
+              ],
+              "python_wheel_task": {
+                "entry_point": "main",
+                "package_name": "my_default_python",
+                "parameters": [
+                  "--catalog",
+                  "hive_metastore",
+                  "--schema",
+                  "[USERNAME]"
+                ]
+              },
+              "task_key": "python_wheel_task"
+            },
+            {
+              "depends_on": [
+                {
+                  "task_key": "notebook_task"
+                }
+              ],
+              "pipeline_task": {
+                "pipeline_id": "[UUID]"
+              },
+              "task_key": "refresh_pipeline"
+            }
+          ],
+          "trigger": {
+            "pause_status": "PAUSED",
+            "periodic": {
+              "interval": 1,
+              "unit": "DAYS"
+            }
+          }
+        }
+      },
+      "remote_state": {
+        "created_time": [UNIX_TIME_MILLIS][0],
+        "creator_user_name": "[USERNAME]",
+        "job_id": [NUMID],
+        "run_as_user_name": "[USERNAME]",
+        "settings": {
+          "deployment": {
+            "kind": "BUNDLE",
+            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
+          },
+          "edit_mode": "UI_LOCKED",
+          "email_notifications": {},
+          "format": "MULTI_TASK",
+          "job_clusters": [
+            {
+              "job_cluster_key": "job_cluster",
+              "new_cluster": {
+                "autoscale": {
+                  "max_workers": 4,
+                  "min_workers": 1
+                },
+                "data_security_mode": "SINGLE_USER",
+                "node_type_id": "[NODE_TYPE_ID]",
+                "num_workers": 0,
+                "spark_version": "16.4.x-scala2.12"
+              }
+            }
+          ],
+          "max_concurrent_runs": 4,
+          "name": "[dev [USERNAME]] sample_job",
+          "parameters": [
+            {
+              "default": "hive_metastore",
+              "name": "catalog"
+            },
+            {
+              "default": "[USERNAME]",
+              "name": "schema"
+            }
+          ],
+          "queue": {
+            "enabled": true
+          },
+          "tags": {
+            "dev": "[USERNAME]"
+          },
+          "tasks": [
+            {
+              "job_cluster_key": "job_cluster",
+              "libraries": [
+                {
+                  "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
+                }
+              ],
+              "notebook_task": {
+                "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook",
+                "source": "WORKSPACE"
+              },
+              "task_key": "notebook_task"
+            },
+            {
+              "depends_on": [
+                {
+                  "task_key": "notebook_task"
+                }
+              ],
+              "job_cluster_key": "job_cluster",
+              "libraries": [
+                {
+                  "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
+                }
+              ],
+              "python_wheel_task": {
+                "entry_point": "main",
+                "package_name": "my_default_python",
+                "parameters": [
+                  "--catalog",
+                  "hive_metastore",
+                  "--schema",
+                  "[USERNAME]"
+                ]
+              },
+              "task_key": "python_wheel_task"
+            },
+            {
+              "depends_on": [
+                {
+                  "task_key": "notebook_task"
+                }
+              ],
+              "pipeline_task": {
+                "pipeline_id": "[UUID]"
+              },
+              "task_key": "refresh_pipeline"
+            }
+          ],
+          "timeout_seconds": 0,
+          "trigger": {
+            "pause_status": "PAUSED",
+            "periodic": {
+              "interval": 1,
+              "unit": "DAYS"
+            }
+          },
+          "webhook_notifications": {}
+        }
+      },
+      "changes": {
+        "local": {
+          "tasks[task_key='notebook_task'].libraries[0].whl": {
+            "action": "update",
+            "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
+            "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
+          },
+          "tasks[task_key='python_wheel_task'].libraries[0].whl": {
+            "action": "update",
+            "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
+            "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
+          }
+        },
+        "remote": {
+          "email_notifications": {
+            "action": "skip",
+            "reason": "server_side_default"
+          },
+          "tasks[task_key='notebook_task'].notebook_task.source": {
+            "action": "skip",
+            "reason": "server_side_default"
+          },
+          "timeout_seconds": {
+            "action": "skip",
+            "reason": "server_side_default"
+          },
+          "webhook_notifications": {
+            "action": "skip",
+            "reason": "server_side_default"
+          }
+        }
+      }
+    },
+    "resources.pipelines.my_default_python_etl": {
+      "action": "skip",
+      "remote_state": {
+        "creator_user_name": "[USERNAME]",
+        "last_modified": [UNIX_TIME_MILLIS][1],
+        "name": "[dev [USERNAME]] my_default_python_etl",
+        "pipeline_id": "[UUID]",
+        "run_as_user_name": "[USERNAME]",
+        "spec": {
+          "channel": "CURRENT",
+          "deployment": {
+            "kind": "BUNDLE",
+            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
+          },
+          "development": true,
+          "edition": "ADVANCED",
+          "environment": {
+            "dependencies": [
+              "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files"
+            ]
+          },
+          "id": "[UUID]",
+          "libraries": [
+            {
+              "glob": {
+                "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**"
+              }
+            }
+          ],
+          "name": "[dev [USERNAME]] my_default_python_etl",
+          "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl",
+          "schema": "[USERNAME]",
+          "storage": "dbfs:/pipelines/[UUID]",
+          "tags": {
+            "dev": "[USERNAME]"
+          }
+        },
+        "state": "IDLE"
+      },
+      "changes": {
+        "remote": {
+          "storage": {
+            "action": "skip",
+            "reason": "server_side_default"
+          }
+        }
+      }
+    }
+  }
+}