diff --git a/acceptance/bundle/migrate/default-python/databricks.yml b/acceptance/bundle/migrate/default-python/databricks.yml new file mode 100644 index 0000000000..87eddb69ad --- /dev/null +++ b/acceptance/bundle/migrate/default-python/databricks.yml @@ -0,0 +1,47 @@ +bundle: + name: migrate-basic-test + +resources: + jobs: + test_job: + name: "Test Migration Job" + tasks: + - task_key: "main" + notebook_task: + notebook_path: "./notebook.py" + volumes: + test_volume: + catalog_name: "mycat" + schema_name: "myschema" + name: "myvol" + + pipelines: + test_pipeline: + name: "Test Migration Pipeline" + tags: + # ids + myjob_id: ${resources.jobs.test_job.id} + myvolume_id: ${resources.volumes.test_volume.id} + + # local field, string: + myjob_name: ${resources.jobs.test_job.name} + volume_catalog_name: ${resources.volumes.test_volume.catalog_name} + + # remote field, int, null + myjob_timeout: ${resources.jobs.test_job.timeout_seconds} + + # remote field, string: + volume_storage_location: ${resources.volumes.test_volume.storage_location} + libraries: + - notebook: + path: "./pipeline.py" + +targets: + dev: + default: true + prod: + resources: + schemas: + test_schema: + catalog_name: mycat + name: myschema diff --git a/acceptance/bundle/migrate/default-python/input.json b/acceptance/bundle/migrate/default-python/input.json new file mode 100644 index 0000000000..2c4416c00c --- /dev/null +++ b/acceptance/bundle/migrate/default-python/input.json @@ -0,0 +1,7 @@ +{ + "project_name": "my_default_python", + "include_notebook": "yes", + "include_dlt": "yes", + "include_python": "yes", + "serverless": "no" +} diff --git a/acceptance/bundle/migrate/default-python/notebook.py b/acceptance/bundle/migrate/default-python/notebook.py new file mode 100644 index 0000000000..250b3c9f76 --- /dev/null +++ b/acceptance/bundle/migrate/default-python/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Hello from test migration job") diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json new file mode 100644 index 0000000000..60d2bf1af5 --- /dev/null +++ b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json @@ -0,0 +1,299 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "update", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } + }, + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [NUMID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "timeout_seconds": 0, + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "webhook_notifications": {} + } + }, + "changes": { + "local": { + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + }, + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip", + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS][1], + "name": "[dev [USERNAME]] my_default_python_etl", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "id": "[UUID]", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "storage": "dbfs:/pipelines/[UUID]", + "tags": { + "dev": "[USERNAME]" + } + }, + "state": "IDLE" + }, + "changes": { + "remote": { + "storage": { + "action": "skip", + "reason": "server_side_default" + } + } + } + } + } +} diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json new file mode 100644 index 0000000000..df22957e03 --- /dev/null +++ b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json @@ -0,0 +1,312 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "update", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } + }, + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [NUMID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "timeout_seconds": 0, + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "webhook_notifications": {} + } + }, + "changes": { + "local": { + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + }, + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "job_clusters[0].new_cluster.num_workers": { + "action": "update", + "old": 0 + }, + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip", + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS][1], + "name": "[dev [USERNAME]] my_default_python_etl", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "id": "[UUID]", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "storage": "dbfs:/pipelines/[UUID]", + "tags": { + "dev": "[USERNAME]" + } + }, + "state": "IDLE" + }, + "changes": { + "remote": { + "storage": { + "action": "skip", + "reason": "server_side_default" + } + } + } + } + } +} diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json new file mode 100644 index 0000000000..8580bd570e --- /dev/null +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -0,0 +1,136 @@ +{ + "lineage": "[UUID]", + "serial": 4, + "state": { + "resources.jobs.sample_job": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } + }, + "resources.pipelines.my_default_python_etl": { + "__id__": "[UUID]", + "state": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "tags": { + "dev": "[USERNAME]" + } + } + } + } +} diff --git a/acceptance/bundle/migrate/default-python/out.state_original.json b/acceptance/bundle/migrate/default-python/out.state_original.json new file mode 100644 index 0000000000..85ccd62219 --- /dev/null +++ b/acceptance/bundle/migrate/default-python/out.state_original.json @@ -0,0 +1,430 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 3, + "lineage": "[UUID]", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "sample_job", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "[NUMID]", + "job_cluster": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": [ + { + "__apply_policy_default_values_allow_list": null, + "apply_policy_default_values": false, + "autoscale": [ + { + "max_workers": 4, + "min_workers": 1 + } + ], + "aws_attributes": [], + "azure_attributes": [], + "cluster_id": "", + "cluster_log_conf": [], + "cluster_mount_info": [], + "cluster_name": "", + "custom_tags": null, + "data_security_mode": "SINGLE_USER", + "docker_image": [], + "driver_instance_pool_id": "", + "driver_node_type_id": "", + "enable_elastic_disk": false, + "enable_local_disk_encryption": false, + "gcp_attributes": [], + "idempotency_token": "", + "init_scripts": [], + "instance_pool_id": "", + "is_single_node": false, + "kind": "", + "library": [], + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "policy_id": "", + "provider_config": [], + "remote_disk_throughput": 0, + "runtime_engine": "", + "single_user_name": "", + "spark_conf": null, + "spark_env_vars": null, + "spark_version": "16.4.x-scala2.12", + "ssh_public_keys": null, + "total_initial_remote_disk_size": 0, + "use_ml_runtime": false, + "workload_type": [] + } + ] + } + ], + "library": [], + "max_concurrent_runs": 4, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "[dev [USERNAME]] sample_job", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "performance_target": null, + "pipeline_task": [], + "provider_config": [], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "service_principal_name": "", + "user_name": "[USERNAME]" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": { + "dev": "[USERNAME]" + }, + "task": [ + { + "clean_rooms_notebook_task": [], + "condition_task": [], + "dashboard_task": [], + "dbt_cloud_task": [], + "dbt_platform_task": [], + "dbt_task": [], + "depends_on": [], + "description": "", + "disable_auto_optimization": false, + "disabled": false, + "email_notifications": [], + "environment_key": "", + "existing_cluster_id": "", + "for_each_task": [], + "gen_ai_compute_task": [], + "health": [], + "job_cluster_key": "job_cluster", + "library": [ + { + "cran": [], + "egg": "", + "jar": "", + "maven": [], + "provider_config": [], + "pypi": [], + "requirements": "", + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "max_retries": 0, + "min_retry_interval_millis": 0, + "new_cluster": [], + "notebook_task": [ + { + "base_parameters": null, + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook", + "source": "WORKSPACE", + "warehouse_id": "" + } + ], + "notification_settings": [], + "pipeline_task": [], + "power_bi_task": [], + "python_wheel_task": [], + "retry_on_timeout": false, + "run_if": "", + "run_job_task": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "sql_task": [], + "task_key": "notebook_task", + "timeout_seconds": 0, + "webhook_notifications": [] + }, + { + "clean_rooms_notebook_task": [], + "condition_task": [], + "dashboard_task": [], + "dbt_cloud_task": [], + "dbt_platform_task": [], + "dbt_task": [], + "depends_on": [ + { + "outcome": "", + "task_key": "notebook_task" + } + ], + "description": "", + "disable_auto_optimization": false, + "disabled": false, + "email_notifications": [], + "environment_key": "", + "existing_cluster_id": "", + "for_each_task": [], + "gen_ai_compute_task": [], + "health": [], + "job_cluster_key": "job_cluster", + "library": [ + { + "cran": [], + "egg": "", + "jar": "", + "maven": [], + "provider_config": [], + "pypi": [], + "requirements": "", + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "max_retries": 0, + "min_retry_interval_millis": 0, + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "pipeline_task": [], + "power_bi_task": [], + "python_wheel_task": [ + { + "entry_point": "main", + "named_parameters": null, + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + } + ], + "retry_on_timeout": false, + "run_if": "", + "run_job_task": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "sql_task": [], + "task_key": "python_wheel_task", + "timeout_seconds": 0, + "webhook_notifications": [] + }, + { + "clean_rooms_notebook_task": [], + "condition_task": [], + "dashboard_task": [], + "dbt_cloud_task": [], + "dbt_platform_task": [], + "dbt_task": [], + "depends_on": [ + { + "outcome": "", + "task_key": "notebook_task" + } + ], + "description": "", + "disable_auto_optimization": false, + "disabled": false, + "email_notifications": [], + "environment_key": "", + "existing_cluster_id": "", + "for_each_task": [], + "gen_ai_compute_task": [], + "health": [], + "job_cluster_key": "", + "library": [], + "max_retries": 0, + "min_retry_interval_millis": 0, + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "pipeline_task": [ + { + "full_refresh": false, + "pipeline_id": "[UUID]" + } + ], + "power_bi_task": [], + "python_wheel_task": [], + "retry_on_timeout": false, + "run_if": "", + "run_job_task": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "sql_task": [], + "task_key": "refresh_pipeline", + "timeout_seconds": 0, + "webhook_notifications": [] + } + ], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [ + { + "file_arrival": [], + "pause_status": "PAUSED", + "periodic": [ + { + "interval": 1, + "unit": "DAYS" + } + ], + "table_update": [] + } + ], + "url": "[DATABRICKS_URL]/#job/[NUMID]", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "databricks_pipeline.my_default_python_etl" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_pipeline", + "name": "my_default_python_etl", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allow_duplicate_names": false, + "budget_policy_id": null, + "catalog": null, + "cause": null, + "channel": "CURRENT", + "cluster": [], + "cluster_id": null, + "configuration": null, + "continuous": false, + "creator_user_name": "[USERNAME]", + "deployment": [ + { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + } + ], + "development": true, + "edition": "ADVANCED", + "environment": [ + { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + } + ], + "event_log": [], + "expected_last_modified": 0, + "filters": [], + "gateway_definition": [], + "health": null, + "id": "[UUID]", + "ingestion_definition": [], + "last_modified": [UNIX_TIME_MILLIS], + "latest_updates": null, + "library": [ + { + "file": [], + "glob": [ + { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + ], + "jar": "", + "maven": [], + "notebook": [], + "whl": "" + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "notification": [], + "photon": false, + "restart_window": [], + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "run_as": [], + "run_as_user_name": "[USERNAME]", + "schema": "[USERNAME]", + "serverless": false, + "state": "IDLE", + "storage": "dbfs:/pipelines/[UUID]", + "tags": { + "dev": "[USERNAME]" + }, + "target": null, + "timeouts": null, + "trigger": [], + "url": "[DATABRICKS_URL]/#joblist/pipelines/[UUID]", + "usage_policy_id": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt new file mode 100644 index 0000000000..ac8cb0eac0 --- /dev/null +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -0,0 +1,198 @@ + +>>> [CLI] bundle init default-python --config-file ./input.json +Welcome to the default Python template for Databricks Asset Bundles! + +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) + +✨ Your new project has been created in the 'my_default_python' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Building python_artifact... +Uploading .databricks/bundle/dev/patched_wheels/python_artifact_my_default_python/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_state.py + +>>> [CLI] bundle deployment migrate +Building python_artifact... +Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/my_default_python/.databricks/bundle/dev/resources.json + +Validate the migration by running "bundle plan", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy". + +To undo the migration, remove [TEST_TMP_DIR]/my_default_python/.databricks/bundle/dev/resources.json and rename [TEST_TMP_DIR]/my_default_python/.databricks/bundle/dev/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/my_default_python/.databricks/bundle/dev/terraform/terraform.tfstate + + +>>> print_state.py + +>>> jq .. | .libraries? | select(.) ../out.state_after_migration.json +[ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } +] +[ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } +] +[ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } +] + +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +Building python_artifact... +update jobs.sample_job + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan -o json +Building python_artifact... + +>>> jq .plan[] | .changes ../out.plan_after_migration.json +{ + "local": { + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" + } + }, + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "job_clusters[0].new_cluster.num_workers": { + "action": "update", + "old": 0 + }, + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } +} +{ + "remote": { + "storage": { + "action": "skip", + "reason": "server_side_default" + } + } +} + +>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle deploy +Building python_artifact... +Uploading .databricks/bundle/dev/patched_wheels/python_artifact_my_default_python/my_default_python-0.0.1+[UNIX_TIME_NANOS][3]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_state.py + +>>> diff.py ../out.state_after_migration.json ../out.state_after_deploy.json + +=== Extra plan: should have no drift +>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan +Building python_artifact... +update jobs.sample_job + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan -o json +Building python_artifact... + +>>> jq .plan[] | .changes ../out.plan_after_migration.json +{ + "local": { + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" + } + }, + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "job_clusters[0].new_cluster.num_workers": { + "action": "update", + "old": 0 + }, + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } +} +{ + "remote": { + "storage": { + "action": "skip", + "reason": "server_side_default" + } + } +} diff --git a/acceptance/bundle/migrate/default-python/pipeline.py b/acceptance/bundle/migrate/default-python/pipeline.py new file mode 100644 index 0000000000..cd19031e96 --- /dev/null +++ b/acceptance/bundle/migrate/default-python/pipeline.py @@ -0,0 +1,2 @@ +# Databricks notebook source +# This is a test pipeline notebook diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/default-python/script new file mode 100755 index 0000000000..58ef1bd66d --- /dev/null +++ b/acceptance/bundle/migrate/default-python/script @@ -0,0 +1,22 @@ +trace $CLI bundle init default-python --config-file ./input.json +cd my_default_python + +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +trace print_state.py > ../out.state_original.json +trace $CLI bundle deployment migrate +trace print_state.py > ../out.state_after_migration.json +trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json + +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_migration.json +trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl and num_workers +trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy + +trace print_state.py > ../out.state_after_deploy.json +trace diff.py ../out.state_after_migration.json ../out.state_after_deploy.json +rm ../out.state_after_deploy.json + +title "Extra plan: should have no drift" +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_deploy.json +trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl diff --git a/acceptance/bundle/migrate/default-python/test.toml b/acceptance/bundle/migrate/default-python/test.toml new file mode 100644 index 0000000000..0fad1e53ff --- /dev/null +++ b/acceptance/bundle/migrate/default-python/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = ["my_default_python"] diff --git a/bundle/phases/build.go b/bundle/phases/build.go index d141bb3916..5a32435f8f 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -7,12 +7,17 @@ import ( "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/bundle/trampoline" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" ) +type LibLocationMap map[string][]libraries.LocationToUpdate + // The build phase builds artifacts. -func Build(ctx context.Context, b *bundle.Bundle) { +func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { log.Info(ctx, "Phase: build") bundle.ApplySeqContext(ctx, b, @@ -25,5 +30,27 @@ func Build(ctx context.Context, b *bundle.Bundle) { mutator.ResolveVariableReferencesOnlyResources( "artifacts", ), + + // libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we + // know what are the actual library paths. + // libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this + // mutator is part of the deploy step rather than validate. + libraries.ExpandGlobReferences(), + libraries.CheckForSameNameLibraries(), + // SwitchToPatchedWheels must be run after ExpandGlobReferences and after build phase because it Artifact.Source and Artifact.Patched populated + libraries.SwitchToPatchedWheels(), + ) + + libs, diags := libraries.ReplaceWithRemotePath(ctx, b) + for _, diag := range diags { + logdiag.LogDiag(ctx, diag) + } + + bundle.ApplyContext(ctx, b, + // TransformWheelTask must be run after ReplaceWithRemotePath so we can use correct remote path in the + // transformed notebook + trampoline.TransformWheelTask(), ) + + return libs } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index b136b24303..e150bc7e91 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -136,7 +136,7 @@ func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]li } // The deploy phase deploys artifacts and resources. -func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, engine engine.EngineType) { +func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, engine engine.EngineType, libs map[string][]libraries.LocationToUpdate) { log.Info(ctx, "Phase: deploy") // Core mutators that CRUD resources and modify deployment state. These @@ -156,11 +156,6 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() - libs := DeployPrepare(ctx, b, false, engine) - if logdiag.HasError(ctx) { - return - } - uploadLibraries(ctx, b, libs) if logdiag.HasError(ctx) { return diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 48ccd9e6bc..4076b10da9 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -11,44 +11,19 @@ import ( "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/statemgmt" - "github.com/databricks/cli/bundle/trampoline" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/logdiag" ) -// DeployPrepare is common set of mutators between "bundle plan" and "bundle deploy". -// This function does not make any mutations in the workspace remotely, only in-memory bundle config mutations -func DeployPrepare(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) map[string][]libraries.LocationToUpdate { +// PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". +// Note, it is not run in "bundle migrate" so it must not modify the config +func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { bundle.ApplySeqContext(ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), deploy.StatePull(), mutator.ValidateGitDetails(), statemgmt.CheckRunningResource(engine), - - // libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we - // know what are the actual library paths. - // libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this - // mutator is part of the deploy step rather than validate. - libraries.ExpandGlobReferences(), - libraries.CheckForSameNameLibraries(), - // SwitchToPatchedWheels must be run after ExpandGlobReferences and after build phase because it Artifact.Source and Artifact.Patched populated - libraries.SwitchToPatchedWheels(), ) - - libs, diags := libraries.ReplaceWithRemotePath(ctx, b) - for _, diag := range diags { - logdiag.LogDiag(ctx, diag) - } - - bundle.ApplySeqContext(ctx, b, - // TransformWheelTask must be run after ReplaceWithRemotePath so we can use correct remote path in the - // transformed notebook - trampoline.TransformWheelTask(), - ) - - return libs } // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 74aa4c4677..2b1da0000e 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -35,10 +35,10 @@ It is useful for previewing changes before running 'bundle deploy'.`, cmd.RunE = func(cmd *cobra.Command, args []string) error { opts := utils.ProcessOptions{ - AlwaysPull: true, - FastValidate: true, - Build: true, - DeployPrepare: true, + AlwaysPull: true, + FastValidate: true, + Build: true, + PreDeployChecks: true, } // Only add InitFunc if we need to set force or cluster ID diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 82b206f78d..12d3efec41 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -59,11 +59,11 @@ type ProcessOptions struct { SkipEngineEnvVar bool // If true, call corresponding phase: - FastValidate bool - Validate bool - Build bool - DeployPrepare bool - Deploy bool + FastValidate bool + Validate bool + Build bool + PreDeployChecks bool + Deploy bool // Indicate whether the bundle operation originates from the pipelines CLI IsPipelinesCLI bool @@ -152,7 +152,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, var stateDesc *statemgmt.StateDesc - shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.DeployPrepare || opts.Deploy + shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy if shouldReadState { // PullResourcesState depends on stateFiler which needs b.Config.Workspace.StatePath which is set in phases.Initialize @@ -206,9 +206,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, } } - if opts.Build { + var libs phases.LibLocationMap + + if opts.Build || opts.Deploy { t2 := time.Now() - phases.Build(ctx, b) + libs = phases.Build(ctx, b) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Build", Value: time.Since(t2).Milliseconds(), @@ -219,12 +221,13 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, } } - if opts.DeployPrepare { - if opts.Deploy { - panic("Deploy already calls DeployPrepare internally") - } + if opts.PreDeployChecks || opts.Deploy { downgradeWarningToError := !opts.Deploy - _ = phases.DeployPrepare(ctx, b, downgradeWarningToError, stateDesc.Engine) + phases.PreDeployChecks(ctx, b, downgradeWarningToError, stateDesc.Engine) + + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } } if opts.Deploy { @@ -236,7 +239,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, } t3 := time.Now() - phases.Deploy(ctx, b, outputHandler, stateDesc.Engine) + phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, libs) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Deploy", Value: time.Since(t3).Milliseconds(),