Skip to content

Commit 33d8ac5

Browse files
denikdenik
committed
WIP migrate command
update export DeployPrepare, make it explicit in plan commands rewrite migration to do plan + dry run deploy warning fix lint & fix Extend jobs to record full state wip migrate command & test Reject same serials in direct and terraform state file update fix warning/error in checkDashboardsModified remotely add missing files update update lint update test ident update test add 'debug states' command add bad_env Use EngineType; enforce env var engine matching state lint fix move to config/engin update message to slash toslash for state files
1 parent 3876283 commit 33d8ac5

File tree

19 files changed

+662
-5
lines changed

19 files changed

+662
-5
lines changed

‎acceptance/bundle/help/bundle-deployment-migrate/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/help/bundle-deployment-migrate/output.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
2+
>>> [CLI] bundle deployment migrate --help
3+
This command converts your bundle from using Terraform for deployment to using
4+
the Direct deployment engine. It reads resource IDs from the existing Terraform
5+
state and creates a Direct deployment state file (resources.json) with the same
6+
lineage and incremented serial number.
7+
8+
Note, the migration is performed locally only. To finalize it, run 'bundle deploy'. This will synchronize the state file
9+
to the workspace so that subsequent deploys of this bundle use direct deployment engine as well.
10+
11+
WARNING: Both direct deployment engine and this command are experimental and not recommended for production targets yet.
12+
13+
Usage:
14+
databricks bundle deployment migrate [flags]
15+
16+
Flags:
17+
-h, --help help for migrate
18+
19+
Global Flags:
20+
--debug enable debug logging
21+
-o, --output type output type: text or json (default text)
22+
-p, --profile string ~/.databrickscfg profile
23+
-t, --target string bundle target to use (if applicable)
24+
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"

‎acceptance/bundle/help/bundle-deployment-migrate/script‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
trace $CLI bundle deployment migrate --help

‎acceptance/bundle/help/bundle-deployment/output.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ Usage:
2121

2222
Available Commands:
2323
bind Bind bundle-defined resources to existing resources
24+
migrate Migrate from Terraform to Direct deployment engine
2425
unbind Unbind bundle-defined resources from its managed remote resource
2526

2627
Flags:

‎acceptance/bundle/migrate/basic/databricks.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
bundle:
2+
name: migrate-basic-test
3+
4+
resources:
5+
jobs:
6+
test_job:
7+
name: "Test Migration Job"
8+
tasks:
9+
- task_key: "main"
10+
notebook_task:
11+
notebook_path: "./notebook.py"
12+
# permissions dont work yet
13+
#permissions:
14+
# - level: CAN_VIEW
15+
# user_name: [email protected]
16+
volumes:
17+
test_volume:
18+
catalog_name: "mycat"
19+
schema_name: "myschema"
20+
name: "myvol"
21+
22+
pipelines:
23+
test_pipeline:
24+
name: "Test Migration Pipeline"
25+
tags:
26+
# ids
27+
myjob_id: ${resources.jobs.test_job.id}
28+
myvolume_id: ${resources.volumes.test_volume.id}
29+
30+
# local field, string:
31+
myjob_name: ${resources.jobs.test_job.name}
32+
volume_catalog_name: ${resources.volumes.test_volume.catalog_name}
33+
34+
# Remote fields cause permanent drift (unrelated to migration)
35+
# remote field, int, null
36+
myjob_timeout: ${resources.jobs.test_job.timeout_seconds}
37+
38+
# remote field, string:
39+
volume_storage_location: ${resources.volumes.test_volume.storage_location}
40+
libraries:
41+
- notebook:
42+
path: "./pipeline.py"
43+
#permissions:
44+
# - level: CAN_MANAGE
45+
# user_name: [email protected]

‎acceptance/bundle/migrate/basic/notebook.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Databricks notebook source
2+
print("Hello from test migration job")

‎acceptance/bundle/migrate/basic/out.new_state.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"lineage": "[UUID]",
3+
"serial": 5,
4+
"state": {
5+
"resources.jobs.test_job": {
6+
"__id__": "[NUMID]",
7+
"state": {
8+
"deployment": {
9+
"kind": "BUNDLE",
10+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
11+
},
12+
"edit_mode": "UI_LOCKED",
13+
"format": "MULTI_TASK",
14+
"max_concurrent_runs": 1,
15+
"name": "Test Migration Job",
16+
"queue": {
17+
"enabled": true
18+
},
19+
"tasks": [
20+
{
21+
"notebook_task": {
22+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/notebook"
23+
},
24+
"task_key": "main"
25+
}
26+
]
27+
}
28+
},
29+
"resources.pipelines.test_pipeline": {
30+
"__id__": "[UUID]",
31+
"state": {
32+
"channel": "CURRENT",
33+
"deployment": {
34+
"kind": "BUNDLE",
35+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
36+
},
37+
"edition": "ADVANCED",
38+
"libraries": [
39+
{
40+
"notebook": {
41+
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline"
42+
}
43+
}
44+
],
45+
"name": "Test Migration Pipeline",
46+
"tags": {
47+
"myjob_id": "[NUMID]",
48+
"myjob_name": "Test Migration Job",
49+
"myjob_timeout": "",
50+
"myvolume_id": "mycat.myschema.myvol",
51+
"volume_catalog_name": "mycat",
52+
"volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
53+
}
54+
}
55+
},
56+
"resources.volumes.test_volume": {
57+
"__id__": "mycat.myschema.myvol",
58+
"state": {
59+
"catalog_name": "mycat",
60+
"name": "myvol",
61+
"schema_name": "myschema",
62+
"volume_type": "MANAGED"
63+
}
64+
}
65+
}
66+
}

‎acceptance/bundle/migrate/basic/out.original_state.json‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
{
2+
"version": 4,
3+
"terraform_version": "1.5.5",
4+
"serial": 4,
5+
"lineage": "[UUID]",
6+
"outputs": {},
7+
"resources": [
8+
{
9+
"mode": "managed",
10+
"type": "databricks_job",
11+
"name": "test_job",
12+
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
13+
"instances": [
14+
{
15+
"schema_version": 2,
16+
"attributes": {
17+
"always_running": false,
18+
"budget_policy_id": null,
19+
"continuous": [],
20+
"control_run_state": false,
21+
"dbt_task": [],
22+
"deployment": [
23+
{
24+
"kind": "BUNDLE",
25+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
26+
}
27+
],
28+
"description": null,
29+
"edit_mode": "UI_LOCKED",
30+
"email_notifications": [],
31+
"environment": [],
32+
"existing_cluster_id": null,
33+
"format": "MULTI_TASK",
34+
"git_source": [],
35+
"health": [],
36+
"id": "[NUMID]",
37+
"job_cluster": [],
38+
"library": [],
39+
"max_concurrent_runs": 1,
40+
"max_retries": 0,
41+
"min_retry_interval_millis": 0,
42+
"name": "Test Migration Job",
43+
"new_cluster": [],
44+
"notebook_task": [],
45+
"notification_settings": [],
46+
"parameter": [],
47+
"performance_target": null,
48+
"pipeline_task": [],
49+
"python_wheel_task": [],
50+
"queue": [
51+
{
52+
"enabled": true
53+
}
54+
],
55+
"retry_on_timeout": false,
56+
"run_as": null,
57+
"run_job_task": [],
58+
"schedule": [],
59+
"spark_jar_task": [],
60+
"spark_python_task": [],
61+
"spark_submit_task": [],
62+
"tags": null,
63+
"task": [
64+
{
65+
"clean_rooms_notebook_task": [],
66+
"condition_task": [],
67+
"dashboard_task": [],
68+
"dbt_cloud_task": [],
69+
"dbt_platform_task": [],
70+
"dbt_task": [],
71+
"depends_on": [],
72+
"description": "",
73+
"disable_auto_optimization": false,
74+
"disabled": false,
75+
"email_notifications": [],
76+
"environment_key": "",
77+
"existing_cluster_id": "",
78+
"for_each_task": [],
79+
"gen_ai_compute_task": [],
80+
"health": [],
81+
"job_cluster_key": "",
82+
"library": [],
83+
"max_retries": 0,
84+
"min_retry_interval_millis": 0,
85+
"new_cluster": [],
86+
"notebook_task": [
87+
{
88+
"base_parameters": null,
89+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/notebook",
90+
"source": "WORKSPACE",
91+
"warehouse_id": ""
92+
}
93+
],
94+
"notification_settings": [],
95+
"pipeline_task": [],
96+
"power_bi_task": [],
97+
"python_wheel_task": [],
98+
"retry_on_timeout": false,
99+
"run_if": "",
100+
"run_job_task": [],
101+
"spark_jar_task": [],
102+
"spark_python_task": [],
103+
"spark_submit_task": [],
104+
"sql_task": [],
105+
"task_key": "main",
106+
"timeout_seconds": 0,
107+
"webhook_notifications": []
108+
}
109+
],
110+
"timeout_seconds": 0,
111+
"timeouts": null,
112+
"trigger": [],
113+
"url": "[DATABRICKS_URL]/#job/[NUMID]",
114+
"usage_policy_id": null,
115+
"webhook_notifications": []
116+
},
117+
"sensitive_attributes": [],
118+
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
119+
}
120+
]
121+
},
122+
{
123+
"mode": "managed",
124+
"type": "databricks_pipeline",
125+
"name": "test_pipeline",
126+
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
127+
"instances": [
128+
{
129+
"schema_version": 0,
130+
"attributes": {
131+
"allow_duplicate_names": false,
132+
"budget_policy_id": null,
133+
"catalog": null,
134+
"cause": null,
135+
"channel": "CURRENT",
136+
"cluster": [],
137+
"cluster_id": null,
138+
"configuration": null,
139+
"continuous": false,
140+
"creator_user_name": "[USERNAME]",
141+
"deployment": [
142+
{
143+
"kind": "BUNDLE",
144+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
145+
}
146+
],
147+
"development": false,
148+
"edition": "ADVANCED",
149+
"environment": [],
150+
"event_log": [],
151+
"expected_last_modified": 0,
152+
"filters": [],
153+
"gateway_definition": [],
154+
"health": null,
155+
"id": "[UUID]",
156+
"ingestion_definition": [],
157+
"last_modified": [UNIX_TIME_MILLIS],
158+
"latest_updates": null,
159+
"library": [
160+
{
161+
"file": [],
162+
"glob": [],
163+
"jar": "",
164+
"maven": [],
165+
"notebook": [
166+
{
167+
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline"
168+
}
169+
],
170+
"whl": ""
171+
}
172+
],
173+
"name": "Test Migration Pipeline",
174+
"notification": [],
175+
"photon": false,
176+
"restart_window": [],
177+
"root_path": null,
178+
"run_as": [],
179+
"run_as_user_name": "[USERNAME]",
180+
"schema": null,
181+
"serverless": false,
182+
"state": "IDLE",
183+
"storage": "dbfs:/pipelines/[UUID]",
184+
"tags": {
185+
"myjob_id": "[NUMID]",
186+
"myjob_name": "Test Migration Job",
187+
"myjob_timeout": "0",
188+
"myvolume_id": "mycat.myschema.myvol",
189+
"volume_catalog_name": "mycat",
190+
"volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
191+
},
192+
"target": null,
193+
"timeouts": null,
194+
"trigger": [],
195+
"url": "[DATABRICKS_URL]/#joblist/pipelines/[UUID]"
196+
},
197+
"sensitive_attributes": [],
198+
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19",
199+
"dependencies": [
200+
"databricks_job.test_job",
201+
"databricks_volume.test_volume"
202+
]
203+
}
204+
]
205+
},
206+
{
207+
"mode": "managed",
208+
"type": "databricks_volume",
209+
"name": "test_volume",
210+
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
211+
"instances": [
212+
{
213+
"schema_version": 0,
214+
"attributes": {
215+
"catalog_name": "mycat",
216+
"comment": null,
217+
"id": "mycat.myschema.myvol",
218+
"name": "myvol",
219+
"owner": null,
220+
"schema_name": "myschema",
221+
"storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
222+
"volume_path": "/Volumes/mycat/myschema/myvol",
223+
"volume_type": "MANAGED"
224+
},
225+
"sensitive_attributes": [],
226+
"private": "bnVsbA=="
227+
}
228+
]
229+
}
230+
],
231+
"check_results": null
232+
}

‎acceptance/bundle/migrate/basic/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)