Skip to content

Commit b4f3df3

Browse files
denikdenik
authored
Fix migrate to store expanded globs in the new state (#4068)
Since migrate includes phases.Build but not phases.DeployPrepare, it did not process globs. I moved globs processing to phases.Build, they are logically related to artifact building.
1 parent aab9b14 commit b4f3df3

File tree

17 files changed

+1515
-53
lines changed

17 files changed

+1515
-53
lines changed

‎acceptance/bundle/migrate/default-python/databricks.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
bundle:
2+
name: migrate-basic-test
3+
4+
resources:
5+
jobs:
6+
test_job:
7+
name: "Test Migration Job"
8+
tasks:
9+
- task_key: "main"
10+
notebook_task:
11+
notebook_path: "./notebook.py"
12+
volumes:
13+
test_volume:
14+
catalog_name: "mycat"
15+
schema_name: "myschema"
16+
name: "myvol"
17+
18+
pipelines:
19+
test_pipeline:
20+
name: "Test Migration Pipeline"
21+
tags:
22+
# ids
23+
myjob_id: ${resources.jobs.test_job.id}
24+
myvolume_id: ${resources.volumes.test_volume.id}
25+
26+
# local field, string:
27+
myjob_name: ${resources.jobs.test_job.name}
28+
volume_catalog_name: ${resources.volumes.test_volume.catalog_name}
29+
30+
# remote field, int, null
31+
myjob_timeout: ${resources.jobs.test_job.timeout_seconds}
32+
33+
# remote field, string:
34+
volume_storage_location: ${resources.volumes.test_volume.storage_location}
35+
libraries:
36+
- notebook:
37+
path: "./pipeline.py"
38+
39+
targets:
40+
dev:
41+
default: true
42+
prod:
43+
resources:
44+
schemas:
45+
test_schema:
46+
catalog_name: mycat
47+
name: myschema

‎acceptance/bundle/migrate/default-python/input.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"project_name": "my_default_python",
3+
"include_notebook": "yes",
4+
"include_dlt": "yes",
5+
"include_python": "yes",
6+
"serverless": "no"
7+
}

‎acceptance/bundle/migrate/default-python/notebook.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Databricks notebook source
2+
print("Hello from test migration job")

‎acceptance/bundle/migrate/default-python/out.plan_after_deploy.json‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
{
2+
"plan": {
3+
"resources.jobs.sample_job": {
4+
"depends_on": [
5+
{
6+
"node": "resources.pipelines.my_default_python_etl",
7+
"label": "${resources.pipelines.my_default_python_etl.id}"
8+
}
9+
],
10+
"action": "update",
11+
"new_state": {
12+
"value": {
13+
"deployment": {
14+
"kind": "BUNDLE",
15+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
16+
},
17+
"edit_mode": "UI_LOCKED",
18+
"format": "MULTI_TASK",
19+
"job_clusters": [
20+
{
21+
"job_cluster_key": "job_cluster",
22+
"new_cluster": {
23+
"autoscale": {
24+
"max_workers": 4,
25+
"min_workers": 1
26+
},
27+
"data_security_mode": "SINGLE_USER",
28+
"node_type_id": "[NODE_TYPE_ID]",
29+
"num_workers": 0,
30+
"spark_version": "16.4.x-scala2.12"
31+
}
32+
}
33+
],
34+
"max_concurrent_runs": 4,
35+
"name": "[dev [USERNAME]] sample_job",
36+
"parameters": [
37+
{
38+
"default": "hive_metastore",
39+
"name": "catalog"
40+
},
41+
{
42+
"default": "[USERNAME]",
43+
"name": "schema"
44+
}
45+
],
46+
"queue": {
47+
"enabled": true
48+
},
49+
"tags": {
50+
"dev": "[USERNAME]"
51+
},
52+
"tasks": [
53+
{
54+
"job_cluster_key": "job_cluster",
55+
"libraries": [
56+
{
57+
"whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
58+
}
59+
],
60+
"notebook_task": {
61+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook"
62+
},
63+
"task_key": "notebook_task"
64+
},
65+
{
66+
"depends_on": [
67+
{
68+
"task_key": "notebook_task"
69+
}
70+
],
71+
"job_cluster_key": "job_cluster",
72+
"libraries": [
73+
{
74+
"whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
75+
}
76+
],
77+
"python_wheel_task": {
78+
"entry_point": "main",
79+
"package_name": "my_default_python",
80+
"parameters": [
81+
"--catalog",
82+
"hive_metastore",
83+
"--schema",
84+
"[USERNAME]"
85+
]
86+
},
87+
"task_key": "python_wheel_task"
88+
},
89+
{
90+
"depends_on": [
91+
{
92+
"task_key": "notebook_task"
93+
}
94+
],
95+
"pipeline_task": {
96+
"pipeline_id": "[UUID]"
97+
},
98+
"task_key": "refresh_pipeline"
99+
}
100+
],
101+
"trigger": {
102+
"pause_status": "PAUSED",
103+
"periodic": {
104+
"interval": 1,
105+
"unit": "DAYS"
106+
}
107+
}
108+
}
109+
},
110+
"remote_state": {
111+
"created_time": [UNIX_TIME_MILLIS][0],
112+
"creator_user_name": "[USERNAME]",
113+
"job_id": [NUMID],
114+
"run_as_user_name": "[USERNAME]",
115+
"settings": {
116+
"deployment": {
117+
"kind": "BUNDLE",
118+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
119+
},
120+
"edit_mode": "UI_LOCKED",
121+
"email_notifications": {},
122+
"format": "MULTI_TASK",
123+
"job_clusters": [
124+
{
125+
"job_cluster_key": "job_cluster",
126+
"new_cluster": {
127+
"autoscale": {
128+
"max_workers": 4,
129+
"min_workers": 1
130+
},
131+
"data_security_mode": "SINGLE_USER",
132+
"node_type_id": "[NODE_TYPE_ID]",
133+
"num_workers": 0,
134+
"spark_version": "16.4.x-scala2.12"
135+
}
136+
}
137+
],
138+
"max_concurrent_runs": 4,
139+
"name": "[dev [USERNAME]] sample_job",
140+
"parameters": [
141+
{
142+
"default": "hive_metastore",
143+
"name": "catalog"
144+
},
145+
{
146+
"default": "[USERNAME]",
147+
"name": "schema"
148+
}
149+
],
150+
"queue": {
151+
"enabled": true
152+
},
153+
"tags": {
154+
"dev": "[USERNAME]"
155+
},
156+
"tasks": [
157+
{
158+
"job_cluster_key": "job_cluster",
159+
"libraries": [
160+
{
161+
"whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
162+
}
163+
],
164+
"notebook_task": {
165+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook",
166+
"source": "WORKSPACE"
167+
},
168+
"task_key": "notebook_task"
169+
},
170+
{
171+
"depends_on": [
172+
{
173+
"task_key": "notebook_task"
174+
}
175+
],
176+
"job_cluster_key": "job_cluster",
177+
"libraries": [
178+
{
179+
"whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
180+
}
181+
],
182+
"python_wheel_task": {
183+
"entry_point": "main",
184+
"package_name": "my_default_python",
185+
"parameters": [
186+
"--catalog",
187+
"hive_metastore",
188+
"--schema",
189+
"[USERNAME]"
190+
]
191+
},
192+
"task_key": "python_wheel_task"
193+
},
194+
{
195+
"depends_on": [
196+
{
197+
"task_key": "notebook_task"
198+
}
199+
],
200+
"pipeline_task": {
201+
"pipeline_id": "[UUID]"
202+
},
203+
"task_key": "refresh_pipeline"
204+
}
205+
],
206+
"timeout_seconds": 0,
207+
"trigger": {
208+
"pause_status": "PAUSED",
209+
"periodic": {
210+
"interval": 1,
211+
"unit": "DAYS"
212+
}
213+
},
214+
"webhook_notifications": {}
215+
}
216+
},
217+
"changes": {
218+
"local": {
219+
"tasks[task_key='notebook_task'].libraries[0].whl": {
220+
"action": "update",
221+
"old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
222+
"new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
223+
},
224+
"tasks[task_key='python_wheel_task'].libraries[0].whl": {
225+
"action": "update",
226+
"old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
227+
"new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
228+
}
229+
},
230+
"remote": {
231+
"email_notifications": {
232+
"action": "skip",
233+
"reason": "server_side_default"
234+
},
235+
"tasks[task_key='notebook_task'].notebook_task.source": {
236+
"action": "skip",
237+
"reason": "server_side_default"
238+
},
239+
"timeout_seconds": {
240+
"action": "skip",
241+
"reason": "server_side_default"
242+
},
243+
"webhook_notifications": {
244+
"action": "skip",
245+
"reason": "server_side_default"
246+
}
247+
}
248+
}
249+
},
250+
"resources.pipelines.my_default_python_etl": {
251+
"action": "skip",
252+
"remote_state": {
253+
"creator_user_name": "[USERNAME]",
254+
"last_modified": [UNIX_TIME_MILLIS][1],
255+
"name": "[dev [USERNAME]] my_default_python_etl",
256+
"pipeline_id": "[UUID]",
257+
"run_as_user_name": "[USERNAME]",
258+
"spec": {
259+
"channel": "CURRENT",
260+
"deployment": {
261+
"kind": "BUNDLE",
262+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
263+
},
264+
"development": true,
265+
"edition": "ADVANCED",
266+
"environment": {
267+
"dependencies": [
268+
"--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files"
269+
]
270+
},
271+
"id": "[UUID]",
272+
"libraries": [
273+
{
274+
"glob": {
275+
"include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**"
276+
}
277+
}
278+
],
279+
"name": "[dev [USERNAME]] my_default_python_etl",
280+
"root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl",
281+
"schema": "[USERNAME]",
282+
"storage": "dbfs:/pipelines/[UUID]",
283+
"tags": {
284+
"dev": "[USERNAME]"
285+
}
286+
},
287+
"state": "IDLE"
288+
},
289+
"changes": {
290+
"remote": {
291+
"storage": {
292+
"action": "skip",
293+
"reason": "server_side_default"
294+
}
295+
}
296+
}
297+
}
298+
}
299+
}

0 commit comments

Comments
 (0)