Skip to content

Commit bf3356b

Browse files
denikdenik
authored
direct: Detect remote changes (#3666)
## Changes Calculate diff of snapshot state with remote state and calculate action based on that. ## Why Remote drift detection for direct. ## Tests New acceptance tests.
1 parent 0f6ea36 commit bf3356b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1270
-149
lines changed

‎acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct-exp.json‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,76 @@
7070
]
7171
}
7272
},
73+
"remote_state": {
74+
"creator_user_name": "[USERNAME]",
75+
"job_id": [NUMID],
76+
"settings": {
77+
"deployment": {
78+
"kind": "BUNDLE",
79+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
80+
},
81+
"edit_mode": "UI_LOCKED",
82+
"environments": [
83+
{
84+
"environment_key": "test_env",
85+
"spec": {
86+
"client": "1",
87+
"dependencies": [
88+
"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
89+
]
90+
}
91+
}
92+
],
93+
"format": "MULTI_TASK",
94+
"max_concurrent_runs": 1,
95+
"name": "[default] My Wheel Job",
96+
"queue": {
97+
"enabled": true
98+
},
99+
"tasks": [
100+
{
101+
"environment_key": "test_env",
102+
"python_wheel_task": {
103+
"entry_point": "run",
104+
"package_name": "my_test_code"
105+
},
106+
"task_key": "ServerlessTestTask"
107+
},
108+
{
109+
"existing_cluster_id": "0717-132531-5opeqon1",
110+
"for_each_task": {
111+
"inputs": "[1]",
112+
"task": {
113+
"existing_cluster_id": "0717-132531-5opeqon1",
114+
"libraries": [
115+
{
116+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
117+
}
118+
],
119+
"python_wheel_task": {
120+
"entry_point": "run",
121+
"package_name": "my_test_code"
122+
},
123+
"task_key": "SubTask"
124+
}
125+
},
126+
"libraries": [
127+
{
128+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
129+
},
130+
{
131+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
132+
}
133+
],
134+
"python_wheel_task": {
135+
"entry_point": "run",
136+
"package_name": "my_test_code"
137+
},
138+
"task_key": "TestTask"
139+
}
140+
]
141+
}
142+
},
73143
"changes": {
74144
"local": {
75145
"environments[0].spec.dependencies[0]": {

‎acceptance/bundle/deploy/models/basic/out.plan_update_description.direct-exp.json‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎acceptance/bundle/deploy/models/basic/out.plan_update_description.terraform.json‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎acceptance/bundle/deploy/models/basic/out.plan_update_name.direct-exp.json‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎acceptance/bundle/deploy/models/basic/out.plan_update_name.terraform.json‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎acceptance/bundle/deploy/models/basic/out.plan_update_tag.direct-exp.json‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎acceptance/bundle/deploy/models/basic/out.plan_update_tag.terraform.json‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎acceptance/bundle/deploy/models/basic/script‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,18 @@ trace export MODEL_DESCRIPTION=new-description-$UNIQUE_NAME
1515
envsubst < templates/one_tag.tmpl > databricks.yml
1616
title "update the description, this should update the description remotely as well"
1717
trace $CLI bundle plan
18-
$CLI bundle debug plan > out.plan_update_description.$DATABRICKS_BUNDLE_ENGINE.json
1918
trace $CLI bundle deploy
2019
trace $CLI model-registry get-model $MODEL_NAME | jq '.registered_model_databricks | {name, description, tags}'
2120

2221
trace export MODEL_NAME=new-name-$UNIQUE_NAME
2322
envsubst < templates/one_tag.tmpl > databricks.yml
2423
title "update the name, this should recreate the model with the new name"
2524
trace $CLI bundle plan
26-
$CLI bundle debug plan > out.plan_update_name.$DATABRICKS_BUNDLE_ENGINE.json
2725
trace $CLI bundle deploy
2826
trace $CLI model-registry get-model $MODEL_NAME | jq '.registered_model_databricks | {name, description, tags}'
2927

3028
title "add a new tag, this should be a no-op; terraform does make an update request but since update request has no tags in it, it does nothing; direct skips request completely"
3129
envsubst < templates/two_tag.tmpl > databricks.yml
3230
trace $CLI bundle plan > out.plan_update_tag.$DATABRICKS_BUNDLE_ENGINE.txt
33-
$CLI bundle debug plan > out.plan_update_tag.$DATABRICKS_BUNDLE_ENGINE.json
3431
trace $CLI bundle deploy
3532
trace $CLI model-registry get-model $MODEL_NAME | jq '.registered_model_databricks | {name, description, tags}'

‎acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct-exp.json‎

Lines changed: 90 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,24 @@
11
{
22
"plan": {
33
"resources.jobs.a": {
4-
"action": "skip"
4+
"action": "skip",
5+
"remote_state": {
6+
"job_id": [NUMID],
7+
"settings": {
8+
"deployment": {
9+
"kind": "BUNDLE",
10+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
11+
},
12+
"description": "aa_new_desc",
13+
"edit_mode": "UI_LOCKED",
14+
"format": "MULTI_TASK",
15+
"max_concurrent_runs": 1,
16+
"name": "aa",
17+
"queue": {
18+
"enabled": true
19+
}
20+
}
21+
}
522
},
623
"resources.jobs.b": {
724
"depends_on": [
@@ -10,7 +27,24 @@
1027
"label": "${resources.jobs.a.id}"
1128
}
1229
],
13-
"action": "skip"
30+
"action": "skip",
31+
"remote_state": {
32+
"job_id": [NUMID],
33+
"settings": {
34+
"deployment": {
35+
"kind": "BUNDLE",
36+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
37+
},
38+
"description": "new_prefix [NUMID]",
39+
"edit_mode": "UI_LOCKED",
40+
"format": "MULTI_TASK",
41+
"max_concurrent_runs": 1,
42+
"name": "bb",
43+
"queue": {
44+
"enabled": true
45+
}
46+
}
47+
}
1448
},
1549
"resources.jobs.c": {
1650
"depends_on": [
@@ -19,7 +53,24 @@
1953
"label": "${resources.jobs.b.id}"
2054
}
2155
],
22-
"action": "skip"
56+
"action": "skip",
57+
"remote_state": {
58+
"job_id": [NUMID],
59+
"settings": {
60+
"deployment": {
61+
"kind": "BUNDLE",
62+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
63+
},
64+
"description": "new_prefix [NUMID]",
65+
"edit_mode": "UI_LOCKED",
66+
"format": "MULTI_TASK",
67+
"max_concurrent_runs": 1,
68+
"name": "cc",
69+
"queue": {
70+
"enabled": true
71+
}
72+
}
73+
}
2374
},
2475
"resources.jobs.d": {
2576
"depends_on": [
@@ -28,7 +79,24 @@
2879
"label": "${resources.jobs.c.id}"
2980
}
3081
],
31-
"action": "skip"
82+
"action": "skip",
83+
"remote_state": {
84+
"job_id": [NUMID],
85+
"settings": {
86+
"deployment": {
87+
"kind": "BUNDLE",
88+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
89+
},
90+
"description": "new_prefix [NUMID]",
91+
"edit_mode": "UI_LOCKED",
92+
"format": "MULTI_TASK",
93+
"max_concurrent_runs": 1,
94+
"name": "dd",
95+
"queue": {
96+
"enabled": true
97+
}
98+
}
99+
}
32100
},
33101
"resources.jobs.e": {
34102
"depends_on": [
@@ -37,7 +105,24 @@
37105
"label": "${resources.jobs.d.id}"
38106
}
39107
],
40-
"action": "skip"
108+
"action": "skip",
109+
"remote_state": {
110+
"job_id": [NUMID],
111+
"settings": {
112+
"deployment": {
113+
"kind": "BUNDLE",
114+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
115+
},
116+
"description": "new_prefix [NUMID]",
117+
"edit_mode": "UI_LOCKED",
118+
"format": "MULTI_TASK",
119+
"max_concurrent_runs": 1,
120+
"name": "ee",
121+
"queue": {
122+
"enabled": true
123+
}
124+
}
125+
}
41126
}
42127
}
43128
}

0 commit comments

Comments
 (0)