Skip to content

Commit 58479e3

Browse files
shreyas-goenkashreyas-goenka
authored
Add support for experiments in direct deployment (#3669)
## Changes This PR adds support for ML experiments to direct deployment, keeping the behaviour the same as the current Terraform implementation. ## Why To make migration to direct possible and seamless. ## Tests New integration test.
1 parent 5a78875 commit 58479e3

File tree

17 files changed

+540
-3
lines changed

17 files changed

+540
-3
lines changed

‎acceptance/bundle/deploy/experiments/basic/out.plan.direct-exp.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged

‎acceptance/bundle/deploy/experiments/basic/out.plan.terraform.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
update experiments.my_experiment
2+
3+
Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged

‎acceptance/bundle/deploy/experiments/basic/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/deploy/experiments/basic/output.txt‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
2+
=== create the experiment
3+
>>> export EXPERIMENT_ARTIFACT_LOCATION=s3://original-[UNIQUE_NAME]
4+
5+
>>> export EXPERIMENT_NAME_PREFIX=original
6+
7+
>>> envsubst
8+
9+
>>> [CLI] bundle plan
10+
create experiments.my_experiment
11+
12+
Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
13+
14+
>>> [CLI] bundle deploy
15+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/experiment-basic-[UNIQUE_NAME]/default/files...
16+
Deploying resources...
17+
Updating deployment state...
18+
Deployment complete!
19+
20+
>>> [CLI] experiments get-experiment [NUMID]
21+
{
22+
"name": "/Users/[USERNAME]/original-[UNIQUE_NAME]",
23+
"artifact_location": "s3://original-[UNIQUE_NAME]",
24+
"tags": [
25+
{
26+
"key": "k1",
27+
"value": "v1"
28+
},
29+
{
30+
"key": "mlflow.ownerId",
31+
"value": "[USERID]"
32+
},
33+
{
34+
"key": "mlflow.experiment.sourceName",
35+
"value": "/Users/[USERNAME]/original-[UNIQUE_NAME]"
36+
},
37+
{
38+
"key": "mlflow.ownerId",
39+
"value": "[USERID]"
40+
},
41+
{
42+
"key": "mlflow.ownerEmail",
43+
"value": "[USERNAME]"
44+
},
45+
{
46+
"key": "mlflow.experimentType",
47+
"value": "MLFLOW_EXPERIMENT"
48+
}
49+
]
50+
}
51+
52+
=== update the name
53+
>>> export EXPERIMENT_NAME_PREFIX=new-name
54+
55+
>>> envsubst
56+
57+
>>> [CLI] bundle plan
58+
update experiments.my_experiment
59+
60+
Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
61+
62+
>>> [CLI] bundle deploy
63+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/experiment-basic-[UNIQUE_NAME]/default/files...
64+
Deploying resources...
65+
Updating deployment state...
66+
Deployment complete!
67+
68+
>>> [CLI] experiments get-experiment [NUMID]
69+
{
70+
"name": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]",
71+
"artifact_location": "s3://original-[UNIQUE_NAME]",
72+
"tags": [
73+
{
74+
"key": "k1",
75+
"value": "v1"
76+
},
77+
{
78+
"key": "mlflow.ownerId",
79+
"value": "[USERID]"
80+
},
81+
{
82+
"key": "mlflow.experiment.sourceName",
83+
"value": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]"
84+
},
85+
{
86+
"key": "mlflow.ownerId",
87+
"value": "[USERID]"
88+
},
89+
{
90+
"key": "mlflow.ownerEmail",
91+
"value": "[USERNAME]"
92+
},
93+
{
94+
"key": "mlflow.experimentType",
95+
"value": "MLFLOW_EXPERIMENT"
96+
}
97+
]
98+
}
99+
100+
=== updating the artifact location should cause a recreation
101+
>>> export EXPERIMENT_ARTIFACT_LOCATION=s3://new-[UNIQUE_NAME]
102+
103+
>>> envsubst
104+
105+
>>> [CLI] bundle plan
106+
recreate experiments.my_experiment
107+
108+
Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
109+
110+
>>> [CLI] bundle deploy
111+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/experiment-basic-[UNIQUE_NAME]/default/files...
112+
Deploying resources...
113+
Updating deployment state...
114+
Deployment complete!
115+
116+
>>> [CLI] experiments get-experiment [NUMID]
117+
{
118+
"name": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]",
119+
"artifact_location": "s3://new-[UNIQUE_NAME]",
120+
"tags": [
121+
{
122+
"key": "k1",
123+
"value": "v1"
124+
},
125+
{
126+
"key": "mlflow.ownerId",
127+
"value": "[USERID]"
128+
},
129+
{
130+
"key": "mlflow.experiment.sourceName",
131+
"value": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]"
132+
},
133+
{
134+
"key": "mlflow.ownerId",
135+
"value": "[USERID]"
136+
},
137+
{
138+
"key": "mlflow.ownerEmail",
139+
"value": "[USERNAME]"
140+
},
141+
{
142+
"key": "mlflow.experimentType",
143+
"value": "MLFLOW_EXPERIMENT"
144+
}
145+
]
146+
}
147+
148+
=== adding a new tag should be a no-op
149+
>>> envsubst
150+
151+
>>> [CLI] bundle plan
152+
153+
>>> [CLI] bundle deploy
154+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/experiment-basic-[UNIQUE_NAME]/default/files...
155+
Deploying resources...
156+
Updating deployment state...
157+
Deployment complete!
158+
159+
>>> [CLI] experiments get-experiment [NUMID]
160+
{
161+
"name": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]",
162+
"artifact_location": "s3://new-[UNIQUE_NAME]",
163+
"tags": [
164+
{
165+
"key": "k1",
166+
"value": "v1"
167+
},
168+
{
169+
"key": "mlflow.ownerId",
170+
"value": "[USERID]"
171+
},
172+
{
173+
"key": "mlflow.experiment.sourceName",
174+
"value": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]"
175+
},
176+
{
177+
"key": "mlflow.ownerId",
178+
"value": "[USERID]"
179+
},
180+
{
181+
"key": "mlflow.ownerEmail",
182+
"value": "[USERNAME]"
183+
},
184+
{
185+
"key": "mlflow.experimentType",
186+
"value": "MLFLOW_EXPERIMENT"
187+
}
188+
]
189+
}
190+
191+
>>> [CLI] bundle destroy --auto-approve
192+
The following resources will be deleted:
193+
delete experiment my_experiment
194+
195+
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/experiment-basic-[UNIQUE_NAME]/default
196+
197+
Deleting files...
198+
Destroy complete!

‎acceptance/bundle/deploy/experiments/basic/script‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
cleanup() {
2+
trace $CLI bundle destroy --auto-approve
3+
}
4+
5+
trap cleanup EXIT
6+
7+
title "create the experiment"
8+
trace export EXPERIMENT_ARTIFACT_LOCATION="s3://original-${UNIQUE_NAME}"
9+
trace export EXPERIMENT_NAME_PREFIX="original"
10+
trace envsubst < templates/one_tag.tmpl > databricks.yml
11+
trace $CLI bundle plan
12+
trace $CLI bundle deploy
13+
experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id')
14+
trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}'
15+
16+
title "update the name"
17+
trace export EXPERIMENT_NAME_PREFIX="new-name"
18+
trace envsubst < templates/one_tag.tmpl > databricks.yml
19+
trace $CLI bundle plan
20+
trace $CLI bundle deploy
21+
experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id')
22+
trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}'
23+
24+
title "updating the artifact location should cause a recreation"
25+
trace export EXPERIMENT_ARTIFACT_LOCATION="s3://new-${UNIQUE_NAME}"
26+
trace envsubst < templates/one_tag.tmpl > databricks.yml
27+
trace $CLI bundle plan
28+
trace $CLI bundle deploy
29+
experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id')
30+
trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}'
31+
32+
title "adding a new tag should be a no-op"
33+
trace envsubst < templates/two_tag.tmpl > databricks.yml
34+
trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt
35+
trace $CLI bundle deploy
36+
experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id')
37+
trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}'

‎acceptance/bundle/deploy/experiments/basic/templates/one_tag.tmpl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
bundle:
2+
name: experiment-basic-$UNIQUE_NAME
3+
4+
resources:
5+
experiments:
6+
my_experiment:
7+
artifact_location: $EXPERIMENT_ARTIFACT_LOCATION
8+
name: /Users/${CURRENT_USER_NAME}/${EXPERIMENT_NAME_PREFIX}-${UNIQUE_NAME}
9+
tags:
10+
- key: "k1"
11+
value: "v1"

‎acceptance/bundle/deploy/experiments/basic/templates/two_tag.tmpl‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
bundle:
2+
name: experiment-basic-$UNIQUE_NAME
3+
4+
resources:
5+
experiments:
6+
my_experiment:
7+
artifact_location: $EXPERIMENT_ARTIFACT_LOCATION
8+
name: /Users/${CURRENT_USER_NAME}/${EXPERIMENT_NAME_PREFIX}-${UNIQUE_NAME}
9+
tags:
10+
- key: "k1"
11+
value: "v1"
12+
- key: "k2"
13+
value: "v2"

‎acceptance/bundle/deploy/experiments/basic/test.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Cloud = true
2+
Local = true
3+
4+
[[Repls]]
5+
Old = '\d{3,}'
6+
New = "[NUMID]"
7+
8+
# Test both terraform and direct deployment engines
9+
[EnvMatrix]
10+
DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"]

‎acceptance/bundle/deploy/mlops-stacks/out.test.toml‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/deploy/mlops-stacks/test.toml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@ Local=false
33

44
Badness = "the newly initialized bundle from the 'mlops-stacks' template contains two validation warnings in the configuration"
55

6-
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] # uses 'experiments' resource
7-
86
Ignore = [
97
"config.json"
108
]

0 commit comments

Comments
 (0)