Skip to content

Commit 9566b99

Browse files
andrewnesterandrewnester
authored
Moved setting default source for tasks to tfdyn (#3528)
## Changes Moved setting default source for tasks to tfdyn ## Why This prevents defaults from being modified after PyDABs have access to them. This fix is "backend" fixup so it shouldn't be in resource mutators anyway ## Tests Existing acceptance tests pass. It required splitting the output for TF and direct because in direct mode, we don't need to send the source field unless it's explicitly set
1 parent 3fa5519 commit 9566b99

File tree

25 files changed

+460
-315
lines changed

25 files changed

+460
-315
lines changed

‎acceptance/bundle/artifacts/whl_implicit_notebook/output.txt‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,10 @@ Deployment complete!
1111
dist/my_test_code-0.0.1-py3-none-any.whl
1212

1313
=== Expecting 1 wheel in libraries section in /jobs/create
14-
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks out.requests.txt
14+
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries out.requests.txt
1515
[
1616
{
17-
"existing_cluster_id": "0717-aaaaa-bbbbbb",
18-
"libraries": [
19-
{
20-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
21-
}
22-
],
23-
"notebook_task": {
24-
"notebook_path": "/notebook.py",
25-
"source": "WORKSPACE"
26-
},
27-
"task_key": "TestTask"
17+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
2818
}
2919
]
3020

‎acceptance/bundle/artifacts/whl_implicit_notebook/script‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ trace $CLI bundle deploy
33
trace find.py --expect 1 whl
44

55
title "Expecting 1 wheel in libraries section in /jobs/create"
6-
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt
6+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries' out.requests.txt
77

88
title "Expecting 1 wheel to be uploaded"
99
trace jq .path < out.requests.txt | grep import | grep whl | sort

‎acceptance/bundle/deploy/jobs/check-metadata/output.txt‎

Lines changed: 3 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,8 @@
22
=== First summary, should not have id and modified_status should be created
33
>>> [CLI] bundle summary -o json
44
{
5-
"deployment": {
6-
"kind": "BUNDLE",
7-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json"
8-
},
9-
"edit_mode": "UI_LOCKED",
10-
"format": "MULTI_TASK",
11-
"max_concurrent_runs": 1,
12-
"modified_status": "created",
13-
"name": "test-job-metadata-1-[UNIQUE_NAME]",
14-
"permissions": [],
15-
"queue": {
16-
"enabled": true
17-
},
18-
"tasks": [
19-
{
20-
"new_cluster": {
21-
"node_type_id": "[NODE_TYPE_ID]",
22-
"num_workers": 1,
23-
"spark_version": "13.3.x-snapshot-scala2.12"
24-
},
25-
"notebook_task": {
26-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/foo",
27-
"source": "WORKSPACE"
28-
},
29-
"task_key": "my_notebook_task"
30-
}
31-
]
5+
"id": null,
6+
"modified_status": "created"
327
}
338

349
>>> [CLI] bundle deploy
@@ -42,34 +17,8 @@ Deployment complete!
4217

4318
>>> cat out.summary.json
4419
{
45-
"deployment": {
46-
"kind": "BUNDLE",
47-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json"
48-
},
49-
"edit_mode": "UI_LOCKED",
50-
"format": "MULTI_TASK",
5120
"id": "[JOB_1_ID]",
52-
"max_concurrent_runs": 1,
53-
"name": "test-job-metadata-1-[UNIQUE_NAME]",
54-
"permissions": [],
55-
"queue": {
56-
"enabled": true
57-
},
58-
"tasks": [
59-
{
60-
"new_cluster": {
61-
"node_type_id": "[NODE_TYPE_ID]",
62-
"num_workers": 1,
63-
"spark_version": "13.3.x-snapshot-scala2.12"
64-
},
65-
"notebook_task": {
66-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/foo",
67-
"source": "WORKSPACE"
68-
},
69-
"task_key": "my_notebook_task"
70-
}
71-
],
72-
"url": "[DATABRICKS_URL]/jobs/[JOB_1_ID]"
21+
"modified_status": null
7322
}
7423

7524
=== Assert job 1 is created

‎acceptance/bundle/deploy/jobs/check-metadata/script‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ cleanup() {
77
trap cleanup EXIT
88

99
title "First summary, should not have id and modified_status should be created"
10-
trace $CLI bundle summary -o json | jq '.resources.jobs.foo'
10+
trace $CLI bundle summary -o json | jq '.resources.jobs.foo | {id, modified_status}'
1111
trace $CLI bundle deploy
1212

1313
title "Second summary, should have id and modified status should be absent"
14-
trace $CLI bundle summary -o json | jq '.resources.jobs.foo' > out.summary.json
14+
trace $CLI bundle summary -o json | jq '.resources.jobs.foo | {id, modified_status}' > out.summary.json
1515
trace cat out.summary.json
1616
JOB_1_ID=$(jq -r '.id' out.summary.json)
1717
echo "$JOB_1_ID:JOB_1_ID" >> ACC_REPLS

‎acceptance/bundle/deploy/jobs/task-source/databricks.yml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,21 @@
11
bundle:
22
name: task-source
33

4+
git_source: &git_source
5+
git_source:
6+
git_branch: main
7+
git_url: https://github.com/databricks/cli.git
8+
git_provider: github
9+
410
resources:
511
jobs:
612
git_job:
713
name: git_job
8-
git_source:
9-
git_branch: main
10-
git_url: https://github.com/databricks/cli.git
11-
git_provider: github
12-
14+
<<: *git_source
1315
tasks:
1416
- task_key: test_task_no_source
1517
notebook_task:
16-
notebook_path: ./notebook
18+
notebook_path: ./notebook.py
1719
- task_key: test_task_source_workspace
1820
notebook_task:
1921
notebook_path: ./notebook.py

‎acceptance/bundle/deploy/jobs/task-source/out.git_job.direct-exp.txt‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"deployment": {
3+
"kind": "BUNDLE",
4+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/state/metadata.json"
5+
},
6+
"edit_mode": "UI_LOCKED",
7+
"format": "MULTI_TASK",
8+
"git_source": {
9+
"git_branch": "main",
10+
"git_provider": "github",
11+
"git_url": "https://github.com/databricks/cli.git"
12+
},
13+
"max_concurrent_runs": 1,
14+
"name": "git_job",
15+
"queue": {
16+
"enabled": true
17+
},
18+
"tasks": [
19+
{
20+
"for_each_task": {
21+
"inputs": "[1]",
22+
"task": {
23+
"notebook_task": {
24+
"notebook_path": "./notebook.py"
25+
},
26+
"task_key": "test_task_foreach_no_source"
27+
}
28+
},
29+
"task_key": "test_task_foreach_no_source"
30+
},
31+
{
32+
"for_each_task": {
33+
"inputs": "[1]",
34+
"task": {
35+
"notebook_task": {
36+
"notebook_path": "./notebook.py",
37+
"source": "WORKSPACE"
38+
},
39+
"task_key": "test_task_foreach_workspace"
40+
}
41+
},
42+
"task_key": "test_task_foreach_workspace"
43+
},
44+
{
45+
"notebook_task": {
46+
"notebook_path": "./notebook.py"
47+
},
48+
"task_key": "test_task_no_source"
49+
},
50+
{
51+
"notebook_task": {
52+
"notebook_path": "./notebook.py",
53+
"source": "WORKSPACE"
54+
},
55+
"task_key": "test_task_source_workspace"
56+
}
57+
]
58+
}
59+
{
60+
"job_id": [NUMID],
61+
"new_settings": {
62+
"deployment": {
63+
"kind": "BUNDLE",
64+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/state/metadata.json"
65+
},
66+
"edit_mode": "UI_LOCKED",
67+
"format": "MULTI_TASK",
68+
"max_concurrent_runs": 1,
69+
"name": "git_job",
70+
"queue": {
71+
"enabled": true
72+
},
73+
"tasks": [
74+
{
75+
"for_each_task": {
76+
"inputs": "[1]",
77+
"task": {
78+
"notebook_task": {
79+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook"
80+
},
81+
"task_key": "test_task_foreach_no_source"
82+
}
83+
},
84+
"task_key": "test_task_foreach_no_source"
85+
},
86+
{
87+
"for_each_task": {
88+
"inputs": "[1]",
89+
"task": {
90+
"notebook_task": {
91+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
92+
"source": "WORKSPACE"
93+
},
94+
"task_key": "test_task_foreach_workspace"
95+
}
96+
},
97+
"task_key": "test_task_foreach_workspace"
98+
},
99+
{
100+
"notebook_task": {
101+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook"
102+
},
103+
"task_key": "test_task_no_source"
104+
},
105+
{
106+
"notebook_task": {
107+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
108+
"source": "WORKSPACE"
109+
},
110+
"task_key": "test_task_source_workspace"
111+
}
112+
]
113+
}
114+
}

‎acceptance/bundle/deploy/jobs/task-source/out.git_job.terraform.txt‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
{
2+
"deployment": {
3+
"kind": "BUNDLE",
4+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/state/metadata.json"
5+
},
6+
"edit_mode": "UI_LOCKED",
7+
"format": "MULTI_TASK",
8+
"git_source": {
9+
"git_branch": "main",
10+
"git_provider": "github",
11+
"git_url": "https://github.com/databricks/cli.git"
12+
},
13+
"max_concurrent_runs": 1,
14+
"name": "git_job",
15+
"queue": {
16+
"enabled": true
17+
},
18+
"tasks": [
19+
{
20+
"for_each_task": {
21+
"inputs": "[1]",
22+
"task": {
23+
"notebook_task": {
24+
"notebook_path": "./notebook.py",
25+
"source": "GIT"
26+
},
27+
"task_key": "test_task_foreach_no_source"
28+
}
29+
},
30+
"task_key": "test_task_foreach_no_source"
31+
},
32+
{
33+
"for_each_task": {
34+
"inputs": "[1]",
35+
"task": {
36+
"notebook_task": {
37+
"notebook_path": "./notebook.py",
38+
"source": "WORKSPACE"
39+
},
40+
"task_key": "test_task_foreach_workspace"
41+
}
42+
},
43+
"task_key": "test_task_foreach_workspace"
44+
},
45+
{
46+
"notebook_task": {
47+
"notebook_path": "./notebook.py",
48+
"source": "GIT"
49+
},
50+
"task_key": "test_task_no_source"
51+
},
52+
{
53+
"notebook_task": {
54+
"notebook_path": "./notebook.py",
55+
"source": "WORKSPACE"
56+
},
57+
"task_key": "test_task_source_workspace"
58+
}
59+
]
60+
}
61+
{
62+
"job_id": [NUMID],
63+
"new_settings": {
64+
"deployment": {
65+
"kind": "BUNDLE",
66+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/state/metadata.json"
67+
},
68+
"edit_mode": "UI_LOCKED",
69+
"format": "MULTI_TASK",
70+
"max_concurrent_runs": 1,
71+
"name": "git_job",
72+
"queue": {
73+
"enabled": true
74+
},
75+
"tasks": [
76+
{
77+
"for_each_task": {
78+
"inputs": "[1]",
79+
"task": {
80+
"notebook_task": {
81+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
82+
"source": "WORKSPACE"
83+
},
84+
"task_key": "test_task_foreach_no_source"
85+
}
86+
},
87+
"task_key": "test_task_foreach_no_source"
88+
},
89+
{
90+
"for_each_task": {
91+
"inputs": "[1]",
92+
"task": {
93+
"notebook_task": {
94+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
95+
"source": "WORKSPACE"
96+
},
97+
"task_key": "test_task_foreach_workspace"
98+
}
99+
},
100+
"task_key": "test_task_foreach_workspace"
101+
},
102+
{
103+
"notebook_task": {
104+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
105+
"source": "WORKSPACE"
106+
},
107+
"task_key": "test_task_no_source"
108+
},
109+
{
110+
"notebook_task": {
111+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
112+
"source": "WORKSPACE"
113+
},
114+
"task_key": "test_task_source_workspace"
115+
}
116+
]
117+
}
118+
}

0 commit comments

Comments
 (0)