Skip to content

Commit 2414730

Browse files
andrewnesterandrewnester
authored
Always set default value for source field for tasks if not explicitly set already (#3359)
## Changes If the job has `git_source` set, DABs will set `source: GIT` to all the tasks where source is not already set If the job has no `git_source` set, DABs will set `source: WORKSPACE` to all the tasks where source is not already set We only do this for a subset of task types which support source field as per https://docs.databricks.com/api/workspace/jobs/create ## Why Fixes #2561 ## Tests Added an acceptance test <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->
1 parent cbd391f commit 2414730

File tree

22 files changed

+327
-38
lines changed

22 files changed

+327
-38
lines changed

‎NEXT_CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
* Show "DEFAULT" as the default profile for `databricks auth login` [#3252](https://github.com/databricks/cli/pull/3252)
1111

1212
### Bundles
13+
* Always set default value for source field for tasks if not explicitly set already ([#3359](https://github.com/databricks/cli/pull/3359))
1314
* Fix panic in auto completion handler for bundler run/open. Regression introduced in 0.259.0. [#3358](https://github.com/databricks/cli/pull/3358)
1415
* Fix error propagation in the `bundle generate dashboard` command. Regression introduced in 0.259.0. [#3354](https://github.com/databricks/cli/pull/3354)
1516

‎acceptance/bundle/artifacts/whl_implicit_notebook/output.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ dist/my_test_code-0.0.1-py3-none-any.whl
2121
}
2222
],
2323
"notebook_task": {
24-
"notebook_path": "/notebook.py"
24+
"notebook_path": "/notebook.py",
25+
"source": "WORKSPACE"
2526
},
2627
"task_key": "TestTask"
2728
}

‎acceptance/bundle/deploy/experimental-python/output.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ Deployment complete!
2424
"tasks": [
2525
{
2626
"notebook_task": {
27-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_notebook"
27+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_notebook",
28+
"source": "WORKSPACE"
2829
},
2930
"task_key": "my_notebook"
3031
}

‎acceptance/bundle/deploy/jobs/check-metadata/output.txt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
"spark_version": "13.3.x-snapshot-scala2.12"
2424
},
2525
"notebook_task": {
26-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/foo"
26+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/foo",
27+
"source": "WORKSPACE"
2728
},
2829
"task_key": "my_notebook_task"
2930
}
@@ -62,7 +63,8 @@ Deployment complete!
6263
"spark_version": "13.3.x-snapshot-scala2.12"
6364
},
6465
"notebook_task": {
65-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/foo"
66+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/foo",
67+
"source": "WORKSPACE"
6668
},
6769
"task_key": "my_notebook_task"
6870
}

‎acceptance/bundle/deploy/jobs/task-source/databricks.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
bundle:
2+
name: task-source
3+
4+
resources:
5+
jobs:
6+
git_job:
7+
name: git_job
8+
git_source:
9+
git_branch: main
10+
git_url: https://github.com/databricks/cli.git
11+
git_provider: github
12+
13+
tasks:
14+
- task_key: test_task_no_source
15+
notebook_task:
16+
notebook_path: ./notebook
17+
- task_key: test_task_source_workspace
18+
notebook_task:
19+
notebook_path: ./notebook.py
20+
source: WORKSPACE
21+
- task_key: test_task_foreach_no_source
22+
for_each_task:
23+
inputs: "[1]"
24+
task:
25+
task_key: test_task_foreach_no_source
26+
notebook_task:
27+
notebook_path: ./notebook.py
28+
- task_key: test_task_foreach_workspace
29+
for_each_task:
30+
inputs: "[1]"
31+
task:
32+
task_key: test_task_foreach_workspace
33+
notebook_task:
34+
notebook_path: ./notebook.py
35+
source: WORKSPACE
36+
37+
workspace_job:
38+
name: workspace_job
39+
tasks:
40+
- task_key: test_task_no_source
41+
notebook_task:
42+
notebook_path: ./notebook.py
43+
- task_key: test_task_foreach
44+
for_each_task:
45+
inputs: "[1]"
46+
task:
47+
task_key: test_task_foreach_no_source
48+
notebook_task:
49+
notebook_path: ./notebook.py

‎acceptance/bundle/deploy/jobs/task-source/notebook.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Databricks notebook source
2+
# %%
3+
print("Hello, World!")
4+
5+
# %%

‎acceptance/bundle/deploy/jobs/task-source/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Local = true
2+
Cloud = false
3+
4+
[EnvMatrix]
5+
DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

‎acceptance/bundle/deploy/jobs/task-source/output.txt‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
2+
>>> [CLI] bundle deploy
3+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/task-source/default/files...
4+
Deploying resources...
5+
Updating deployment state...
6+
Deployment complete!
7+
8+
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body out.requests.txt
9+
{
10+
"deployment": {
11+
"kind": "BUNDLE",
12+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/state/metadata.json"
13+
},
14+
"edit_mode": "UI_LOCKED",
15+
"format": "MULTI_TASK",
16+
"git_source": {
17+
"git_branch": "main",
18+
"git_provider": "github",
19+
"git_url": "https://github.com/databricks/cli.git"
20+
},
21+
"max_concurrent_runs": 1,
22+
"name": "git_job",
23+
"queue": {
24+
"enabled": true
25+
},
26+
"tasks": [
27+
{
28+
"for_each_task": {
29+
"inputs": "[1]",
30+
"task": {
31+
"notebook_task": {
32+
"notebook_path": "./notebook.py",
33+
"source": "GIT"
34+
},
35+
"task_key": "test_task_foreach_no_source"
36+
}
37+
},
38+
"task_key": "test_task_foreach_no_source"
39+
},
40+
{
41+
"for_each_task": {
42+
"inputs": "[1]",
43+
"task": {
44+
"notebook_task": {
45+
"notebook_path": "./notebook.py",
46+
"source": "WORKSPACE"
47+
},
48+
"task_key": "test_task_foreach_workspace"
49+
}
50+
},
51+
"task_key": "test_task_foreach_workspace"
52+
},
53+
{
54+
"notebook_task": {
55+
"notebook_path": "./notebook",
56+
"source": "GIT"
57+
},
58+
"task_key": "test_task_no_source"
59+
},
60+
{
61+
"notebook_task": {
62+
"notebook_path": "./notebook.py",
63+
"source": "WORKSPACE"
64+
},
65+
"task_key": "test_task_source_workspace"
66+
}
67+
]
68+
}
69+
70+
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body out.requests.txt
71+
{
72+
"deployment": {
73+
"kind": "BUNDLE",
74+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/state/metadata.json"
75+
},
76+
"edit_mode": "UI_LOCKED",
77+
"format": "MULTI_TASK",
78+
"max_concurrent_runs": 1,
79+
"name": "workspace_job",
80+
"queue": {
81+
"enabled": true
82+
},
83+
"tasks": [
84+
{
85+
"for_each_task": {
86+
"inputs": "[1]",
87+
"task": {
88+
"notebook_task": {
89+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
90+
"source": "WORKSPACE"
91+
},
92+
"task_key": "test_task_foreach_no_source"
93+
}
94+
},
95+
"task_key": "test_task_foreach"
96+
},
97+
{
98+
"notebook_task": {
99+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/task-source/default/files/notebook",
100+
"source": "WORKSPACE"
101+
},
102+
"task_key": "test_task_no_source"
103+
}
104+
]
105+
}

‎acceptance/bundle/deploy/jobs/task-source/script‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
trace $CLI bundle deploy
2+
3+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body' out.requests.txt | jq --sort-keys
4+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body' out.requests.txt | jq --sort-keys
5+
6+
rm out.requests.txt

‎acceptance/bundle/deploy/jobs/task-source/test.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Cloud = false
2+
Local = true
3+
4+
RecordRequests = true

0 commit comments

Comments
 (0)