Skip to content

Commit 5053e0e

Browse files
andrewnesterandrewnester
authored
acc: Convert artifacts_test and upload_test into acceptance tests (#3477)
## Changes Convert artifacts_test and upload_test into acceptance tests ## Why Needed for #3442 <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->
1 parent 8689a77 commit 5053e0e

File tree

22 files changed

+559
-554
lines changed

22 files changed

+559
-554
lines changed

‎acceptance/bundle/artifacts/artifact_upload_for_volumes/databricks.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
workspace:
2+
artifact_path: /Volumes/foo/bar/artifacts
3+
4+
resources:
5+
jobs:
6+
job:
7+
name: "[${bundle.target}] Test Job for Volumes"
8+
tasks:
9+
- task_key: TestTask1
10+
existing_cluster_id: "0717-132531-5opeqon1"
11+
python_wheel_task:
12+
package_name: "whl"
13+
entry_point: "run"
14+
libraries:
15+
- whl: whl/*.whl
16+
- whl: /Volumes/some/path/mywheel.whl
17+
- task_key: TestTask2
18+
for_each_task:
19+
inputs: "[1]"
20+
task:
21+
task_key: TestTask2
22+
existing_cluster_id: "0717-132531-5opeqon1"
23+
python_wheel_task:
24+
package_name: "whl"
25+
entry_point: "run"
26+
libraries:
27+
- whl: whl/*.whl
28+
- whl: /Volumes/some/path/mywheel.whl
29+
environments:
30+
- environment_key: "test_env"
31+
spec:
32+
client: "1"
33+
dependencies:
34+
- whl/*.whl
35+
- /Volumes/some/path/mywheel.whl

‎acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Local = true
2+
Cloud = false
3+
4+
[EnvMatrix]
5+
DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

‎acceptance/bundle/artifacts/artifact_upload_for_volumes/output.txt‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
2+
>>> [CLI] bundle deploy
3+
Uploading whl/source.whl...
4+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
5+
Deploying resources...
6+
Updating deployment state...
7+
Deployment complete!
8+
9+
=== Expecting 2 wheels in libraries section in /jobs/create
10+
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks out.requests.txt
11+
[
12+
{
13+
"existing_cluster_id": "0717-132531-5opeqon1",
14+
"libraries": [
15+
{
16+
"whl": "/Volumes/foo/bar/artifacts/.internal/source.whl"
17+
},
18+
{
19+
"whl": "/Volumes/some/path/mywheel.whl"
20+
}
21+
],
22+
"python_wheel_task": {
23+
"entry_point": "run",
24+
"package_name": "whl"
25+
},
26+
"task_key": "TestTask1"
27+
},
28+
{
29+
"for_each_task": {
30+
"inputs": "[1]",
31+
"task": {
32+
"existing_cluster_id": "0717-132531-5opeqon1",
33+
"libraries": [
34+
{
35+
"whl": "/Volumes/foo/bar/artifacts/.internal/source.whl"
36+
},
37+
{
38+
"whl": "/Volumes/some/path/mywheel.whl"
39+
}
40+
],
41+
"python_wheel_task": {
42+
"entry_point": "run",
43+
"package_name": "whl"
44+
},
45+
"task_key": "TestTask2"
46+
}
47+
},
48+
"task_key": "TestTask2"
49+
}
50+
]
51+
52+
=== Expecting environment dependencies to be updated
53+
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt
54+
[
55+
{
56+
"environment_key": "test_env",
57+
"spec": {
58+
"client": "1",
59+
"dependencies": [
60+
"/Volumes/foo/bar/artifacts/.internal/source.whl",
61+
"/Volumes/some/path/mywheel.whl"
62+
]
63+
}
64+
}
65+
]
66+
67+
=== Expecting wheel to be uploaded
68+
>>> jq -s .[] | select(.path=="/api/2.0/fs/files/Volumes/foo/bar/artifacts/.internal/source.whl") out.requests.txt
69+
{
70+
"method": "PUT",
71+
"path": "/api/2.0/fs/files/Volumes/foo/bar/artifacts/.internal/source.whl",
72+
"raw_body": "test wheel content\n"
73+
}

‎acceptance/bundle/artifacts/artifact_upload_for_volumes/script‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Create test wheel files
2+
mkdir -p whl
3+
echo "test wheel content" > whl/source.whl
4+
5+
trace $CLI bundle deploy
6+
7+
title "Expecting 2 wheels in libraries section in /jobs/create"
8+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt
9+
10+
title "Expecting environment dependencies to be updated"
11+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt
12+
13+
title "Expecting wheel to be uploaded"
14+
trace jq -s '.[] | select(.path=="/api/2.0/fs/files/Volumes/foo/bar/artifacts/.internal/source.whl")' out.requests.txt
15+
16+
rm out.requests.txt

‎acceptance/bundle/artifacts/artifact_upload_for_volumes/test.toml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
RecordRequests = true
2+
Ignore = [
3+
'.venv',
4+
'dist',
5+
'build',
6+
'*egg-info',
7+
'.databricks',
8+
'*.whl',
9+
]
10+
11+
[[Server]]
12+
Pattern = "GET /api/2.1/clusters/get"
13+
Response.Body = '''
14+
{
15+
"cluster_id": "0717-132531-5opeqon1",
16+
"spark_version": "13.3.x-scala2.12"
17+
}
18+
'''
19+
20+
[[Server]]
21+
Pattern = "GET /api/2.1/unity-catalog/volumes/foo.bar.artifacts"
22+
Response.Body = '''
23+
{
24+
"catalog_name": "foo",
25+
"schema_name": "bar",
26+
"name": "artifacts",
27+
"volume_type": "MANAGED",
28+
"full_name": "foo.bar.artifacts"
29+
}
30+
'''
31+
32+
[[Server]]
33+
Pattern = "HEAD /api/2.0/fs/directories/Volumes/foo/bar/artifacts/.internal"
34+
Response.StatusCode = 404
35+
36+
[[Server]]
37+
Pattern = "HEAD /api/2.0/fs/files/Volumes/foo/bar/artifacts/.internal"
38+
Response.StatusCode = 404
39+
40+
[[Server]]
41+
Pattern = "PUT /api/2.0/fs/directories/Volumes/foo/bar/artifacts/.internal"
42+
Response.StatusCode = 200
43+
44+
[[Server]]
45+
Pattern = "PUT /api/2.0/fs/files/Volumes/foo/bar/artifacts/.internal/source.whl"
46+
Response.StatusCode = 200

‎acceptance/bundle/artifacts/artifact_upload_for_workspace/databricks.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
workspace:
2+
artifact_path: /foo/bar/artifacts
3+
4+
resources:
5+
jobs:
6+
job:
7+
name: "[${bundle.target}] Test Job with Multiple Libraries"
8+
tasks:
9+
- task_key: TestTask1
10+
existing_cluster_id: "0717-132531-5opeqon1"
11+
python_wheel_task:
12+
package_name: "whl"
13+
entry_point: "run"
14+
libraries:
15+
- whl: whl/*.whl
16+
- whl: /Workspace/Users/[email protected]/mywheel.whl
17+
- task_key: TestTask2
18+
for_each_task:
19+
inputs: "[1]"
20+
task:
21+
task_key: TestTask2
22+
existing_cluster_id: "0717-132531-5opeqon1"
23+
python_wheel_task:
24+
package_name: "whl"
25+
entry_point: "run"
26+
libraries:
27+
- whl: whl/*.whl
28+
- whl: /Workspace/Users/[email protected]/mywheel.whl
29+
environments:
30+
- environment_key: "test_env"
31+
spec:
32+
client: "1"
33+
dependencies:
34+
- whl/source.whl
35+
- /Workspace/Users/[email protected]/mywheel.whl

‎acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Local = true
2+
Cloud = false
3+
4+
[EnvMatrix]
5+
DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

‎acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
2+
>>> [CLI] bundle deploy
3+
Uploading whl/source.whl...
4+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
5+
Deploying resources...
6+
Updating deployment state...
7+
Deployment complete!
8+
9+
=== Expecting 2 wheels in libraries section in /jobs/create
10+
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks out.requests.txt
11+
[
12+
{
13+
"existing_cluster_id": "0717-132531-5opeqon1",
14+
"libraries": [
15+
{
16+
"whl": "/Workspace/foo/bar/artifacts/.internal/source.whl"
17+
},
18+
{
19+
"whl": "/Workspace/Users/[email protected]/mywheel.whl"
20+
}
21+
],
22+
"python_wheel_task": {
23+
"entry_point": "run",
24+
"package_name": "whl"
25+
},
26+
"task_key": "TestTask1"
27+
},
28+
{
29+
"for_each_task": {
30+
"inputs": "[1]",
31+
"task": {
32+
"existing_cluster_id": "0717-132531-5opeqon1",
33+
"libraries": [
34+
{
35+
"whl": "/Workspace/foo/bar/artifacts/.internal/source.whl"
36+
},
37+
{
38+
"whl": "/Workspace/Users/[email protected]/mywheel.whl"
39+
}
40+
],
41+
"python_wheel_task": {
42+
"entry_point": "run",
43+
"package_name": "whl"
44+
},
45+
"task_key": "TestTask2"
46+
}
47+
},
48+
"task_key": "TestTask2"
49+
}
50+
]
51+
52+
=== Expecting wheel to be uploaded
53+
>>> jq .path
54+
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl"
55+
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl"
56+
57+
=== Expecting environment dependencies to be updated
58+
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt
59+
[
60+
{
61+
"environment_key": "test_env",
62+
"spec": {
63+
"client": "1",
64+
"dependencies": [
65+
"/Workspace/foo/bar/artifacts/.internal/source.whl",
66+
"/Workspace/Users/[email protected]/mywheel.whl"
67+
]
68+
}
69+
}
70+
]
71+
72+
=== Expecting delete request to artifact_path/.internal folder
73+
>>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt
74+
{
75+
"method": "POST",
76+
"path": "/api/2.0/workspace/delete",
77+
"body": {
78+
"path": "/Workspace/foo/bar/artifacts/.internal",
79+
"recursive": true
80+
}
81+
}
82+
83+
=== Expecting mkdirs request to artifact_path/.internal folder
84+
>>> jq -s .[] | select(.path=="/api/2.0/workspace/mkdirs") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt
85+
{
86+
"method": "POST",
87+
"path": "/api/2.0/workspace/mkdirs",
88+
"body": {
89+
"path": "/Workspace/foo/bar/artifacts/.internal"
90+
}
91+
}

‎acceptance/bundle/artifacts/artifact_upload_for_workspace/script‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Create test wheel files
2+
mkdir -p whl
3+
echo "test wheel content" > whl/source.whl
4+
5+
trace $CLI bundle deploy
6+
7+
title "Expecting 2 wheels in libraries section in /jobs/create"
8+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt
9+
10+
title "Expecting wheel to be uploaded"
11+
trace jq .path < out.requests.txt | grep import | grep whl | sort
12+
13+
title "Expecting environment dependencies to be updated"
14+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt
15+
16+
title "Expecting delete request to artifact_path/.internal folder"
17+
trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt
18+
19+
title "Expecting mkdirs request to artifact_path/.internal folder"
20+
trace jq -s '.[] | select(.path=="/api/2.0/workspace/mkdirs") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt
21+
22+
rm out.requests.txt

‎acceptance/bundle/artifacts/artifact_upload_for_workspace/test.toml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
RecordRequests = true
2+
Ignore = [
3+
'.venv',
4+
'dist',
5+
'build',
6+
'*egg-info',
7+
'.databricks',
8+
'*.whl',
9+
]
10+
11+
[[Server]]
12+
Pattern = "GET /api/2.1/clusters/get"
13+
Response.Body = '''
14+
{
15+
"cluster_id": "0717-132531-5opeqon1",
16+
"spark_version": "13.3.x-scala2.12"
17+
}
18+
'''

0 commit comments

Comments
 (0)