Skip to content

Commit de41248

Browse files
alyssa-dbalyssa-db
authored
acc: pipelines and bundle run --refresh and --full-refresh flags (#3301)
## Changes Added acceptance tests to check if both flags can be specified at once, --refresh and --full-refresh flags Added tests for bundle run pipeline [refresh-flags] Acceptance tests: - for pipelines, adds one new test for --full-refresh table1,table2 --refresh table3,table4 - for bundles, adds all tests to match tests present in the pipelines tests Follows #3300
1 parent 508d9a3 commit de41248

File tree

8 files changed

+146
-3
lines changed

8 files changed

+146
-3
lines changed

‎acceptance/bundle/run/refresh-flags/databricks.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
bundle:
2+
name: test-bundle-run-flags
3+
4+
resources:
5+
pipelines:
6+
my_pipeline:
7+
name: test-pipeline-flags
8+
libraries:
9+
- file:
10+
path: pipeline_file.py

‎acceptance/bundle/run/refresh-flags/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Local = true
2+
Cloud = false
3+
4+
[EnvMatrix]
5+
DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

‎acceptance/bundle/run/refresh-flags/output.txt‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
2+
>>> [CLI] bundle deploy
3+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-run-flags/default/files...
4+
Deploying resources...
5+
Updating deployment state...
6+
Deployment complete!
7+
8+
=== Running pipeline with --refresh flag and specific tables
9+
>>> [CLI] bundle run my_pipeline --refresh table1,table2
10+
Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
11+
12+
13+
>>> print_requests
14+
{
15+
"body": {
16+
"refresh_selection": [
17+
"table1",
18+
"table2"
19+
]
20+
},
21+
"method": "POST",
22+
"path": "/api/2.0/pipelines/[UUID]/updates"
23+
}
24+
25+
=== Running pipeline with --full-refresh-all flag
26+
>>> [CLI] bundle run my_pipeline --full-refresh-all
27+
Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
28+
29+
30+
>>> print_requests
31+
{
32+
"body": {
33+
"full_refresh": true
34+
},
35+
"method": "POST",
36+
"path": "/api/2.0/pipelines/[UUID]/updates"
37+
}
38+
39+
=== Running pipeline with --full-refresh flag and specific tables
40+
>>> [CLI] bundle run my_pipeline --full-refresh table1,table2
41+
Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
42+
43+
44+
>>> print_requests
45+
{
46+
"body": {
47+
"full_refresh_selection": [
48+
"table1",
49+
"table2"
50+
]
51+
},
52+
"method": "POST",
53+
"path": "/api/2.0/pipelines/[UUID]/updates"
54+
}
55+
56+
=== Running pipeline with --full-refresh flag and --refresh flag
57+
>>> [CLI] bundle run my_pipeline --full-refresh table1,table2 --refresh table3,table4
58+
Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
59+
60+
61+
>>> print_requests
62+
{
63+
"body": {
64+
"full_refresh_selection": [
65+
"table1",
66+
"table2"
67+
],
68+
"refresh_selection": [
69+
"table3",
70+
"table4"
71+
]
72+
},
73+
"method": "POST",
74+
"path": "/api/2.0/pipelines/[UUID]/updates"
75+
}

‎acceptance/bundle/run/refresh-flags/pipeline_file.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import dlt
2+
from pyspark.sql import SparkSession
3+
4+
5+
@dlt.table
6+
def my_table():
7+
return spark.range(10)

‎acceptance/bundle/run/refresh-flags/script‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
print_requests() {
2+
jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt
3+
rm out.requests.txt
4+
}
5+
6+
trace $CLI bundle deploy
7+
rm out.requests.txt
8+
9+
title "Running pipeline with --refresh flag and specific tables"
10+
trace $CLI bundle run my_pipeline --refresh table1,table2
11+
trace print_requests
12+
13+
title "Running pipeline with --full-refresh-all flag"
14+
trace $CLI bundle run my_pipeline --full-refresh-all
15+
trace print_requests
16+
17+
title "Running pipeline with --full-refresh flag and specific tables"
18+
trace $CLI bundle run my_pipeline --full-refresh table1,table2
19+
trace print_requests
20+
21+
title "Running pipeline with --full-refresh flag and --refresh flag"
22+
trace $CLI bundle run my_pipeline --full-refresh table1,table2 --refresh table3,table4
23+
trace print_requests

‎acceptance/bundle/run/refresh-flags/test.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
RecordRequests = true

‎acceptance/pipelines/run/refresh-flags/output.txt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,24 @@ Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
5252
"method": "POST",
5353
"path": "/api/2.0/pipelines/[UUID]/updates"
5454
}
55+
56+
=== Running pipeline with --full-refresh flag and --refresh flag
57+
>>> [PIPELINES] run --full-refresh table1,table2 --refresh table3,table4
58+
Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
59+
60+
61+
>>> print_requests
62+
{
63+
"body": {
64+
"full_refresh_selection": [
65+
"table1",
66+
"table2"
67+
],
68+
"refresh_selection": [
69+
"table3",
70+
"table4"
71+
]
72+
},
73+
"method": "POST",
74+
"path": "/api/2.0/pipelines/[UUID]/updates"
75+
}

‎acceptance/pipelines/run/refresh-flags/script‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,17 @@ trace $PIPELINES deploy
77
rm out.requests.txt
88

99
title "Running pipeline with --refresh flag and specific tables"
10-
# Should refresh only the specified tables
1110
trace $PIPELINES run --refresh table1,table2
1211
trace print_requests
1312

1413
title "Running pipeline with --full-refresh-all flag"
15-
# Should have full refresh set to true
1614
trace $PIPELINES run --full-refresh-all
1715
trace print_requests
1816

1917
title "Running pipeline with --full-refresh flag and specific tables"
20-
# Should reset and recompute only the specified tables
2118
trace $PIPELINES run --full-refresh table1,table2
2219
trace print_requests
20+
21+
title "Running pipeline with --full-refresh flag and --refresh flag"
22+
trace $PIPELINES run --full-refresh table1,table2 --refresh table3,table4
23+
trace print_requests

0 commit comments

Comments
 (0)