Skip to content

Commit 6e1810a

Browse files
denikpietern
denik
and
pietern
authored
Stricter validation of DATABRICKS_BUNDLE_ENGINE (#3875)
## Changes ### User visible - Since #3797 this env var is only consulted if there is no state. Now it is always consulted and must match the state, otherwise error is raised. - New “bundle debug states” to print info about available state files. - Stricter state validation in case both terraform and direct state files are present. If serial numbers are the same, error rather than preferring direct. - Better error messages when state validation fails. ### Internal - Use string enum for engine instead if bool and update all functions to use the enum. - Move all engine parsing/configuration to bundle/config/engine package. - Engine configuration is done in top level command handler rather than inside PullResourcesState. - PullResourcesState() returns winning StateDesc object. ## Why Ensure users get the engine they select via env var (and in the future this will extend to config setting). ## Tests New acceptance tests. New test helper print_state.py to print state as is. --------- Co-authored-by: Pieter Noordhuis <[email protected]>
1 parent 8d41fe3 commit 6e1810a

39 files changed

+801
-222
lines changed

‎acceptance/bin/print_state.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Print resources state from default target.
4+
5+
Note, this intentionally has no logic on guessing what is the right state file (e.g. via DATABRICKS_BUNDLE_ENGINE),
6+
the goal is to record all states that are available.
7+
"""
8+
9+
import os
10+
11+
12+
def write(filename):
13+
data = open(filename).read()
14+
print(data, end="")
15+
if not data.endswith("\n"):
16+
print()
17+
18+
19+
filename = ".databricks/bundle/default/terraform/terraform.tfstate"
20+
if os.path.exists(filename):
21+
write(filename)
22+
23+
filename = ".databricks/bundle/default/resources.json"
24+
if os.path.exists(filename):
25+
write(filename)

‎acceptance/bundle/debug/output.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ Usage:
66
Available Commands:
77
plan Show deployment plan in JSON format (experimental)
88
refschema Dump all relevant fields all bundle resources
9+
states Show available state files
910

1011
Flags:
1112
-h, --help help for debug

‎acceptance/bundle/resources/jobs/update/out.state.direct.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"lineage": "[UUID]",
3+
"serial": 0,
4+
"state": {
5+
"resources.jobs.foo": {
6+
"__id__": "[JOB_ID]",
7+
"state": {
8+
"deployment": {
9+
"kind": "BUNDLE",
10+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
11+
},
12+
"edit_mode": "UI_LOCKED",
13+
"format": "MULTI_TASK",
14+
"job_clusters": [
15+
{
16+
"job_cluster_key": "key",
17+
"new_cluster": {
18+
"num_workers": 0,
19+
"spark_version": "13.3.x-scala2.12"
20+
}
21+
}
22+
],
23+
"max_concurrent_runs": 1,
24+
"name": "foo",
25+
"queue": {
26+
"enabled": true
27+
},
28+
"trigger": {
29+
"pause_status": "UNPAUSED",
30+
"periodic": {
31+
"interval": 1,
32+
"unit": "DAYS"
33+
}
34+
}
35+
}
36+
}
37+
}
38+
}

‎acceptance/bundle/resources/jobs/update/out.state.terraform.json‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
{
2+
"version": 4,
3+
"terraform_version": "1.5.5",
4+
"serial": 1,
5+
"lineage": "[UUID]",
6+
"outputs": {},
7+
"resources": [
8+
{
9+
"mode": "managed",
10+
"type": "databricks_job",
11+
"name": "foo",
12+
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
13+
"instances": [
14+
{
15+
"schema_version": 2,
16+
"attributes": {
17+
"always_running": false,
18+
"budget_policy_id": null,
19+
"continuous": [],
20+
"control_run_state": false,
21+
"dbt_task": [],
22+
"deployment": [
23+
{
24+
"kind": "BUNDLE",
25+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
26+
}
27+
],
28+
"description": null,
29+
"edit_mode": "UI_LOCKED",
30+
"email_notifications": [],
31+
"environment": [],
32+
"existing_cluster_id": null,
33+
"format": "MULTI_TASK",
34+
"git_source": [],
35+
"health": [],
36+
"id": "[JOB_ID]",
37+
"job_cluster": [
38+
{
39+
"job_cluster_key": "key",
40+
"new_cluster": [
41+
{
42+
"__apply_policy_default_values_allow_list": null,
43+
"apply_policy_default_values": false,
44+
"autoscale": [],
45+
"aws_attributes": [],
46+
"azure_attributes": [],
47+
"cluster_id": "",
48+
"cluster_log_conf": [],
49+
"cluster_mount_info": [],
50+
"cluster_name": "",
51+
"custom_tags": null,
52+
"data_security_mode": "",
53+
"docker_image": [],
54+
"driver_instance_pool_id": "",
55+
"driver_node_type_id": "",
56+
"enable_elastic_disk": false,
57+
"enable_local_disk_encryption": false,
58+
"gcp_attributes": [],
59+
"idempotency_token": "",
60+
"init_scripts": [],
61+
"instance_pool_id": "",
62+
"is_single_node": false,
63+
"kind": "",
64+
"library": [],
65+
"node_type_id": "",
66+
"num_workers": 0,
67+
"policy_id": "",
68+
"provider_config": [],
69+
"remote_disk_throughput": 0,
70+
"runtime_engine": "",
71+
"single_user_name": "",
72+
"spark_conf": null,
73+
"spark_env_vars": null,
74+
"spark_version": "13.3.x-scala2.12",
75+
"ssh_public_keys": null,
76+
"total_initial_remote_disk_size": 0,
77+
"use_ml_runtime": false,
78+
"workload_type": []
79+
}
80+
]
81+
}
82+
],
83+
"library": [],
84+
"max_concurrent_runs": 1,
85+
"max_retries": 0,
86+
"min_retry_interval_millis": 0,
87+
"name": "foo",
88+
"new_cluster": [],
89+
"notebook_task": [],
90+
"notification_settings": [],
91+
"parameter": [],
92+
"performance_target": null,
93+
"pipeline_task": [],
94+
"provider_config": [],
95+
"python_wheel_task": [],
96+
"queue": [
97+
{
98+
"enabled": true
99+
}
100+
],
101+
"retry_on_timeout": false,
102+
"run_as": null,
103+
"run_job_task": [],
104+
"schedule": [],
105+
"spark_jar_task": [],
106+
"spark_python_task": [],
107+
"spark_submit_task": [],
108+
"tags": null,
109+
"task": [],
110+
"timeout_seconds": 0,
111+
"timeouts": null,
112+
"trigger": [
113+
{
114+
"file_arrival": [],
115+
"pause_status": "UNPAUSED",
116+
"periodic": [
117+
{
118+
"interval": 1,
119+
"unit": "DAYS"
120+
}
121+
],
122+
"table_update": []
123+
}
124+
],
125+
"url": "[DATABRICKS_URL]/#job/[JOB_ID]",
126+
"usage_policy_id": null,
127+
"webhook_notifications": []
128+
},
129+
"sensitive_attributes": [],
130+
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
131+
}
132+
]
133+
}
134+
],
135+
"check_results": null
136+
}

‎acceptance/bundle/resources/jobs/update/script‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ echo "*" > .gitignore
22
trace $CLI bundle plan
33
$CLI bundle debug plan > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
44
trace $CLI bundle deploy
5+
print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json
56
trace $CLI bundle plan
67
trace $CLI bundle debug plan > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json
78

‎acceptance/bundle/state/bad_env/databricks.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
bundle:
2+
name: test-bundle
3+
4+
resources:
5+
schemas:
6+
foo:
7+
catalog_name: bar
8+
name: foo

‎acceptance/bundle/state/bad_env/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/state/bad_env/output.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
2+
>>> musterr [CLI] bundle plan
3+
Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'terraform' or 'direct')
4+
5+
>>> musterr [CLI] bundle deploy
6+
Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'terraform' or 'direct')
7+
8+
>>> musterr [CLI] bundle summary
9+
Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'terraform' or 'direct')
10+
11+
>>> musterr [CLI] bundle destroy --auto-approve
12+
Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'terraform' or 'direct')
13+
14+
>>> musterr [CLI] bundle validate
15+
Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'terraform' or 'direct')
16+
17+
Found 1 error

‎acceptance/bundle/state/bad_env/script‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
export DATABRICKS_BUNDLE_ENGINE=abc
2+
trace musterr $CLI bundle plan
3+
trace musterr $CLI bundle deploy
4+
trace musterr $CLI bundle summary
5+
trace musterr $CLI bundle destroy --auto-approve
6+
trace musterr $CLI bundle validate

‎acceptance/bundle/state/lineage_different/output.txt‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,43 @@
11

2-
=== Terraform state is present, env var does not matter, we select terraform
3-
4-
>>> [CLI] bundle deploy
5-
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
6-
Deploying resources...
7-
Updating deployment state...
8-
Deployment complete!
9-
10-
=== Adding resources.json with different lineage
112
>>> musterr [CLI] bundle plan
12-
Error: lineage mismatch in state files: <local direct state serial=0 lineage="222">, <remote terraform state serial=2 lineage="111">, <local terraform state serial=2 lineage="111">
3+
Error: lineage mismatch in state files
4+
5+
Available state files:
6+
- [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=0 lineage="111"
7+
- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=0 lineage="222"
138

149

1510
>>> musterr [CLI] bundle deploy
16-
Error: lineage mismatch in state files: <local direct state serial=0 lineage="222">, <remote terraform state serial=2 lineage="111">, <local terraform state serial=2 lineage="111">
11+
Error: lineage mismatch in state files
12+
13+
Available state files:
14+
- [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=0 lineage="111"
15+
- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=0 lineage="222"
1716

1817

1918
>>> musterr [CLI] bundle summary
20-
Error: lineage mismatch in state files: <local direct state serial=0 lineage="222">, <local terraform state serial=2 lineage="111">
19+
Error: lineage mismatch in state files
20+
21+
Available state files:
22+
- [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=0 lineage="111"
23+
- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=0 lineage="222"
2124

2225

2326
>>> musterr [CLI] bundle destroy --auto-approve
24-
Error: lineage mismatch in state files: <local direct state serial=0 lineage="222">, <remote terraform state serial=2 lineage="111">, <local terraform state serial=2 lineage="111">
27+
Error: lineage mismatch in state files
28+
29+
Available state files:
30+
- [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=0 lineage="111"
31+
- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=0 lineage="222"
32+
33+
34+
>>> [CLI] bundle validate
35+
Name: test-bundle
36+
Target: default
37+
Workspace:
38+
User: [USERNAME]
39+
Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
40+
41+
Validation OK!
2542

43+
>>> [CLI] bundle

0 commit comments

Comments
 (0)