Skip to content

Commit 0b84364

Browse files
denikdenik
authored
acc: add remote_field_storage_location test; improve testserver volumes (#3474)
## Changes - Update testserver volume to process storage_location like backend does on AWS. - Update acceptance test runner to treat Local more like AWS when it comes to CLOUD_ENV_BASE, so that tests can be run on both Local + Cloud - New test that sets storage_location to string and gets an error. - New test that sets storage_location to reference to another storage location and triggers "Provider produced inconsistent final plan" from terraform. ## Why New tests will be used for testing direct backend once it supports arbitrary $resources references.
1 parent 54932e4 commit 0b84364

File tree

15 files changed

+266
-2
lines changed

15 files changed

+266
-2
lines changed

‎acceptance/acceptance_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,6 @@ func runTest(t *testing.T,
552552
// populate CLOUD_ENV_BASE
553553
envBase := getCloudEnvBase(cloudEnv)
554554
cmd.Env = append(cmd.Env, "CLOUD_ENV_BASE="+envBase)
555-
repls.Set(envBase, "[CLOUD_ENV_BASE]")
556555

557556
// Must be added PrepareReplacementsUser, otherwise conflicts with [USERNAME]
558557
testdiff.PrepareReplacementsUUID(t, &repls)
@@ -1163,7 +1162,7 @@ func getCloudEnvBase(cloudEnv string) string {
11631162
case "gcp", "gcp-ucws":
11641163
return "gcp"
11651164
case "":
1166-
return ""
1165+
return "aws"
11671166
default:
11681167
return "unknown-cloudEnv-" + cloudEnv
11691168
}

‎acceptance/bundle/deploy/mlops-stacks/test.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,7 @@ EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # uses 'experiments' resourc
88
Ignore = [
99
"config.json"
1010
]
11+
12+
[[Repls]]
13+
Old = "aws|azure|gcp"
14+
New = "[CLOUD_ENV_BASE]"

‎acceptance/bundle/resource_deps/remote_field_storage_location/databricks.yml.tmpl‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
bundle:
2+
name: testbundle-${UNIQUE_NAME}
3+
4+
resources:
5+
schemas:
6+
my:
7+
catalog_name: main
8+
name: myschema-${UNIQUE_NAME}
9+
10+
volumes:
11+
bar:
12+
catalog_name: main
13+
schema_name: myschema-${UNIQUE_NAME}
14+
name: volumebar-${UNIQUE_NAME}
15+
foo:
16+
catalog_name: main
17+
schema_name: myschema-${UNIQUE_NAME}
18+
name: volumefoo-${UNIQUE_NAME}
19+
storage_location: ${resources.volumes.bar.storage_location}

‎acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Local = true
2+
Cloud = true
3+
RequiresUnityCatalog = true
4+
5+
[CloudEnvs]
6+
azure = false
7+
gcp = false
8+
9+
[EnvMatrix]
10+
DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

‎acceptance/bundle/resource_deps/remote_field_storage_location/output.txt‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
2+
>>> [CLI] bundle validate -o json
3+
{
4+
"schemas": {
5+
"my": {
6+
"catalog_name": "main",
7+
"name": "myschema-[UNIQUE_NAME]"
8+
}
9+
},
10+
"volumes": {
11+
"bar": {
12+
"catalog_name": "main",
13+
"name": "volumebar-[UNIQUE_NAME]",
14+
"schema_name": "${resources.schemas.my.name}",
15+
"volume_type": "MANAGED"
16+
},
17+
"foo": {
18+
"catalog_name": "main",
19+
"name": "volumefoo-[UNIQUE_NAME]",
20+
"schema_name": "${resources.schemas.my.name}",
21+
"storage_location": "${resources.volumes.bar.storage_location}",
22+
"volume_type": "MANAGED"
23+
}
24+
}
25+
}
26+
27+
>>> [CLI] bundle plan
28+
create schemas.my
29+
create volumes.bar
30+
create volumes.foo
31+
32+
>>> print_requests
33+
34+
>>> [CLI] bundle deploy
35+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files...
36+
Deploying resources...
37+
Error: terraform apply: exit status 1
38+
39+
Error: Provider produced inconsistent final plan
40+
41+
When expanding the plan for databricks_volume.foo to include new values
42+
learned so far during apply, provider
43+
"registry.terraform.io/databricks/databricks" produced an invalid new value
44+
for .storage_location: was null, but now
45+
cty.StringVal("s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]").
46+
47+
This is a bug in the provider, which should be reported in the provider's own
48+
issue tracker.
49+
50+
51+
Updating deployment state...
52+
53+
>>> [CLI] bundle destroy --auto-approve
54+
The following resources will be deleted:
55+
delete schema my
56+
delete volume bar
57+
58+
This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
59+
delete schema my
60+
61+
This action will result in the deletion of the following volumes.
62+
For managed volumes, the files stored in the volume are also deleted from your
63+
cloud tenant within 30 days. For external volumes, the metadata about the volume
64+
is removed from the catalog, but the underlying files are not deleted:
65+
delete volume bar
66+
67+
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default
68+
69+
Deleting files...
70+
Destroy complete!
71+
72+
Exit code: 1

‎acceptance/bundle/resource_deps/remote_field_storage_location/script‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
print_requests() {
2+
jq --sort-keys 'select(.path | contains("/unity"))' < out.requests.txt
3+
rm out.requests.txt
4+
}
5+
6+
cleanup() {
7+
trace $CLI bundle destroy --auto-approve
8+
rm out.requests.txt
9+
}
10+
11+
envsubst < databricks.yml.tmpl > databricks.yml
12+
trace $CLI bundle validate -o json | jq .resources
13+
trace $CLI bundle plan
14+
trace print_requests
15+
trap cleanup EXIT
16+
trace $CLI bundle deploy
17+
trace print_requests

‎acceptance/bundle/resource_deps/remote_field_storage_location/test.toml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
Badness = "Terraform says it is a bug in the provider; expected either to reject this config or to evaluate it and pass it to backend which will reject it"
2+
3+
Local = true
4+
Cloud = true
5+
RequiresUnityCatalog = true
6+
7+
# Each cloud has its specific storage url, we record AWS's one in the output (which is also emulated by Local):
8+
CloudEnvs.gcp = false
9+
CloudEnvs.azure = false
10+
11+
EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

‎acceptance/bundle/resources/volumes/change-comment/output.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ Deployment complete!
4747
"full_name":"main.myschema.myvolume",
4848
"name":"myvolume",
4949
"schema_name":"myschema",
50+
"storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
5051
"volume_type":"MANAGED"
5152
}
5253

‎acceptance/bundle/resources/volumes/change-name/output.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ Deployment complete!
6262
"full_name":"main.myschema.mynewvolume",
6363
"name":"mynewvolume",
6464
"schema_name":"myschema",
65+
"storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
6566
"volume_type":"MANAGED"
6667
}
6768

‎acceptance/bundle/resources/volumes/set-storage-location/databricks.yml.tmpl‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
bundle:
2+
name: testbundle-$UNIQUE_NAME
3+
4+
resources:
5+
schemas:
6+
myschema:
7+
catalog_name: main
8+
name: testschema-$UNIQUE_NAME
9+
10+
volumes:
11+
volume1:
12+
catalog_name: main
13+
schema_name: testschema-$UNIQUE_NAME
14+
name: testvolume-$UNIQUE_NAME
15+
comment: COMMENT1
16+
storage_location: "s3://my_bucket/my_location"

0 commit comments

Comments
 (0)