Skip to content

Commit 7e466f4

Browse files
denikdenik
committed
targets test
1 parent e4166ff commit 7e466f4

File tree

6 files changed

+64
-31
lines changed

6 files changed

+64
-31
lines changed

‎acceptance/bin/print_state.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"""
88

99
import os
10+
import argparse
1011

1112

1213
def write(filename):
@@ -16,10 +17,20 @@ def write(filename):
1617
print()
1718

1819

19-
filename = ".databricks/bundle/default/terraform/terraform.tfstate"
20-
if os.path.exists(filename):
21-
write(filename)
2220

23-
filename = ".databricks/bundle/default/resources.json"
24-
if os.path.exists(filename):
25-
write(filename)
21+
def main():
22+
parser = argparse.ArgumentParser()
23+
parser.add_argument("-t", "--target", default="default")
24+
args = parser.parse_args()
25+
26+
filename = f".databricks/bundle/{args.target}/terraform/terraform.tfstate"
27+
if os.path.exists(filename):
28+
write(filename)
29+
30+
filename = f".databricks/bundle/{args.target}/resources.json"
31+
if os.path.exists(filename):
32+
write(filename)
33+
34+
35+
if __name__ == "__main__":
36+
main()

‎acceptance/bundle/migrate/basic/databricks.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,13 @@ resources:
4343
#permissions:
4444
# - level: CAN_MANAGE
4545
# user_name: [email protected]
46+
47+
targets:
48+
dev:
49+
default: true
50+
prod:
51+
resources:
52+
schemas:
53+
test_schema:
54+
catalog_name: mycat
55+
name: myschema

‎acceptance/bundle/migrate/basic/out.new_state.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"state": {
88
"deployment": {
99
"kind": "BUNDLE",
10-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
10+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
1111
},
1212
"edit_mode": "UI_LOCKED",
1313
"format": "MULTI_TASK",
@@ -19,7 +19,7 @@
1919
"tasks": [
2020
{
2121
"notebook_task": {
22-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/notebook"
22+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/notebook"
2323
},
2424
"task_key": "main"
2525
}
@@ -32,13 +32,13 @@
3232
"channel": "CURRENT",
3333
"deployment": {
3434
"kind": "BUNDLE",
35-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
35+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
3636
},
3737
"edition": "ADVANCED",
3838
"libraries": [
3939
{
4040
"notebook": {
41-
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline"
41+
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/pipeline"
4242
}
4343
}
4444
],

‎acceptance/bundle/migrate/basic/out.original_state.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
"deployment": [
2323
{
2424
"kind": "BUNDLE",
25-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
25+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
2626
}
2727
],
2828
"description": null,
@@ -87,7 +87,7 @@
8787
"notebook_task": [
8888
{
8989
"base_parameters": null,
90-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/notebook",
90+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/notebook",
9191
"source": "WORKSPACE",
9292
"warehouse_id": ""
9393
}
@@ -142,7 +142,7 @@
142142
"deployment": [
143143
{
144144
"kind": "BUNDLE",
145-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
145+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
146146
}
147147
],
148148
"development": false,
@@ -165,7 +165,7 @@
165165
"maven": [],
166166
"notebook": [
167167
{
168-
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline"
168+
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/pipeline"
169169
}
170170
],
171171
"whl": ""

‎acceptance/bundle/migrate/basic/output.txt‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,19 @@ This command migrates the existing Terraform state file (terraform.tfstate) to a
55
To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env var set.
66

77
>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy
8-
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files...
8+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files...
99
Deploying resources...
1010
Updating deployment state...
1111
Deployment complete!
1212

1313
>>> [CLI] bundle deployment migrate
14-
Migrated 3 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
14+
Migrated 3 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json
1515
Validate the migration by running "bundle debug plan", there should be no actions.
1616

1717
The state file is not synchronized to the workspace yet. To do that, run "bundle deploy".
1818

1919
To finalize deployment, run "bundle deploy".
2020

21-
>>> ls -1 .databricks/bundle/default/
22-
deployment.json
23-
resources.json
24-
sync-snapshots
25-
terraform
26-
2721
>>> jq .state."resources.pipelines.test_pipeline".state.tags.volume_storage_location out.new_state.json
2822
"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
2923

@@ -32,15 +26,15 @@ Error: Required engine "terraform" does not match present state files. Set requi
3226

3327
Available state files:
3428
- terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
35-
- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
29+
- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
3630

3731

3832
>>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
3933
Error: Required engine "terraform" does not match present state files. Set required engine via "DATABRICKS_BUNDLE_ENGINE" env var.
4034

4135
Available state files:
4236
- terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
43-
- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
37+
- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
4438

4539

4640
>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
@@ -80,7 +74,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
8074
}
8175

8276
>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle deploy
83-
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files...
77+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files...
8478
Deploying resources...
8579
Updating deployment state...
8680
Deployment complete!
@@ -92,12 +86,26 @@ Deployment complete!
9286
=== Should show that it's already migrated
9387
>>> musterr [CLI] bundle deployment migrate
9488
Error: already using direct engine
95-
Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
89+
Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
9690

9791
>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
98-
[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
92+
[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
9993

10094
>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull
10195
terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
10296
resources.json: remote direct state serial=5 lineage="[UUID]"
103-
[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=5 lineage="[UUID]"
97+
[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
98+
99+
=== Different target, still on terraform
100+
>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle deploy -t prod
101+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/prod/files...
102+
Deploying resources...
103+
Updating deployment state...
104+
Deployment complete!
105+
106+
>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -t prod
107+
[TEST_TMP_DIR]/.databricks/bundle/prod/terraform/terraform.tfstate: local terraform state serial=5 lineage="[UUID]"
108+
109+
>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -t prod --force-pull
110+
terraform.tfstate: remote terraform state serial=5 lineage="[UUID]"
111+
[TEST_TMP_DIR]/.databricks/bundle/prod/terraform/terraform.tfstate: local terraform state serial=5 lineage="[UUID]"

‎acceptance/bundle/migrate/basic/script‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,10 @@
22
trace musterr $CLI bundle deployment migrate
33

44
trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
5-
print_state.py > out.original_state.json
5+
print_state.py -t dev > out.original_state.json
66

77
trace $CLI bundle deployment migrate 2>&1 | contains.py 'Migrated 3 resources'
8-
print_state.py > out.new_state.json
9-
trace ls -1 .databricks/bundle/default/ | contains.py "resources.json"
8+
print_state.py -t dev > out.new_state.json
109
# check that this is stored as resolved value, not as reference:
1110
trace jq '.state."resources.pipelines.test_pipeline".state.tags.volume_storage_location' out.new_state.json | contains.py 's3://'
1211

@@ -29,3 +28,8 @@ trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states
2928
trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states --force-pull
3029

3130
rm out.requests.txt
31+
32+
title "Different target, still on terraform"
33+
trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy -t prod
34+
trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states -t prod
35+
trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states -t prod --force-pull

0 commit comments

Comments
 (0)