Skip to content

Commit 5d2570a

Browse files
denikdenik
authored
direct: pipelines: Ignore run_as change from the backend (#4037)
## Changes This matches terraform and works around backend issue where run_as is dropped. ## Tests Our testserver happens to drop run_as as well (accidentally), added new acceptance test that shows that tf and direct have no drift despite that and that migration works. Fixed print_state.py helper to infer target if -t is not passed.
1 parent 137b596 commit 5d2570a

File tree

13 files changed

+573
-3
lines changed

13 files changed

+573
-3
lines changed

‎acceptance/bin/print_state.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"""
88

99
import os
10+
import glob
1011
import argparse
1112

1213

@@ -19,14 +20,22 @@ def write(filename):
1920

2021
def main():
2122
parser = argparse.ArgumentParser()
22-
parser.add_argument("-t", "--target", default="default")
23+
parser.add_argument("-t", "--target")
2324
parser.add_argument("--backup", action="store_true")
2425
args = parser.parse_args()
2526

2627
if args.target:
2728
target_dir = f".databricks/bundle/{args.target}"
2829
if not os.path.exists(target_dir):
2930
raise SystemExit(f"Invalid target {args.target!r}: {target_dir} does not exist")
31+
else:
32+
targets = glob.glob(".databricks/bundle/*")
33+
if not targets:
34+
return
35+
targets = [os.path.basename(x) for x in targets]
36+
if len(targets) > 1:
37+
raise SystemExit("Many targets found, specify one to use with -t: " + ", ".join(sorted(targets)))
38+
args.target = targets[0]
3039

3140
if args.backup:
3241
filename = f".databricks/bundle/{args.target}/terraform/terraform.tfstate.backup"

‎acceptance/bundle/migrate/runas/databricks.yml.tmpl‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
bundle:
2+
name: dabs_revenue-$UNIQUE_NAME
3+
4+
permissions:
5+
- level: CAN_VIEW
6+
group_name: users
7+
8+
resources:
9+
pipelines:
10+
foo:
11+
name: DABs Revenue Pipeline
12+
13+
libraries:
14+
- notebook:
15+
path: sql.sql
16+
17+
serverless: true
18+
19+
targets:
20+
production:
21+
run_as:
22+
service_principal_name: $TEST_SP_APPLICATION_ID
23+
24+
resources:
25+
pipelines:
26+
# Make the pipeline write to the team's schema in logfood.
27+
foo:
28+
catalog: main
29+
target: team_eng_deco

‎acceptance/bundle/migrate/runas/out.create_requests.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"headers": {
3+
"User-Agent": [
4+
"databricks-tf-provider/1.97.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/pipeline auth/pat"
5+
]
6+
},
7+
"method": "POST",
8+
"path": "/api/2.0/pipelines",
9+
"body": {
10+
"catalog": "main",
11+
"channel": "CURRENT",
12+
"deployment": {
13+
"kind": "BUNDLE",
14+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json"
15+
},
16+
"edition": "ADVANCED",
17+
"libraries": [
18+
{
19+
"notebook": {
20+
"path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql"
21+
}
22+
}
23+
],
24+
"name": "DABs Revenue Pipeline",
25+
"run_as": {
26+
"service_principal_name": "[UUID]"
27+
},
28+
"serverless": true,
29+
"target": "team_eng_deco"
30+
}
31+
}
32+
{
33+
"headers": {
34+
"User-Agent": [
35+
"databricks-tf-provider/1.97.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/permissions auth/pat"
36+
]
37+
},
38+
"method": "PUT",
39+
"path": "/api/2.0/permissions/pipelines/[UUID]",
40+
"body": {
41+
"access_control_list": [
42+
{
43+
"group_name": "users",
44+
"permission_level": "CAN_VIEW"
45+
},
46+
{
47+
"permission_level": "IS_OWNER",
48+
"user_name": "[USERNAME]"
49+
}
50+
]
51+
}
52+
}

‎acceptance/bundle/migrate/runas/out.new_state.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"lineage": "[UUID]",
3+
"serial": 4,
4+
"state": {
5+
"resources.pipelines.foo": {
6+
"__id__": "[UUID]",
7+
"state": {
8+
"catalog": "main",
9+
"channel": "CURRENT",
10+
"deployment": {
11+
"kind": "BUNDLE",
12+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json"
13+
},
14+
"edition": "ADVANCED",
15+
"libraries": [
16+
{
17+
"notebook": {
18+
"path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql"
19+
}
20+
}
21+
],
22+
"name": "DABs Revenue Pipeline",
23+
"run_as": {
24+
"service_principal_name": "[UUID]"
25+
},
26+
"serverless": true,
27+
"target": "team_eng_deco"
28+
}
29+
},
30+
"resources.pipelines.foo.permissions": {
31+
"__id__": "/pipelines/[UUID]",
32+
"state": {
33+
"object_id": "/pipelines/[UUID]",
34+
"permissions": [
35+
{
36+
"group_name": "users",
37+
"permission_level": "CAN_VIEW"
38+
},
39+
{
40+
"permission_level": "IS_OWNER",
41+
"user_name": "[USERNAME]"
42+
}
43+
]
44+
}
45+
}
46+
}
47+
}

‎acceptance/bundle/migrate/runas/out.old_state.json‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
{
2+
"version": 4,
3+
"terraform_version": "1.5.5",
4+
"serial": 3,
5+
"lineage": "[UUID]",
6+
"outputs": {},
7+
"resources": [
8+
{
9+
"mode": "managed",
10+
"type": "databricks_permissions",
11+
"name": "pipeline_foo",
12+
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
13+
"instances": [
14+
{
15+
"schema_version": 0,
16+
"attributes": {
17+
"access_control": [
18+
{
19+
"group_name": "",
20+
"permission_level": "IS_OWNER",
21+
"service_principal_name": "",
22+
"user_name": "[USERNAME]"
23+
},
24+
{
25+
"group_name": "users",
26+
"permission_level": "CAN_VIEW",
27+
"service_principal_name": "",
28+
"user_name": ""
29+
}
30+
],
31+
"alert_v2_id": null,
32+
"app_name": null,
33+
"authorization": null,
34+
"cluster_id": null,
35+
"cluster_policy_id": null,
36+
"dashboard_id": null,
37+
"database_instance_name": null,
38+
"directory_id": null,
39+
"directory_path": null,
40+
"experiment_id": null,
41+
"id": "/pipelines/[UUID]",
42+
"instance_pool_id": null,
43+
"job_id": null,
44+
"notebook_id": null,
45+
"notebook_path": null,
46+
"object_type": "pipelines",
47+
"pipeline_id": "[UUID]",
48+
"registered_model_id": null,
49+
"repo_id": null,
50+
"repo_path": null,
51+
"serving_endpoint_id": null,
52+
"sql_alert_id": null,
53+
"sql_dashboard_id": null,
54+
"sql_endpoint_id": null,
55+
"sql_query_id": null,
56+
"vector_search_endpoint_id": null,
57+
"workspace_file_id": null,
58+
"workspace_file_path": null
59+
},
60+
"sensitive_attributes": [],
61+
"private": "bnVsbA==",
62+
"dependencies": [
63+
"databricks_pipeline.foo"
64+
]
65+
}
66+
]
67+
},
68+
{
69+
"mode": "managed",
70+
"type": "databricks_pipeline",
71+
"name": "foo",
72+
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
73+
"instances": [
74+
{
75+
"schema_version": 0,
76+
"attributes": {
77+
"allow_duplicate_names": false,
78+
"budget_policy_id": null,
79+
"catalog": "main",
80+
"cause": null,
81+
"channel": "CURRENT",
82+
"cluster": [],
83+
"cluster_id": null,
84+
"configuration": null,
85+
"continuous": false,
86+
"creator_user_name": "[USERNAME]",
87+
"deployment": [
88+
{
89+
"kind": "BUNDLE",
90+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json"
91+
}
92+
],
93+
"development": false,
94+
"edition": "ADVANCED",
95+
"environment": [],
96+
"event_log": [],
97+
"expected_last_modified": 0,
98+
"filters": [],
99+
"gateway_definition": [],
100+
"health": null,
101+
"id": "[UUID]",
102+
"ingestion_definition": [],
103+
"last_modified": [UNIX_TIME_MILLIS],
104+
"latest_updates": null,
105+
"library": [
106+
{
107+
"file": [],
108+
"glob": [],
109+
"jar": "",
110+
"maven": [],
111+
"notebook": [
112+
{
113+
"path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql"
114+
}
115+
],
116+
"whl": ""
117+
}
118+
],
119+
"name": "DABs Revenue Pipeline",
120+
"notification": [],
121+
"photon": false,
122+
"restart_window": [],
123+
"root_path": null,
124+
"run_as": [
125+
{
126+
"service_principal_name": "[UUID]",
127+
"user_name": ""
128+
}
129+
],
130+
"run_as_user_name": "[USERNAME]",
131+
"schema": null,
132+
"serverless": true,
133+
"state": "IDLE",
134+
"storage": null,
135+
"tags": null,
136+
"target": "team_eng_deco",
137+
"timeouts": null,
138+
"trigger": [],
139+
"url": "[DATABRICKS_URL]/#joblist/pipelines/[UUID]",
140+
"usage_policy_id": null
141+
},
142+
"sensitive_attributes": [],
143+
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
144+
}
145+
]
146+
}
147+
],
148+
"check_results": null
149+
}

‎acceptance/bundle/migrate/runas/out.pipelines_get.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"creator_user_name":"[USERNAME]",
3+
"last_modified":[UNIX_TIME_MILLIS],
4+
"name":"DABs Revenue Pipeline",
5+
"pipeline_id":"[UUID]",
6+
"run_as_user_name":"[USERNAME]",
7+
"spec": {
8+
"catalog":"main",
9+
"channel":"CURRENT",
10+
"deployment": {
11+
"kind":"BUNDLE",
12+
"metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json"
13+
},
14+
"edition":"ADVANCED",
15+
"id":"[UUID]",
16+
"libraries": [
17+
{
18+
"notebook": {
19+
"path":"/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql"
20+
}
21+
}
22+
],
23+
"name":"DABs Revenue Pipeline",
24+
"serverless":true,
25+
"target":"team_eng_deco"
26+
},
27+
"state":"IDLE"
28+
}

0 commit comments

Comments
 (0)