Skip to content

Commit 2e743dc

Browse files
pieternpietern
committed
Test the catalog/schema override e2e
1 parent 7d5b7dd commit 2e743dc

File tree

8 files changed

+74
-12
lines changed

8 files changed

+74
-12
lines changed

‎acceptance/bundle/refschema/out.fields.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,8 @@ resources.clusters.*.permissions.permissions[*].service_principal_name string AL
424424
resources.clusters.*.permissions.permissions[*].user_name string ALL
425425
resources.dashboards.*.create_time string ALL
426426
resources.dashboards.*.dashboard_id string ALL
427+
resources.dashboards.*.dataset_catalog string ALL
428+
resources.dashboards.*.dataset_schema string ALL
427429
resources.dashboards.*.display_name string ALL
428430
resources.dashboards.*.embed_credentials bool ALL
429431
resources.dashboards.*.etag string ALL

‎acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
"lifecycle_state": "ACTIVE",
1414
"parent_path": "/Workspace/Users/[USERNAME]",
1515
"path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json",
16-
"serialized_dashboard": "{\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n",
16+
"serialized_dashboard": "[SERIALIZED_FIXTURE_NEW]",
1717
"update_time": "[TIMESTAMP]",
1818
"warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
1919
},
@@ -35,8 +35,8 @@
3535
},
3636
"serialized_dashboard": {
3737
"action": "skip",
38-
"old": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n",
39-
"new": "{\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
38+
"old": "[SERIALIZED_FIXTURE_OLD]",
39+
"new": "[SERIALIZED_FIXTURE_NEW]"
4040
}
4141
}
4242
}

‎acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.direct.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"body": {
99
"display_name": "test bundle-deploy-dashboard-dataset [UUID]",
1010
"parent_path": "/Workspace/Users/[USERNAME]",
11-
"serialized_dashboard": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n",
11+
"serialized_dashboard": "{\n \"pages\": [\n {\n \"name\": \"test_page\",\n \"displayName\": \"Test Page\",\n \"pageType\": \"PAGE_TYPE_CANVAS\"\n }\n ],\n \"datasets\": [\n {\n \"name\": \"bf8f76f4\",\n \"displayName\": \"Test Dataset\",\n \"queryLines\": [\n \"SELECT 1\\n\"\n ],\n \"catalog\": \"foobar\",\n \"schema\": \"foobar\"\n }\n ]\n}\n",
1212
"warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
1313
}
1414
}

‎acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.terraform.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"body": {
99
"display_name": "test bundle-deploy-dashboard-dataset [UUID]",
1010
"parent_path": "/Workspace/Users/[USERNAME]",
11-
"serialized_dashboard": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n",
11+
"serialized_dashboard": "{\n \"pages\": [\n {\n \"name\": \"test_page\",\n \"displayName\": \"Test Page\",\n \"pageType\": \"PAGE_TYPE_CANVAS\"\n }\n ],\n \"datasets\": [\n {\n \"name\": \"bf8f76f4\",\n \"displayName\": \"Test Dataset\",\n \"queryLines\": [\n \"SELECT 1\\n\"\n ],\n \"catalog\": \"foobar\",\n \"schema\": \"foobar\"\n }\n ]\n}\n",
1212
"warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
1313
}
1414
}

‎acceptance/bundle/resources/dashboards/dataset-catalog-schema/output.txt‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,13 @@ Deployment complete!
99
{
1010
"lifecycle_state": "ACTIVE",
1111
"parent_path": "/Users/[USERNAME]",
12-
"path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json",
13-
"serialized_dashboard": "{\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
12+
"path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json"
13+
}
14+
15+
>>> [CLI] lakeview get [DASHBOARD_ID]
16+
{
17+
"catalog": "main",
18+
"schema": "default"
1419
}
1520

1621
>>> [CLI] bundle plan -o json

‎acceptance/bundle/resources/dashboards/dataset-catalog-schema/sample-dashboard.lvdash.json‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,20 @@
1-
{"pages":[{"name":"test_page","displayName":"Test Page"}]}
1+
{
2+
"pages": [
3+
{
4+
"name": "test_page",
5+
"displayName": "Test Page",
6+
"pageType": "PAGE_TYPE_CANVAS"
7+
}
8+
],
9+
"datasets": [
10+
{
11+
"name": "bf8f76f4",
12+
"displayName": "Test Dataset",
13+
"queryLines": [
14+
"SELECT 1\n"
15+
],
16+
"catalog": "foobar",
17+
"schema": "foobar"
18+
}
19+
]
20+
}

‎acceptance/bundle/resources/dashboards/dataset-catalog-schema/script‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,26 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
2020
# Capture the dashboard ID as a replacement.
2121
echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
2222

23-
trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
23+
trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path}'
24+
25+
# Verify that the serialized_dashboard datasets have the overridden catalog/schema values.
26+
# The dataset_catalog and dataset_schema parameters should override the values in the datasets.
27+
trace $CLI lakeview get $DASHBOARD_ID | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}'
2428

2529
# Verify that there is no drift right after deploy.
2630
trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
2731

32+
# Modify the direct plan to replace "serialized_dashboard" with a fixture.
33+
# It is normalized on the backend so we cannot compare reliably across local and cloud.
34+
if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then
35+
jq '.plan["resources.dashboards.dashboard1"].remote_state.serialized_dashboard |=
36+
if . then "[SERIALIZED_FIXTURE]" else . end' \
37+
out.plan.direct.json > out.plan.direct.json.tmp && mv out.plan.direct.json.tmp out.plan.direct.json
38+
jq '.plan["resources.dashboards.dashboard1"].changes.remote.serialized_dashboard |=
39+
if . then {"action": .action, "old": "[SERIALIZED_FIXTURE_OLD]", "new": "[SERIALIZED_FIXTURE_NEW]"} else . end' \
40+
out.plan.direct.json > out.plan.direct.json.tmp && mv out.plan.direct.json.tmp out.plan.direct.json
41+
fi
42+
2843
# Print API requests made to create the dashboard.
2944
# This verifies that dataset_catalog and dataset_schema are passed to the API.
3045
cat out.requests.txt | \

‎libs/testserver/dashboards.go‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ func generateDashboardId() (string, error) {
2525
}
2626

2727
// Transform the serialized dashboard to mimic remote behavior.
28-
func transformSerializedDashboard(serializedDashboard string) string {
28+
func transformSerializedDashboard(serializedDashboard, datasetCatalog, datasetSchema string) string {
2929
var dashboardContent map[string]any
3030
err := json.Unmarshal([]byte(serializedDashboard), &dashboardContent)
3131
if err != nil {
@@ -41,6 +41,20 @@ func transformSerializedDashboard(serializedDashboard string) string {
4141
}
4242
}
4343

44+
// Apply dataset_catalog and dataset_schema overrides to all datasets
45+
if datasets, ok := dashboardContent["datasets"].([]any); ok {
46+
for _, dataset := range datasets {
47+
if datasetMap, ok := dataset.(map[string]any); ok {
48+
if datasetCatalog != "" {
49+
datasetMap["catalog"] = datasetCatalog
50+
}
51+
if datasetSchema != "" {
52+
datasetMap["schema"] = datasetSchema
53+
}
54+
}
55+
}
56+
}
57+
4458
updatedContent, err := json.Marshal(dashboardContent)
4559
if err != nil {
4660
return serializedDashboard
@@ -98,9 +112,13 @@ func (s *FakeWorkspace) DashboardCreate(req Request) Response {
98112

99113
inputSerializedDashboard := dashboard.SerializedDashboard
100114

115+
// Extract dataset_catalog and dataset_schema from query parameters
116+
datasetCatalog := req.URL.Query().Get("dataset_catalog")
117+
datasetSchema := req.URL.Query().Get("dataset_schema")
118+
101119
// Parse serializedDashboard into json and put it back as a string
102120
if dashboard.SerializedDashboard != "" {
103-
dashboard.SerializedDashboard = transformSerializedDashboard(dashboard.SerializedDashboard)
121+
dashboard.SerializedDashboard = transformSerializedDashboard(dashboard.SerializedDashboard, datasetCatalog, datasetSchema)
104122
}
105123
dashboard.Etag = "80611980"
106124

@@ -170,7 +188,10 @@ func (s *FakeWorkspace) DashboardUpdate(req Request) Response {
170188
dashboard.Path = path.Join(dir, base)
171189
}
172190
if updateReq.SerializedDashboard != "" {
173-
dashboard.SerializedDashboard = transformSerializedDashboard(updateReq.SerializedDashboard)
191+
// Extract dataset_catalog and dataset_schema from query parameters
192+
datasetCatalog := req.URL.Query().Get("dataset_catalog")
193+
datasetSchema := req.URL.Query().Get("dataset_schema")
194+
dashboard.SerializedDashboard = transformSerializedDashboard(updateReq.SerializedDashboard, datasetCatalog, datasetSchema)
174195
}
175196
if updateReq.WarehouseId != "" {
176197
dashboard.WarehouseId = updateReq.WarehouseId

0 commit comments

Comments
 (0)