Extend dataset_catalog and dataset_schema support to Terraform engine

pietern · pietern · commit 3f8b827f1547 · 2025-12-17T13:24:42.000+01:00
- Add Terraform conversion test for dataset_catalog and dataset_schema
- Update acceptance tests to run for both direct and terraform engines
- Fix direct engine to skip write-only fields in remote diff computation
- Add engine-specific expected output files for requests and plans
- Update test script to generate engine-specific request files
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json
@@ -1,9 +1,45 @@
 {
-  "all_resources": [
-    {
-      "resource_type": "databricks_dashboard",
-      "resource_key": "dashboard1",
-      "action": "noop"
+  "plan_version": 1,
+  "cli_version": "[DEV_VERSION]",
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "skip",
+      "remote_state": {
+        "create_time": "[TIMESTAMP]",
+        "dashboard_id": "[DASHBOARD_ID]",
+        "display_name": "test bundle-deploy-dashboard-dataset [UUID]",
+        "embed_credentials": true,
+        "etag": [ETAG],
+        "lifecycle_state": "ACTIVE",
+        "parent_path": "/Workspace/Users/[USERNAME]",
+        "path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json",
+        "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n",
+        "update_time": "[TIMESTAMP]",
+        "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+      },
+      "changes": {
+        "local": {
+          "etag": {
+            "action": "skip",
+            "old": [ETAG]
+          }
+        },
+        "remote": {
+          "dataset_catalog": {
+            "action": "skip",
+            "old": "main"
+          },
+          "dataset_schema": {
+            "action": "skip",
+            "old": "default"
+          },
+          "serialized_dashboard": {
+            "action": "skip",
+            "old": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n",
+            "new": "{\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
+          }
+        }
+      }
     }
-  ]
+  }
 }
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.terraform.json b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.terraform.json
@@ -0,0 +1,9 @@
+{
+  "plan_version": 1,
+  "cli_version": "[DEV_VERSION]",
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "skip"
+    }
+  }
+}
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.direct.txt b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.direct.txt
@@ -0,0 +1,14 @@
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards",
+  "q": {
+    "dataset_catalog": "main",
+    "dataset_schema": "default"
+  },
+  "body": {
+    "display_name": "test bundle-deploy-dashboard-dataset [UUID]",
+    "parent_path": "/Workspace/Users/[USERNAME]",
+    "serialized_dashboard": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n",
+    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+  }
+}
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.terraform.txt b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.terraform.txt
@@ -0,0 +1,14 @@
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards",
+  "q": {
+    "dataset_catalog": "main",
+    "dataset_schema": "default"
+  },
+  "body": {
+    "display_name": "test bundle-deploy-dashboard-dataset [UUID]",
+    "parent_path": "/Workspace/Users/[USERNAME]",
+    "serialized_dashboard": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n",
+    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+  }
+}
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.txt b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.post.requests.txt
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/output.txt b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/output.txt
@@ -10,7 +10,7 @@ Deployment complete!
   "lifecycle_state": "ACTIVE",
   "parent_path": "/Users/[USERNAME]",
   "path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json",
-  "serialized_dashboard": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n"
+  "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
 }
 
 >>> [CLI] bundle plan -o json
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script
@@ -31,4 +31,4 @@ cat out.requests.txt | \
  jq 'select(.method == "POST")' | \
  jq 'select(.path | contains("/api/2.0/lakeview/dashboards"))' | \
  jq 'select(.path | contains("/published") | not)' \
- > out.post.requests.txt
+ > out.post.requests.$DATABRICKS_BUNDLE_ENGINE.txt
diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml
@@ -2,10 +2,10 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = true
+EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE"
 
-# TODO: test terraform once commit had been released (https://github.com/databricks/terraform-provider-databricks/pull/5259)
 [EnvMatrix]
-  DATABRICKS_BUNDLE_ENGINE = ["direct"]
+  DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]
 
 Ignore = [
     "databricks.yml",
diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go
@@ -32,6 +32,7 @@ func GenerateMaterializedConfig(config TestConfig) (string, error) {
 		RequiresUnityCatalog: config.RequiresUnityCatalog,
 		RequiresCluster:      config.RequiresCluster,
 		RequiresWarehouse:    config.RequiresWarehouse,
+		RecordRequests:       config.RecordRequests,
 		EnvMatrix:            config.EnvMatrix,
 	}
 
diff --git a/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go b/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go
@@ -109,3 +109,34 @@ func TestConvertDashboardSerializedDashboardAny(t *testing.T) {
 	// Assert that the "file_path" is dropped.
 	assert.NotContains(t, out.Dashboard["my_dashboard"], "file_path")
 }
+
+func TestConvertDashboardDatasetCatalogSchema(t *testing.T) {
+	src := resources.Dashboard{
+		DashboardConfig: resources.DashboardConfig{
+			DisplayName:      "my dashboard",
+			WarehouseId:      "f00dcafe",
+			ParentPath:       "/some/path",
+			DatasetCatalog:   "main",
+			DatasetSchema:    "default",
+			EmbedCredentials: true,
+		},
+	}
+
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
+	require.NoError(t, err)
+
+	// Assert that dataset_catalog and dataset_schema are included.
+	assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{
+		"display_name":      "my dashboard",
+		"warehouse_id":      "f00dcafe",
+		"parent_path":       "/some/path",
+		"dataset_catalog":   "main",
+		"dataset_schema":    "default",
+		"embed_credentials": true,
+	})
+}
diff --git a/bundle/direct/dresources/dashboard.go b/bundle/direct/dresources/dashboard.go
@@ -124,8 +124,8 @@ func (r *ResourceDashboard) DoRead(ctx context.Context, id string) (*resources.D
 		SerializedDashboard: dashboard.SerializedDashboard,
 		ParentPath:          ensureWorkspacePrefix(dashboard.ParentPath),
 		// diffs are detected via etags, which will change if dataset_catalog/dataset_schema is updated.
-		DatasetCatalog:      "",
-		DatasetSchema:       "",
+		DatasetCatalog: "",
+		DatasetSchema:  "",
 
 		// Output only fields.
 		CreateTime:      dashboard.CreateTime,
@@ -321,6 +321,12 @@ func (*ResourceDashboard) FieldTriggers(isLocal bool) map[string]deployplan.Acti
 		// "serialized_dashboard" locally and remotely will have different diffs.
 		// We only need to rely on etag here, and can skip this field for diff computation.
 		triggers["serialized_dashboard"] = deployplan.ActionTypeSkip
+
+		// "dataset_catalog" and "dataset_schema" are write-only fields that are not returned by the server.
+		// They will always differ between local config (which has values) and remote state (which has empty strings),
+		// so we skip them for remote diff computation to avoid false positives.
+		triggers["dataset_catalog"] = deployplan.ActionTypeSkip
+		triggers["dataset_schema"] = deployplan.ActionTypeSkip
 	}
 
 	return triggers

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ Deployment complete!`
`10`	`10`	`"lifecycle_state": "ACTIVE",`
`11`	`11`	`"parent_path": "/Users/[USERNAME]",`
`12`	`12`	`"path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json",`
`13`		`- "serialized_dashboard": "{\"pages\":[{\"name\":\"test_page\",\"displayName\":\"Test Page\"}]}\n"`
	`13`	`+ "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"`
`14`	`14`	`}`
`15`	`15`
`16`	`16`	`>>> [CLI] bundle plan -o json`
Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ func GenerateMaterializedConfig(config TestConfig) (string, error) {`
`32`	`32`	`RequiresUnityCatalog: config.RequiresUnityCatalog,`
`33`	`33`	`RequiresCluster: config.RequiresCluster,`
`34`	`34`	`RequiresWarehouse: config.RequiresWarehouse,`
	`35`	`+ RecordRequests: config.RecordRequests,`
`35`	`36`	`EnvMatrix: config.EnvMatrix,`
`36`	`37`	`}`
`37`	`38`