[Fix] Fix filling of missing attributes in databricks_quality_monitor (#4519)

## Changes
<!-- Summary of your changes that are easy to understand -->

The quality monitor Get API doesn't return attributes `warehouse_id` and
`skip_builtin_dashboard` set by the Create API, leading to configuration
drift and errors. We can backfill them from the existing state.

Resolves #4518

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [ ] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] using Go SDK
- [ ] using TF Plugin Framework

Author: alexott

NEXT_CHANGELOG.md

@@ -13,6 +13,7 @@
  * Only allow `authorized_paths` to be updated in the `options` field of `databricks_catalog` ([#4517](https://github.com/databricks/terraform-provider-databricks/pull/4517)).
  * Mark `default_catalog_name` attribute in `databricks_metastore_assignment` as deprecated ([#4522](https://github.com/databricks/terraform-provider-databricks/pull/4522))
  * Delete `databricks_sql_endpoint` that failed to start ([#4520](https://github.com/databricks/terraform-provider-databricks/pull/4520))
+ * Fix filling of missing attributes in `databricks_quality_monitor` ([#4519](https://github.com/databricks/terraform-provider-databricks/pull/4519)).
 
 ### Documentation
 
@@ -25,4 +26,4 @@
 
 ### Internal Changes
 
- * Bump golang version to 1.24.0 ([#4508](https://github.com/databricks/terraform-provider-databricks/pull/4508)).
+ * Bump golang version to 1.24.0 ([#4508](https://github.com/databricks/terraform-provider-databricks/pull/4508)).

catalog/quality_monitor_test.go

@@ -32,6 +32,7 @@ resource "databricks_sql_table" "myInferenceTable" {
 	name = "bar{var.STICKY_RANDOM}_inference"
 	table_type = "MANAGED"
 	data_source_format = "DELTA"
+	warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 	
 	column {
 		name = "model_id"
@@ -79,6 +80,7 @@ func TestUcAccQualityMonitor(t *testing.T) {
 				name = "bar{var.STICKY_RANDOM}_timeseries"
 				table_type = "MANAGED"
 				data_source_format = "DELTA"
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 
 				column {
 					name = "timestamp"
@@ -106,6 +108,7 @@ func TestUcAccQualityMonitor(t *testing.T) {
 				name = "bar{var.STICKY_RANDOM}_snapshot"
 				table_type = "MANAGED"
 				data_source_format = "DELTA"
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 
 				column {
 					name = "timestamp"

catalog/resource_quality_monitor.go

@@ -90,7 +90,18 @@ func ResourceQualityMonitor() common.Resource {
 				return err
 
 			}
-			return common.StructToData(endpoint, monitorSchema, d)
+			oldWarehouseId := d.Get("warehouse_id").(string)
+			oldSkipBuiltinDashboard := d.Get("skip_builtin_dashboard").(bool)
+
+			err = common.StructToData(endpoint, monitorSchema, d)
+			if err != nil {
+				return err
+			}
+			// we need this because Get API doesn't return that information
+			d.Set("warehouse_id", oldWarehouseId)
+			d.Set("skip_builtin_dashboard", oldSkipBuiltinDashboard)
+
+			return nil
 		},
 		Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
 			w, err := c.WorkspaceClient()

catalog/resource_quality_monitor_test.go

@@ -18,9 +18,11 @@ func TestQualityMonitorCreateTimeseries(t *testing.T) {
 		MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
 			e := w.GetMockQualityMonitorsAPI().EXPECT()
 			e.Create(mock.Anything, catalog.CreateMonitor{
-				TableName:        "test_table",
-				OutputSchemaName: "output.schema",
-				AssetsDir:        "sample.dir",
+				TableName:            "test_table",
+				OutputSchemaName:     "output.schema",
+				AssetsDir:            "sample.dir",
+				WarehouseId:          "1234",
+				SkipBuiltinDashboard: true,
 				TimeSeries: &catalog.MonitorTimeSeries{
 					Granularities: []string{"1 day"},
 					TimestampCol:  "timestamp",
@@ -45,13 +47,18 @@ func TestQualityMonitorCreateTimeseries(t *testing.T) {
 			table_name = "test_table",
 			assets_dir = "sample.dir",
 			output_schema_name = "output.schema",
+			warehouse_id = "1234",
+			skip_builtin_dashboard = true,
 			time_series = {
 				granularities = ["1 day"],
 				timestamp_col = "timestamp"
 			} 
 		`,
 		Create: true,
-	}.ApplyNoError(t)
+	}.ApplyAndExpectData(t, map[string]any{
+		"warehouse_id":           "1234",
+		"skip_builtin_dashboard": true,
+	})
 }
 
 func TestQualityMonitorCreateInference(t *testing.T) {

docs/resources/quality_monitor.md

@@ -5,9 +5,10 @@ subcategory: "Unity Catalog"
 
 This resource allows you to manage [Lakehouse Monitors](https://docs.databricks.com/en/lakehouse-monitoring/index.html) in Databricks. 
 
-A `databricks_quality_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference. 
+A `databricks_quality_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference.
 
 ## Plugin Framework Migration
+
 The quality monitor resource has been migrated from sdkv2 to plugin framework。 If you encounter any problem with this resource and suspect it is due to the migration, you can fallback to sdkv2 by setting the environment variable in the following way `export USE_SDK_V2_RESOURCES="databricks_quality_monitor"`.
 
 ## Example Usage
@@ -114,9 +115,9 @@ table.
 * `schedule` - The schedule for automatically updating and refreshing metric tables.  This block consists of following fields:
     * `quartz_cron_expression` - string expression that determines when to run the monitor. See [Quartz documentation](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for examples.
     * `timezone_id` - string with timezone id (e.g., `PST`) in which to evaluate the Quartz expression.
-* `skip_builtin_dashboard` - Whether to skip creating a default dashboard summarizing data quality metrics.
+* `skip_builtin_dashboard` - Whether to skip creating a default dashboard summarizing data quality metrics.  (Can't be updated after creation).
 * `slicing_exprs` - List of column expressions to slice data with for targeted analysis. The data is grouped by each expression independently, resulting in a separate slice for each predicate and its complements. For high-cardinality columns, only the top 100 unique values by frequency will generate slices.
-* `warehouse_id` - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used.
+* `warehouse_id` - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used.  (Can't be updated after creation)
 
 ## Attribute Reference
 

internal/providers/pluginfw/products/qualitymonitor/resource_quality_monitor.go

@@ -146,6 +146,9 @@ func (r *QualityMonitorResource) Create(ctx context.Context, req resource.Create
 
 	// Set the ID to the table name
 	newMonitorInfoTfSDK.ID = newMonitorInfoTfSDK.TableName
+	// We need it to fill additional fields as they are not returned by the API
+	newMonitorInfoTfSDK.WarehouseId = monitorInfoTfSDK.WarehouseId
+	newMonitorInfoTfSDK.SkipBuiltinDashboard = monitorInfoTfSDK.SkipBuiltinDashboard
 
 	resp.Diagnostics.Append(resp.State.Set(ctx, newMonitorInfoTfSDK)...)
 }
@@ -179,6 +182,20 @@ func (r *QualityMonitorResource) Read(ctx context.Context, req resource.ReadRequ
 	}
 
 	monitorInfoTfSDK.ID = monitorInfoTfSDK.TableName
+	// We need it to fill additional fields as they are not returned by the API
+	var origWarehouseId types.String
+	var origSkipBuiltinDashboard types.Bool
+	resp.Diagnostics.Append(req.State.GetAttribute(ctx, path.Root("warehouse_id"), &origWarehouseId)...)
+	resp.Diagnostics.Append(req.State.GetAttribute(ctx, path.Root("skip_builtin_dashboard"), &origSkipBuiltinDashboard)...)
+	if resp.Diagnostics.HasError() {
+		return
+	}
+	if origWarehouseId.ValueString() != "" {
+		monitorInfoTfSDK.WarehouseId = origWarehouseId
+	}
+	if origSkipBuiltinDashboard.ValueBool() {
+		monitorInfoTfSDK.SkipBuiltinDashboard = origSkipBuiltinDashboard
+	}
 
 	resp.Diagnostics.Append(resp.State.Set(ctx, monitorInfoTfSDK)...)
 }
@@ -225,6 +242,12 @@ func (r *QualityMonitorResource) Update(ctx context.Context, req resource.Update
 	if resp.Diagnostics.HasError() {
 		return
 	}
+	// We need it to fill additional fields as they are not returned by the API
+	resp.Diagnostics.Append(req.State.GetAttribute(ctx, path.Root("warehouse_id"), &newMonitorInfoTfSDK.WarehouseId)...)
+	resp.Diagnostics.Append(req.State.GetAttribute(ctx, path.Root("skip_builtin_dashboard"), &newMonitorInfoTfSDK.SkipBuiltinDashboard)...)
+	if resp.Diagnostics.HasError() {
+		return
+	}
 
 	resp.Diagnostics.Append(resp.State.Set(ctx, newMonitorInfoTfSDK)...)
 }

internal/providers/pluginfw/products/qualitymonitor/resource_quality_monitor_acc_test.go

@@ -35,6 +35,7 @@ resource "databricks_sql_table" "myInferenceTable" {
 	name = "bar{var.STICKY_RANDOM}_inference"
 	table_type = "MANAGED"
 	data_source_format = "DELTA"
+	warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 	
 	column {
 		name = "model_id"
@@ -63,6 +64,7 @@ func TestUcAccQualityMonitor(t *testing.T) {
 				table_name = databricks_sql_table.myInferenceTable.id
 				assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}"
 				output_schema_name = databricks_schema.things.id
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 				inference_log {
 				  granularities = ["1 day"]
 				  timestamp_col = "timestamp"
@@ -78,6 +80,7 @@ func TestUcAccQualityMonitor(t *testing.T) {
 				name = "bar{var.STICKY_RANDOM}_timeseries"
 				table_type = "MANAGED"
 				data_source_format = "DELTA"
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 
 				column {
 					name = "timestamp"
@@ -89,6 +92,7 @@ func TestUcAccQualityMonitor(t *testing.T) {
 				table_name = databricks_sql_table.myTimeseries.id
 				assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}"
 				output_schema_name = databricks_schema.things.id
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 				time_series {
 				  granularities = ["1 day"]
 				  timestamp_col = "timestamp"
@@ -101,6 +105,7 @@ func TestUcAccQualityMonitor(t *testing.T) {
 				name = "bar{var.STICKY_RANDOM}_snapshot"
 				table_type = "MANAGED"
 				data_source_format = "DELTA"
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 
 				column {
 					name = "timestamp"
@@ -112,6 +117,7 @@ func TestUcAccQualityMonitor(t *testing.T) {
 				table_name = databricks_sql_table.mySnapshot.id
 				assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}"
 				output_schema_name = databricks_schema.things.id
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 				snapshot {
 				} 
 			}
@@ -129,6 +135,7 @@ func TestUcAccUpdateQualityMonitor(t *testing.T) {
 				table_name = databricks_sql_table.myInferenceTable.id
 				assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}"
 				output_schema_name = databricks_schema.things.id
+				warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 				inference_log {
 				  granularities = ["1 day"]
 				  timestamp_col = "timestamp"
@@ -144,6 +151,7 @@ func TestUcAccUpdateQualityMonitor(t *testing.T) {
 			table_name = databricks_sql_table.myInferenceTable.id
 			assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}"
 			output_schema_name = databricks_schema.things.id
+			warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 			inference_log {
 				granularities = ["1 hour"]
 				timestamp_col = "timestamp"
@@ -228,6 +236,7 @@ func TestUcAccUpdateQualityMonitorTransitionFromPluginFw(t *testing.T) {
 			table_name = databricks_sql_table.myInferenceTable.id
 			assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}"
 			output_schema_name = databricks_schema.things.id
+			warehouse_id = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
 			inference_log {
 				granularities = ["1 hour"]
 				timestamp_col = "timestamp"