Skip to content

Commit f2ecb5c

Browse files
arpitjasa-dbnkvuong
arpitjasa-db
and
nkvuong
authored
Rename lakehouse monitor to quality monitor (#3584)
Deprecate resource instead of breaking rename fix Apply suggestions from code review Co-authored-by: vuong-nguyen <[email protected]>
1 parent 2a0ea25 commit f2ecb5c

File tree

9 files changed

+298
-146
lines changed

9 files changed

+298
-146
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
| [databricks_permissions](docs/resources/permissions.md)
6565
| [databricks_pipeline](docs/resources/pipeline.md)
6666
| [databricks_pipelines](docs/data-sources/pipelines.md) data
67+
| [databricks_quality_monitor](docs/resources/quality_monitor.md)
6768
| [databricks_repo](docs/resources/repo.md)
6869
| [databricks_schema](docs/resources/schema.md)
6970
| [databricks_schemas](docs/data-sources/schema.md) data

‎catalog/resource_lakehouse_monitor.go‎

Lines changed: 3 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -1,120 +1,11 @@
11
package catalog
22

33
import (
4-
"context"
5-
"fmt"
6-
"time"
7-
8-
"github.com/databricks/databricks-sdk-go"
9-
"github.com/databricks/databricks-sdk-go/service/catalog"
104
"github.com/databricks/terraform-provider-databricks/common"
11-
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry"
12-
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
135
)
146

15-
const lakehouseMonitorDefaultProvisionTimeout = 15 * time.Minute
16-
17-
func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorName string) error {
18-
return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError {
19-
endpoint, err := w.QualityMonitors.GetByTableName(ctx, monitorName)
20-
if err != nil {
21-
return retry.NonRetryableError(err)
22-
}
23-
24-
switch endpoint.Status {
25-
case catalog.MonitorInfoStatusMonitorStatusActive:
26-
return nil
27-
case catalog.MonitorInfoStatusMonitorStatusError, catalog.MonitorInfoStatusMonitorStatusFailed:
28-
return retry.NonRetryableError(fmt.Errorf("monitor status retrund %s for monitor: %s", endpoint.Status, monitorName))
29-
}
30-
return retry.RetryableError(fmt.Errorf("monitor %s is still pending", monitorName))
31-
})
32-
}
33-
347
func ResourceLakehouseMonitor() common.Resource {
35-
monitorSchema := common.StructToSchema(
36-
catalog.MonitorInfo{},
37-
func(m map[string]*schema.Schema) map[string]*schema.Schema {
38-
common.CustomizeSchemaPath(m, "assets_dir").SetRequired()
39-
common.CustomizeSchemaPath(m, "output_schema_name").SetRequired()
40-
common.CustomizeSchemaPath(m, "table_name").SetRequired()
41-
common.CustomizeSchemaPath(m).AddNewField("skip_builtin_dashboard", &schema.Schema{
42-
Type: schema.TypeBool,
43-
Optional: true,
44-
Required: false,
45-
})
46-
common.CustomizeSchemaPath(m).AddNewField("warehouse_id", &schema.Schema{
47-
Type: schema.TypeString,
48-
Optional: true,
49-
Required: false,
50-
})
51-
common.CustomizeSchemaPath(m, "monitor_version").SetReadOnly()
52-
common.CustomizeSchemaPath(m, "drift_metrics_table_name").SetReadOnly()
53-
common.CustomizeSchemaPath(m, "profile_metrics_table_name").SetReadOnly()
54-
common.CustomizeSchemaPath(m, "status").SetReadOnly()
55-
common.CustomizeSchemaPath(m, "dashboard_id").SetReadOnly()
56-
return m
57-
},
58-
)
59-
60-
return common.Resource{
61-
Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
62-
w, err := c.WorkspaceClient()
63-
if err != nil {
64-
return err
65-
}
66-
67-
var create catalog.CreateMonitor
68-
common.DataToStructPointer(d, monitorSchema, &create)
69-
create.TableName = d.Get("table_name").(string)
70-
71-
endpoint, err := w.QualityMonitors.Create(ctx, create)
72-
if err != nil {
73-
return err
74-
}
75-
err = WaitForMonitor(w, ctx, create.TableName)
76-
if err != nil {
77-
return err
78-
}
79-
d.SetId(endpoint.TableName)
80-
return nil
81-
},
82-
Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
83-
w, err := c.WorkspaceClient()
84-
if err != nil {
85-
return err
86-
}
87-
endpoint, err := w.QualityMonitors.GetByTableName(ctx, d.Id())
88-
if err != nil {
89-
return err
90-
91-
}
92-
return common.StructToData(endpoint, monitorSchema, d)
93-
},
94-
Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
95-
w, err := c.WorkspaceClient()
96-
if err != nil {
97-
return err
98-
}
99-
var update catalog.UpdateMonitor
100-
common.DataToStructPointer(d, monitorSchema, &update)
101-
update.TableName = d.Get("table_name").(string)
102-
_, err = w.QualityMonitors.Update(ctx, update)
103-
if err != nil {
104-
return err
105-
}
106-
return WaitForMonitor(w, ctx, update.TableName)
107-
},
108-
Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
109-
w, err := c.WorkspaceClient()
110-
if err != nil {
111-
return err
112-
}
113-
return w.QualityMonitors.DeleteByTableName(ctx, d.Id())
114-
},
115-
Schema: monitorSchema,
116-
Timeouts: &schema.ResourceTimeout{
117-
Create: schema.DefaultTimeout(lakehouseMonitorDefaultProvisionTimeout),
118-
},
119-
}
8+
r := ResourceQualityMonitor()
9+
r.DeprecationMessage = "Use `databricks_quality_monitor` instead."
10+
return r
12011
}

‎catalog/resource_online_table.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717
const onlineTableDefaultProvisionTimeout = 45 * time.Minute
1818

1919
func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error {
20-
return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError {
20+
return retry.RetryContext(ctx, onlineTableDefaultProvisionTimeout, func() *retry.RetryError {
2121
endpoint, err := w.OnlineTables.GetByName(ctx, onlineTableName)
2222
if err != nil {
2323
return retry.NonRetryableError(err)
@@ -40,7 +40,7 @@ func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Conte
4040
}
4141

4242
func waitForOnlineTableDeletion(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error {
43-
return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError {
43+
return retry.RetryContext(ctx, onlineTableDefaultProvisionTimeout, func() *retry.RetryError {
4444
_, err := w.OnlineTables.GetByName(ctx, onlineTableName)
4545
if err == nil {
4646
return retry.RetryableError(fmt.Errorf("online table %s is still not deleted", onlineTableName))

‎catalog/resource_quality_monitor.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package catalog
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
"github.com/databricks/databricks-sdk-go"
9+
"github.com/databricks/databricks-sdk-go/service/catalog"
10+
"github.com/databricks/terraform-provider-databricks/common"
11+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry"
12+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
13+
)
14+
15+
const qualityMonitorDefaultProvisionTimeout = 15 * time.Minute
16+
17+
func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorName string) error {
18+
return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError {
19+
endpoint, err := w.QualityMonitors.GetByTableName(ctx, monitorName)
20+
if err != nil {
21+
return retry.NonRetryableError(err)
22+
}
23+
24+
switch endpoint.Status {
25+
case catalog.MonitorInfoStatusMonitorStatusActive:
26+
return nil
27+
case catalog.MonitorInfoStatusMonitorStatusError, catalog.MonitorInfoStatusMonitorStatusFailed:
28+
return retry.NonRetryableError(fmt.Errorf("monitor status retrund %s for monitor: %s", endpoint.Status, monitorName))
29+
}
30+
return retry.RetryableError(fmt.Errorf("monitor %s is still pending", monitorName))
31+
})
32+
}
33+
34+
func ResourceQualityMonitor() common.Resource {
35+
monitorSchema := common.StructToSchema(
36+
catalog.MonitorInfo{},
37+
func(m map[string]*schema.Schema) map[string]*schema.Schema {
38+
common.CustomizeSchemaPath(m, "assets_dir").SetRequired()
39+
common.CustomizeSchemaPath(m, "output_schema_name").SetRequired()
40+
common.CustomizeSchemaPath(m, "table_name").SetRequired()
41+
common.CustomizeSchemaPath(m).AddNewField("skip_builtin_dashboard", &schema.Schema{
42+
Type: schema.TypeBool,
43+
Optional: true,
44+
Required: false,
45+
})
46+
common.CustomizeSchemaPath(m).AddNewField("warehouse_id", &schema.Schema{
47+
Type: schema.TypeString,
48+
Optional: true,
49+
Required: false,
50+
})
51+
common.CustomizeSchemaPath(m, "monitor_version").SetReadOnly()
52+
common.CustomizeSchemaPath(m, "drift_metrics_table_name").SetReadOnly()
53+
common.CustomizeSchemaPath(m, "profile_metrics_table_name").SetReadOnly()
54+
common.CustomizeSchemaPath(m, "status").SetReadOnly()
55+
common.CustomizeSchemaPath(m, "dashboard_id").SetReadOnly()
56+
return m
57+
},
58+
)
59+
60+
return common.Resource{
61+
Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
62+
w, err := c.WorkspaceClient()
63+
if err != nil {
64+
return err
65+
}
66+
67+
var create catalog.CreateMonitor
68+
common.DataToStructPointer(d, monitorSchema, &create)
69+
create.TableName = d.Get("table_name").(string)
70+
71+
endpoint, err := w.QualityMonitors.Create(ctx, create)
72+
if err != nil {
73+
return err
74+
}
75+
err = WaitForMonitor(w, ctx, create.TableName)
76+
if err != nil {
77+
return err
78+
}
79+
d.SetId(endpoint.TableName)
80+
return nil
81+
},
82+
Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
83+
w, err := c.WorkspaceClient()
84+
if err != nil {
85+
return err
86+
}
87+
endpoint, err := w.QualityMonitors.GetByTableName(ctx, d.Id())
88+
if err != nil {
89+
return err
90+
91+
}
92+
return common.StructToData(endpoint, monitorSchema, d)
93+
},
94+
Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
95+
w, err := c.WorkspaceClient()
96+
if err != nil {
97+
return err
98+
}
99+
var update catalog.UpdateMonitor
100+
common.DataToStructPointer(d, monitorSchema, &update)
101+
update.TableName = d.Get("table_name").(string)
102+
_, err = w.QualityMonitors.Update(ctx, update)
103+
if err != nil {
104+
return err
105+
}
106+
return WaitForMonitor(w, ctx, update.TableName)
107+
},
108+
Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
109+
w, err := c.WorkspaceClient()
110+
if err != nil {
111+
return err
112+
}
113+
return w.QualityMonitors.DeleteByTableName(ctx, d.Id())
114+
},
115+
Schema: monitorSchema,
116+
Timeouts: &schema.ResourceTimeout{
117+
Create: schema.DefaultTimeout(qualityMonitorDefaultProvisionTimeout),
118+
},
119+
}
120+
}

‎catalog/resource_lakehouse_monitor_test.go‎ renamed to ‎catalog/resource_quality_monitor_test.go‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ import (
99
"github.com/stretchr/testify/mock"
1010
)
1111

12-
func TestLakehouseMonitorCornerCases(t *testing.T) {
13-
qa.ResourceCornerCases(t, ResourceLakehouseMonitor())
12+
func TestQualityMonitorCornerCases(t *testing.T) {
13+
qa.ResourceCornerCases(t, ResourceQualityMonitor())
1414
}
1515

16-
func TestLakehouseMonitorCreateTimeseries(t *testing.T) {
16+
func TestQualityMonitorCreateTimeseries(t *testing.T) {
1717
qa.ResourceFixture{
1818
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
1919
e := w.GetMockQualityMonitorsAPI().EXPECT()
@@ -40,7 +40,7 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) {
4040
DriftMetricsTableName: "test_table_drift",
4141
}, nil)
4242
},
43-
Resource: ResourceLakehouseMonitor(),
43+
Resource: ResourceQualityMonitor(),
4444
HCL: `
4545
table_name = "test_table",
4646
assets_dir = "sample.dir",
@@ -54,7 +54,7 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) {
5454
}.ApplyNoError(t)
5555
}
5656

57-
func TestLakehouseMonitorCreateInference(t *testing.T) {
57+
func TestQualityMonitorCreateInference(t *testing.T) {
5858
qa.ResourceFixture{
5959
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
6060
e := w.GetMockQualityMonitorsAPI().EXPECT()
@@ -89,7 +89,7 @@ func TestLakehouseMonitorCreateInference(t *testing.T) {
8989
},
9090
}, nil)
9191
},
92-
Resource: ResourceLakehouseMonitor(),
92+
Resource: ResourceQualityMonitor(),
9393
HCL: `
9494
table_name = "test_table",
9595
assets_dir = "sample.dir",
@@ -106,7 +106,7 @@ func TestLakehouseMonitorCreateInference(t *testing.T) {
106106
}.ApplyNoError(t)
107107
}
108108

109-
func TestLakehouseMonitorCreateSnapshot(t *testing.T) {
109+
func TestQualityMonitorCreateSnapshot(t *testing.T) {
110110
qa.ResourceFixture{
111111
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
112112
e := w.GetMockQualityMonitorsAPI().EXPECT()
@@ -129,7 +129,7 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) {
129129
Snapshot: &catalog.MonitorSnapshot{},
130130
}, nil)
131131
},
132-
Resource: ResourceLakehouseMonitor(),
132+
Resource: ResourceQualityMonitor(),
133133
HCL: `
134134
table_name = "test_table",
135135
assets_dir = "sample.dir",
@@ -140,7 +140,7 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) {
140140
}.ApplyNoError(t)
141141
}
142142

143-
func TestLakehouseMonitorGet(t *testing.T) {
143+
func TestQualityMonitorGet(t *testing.T) {
144144
qa.ResourceFixture{
145145
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
146146
e := w.GetMockQualityMonitorsAPI().EXPECT()
@@ -157,13 +157,13 @@ func TestLakehouseMonitorGet(t *testing.T) {
157157
},
158158
DriftMetricsTableName: "test_table_drift"}, nil)
159159
},
160-
Resource: ResourceLakehouseMonitor(),
160+
Resource: ResourceQualityMonitor(),
161161
Read: true,
162162
ID: "test_table",
163163
}.ApplyNoError(t)
164164
}
165165

166-
func TestLakehouseMonitorUpdate(t *testing.T) {
166+
func TestQualityMonitorUpdate(t *testing.T) {
167167
qa.ResourceFixture{
168168
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
169169
e := w.GetMockQualityMonitorsAPI().EXPECT()
@@ -204,7 +204,7 @@ func TestLakehouseMonitorUpdate(t *testing.T) {
204204
DriftMetricsTableName: "test_table_drift",
205205
}, nil)
206206
},
207-
Resource: ResourceLakehouseMonitor(),
207+
Resource: ResourceQualityMonitor(),
208208
Update: true,
209209
ID: "test_table",
210210
InstanceState: map[string]string{
@@ -225,13 +225,13 @@ func TestLakehouseMonitorUpdate(t *testing.T) {
225225
}.ApplyNoError(t)
226226
}
227227

228-
func TestLakehouseMonitorDelete(t *testing.T) {
228+
func TestQualityMonitorDelete(t *testing.T) {
229229
qa.ResourceFixture{
230230
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
231231
e := w.GetMockQualityMonitorsAPI().EXPECT()
232232
e.DeleteByTableName(mock.Anything, "test_table").Return(nil)
233233
},
234-
Resource: ResourceLakehouseMonitor(),
234+
Resource: ResourceQualityMonitor(),
235235
Delete: true,
236236
ID: "test_table",
237237
}.ApplyNoError(t)

‎docs/resources/lakehouse_monitor.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ subcategory: "Unity Catalog"
33
---
44
# databricks_lakehouse_monitor Resource
55

6+
NOTE: This resource has been deprecated and will be removed soon. Please use the [databricks_quality_monitor resource](./quality_monitor.md) instead.
7+
68
This resource allows you to manage [Lakehouse Monitors](https://docs.databricks.com/en/lakehouse-monitoring/index.html) in Databricks.
79

810
A `databricks_lakehouse_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference.

0 commit comments

Comments
 (0)