[Feature] Add databricks_alert resource to replace databricks_sql_alert (#4051)

alexott · mgyucht · web-flow · commit b9fb47c28f32 · 2024-10-18T08:42:41.000Z
## Changes  The new resource uses the [new Alerts API](https://docs.databricks.com/api/workspace/alerts/create) instead of the legacy one that will be deprecated. Since the new resource has a slightly different set of parameters, it was decided to create a new resource and deprecate the old one. This resource uses old TF SDK to be compatible with TF exporter (until #4050 is implemented). TODOs: - Need to discuss how to handle permissions - `sql_alert` permissions look like working, but not sure if we should continue to use that API - Support in the exporter will be in a separate PR ## Tests  - [x] `make test` run locally - [x] relevant change in `docs/` folder - [x] covered with integration tests in `internal/acceptance` - [x] relevant acceptance tests are passing - [x] using Go SDK --------- Co-authored-by: Miles Yucht <miles@databricks.com>
diff --git a/common/resource.go b/common/resource.go
@@ -443,13 +443,20 @@ func genericDatabricksData[T, P, C any](
 // WorkspacePathPrefixDiffSuppress suppresses diffs for workspace paths where both sides
 // may or may not include the `/Workspace` prefix.
 //
-// This is the case for dashboards where at create time, the user may include the `/Workspace`
+// This is the case for dashboards, alerts and queries where at create time, the user may include the `/Workspace`
 // prefix for the `parent_path` field, but the read response will not include the prefix.
 func WorkspacePathPrefixDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
 	const prefix = "/Workspace"
 	return strings.TrimPrefix(old, prefix) == strings.TrimPrefix(new, prefix)
 }
 
+// WorkspaceOrEmptyPathPrefixDiffSuppress is similar WorkspacePathPrefixDiffSuppress but also suppresses diffs
+// when the new value is empty (not specified by user).
+func WorkspaceOrEmptyPathPrefixDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
+	const prefix = "/Workspace"
+	return (old != "" && new == "") || strings.TrimPrefix(old, prefix) == strings.TrimPrefix(new, prefix)
+}
+
 func EqualFoldDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
 	if strings.EqualFold(old, new) {
 		log.Printf("[INFO] Suppressing diff on %s", k)
diff --git a/common/resource_test.go b/common/resource_test.go
@@ -187,6 +187,15 @@ func TestWorkspacePathPrefixDiffSuppress(t *testing.T) {
 	assert.False(t, WorkspacePathPrefixDiffSuppress("k", "/Workspace/1", "/Workspace/2", nil))
 }
 
+func TestWorkspaceOrEmptyPathPrefixDiffSuppress(t *testing.T) {
+	assert.True(t, WorkspaceOrEmptyPathPrefixDiffSuppress("k", "/Workspace/foo/bar", "/Workspace/foo/bar", nil))
+	assert.True(t, WorkspaceOrEmptyPathPrefixDiffSuppress("k", "/Workspace/foo/bar", "/foo/bar", nil))
+	assert.True(t, WorkspaceOrEmptyPathPrefixDiffSuppress("k", "/foo/bar", "/Workspace/foo/bar", nil))
+	assert.True(t, WorkspaceOrEmptyPathPrefixDiffSuppress("k", "/foo/bar", "/foo/bar", nil))
+	assert.True(t, WorkspaceOrEmptyPathPrefixDiffSuppress("k", "/foo/bar", "", nil))
+	assert.False(t, WorkspaceOrEmptyPathPrefixDiffSuppress("k", "/Workspace/1", "/Workspace/2", nil))
+}
+
 func TestEqualFoldDiffSuppress(t *testing.T) {
 	assert.True(t, EqualFoldDiffSuppress("k", "A", "a", nil))
 	assert.False(t, EqualFoldDiffSuppress("k", "A", "A2", nil))
diff --git a/docs/resources/alert.md b/docs/resources/alert.md
@@ -0,0 +1,196 @@
+---
+subcategory: "Databricks SQL"
+---
+# databricks_alert Resource
+
+This resource allows you to manage [Databricks SQL Alerts](https://docs.databricks.com/en/sql/user/alerts/index.html).  It supersedes [databricks_sql_alert](sql_alert.md) resource - see migration guide below for more details.
+
+## Example Usage
+
+```hcl
+resource "databricks_directory" "shared_dir" {
+  path = "/Shared/Queries"
+}
+
+# This will be replaced with new databricks_query resource
+resource "databricks_sql_query" "this" {
+  data_source_id = databricks_sql_endpoint.example.data_source_id
+  name           = "My Query Name"
+  query          = "SELECT 42 as value"
+  parent         = "folders/${databricks_directory.shared_dir.object_id}"
+}
+
+resource "databricks_alert" "alert" {
+  query_id     = databricks_sql_query.this.id
+  display_name = "TF new alert"
+  parent_path  = databricks_directory.shared_dir.path
+  condition {
+    op = "GREATER_THAN"
+    operand {
+      column {
+        name = "value"
+      }
+    }
+    threshold {
+      value {
+        double_value = 42
+      }
+    }
+  }
+}
+```
+
+## Argument Reference
+
+The following arguments are available:
+
+* `query_id` - (Required, String) ID of the query evaluated by the alert.
+* `display_name` - (Required, String) Name of the alert.
+* `condition` - (Required) Trigger conditions of the alert. Block consists of the following attributes:
+  * `op` - (Required, String Enum) Operator used for comparison in alert evaluation. (Enum: `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL`, `EQUAL`, `NOT_EQUAL`, `IS_NULL`)
+  * `operand` - (Required, Block) Name of the column from the query result to use for comparison in alert evaluation:
+    * `column` - (Required, Block) Block describing the column from the query result to use for comparison in alert evaluation:
+      * `name` - (Required, String) Name of the column.
+  * `threshold` - (Optional for `IS_NULL` operation, Block) Threshold value used for comparison in alert evaluation:
+    * `value` - (Required, Block) actual value used in comparison (one of the attributes is required):
+      * `string_value` - string value to compare against string results.
+      * `double_value` - double value to compare against integer and double results.
+      * `bool_value` - boolean value (`true` or `false`) to compare against boolean results.
+  * `empty_result_state` - (Optional, String Enum) Alert state if the result is empty (`UNKNOWN`, `OK`, `TRIGGERED`)
+* `custom_subject` - (Optional, String) Custom subject of alert notification, if it exists. This includes email subject, Slack notification header, etc. See [Alerts API reference](https://docs.databricks.com/en/sql/user/alerts/index.html) for custom templating instructions.
+* `custom_body` - (Optional, String) Custom body of alert notification, if it exists. See [Alerts API reference](https://docs.databricks.com/en/sql/user/alerts/index.html) for custom templating instructions.
+* `parent_path` - (Optional, String) The path to a workspace folder containing the alert. The default is the user's home folder.  If changed, the alert will be recreated.
+* `seconds_to_retrigger` - (Optional, Integer) Number of seconds an alert must wait after being triggered to rearm itself. After rearming, it can be triggered again. If 0 or not specified, the alert will not be triggered again.
+* `owner_user_name` - (Optional, String) Alert owner's username.
+* `notify_on_ok` - (Optional, Boolean) Whether to notify alert subscribers when alert returns back to normal.
+
+## Attribute Reference
+
+In addition to all the arguments above, the following attributes are exported:
+
+* `id` - unique ID of the Alert.
+* `lifecycle_state` - The workspace state of the alert. Used for tracking trashed status. (Possible values are `ACTIVE` or `TRASHED`).
+* `state` - Current state of the alert's trigger status (`UNKNOWN`, `OK`, `TRIGGERED`). This field is set to `UNKNOWN` if the alert has not yet been evaluated or ran into an error during the last evaluation.
+* `create_time` - The timestamp string indicating when the alert was created.
+* `update_time` - The timestamp string indicating when the alert was updated.
+* `trigger_time` - The timestamp string when the alert was last triggered if the alert has been triggered before.
+
+## Migrating from `databricks_sql_alert` resource
+
+Under the hood, the new resource uses the same data as the `databricks_sql_alert`, but is exposed via a different API. This means that we can migrate existing alerts without recreating them.  This operation is done in few steps:
+
+* Record the ID of existing `databricks_sql_alert`, for example, by executing the `terraform state show databricks_sql_alert.alert` command.
+* Create the code for the new implementation by performing the following changes:
+  * the `name` attribute is now named `display_name`
+  * the `parent` (if exists) is renamed to `parent_path` attribute and should be converted from `folders/object_id` to the actual path.
+  * the `options` block is converted into the `condition` block with the following changes:
+    * the value of the `op` attribute should be converted from a mathematical operator into a string name, like, `>` is becoming `GREATER_THAN`, `==` is becoming `EQUAL`, etc.
+    * the `column` attribute is becoming the `operand` block
+    * the `value` attribute is becoming the `threshold` block.  **Please note that the old implementation always used strings so you may have changes after import if you use `double_value` or `bool_value` inside the block.**
+  * the `rearm` attribute is renamed to `seconds_to_retrigger`.
+  
+For example, if we have the original `databricks_sql_alert` defined as:
+
+```hcl
+resource "databricks_sql_alert" "alert" {
+  query_id = databricks_sql_query.this.id
+  name     = "My Alert"
+  parent   = "folders/${databricks_directory.shared_dir.object_id}"
+  options {
+    column = "value"
+    op     = ">"
+    value  = "42"
+    muted  = false
+  }
+}
+```
+
+we'll have a new resource defined as:
+
+```hcl
+resource "databricks_alert" "alert" {
+  query_id     = databricks_sql_query.this.id
+  display_name = "My Alert"
+  parent_path  = databricks_directory.shared_dir.path
+  condition {
+    op = "GREATER_THAN"
+    operand {
+      column {
+        name = "value"
+      }
+    }
+    threshold {
+      value {
+        double_value = 42
+      }
+    }
+  }
+}
+```
+
+### For Terraform version >= 1.7.0
+
+Terraform 1.7 introduced the [removed](https://developer.hashicorp.com/terraform/language/resources/syntax#removing-resources) block in addition to the [import](https://developer.hashicorp.com/terraform/language/import) block introduced in Terraform 1.5.  Together they make import and removal of resources easier, avoiding manual execution of `terraform import` and `terraform state rm` commands.
+
+So with Terraform 1.7+, the migration looks as the following:
+
+* remove the old alert definition and replace it with the new one.
+* Adjust references, like, `databricks_permissions`.
+* Add `import` and `removed` blocks like this:
+
+```hcl
+import {
+  to = databricks_alert.alert
+  id = "<alert-id>"
+}
+
+removed {
+  from = databricks_sql_alert.alert
+
+  lifecycle {
+    destroy = false
+  }
+}
+```
+
+* Run the `terraform plan` command to check possible changes, such as value type change, etc.
+* Run the `terraform apply` command to apply changes.
+* Remove the `import` and `removed` blocks from the code.
+
+### For Terraform version < 1.7.0
+
+* Remove the old alert definition and replace it with the new one.
+* Remove the old resource from the state with the `terraform state rm databricks_sql_alert.alert` command.
+* Import new resource with the `terraform import databricks_alert.alert <alert-id>` command.
+* Adjust references, like, `databricks_permissions`.
+* Run the `terraform plan` command to check possible changes, such as value type change, etc.
+
+## Access Control
+
+[databricks_permissions](permissions.md#sql-alert-usage) can control which groups or individual users can *Manage*, *Edit*, *Run* or *View* individual alerts.
+
+```hcl
+resource "databricks_permissions" "alert_usage" {
+  sql_alert_id = databricks_alert.alert.id
+  access_control {
+    group_name       = "users"
+    permission_level = "CAN_RUN"
+  }
+}
+```
+
+## Import
+
+This resource can be imported using alert ID:
+
+```bash
+terraform import databricks_alert.this <alert-id>
+```
+
+## Related Resources
+
+The following resources are often used in the same context:
+
+* [databricks_sql_query](sql_query.md) to manage Databricks SQL [Queries](https://docs.databricks.com/sql/user/queries/index.html).
+* [databricks_sql_endpoint](sql_endpoint.md) to manage Databricks SQL [Endpoints](https://docs.databricks.com/sql/admin/sql-endpoints.html).
+* [databricks_directory](directory.md) to manage directories in [Databricks Workpace](https://docs.databricks.com/workspace/workspace-objects.html).
diff --git a/docs/resources/sql_alert.md b/docs/resources/sql_alert.md
@@ -58,6 +58,18 @@ In addition to all arguments above, the following attributes are exported:
 
 * `id` - unique ID of the SQL Alert.
 
+## Access Control
+
+[databricks_permissions](permissions.md#sql-alert-usage) can control which groups or individual users can *Manage*, *Edit*, *Run* or *View* individual alerts.
+
+## Import
+
+This resource can be imported using alert ID:
+
+```bash
+terraform import databricks_sql_alert.this <alert-id>
+```
+
 ## Related Resources
 
 The following resources are often used in the same context:
diff --git a/internal/acceptance/alert_test.go b/internal/acceptance/alert_test.go
@@ -0,0 +1,60 @@
+package acceptance
+
+import (
+	"testing"
+)
+
+func TestAccAlert(t *testing.T) {
+	WorkspaceLevel(t, Step{
+		Template: `
+		resource "databricks_sql_query" "this" {
+			data_source_id = "{env.TEST_DEFAULT_WAREHOUSE_DATASOURCE_ID}"
+			name = "tf-{var.RANDOM}"
+			query = "SELECT 1 AS p1, 2 as p2"
+		}
+
+		resource "databricks_alert" "alert" {
+			query_id = databricks_sql_query.this.id
+			display_name = "tf-alert-{var.RANDOM}"
+			condition {
+			    op = "EQUAL"
+			    operand {
+			      column {
+			        name = "p2"
+			      }
+			    }
+			    threshold {
+			      value {
+			        double_value = 2
+			      }
+			    }
+			}
+		}
+`,
+	}, Step{
+		Template: `
+		resource "databricks_sql_query" "this" {
+			data_source_id = "{env.TEST_DEFAULT_WAREHOUSE_DATASOURCE_ID}"
+			name = "tf-{var.RANDOM}"
+			query = "SELECT 1 AS p1, 2 as p2"
+		}
+
+		resource "databricks_alert" "alert" {
+			query_id = databricks_sql_query.this.id
+			display_name = "tf-alert-{var.RANDOM}"
+			condition {
+			    op = "GREATER_THAN"
+			    operand {
+			      column {
+			        name = "p2"
+			      }
+			    }
+			    threshold {
+			      value {
+			        double_value = 3
+			      }
+			    }
+			}
+		}`,
+	})
+}
diff --git a/internal/acceptance/permissions_test.go b/internal/acceptance/permissions_test.go
@@ -837,3 +837,42 @@ func TestAccPermissions_ServingEndpoint(t *testing.T) {
 		ExpectError: regexp.MustCompile("cannot remove management permissions for the current user for serving-endpoint, allowed levels: CAN_MANAGE"),
 	})
 }
+
+func TestAccPermissions_Alert(t *testing.T) {
+	loadDebugEnvIfRunsFromIDE(t, "workspace")
+	alertTemplate := `
+		resource "databricks_sql_query" "this" {
+			name = "{var.STICKY_RANDOM}-query"
+			query = "SELECT 1 AS p1, 2 as p2"
+			data_source_id = "{env.TEST_DEFAULT_WAREHOUSE_DATASOURCE_ID}"
+		}
+
+		resource "databricks_alert" "this" {
+  			query_id     = databricks_sql_query.this.id
+  			display_name = "{var.STICKY_RANDOM}-alert"
+			condition {
+    			op = "GREATER_THAN"
+    			operand {
+      				column {
+        				name = "value"
+      				}
+    			}
+    			threshold {
+      				value {
+        				double_value = 42
+      				}
+    			}
+  			}
+		}
+`
+	WorkspaceLevel(t, Step{
+		Template: alertTemplate + makePermissionsTestStage("sql_alert_id", "databricks_alert.this.id", groupPermissions("CAN_VIEW")),
+	}, Step{
+		Template: alertTemplate + makePermissionsTestStage("sql_alert_id", "databricks_alert.this.id",
+			currentPrincipalPermission(t, "CAN_MANAGE"), groupPermissions("CAN_VIEW", "CAN_EDIT", "CAN_RUN", "CAN_MANAGE")),
+	}, Step{
+		Template: alertTemplate + makePermissionsTestStage("sql_alert_id", "databricks_alert.this.id",
+			currentPrincipalPermission(t, "CAN_VIEW"), groupPermissions("CAN_VIEW", "CAN_EDIT", "CAN_RUN", "CAN_MANAGE")),
+		ExpectError: regexp.MustCompile("cannot remove management permissions for the current user for alert, allowed levels: CAN_MANAGE"),
+	})
+}
diff --git a/internal/acceptance/sql_alert_test.go b/internal/acceptance/sql_alert_test.go
@@ -4,7 +4,7 @@ import (
 	"testing"
 )
 
-func TestAccAlert(t *testing.T) {
+func TestAccSqlAlert(t *testing.T) {
 	WorkspaceLevel(t, Step{
 		Template: `
 		resource "databricks_sql_query" "this" {
diff --git a/internal/providers/sdkv2/sdkv2.go b/internal/providers/sdkv2/sdkv2.go
@@ -128,6 +128,7 @@ func DatabricksProvider() *schema.Provider {
 		},
 		ResourcesMap: map[string]*schema.Resource{ // must be in alphabetical order
 			"databricks_access_control_rule_set":         permissions.ResourceAccessControlRuleSet().ToResource(),
+			"databricks_alert":                           sql.ResourceAlert().ToResource(),
 			"databricks_artifact_allowlist":              catalog.ResourceArtifactAllowlist().ToResource(),
 			"databricks_aws_s3_mount":                    storage.ResourceAWSS3Mount().ToResource(),
 			"databricks_azure_adls_gen1_mount":           storage.ResourceAzureAdlsGen1Mount().ToResource(),
diff --git a/sql/resource_alert.go b/sql/resource_alert.go
diff --git a/sql/resource_alert_test.go b/sql/resource_alert_test.go
diff --git a/sql/resource_sql_alerts.go b/sql/resource_sql_alerts.go

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@ import (`
`4`	`4`	`"testing"`
`5`	`5`	`)`
`6`	`6`
`7`		`-func TestAccAlert(t *testing.T) {`
	`7`	`+func TestAccSqlAlert(t *testing.T) {`
`8`	`8`	`WorkspaceLevel(t, Step{`
`9`	`9`	Template: `
`10`	`10`	`resource "databricks_sql_query" "this" {`