[Feature] Proper support for file events for databricks_external_location (#4749)

## Changes
<!-- Summary of your changes that are easy to understand -->

Proper support of file events requires customization and additional
workarounds for API problems.

Resolves #4747, resolves #4748

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [x] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [x] using Go SDK
- [ ] using TF Plugin Framework

Author: alexott

NEXT_CHANGELOG.md

@@ -6,6 +6,8 @@
 
 ### New Features and Improvements
 
+ * Support configuration of file events in `databricks_external_location` [#4749](https://github.com/databricks/terraform-provider-databricks/pull/4749).
+
 ### Bug Fixes
 
  * Populate `partitions` when reading `databricks_sql_table` ([#4674](https://github.com/databricks/terraform-provider-databricks/pull/4674)).

catalog/resource_external_location.go

@@ -38,6 +38,25 @@ func ResourceExternalLocation() common.Resource {
 			for _, key := range []string{"created_at", "created_by", "credential_id", "updated_at", "updated_by", "browse_only"} {
 				common.CustomizeSchemaPath(m, key).SetReadOnly()
 			}
+			// customize file event queue
+			supportedQueues := []string{"managed_pubsub", "managed_aqs", "managed_sqs", "provided_pubsub", "provided_aqs", "provided_sqs"}
+			for _, key := range supportedQueues {
+				common.CustomizeSchemaPath(m, "file_event_queue", key, "managed_resource_id").SetReadOnly()
+				conflicts := make([]string, 0, len(supportedQueues)-1)
+				for _, otherKey := range supportedQueues {
+					if key != otherKey {
+						conflicts = append(conflicts, "file_event_queue.0."+otherKey)
+					}
+				}
+				common.CustomizeSchemaPath(m, "file_event_queue", key).SetConflictsWith(conflicts)
+			}
+			common.CustomizeSchemaPath(m, "file_event_queue", "provided_pubsub", "subscription_name").SetRequired()
+			common.CustomizeSchemaPath(m, "file_event_queue", "provided_aqs", "queue_url").SetRequired()
+			common.CustomizeSchemaPath(m, "file_event_queue", "provided_sqs", "queue_url").SetRequired()
+			common.CustomizeSchemaPath(m, "file_event_queue", "managed_aqs", "resource_group").SetRequired()
+			common.CustomizeSchemaPath(m, "file_event_queue", "managed_aqs", "subscription_id").SetRequired()
+			common.CustomizeSchemaPath(m, "file_event_queue").SetMaxItems(1)
+
 			return m
 		})
 	return common.Resource{
@@ -113,13 +132,37 @@ func ResourceExternalLocation() common.Resource {
 			if !d.HasChangeExcept("owner") {
 				return nil
 			}
-			if d.HasChange("read_only") {
-				updateExternalLocationRequest.ForceSendFields = append(updateExternalLocationRequest.ForceSendFields, "ReadOnly")
+			booleanFields := map[string]string{"read_only": "ReadOnly", "fallback": "Fallback", "enable_file_events": "EnableFileEvents"}
+			for key, value := range booleanFields {
+				if d.HasChange(key) {
+					updateExternalLocationRequest.ForceSendFields = append(updateExternalLocationRequest.ForceSendFields, value)
+				}
+			}
+			// ugly hack until API is fixed
+			if updateExternalLocationRequest.FileEventQueue != nil {
+				if updateExternalLocationRequest.FileEventQueue.ManagedAqs != nil {
+					updateExternalLocationRequest.FileEventQueue.ManagedAqs.ManagedResourceId = ""
+				}
+				if updateExternalLocationRequest.FileEventQueue.ManagedPubsub != nil {
+					updateExternalLocationRequest.FileEventQueue.ManagedPubsub.ManagedResourceId = ""
+				}
+				if updateExternalLocationRequest.FileEventQueue.ManagedSqs != nil {
+					updateExternalLocationRequest.FileEventQueue.ManagedSqs.ManagedResourceId = ""
+				}
+				if updateExternalLocationRequest.FileEventQueue.ProvidedPubsub != nil {
+					updateExternalLocationRequest.FileEventQueue.ProvidedPubsub.SubscriptionName = ""
+				}
+				if updateExternalLocationRequest.FileEventQueue.ProvidedAqs != nil {
+					updateExternalLocationRequest.FileEventQueue.ProvidedAqs.QueueUrl = ""
+				}
+				if updateExternalLocationRequest.FileEventQueue.ProvidedSqs != nil {
+					updateExternalLocationRequest.FileEventQueue.ProvidedSqs.QueueUrl = ""
+				}
 			}
-			if d.HasChange("fallback") {
-				updateExternalLocationRequest.ForceSendFields = append(updateExternalLocationRequest.ForceSendFields, "Fallback")
+			// if file events are disabled we shouldn't send FileEventQueue
+			if !updateExternalLocationRequest.EnableFileEvents {
+				updateExternalLocationRequest.FileEventQueue = nil
 			}
-
 			updateExternalLocationRequest.Owner = ""
 			_, err = w.ExternalLocations.Update(ctx, updateExternalLocationRequest)
 			if err != nil {

catalog/resource_external_location_test.go

@@ -372,12 +372,13 @@ func TestUpdateExternalLocation_FromReadOnly(t *testing.T) {
 				Method:   "PATCH",
 				Resource: "/api/2.1/unity-catalog/external-locations/abc",
 				ExpectedRequest: catalog.UpdateExternalLocation{
-					Url:             "s3://foo/bar",
-					CredentialName:  "bcd",
-					Comment:         "def",
-					ReadOnly:        false,
-					Fallback:        false,
-					ForceSendFields: []string{"ReadOnly", "Fallback"},
+					Url:              "s3://foo/bar",
+					CredentialName:   "bcd",
+					Comment:          "def",
+					ReadOnly:         false,
+					Fallback:         false,
+					EnableFileEvents: false,
+					ForceSendFields:  []string{"ReadOnly", "Fallback", "EnableFileEvents"},
 				},
 			},
 			{
@@ -396,12 +397,13 @@ func TestUpdateExternalLocation_FromReadOnly(t *testing.T) {
 		Update:   true,
 		ID:       "abc",
 		InstanceState: map[string]string{
-			"name":            "abc",
-			"url":             "s3://foo/bar",
-			"credential_name": "abc",
-			"comment":         "def",
-			"read_only":       "true",
-			"fallback":        "true",
+			"name":               "abc",
+			"url":                "s3://foo/bar",
+			"credential_name":    "abc",
+			"comment":            "def",
+			"read_only":          "true",
+			"fallback":           "true",
+			"enable_file_events": "true",
 		},
 		HCL: `
 		name = "abc"
@@ -410,6 +412,7 @@ func TestUpdateExternalLocation_FromReadOnly(t *testing.T) {
 		comment = "def"
 		read_only = false
 		fallback = false
+		enable_file_events = false
 		`,
 	}.ApplyNoError(t)
 }

docs/resources/external_location.md

@@ -126,12 +126,33 @@ The following arguments are required:
 - `skip_validation` - (Optional) Suppress validation errors if any & force save the external location
 - `fallback` - (Optional) Indicates whether fallback mode is enabled for this external location. When fallback mode is enabled (disabled by default), the access to the location falls back to cluster credentials if UC credentials are not sufficient.
 - `read_only` - (Optional) Indicates whether the external location is read-only.
+- `enable_file_events` - (Optional) indicates if managed file events are enabled for this external location.  Requires `file_event_queue` block.
 - `force_destroy` - (Optional) Destroy external location regardless of its dependents.
 - `force_update` - (Optional) Update external location regardless of its dependents.
 - `access_point` - (Optional) The ARN of the s3 access point to use with the external location (AWS).
 - `encryption_details` - (Optional) The options for Server-Side Encryption to be used by each Databricks s3 client when connecting to S3 cloud storage (AWS).
 - `isolation_mode` - (Optional) Whether the external location is accessible from all workspaces or a specific set of workspaces. Can be `ISOLATION_MODE_ISOLATED` or `ISOLATION_MODE_OPEN`. Setting the external location to `ISOLATION_MODE_ISOLATED` will automatically allow access from the current workspace.
 
+### file_event_queue block
+
+The `file_event_queue` block supports the following:
+
+- `managed_pubsub` - (Optional) Configuration for managed Google Cloud Pub/Sub queue.
+  - `managed_resource_id` - (Computed) The ID of the managed resource.
+- `managed_aqs` - (Optional) Configuration for managed Azure Queue Storage queue.
+  - `managed_resource_id` - (Computed) The ID of the managed resource.
+  - `resource_group` - (Required) The Azure resource group.
+  - `subscription_id` - (Required) The Azure subscription ID.
+- `managed_sqs` - (Optional) Configuration for managed Amazon SQS queue.
+  - `managed_resource_id` - (Computed) The ID of the managed resource.
+- `provided_pubsub` - (Optional) Configuration for provided Google Cloud Pub/Sub queue.
+  - `subscription_name` - (Required) The name of the subscription.
+- `provided_aqs` - (Optional) Configuration for provided Azure Storage Queue.
+  - `queue_url` - (Required) The URL of the queue.
+- `provided_sqs` - (Optional) Configuration for provided Amazon SQS queue.
+  - `queue_url` - (Required) The URL of the SQS queue.
+
+
 ## Attribute Reference
 
 In addition to all arguments above, the following attributes are exported: