[Fix] Remove configuration drift when configuring databricks_connection to built-in Hive Metastore (#4505)

## Changes
<!-- Summary of your changes that are easy to understand -->

When configuring built-in Hive Metastore, the backend returns a number
of additional options. But it's hard to suppress them as they are valid
when `builtin = "false"`, and each option is evaluated separately. So
it's easier just to remove them before saving to the state.

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [x] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] using Go SDK
- [ ] using TF Plugin Framework

---------

Co-authored-by: vuong-nguyen <[email protected]>

Author: alexott

NEXT_CHANGELOG.md

@@ -9,6 +9,7 @@
 
 ### Bug Fixes
 
+ * Fix configuration drift when configuring `databricks_connection` to builtin Hive Metastore ([#4505](https://github.com/databricks/terraform-provider-databricks/pull/4505)).
  * Only allow `authorized_paths` to be updated in the `options` field of `databricks_catalog` ([#4517](https://github.com/databricks/terraform-provider-databricks/pull/4517)).
  * Mark `default_catalog_name` attribute in `databricks_metastore_assignment` as deprecated ([#4522](https://github.com/databricks/terraform-provider-databricks/pull/4522))
  * Delete `databricks_sql_endpoint` that failed to start ([#4520](https://github.com/databricks/terraform-provider-databricks/pull/4520))

catalog/resource_connection.go

@@ -87,13 +87,20 @@ func ResourceConnection() common.Resource {
 			if err != nil {
 				return err
 			}
-			// We need to preserve original sensitive options as API doesn't return them
-			var cOrig catalog.CreateConnection
-			common.DataToStructPointer(d, s, &cOrig)
 			// If there are no options returned, need to initialize the map
 			if conn.Options == nil {
 				conn.Options = map[string]string{}
 			}
+			// remove not necessary parameters for builtin HMS to avoid configuration drift
+			if val, exists := conn.Options["builtin"]; exists && val == "true" {
+				delete(conn.Options, "host")
+				delete(conn.Options, "port")
+				delete(conn.Options, "home_workspace_id")
+				delete(conn.Options, "database")
+			}
+			// We need to preserve original sensitive options as API doesn't return them
+			var cOrig catalog.CreateConnection
+			common.DataToStructPointer(d, s, &cOrig)
 			for key, element := range cOrig.Options {
 				if slices.Contains(sensitiveOptions, key) {
 					conn.Options[key] = element

catalog/resource_connection_test.go

@@ -131,6 +131,62 @@ func TestConnectionsCreate(t *testing.T) {
 	assert.Equal(t, map[string]interface{}{"purpose": "testing"}, d.Get("properties"))
 }
 
+func TestConnectionsCreate_BuiltinHms(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   http.MethodPost,
+				Resource: "/api/2.1/unity-catalog/connections",
+				ExpectedRequest: catalog.CreateConnection{
+					Name:           "hms",
+					ConnectionType: catalog.ConnectionType("HIVE_METASTORE"),
+					Options: map[string]string{
+						"builtin": "true",
+					},
+				},
+				Response: catalog.ConnectionInfo{
+					Name:           "hms",
+					ConnectionType: catalog.ConnectionType("HIVE_METASTORE"),
+					FullName:       "hms",
+					MetastoreId:    "abc",
+					Options: map[string]string{
+						"builtin": "true",
+						"host":    "test.com",
+						"port":    "3306",
+					},
+				},
+			},
+			{
+				Method:   http.MethodGet,
+				Resource: "/api/2.1/unity-catalog/connections/hms?",
+				Response: catalog.ConnectionInfo{
+					Name:           "hms",
+					ConnectionType: catalog.ConnectionType("HIVE_METASTORE"),
+					FullName:       "hms",
+					MetastoreId:    "abc",
+					Options: map[string]string{
+						"builtin": "true",
+						"host":    "test.com",
+						"port":    "3306",
+					},
+				},
+			},
+		},
+		Resource: ResourceConnection(),
+		Create:   true,
+		HCL: `
+		name = "hms"
+		connection_type = "HIVE_METASTORE"
+		options = {
+			builtin = "true"
+		}
+		`,
+	}.ApplyAndExpectData(t, map[string]any{
+		"name":            "hms",
+		"connection_type": "HIVE_METASTORE",
+	})
+}
+
 func TestConnectionsCreate_Error(t *testing.T) {
 	_, err := qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{

docs/resources/connection.md

@@ -61,12 +61,25 @@ resource "databricks_connection" "bigquery" {
 }
 ```
 
+Create a connection to builtin Hive Metastore
+
+```hcl
+resource "databricks_connection" "this" {
+  name            = "hms-builtin"
+  connection_type = "HIVE_METASTORE"
+  comment         = "This is a connection to builtin HMS"
+  options = {
+    builtin = "true"
+  }
+}
+```
+
 ## Argument Reference
 
 The following arguments are supported:
 
 - `name` - Name of the Connection.
-- `connection_type` - Connection type. `BIGQUERY` `MYSQL` `POSTGRESQL` `SNOWFLAKE` `REDSHIFT` `SQLDW` `SQLSERVER`, `SALESFORCE` or `DATABRICKS` are supported. [Up-to-date list of connection type supported](https://docs.databricks.com/query-federation/index.html#supported-data-sources)
+- `connection_type` - Connection type. `BIGQUERY` `MYSQL` `POSTGRESQL` `SNOWFLAKE` `REDSHIFT` `SQLDW` `SQLSERVER`, `SALESFORCE`, `HIVE_METASTORE`, `GLUE`, `TERADATA`, `ORACLE` or `DATABRICKS` are supported. Up-to-date list of connection type supported is in the [documentation](https://docs.databricks.com/query-federation/index.html#supported-data-sources)
 - `options` - The key value of options required by the connection, e.g. `host`, `port`, `user`, `password` or `GoogleServiceAccountKeyJson`. Please consult the [documentation](https://docs.databricks.com/query-federation/index.html#supported-data-sources) for the required option.
 - `owner` - (Optional) Name of the connection owner.
 - `properties` -  (Optional) Free-form connection properties.