[Feature] add automatic clustering support for databricks_sql_table (#4607)

## Changes
- Allow setting `cluster_keys` to `auto` for automatic clustering

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [x] relevant change in `docs/` folder
- [x] covered with integration tests in `internal/acceptance`
- [x] using Go SDK

Author: nkvuong

NEXT_CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### New Features and Improvements
 
+ * Add automatic clustering support for `databricks_sql_table` ([#4607](https://github.com/databricks/terraform-provider-databricks/pull/4607))
+
 ### Bug Fixes
 
  * Suppress diff in `databricks_mlflow_experiment` name ([#4606](https://github.com/databricks/terraform-provider-databricks/pull/4606))

catalog/resource_sql_table.go

@@ -304,7 +304,7 @@ func (ti *SqlTableInfo) buildTableCreateStatement() string {
 	}
 
 	if len(ti.ClusterKeys) > 0 {
-		statements = append(statements, fmt.Sprintf("\nCLUSTER BY (%s)", ti.getWrappedClusterKeys())) // CLUSTER BY (`university`, `major`)
+		statements = append(statements, fmt.Sprintf("\nCLUSTER BY %s", ti.getWrappedClusterKeys())) // CLUSTER BY (`university`, `major`)
 	}
 
 	if ti.Comment != "" {
@@ -339,7 +339,14 @@ func (ci SqlColumnInfo) getWrappedColumnName() string {
 
 // Wrapping column name with backticks to avoid special character messing things up.
 func (ti *SqlTableInfo) getWrappedClusterKeys() string {
-	return "`" + strings.Join(ti.ClusterKeys, "`,`") + "`"
+	if len(ti.ClusterKeys) == 1 {
+		clusterKey := strings.ToUpper(ti.ClusterKeys[0])
+		// If the cluster key is AUTO or NONE, we don't need to wrap it with backticks.
+		if slices.Contains([]string{"AUTO", "NONE"}, clusterKey) {
+			return clusterKey
+		}
+	}
+	return "(`" + strings.Join(ti.ClusterKeys, "`,`") + "`)"
 }
 
 func (ti *SqlTableInfo) getStatementsForColumnDiffs(oldti *SqlTableInfo, statements []string, typestring string) []string {
@@ -429,7 +436,7 @@ func (ti *SqlTableInfo) diff(oldti *SqlTableInfo) ([]string, error) {
 		}
 		equal := slices.Equal(ti.ClusterKeys, oldti.ClusterKeys)
 		if !equal {
-			statements = append(statements, fmt.Sprintf("ALTER TABLE %s CLUSTER BY (%s)", ti.SQLFullName(), ti.getWrappedClusterKeys()))
+			statements = append(statements, fmt.Sprintf("ALTER TABLE %s CLUSTER BY %s", ti.SQLFullName(), ti.getWrappedClusterKeys()))
 		}
 	}
 

catalog/resource_sql_table_test.go

@@ -161,6 +161,26 @@ func TestResourceSqlTableCreateStatement_Liquid(t *testing.T) {
 	assert.Contains(t, stmt, "CLUSTER BY (`baz`,`bazz`)")
 }
 
+func TestResourceSqlTableCreateStatement_AutoLiquid(t *testing.T) {
+	ti := &SqlTableInfo{
+		Name:                  "bar",
+		CatalogName:           "main",
+		SchemaName:            "foo",
+		TableType:             "EXTERNAL",
+		DataSourceFormat:      "DELTA",
+		StorageLocation:       "s3://ext-main/foo/bar1",
+		StorageCredentialName: "somecred",
+		Comment:               "terraform managed",
+		ClusterKeys:           []string{"auto"},
+	}
+	stmt := ti.buildTableCreateStatement()
+	assert.Contains(t, stmt, "CREATE EXTERNAL TABLE `main`.`foo`.`bar`")
+	assert.Contains(t, stmt, "USING DELTA")
+	assert.Contains(t, stmt, "LOCATION 's3://ext-main/foo/bar1' WITH (CREDENTIAL `somecred`)")
+	assert.Contains(t, stmt, "COMMENT 'terraform managed'")
+	assert.Contains(t, stmt, "CLUSTER BY AUTO")
+}
+
 func TestResourceSqlTableSerializeProperties(t *testing.T) {
 	ti := &SqlTableInfo{
 		Properties: map[string]string{

docs/resources/sql_table.md

@@ -37,8 +37,6 @@ resource "databricks_sql_table" "thing" {
   catalog_name       = databricks_catalog.sandbox.name
   schema_name        = databricks_schema.things.name
   table_type         = "MANAGED"
-  data_source_format = "DELTA"
-  storage_location   = ""
 
   column {
     name = "id"
@@ -81,8 +79,6 @@ resource "databricks_sql_table" "thing" {
   catalog_name       = databricks_catalog.sandbox.name
   schema_name        = databricks_schema.things.name
   table_type         = "MANAGED"
-  data_source_format = "DELTA"
-  storage_location   = ""
   warehouse_id       = databricks_sql_endpoint.this.id
 
   column {
@@ -131,12 +127,10 @@ resource "databricks_schema" "things" {
 }
 resource "databricks_sql_table" "thing" {
   provider           = databricks.workspace
-  name               = "quickstart_table"
+  name               = "identity_table"
   catalog_name       = databricks_catalog.sandbox.name
   schema_name        = databricks_schema.things.name
   table_type         = "MANAGED"
-  data_source_format = "DELTA"
-  storage_location   = ""
   column {
     name     = "id"
     type     = "bigint"
@@ -151,6 +145,26 @@ resource "databricks_sql_table" "thing" {
 }
 ```
 
+## Enable automatic clustering
+
+```hcl
+resource "databricks_sql_table" "thing" {
+  provider           = databricks.workspace
+  name               = "auto_cluster_table"
+  catalog_name       = databricks_catalog.sandbox.name
+  schema_name        = databricks_schema.things.name
+  table_type         = "MANAGED"
+  cluster_keys       = ["AUTO"]
+  
+  column {
+    name    = "name"
+    type    = "string"
+    comment = "name of thing"
+  }
+  comment = "this table is managed by terraform"
+}
+```
+
 ## Argument Reference
 
 The following arguments are supported:
@@ -164,7 +178,7 @@ The following arguments are supported:
 * `view_definition` - (Optional) SQL text defining the view (for `table_type == "VIEW"`). Not supported for `MANAGED` or `EXTERNAL` table_type.
 * `cluster_id` - (Optional) All table CRUD operations must be executed on a running cluster or SQL warehouse. If a cluster_id is specified, it will be used to execute SQL commands to manage this table. If empty, a cluster will be created automatically with the name `terraform-sql-table`. Conflicts with `warehouse_id`.
 * `warehouse_id` - (Optional) All table CRUD operations must be executed on a running cluster or SQL warehouse. If a `warehouse_id` is specified, that SQL warehouse will be used to execute SQL commands to manage this table. Conflicts with `cluster_id`.
-* `cluster_keys` - (Optional) a subset of columns to liquid cluster the table by. Conflicts with `partitions`.
+* `cluster_keys` - (Optional) a subset of columns to liquid cluster the table by. For automatic clustering, set `cluster_keys` to `["AUTO"]`. To turn off clustering, set it to `["NONE"]`. Conflicts with `partitions`.
 * `partitions` - (Optional) a subset of columns to partition the table by. Change forces the creation of a new resource. Conflicts with `cluster_keys`.
 * `storage_credential_name` - (Optional) For EXTERNAL Tables only: the name of storage credential to use. Change forces the creation of a new resource.
 * `owner` - (Optional) User name/group name/sp application_id of the table owner.

sql/sql_table_test.go

@@ -326,6 +326,38 @@ func TestUcAccResourceSqlTable_Liquid(t *testing.T) {
 			cluster_keys = ["id", "name"]			
 			comment = "this table is managed by terraform..."
 		}`,
+	}, acceptance.Step{
+		Template: `
+		resource "databricks_schema" "this" {
+			name         = "{var.STICKY_RANDOM}"
+			catalog_name = "main"
+		}
+
+		resource "databricks_sql_table" "this" {
+			name               = "bar"
+			catalog_name       = "main"
+			schema_name        = databricks_schema.this.name
+			table_type         = "MANAGED"
+			warehouse_id       = "{env.TEST_DEFAULT_WAREHOUSE_ID}"
+			properties         = {
+				them      = "that"
+				something = "else"
+			}
+			options         = {
+				this      = "blue"
+				that      = "green"
+			}
+			column {
+				name      = "id"
+				type      = "int"
+			}
+			column {
+				name      = "name"
+				type      = "varchar(64)"
+			}
+			cluster_keys = ["auto"]			
+			comment = "this table is managed by terraform..."
+		}`,
 	})
 }