[Fix] Support updating databricks_cluster to Auto AZ (#4743)

## Changes
- Cluster API now correctly returns `zone_id`, so removing
`ZoneDiffSuppress`
- Resolves #1700

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [x] relevant change in `docs/` folder
- [x] covered with integration tests in `internal/acceptance`

Author: nkvuong

NEXT_CHANGELOG.md

@@ -8,6 +8,8 @@
 
 ### Bug Fixes
 
+ * Support updating `databricks_cluster` to Auto AZ [#4743](https://github.com/databricks/terraform-provider-databricks/pull/4743).
+
 ### Documentation
 
 ### Exporter

clusters/cluster_test.go

@@ -93,6 +93,7 @@ func awsClusterTemplate(availability string) string {
 			autotermination_minutes = 10
 			aws_attributes {
 				availability = "%s"
+				zone_id      = "auto"
 			}
 			custom_tags = {
 				"Owner" = "[email protected]"

clusters/resource_cluster.go

@@ -102,14 +102,6 @@ func SparkConfDiffSuppressFunc(k, old, new string, d *schema.ResourceData) bool
 	return false
 }
 
-func ZoneDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
-	if old != "" && (new == "auto" || new == "") {
-		log.Printf("[INFO] Suppressing diff on availability zone")
-		return true
-	}
-	return false
-}
-
 // This method is a duplicate of ModifyRequestOnInstancePool() in clusters/clusters_api.go that uses Go SDK.
 // Long term, ModifyRequestOnInstancePool() in clusters_api.go will be removed once all the resources using clusters are migrated to Go SDK.
 func ModifyRequestOnInstancePool(cluster any) error {
@@ -349,7 +341,6 @@ func (ClusterSpec) CustomizeSchema(s *common.CustomizableSchema) *common.Customi
 	s.SchemaPath("docker_image", "basic_auth", "username").SetRequired()
 	s.SchemaPath("spark_conf").SetCustomSuppressDiff(SparkConfDiffSuppressFunc)
 	s.SchemaPath("aws_attributes").SetSuppressDiff().SetConflictsWith([]string{"azure_attributes", "gcp_attributes"})
-	s.SchemaPath("aws_attributes", "zone_id").SetCustomSuppressDiff(ZoneDiffSuppress)
 	s.SchemaPath("azure_attributes").SetSuppressDiff().SetConflictsWith([]string{"aws_attributes", "gcp_attributes"})
 	s.SchemaPath("gcp_attributes").SetSuppressDiff().SetConflictsWith([]string{"aws_attributes", "azure_attributes"})
 	s.SchemaPath("autoscale", "max_workers").SetOptional()

clusters/resource_cluster_test.go

@@ -1438,7 +1438,7 @@ func TestResourceClusterUpdate_AutoAz(t *testing.T) {
 					AwsAttributes: &compute.AwsAttributes{
 						Availability:  "SPOT",
 						FirstOnDemand: 1,
-						ZoneId:        "us-west-2a",
+						ZoneId:        "auto",
 					},
 				},
 			},

pipelines/resource_pipeline.go

@@ -231,7 +231,6 @@ func (Pipeline) CustomizeSchema(s *common.CustomizableSchema) *common.Customizab
 	s.SchemaPath("edition").SetSuppressDiff()
 	s.SchemaPath("channel").SetSuppressDiff()
 	s.SchemaPath("cluster", "spark_conf").SetCustomSuppressDiff(clusters.SparkConfDiffSuppressFunc)
-	s.SchemaPath("cluster", "aws_attributes", "zone_id").SetCustomSuppressDiff(clusters.ZoneDiffSuppress)
 	s.SchemaPath("cluster", "autoscale", "mode").SetCustomSuppressDiff(common.EqualFoldDiffSuppress)
 	s.SchemaPath("edition").SetCustomSuppressDiff(common.EqualFoldDiffSuppress)
 	s.SchemaPath("storage").SetCustomSuppressDiff(suppressStorageDiff)