Treat auto-purged databricks_cluster as removed (#1178)

Added special case for handling `Cannot access cluster that was terminated or unpinned more than 30 days ago` error in `databricks_cluster` as an indication of resource removed on the platform side.

Fix #1177

Author: nfx

CHANGELOG.md

@@ -6,6 +6,7 @@
 * Removed client-side validation in `databricks_service_principal` for `application_id`, that may not always be available in the planning stage ([#1165](https://github.com/databrickslabs/terraform-provider-databricks/issues/1165)).
 * Use correct HTTP verb for modifying `databricks_permissions` on `databricks_sql_endpoint` entities. Authorized user, assumingly part of `admins` group, is no longer sending `CAN_MANAGE` permission in the HTTP PUT request ([#1163](https://github.com/databrickslabs/terraform-provider-databricks/issues/1163)).
 * Added diff suppression for `min_num_clusters` field in `databricks_sql_endpoint` ([#1172](https://github.com/databrickslabs/terraform-provider-databricks/pull/1172)).
+* Added special case for handling `Cannot access cluster that was terminated or unpinned more than 30 days ago` error in `databricks_cluster` as an indication of resource removed on the platform side ([#1177](https://github.com/databrickslabs/terraform-provider-databricks/issues/1177)).
 
 Updated dependency versions:
 

clusters/clusters_api.go

@@ -600,6 +600,7 @@ func (a ClustersAPI) StartAndGetInfo(clusterID string) (ClusterInfo, error) {
 	return a.waitForClusterStatus(clusterID, ClusterStateRunning)
 }
 
+// make common/resource.go#ToResource read behavior consistent with "normal" resources
 func wrapMissingClusterError(err error, id string) error {
 	if err == nil {
 		return nil
@@ -611,6 +612,14 @@ func wrapMissingClusterError(err error, id string) error {
 	if apiErr.IsMissing() {
 		return err
 	}
+	// https://github.com/databrickslabs/terraform-provider-databricks/issues/1177
+	// Aligned with Clusters Core team to keep behavior of these workarounds 
+	// as is in the longer term, so that this keeps working.
+	if apiErr.ErrorCode == "INVALID_STATE" {
+		log.Printf("[WARN] assuming that cluster is removed on backend: %s", apiErr)
+		apiErr.StatusCode = 404
+		return apiErr
+	}
 	// fix non-compliant error code
 	if strings.Contains(apiErr.Message,
 		fmt.Sprintf("Cluster %s does not exist", id)) {

clusters/clusters_api_test.go

@@ -1202,3 +1202,12 @@ func TestWrapMissingClusterError(t *testing.T) {
 		Message: "Cluster abc does not exist",
 	}, "abc"), "Cluster abc does not exist")
 }
+
+func TestExpiredClusterAssumedAsRemoved(t *testing.T) {
+	err := wrapMissingClusterError(common.APIError{
+		ErrorCode: "INVALID_STATE",
+		Message: "Cannot access cluster X that was terminated or unpinned more than Y days ago.",
+	}, "X")
+	ae, _ := err.(common.APIError)
+	assert.Equal(t, 404, ae.StatusCode)
+}