Retry internal errors from cloud SQL. (#3469) (#2045)

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/3469.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+Fix occasional failure to delete `google_sql_database_instance` and `google_sql_user`.
+```

google-beta/error_retry_predicates.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"google.golang.org/api/googleapi"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 type RetryErrorPredicateFunc func(error) (bool, string)
@@ -160,6 +161,22 @@ func pubsubTopicProjectNotReady(err error) (bool, string) {
 	return false, ""
 }
 
+// Retry if Cloud SQL operation returns a 429 with a specific message for
+// concurrent operations.
+func isSqlInternalError(err error) (bool, string) {
+	if gerr, ok := err.(*SqlAdminOperationError); ok {
+		// SqlAdminOperationError is a non-interface type so we need to cast it through
+		// a layer of interface{}.  :)
+		var ierr interface{}
+		ierr = gerr
+		if serr, ok := ierr.(*sqladmin.OperationErrors); ok && serr.Errors[0].Code == "INTERNAL_ERROR" {
+			return true, "Received an internal error, which is sometimes retryable for some SQL resources.  Optimistically retrying."
+		}
+
+	}
+	return false, ""
+}
+
 // Retry if Cloud SQL operation returns a 429 with a specific message for
 // concurrent operations.
 func isSqlOperationInProgressError(err error) (bool, string) {

google-beta/resource_sql_database_instance.go

@@ -927,17 +927,18 @@ func resourceSqlDatabaseInstanceDelete(d *schema.ResourceData, meta interface{})
 	var op *sqladmin.Operation
 	err = retryTimeDuration(func() (rerr error) {
 		op, rerr = config.clientSqlAdmin.Instances.Delete(project, d.Get("name").(string)).Do()
-		return rerr
-	}, d.Timeout(schema.TimeoutDelete))
+		if rerr != nil {
+			return rerr
+		}
+		err = sqlAdminOperationWaitTime(config, op, project, "Delete Instance", d.Timeout(schema.TimeoutDelete))
+		if err != nil {
+			return err
+		}
+		return nil
+	}, d.Timeout(schema.TimeoutDelete), isSqlOperationInProgressError, isSqlInternalError)
 	if err != nil {
 		return fmt.Errorf("Error, failed to delete instance %s: %s", d.Get("name").(string), err)
 	}
-
-	err = sqlAdminOperationWaitTime(config, op, project, "Delete Instance", d.Timeout(schema.TimeoutDelete))
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 

google-beta/resource_sql_user.go

@@ -227,22 +227,22 @@ func resourceSqlUserDelete(d *schema.ResourceData, meta interface{}) error {
 	var op *sqladmin.Operation
 	err = retryTimeDuration(func() error {
 		op, err = config.clientSqlAdmin.Users.Delete(project, instance).Host(host).Name(name).Do()
-		return err
-	}, d.Timeout(schema.TimeoutDelete))
+		if err != nil {
+			return err
+		}
+
+		if err := sqlAdminOperationWaitTime(config, op, project, "Delete User", d.Timeout(schema.TimeoutDelete)); err != nil {
+			return err
+		}
+		return nil
+	}, d.Timeout(schema.TimeoutDelete), isSqlOperationInProgressError, isSqlInternalError)
 
 	if err != nil {
 		return fmt.Errorf("Error, failed to delete"+
 			"user %s in instance %s: %s", name,
 			instance, err)
 	}
 
-	err = sqlAdminOperationWaitTime(config, op, project, "Delete User", d.Timeout(schema.TimeoutDelete))
-
-	if err != nil {
-		return fmt.Errorf("Error, failure waiting for deletion of %s "+
-			"in %s: %s", name, instance, err)
-	}
-
 	return nil
 }