Retry GCE's (new?) 403 ReadRequests errors (#5723) (#4064)

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/5723.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+provider: added retries for `ReadRequest` errors incorrectly coded as `403` errors, particularly in Google Compute Engine
+```

google-beta/error_retry_predicates.go

@@ -32,11 +32,20 @@ var defaultErrorRetryPredicates = []RetryErrorPredicateFunc{
 	// Keeping it as a default for now.
 	is409OperationInProgressError,
 
+	// GCE Error codes- we don't have a way to add these to all GCE resources
+	// easily, so add them globally.
+
 	// GCE Subnetworks are considered unready for a brief period when certain
 	// operations are performed on them, and the scope is likely too broad to
 	// apply a mutex. If we attempt an operation w/ an unready subnetwork, retry
 	// it.
 	isSubnetworkUnreadyError,
+
+	// As of February 2022 GCE seems to have added extra quota enforcement on
+	// reads, causing significant failure for our CI and for large customers.
+	// GCE returns the wrong error code, as this should be a 429, which we retry
+	// already.
+	is403ReadRequestsForMinuteError,
 }
 
 /** END GLOBAL ERROR RETRY PREDICATES HERE **/
@@ -116,6 +125,21 @@ func isSubnetworkUnreadyError(err error) (bool, string) {
 	return false, ""
 }
 
+// GCE (and possibly other APIs) incorrectly return a 403 rather than a 429 on
+// rate limits.
+func is403ReadRequestsForMinuteError(err error) (bool, string) {
+	gerr, ok := err.(*googleapi.Error)
+	if !ok {
+		return false, ""
+	}
+
+	if gerr.Code == 403 && strings.Contains(gerr.Body, "Quota exceeded for quota metric") && strings.Contains(gerr.Body, "Read requests per minute") {
+		log.Printf("[DEBUG] Dismissed an error as retryable based on error code 403 and error message 'Quota exceeded for quota metric' on metric `Read requests per minute`: %s", err)
+		return true, "Read requests per minute"
+	}
+	return false, ""
+}
+
 // Retry on comon googleapi error codes for retryable errors.
 // TODO(#5609): This may not need to be applied globally - figure out
 // what retryable error codes apply to which API.