Add operation retry for exceeded quota group OperationReadGroup (#4599) (#3077)

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/4599.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+compute: fixed an issue where exceeding the operation rate limit would fail without retrying
+```

google-beta/common_operation.go

@@ -111,7 +111,7 @@ func CommonRefreshFunc(w Waiter) resource.StateRefreshFunc {
 		op, err := w.QueryOp()
 		if err != nil {
 			// Retry 404 when getting operation (not resource state)
-			if isRetryableError(err, isNotFoundRetryableError("GET operation")) {
+			if isRetryableError(err, isNotFoundRetryableError("GET operation"), isOperationReadQuotaError) {
 				log.Printf("[DEBUG] Dismissed retryable error on GET operation %q: %s", w.OpName(), err)
 				return nil, "done: false", nil
 			}

google-beta/error_retry_predicates.go

@@ -216,6 +216,17 @@ func isBigqueryIAMQuotaError(err error) (bool, string) {
 	return false, ""
 }
 
+// Retry if operation returns a 403 with the message for
+// exceeding the quota limit for 'OperationReadGroup'
+func isOperationReadQuotaError(err error) (bool, string) {
+	if gerr, ok := err.(*googleapi.Error); ok {
+		if gerr.Code == 403 && strings.Contains(gerr.Body, "Quota exceeded for quota group 'OperationReadGroup'") {
+			return true, "Waiting for quota to refresh"
+		}
+	}
+	return false, ""
+}
+
 // Retry if Monitoring operation returns a 409 with a specific message for
 // concurrent operations.
 func isMonitoringConcurrentEditError(err error) (bool, string) {

google-beta/error_retry_predicates_test.go

@@ -78,3 +78,14 @@ func TestIsCommonRetryableErrorCode_otherError(t *testing.T) {
 		t.Errorf("Error incorrectly detected as retryable")
 	}
 }
+
+func TestIsOperationReadQuotaError_quotaExceeded(t *testing.T) {
+	err := googleapi.Error{
+		Code: 403,
+		Body: "Quota exceeded for quota group 'OperationReadGroup' and limit 'Operation read requests per 100 seconds' of service 'compute.googleapis.com' for consumer 'project_number:11111111'.",
+	}
+	isRetryable, _ := isOperationReadQuotaError(&err)
+	if !isRetryable {
+		t.Errorf("Error not detected as retryable")
+	}
+}