Add check for retry ruota exceeded

Author: edibble21

pkg/controllers/nodeclass/validation.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/mitchellh/hashstructure/v2"
 	"github.com/patrickmn/go-cache"
@@ -130,7 +131,11 @@ func (v *Validation) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass)
 		v.validateRunInstancesAuthorization,
 	} {
 		if failureReason, requeue, err := isValid(ctx, nodeClass, nodeClaim, tags); err != nil {
-			return reconcile.Result{}, err
+			// if the error is not a retry quota exceeded error then enter exponential backoff
+			if awserrors.IgnoreRetryQuotaExceededError(err) != nil {
+				return reconcile.Result{}, err
+			}
+			return reconcile.Result{RequeueAfter: time.Minute}, nil
 		} else if requeue {
 			return reconcile.Result{Requeue: true}, nil
 		} else if failureReason != "" {

pkg/errors/errors.go

@@ -29,6 +29,7 @@ const (
 	DryRunOperationErrorCode              = "DryRunOperation"
 	UnauthorizedOperationErrorCode        = "UnauthorizedOperation"
 	RateLimitingErrorCode                 = "RequestLimitExceeded"
+	RetryQuotaExceededErrorMessage        = "retry quota exceeded"
 )
 
 var (
@@ -123,6 +124,23 @@ func IsUnauthorizedOperationError(err error) bool {
 	return false
 }
 
+func IgnoreRetryQuotaExceededError(err error) error {
+	if IsRetryQuotaExceededError(err) {
+		return nil
+	}
+	return err
+}
+
+func IsRetryQuotaExceededError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if opErr, ok := lo.ErrorsAs[*smithy.OperationError](err); ok {
+		return strings.Contains(opErr.Unwrap().Error(), RetryQuotaExceededErrorMessage)
+	}
+	return false
+}
+
 func IgnoreUnauthorizedOperationError(err error) error {
 	if IsUnauthorizedOperationError(err) {
 		return nil