Merge pull request #3381 from sonasingh46/fix_intermittent_statuscode=0

mark context canceled or exceeded as transient failures

Author: k8s-ci-robot

azure/errors.go

@@ -17,6 +17,7 @@ limitations under the License.
 package azure
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"time"
@@ -153,3 +154,12 @@ func IsOperationNotDoneError(target error) bool {
 	}
 	return errors.As(target, &OperationNotDoneError{})
 }
+
+// IsContextDeadlineExceededOrCanceledError checks if it's a context deadline
+// exceeded or canceled error.
+func IsContextDeadlineExceededOrCanceledError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)
+}

azure/errors_test.go

@@ -0,0 +1,53 @@
+package azure
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+func TestIsContextDeadlineExceededOrCanceled(t *testing.T) {
+	tests := []struct {
+		name string
+		want bool
+		err  error
+	}{
+		{
+			name: "Context deadline exceeded error",
+			err: func() error {
+				ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-7*time.Hour))
+				defer cancel()
+				return ctx.Err()
+			}(),
+			want: true,
+		},
+		{
+			name: "Context canceled exceeded error",
+			err: func() error {
+				ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(1*time.Hour))
+				cancel()
+				return ctx.Err()
+			}(),
+			want: true,
+		},
+		{
+			name: "Nil error",
+			err:  nil,
+			want: false,
+		},
+		{
+			name: "Error other than context deadline exceeded or canceled error",
+			err:  errors.New("dummy error"),
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsContextDeadlineExceededOrCanceledError(tt.err); got != tt.want {
+				t.Errorf("IsContextDeadlineExceededOrCanceled() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

azure/services/async/async.go

@@ -140,6 +140,7 @@ func (s *Service) CreateOrUpdateResource(ctx context.Context, spec azure.Resourc
 	log.V(2).Info(fmt.Sprintf("%sing resource", logMessageVerbPrefix), "service", serviceName, "resource", resourceName, "resourceGroup", rgName)
 	result, sdkFuture, err := s.Creator.CreateOrUpdateAsync(ctx, spec, parameters)
 	errWrapped := errors.Wrapf(err, fmt.Sprintf("failed to %se resource %s/%s (service: %s)", logMessageVerbPrefix, rgName, resourceName, serviceName))
+
 	if sdkFuture != nil {
 		future, err := converters.SDKToFuture(sdkFuture, infrav1.PutFuture, serviceName, resourceName, rgName)
 		if err != nil {
@@ -148,6 +149,11 @@ func (s *Service) CreateOrUpdateResource(ctx context.Context, spec azure.Resourc
 		s.Scope.SetLongRunningOperationState(future)
 		return nil, azure.WithTransientError(azure.NewOperationNotDoneError(future), getRequeueAfterFromFuture(sdkFuture))
 	} else if err != nil {
+		// If it is an intermittent failure with context deadline exceeded or canceled as the reconciler could not complete
+		// in the max amount of time, mark it as a transient error and return.
+		if azure.IsContextDeadlineExceededOrCanceledError(ctx.Err()) {
+			return nil, azure.WithTransientError(errWrapped, getRetryAfterFromError(err))
+		}
 		return nil, errWrapped
 	}
 
@@ -174,6 +180,7 @@ func (s *Service) DeleteResource(ctx context.Context, spec azure.ResourceSpecGet
 	// No long running operation is active, so delete the resource.
 	log.V(2).Info("deleting resource", "service", serviceName, "resource", resourceName, "resourceGroup", rgName)
 	sdkFuture, err := s.Deleter.DeleteAsync(ctx, spec)
+
 	if sdkFuture != nil {
 		future, err := converters.SDKToFuture(sdkFuture, infrav1.DeleteFuture, serviceName, resourceName, rgName)
 		if err != nil {
@@ -186,6 +193,11 @@ func (s *Service) DeleteResource(ctx context.Context, spec azure.ResourceSpecGet
 			// already deleted
 			return nil
 		}
+		// If it is an intermittent failure with context deadline exceeded or canceled as the reconciler could not complete
+		// in the max amount of time, mark it as a transient error and return.
+		if azure.IsContextDeadlineExceededOrCanceledError(ctx.Err()) {
+			return azure.WithTransientError(err, getRetryAfterFromError(err))
+		}
 		return errors.Wrapf(err, "failed to delete resource %s/%s (service: %s)", rgName, resourceName, serviceName)
 	}