Merge pull request #1914 from RomanBednar/OCPBUGS-24588

OCPBUGS-24588: degrade when deployment progressing too long

Author: openshift-merge-bot[bot]

pkg/operator/apiserver/controller/workload/workload.go

@@ -4,9 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strings"
-	"time"
-
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -17,6 +14,7 @@ import (
 	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
+	"strings"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
 	openshiftconfigclientv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
@@ -293,7 +291,7 @@ func (c *Controller) updateOperatorStatus(ctx context.Context, previousStatus *o
 	// Update is done when all pods have been updated to the latest revision
 	// and the deployment controller has reported NewReplicaSetAvailable
 	workloadIsBeingUpdated := !workloadAtHighestGeneration || !hasDeploymentProgressed(workload.Status)
-	workloadIsBeingUpdatedTooLong, err := isUpdatingTooLong(previousStatus, *deploymentProgressingCondition.Type)
+	workloadIsBeingUpdatedTooLong := v1helpers.IsUpdatingTooLong(previousStatus, *deploymentProgressingCondition.Type)
 	if !workloadAtHighestGeneration {
 		deploymentProgressingCondition = deploymentProgressingCondition.
 			WithStatus(operatorv1.ConditionTrue).
@@ -356,13 +354,6 @@ func (c *Controller) updateOperatorStatus(ctx context.Context, previousStatus *o
 	return nil
 }
 
-// isUpdatingTooLong determines if updating operands takes too long.
-// it returns true if the progressing condition has been set to True for at least 15 minutes
-func isUpdatingTooLong(operatorStatus *operatorv1.OperatorStatus, progressingConditionType string) (bool, error) {
-	progressing := v1helpers.FindOperatorCondition(operatorStatus.Conditions, progressingConditionType)
-	return progressing != nil && progressing.Status == operatorv1.ConditionTrue && time.Now().After(progressing.LastTransitionTime.Add(15*time.Minute)), nil
-}
-
 // hasDeploymentProgressed returns true if the deployment reports NewReplicaSetAvailable
 // via the DeploymentProgressing condition
 func hasDeploymentProgressed(status appsv1.DeploymentStatus) bool {

pkg/operator/deploymentcontroller/deployment_controller.go

@@ -300,6 +300,11 @@ func (c *DeploymentController) syncManaged(ctx context.Context, opSpec *opv1.Ope
 				WithMessage(msg).
 				WithReason("Deploying")
 		}
+
+		// Degrade when operator is progressing too long.
+		if v1helpers.IsUpdatingTooLong(opStatus, c.instanceName+opv1.OperatorStatusTypeProgressing) {
+			return fmt.Errorf("Deployment was progressing too long")
+		}
 		status = status.WithConditions(progressingCondition)
 	}
 

pkg/operator/v1helpers/helpers.go

@@ -23,6 +23,10 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+const (
+	progressingConditionTimeout = 15 * time.Minute
+)
+
 // SetOperandVersion sets the new version and returns the previous value.
 func SetOperandVersion(versions *[]configv1.OperandVersion, operandVersion configv1.OperandVersion) string {
 	if versions == nil {
@@ -554,3 +558,10 @@ func IsConditionPresentAndEqual(conditions []metav1.Condition, conditionType str
 	}
 	return false
 }
+
+// IsUpdatingTooLong determines if updating operands condition takes too long.
+// It returns true if the given condition was found and has been set to True longer than progressingConditionTimeout.
+func IsUpdatingTooLong(operatorStatus *operatorv1.OperatorStatus, progressingConditionType string) bool {
+	progressing := FindOperatorCondition(operatorStatus.Conditions, progressingConditionType)
+	return progressing != nil && progressing.Status == operatorv1.ConditionTrue && time.Now().After(progressing.LastTransitionTime.Add(progressingConditionTimeout))
+}