fix(olm): handle CSV install failure gracefully

Specifically, don't continuously retry CSV installation when a forbidden
error is returned. A new deployment error and CSV reason has been created
for this scenario. The CSV sync loop will not process a CSV that has the
no retry reason set.

Author: Jeff Peeler

pkg/api/apis/operators/v1alpha1/clusterserviceversion_types.go

@@ -231,6 +231,7 @@ const (
 	CSVReasonRequirementsMet                             ConditionReason = "AllRequirementsMet"
 	CSVReasonOwnerConflict                               ConditionReason = "OwnerConflict"
 	CSVReasonComponentFailed                             ConditionReason = "InstallComponentFailed"
+	CSVReasonComponentFailedNoRetry						 ConditionReason = "InstallComponentFailedNoRetry"
 	CSVReasonInvalidStrategy                             ConditionReason = "InvalidInstallStrategy"
 	CSVReasonWaiting                                     ConditionReason = "InstallWaiting"
 	CSVReasonInstallSuccessful                           ConditionReason = "InstallSucceeded"

pkg/controller/install/deployment.go

@@ -7,6 +7,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	rbac "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/util/diff"
 
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
@@ -160,6 +161,9 @@ func (i *StrategyDeploymentInstaller) Install(s Strategy) error {
 	}
 
 	if err := i.installDeployments(strategy.DeploymentSpecs); err != nil {
+		if k8serrors.IsForbidden(err) {
+			return StrategyError{Reason: StrategyErrInsufficientPermissions, Message: fmt.Sprintf("install strategy failed: %s", err)}
+		}
 		return err
 	}
 

pkg/controller/install/errors.go

@@ -11,13 +11,15 @@ const (
 	StrategyErrReasonUnknown            = "Unknown"
 	StrategyErrBadPatch                 = "PatchUnsuccessful"
 	StrategyErrDeploymentUpdated        = "DeploymentUpdated"
+	StrategyErrInsufficientPermissions  = "InsufficentPermissions"
 )
 
 // unrecoverableErrors are the set of errors that mean we can't recover an install strategy
 var unrecoverableErrors = map[string]struct{}{
-	StrategyErrReasonInvalidStrategy: {},
-	StrategyErrReasonTimeout:         {},
-	StrategyErrBadPatch:              {},
+	StrategyErrReasonInvalidStrategy:   {},
+	StrategyErrReasonTimeout:           {},
+	StrategyErrBadPatch:                {},
+	StrategyErrInsufficientPermissions: {},
 }
 
 // StrategyError is used to represent error types for install strategies

pkg/controller/operators/olm/operator.go

@@ -1119,6 +1119,12 @@ func (a *Operator) transitionCSVState(in v1alpha1.ClusterServiceVersion) (out *v
 		"phase":     in.Status.Phase,
 	})
 
+	if in.Status.Reason == v1alpha1.CSVReasonComponentFailedNoRetry {
+		// will change phase out of failed-no-retry in the event of an intentional requeue
+		logger.Debugf("skipping sync for CSV in failed-no-retry state")
+		return
+	}
+
 	out = in.DeepCopy()
 	now := a.now()
 
@@ -1308,6 +1314,11 @@ func (a *Operator) transitionCSVState(in v1alpha1.ClusterServiceVersion) (out *v
 		}
 
 		if syncError = installer.Install(strategy); syncError != nil {
+			if install.IsErrorUnrecoverable(syncError) {
+				logger.Infof("Setting CSV reason to failed without retry: %v", syncError)
+				out.SetPhaseWithEvent(v1alpha1.CSVPhaseFailed, v1alpha1.CSVReasonComponentFailedNoRetry, fmt.Sprintf("install strategy failed: %s", syncError), now, a.recorder)
+				return
+			}
 			out.SetPhaseWithEvent(v1alpha1.CSVPhaseFailed, v1alpha1.CSVReasonComponentFailed, fmt.Sprintf("install strategy failed: %s", syncError), now, a.recorder)
 			return
 		}