Gracefully handle service unavailable errors from kube-apiserver

Signed-off-by: Joe Lanford <[email protected]>

Author: joelanford

pkg/controller/operators/olm/apiservices.go

@@ -239,8 +239,8 @@ func (a *Operator) areAPIServicesAvailable(csv *v1alpha1.ClusterServiceVersion)
 			return false, nil
 		}
 
-		if err := a.isGVKRegistered(desc.Group, desc.Version, desc.Kind); err != nil {
-			return false, nil
+		if ok, err := a.isGVKRegistered(desc.Group, desc.Version, desc.Kind); !ok || err != nil {
+			return false, err
 		}
 	}
 

pkg/controller/operators/olm/operator.go

@@ -1782,6 +1782,11 @@ func (a *Operator) updateInstallStatus(csv *v1alpha1.ClusterServiceVersion, inst
 		return nil
 	}
 
+	if err := findFirstError(k8serrors.IsServiceUnavailable, strategyErr, apiServiceErr, webhookErr); err != nil {
+		csv.SetPhaseWithEventIfChanged(v1alpha1.CSVPhasePending, v1alpha1.CSVReasonDetectedClusterChange, err.Error(), now, a.recorder)
+		return err
+	}
+
 	// installcheck determined we can't progress (e.g. deployment failed to come up in time)
 	if install.IsErrorUnrecoverable(strategyErr) {
 		csv.SetPhaseWithEventIfChanged(v1alpha1.CSVPhaseFailed, v1alpha1.CSVReasonInstallCheckFailed, fmt.Sprintf("install failed: %s", strategyErr), now, a.recorder)
@@ -1829,6 +1834,15 @@ func (a *Operator) updateInstallStatus(csv *v1alpha1.ClusterServiceVersion, inst
 	return nil
 }
 
+func findFirstError(f func(error) bool, errs ...error) error {
+	for _, err := range errs {
+		if f(err) {
+			return err
+		}
+	}
+	return nil
+}
+
 // parseStrategiesAndUpdateStatus returns a StrategyInstaller and a Strategy for a CSV if it can, else it sets a status on the CSV and returns
 func (a *Operator) parseStrategiesAndUpdateStatus(csv *v1alpha1.ClusterServiceVersion) (install.StrategyInstaller, install.Strategy) {
 	strategy, err := a.resolver.UnmarshalStrategy(csv.Spec.InstallStrategy)

pkg/controller/operators/olm/requirements.go

@@ -11,7 +11,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
-	olmErrors "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/errors"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
 )
@@ -156,7 +155,7 @@ func (a *Operator) requirementStatus(strategyDetailsDeployment *v1alpha1.Strateg
 		}
 
 		// Check if GVK exists
-		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
+		if ok, err := a.isGVKRegistered(r.Group, r.Version, r.Kind); !ok || err != nil {
 			status.Status = "NotPresent"
 			met = false
 			statuses = append(statuses, status)
@@ -219,7 +218,7 @@ func (a *Operator) requirementStatus(strategyDetailsDeployment *v1alpha1.Strateg
 			Name:    name,
 		}
 
-		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
+		if ok, err := a.isGVKRegistered(r.Group, r.Version, r.Kind); !ok || err != nil {
 			status.Status = v1alpha1.RequirementStatusReasonNotPresent
 			status.Message = "Native API does not exist"
 			met = false
@@ -397,7 +396,7 @@ func (a *Operator) requirementAndPermissionStatus(csv *v1alpha1.ClusterServiceVe
 	return met, statuses, nil
 }
 
-func (a *Operator) isGVKRegistered(group, version, kind string) error {
+func (a *Operator) isGVKRegistered(group, version, kind string) (bool, error) {
 	logger := a.logger.WithFields(logrus.Fields{
 		"group":   group,
 		"version": version,
@@ -408,15 +407,15 @@ func (a *Operator) isGVKRegistered(group, version, kind string) error {
 	resources, err := a.opClient.KubernetesInterface().Discovery().ServerResourcesForGroupVersion(gv.String())
 	if err != nil {
 		logger.WithField("err", err).Info("could not query for GVK in api discovery")
-		return err
+		return false, err
 	}
 
 	for _, r := range resources.APIResources {
 		if r.Kind == kind {
-			return nil
+			return true, nil
 		}
 	}
 
 	logger.Info("couldn't find GVK in api discovery")
-	return olmErrors.GroupVersionKindNotFoundError{group, version, kind}
+	return false, nil
 }