Merge pull request #3093 from wgahnagl/OCPBUGS-57399

OCPBUGS-57399: Prevent upgrade log spam

Author: openshift-merge-bot[bot]

controllers/configmap_controller.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -183,7 +184,13 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context,
 	case servicescm.Name:
 		return ctrl.Result{}, r.reconcileServices(ctx, configMap)
 	case wiparser.InstanceConfigMap:
-		return ctrl.Result{}, r.reconcileNodes(ctx, configMap)
+		err := r.reconcileNodes(ctx, configMap)
+		var upgradeErr *UpgradeLimitExceededError
+		if !errors.As(err, &upgradeErr) {
+			return ctrl.Result{}, err
+		}
+		r.log.Info(upgradeErr.Error())
+		return ctrl.Result{Requeue: true}, nil
 	case certificates.ProxyCertsConfigMap:
 		return ctrl.Result{}, r.reconcileProxyCerts(ctx, configMap)
 	default:

controllers/controllers.go

@@ -281,8 +281,10 @@ func markNodeAsUpgrading(ctx context.Context, c client.Client, currentNode *core
 		}
 	}
 	if len(upgradingNodes.Items) >= MaxParallelUpgrades {
-		return fmt.Errorf("cannot mark node %s as upgrading, maximum number of parallel upgrading nodes reached (%d)",
-			currentNode.Name, MaxParallelUpgrades)
+		return &UpgradeLimitExceededError{
+			NodeName: currentNode.Name,
+			Count:    len(upgradingNodes.Items),
+		}
 	}
 	return metadata.ApplyUpgradingLabel(ctx, c, currentNode)
 }

controllers/errors.go

@@ -0,0 +1,15 @@
+package controllers
+
+import (
+	"fmt"
+)
+
+type UpgradeLimitExceededError struct {
+	NodeName string
+	Count    int
+	Max      int
+}
+
+func (e *UpgradeLimitExceededError) Error() string {
+	return fmt.Sprintf("Cannot mark node %s as upgrading. Current number of upgrading nodes is (%d). Max number of upgrading nodes is (%d)", e.NodeName, e.Count, e.Max)
+}

controllers/windowsmachine_controller.go

@@ -211,7 +211,12 @@ func (r *WindowsMachineReconciler) Reconcile(ctx context.Context,
 
 	// Prevent WMCO upgrades while Machine nodes are being processed
 	if err := condition.MarkAsBusy(ctx, r.client, r.watchNamespace, r.recorder, WindowsMachineController); err != nil {
-		return ctrl.Result{}, err
+		var upgradeErr *UpgradeLimitExceededError
+		if !errors.As(err, &upgradeErr) {
+			return ctrl.Result{}, err
+		}
+		r.log.Info(upgradeErr.Error())
+		return ctrl.Result{Requeue: true}, nil
 	}
 	defer func() {
 		reconcileErr = markAsFreeOnSuccess(ctx, r.client, r.watchNamespace, r.recorder, WindowsMachineController,