nfd-master: retry node updates indefinitely

Treat node updates like a reconciliation loop. Keep trying on node
update as long as it fails. Node update permafailing likely indicates a
bug in the nfd code (there should be no reason for it to fail forever)
and it's better to clearly see it in the logs/metrics rather than giving
up after a few retries.

Author: marquiz

pkg/nfd-master/node-updater-pool.go

@@ -56,14 +56,15 @@ func (u *nodeUpdaterPool) processNodeUpdateRequest(queue workqueue.RateLimitingI
 	if _, err := u.nfdMaster.getNode(nodeName); apierrors.IsNotFound(err) {
 		klog.InfoS("node not found, skip update", "nodeName", nodeName)
 	} else if err := u.nfdMaster.nfdAPIUpdateOneNode(nodeName); err != nil {
-		if queue.NumRequeues(nodeName) < 15 {
-			klog.InfoS("retrying node update", "nodeName", nodeName, "lastError", err)
-			queue.AddRateLimited(nodeName)
-			return true
+		if n := queue.NumRequeues(nodeName); n < 15 {
+			klog.InfoS("retrying node update", "nodeName", nodeName, "lastError", err, "numRetries", n)
 		} else {
-			klog.ErrorS(err, "failed to update node", "nodeName", nodeName)
+			klog.ErrorS(err, "node update failed, queuing for retry ", "nodeName", nodeName, "numRetries", n)
+			// Count only long-failing attempts
 			nodeUpdateFailures.Inc()
 		}
+		queue.AddRateLimited(nodeName)
+		return true
 	}
 	queue.Forget(nodeName)
 	return true