[backport] fix: [NPM] [Linux] panic if applyIPSets continues to fail (#2969)

[backport] fix: [NPM] [Linux] panic if applyIPSets continues to fail (#2964)

Signed-off-by: Hunter Gregory <[email protected]>

Author: huntergregory

npm/pkg/dataplane/ipsets/ipsetmanager.go

@@ -51,6 +51,9 @@ type IPSetManager struct {
 	setMap     map[string]*IPSet
 	dirtyCache dirtyCacheInterface
 	ioShim     *common.IOShim
+	// consecutiveApplyFailures is used in Linux to count the number of consecutive failures to apply ipsets
+	// if this count exceeds a threshold, we will panic
+	consecutiveApplyFailures int
 	sync.RWMutex
 }
 
@@ -71,6 +74,8 @@ func NewIPSetManager(iMgrCfg *IPSetManagerCfg, ioShim *common.IOShim) *IPSetMana
 		setMap:     make(map[string]*IPSet),
 		dirtyCache: newDirtyCache(),
 		ioShim:     ioShim,
+		// set to 0 to avoid lint error for windows
+		consecutiveApplyFailures: 0,
 	}
 }
 

npm/pkg/dataplane/ipsets/ipsetmanager_linux.go

@@ -54,6 +54,8 @@ const (
 	destroySectionPrefix           = "delete"
 	addOrUpdateSectionPrefix       = "add/update"
 	ipsetRestoreLineFailurePattern = "Error in line (\\d+):"
+
+	maxConsecutiveFailures = 100
 )
 
 var (
@@ -408,8 +410,19 @@ func (iMgr *IPSetManager) applyIPSets() error {
 	creator := iMgr.fileCreatorForApply(maxTryCount)
 	restoreError := creator.RunCommandWithFile(ipsetCommand, ipsetRestoreFlag)
 	if restoreError != nil {
+		iMgr.consecutiveApplyFailures++
+		if iMgr.consecutiveApplyFailures >= maxConsecutiveFailures {
+			msg := fmt.Sprintf("exceeded max consecutive failures (%d) when applying ipsets. final error: %s", maxConsecutiveFailures, restoreError.Error())
+			klog.Error(msg)
+			metrics.SendErrorLogAndMetric(util.IpsmID, msg)
+			panic(msg)
+		}
+
 		return npmerrors.SimpleErrorWrapper("ipset restore failed when applying ipsets", restoreError)
 	}
+
+	iMgr.consecutiveApplyFailures = 0
+
 	return nil
 }