Merge pull request #2131 from marquiz/devel/api-controller-memleak

nfd-master: fix memory leak when leader election is enabled

Author: k8s-ci-robot

pkg/nfd-master/nfd-master.go

@@ -149,6 +149,9 @@ type nfdMaster struct {
 	updaterPool    *updaterPool
 	deniedNs
 	config *NFDConfig
+
+	// isLeader indicates if this instance is the leader, changing dynamically
+	isLeader bool
 }
 
 // NewNfdMaster creates a new NfdMaster server instance.
@@ -305,10 +308,11 @@ func (m *nfdMaster) Run() error {
 
 	// Run updater that handles events from the nfd CRD API.
 	if m.args.EnableLeaderElection {
-		go m.nfdAPIUpdateHandlerWithLeaderElection()
+		go m.startLeaderElectionHandler()
 	} else {
-		go m.nfdAPIUpdateHandler()
+		m.isLeader = true
 	}
+	go m.nfdAPIUpdateHandler()
 
 	// Register health probe (at this point we're "ready and live")
 	httpMux.HandleFunc("/healthz", m.Healthz)
@@ -349,6 +353,12 @@ func (m *nfdMaster) nfdAPIUpdateHandler() {
 		case nodeFeatureGroupName := <-m.nfdController.updateNodeFeatureGroupChan:
 			nodeFeatureGroup[nodeFeatureGroupName] = struct{}{}
 		case <-rateLimit:
+			// If we're not the leader, don't do anything, sleep a bit longer
+			if !m.isLeader {
+				rateLimit = time.After(5 * time.Second)
+				break
+			}
+
 			// NodeFeature
 			errUpdateAll := false
 			if updateAll {
@@ -1289,7 +1299,7 @@ func (m *nfdMaster) startNfdApiController() error {
 	return nil
 }
 
-func (m *nfdMaster) nfdAPIUpdateHandlerWithLeaderElection() {
+func (m *nfdMaster) startLeaderElectionHandler() {
 	ctx := context.Background()
 	lock := &resourcelock.LeaseLock{
 		LeaseMeta: metav1.ObjectMeta{
@@ -1310,11 +1320,15 @@ func (m *nfdMaster) nfdAPIUpdateHandlerWithLeaderElection() {
 		RenewDeadline: m.config.LeaderElection.RenewDeadline.Duration,
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(_ context.Context) {
-				m.nfdAPIUpdateHandler()
+				m.isLeader = true
 			},
 			OnStoppedLeading: func() {
 				// We lost the lock.
 				klog.InfoS("leaderelection lock was lost")
+				// We stop (i.e. exit), makes sure that in-flight
+				// requests/re-tries will be stopped TODO: more graceful
+				// handling that does not exit the pod (set m.isLeader to false
+				// and flush updater queue and flush updater queues...)
 				m.Stop()
 			},
 		},