fix race condition issues with health checks (#460)

* fix race condition issues with health checks

* better log meesage when skipping heartbeat

Author: murali-reddy

pkg/cmd/kube-router.go

@@ -87,13 +87,6 @@ func (kr *KubeRouter) Run() error {
 		os.Exit(0)
 	}
 
-	hc, err := healthcheck.NewHealthController(kr.Config)
-	if err != nil {
-		return errors.New("Failed to create health controller: " + err.Error())
-	}
-	wg.Add(1)
-	go hc.Run(healthChan, stopCh, &wg)
-
 	if (kr.Config.MetricsPort > 0) && (kr.Config.MetricsPort <= 65535) {
 		kr.Config.MetricsEnabled = true
 		mc, err := metrics.NewMetricsController(kr.Client, kr.Config)
@@ -165,6 +158,13 @@ func (kr *KubeRouter) Run() error {
 		go nsc.Run(healthChan, stopCh, &wg)
 	}
 
+	hc, err := healthcheck.NewHealthController(kr.Config)
+	if err != nil {
+		return errors.New("Failed to create health controller: " + err.Error())
+	}
+	wg.Add(1)
+	go hc.Run(healthChan, stopCh, &wg)
+
 	// Handle SIGINT and SIGTERM
 	ch := make(chan os.Signal)
 	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)

pkg/controllers/netpol/network_policy_controller.go

@@ -136,7 +136,8 @@ func (npc *NetworkPolicyController) Run(healthChan chan<- *healthcheck.Controlle
 		glog.V(1).Info("Performing periodic sync of iptables to reflect network policies")
 		err := npc.Sync()
 		if err != nil {
-			glog.Errorf("Error during periodic sync: " + err.Error())
+			glog.Errorf("Error during periodic sync of network policies in network policy controller. Error: " + err.Error())
+			glog.Errorf("Skipping sending heartbeat from network policy controller as periodic sync failed.")
 		} else {
 			healthcheck.SendHeartBeat(healthChan, "NPC")
 		}

pkg/controllers/proxy/network_services_controller.go

@@ -265,7 +265,8 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control
 		glog.V(1).Info("Performing periodic sync of ipvs services")
 		err := nsc.sync()
 		if err != nil {
-			glog.Errorf("Error during periodic ipvs sync: " + err.Error())
+			glog.Errorf("Error during periodic ipvs sync in network service controller. Error: " + err.Error())
+			glog.Errorf("Skipping sending heartbeat from network service controller as periodic sync failed.")
 		} else {
 			healthcheck.SendHeartBeat(healthChan, "NSC")
 		}

pkg/controllers/routing/network_routes_controller.go

@@ -216,6 +216,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll
 
 	// loop forever till notified to stop on stopCh
 	for {
+		var err error
 		select {
 		case <-stopCh:
 			glog.Infof("Shutting down network routes controller")
@@ -226,7 +227,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll
 		// Update ipset entries
 		if nrc.enablePodEgress || nrc.enableOverlays {
 			glog.V(1).Info("Syncing ipsets")
-			err := nrc.syncNodeIPSets()
+			err = nrc.syncNodeIPSets()
 			if err != nil {
 				glog.Errorf("Error synchronizing ipsets: %s", err.Error())
 			}
@@ -257,7 +258,12 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll
 			nrc.syncInternalPeers()
 		}
 
-		healthcheck.SendHeartBeat(healthChan, "NRC")
+		if err == nil {
+			healthcheck.SendHeartBeat(healthChan, "NRC")
+		} else {
+			glog.Errorf("Error during periodic sync in network routing controller. Error: " + err.Error())
+			glog.Errorf("Skipping sending heartbeat from network routing controller as periodic sync failed.")
+		}
 
 		select {
 		case <-stopCh:

pkg/healthcheck/health_controller.go

@@ -121,8 +121,7 @@ func (hc *HealthController) CheckHealth() bool {
 
 //Run starts the HealthController
 func (hc *HealthController) Run(healthChan <-chan *ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
-	Started := time.Now()
-	t := time.NewTicker(500 * time.Millisecond)
+	t := time.NewTicker(5000 * time.Millisecond)
 	defer wg.Done()
 	glog.Info("Starting health controller")
 
@@ -144,11 +143,10 @@ func (hc *HealthController) Run(healthChan <-chan *ControllerHeartbeat, stopCh <
 		hc.HTTPEnabled = false
 	}
 
+	//Give the controllers a few seconds to start before checking health
+	time.Sleep(60 * time.Second)
+
 	for {
-		//Give the controllers a few seconds to start before checking health
-		if time.Since(Started) > 5*time.Second {
-			hc.Status.Healthy = hc.CheckHealth()
-		}
 		select {
 		case <-stopCh:
 			glog.Infof("Shutting down health controller")
@@ -163,6 +161,7 @@ func (hc *HealthController) Run(healthChan <-chan *ControllerHeartbeat, stopCh <
 		case <-t.C:
 			glog.V(4).Info("Health controller tick")
 		}
+		hc.Status.Healthy = hc.CheckHealth()
 	}
 
 }
@@ -173,7 +172,11 @@ func NewHealthController(config *options.KubeRouterConfig) (*HealthController, e
 		Config:     config,
 		HealthPort: config.HealthPort,
 		Status: HealthStats{
-			Healthy: false,
+			Healthy:                        true,
+			MetricsControllerAlive:         time.Now(),
+			NetworkPolicyControllerAlive:   time.Now(),
+			NetworkRoutingControllerAlive:  time.Now(),
+			NetworkServicesControllerAlive: time.Now(),
 		},
 	}
 	return &hc, nil