fix: list the pods again if the pod ip is not yet assigned

Author: meysam81

.goreleaser.yaml

@@ -85,7 +85,6 @@ release:
     owner: "meysam81"
     name: "{{ .ProjectName }}"
   draft: false
-  mode: replace
   prerelease: auto
   header: |
     ## What's Changed

cmd/checker/http.go

@@ -3,7 +3,6 @@ package checker
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"net/http"
 	"time"
 )
@@ -15,38 +14,20 @@ type StaticHTTPChecker struct {
 }
 
 func (h *StaticHTTPChecker) Check(ctx context.Context) error {
-	var attempts uint
 
-	for {
+	retriable := func() error {
 		result := h.Common.performSingleCheck(ctx, h.Upstream)
-		attempts++
 
 		if result.Success {
 			h.Common.Logger.Info().Msgf("check successful in %s with status: %s",
 				result.Duration.Round(time.Millisecond), result.Status)
 			return nil
 		}
 
-		numTries := fmt.Sprintf("%d", attempts)
-		if h.Common.Retries > 0 {
-			numTries = fmt.Sprintf("%d/%d", attempts, h.Common.Retries)
-
-			if attempts >= h.Common.Retries {
-				h.Common.Logger.Error().Msgf("max retries reached: %d", h.Common.Retries)
-				return fmt.Errorf("max retries (%d) exceeded", h.Common.Retries)
-			}
-		}
-
-		jitterSeconds := rand.Intn(6) + 5 // 5-10 seconds
-		if result.Error != nil {
-			h.Common.Logger.Info().Err(result.Error).Msgf("[%s] check failed, retrying in %ds...", numTries, jitterSeconds)
-		}
-
-		if err := waitWithJitter(ctx, jitterSeconds); err != nil {
-			h.Common.Logger.Info().Msg("shutdown signal received")
-			return err
-		}
+		return result.Error
 	}
+
+	return h.Common.runWithJitterBackoff(ctx, retriable)
 }
 
 func (c *HTTPCommon) performSingleCheck(ctx context.Context, upstream string) checkResult {

cmd/checker/k8s.go

@@ -2,8 +2,8 @@ package checker
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"strings"
@@ -76,46 +76,47 @@ func (k *K8sPodChecker) Check(ctx context.Context) error {
 		return err
 	}
 
-	k.Common.Logger.Info().Strs("labels", k.LabelSelector).Str("namespace", k.Namespace).Msg("listing pods")
-
-	pods, err := clientset.CoreV1().Pods(k.Namespace).List(ctx, metav1.ListOptions{
-		LabelSelector: strings.Join(k.LabelSelector, ","),
-	})
-	if err != nil {
-		return err
-	}
+	var latestPod *corev1.Pod
+	var podIP string
 
-	k.Common.Logger.Info().Msgf("found %d pods with the selected filters", len(pods.Items))
+	retriable := func() error {
+		k.Common.Logger.Info().Strs("labels", k.LabelSelector).Str("namespace", k.Namespace).Msg("listing pods")
 
-	var latestPod *corev1.Pod
-	for _, pod := range pods.Items {
-		if k.Image != "" && k.Image != pod.Spec.Containers[0].Image {
-			k.Common.Logger.Info().Msgf("found pod %s/%s but did not match on image", pod.Namespace, pod.Name)
-			continue
+		pods, err := clientset.CoreV1().Pods(k.Namespace).List(ctx, metav1.ListOptions{
+			LabelSelector: strings.Join(k.LabelSelector, ","),
+		})
+		if err != nil {
+			return err
 		}
 
-		if latestPod == nil || latestPod.CreationTimestamp.After(pod.CreationTimestamp.Time) {
-			latestPod = &pod
-		}
-	}
+		k.Common.Logger.Info().Msgf("found %d pods with the selected filters", len(pods.Items))
 
-	if latestPod == nil {
-		return fmt.Errorf("no matching pod found in %s namespace with labels: %s", k.Namespace, k.LabelSelector)
-	}
+		for _, pod := range pods.Items {
+			if k.Image != "" && k.Image != pod.Spec.Containers[0].Image {
+				k.Common.Logger.Info().Msgf("found pod %s/%s but did not match on image", pod.Namespace, pod.Name)
+				continue
+			}
 
-	var podIP string
-	for {
+			if latestPod == nil || latestPod.CreationTimestamp.After(pod.CreationTimestamp.Time) {
+				latestPod = &pod
+			}
+		}
+
+		if latestPod == nil {
+			return fmt.Errorf("no matching pod found in %s namespace with labels: %s", k.Namespace, k.LabelSelector)
+		}
 		podIP = latestPod.Status.PodIP
 		if podIP != "" {
 			k.Common.Logger.Info().Msgf("pod %s has an IP assigned: %s", latestPod.Name, latestPod.Status.PodIP)
-			break
+			return nil
 		}
 
-		jitterSeconds := rand.Intn(6) + 5
-		err := waitWithJitter(ctx, jitterSeconds)
-		if err != nil {
-			return err
-		}
+		return errors.New("no IP assigned to the pod yet")
+	}
+
+	err = k.Common.runWithJitterBackoff(ctx, retriable)
+	if err != nil {
+		return err
 	}
 
 	port := k.Port

cmd/checker/types.go

@@ -11,6 +11,8 @@ type HTTPCommon struct {
 	HTTPClient *http.Client
 	Retries    uint
 	StatusCode int
+	JitterMin  int
+	JitterMax  int
 	Logger     *logging.Logger
 }
 

cmd/checker/utils.go

@@ -2,9 +2,48 @@ package checker
 
 import (
 	"context"
+	"fmt"
+	"math/rand"
 	"time"
 )
 
+func (c *HTTPCommon) runWithJitterBackoff(ctx context.Context, do func() error) error {
+	low := c.JitterMin
+	high := c.JitterMax - c.JitterMin + 1
+
+	var attempts uint
+
+	for {
+		jitterSeconds := rand.Intn(high) + low
+
+		err := do()
+		attempts++
+
+		tries := fmt.Sprintf("%d", attempts)
+		if c.Retries > 0 {
+			tries = fmt.Sprintf("%d/%d", attempts, c.Retries)
+		}
+
+		if err != nil {
+			c.Logger.Info().Err(err).Msgf("[%s] failed, retrying in %ds...", tries, jitterSeconds)
+		} else {
+			return nil
+		}
+
+		if c.Retries > 0 {
+			if attempts >= c.Retries {
+				c.Logger.Error().Msgf("max retries reached: %d", c.Retries)
+				return fmt.Errorf("max retries (%d) exceeded", c.Retries)
+			}
+		}
+
+		if err := waitWithJitter(ctx, jitterSeconds); err != nil {
+			c.Logger.Info().Msg("shutdown signal received")
+			return err
+		}
+	}
+}
+
 // waitWithJitter will only return error if the context is canceled/done
 func waitWithJitter(ctx context.Context, jitterSeconds int) error {
 	t := time.NewTicker(time.Duration(jitterSeconds) * time.Second)

cmd/cmd.go

@@ -95,6 +95,20 @@ func (a *app) createGlobalFlags() []cli.Flag {
 			Destination: &a.Config.StatusCode,
 			Sources:     cli.EnvVars("STATUS_CODE"),
 		},
+		&cli.IntFlag{
+			Name:        "jitter-min-seconds",
+			Usage:       "The min seconds when picking a random time for backoff",
+			Value:       5,
+			Destination: &a.Config.JitterMin,
+			Sources:     cli.EnvVars("JITTER_MIN"),
+		},
+		&cli.IntFlag{
+			Name:        "jitter-max-seconds",
+			Usage:       "The max seconds when picking a random time for backoff",
+			Value:       10,
+			Destination: &a.Config.JitterMax,
+			Sources:     cli.EnvVars("JITTER_MAX"),
+		},
 	}
 }
 
@@ -186,6 +200,8 @@ func (a *app) createK8sCheckCommand() *cli.Command {
 					Retries:    a.Config.Retries,
 					StatusCode: a.Config.StatusCode,
 					Logger:     a.Logger,
+					JitterMin:  a.Config.JitterMin,
+					JitterMax:  a.Config.JitterMax,
 				},
 			}
 
@@ -225,6 +241,8 @@ func (a *app) staticHTTPCheck(ctx context.Context, c *cli.Command) error {
 			Retries:    a.Config.Retries,
 			StatusCode: a.Config.StatusCode,
 			Logger:     a.Logger,
+			JitterMin:  a.Config.JitterMin,
+			JitterMax:  a.Config.JitterMax,
 		},
 	}
 

cmd/config/config.go

@@ -26,6 +26,8 @@ type Config struct {
 	LogLevel   string
 	Retries    uint
 	Timeout    uint
+	JitterMin  int
+	JitterMax  int
 	StatusCode int
 
 	StaticHTTPTarget *StaticHTTPTarget