Use sync timeout to decide how long to retry waiting for k8s api for prometheus.operator.* components (#4568)

* use sync timeout to decide how long to retry waiting for k8s api

Author: dehaansa

CHANGELOG.md

@@ -52,6 +52,8 @@ Main (unreleased)
 
 - Fix potential deadlock in `loki.source.journal` when stopping or reloading the component. (@thampiotr)
 
+- Honor sync timeout when waiting for network availability for prometheus.operator.* components. (@dehaansa)
+
 v1.11.0
 -----------------
 

internal/component/prometheus/operator/common/crdmanager.go

@@ -364,21 +364,26 @@ func (c *crdManager) configureInformers(ctx context.Context, informers cache.Inf
 	var informer cache.Informer
 	var err error
 
+	timeoutCtx, cancel := context.WithTimeout(ctx, c.args.InformerSyncTimeout)
+	deadline, _ := timeoutCtx.Deadline()
+	defer cancel()
 	backoff := backoff.New(
-		ctx,
+		timeoutCtx,
 		backoff.Config{
 			MinBackoff: 1 * time.Second,
 			MaxBackoff: 10 * time.Second,
-			MaxRetries: 3, // retry up to 3 times
+			MaxRetries: 0, // Will retry until InformerSyncTimeout is reached
 		},
 	)
-	for backoff.Ongoing() {
+	for {
 		// Retry to get the informer in case of a timeout.
 		informer, err = getInformer(ctx, informers, prototype, c.args.InformerSyncTimeout)
-		if err == nil {
+		nextDelay := backoff.NextDelay()
+		// exit loop on success, timeout, max retries reached, or if next backoff exceeds timeout
+		if err == nil || !backoff.Ongoing() || time.Now().Add(nextDelay).After(deadline) {
 			break
 		}
-		level.Warn(c.logger).Log("msg", "failed to get informer, retrying", "next backoff", backoff.NextDelay(), "err", err)
+		level.Warn(c.logger).Log("msg", "failed to get informer, retrying", "next backoff", nextDelay, "err", err)
 		backoff.Wait()
 	}
 	if err != nil {