Merge pull request #30256 from machine424/rtt

OCPBUGS-61193: chore(extended/prometheus): 2/2: make 'targets auth' test more lenient and more resilient

Author: openshift-merge-bot[bot]

test/extended/prometheus/prometheus.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"regexp"
 	"runtime"
+	"slices"
 	"strings"
 	"time"
 
@@ -105,33 +106,44 @@ var _ = g.Describe("[sig-instrumentation][Late] Platform Prometheus targets", fu
 			o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Delete pod %s/%s", execPod.Namespace, execPod.Name))
 		}()
 
-		contents, err := helper.GetURLWithToken(helper.MustJoinUrlPath(prometheusURL, "api/v1/targets"), bearerToken)
-		o.Expect(err).NotTo(o.HaveOccurred())
+		promTargets := func() (*prometheusTargets, error) {
+			contents, err := helper.GetURLWithToken(helper.MustJoinUrlPath(prometheusURL, "api/v1/targets"), bearerToken)
+			if err != nil {
+				return nil, err
+			}
+			targets := &prometheusTargets{}
+			err = json.Unmarshal([]byte(contents), targets)
+			if err != nil {
+				return nil, err
+			}
+			// sanity check.
+			if len(targets.Data.ActiveTargets) < 5 {
+				return nil, fmt.Errorf("only got %d targets, something is wrong", len(targets.Data.ActiveTargets))
+			}
+			return targets, nil
+		}
 
-		targets := &prometheusTargets{}
-		err = json.Unmarshal([]byte(contents), targets)
+		initialPromTargets, err := promTargets()
 		o.Expect(err).NotTo(o.HaveOccurred())
-		o.Expect(len(targets.Data.ActiveTargets)).Should(o.BeNumerically(">=", 5))
-
 		eg := errgroup.Group{}
 		eg.SetLimit(runtime.GOMAXPROCS(0))
-		errChan := make(chan error, len(targets.Data.ActiveTargets))
-		for _, target := range targets.Data.ActiveTargets {
+		errChan := make(chan error, len(initialPromTargets.Data.ActiveTargets))
+		for _, target := range initialPromTargets.Data.ActiveTargets {
 			eg.Go(func() error {
-				ns := target.Labels["namespace"]
-				o.Expect(ns).NotTo(o.BeEmpty())
-				if namespaceUnderTest != "" && ns != namespaceUnderTest {
+				targetNs, targetJob, targetPod, targetScrapeURL := target.Labels["namespace"], target.Labels["job"], target.Labels["pod"], target.ScrapeUrl
+				o.Expect(targetNs).NotTo(o.BeEmpty())
+				if namespaceUnderTest != "" && targetNs != namespaceUnderTest {
 					return nil
 				}
-				job, pod := target.Labels["job"], target.Labels["pod"]
-				err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 5*time.Minute, true, func(context.Context) (bool, error) {
-					statusCode, execError := helper.URLStatusCodeExecViaPod(execPod.Namespace, execPod.Name, target.ScrapeUrl)
-					e2e.Logf("Scraping target %s of pod %s/%s/%s without auth returned %d, err: %v (skip=%t)", target.ScrapeUrl, ns, job, pod, statusCode, execError, namespacesToSkip.Has(ns))
+				scrapeErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 5*time.Minute, true, func(context.Context) (bool, error) {
+					statusCode, err := helper.URLStatusCodeExecViaPod(execPod.Namespace, execPod.Name, targetScrapeURL)
+					e2e.Logf("scraping target %s of pod %s/%s/%s without auth returned %d, err: %v (skip=%t)", targetScrapeURL, targetNs, targetJob, targetPod, statusCode, err, namespacesToSkip.Has(targetNs))
 					if expectedStatusCodes.Has(statusCode) {
 						return true, nil
 					}
-					// retry on those cases
-					if execError != nil ||
+
+					// retry
+					if err != nil ||
 						statusCode/100 == 5 ||
 						statusCode == http.StatusRequestTimeout ||
 						statusCode == http.StatusTooManyRequests {
@@ -140,12 +152,25 @@ var _ = g.Describe("[sig-instrumentation][Late] Platform Prometheus targets", fu
 					return false, fmt.Errorf("expecting status code %v but returned %d", expectedStatusCodes.UnsortedList(), statusCode)
 				})
 
-				// Decided to ignore targets that Prometheus itself failed to scrape; may be leftovers from earlier tests.
-				// See: https://issues.redhat.com/browse/OCPBUGS-61193
-				if err != nil && target.Health == "up" && !namespacesToSkip.Has(ns) {
-					errChan <- fmt.Errorf("Scraping target %s of pod %s/%s/%s is probably possible without auth: %w", target.ScrapeUrl, ns, job, pod, err)
+				// Ignoring targets that Prometheus no longer scrapes or fails to scrape.
+				// These may be leftovers from earlier tests.
+				// Reference: https://issues.redhat.com/browse/OCPBUGS-61193
+				if scrapeErr != nil && !namespacesToSkip.Has(targetNs) {
+					targets, err := promTargets()
+					if err != nil {
+						e2e.Logf("refreshing state of target %s of pod %s/%s/%s failed, err: %v (skip=%t)", targetScrapeURL, targetNs, targetJob, targetPod, err, namespacesToSkip.Has(targetNs))
+						targets = initialPromTargets
+					}
+					idx := slices.IndexFunc(targets.Data.ActiveTargets, func(t prometheusTarget) bool {
+						return t.Labels["namespace"] == targetNs &&
+							t.Labels["job"] == targetJob &&
+							t.Labels["pod"] == targetPod &&
+							t.ScrapeUrl == targetScrapeURL
+					})
+					if idx >= 0 && targets.Data.ActiveTargets[idx].Health == "up" {
+						errChan <- fmt.Errorf("failed to ensure scraping target %s of pod %s/%s/%s requires auth: %w", targetScrapeURL, targetNs, targetJob, targetPod, scrapeErr)
+					}
 				}
-
 				return nil
 			})
 		}
@@ -929,13 +954,15 @@ func all(errs ...error) []error {
 	return result
 }
 
+type prometheusTarget struct {
+	Labels    map[string]string
+	Health    string
+	ScrapeUrl string
+}
+
 type prometheusTargets struct {
 	Data struct {
-		ActiveTargets []struct {
-			Labels    map[string]string
-			Health    string
-			ScrapeUrl string
-		}
+		ActiveTargets []prometheusTarget
 	}
 	Status string
 }