wait until /metrics are ready on e2e test

Some e2e tests depend on the controller-manager to expose metrics
on the path /metrics.

It may happen that when the test runs, the pod is not available or the
URL not ready, causing it to fail.

Previously, the test were waiting until the pod was running, but we
need to wait until the /metrics URL is ready.

Author: aojea

test/e2e/framework/metrics/BUILD

@@ -28,6 +28,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
         "//test/e2e/framework/log:go_default_library",

test/e2e/framework/metrics/metrics_grabber.go

@@ -20,9 +20,11 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/master/ports"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -164,10 +166,27 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error)
 		return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
 	}
 
+	var err error
 	podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName)
 	g.waitForControllerManagerReadyOnce.Do(func() {
-		e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem)
+		if runningErr := e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem); runningErr != nil {
+			err = fmt.Errorf("error waiting for controller manager pod to be running: %w", err)
+			return
+		}
+
+		var lastMetricsFetchErr error
+		if metricsWaitErr := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
+			_, lastMetricsFetchErr = g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
+			return lastMetricsFetchErr == nil, nil
+		}); metricsWaitErr != nil {
+			err = fmt.Errorf("error waiting for controller manager pod to expose metrics: %v; %v", err, lastMetricsFetchErr)
+			return
+		}
 	})
+	if err != nil {
+		return ControllerManagerMetrics{}, err
+	}
+
 	output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
 	if err != nil {
 		return ControllerManagerMetrics{}, err