Skip to content

Commit b1c1187

Browse files
k8s-ci-robotaojea
authored andcommitted
e2e: wait for controller manager pod to be ready
The MetricsGrabber may use the controller-manager pod to gather metrics, however, it doesn't wait until it is ready to serve, failing the test if this is the case. We wait until the controller-manager pod is running before trying to get metrics from it.
1 parent 5698716 commit b1c1187

File tree

2 files changed

+19
-10
lines changed

2 files changed

+19
-10
lines changed

test/e2e/framework/metrics/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ go_library(
3232
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
3333
"//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
3434
"//test/e2e/framework/log:go_default_library",
35+
"//test/e2e/framework/pod:go_default_library",
3536
"//test/e2e/perftype:go_default_library",
3637
"//test/e2e/system:go_default_library",
3738
"//vendor/k8s.io/klog:go_default_library",

test/e2e/framework/metrics/metrics_grabber.go

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ package metrics
1919
import (
2020
"context"
2121
"fmt"
22+
"sync"
2223

2324
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2425
"k8s.io/apimachinery/pkg/fields"
2526
clientset "k8s.io/client-go/kubernetes"
2627
api "k8s.io/kubernetes/pkg/apis/core"
2728
"k8s.io/kubernetes/pkg/master/ports"
29+
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
2830
"k8s.io/kubernetes/test/e2e/system"
2931

3032
"k8s.io/klog"
@@ -41,15 +43,16 @@ type Collection struct {
4143

4244
// Grabber provides functions which grab metrics from components
4345
type Grabber struct {
44-
client clientset.Interface
45-
externalClient clientset.Interface
46-
grabFromAPIServer bool
47-
grabFromControllerManager bool
48-
grabFromKubelets bool
49-
grabFromScheduler bool
50-
grabFromClusterAutoscaler bool
51-
masterName string
52-
registeredMaster bool
46+
client clientset.Interface
47+
externalClient clientset.Interface
48+
grabFromAPIServer bool
49+
grabFromControllerManager bool
50+
grabFromKubelets bool
51+
grabFromScheduler bool
52+
grabFromClusterAutoscaler bool
53+
masterName string
54+
registeredMaster bool
55+
waitForControllerManagerReadyOnce sync.Once
5356
}
5457

5558
// NewMetricsGrabber returns new metrics which are initialized.
@@ -161,7 +164,12 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error)
161164
if !g.registeredMaster {
162165
return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
163166
}
164-
output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
167+
168+
podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName)
169+
g.waitForControllerManagerReadyOnce.Do(func() {
170+
e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem)
171+
})
172+
output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
165173
if err != nil {
166174
return ControllerManagerMetrics{}, err
167175
}

0 commit comments

Comments
 (0)