Merge pull request kubernetes#92485 from oomichi/metrics-grabber

Avoid DeprecatedMightBeMasterNode() in e2e metrics

Author: k8s-ci-robot

test/e2e/framework/metrics/BUILD

@@ -31,7 +31,6 @@ go_library(
         "//test/e2e/framework/log:go_default_library",
         "//test/e2e/framework/pod:go_default_library",
         "//test/e2e/perftype:go_default_library",
-        "//test/e2e/system:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],
 )

test/e2e/framework/metrics/metrics_grabber.go

@@ -19,6 +19,7 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"regexp"
 	"sync"
 	"time"
 
@@ -27,7 +28,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-	"k8s.io/kubernetes/test/e2e/system"
 
 	"k8s.io/klog/v2"
 )
@@ -61,38 +61,48 @@ type Grabber struct {
 	grabFromKubelets                  bool
 	grabFromScheduler                 bool
 	grabFromClusterAutoscaler         bool
-	masterName                        string
-	registeredMaster                  bool
+	kubeScheduler                     string
+	kubeControllerManager             string
 	waitForControllerManagerReadyOnce sync.Once
 }
 
 // NewMetricsGrabber returns new metrics which are initialized.
 func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool) (*Grabber, error) {
-	registeredMaster := false
-	masterName := ""
-	nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+
+	kubeScheduler := ""
+	kubeControllerManager := ""
+
+	regKubeScheduler := regexp.MustCompile("kube-scheduler-.*")
+	regKubeControllerManager := regexp.MustCompile("kube-controller-manager-.*")
+
+	podList, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		return nil, err
 	}
-	if len(nodeList.Items) < 1 {
-		klog.Warning("Can't find any Nodes in the API server to grab metrics from")
+	if len(podList.Items) < 1 {
+		klog.Warningf("Can't find any pods in namespace %s to grab metrics from", metav1.NamespaceSystem)
 	}
-	for _, node := range nodeList.Items {
-		if system.DeprecatedMightBeMasterNode(node.Name) {
-			registeredMaster = true
-			masterName = node.Name
+	for _, pod := range podList.Items {
+		if regKubeScheduler.MatchString(pod.Name) {
+			kubeScheduler = pod.Name
+		}
+		if regKubeControllerManager.MatchString(pod.Name) {
+			kubeControllerManager = pod.Name
+		}
+		if kubeScheduler != "" && kubeControllerManager != "" {
 			break
 		}
 	}
-	if !registeredMaster {
+	if kubeScheduler == "" {
 		scheduler = false
+		klog.Warningf("Can't find kube-scheduler pod. Grabbing metrics from kube-scheduler is disabled.")
+	}
+	if kubeControllerManager == "" {
 		controllers = false
-		clusterAutoscaler = ec != nil
-		if clusterAutoscaler {
-			klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager is disabled.")
-		} else {
-			klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager and ClusterAutoscaler is disabled.")
-		}
+		klog.Warningf("Can't find kube-controller-manager pod. Grabbing metrics from kube-controller-manager is disabled.")
+	}
+	if ec == nil {
+		klog.Warningf("Did not receive an external client interface. Grabbing metrics from ClusterAutoscaler is disabled.")
 	}
 
 	return &Grabber{
@@ -103,14 +113,14 @@ func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets b
 		grabFromKubelets:          kubelets,
 		grabFromScheduler:         scheduler,
 		grabFromClusterAutoscaler: clusterAutoscaler,
-		masterName:                masterName,
-		registeredMaster:          registeredMaster,
+		kubeScheduler:             kubeScheduler,
+		kubeControllerManager:     kubeControllerManager,
 	}, nil
 }
 
-// HasRegisteredMaster returns if metrics grabber was able to find a master node
-func (g *Grabber) HasRegisteredMaster() bool {
-	return g.registeredMaster
+// HasControlPlanePods returns true if metrics grabber was able to find control-plane pods
+func (g *Grabber) HasControlPlanePods() bool {
+	return g.kubeScheduler != "" && g.kubeControllerManager != ""
 }
 
 // GrabFromKubelet returns metrics from kubelet
@@ -139,10 +149,10 @@ func (g *Grabber) grabFromKubeletInternal(nodeName string, kubeletPort int) (Kub
 
 // GrabFromScheduler returns metrics from scheduler
 func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
-	if !g.registeredMaster {
-		return SchedulerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping Scheduler's metrics gathering")
+	if g.kubeScheduler == "" {
+		return SchedulerMetrics{}, fmt.Errorf("kube-scheduler pod is not registered. Skipping Scheduler's metrics gathering")
 	}
-	output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, insecureSchedulerPort)
+	output, err := g.getMetricsFromPod(g.client, g.kubeScheduler, metav1.NamespaceSystem, insecureSchedulerPort)
 	if err != nil {
 		return SchedulerMetrics{}, err
 	}
@@ -151,8 +161,8 @@ func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
 
 // GrabFromClusterAutoscaler returns metrics from cluster autoscaler
 func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) {
-	if !g.registeredMaster && g.externalClient == nil {
-		return ClusterAutoscalerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ClusterAutoscaler's metrics gathering")
+	if !g.HasControlPlanePods() && g.externalClient == nil {
+		return ClusterAutoscalerMetrics{}, fmt.Errorf("Did not find control-plane pods. Skipping ClusterAutoscaler's metrics gathering")
 	}
 	var client clientset.Interface
 	var namespace string
@@ -172,12 +182,12 @@ func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error)
 
 // GrabFromControllerManager returns metrics from controller manager
 func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) {
-	if !g.registeredMaster {
-		return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
+	if g.kubeControllerManager == "" {
+		return ControllerManagerMetrics{}, fmt.Errorf("kube-controller-manager pod is not registered. Skipping ControllerManager's metrics gathering")
 	}
 
 	var err error
-	podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName)
+	podName := g.kubeControllerManager
 	g.waitForControllerManagerReadyOnce.Do(func() {
 		if readyErr := e2epod.WaitForPodsReady(g.client, metav1.NamespaceSystem, podName, 0); readyErr != nil {
 			err = fmt.Errorf("error waiting for controller manager pod to be ready: %w", readyErr)

test/e2e/storage/testsuites/base.go

@@ -625,7 +625,7 @@ func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts {
 		framework.ExpectNoError(err, "Error creating metrics grabber: %v", err)
 	}
 
-	if !metricsGrabber.HasRegisteredMaster() {
+	if !metricsGrabber.HasControlPlanePods() {
 		framework.Logf("Warning: Environment does not support getting controller-manager metrics")
 		return opCounts{}
 	}

test/e2e/storage/volume_metrics.go

@@ -100,7 +100,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 	ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() {
 		var err error
 
-		if !metricsGrabber.HasRegisteredMaster() {
+		if !metricsGrabber.HasControlPlanePods() {
 			e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
 		}
 
@@ -146,7 +146,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 	ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() {
 		var err error
 
-		if !metricsGrabber.HasRegisteredMaster() {
+		if !metricsGrabber.HasControlPlanePods() {
 			e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
 		}
 
@@ -446,7 +446,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		}
 
 		ginkgo.BeforeEach(func() {
-			if !metricsGrabber.HasRegisteredMaster() {
+			if !metricsGrabber.HasControlPlanePods() {
 				e2eskipper.Skipf("Environment does not support getting controller-manager metrics - skipping")
 			}