Merge pull request #3583 from slashpai/endpointslice

MON-4437: Migrate windows-exporter ServiceMonitor to EndpointSlice

Author: openshift-merge-bot[bot]

bundle/manifests/prometheus-k8s_rbac.authorization.k8s.io_v1_role.yaml

@@ -13,3 +13,11 @@ rules:
   verbs:
   - list
   - watch
+- apiGroups:
+  - "discovery.k8s.io"
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch

config/windows-exporter/windows-exporter-role.yaml

@@ -13,3 +13,11 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch

controllers/metric_controller.go

@@ -132,6 +132,7 @@ func (r *metricReconciler) ensureServiceMonitor(ctx context.Context) error {
 	replacement1 := "$1:9182"
 	replacement2 := metrics.WindowsMetricsResource
 	attachMetadataBool := true
+	endpointSliceRole := monv1.EndpointSliceRole
 	expectedSM := &monv1.ServiceMonitor{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      metrics.WindowsMetricsResource,
@@ -141,6 +142,7 @@ func (r *metricReconciler) ensureServiceMonitor(ctx context.Context) error {
 			},
 		},
 		Spec: monv1.ServiceMonitorSpec{
+			ServiceDiscoveryRole: &endpointSliceRole,
 			AttachMetadata: &monv1.AttachMetadata{
 				Node: &attachMetadataBool,
 			},
@@ -165,7 +167,7 @@ func (r *metricReconciler) ensureServiceMonitor(ctx context.Context) error {
 							Replacement: &replacement0,
 							TargetLabel: "instance",
 							SourceLabels: []monv1.LabelName{
-								"__meta_kubernetes_endpoint_address_target_name",
+								"__meta_kubernetes_endpointslice_endpoint_target_name",
 							},
 						},
 						{ // Include only Windows nodes for this serviceMonitor

go.mod

@@ -23,8 +23,8 @@ require (
 	github.com/operator-framework/operator-lib v0.4.0
 	github.com/operator-framework/operator-lifecycle-manager v0.22.0
 	github.com/pkg/sftp v1.13.10
-	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.85.0
-	github.com/prometheus-operator/prometheus-operator/pkg/client v0.85.0
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2
+	github.com/prometheus-operator/prometheus-operator/pkg/client v0.86.2
 	github.com/spf13/cobra v1.10.1
 	github.com/spf13/pflag v1.0.10
 	github.com/stretchr/testify v1.11.1

go.sum

@@ -544,10 +544,10 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.85.0 h1:oY+F5FZFmCjCyzkHWPjVQpzvnvEB/0FP+iyzDUUlqFc=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.85.0/go.mod h1:VB7wtBmDT6W2RJHzsvPZlBId+EnmeQA0d33fFTXvraM=
-github.com/prometheus-operator/prometheus-operator/pkg/client v0.85.0 h1:OdW3Vnmoa2pM5PfRyQaSEO+UN5yamEUS1MuhwE0PxbY=
-github.com/prometheus-operator/prometheus-operator/pkg/client v0.85.0/go.mod h1:5ctipSFkXKeXig01Or0aXKNaaBQFWZAAK1zzYWz9YZY=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2 h1:VRXUgbGmpmjZgFYiUnTwlC+JjfCUs5KKFsorJhI1ZKQ=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.86.2/go.mod h1:nPk0OteXBkbT0CRCa2oZQL1jRLW6RJ2fuIijHypeJdk=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.86.2 h1:aD+r5a/96ZVT11Uo6Jpt3gO2Hutq3NROB8lXYKMZlOI=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.86.2/go.mod h1:fXZB2vXirMxIjFEIggDrFirYJQh7GkHCZRwm8aKD0e8=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=

test/e2e/delete_test.go

@@ -273,8 +273,9 @@ func (tc *testContext) testWindowsNodeDeletion(t *testing.T) {
 	_, err = tc.waitForWindowsMachines(int(expectedNodeCount), "", true)
 	require.NoError(t, err, "ConfigMap controller Windows machine deletion failed")
 
-	// Test if prometheus configuration is updated to have no node entries in the endpoints object
+	// Test if prometheus configuration is updated to have no node entries in the endpointslice objects.
 	t.Run("Prometheus configuration", tc.testPrometheus)
+	t.Run("Prometheus endpoint slice cleanup", tc.testPrometheusEndpointSliceCleanup)
 
 	// Cleanup windows-instances ConfigMap
 	tc.deleteWindowsInstanceConfigMap()

test/e2e/metrics_test.go

@@ -82,6 +82,41 @@ func (tc *testContext) testPrometheus(t *testing.T) {
 
 }
 
+// testPrometheusEndpointSliceCleanup verifies that no Windows nodes remain in kubelet EndpointSlices
+func (tc *testContext) testPrometheusEndpointSliceCleanup(t *testing.T) {
+	// List all EndpointSlices for the kubelet service in kube-system namespace
+	endpointSlices, err := tc.client.K8s.DiscoveryV1().EndpointSlices("kube-system").List(
+		context.TODO(),
+		metav1.ListOptions{
+			LabelSelector: "kubernetes.io/service-name=kubelet",
+		},
+	)
+	require.NoError(t, err, "error listing EndpointSlices for kubelet service")
+
+	// Verify that no Windows node addresses appear in any EndpointSlice
+	var foundWindowsNodes []string
+	for _, slice := range endpointSlices.Items {
+		for _, endpoint := range slice.Endpoints {
+			// Check if this endpoint references a node
+			if endpoint.TargetRef == nil || endpoint.TargetRef.Kind != "Node" {
+				continue
+			}
+			// Try to get the node to check its OS
+			node, err := tc.client.K8s.CoreV1().Nodes().Get(context.Background(),
+				endpoint.TargetRef.Name, metav1.GetOptions{})
+			require.NoError(t, err, "node %s referenced in EndpointSlice not found - EndpointSlice not synced properly",
+				endpoint.TargetRef.Name)
+			// Check if this is a Windows node
+			if nodeOS, exists := node.Labels["kubernetes.io/os"]; exists && nodeOS == "windows" {
+				foundWindowsNodes = append(foundWindowsNodes, node.Name)
+			}
+		}
+	}
+
+	require.Empty(t, foundWindowsNodes,
+		"Found Windows nodes in kubelet EndpointSlices after deletion: %v", foundWindowsNodes)
+}
+
 // PrometheusQuery defines the result of the /query request
 // Example Reference of Prometheus Query Response: https://prometheus.io/docs/prometheus/latest/querying/api/
 type PrometheusQuery struct {
@@ -239,9 +274,12 @@ func (tc *testContext) testPodMetrics(t *testing.T, podName string) {
 		fmt.Sprintf("pod_interface_network:container_network_receive_bytes:irate5m{pod='%s',namespace='%s'}", podName, tc.workloadNamespace),
 		fmt.Sprintf("pod_interface_network:container_network_transmit_bytes_total:irate5m{pod='%s',namespace='%s'}", podName, tc.workloadNamespace),
 	}
+	// Use extended timeout to account for EndpointSlice propagation, metric scraping,
+	// and recording rule evaluation when using EndpointSlice-based service discovery
+	podMetricsTimeout := 5 * time.Minute
 	for i, query := range queries {
 		t.Run("query "+strconv.Itoa(i), func(t *testing.T) {
-			err := wait.PollUntilContextTimeout(context.TODO(), retry.Interval, retry.ResourceChangeTimeout, true,
+			err := wait.PollUntilContextTimeout(context.TODO(), retry.Interval, podMetricsTimeout, true,
 				func(ctx context.Context) (done bool, err error) {
 					results, err := makePrometheusQuery(prometheusRoute.Spec.Host, query, prometheusToken)
 					if err != nil {