OCPBUGS-61508: IngressOperator not exposing some metrics for degraded IngressController

Signed-off-by: Davide Salerno <[email protected]>

Author: davidesalerno

pkg/operator/controller/ingress/status.go

@@ -104,10 +104,10 @@ func (r *reconciler) syncIngressControllerStatus(ic *operatorv1.IngressControlle
 			errs = append(errs, fmt.Errorf("failed to update ingresscontroller status: %v", err))
 		} else {
 			updatedIc = true
-			SetIngressControllerConditionsMetric(updated)
 		}
 	}
-
+	//OCPBUGS-61508 set at every reconcile the ingress_controller_conditions metrics
+	SetIngressControllerConditionsMetric(updated)
 	return retryableerror.NewMaybeRetryableAggregate(errs), updatedIc
 }
 

test/e2e/all_test.go

@@ -136,5 +136,6 @@ func TestAll(t *testing.T) {
 		// Serializing the test ensures it runs in isolation with other tests,
 		// preventing any impact of the mutating webhook on pod creation in the cluster
 		t.Run("TestGatewayAPI", TestGatewayAPI)
+		t.Run("TestIngressControllerConditionsMetricAfterRestart", TestIngressControllerConditionsMetricAfterRestart)
 	})
 }

test/e2e/ic_conditions_metric_test.go

@@ -0,0 +1,142 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	routev1client "github.com/openshift/client-go/route/clientset/versioned"
+	"github.com/openshift/library-go/test/library/metrics"
+	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+// waitForIngressControllerConditionsMetrics waits for the metrics for ingress_controller_conditions to be present.
+func waitForIngressControllerConditionsMetrics(t *testing.T, prometheusClient prometheusv1.API, start time.Time) error {
+	t.Logf("Waiting for ingress_controller_conditions to be present")
+	if err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, 2*time.Minute, false, func(context context.Context) (bool, error) {
+		r := prometheusv1.Range{
+			Start: start,
+			End:   time.Now(),
+			Step:  time.Minute,
+		}
+		result, _, err := prometheusClient.QueryRange(context, "ingress_controller_conditions", r)
+		if err != nil {
+			t.Logf("Failed to fetch metrics: %v, retrying...", err)
+			return false, nil
+		}
+
+		// Check if fetched metrics is of Vector type.
+		matrix, ok := result.(model.Matrix)
+		if !ok {
+			t.Logf("Unexpected metric type, retrying...")
+			return false, nil
+		}
+
+		// Check if length of returned metric Vector is zero.
+		if len(matrix) == 0 {
+			t.Logf("Metric is empty, retrying...")
+			return false, nil
+		}
+
+		return true, nil
+	}); err != nil {
+		return fmt.Errorf("Error waiting for route metrics: %w", err)
+	}
+	return nil
+}
+func restartOperatorPod(t *testing.T, kubeClient kubernetes.Interface) time.Time {
+	// Find the operator pod
+	t.Logf("Restarting Ingress operator pod...")
+	podsList, err := kubeClient.CoreV1().Pods("openshift-ingress-operator").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		t.Fatalf("Failed listing pods in openshift-ingress-operator namespace: %v", err)
+	}
+	operatorPodName := ""
+	for _, pod := range podsList.Items {
+		if strings.HasPrefix(pod.Name, "ingress-operator") {
+			operatorPodName = pod.Name
+			break
+		}
+	}
+	if operatorPodName == "" {
+		t.Fatalf("Unable to find ingress operator pod")
+	}
+	// Delete the operator pod
+	if err := kubeClient.CoreV1().Pods("openshift-ingress-operator").Delete(context.TODO(), operatorPodName, metav1.DeleteOptions{}); err != nil {
+		t.Fatalf("failed to find the ingresscontroller operator pods: %v", err)
+	}
+
+	interval, timeout := 5*time.Second, 5*time.Minute
+	// Wait for new pod to be ready
+	t.Logf("Polling for up to %v to verify that the oprator restart is terminated...", timeout)
+	if err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, false, func(context context.Context) (bool, error) {
+		if podsList, err = kubeClient.CoreV1().Pods("openshift-ingress-operator").List(context, metav1.ListOptions{}); err != nil || len(podsList.Items) == 0 {
+			return false, err
+		}
+		return true, nil
+	}); err != nil {
+		t.Fatalf("Operator pod did not restart in time")
+	}
+	return time.Now()
+}
+
+// TestIngressControllerConditionsMetricAfterRestart verifies that metric ingress_controller_conditions(router,status) is
+// available after an operator pod restart too.
+//
+// This test:
+//
+// 1. Verifies that the metric is available in a normal situation when the operator pod is up&running (i.e. before restart)
+//
+// 2. Restarts the operator pod, waiting it will be available again
+//
+// 3. Repeats the step 1 expecting the same result again and so the presence of the metric
+//
+// NB:
+//  1. this test requires an OpenShift version with the monitoring stack up&running
+//  2. due to the fact that this test is restarting the operator pod it cannot be executed in parallel with other tests
+func TestIngressControllerConditionsMetricAfterRestart(t *testing.T) {
+	metricName := "ingress_controller_conditions"
+
+	// Create a new prometheus client for fetching metrics and dependencies needed
+	kubeConfig, err := config.GetConfig()
+	if err != nil {
+		t.Fatalf("Failed to get kube config: %s", err)
+	}
+	kubeClient, err := kubernetes.NewForConfig(kubeConfig)
+	if err != nil {
+		t.Fatalf("Failed to create kube client: %v", err)
+	}
+	routeClient, err := routev1client.NewForConfig(kubeConfig)
+	if err != nil {
+		t.Fatalf("Failed to create route client: %v", err)
+	}
+	prometheusClient, err := metrics.NewPrometheusClient(context.Background(), kubeClient, routeClient)
+	if err != nil {
+		t.Fatalf("Failed to create prometheus client: %v", err)
+	}
+
+	// Check metric before restart
+	t.Logf("Verifying that in Prometheus metrics there are %s metrics before resart", metricName)
+	// Wait for metrics to be added and set to 0.
+	if err := waitForIngressControllerConditionsMetrics(t, prometheusClient, time.Now().Add(-5*time.Minute)); err != nil {
+		t.Fatalf("Failed to fetch expected metrics: %v", err)
+	}
+
+	// Restart operator pod
+	restartTime := restartOperatorPod(t, kubeClient)
+
+	// Check metric after restart
+	if err := waitForIngressControllerConditionsMetrics(t, prometheusClient, restartTime); err != nil {
+		t.Fatalf("Failed to fetch expected metrics: %v", err)
+	}
+}