OCPBUGS-61508: IngressOperator not exposing some metrics for degraded IngressController

Signed-off-by: Davide Salerno <[email protected]>

Author: davidesalerno

pkg/operator/controller/ingress/status.go

@@ -104,10 +104,10 @@ func (r *reconciler) syncIngressControllerStatus(ic *operatorv1.IngressControlle
 			errs = append(errs, fmt.Errorf("failed to update ingresscontroller status: %v", err))
 		} else {
 			updatedIc = true
-			SetIngressControllerConditionsMetric(updated)
 		}
 	}
-
+	//OCPBUGS-61508 set at every reconcile the ingress_controller_conditions metrics
+	SetIngressControllerConditionsMetric(updated)
 	return retryableerror.NewMaybeRetryableAggregate(errs), updatedIc
 }
 

test/e2e/all_test.go

@@ -136,5 +136,6 @@ func TestAll(t *testing.T) {
 		// Serializing the test ensures it runs in isolation with other tests,
 		// preventing any impact of the mutating webhook on pod creation in the cluster
 		t.Run("TestGatewayAPI", TestGatewayAPI)
+		t.Run("TestIngressControllerConditionsMetricAfterRestart", TestIngressControllerConditionsMetricAfterRestart)
 	})
 }

test/e2e/ic_conditions_metric_test.go

@@ -0,0 +1,79 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func queryPrometheusForMetric(t *testing.T, metric string) string {
+	// You might need to adjust the namespace and Prometheus route
+	t.Log("Retrieving Prometheus metric from Ingress operator pod..")
+	cmd := exec.Command("oc", "exec", "-n", "openshift-ingress-operator", "deploy/ingress-operator", "--",
+		"curl", "-s", "localhost:60000/metrics")
+	out, err := cmd.Output()
+	require.NoError(t, err, "Failed to query Prometheus for metric due to error %s", err)
+	return string(out)
+}
+
+func restartOperatorPod(t *testing.T) {
+	// Find the operator pod
+	t.Logf("Restarting Ingress operator pod...")
+	getPodCmd := exec.Command("oc", "get", "pod", "-n", "openshift-ingress-operator", "-l", "name=ingress-operator", "-o", "jsonpath={.items[0].metadata.name}")
+	podNameBytes, err := getPodCmd.Output()
+	require.NoError(t, err)
+	podName := strings.TrimSpace(string(podNameBytes))
+
+	// Delete the pod
+	delPodCmd := exec.Command("oc", "delete", "pod", podName, "-n", "openshift-ingress-operator")
+	err = delPodCmd.Run()
+	require.NoError(t, err)
+
+	// Wait for new pod to be ready
+	for i := 0; i < 30; i++ {
+		getPodCmd := exec.Command("oc", "get", "pod", "-n", "openshift-ingress-operator", "-l", "name=ingress-operator", "-o", "jsonpath={.items[0].status.phase}")
+		phaseBytes, _ := getPodCmd.Output()
+		if strings.TrimSpace(string(phaseBytes)) == "Running" {
+			time.Sleep(10 * time.Second) // wait for full readiness
+			return
+		}
+		time.Sleep(5 * time.Second)
+	}
+	t.Fatalf("Operator pod did not restart in time")
+}
+
+// TestIngressControllerConditionsMetricAfterRestart verifies that metric ingress_controller_conditions(router,status) is
+// available after an operator pod restart too.
+//
+// This test:
+//
+// 1. Verifies that the metric is available in a normal situation when the operator pod is up&running (i.e. before restart)
+//
+// 2. Restarts the operator pod, waiting it will be available again
+//
+// 3. Repeats the step 1 expecting the same result again and so the presence of the metric
+//
+// NB: due to the fact that this test is restarting the operator pod it cannot be executed in parallel with other tests
+func TestIngressControllerConditionsMetricAfterRestart(t *testing.T) {
+	metricName := "ingress_controller_conditions"
+
+	// Check metric before restart
+	outBefore := queryPrometheusForMetric(t, metricName)
+	t.Logf("Verifying that in Prometheus metrics there are %s metrics before resart", metricName)
+	require.Contains(t, outBefore, metricName, fmt.Sprintf("Metrics %s not found before operator restart", metricName))
+
+	// Restart operator pod
+	restartOperatorPod(t)
+
+	// Check metric after restart
+	outAfter := queryPrometheusForMetric(t, metricName)
+	t.Logf("Verifying that in Prometheus metrics there are %s after a pod restart too", metricName)
+	require.Contains(t, outAfter, metricName, fmt.Sprintf("Metrics %s not found after operator restart", metricName))
+}