POC - Test Gathering Prometheus data

Author: camilamacedo86

.github/workflows/e2e.yaml

@@ -35,6 +35,11 @@ jobs:
       - name: Run e2e tests
         run: ARTIFACT_PATH=/tmp/artifacts make test-e2e
 
+      - uses: actions/upload-artifact@v4
+        with:
+          name: upgrade-e2e-artifacts
+          path: test/e2e/results/**
+
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

internal/catalogd/controllers/core/clustercatalog_controller.go

@@ -152,6 +152,7 @@ func (r *ClusterCatalogReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&ocv1.ClusterCatalog{}).
+		Named("catalogd-clustercatalog-controller").
 		Complete(r)
 }
 

internal/catalogd/controllers/core/pull_secret_controller.go

@@ -64,6 +64,7 @@ func (r *PullSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 func (r *PullSecretReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	_, err := ctrl.NewControllerManagedBy(mgr).
 		For(&corev1.Secret{}).
+		Named("catalogd-pull-secret-controller").
 		WithEventFilter(newSecretPredicate(r.SecretKey)).
 		Build(r)
 

internal/operator-controller/controllers/clustercatalog_controller.go

@@ -93,6 +93,7 @@ func (r *ClusterCatalogReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 // SetupWithManager sets up the controller with the Manager.
 func (r *ClusterCatalogReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	_, err := ctrl.NewControllerManagedBy(mgr).
+		Named("con-oper-clustercatalog-controller").
 		For(&ocv1.ClusterCatalog{}).
 		Build(r)
 

internal/operator-controller/controllers/clusterextension_controller.go

@@ -407,6 +407,7 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, bundleName string, depreca
 func (r *ClusterExtensionReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	controller, err := ctrl.NewControllerManagedBy(mgr).
 		For(&ocv1.ClusterExtension{}).
+		Named("con-oper-cluster-extension-controller").
 		Watches(&ocv1.ClusterCatalog{},
 			crhandler.EnqueueRequestsFromMapFunc(clusterExtensionRequestsForCatalog(mgr.GetClient(), mgr.GetLogger())),
 			builder.WithPredicates(predicate.Funcs{

internal/operator-controller/controllers/pull_secret_controller.go

@@ -63,6 +63,7 @@ func (r *PullSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 // SetupWithManager sets up the controller with the Manager.
 func (r *PullSecretReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	_, err := ctrl.NewControllerManagedBy(mgr).
+		Named("con-oper-pull-secret-controller").
 		For(&corev1.Secret{}).
 		WithEventFilter(newSecretPredicate(r.SecretKey)).
 		Build(r)

test/e2e/cluster_extension_install_test.go

@@ -377,6 +377,9 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {
 					assert.NotEmpty(ct, clusterExtension.Status.Install.Bundle)
 				}
 			}, pollDuration, pollInterval)
+
+			FetchCatalogdMetricsExportedEndpoint(t)
+			FetchOperatorControllerMetricsExportedEndpoint(t)
 		})
 	}
 }
@@ -455,6 +458,9 @@ location = "docker-registry.operator-controller-e2e.svc.cluster.local:5000"`,
 			assert.NotEmpty(ct, clusterExtension.Status.Install.Bundle)
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
@@ -505,6 +511,9 @@ func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
 			assert.Contains(ct, cond.Message, "in multiple catalogs with the same priority [extra-test-catalog test-catalog]")
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) {
@@ -568,6 +577,9 @@ func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) {
 			assert.Equal(ct, "error upgrading from currently installed version \"1.0.0\": no bundles found for package \"test\" matching version \"1.2.0\"", cond.Message)
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) {
@@ -618,6 +630,9 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) {
 			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionInstallSuccessorVersion(t *testing.T) {
@@ -666,6 +681,9 @@ func TestClusterExtensionInstallSuccessorVersion(t *testing.T) {
 			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) {
@@ -733,6 +751,9 @@ func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) {
 			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) {
@@ -814,6 +835,9 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) {
 			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T) {
@@ -866,6 +890,9 @@ func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T
 	require.EventuallyWithT(t, func(ct *assert.CollectT) {
 		assert.NoError(ct, globalClient.Get(context.Background(), types.NamespacedName{Name: testConfigMap.Name, Namespace: testConfigMap.Namespace}, testConfigMap))
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }
 
 func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *testing.T) {
@@ -959,4 +986,7 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes
 			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
 		}
 	}, pollDuration, pollInterval)
+
+	FetchCatalogdMetricsExportedEndpoint(t)
+	FetchOperatorControllerMetricsExportedEndpoint(t)
 }

test/e2e/metrics_test.go

@@ -17,6 +17,8 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -36,8 +38,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 )
 
-// TestOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller
-func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) {
+// FetchOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller
+func FetchOperatorControllerMetricsExportedEndpoint(t *testing.T) {
 	kubeClient, restConfig := findK8sClient(t)
 	mtc := NewMetricsTestConfig(
 		t,
@@ -49,13 +51,14 @@ func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) {
 		"operator-controller-controller-manager",
 		"oper-curl-metrics",
 		"https://operator-controller-service.NAMESPACE.svc.cluster.local:8443/metrics",
+		"oper-con",
 	)
 
 	mtc.run()
 }
 
-// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd
-func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
+// FetchCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd
+func FetchCatalogdMetricsExportedEndpoint(t *testing.T) {
 	kubeClient, restConfig := findK8sClient(t)
 	mtc := NewMetricsTestConfig(
 		t,
@@ -67,11 +70,68 @@ func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
 		"catalogd-controller-manager",
 		"catalogd-curl-metrics",
 		"https://catalogd-service.NAMESPACE.svc.cluster.local:7443/metrics",
+		"catalogd",
 	)
 
 	mtc.run()
 }
 
+// fetchMetrics retrieves Prometheus metrics from the endpoint
+func (c *MetricsTestConfig) fetchMetrics(ctx context.Context, token string) string {
+	c.t.Log("Fetching Prometheus metrics after test execution")
+
+	// Command to execute inside the pod
+	cmd := []string{
+		"curl", "-s", "-k",
+		"-H", "Authorization: Bearer " + token,
+		c.metricsURL,
+	}
+
+	// Execute command in pod
+	req := c.kubeClient.CoreV1().RESTClient().
+		Post().
+		Resource("pods").
+		Namespace(c.namespace).
+		Name(c.curlPodName).
+		SubResource("exec").
+		VersionedParams(&corev1.PodExecOptions{
+			Container: "curl",
+			Command:   cmd,
+			Stdin:     false,
+			Stdout:    true,
+			Stderr:    true,
+			TTY:       false,
+		}, scheme.ParameterCodec)
+
+	executor, err := remotecommand.NewSPDYExecutor(c.restConfig, "POST", req.URL())
+	require.NoError(c.t, err, "Error creating SPDY executor")
+
+	var stdout, stderr bytes.Buffer
+	streamOpts := remotecommand.StreamOptions{
+		Stdout: &stdout,
+		Stderr: &stderr,
+	}
+
+	err = executor.StreamWithContext(ctx, streamOpts)
+	require.NoError(c.t, err, "Error streaming exec request: %v", stderr.String())
+
+	return stdout.String()
+}
+
+// saveMetricsToFile writes the fetched metrics to a text file
+func (c *MetricsTestConfig) saveMetricsToFile(metrics string) {
+	dir := "results"
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		c.t.Fatalf("Failed to create directory %s: %v", dir, err)
+	}
+
+	filePath := fmt.Sprintf("%s/metrics_%s_%s.txt", dir, c.name, c.t.Name())
+	err := os.WriteFile(filePath, []byte(metrics), 0644)
+	require.NoError(c.t, err, "Failed to save metrics to file")
+
+	c.t.Logf("Metrics saved to: %s", filePath)
+}
+
 func findK8sClient(t *testing.T) (kubernetes.Interface, *rest.Config) {
 	cfg, err := config.GetConfig()
 	require.NoError(t, err, "Failed to get Kubernetes config")
@@ -94,6 +154,7 @@ type MetricsTestConfig struct {
 	serviceAccount string
 	curlPodName    string
 	metricsURL     string
+	name           string
 }
 
 // NewMetricsTestConfig initializes a new MetricsTestConfig.
@@ -107,6 +168,7 @@ func NewMetricsTestConfig(
 	serviceAccount string,
 	curlPodName string,
 	metricsURL string,
+	name string,
 ) *MetricsTestConfig {
 	// Discover which namespace the relevant Pod is running in
 	namespace := getComponentNamespace(t, kubeClient, selector)
@@ -124,24 +186,41 @@ func NewMetricsTestConfig(
 		serviceAccount: serviceAccount,
 		curlPodName:    curlPodName,
 		metricsURL:     metricsURL,
+		name:           name,
 	}
 }
 
 // run executes the entire test flow
 func (c *MetricsTestConfig) run() {
 	ctx := context.Background()
-	defer c.cleanup(ctx)
+	// To speed up
+	// defer c.cleanup(ctx)
 	c.createMetricsClusterRoleBinding(ctx)
 	token := c.getServiceAccountToken(ctx)
 	c.createCurlMetricsPod(ctx)
 	c.waitForPodReady(ctx)
 	// Exec `curl` in the Pod to validate the metrics
 	c.validateMetricsEndpoint(ctx, token)
+
+	// Fetch and save Prometheus metrics after test execution
+	metrics := c.fetchMetrics(ctx, token)
+	c.saveMetricsToFile(metrics)
 }
 
 // createMetricsClusterRoleBinding to bind the cluster role so metrics are accessible
 func (c *MetricsTestConfig) createMetricsClusterRoleBinding(ctx context.Context) {
-	c.t.Logf("Creating ClusterRoleBinding %q in namespace %q", c.clusterBinding, c.namespace)
+	c.t.Logf("Ensuring ClusterRoleBinding %q exists in namespace %q", c.clusterBinding, c.namespace)
+
+	_, err := c.kubeClient.RbacV1().ClusterRoleBindings().Get(ctx, c.clusterBinding, metav1.GetOptions{})
+	if err == nil {
+		c.t.Logf("ClusterRoleBinding %q already exists, skipping creation", c.clusterBinding)
+		return
+	}
+
+	if !apierrors.IsNotFound(err) {
+		require.NoError(c.t, err, "Error checking for existing ClusterRoleBinding")
+		return
+	}
 
 	crb := &rbacv1.ClusterRoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
@@ -161,8 +240,9 @@ func (c *MetricsTestConfig) createMetricsClusterRoleBinding(ctx context.Context)
 		},
 	}
 
-	_, err := c.kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{})
+	_, err = c.kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{})
 	require.NoError(c.t, err, "Error creating ClusterRoleBinding")
+	c.t.Logf("Successfully created ClusterRoleBinding %q", c.clusterBinding)
 }
 
 // getServiceAccountToken creates a TokenRequest for the service account
@@ -188,7 +268,18 @@ func (c *MetricsTestConfig) getServiceAccountToken(ctx context.Context) string {
 
 // createCurlMetricsPod spawns a pod running `curlimages/curl` to check metrics
 func (c *MetricsTestConfig) createCurlMetricsPod(ctx context.Context) {
-	c.t.Logf("Creating curl pod (%s/%s) to validate the metrics endpoint", c.namespace, c.curlPodName)
+	c.t.Logf("Ensuring curl pod (%s/%s) exists to validate the metrics endpoint", c.namespace, c.curlPodName)
+
+	_, err := c.kubeClient.CoreV1().Pods(c.namespace).Get(ctx, c.curlPodName, metav1.GetOptions{})
+	if err == nil {
+		c.t.Logf("Curl pod %q already exists, skipping creation", c.curlPodName)
+		return
+	}
+
+	if !apierrors.IsNotFound(err) {
+		require.NoError(c.t, err, "Error checking for existing curl pod")
+		return
+	}
 
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
@@ -220,7 +311,7 @@ func (c *MetricsTestConfig) createCurlMetricsPod(ctx context.Context) {
 		},
 	}
 
-	_, err := c.kubeClient.CoreV1().Pods(c.namespace).Create(ctx, pod, metav1.CreateOptions{})
+	_, err = c.kubeClient.CoreV1().Pods(c.namespace).Create(ctx, pod, metav1.CreateOptions{})
 	require.NoError(c.t, err, "Error creating curl pod")
 }