Merge pull request #1055 from awgreene/csv-health-metrics

feat(metrics) record CSV failures

Author: openshift-merge-robot

pkg/controller/operators/olm/operator.go

@@ -930,6 +930,8 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error)
 	})
 	logger.Debug("syncing CSV")
 
+	metrics.EmitCSVMetric(clusterServiceVersion)
+
 	if a.csvNotification != nil {
 		a.csvNotification.OnAddOrUpdate(clusterServiceVersion)
 	}

pkg/metrics/metrics.go

@@ -7,11 +7,16 @@ import (
 
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
 	v1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
+	olmv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
+
 )
 
 const (
 	NAME_LABEL      = "name"
 	INSTALLED_LABEL = "installed"
+	VERSION_LABEL   = "version"
+	PHASE_LABEL    = "phase"
+	REASON_LABEL    = "reason"
 )
 
 // TODO(alecmerdler): Can we use this to emit Kubernetes events?
@@ -145,10 +150,19 @@ var (
 		},
 		[]string{NAME_LABEL, INSTALLED_LABEL},
 	)
+
+	csvSyncCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "csv_sync_total",
+			Help: "Monotonic count of CSV syncs",
+		},
+		[]string{NAME_LABEL, VERSION_LABEL, PHASE_LABEL, REASON_LABEL},
+	)
 )
 
 func RegisterOLM() {
 	prometheus.MustRegister(csvCount)
+	prometheus.MustRegister(csvSyncCounter)
 	prometheus.MustRegister(CSVUpgradeCount)
 }
 
@@ -162,3 +176,7 @@ func RegisterCatalog() {
 func CounterForSubscription(name, installedCSV string) prometheus.Counter {
 	return SubscriptionSyncCount.WithLabelValues(name, installedCSV)
 }
+
+func EmitCSVMetric(csv *olmv1alpha1.ClusterServiceVersion){
+	csvSyncCounter.WithLabelValues(csv.Name, csv.Spec.Version.String(), string(csv.Status.Phase), string(csv.Status.Reason)).Inc()
+}

test/e2e/metrics_e2e_test.go

@@ -6,34 +6,72 @@ import (
 	"testing"
 
 	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/net"
 
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
 )
 
 // TestMetrics tests the metrics endpoint of the OLM pod.
 func TestMetricsEndpoint(t *testing.T) {
 	c := newKubeClient(t)
+	crc := newCRClient(t)
 
+	failingCSV := v1alpha1.ClusterServiceVersion{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       v1alpha1.ClusterServiceVersionKind,
+			APIVersion: v1alpha1.ClusterServiceVersionAPIVersion,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: genName("failing-csv-test-"),
+		},
+		Spec: v1alpha1.ClusterServiceVersionSpec{
+			InstallStrategy: v1alpha1.NamedInstallStrategy{
+				StrategyName:    install.InstallStrategyNameDeployment,
+				StrategySpecRaw: strategyRaw,
+			},
+		},
+	}
+
+	cleanupCSV, err := createCSV(t, c, crc, failingCSV, testNamespace, false, false)
+	require.NoError(t, err)
+	defer cleanupCSV()
+
+	_, err = fetchCSV(t, crc, failingCSV.Name, testNamespace, csvFailedChecker)
+	require.NoError(t, err)
+
+	rawOutput, err := getMetricsFromPod(t, c, getOLMPodName(t, c), operatorNamespace, "8081")
+	if err != nil {
+		t.Fatalf("Metrics test failed: %v\n", err)
+	}
+
+	// Verify metrics have been emitted for packageserver csv
+	require.Contains(t, rawOutput, "csv_sync_total")
+	require.Contains(t, rawOutput, "name=\""+failingCSV.Name+"\"")
+	require.Contains(t, rawOutput, "phase=\"Failed\"")
+	require.Contains(t, rawOutput, "reason=\"UnsupportedOperatorGroup\"")
+	require.Contains(t, rawOutput, "version=\"0.0.0\"")
+	log.Info(rawOutput)
+}
+
+func getOLMPodName(t *testing.T, client operatorclient.ClientInterface) string {
 	listOptions := metav1.ListOptions{LabelSelector: "app=olm-operator"}
-	podList, err := c.KubernetesInterface().CoreV1().Pods(operatorNamespace).List(listOptions)
+	podList, err := client.KubernetesInterface().CoreV1().Pods(operatorNamespace).List(listOptions)
 	if err != nil {
 		log.Infof("Error %v\n", err)
 		t.Fatalf("Listing pods failed: %v\n", err)
 	}
-	if len(podList.Items) > 1 {
-		t.Fatalf("Expected only 1 olm-operator pod, got %v", len(podList.Items))
+	if len(podList.Items) != 1 {
+		t.Fatalf("Expected 1 olm-operator pod, got %v", len(podList.Items))
 	}
 
 	podName := podList.Items[0].GetName()
 	log.Infof("Looking at pod %v in namespace %v", podName, operatorNamespace)
+	return podName
 
-	rawOutput, err := getMetricsFromPod(t, c, podName, operatorNamespace, "8081")
-	if err != nil {
-		t.Fatalf("Metrics test failed: %v\n", err)
-	}
-	log.Info(rawOutput)
 }
 
 func getMetricsFromPod(t *testing.T, client operatorclient.ClientInterface, podName string, namespace string, port string) (string, error) {