feat(metrics): record sync count for CSVs, labeled with name and phase

Author: ecordell

pkg/controller/operators/olm/operator.go

@@ -7,9 +7,9 @@ import (
 	"strings"
 	"time"
 
-	log "github.com/sirupsen/logrus"
 	v1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1"
 	"github.com/sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	extinf "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -678,6 +678,8 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error)
 		return
 	}
 
+	a.recordMetrics(logger, clusterServiceVersion)
+
 	outCSV, syncError := a.transitionCSVState(*clusterServiceVersion)
 
 	if outCSV == nil {
@@ -777,6 +779,18 @@ func (a *Operator) syncGcCsv(obj interface{}) (syncError error) {
 	return
 }
 
+func (a *Operator) recordMetrics(logger *logrus.Entry, csv *v1alpha1.ClusterServiceVersion) {
+	if csv.IsCopied() {
+		// don't record for copied CSVs
+		return
+	}
+	counter, err := metrics.CounterForCSV(csv.GetName(), string(csv.Status.Phase))
+	if err != nil {
+		logger.WithError(err).Warn("could not record metrics")
+	}
+	counter.Inc()
+}
+
 // operatorGroupFromAnnotations returns the OperatorGroup for the CSV only if the CSV is active one in the group
 func (a *Operator) operatorGroupFromAnnotations(logger *logrus.Entry, csv *v1alpha1.ClusterServiceVersion) *v1.OperatorGroup {
 	annotations := csv.GetAnnotations()
@@ -1426,7 +1440,7 @@ func (a *Operator) apiServiceOwnerConflicts(csv *v1alpha1.ClusterServiceVersion)
 			continue
 		}
 
-		adoptable, err := a.isAPIServiceAdoptable(csv, apiService) 
+		adoptable, err := a.isAPIServiceAdoptable(csv, apiService)
 		if err != nil {
 			a.logger.WithFields(log.Fields{"obj": "apiService", "labels": apiService.GetLabels()}).Errorf("adoption check failed - %v", err)
 		}

pkg/metrics/metrics.go

@@ -9,6 +9,11 @@ import (
 	v1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
 )
 
+const (
+	PHASE_LABEL = "phase"
+	NAME_LABEL  = "name"
+)
+
 // TODO(alecmerdler): Can we use this to emit Kubernetes events?
 type MetricsProvider interface {
 	HandleMetrics() error
@@ -132,15 +137,31 @@ var (
 			Help: "Monotonic count of CSV upgrades",
 		},
 	)
+
+	CSVSyncCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "csv_sync_total",
+			Help: "Monotonic count of CSV syncs",
+		},
+		[]string{PHASE_LABEL, NAME_LABEL},
+	)
 )
 
 func RegisterOLM() {
 	prometheus.MustRegister(csvCount)
 	prometheus.MustRegister(CSVUpgradeCount)
+	prometheus.MustRegister(CSVSyncCount)
 }
 
 func RegisterCatalog() {
 	prometheus.MustRegister(installPlanCount)
 	prometheus.MustRegister(subscriptionCount)
 	prometheus.MustRegister(catalogSourceCount)
 }
+
+func CounterForCSV(name, phase string) (prometheus.Counter, error) {
+	return CSVSyncCount.GetMetricWith(map[string]string{
+		PHASE_LABEL: phase,
+		NAME_LABEL:  name,
+	})
+}

test/e2e/metrics_e2e_test.go

@@ -5,10 +5,12 @@ package e2e
 import (
 	"testing"
 
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
 	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/net"
+
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
 )
 
 // TestMetrics tests the metrics endpoint of the OLM pod.
@@ -33,7 +35,10 @@ func TestMetricsEndpoint(t *testing.T) {
 		t.Fatalf("Metrics test failed: %v\n", err)
 	}
 
-	log.Debugf("Metrics:\n%v", rawOutput)
+	// Verify metrics have been emitted for packageserver csv
+	require.Contains(t, rawOutput, "csv_sync_total counter")
+	require.Contains(t, rawOutput, "phase=\"Succeeded\"")
+	require.Contains(t, rawOutput, "packageserver")
 }
 
 func getMetricsFromPod(t *testing.T, client operatorclient.ClientInterface, podName string, namespace string, port string) (string, error) {