feat(metrics): record sync count for subscriptions, labeled with name

and most recently installed CSV version

Author: ecordell

Makefile

@@ -56,7 +56,7 @@ build-linux: build_cmd=build
 build-linux: arch_flags=GOOS=linux GOARCH=386
 build-linux: clean $(CMDS)
 
-$(CMDS): version_flags=-ldflags "-w -X $(PKG)/pkg/version.GitCommit=`git rev-parse --short HEAD` -X $(PKG)/pkg/version.OLMVersion=`cat OLM_VERSION`"
+$(CMDS): version_flags=-ldflags "-X $(PKG)/pkg/version.GitCommit=`git rev-parse --short HEAD` -X $(PKG)/pkg/version.OLMVersion=`cat OLM_VERSION`"
 $(CMDS):
 	CGO_ENABLED=0 $(arch_flags) go $(build_cmd) $(MOD_FLAGS) $(version_flags) -o bin/$(shell basename $@) $@
 
@@ -87,7 +87,7 @@ setup-bare: clean e2e.namespace
 	. ./scripts/install_bare.sh $(shell cat ./e2e.namespace) test/e2e/resources
 
 e2e:
-	go test -v -failfast -timeout 70m ./test/e2e/... -namespace=openshift-operators -kubeconfig=${KUBECONFIG} -olmNamespace=openshift-operator-lifecycle-manager
+	go test -v $(MOD_FLAGS) -failfast -timeout 70m ./test/e2e/... -namespace=openshift-operators -kubeconfig=${KUBECONFIG} -olmNamespace=openshift-operator-lifecycle-manager
 
 e2e-local: build-linux
 	. ./scripts/build_local.sh

pkg/controller/operators/catalog/subscription/syncer.go

@@ -15,6 +15,7 @@ import (
 	listers "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/kubestate"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 )
 
 var scheme = runtime.NewScheme()
@@ -49,6 +50,8 @@ func (s *subscriptionSyncer) Sync(ctx context.Context, event kubestate.ResourceE
 		return err
 	}
 
+	s.recordMetrics(res)
+
 	logger := s.logger.WithFields(logrus.Fields{
 		"reconciling": fmt.Sprintf("%T", res),
 		"selflink":    res.GetSelfLink(),
@@ -82,6 +85,10 @@ func (s *subscriptionSyncer) Sync(ctx context.Context, event kubestate.ResourceE
 	return nil
 }
 
+func (o *subscriptionSyncer) recordMetrics(sub *v1alpha1.Subscription) {
+	metrics.CounterForSubscription(sub.GetName(), sub.Status.InstalledCSV).Inc()
+}
+
 func (s *subscriptionSyncer) Notify(event kubestate.ResourceEvent) {
 	s.notify(event)
 }

pkg/controller/operators/olm/operator.go

@@ -678,8 +678,6 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error)
 		return
 	}
 
-	a.recordMetrics(logger, clusterServiceVersion)
-
 	outCSV, syncError := a.transitionCSVState(*clusterServiceVersion)
 
 	if outCSV == nil {
@@ -779,18 +777,6 @@ func (a *Operator) syncGcCsv(obj interface{}) (syncError error) {
 	return
 }
 
-func (a *Operator) recordMetrics(logger *logrus.Entry, csv *v1alpha1.ClusterServiceVersion) {
-	if csv.IsCopied() {
-		// don't record for copied CSVs
-		return
-	}
-	counter, err := metrics.CounterForCSV(csv.GetName(), string(csv.Status.Phase))
-	if err != nil {
-		logger.WithError(err).Warn("could not record metrics")
-	}
-	counter.Inc()
-}
-
 // operatorGroupFromAnnotations returns the OperatorGroup for the CSV only if the CSV is active one in the group
 func (a *Operator) operatorGroupFromAnnotations(logger *logrus.Entry, csv *v1alpha1.ClusterServiceVersion) *v1.OperatorGroup {
 	annotations := csv.GetAnnotations()

pkg/metrics/metrics.go

@@ -10,8 +10,8 @@ import (
 )
 
 const (
-	PHASE_LABEL = "phase"
-	NAME_LABEL  = "name"
+	NAME_LABEL      = "name"
+	INSTALLED_LABEL = "installed"
 )
 
 // TODO(alecmerdler): Can we use this to emit Kubernetes events?
@@ -138,30 +138,27 @@ var (
 		},
 	)
 
-	CSVSyncCount = prometheus.NewCounterVec(
+	SubscriptionSyncCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "csv_sync_total",
-			Help: "Monotonic count of CSV syncs",
+			Name: "subscription_sync_total",
+			Help: "Monotonic count of subscription syncs",
 		},
-		[]string{PHASE_LABEL, NAME_LABEL},
+		[]string{NAME_LABEL, INSTALLED_LABEL},
 	)
 )
 
 func RegisterOLM() {
 	prometheus.MustRegister(csvCount)
 	prometheus.MustRegister(CSVUpgradeCount)
-	prometheus.MustRegister(CSVSyncCount)
 }
 
 func RegisterCatalog() {
 	prometheus.MustRegister(installPlanCount)
 	prometheus.MustRegister(subscriptionCount)
 	prometheus.MustRegister(catalogSourceCount)
+	prometheus.MustRegister(SubscriptionSyncCount)
 }
 
-func CounterForCSV(name, phase string) (prometheus.Counter, error) {
-	return CSVSyncCount.GetMetricWith(map[string]string{
-		PHASE_LABEL: phase,
-		NAME_LABEL:  name,
-	})
+func CounterForSubscription(name, installedCSV string) prometheus.Counter {
+	return SubscriptionSyncCount.WithLabelValues(name, installedCSV)
 }

test/e2e/metrics_e2e_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/net"
 
@@ -34,11 +33,7 @@ func TestMetricsEndpoint(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Metrics test failed: %v\n", err)
 	}
-
-	// Verify metrics have been emitted for packageserver csv
-	require.Contains(t, rawOutput, "csv_sync_total counter")
-	require.Contains(t, rawOutput, "phase=\"Succeeded\"")
-	require.Contains(t, rawOutput, "packageserver")
+	log.Info(rawOutput)
 }
 
 func getMetricsFromPod(t *testing.T, client operatorclient.ClientInterface, podName string, namespace string, port string) (string, error) {