Merge pull request #768 from ricardomaraschini/imagestream-metrics

IR-120: Implementing metrics for ImageStreams

Author: openshift-merge-robot

pkg/metrics/metrics.go

@@ -19,12 +19,28 @@ var (
 		},
 		[]string{"result"},
 	)
+	imageStreamTags = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "image_registry_image_stream_tags_total",
+			Help: "Number of image stream tags. Source is either 'imported' or 'pushed'. 'location' label shows if the tag lives in one of the 'openshift' namespaces or 'other'",
+		},
+		[]string{"source", "location"},
+	)
+	storageType = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "image_registry_storage_type",
+			Help: "Holds the storage in use for the image registry",
+		},
+		[]string{"storage"},
+	)
 )
 
 func init() {
 	registry.MustRegister(
 		storageReconfigured,
 		imagePrunerInstallStatus,
 		azurePrimaryKeyCache,
+		imageStreamTags,
+		storageType,
 	)
 }

pkg/metrics/server.go

@@ -60,6 +60,25 @@ func ImagePrunerInstallStatus(installed bool, enabled bool) {
 	imagePrunerInstallStatus.Set(2)
 }
 
+// ReportOpenShiftImageStreamTags reports the amount of seen ImageStream tags existing in openshift
+// namespaces. Receives the total of 'imported' and 'pushed' image streams tags.
+func ReportOpenShiftImageStreamTags(imported float64, pushed float64) {
+	imageStreamTags.WithLabelValues("imported", "openshift").Set(imported)
+	imageStreamTags.WithLabelValues("pushed", "openshift").Set(pushed)
+}
+
+// ReportOtherImageStreamTags reports the amount of seen ImageStream tags existing outside the
+// openshift namespaces. Receives the total of 'imported' and 'pushed' image streams tags.
+func ReportOtherImageStreamTags(imported float64, pushed float64) {
+	imageStreamTags.WithLabelValues("imported", "other").Set(imported)
+	imageStreamTags.WithLabelValues("pushed", "other").Set(pushed)
+}
+
+// ReportStorageType sets the storage in use.
+func ReportStorageType(stype string) {
+	storageType.WithLabelValues(stype).Set(1)
+}
+
 // AzureKeyCacheHit registers a hit on Azure key cache.
 func AzureKeyCacheHit() {
 	azurePrimaryKeyCache.With(map[string]string{"result": "hit"}).Inc()

pkg/operator/metrics.go

@@ -0,0 +1,97 @@
+package operator
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
+
+	imagev1 "github.com/openshift/api/image/v1"
+	imageinformers "github.com/openshift/client-go/image/informers/externalversions/image/v1"
+	imagelisters "github.com/openshift/client-go/image/listers/image/v1"
+
+	"github.com/openshift/cluster-image-registry-operator/pkg/metrics"
+)
+
+// MetricsController is a controller that runs from time to time and reports some metrics about
+// the current status of the system.
+type MetricsController struct {
+	lister imagelisters.ImageStreamLister
+	caches []cache.InformerSynced
+}
+
+// NewMetricsController returns a new MetricsController.
+func NewMetricsController(informer imageinformers.ImageStreamInformer) *MetricsController {
+	return &MetricsController{
+		lister: informer.Lister(),
+		caches: []cache.InformerSynced{informer.Informer().HasSynced},
+	}
+}
+
+// report gathers all metrics reported by this operator and calls approprate function in the
+// metrics package to report the current values.
+func (m *MetricsController) report(_ context.Context) {
+	imgstreams, err := m.lister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("unable to list image streams: %s", err)
+		return
+	}
+
+	var importedOpenShift float64
+	var pushedOpenShift float64
+	var importedOther float64
+	var pushedOther float64
+	for _, is := range imgstreams {
+		imported, pushed := m.assessImageStream(is.DeepCopy())
+		if strings.HasPrefix(is.Namespace, "openshift") {
+			importedOpenShift += imported
+			pushedOpenShift += pushed
+			continue
+		}
+
+		importedOther += imported
+		pushedOther += pushed
+	}
+
+	metrics.ReportOpenShiftImageStreamTags(importedOpenShift, pushedOpenShift)
+	metrics.ReportOtherImageStreamTags(importedOther, pushedOther)
+}
+
+// assessImageStream returns the number of imported and the number of pushed tags for the provided
+// image stream reference.
+func (m *MetricsController) assessImageStream(is *imagev1.ImageStream) (float64, float64) {
+	spectags := map[string]bool{}
+	for _, tag := range is.Spec.Tags {
+		spectags[tag.Name] = true
+	}
+
+	var imported float64
+	var pushed float64
+	for _, tag := range is.Status.Tags {
+		if _, ok := spectags[tag.Tag]; ok {
+			imported++
+		} else {
+			pushed++
+		}
+	}
+
+	return imported, pushed
+}
+
+// Run starts this controller. Runs the main loop in a separate go routine and bails out when
+// the provided context is finished.
+func (m *MetricsController) Run(ctx context.Context) {
+	klog.Infof("Starting MetricsController")
+	if !cache.WaitForCacheSync(ctx.Done(), m.caches...) {
+		return
+	}
+
+	go wait.UntilWithContext(ctx, m.report, time.Hour)
+	klog.Infof("Started MetricsController")
+	<-ctx.Done()
+	klog.Infof("Shutting down MetricsController")
+}

pkg/operator/starter.go

@@ -10,6 +10,8 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	configinformers "github.com/openshift/client-go/config/informers/externalversions"
+	imageclient "github.com/openshift/client-go/image/clientset/versioned"
+	imageinformers "github.com/openshift/client-go/image/informers/externalversions"
 	imageregistryclient "github.com/openshift/client-go/imageregistry/clientset/versioned"
 	imageregistryinformers "github.com/openshift/client-go/imageregistry/informers/externalversions"
 	routeclient "github.com/openshift/client-go/route/clientset/versioned"
@@ -38,6 +40,10 @@ func RunOperator(ctx context.Context, kubeconfig *restclient.Config) error {
 	if err != nil {
 		return err
 	}
+	imageClient, err := imageclient.NewForConfig(kubeconfig)
+	if err != nil {
+		return err
+	}
 
 	kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncDuration, kubeinformers.WithNamespace(defaults.ImageRegistryOperatorNamespace))
 	kubeInformersForOpenShiftConfig := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncDuration, kubeinformers.WithNamespace(defaults.OpenShiftConfigNamespace))
@@ -46,6 +52,7 @@ func RunOperator(ctx context.Context, kubeconfig *restclient.Config) error {
 	configInformers := configinformers.NewSharedInformerFactory(configClient, defaultResyncDuration)
 	imageregistryInformers := imageregistryinformers.NewSharedInformerFactory(imageregistryClient, defaultResyncDuration)
 	routeInformers := routeinformers.NewSharedInformerFactoryWithOptions(routeClient, defaultResyncDuration, routeinformers.WithNamespace(defaults.ImageRegistryOperatorNamespace))
+	imageInformers := imageinformers.NewSharedInformerFactory(imageClient, defaultResyncDuration)
 
 	configOperatorClient := client.NewConfigOperatorClient(
 		imageregistryClient.ImageregistryV1().Configs(),
@@ -124,13 +131,16 @@ func RunOperator(ctx context.Context, kubeconfig *restclient.Config) error {
 		kubeInformersForOpenShiftConfig.Core().V1().ConfigMaps(),
 	)
 
+	metricsController := NewMetricsController(imageInformers.Image().V1().ImageStreams())
+
 	kubeInformers.Start(ctx.Done())
 	kubeInformersForOpenShiftConfig.Start(ctx.Done())
 	kubeInformersForOpenShiftConfigManaged.Start(ctx.Done())
 	kubeInformersForKubeSystem.Start(ctx.Done())
 	configInformers.Start(ctx.Done())
 	imageregistryInformers.Start(ctx.Done())
 	routeInformers.Start(ctx.Done())
+	imageInformers.Start(ctx.Done())
 
 	go controller.Run(ctx.Done())
 	go clusterOperatorStatusController.Run(ctx.Done())
@@ -140,6 +150,7 @@ func RunOperator(ctx context.Context, kubeconfig *restclient.Config) error {
 	go imagePrunerController.Run(ctx.Done())
 	go loggingController.Run(ctx, 1)
 	go azureStackCloudController.Run(ctx)
+	go metricsController.Run(ctx)
 
 	<-ctx.Done()
 	return nil

pkg/storage/storage.go

@@ -13,6 +13,7 @@ import (
 	regopclient "github.com/openshift/cluster-image-registry-operator/pkg/client"
 	"github.com/openshift/cluster-image-registry-operator/pkg/defaults"
 	"github.com/openshift/cluster-image-registry-operator/pkg/envvar"
+	"github.com/openshift/cluster-image-registry-operator/pkg/metrics"
 	"github.com/openshift/cluster-image-registry-operator/pkg/storage/azure"
 	"github.com/openshift/cluster-image-registry-operator/pkg/storage/emptydir"
 	"github.com/openshift/cluster-image-registry-operator/pkg/storage/gcs"
@@ -113,6 +114,7 @@ func NewDriver(cfg *imageregistryv1.ImageRegistryConfigStorage, kubeconfig *rest
 	case 0:
 		return nil, ErrStorageNotConfigured
 	case 1:
+		metrics.ReportStorageType(names[0])
 		return drivers[0], nil
 	}