Adding a LastUpdated metric so that we can see when images were lookedup and if we have stale data (#346)

Author: davidcollom

pkg/metrics/metrics.go

@@ -25,6 +25,7 @@ type Metrics struct {
 
 	registry               *prometheus.Registry
 	containerImageVersion  *prometheus.GaugeVec
+	containerImageChecked  *prometheus.GaugeVec
 	containerImageDuration *prometheus.GaugeVec
 	containerImageErrors   *prometheus.CounterVec
 
@@ -60,6 +61,16 @@ func NewServer(log *logrus.Entry) *Metrics {
 			"namespace", "pod", "container", "container_type", "image", "current_version", "latest_version",
 		},
 	)
+	containerImageChecked := promauto.With(reg).NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "version_checker",
+			Name:      "last_checked",
+			Help:      "Timestamp when the image was checked",
+		},
+		[]string{
+			"namespace", "pod", "container", "container_type", "image",
+		},
+	)
 	containerImageDuration := promauto.With(reg).NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: "version_checker",
@@ -84,6 +95,7 @@ func NewServer(log *logrus.Entry) *Metrics {
 		registry:               reg,
 		containerImageVersion:  containerImageVersion,
 		containerImageDuration: containerImageDuration,
+		containerImageChecked:  containerImageChecked,
 		containerImageErrors:   containerImageErrors,
 		containerCache:         make(map[string]cacheItem),
 		roundTripper:           NewRoundTripper(reg),
@@ -135,9 +147,14 @@ func (m *Metrics) AddImage(namespace, pod, container, containerType, imageURL st
 	}
 
 	m.containerImageVersion.With(
-		m.buildLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion),
+		m.buildFullLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion),
 	).Set(isLatestF)
 
+	// Bump last updated timestamp
+	m.containerImageChecked.With(
+		m.buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL),
+	).Set(float64(time.Now().Unix()))
+
 	index := m.latestImageIndex(namespace, pod, container, containerType)
 	m.containerCache[index] = cacheItem{
 		image:          imageURL,
@@ -162,6 +179,9 @@ func (m *Metrics) RemoveImage(namespace, pod, container, containerType string) {
 	m.containerImageDuration.DeletePartialMatch(
 		m.buildPartialLabels(namespace, pod),
 	)
+	m.containerImageChecked.DeletePartialMatch(
+		m.buildPartialLabels(namespace, pod),
+	)
 	delete(m.containerCache, index)
 }
 
@@ -178,11 +198,10 @@ func (m *Metrics) latestImageIndex(namespace, pod, container, containerType stri
 }
 
 func (m *Metrics) ErrorsReporting(namespace, pod, container, imageURL string) {
-
 	m.containerImageErrors.WithLabelValues(namespace, pod, container, imageURL).Inc()
 }
 
-func (m *Metrics) buildLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion string) prometheus.Labels {
+func (m *Metrics) buildFullLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion string) prometheus.Labels {
 	return prometheus.Labels{
 		"namespace":       namespace,
 		"pod":             pod,
@@ -194,6 +213,16 @@ func (m *Metrics) buildLabels(namespace, pod, container, containerType, imageURL
 	}
 }
 
+func (m *Metrics) buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL string) prometheus.Labels {
+	return prometheus.Labels{
+		"namespace":      namespace,
+		"pod":            pod,
+		"container_type": containerType,
+		"container":      container,
+		"image":          imageURL,
+	}
+}
+
 func (m *Metrics) buildPartialLabels(namespace, pod string) prometheus.Labels {
 	return prometheus.Labels{
 		"namespace": namespace,

pkg/metrics/metrics_test.go

@@ -3,40 +3,60 @@ package metrics
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/sirupsen/logrus"
+
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestCache(t *testing.T) {
 	m := NewServer(logrus.NewEntry(logrus.New()))
 
+	// Lets add some Images/Metrics...
 	for i, typ := range []string{"init", "container"} {
 		version := fmt.Sprintf("0.1.%d", i)
 		m.AddImage("namespace", "pod", "container", typ, "url", true, version, version)
 	}
 
+	// Check and ensure that the metrics are available...
 	for i, typ := range []string{"init", "container"} {
 		version := fmt.Sprintf("0.1.%d", i)
-		mt, _ := m.containerImageVersion.GetMetricWith(m.buildLabels("namespace", "pod", "container", typ, "url", version, version))
+		mt, err := m.containerImageVersion.GetMetricWith(m.buildFullLabels("namespace", "pod", "container", typ, "url", version, version))
+		require.NoError(t, err)
+		count := testutil.ToFloat64(mt)
+		assert.Equal(t, count, float64(1))
+	}
+
+	// as well as the lastUpdated...
+	for _, typ := range []string{"init", "container"} {
+		mt, err := m.containerImageChecked.GetMetricWith(m.buildLastUpdatedLabels("namespace", "pod", "container", typ, "url"))
+		require.NoError(t, err)
 		count := testutil.ToFloat64(mt)
-		if count != 1 {
-			t.Error("Should have added metric")
-		}
+		assert.GreaterOrEqual(t, count, float64(time.Now().Unix()))
 	}
 
+	// Remove said metrics...
 	for _, typ := range []string{"init", "container"} {
 		m.RemoveImage("namespace", "pod", "container", typ)
 	}
+	// Ensure metrics and values return 0
 	for i, typ := range []string{"init", "container"} {
 		version := fmt.Sprintf("0.1.%d", i)
-		mt, _ := m.containerImageVersion.GetMetricWith(m.buildLabels("namespace", "pod", "container", typ, "url", version, version))
+		mt, err := m.containerImageVersion.GetMetricWith(m.buildFullLabels("namespace", "pod", "container", typ, "url", version, version))
+		require.NoError(t, err)
+		count := testutil.ToFloat64(mt)
+		assert.Equal(t, count, float64(0))
+	}
+	// And the Last Updated is removed too
+	for _, typ := range []string{"init", "container"} {
+		mt, err := m.containerImageChecked.GetMetricWith(m.buildLastUpdatedLabels("namespace", "pod", "container", typ, "url"))
+		require.NoError(t, err)
 		count := testutil.ToFloat64(mt)
-		if count != 0 {
-			t.Error("Should have removed metric")
-		}
+		assert.Equal(t, count, float64(0))
 	}
 }
 
@@ -71,7 +91,7 @@ func TestErrorsReporting(t *testing.T) {
 				"container": tc.container,
 				"image":     tc.image,
 			})
-			assert.NoError(t, err, "Failed to get metric with labels")
+			require.NoError(t, err, "Failed to get metric with labels")
 
 			// Validate metric count
 			fetchErrorCount := testutil.ToFloat64(metric)