fix evict metric

Signed-off-by: alanprot <[email protected]>

Author: alanprot

pkg/ingester/ingester.go

@@ -719,7 +719,7 @@ func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registe
 
 	if cfg.MatchersCacheMaxItems > 0 {
 		r := prometheus.NewRegistry()
-		registerer.MustRegister(newMatchCacheMetrics(r))
+		registerer.MustRegister(newMatchCacheMetrics(r, logger))
 		i.matchersCache, err = storecache.NewMatchersCache(storecache.WithSize(cfg.MatchersCacheMaxItems), storecache.WithPromRegistry(r))
 		if err != nil {
 			return nil, err

pkg/ingester/ingester_test.go

@@ -130,8 +130,9 @@ func TestMatcherCache(t *testing.T) {
 	blocksDir := filepath.Join(dir, "blocks")
 	require.NoError(t, os.Mkdir(chunksDir, os.ModePerm))
 	require.NoError(t, os.Mkdir(blocksDir, os.ModePerm))
-
-	ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry, true)
+	cfg := defaultIngesterTestConfig(t)
+	cfg.MatchersCacheMaxItems = 50
+	ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry, true)
 	require.NoError(t, err)
 	require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing))
 
@@ -142,7 +143,8 @@ func TestMatcherCache(t *testing.T) {
 		return ing.lifecycler.GetState()
 	})
 	ctx := user.InjectOrgID(context.Background(), userID)
-	numberOfDifferentMatchers := 50
+	// Lets have 1 key evicted
+	numberOfDifferentMatchers := cfg.MatchersCacheMaxItems + 1
 	callPerMatcher := 10
 	for j := 0; j < numberOfDifferentMatchers; j++ {
 		for i := 0; i < callPerMatcher; i++ {
@@ -159,7 +161,7 @@ func TestMatcherCache(t *testing.T) {
 	require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(fmt.Sprintf(`
 				# HELP ingester_matchers_cache_evicted_total Total number of items evicted from the cache
 				# TYPE ingester_matchers_cache_evicted_total counter
-				ingester_matchers_cache_evicted_total 0
+				ingester_matchers_cache_evicted_total 1
 				# HELP ingester_matchers_cache_hits_total Total number of cache hits for series matchers
 				# TYPE ingester_matchers_cache_hits_total counter
 				ingester_matchers_cache_hits_total %v
@@ -172,7 +174,7 @@ func TestMatcherCache(t *testing.T) {
 				# HELP ingester_matchers_cache_requests_total Total number of cache requests for series matchers
 				# TYPE ingester_matchers_cache_requests_total counter
 				ingester_matchers_cache_requests_total %v
-	`, callPerMatcher*numberOfDifferentMatchers-numberOfDifferentMatchers, numberOfDifferentMatchers, callPerMatcher*numberOfDifferentMatchers)), "ingester_matchers_cache_requests_total", "ingester_matchers_cache_hits_total", "ingester_matchers_cache_items", "ingester_matchers_cache_max_items", "ingester_matchers_cache_evicted_total"))
+	`, callPerMatcher*numberOfDifferentMatchers-numberOfDifferentMatchers, cfg.MatchersCacheMaxItems, callPerMatcher*numberOfDifferentMatchers)), "ingester_matchers_cache_requests_total", "ingester_matchers_cache_hits_total", "ingester_matchers_cache_items", "ingester_matchers_cache_max_items", "ingester_matchers_cache_evicted_total"))
 }
 
 func TestIngesterPerLabelsetLimitExceeded(t *testing.T) {

pkg/ingester/metrics.go

@@ -1,13 +1,13 @@
 package ingester
 
 import (
+	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/cortexproject/cortex/pkg/storage/tsdb"
 	"github.com/cortexproject/cortex/pkg/util"
-	util_log "github.com/cortexproject/cortex/pkg/util/log"
 	util_math "github.com/cortexproject/cortex/pkg/util/math"
 )
 
@@ -680,7 +680,8 @@ func (sm *tsdbMetrics) removeRegistryForUser(userID string) {
 }
 
 type matcherCacheMetrics struct {
-	r *prometheus.Registry
+	r      *prometheus.Registry
+	logger log.Logger
 
 	requestsTotal *prometheus.Desc
 	hitsTotal     *prometheus.Desc
@@ -689,9 +690,10 @@ type matcherCacheMetrics struct {
 	evicted       *prometheus.Desc
 }
 
-func newMatchCacheMetrics(r *prometheus.Registry) *matcherCacheMetrics {
+func newMatchCacheMetrics(r *prometheus.Registry, l log.Logger) *matcherCacheMetrics {
 	m := &matcherCacheMetrics{
-		r: r,
+		r:      r,
+		logger: l,
 		requestsTotal: prometheus.NewDesc(
 			"ingester_matchers_cache_requests_total",
 			"Total number of cache requests for series matchers",
@@ -727,20 +729,20 @@ func (m *matcherCacheMetrics) Describe(out chan<- *prometheus.Desc) {
 func (m *matcherCacheMetrics) Collect(out chan<- prometheus.Metric) {
 	gm, err := m.r.Gather()
 	if err != nil {
-		level.Warn(util_log.Logger).Log("msg", "failed to gather metrics from registry", "err", err)
+		level.Warn(m.logger).Log("msg", "failed to gather metrics from registry", "err", err)
 		return
 	}
 
 	mfm, err := util.NewMetricFamilyMap(gm)
 
 	if err != nil {
-		level.Warn(util_log.Logger).Log("msg", "failed to create metric family map", "err", err)
+		level.Warn(m.logger).Log("msg", "failed to create metric family map", "err", err)
 		return
 	}
 
 	out <- prometheus.MustNewConstMetric(m.requestsTotal, prometheus.CounterValue, mfm.SumCounters("thanos_matchers_cache_requests_total"))
 	out <- prometheus.MustNewConstMetric(m.hitsTotal, prometheus.CounterValue, mfm.SumCounters("thanos_matchers_cache_hits_total"))
 	out <- prometheus.MustNewConstMetric(m.numItems, prometheus.GaugeValue, mfm.SumGauges("thanos_matchers_cache_items"))
 	out <- prometheus.MustNewConstMetric(m.maxItems, prometheus.GaugeValue, mfm.SumGauges("thanos_matchers_cache_max_items"))
-	out <- prometheus.MustNewConstMetric(m.evicted, prometheus.CounterValue, mfm.SumCounters("thanos_matchers_cache_max_items"))
+	out <- prometheus.MustNewConstMetric(m.evicted, prometheus.CounterValue, mfm.SumCounters("thanos_matchers_cache_evicted_total"))
 }