util/metric: introduce HighCardinalityCounter in aggregated metrics

Previously, aggregated metrics support unbounded child metrics with BTree as
child storage. The child metric creation was static. That means we have to
explicitly create the child metric and link it to the parent metric. We are
going to support high cardinality metrics. This requires dynamic child creation
based on label values combination. The existing implementation is inadequate
because it has staic child creation and there is no upper bound on the number
of child metrics that you can create. This patch introduces
`HighCardinalityCounter` which uses unordered cache with LRU eviction as child
storage. The child metrics values are only exported and aggregated values is
persisted in CRDB. It relies on `LabelSliceCache` to efficiently store label
values at registry. The child metric eviction policy is combination of max
cache size of `5000` and minimum retention time of `20` seconds. This
guarantees that we would see the child metric values at least in one scrape
with default interval of `10` seconds before getting evicted due to cache size.
The child eviction won't impact the parent value.

Epic: CRDB-53398
Part of: CRDB-53831
Release note: None

Author: aa-joshi

pkg/util/metric/aggmetric/BUILD.bazel

@@ -12,11 +12,13 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/util/cache",
+        "//pkg/util/log",
         "//pkg/util/metric",
         "//pkg/util/metric/tick",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_errors//:errors",
+        "@com_github_cockroachdb_redact//:redact",
         "@com_github_gogo_protobuf//proto",
         "@com_github_google_btree//:btree",
         "@com_github_prometheus_client_model//go",
@@ -43,6 +45,7 @@ go_test(
         "//pkg/util/cache",
         "//pkg/util/leaktest",
         "//pkg/util/metric",
+        "//pkg/util/timeutil",
         "@com_github_cockroachdb_crlib//testutils/require",
         "@com_github_prometheus_client_model//go",
         "@com_github_prometheus_common//expfmt",

pkg/util/metric/aggmetric/agg_metric.go

@@ -9,27 +9,34 @@
 package aggmetric
 
 import (
+	"context"
 	"hash/fnv"
 	"strings"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/util/cache"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/redact"
 	"github.com/google/btree"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
 
 var delimiter = []byte{'_'}
 
 const (
-	dbLabel        = "database"
-	appLabel       = "application_name"
-	cacheSize      = 5000
-	childMetricTTL = 20 * time.Second
+	dbLabel                   = "database"
+	appLabel                  = "application_name"
+	cacheSize                 = 5000
+	retentionTimeTillEviction = 20 * time.Second
 )
 
+// This is a no-op context used during logging.
+var noOpCtx = context.TODO()
+
 // Builder is used to ease constructing metrics with the same labels.
 type Builder struct {
 	labels []string
@@ -99,6 +106,41 @@ func (cs *childSet) initWithBTreeStorageType(labels []string) {
 	}
 }
 
+func (cs *childSet) initWithCacheStorageType(labels []string, metricName string) {
+	cs.labels = labels
+
+	cs.mu.children = &UnorderedCacheWrapper{
+		cache: cache.NewUnorderedCache(cache.Config{
+			Policy: cache.CacheLRU,
+			ShouldEvict: func(size int, key, value any) bool {
+				if childMetric, ok := value.(ChildMetric); ok {
+					// Check if the child metric has exceeded 20 seconds and cache size is greater than 5000
+					if labelSliceCachedChildMetric, ok := childMetric.(LabelSliceCachedChildMetric); ok {
+						currentTime := timeutil.Now()
+						age := currentTime.Sub(labelSliceCachedChildMetric.CreatedAt())
+						return size > cacheSize && age > retentionTimeTillEviction
+					}
+				}
+				return size > cacheSize
+			},
+			OnEvictedEntry: func(entry *cache.Entry) {
+				if childMetric, ok := entry.Value.(ChildMetric); ok {
+					labelValues := childMetric.labelValues()
+
+					// log metric name and label values of evicted entry
+					log.Dev.Infof(noOpCtx, "evicted child of metric %s with label values: %s\n",
+						redact.SafeString(metricName), redact.SafeString(strings.Join(labelValues, ",")))
+
+					// Invoke DecrementAndDeleteIfZero from ChildMetric which relies on LabelSliceCache
+					if boundedChild, ok := childMetric.(LabelSliceCachedChildMetric); ok {
+						boundedChild.DecrementLabelSliceCacheReference()
+					}
+				}
+			},
+		}),
+	}
+}
+
 func getCacheStorage() *cache.UnorderedCache {
 	cacheStorage := cache.NewUnorderedCache(cache.Config{
 		Policy: cache.CacheLRU,
@@ -165,6 +207,75 @@ func (cs *childSet) get(labelVals ...string) (ChildMetric, bool) {
 	return cs.mu.children.Get(labelVals...)
 }
 
+func (cs *childSet) getOrAddWithLabelSliceCache(
+	metricName string,
+	createFn func(key uint64, cache *metric.LabelSliceCache) LabelSliceCachedChildMetric,
+	labelSliceCache *metric.LabelSliceCache,
+	labelVals ...string,
+) ChildMetric {
+	// Validate label values count
+	if len(labelVals) != len(cs.labels) {
+		if log.V(2) {
+			log.Dev.Errorf(noOpCtx,
+				"cannot add child with %d label values %v to  metric %s with %d labels %s",
+				len(labelVals), redact.SafeString(metricName), redact.SafeString(strings.Join(labelVals, ",")),
+				len(cs.labels), redact.SafeString(strings.Join(cs.labels, ",")))
+		}
+		return nil
+	}
+
+	cs.mu.Lock()
+	defer cs.mu.Unlock()
+
+	// Create a LabelSliceCacheKey from the label.
+	key := metricKey(labelVals...)
+
+	// Check if the child already exists
+	if child, ok := cs.mu.children.GetValue(key); ok {
+		return child
+	}
+
+	// Create and add the new child
+	child := createFn(key, labelSliceCache)
+	err := cs.mu.children.AddKey(key, child)
+	if err != nil {
+		if log.V(2) {
+			log.Dev.Errorf(context.TODO(), "child metric creation failed for metric %s with error %v", redact.SafeString(metricName), err)
+		}
+		return nil
+	}
+	return child
+}
+
+// EachWithLabels is a generic implementation for iterating over child metrics and building prometheus metrics.
+// This can be used by any aggregate metric type that embeds childSet.
+func (cs *childSet) EachWithLabels(
+	labels []*io_prometheus_client.LabelPair,
+	f func(metric *io_prometheus_client.Metric),
+	labelCache *metric.LabelSliceCache,
+) {
+	cs.mu.Lock()
+	defer cs.mu.Unlock()
+
+	cs.mu.children.ForEach(func(cm ChildMetric) {
+		m := cm.ToPrometheusMetric()
+		childLabels := make([]*io_prometheus_client.LabelPair, 0, len(labels)+len(cs.labels))
+		childLabels = append(childLabels, labels...)
+		lvs := cm.labelValues()
+		key := metricKey(lvs...)
+		labelValueCacheValues, _ := labelCache.Get(metric.LabelSliceCacheKey(key))
+		for i := range cs.labels {
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &cs.labels[i],
+				Value: &labelValueCacheValues.LabelValues[i],
+			})
+		}
+
+		m.Label = childLabels
+		f(m)
+	})
+}
+
 // clear method removes all children from the childSet. It does not reset parent metric values.
 // Method should cautiously be used when childSet is reinitialised/updated. Today, it is
 // only used when cluster settings are updated to support app and db label values. For normal
@@ -321,6 +432,7 @@ type ChildMetric interface {
 type LabelSliceCachedChildMetric interface {
 	ChildMetric
 	CreatedAt() time.Time
+	DecrementLabelSliceCacheReference()
 }
 
 type labelValuer interface {
@@ -368,7 +480,7 @@ func (ucw *UnorderedCacheWrapper) GetValue(key uint64) (ChildMetric, bool) {
 
 func (ucw *UnorderedCacheWrapper) AddKey(key uint64, metric ChildMetric) error {
 	if _, ok := ucw.cache.Get(key); ok {
-		return errors.Newf("child %v already exists\n", metric.labelValues())
+		return errors.Newf("child %s already exists\n", redact.SafeString(strings.Join(metric.labelValues(), ",")))
 	}
 	ucw.cache.Add(key, metric)
 	return nil

pkg/util/metric/aggmetric/counter.go

@@ -7,8 +7,10 @@ package aggmetric
 
 import (
 	"sync/atomic"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/gogo/protobuf/proto"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
@@ -394,3 +396,170 @@ func (s *SQLChildCounter) Value() int64 {
 func (s *SQLChildCounter) Inc(i int64) {
 	s.value.Inc(i)
 }
+
+// HighCardinalityCounter is similar to AggCounter but uses cache storage instead of B-tree,
+// allowing for automatic eviction of less frequently used child metrics.
+// This is useful when dealing with high cardinality metrics that might exceed resource limits.
+type HighCardinalityCounter struct {
+	g metric.Counter
+	childSet
+	labelSliceCache *metric.LabelSliceCache
+}
+
+var _ metric.Iterable = (*HighCardinalityCounter)(nil)
+var _ metric.PrometheusEvictable = (*HighCardinalityCounter)(nil)
+
+// NewHighCardinalityCounter constructs a new HighCardinalityCounter that uses cache storage
+// with eviction for child metrics.
+func NewHighCardinalityCounter(
+	metadata metric.Metadata, childLabels ...string,
+) *HighCardinalityCounter {
+	c := &HighCardinalityCounter{g: *metric.NewCounter(metadata)}
+	c.initWithCacheStorageType(childLabels, metadata.Name)
+	return c
+}
+
+// GetName is part of the metric.Iterable interface.
+func (c *HighCardinalityCounter) GetName(useStaticLabels bool) string {
+	return c.g.GetName(useStaticLabels)
+}
+
+// GetHelp is part of the metric.Iterable interface.
+func (c *HighCardinalityCounter) GetHelp() string { return c.g.GetHelp() }
+
+// GetMeasurement is part of the metric.Iterable interface.
+func (c *HighCardinalityCounter) GetMeasurement() string { return c.g.GetMeasurement() }
+
+// GetUnit is part of the metric.Iterable interface.
+func (c *HighCardinalityCounter) GetUnit() metric.Unit { return c.g.GetUnit() }
+
+// GetMetadata is part of the metric.Iterable interface.
+func (c *HighCardinalityCounter) GetMetadata() metric.Metadata { return c.g.GetMetadata() }
+
+// Inspect is part of the metric.Iterable interface.
+func (c *HighCardinalityCounter) Inspect(f func(interface{})) { f(c) }
+
+// GetType is part of the metric.PrometheusExportable interface.
+func (c *HighCardinalityCounter) GetType() *io_prometheus_client.MetricType {
+	return c.g.GetType()
+}
+
+// GetLabels is part of the metric.PrometheusExportable interface.
+func (c *HighCardinalityCounter) GetLabels(useStaticLabels bool) []*io_prometheus_client.LabelPair {
+	return c.g.GetLabels(useStaticLabels)
+}
+
+// ToPrometheusMetric is part of the metric.PrometheusExportable interface.
+func (c *HighCardinalityCounter) ToPrometheusMetric() *io_prometheus_client.Metric {
+	return c.g.ToPrometheusMetric()
+}
+
+// Count returns the aggregate count of all of its current and past children.
+func (c *HighCardinalityCounter) Count() int64 {
+	return c.g.Count()
+}
+
+// Inc increments the counter value by i for the given label values. If a
+// counter with the given label values doesn't exist yet, it creates a new
+// counter and increments it. Inc increments parent metrics as well.
+func (c *HighCardinalityCounter) Inc(i int64, labelValues ...string) {
+	c.g.Inc(i)
+
+	childMetric := c.GetOrAddChild(labelValues...)
+
+	if childMetric != nil {
+		childMetric.Inc(i)
+	}
+
+}
+
+// Each is part of the metric.PrometheusIterable interface.
+func (c *HighCardinalityCounter) Each(
+	labels []*io_prometheus_client.LabelPair, f func(metric *io_prometheus_client.Metric),
+) {
+	c.EachWithLabels(labels, f, c.labelSliceCache)
+}
+
+// InitializeMetrics is part of the PrometheusEvictable interface.
+func (c *HighCardinalityCounter) InitializeMetrics(labelCache *metric.LabelSliceCache) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.labelSliceCache = labelCache
+}
+
+// GetOrAddChild returns the existing child counter for the given label values,
+// or creates a new one if it doesn't exist. This is the preferred method for
+// cache-based storage to avoid panics on existing keys.
+func (c *HighCardinalityCounter) GetOrAddChild(labelVals ...string) *HighCardinalityChildCounter {
+
+	if len(labelVals) == 0 {
+		return nil
+	}
+
+	// Create a LabelSliceCacheKey from the tenantID.
+	key := metric.LabelSliceCacheKey(metricKey(labelVals...))
+
+	child := c.getOrAddWithLabelSliceCache(c.GetMetadata().Name, c.createHighCardinalityChildCounter, c.labelSliceCache, labelVals...)
+
+	c.labelSliceCache.Upsert(key, &metric.LabelSliceCacheValue{
+		LabelValues: labelVals,
+	})
+
+	return child.(*HighCardinalityChildCounter)
+}
+
+func (c *HighCardinalityCounter) createHighCardinalityChildCounter(
+	key uint64, cache *metric.LabelSliceCache,
+) LabelSliceCachedChildMetric {
+	return &HighCardinalityChildCounter{
+		LabelSliceCacheKey: metric.LabelSliceCacheKey(key),
+		LabelSliceCache:    cache,
+		createdAt:          timeutil.Now(),
+	}
+}
+
+// HighCardinalityChildCounter is a child of a HighCardinalityCounter. When metrics are
+// collected by prometheus, each of the children will appear with a distinct label,
+// however, when cockroach internally collects  metrics, only the parent is collected.
+type HighCardinalityChildCounter struct {
+	metric.LabelSliceCacheKey
+	value metric.Counter
+	*metric.LabelSliceCache
+	createdAt time.Time
+}
+
+func (c *HighCardinalityChildCounter) CreatedAt() time.Time {
+	return c.createdAt
+}
+
+func (c *HighCardinalityChildCounter) DecrementLabelSliceCacheReference() {
+	c.LabelSliceCache.DecrementAndDeleteIfZero(c.LabelSliceCacheKey)
+}
+
+// ToPrometheusMetric constructs a prometheus metric for this HighCardinalityChildCounter.
+func (c *HighCardinalityChildCounter) ToPrometheusMetric() *io_prometheus_client.Metric {
+	return &io_prometheus_client.Metric{
+		Counter: &io_prometheus_client.Counter{
+			Value: proto.Float64(float64(c.Value())),
+		},
+	}
+}
+
+func (c *HighCardinalityChildCounter) labelValues() []string {
+	lv, ok := c.LabelSliceCache.Get(c.LabelSliceCacheKey)
+	if !ok {
+		return nil
+	}
+	return lv.LabelValues
+}
+
+// Value returns the HighCardinalityChildCounter's current value.
+func (c *HighCardinalityChildCounter) Value() int64 {
+	return c.value.Count()
+}
+
+// Inc increments the HighCardinalityChildCounter's value.
+func (c *HighCardinalityChildCounter) Inc(i int64) {
+	c.value.Inc(i)
+}