util: introduce LabelSliceCache in metric

Previously, we were persisting label value slice at each child metrics of
aggregated metric type. This was inadequate because it will create redundant
memory allocations for same label value slice across difference metrics. To
address this, this patch introduces `LabelSliceCache` which would persist the
label values at registry level and metrics would reference them through the
key. The LabelSliceCache is referenced in metrics through `PrometheusEvictable`
interface. The LabelSliceCache contains 2 critical methods:
	1. Upsert: This method implements the "add reference" part of the
reference counting mechanism.  This method increments the reference count for
label values by 1, if already exists. Otherwise, It will create a new entry for
the label values with default value as 1.
	2. DecrementAndDeleteIfZero: This method decrements the reference
counter for the given label values by 1. If the count is reached to zero then
it means that no metrics are relying on the particular label values. In that
case, it deletes the entry from cache.
These methods ensure that the cache tracks how many metrics are currently using
each label combination, enabling proper cleanup when metrics are no longer
needed.

Epic: CRDB-53398
Part of: CRDB-53830
Release note: None

Author: aa-joshi

pkg/util/metric/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
         "hdrhistogram.go",
         "histogram_buckets.go",
         "histogram_snapshot.go",
+        "label_slice_cache.go",
         "metric.go",
         "prometheus_exporter.go",
         "prometheus_rule_exporter.go",
@@ -24,6 +25,7 @@ go_library(
     deps = [
         "//pkg/settings",
         "//pkg/util/buildutil",
+        "//pkg/util/cache",
         "//pkg/util/envutil",
         "//pkg/util/log",
         "//pkg/util/metamorphic",
@@ -48,6 +50,7 @@ go_test(
     size = "small",
     srcs = [
         "histogram_buckets_test.go",
+        "label_slice_cache_test.go",
         "metric_ext_test.go",
         "metric_test.go",
         "prometheus_exporter_test.go",

pkg/util/metric/aggmetric/agg_metric.go

@@ -11,6 +11,7 @@ package aggmetric
 import (
 	"hash/fnv"
 	"strings"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/util/cache"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
@@ -23,8 +24,10 @@ import (
 var delimiter = []byte{'_'}
 
 const (
-	dbLabel  = "database"
-	appLabel = "application_name"
+	dbLabel        = "database"
+	appLabel       = "application_name"
+	cacheSize      = 5000
+	childMetricTTL = 20 * time.Second
 )
 
 // Builder is used to ease constructing metrics with the same labels.
@@ -97,7 +100,6 @@ func (cs *childSet) initWithBTreeStorageType(labels []string) {
 }
 
 func getCacheStorage() *cache.UnorderedCache {
-	const cacheSize = 5000
 	cacheStorage := cache.NewUnorderedCache(cache.Config{
 		Policy: cache.CacheLRU,
 		//TODO (aa-joshi) : make cacheSize configurable in the future
@@ -311,6 +313,16 @@ type ChildMetric interface {
 	ToPrometheusMetric() *io_prometheus_client.Metric
 }
 
+// LabelSliceCachedChildMetric extends ChildMetric with label slice caching capabilities.
+// This interface is designed for child metrics that relies on label slice reference
+// counting system. Metrics implementing this interface can have their label values
+// cached and shared among multiple metrics with identical label combinations,
+// reducing memory usage and improving performance in scenarios with many similar metrics.
+type LabelSliceCachedChildMetric interface {
+	ChildMetric
+	CreatedAt() time.Time
+}
+
 type labelValuer interface {
 	labelValues() []string
 }
@@ -330,9 +342,10 @@ func metricKey(labels ...string) uint64 {
 
 type ChildrenStorage interface {
 	Get(labelVals ...string) (ChildMetric, bool)
+	GetValue(key uint64) (ChildMetric, bool)
 	Add(metric ChildMetric)
+	AddKey(key uint64, metric ChildMetric) error
 	Del(key ChildMetric)
-
 	// ForEach calls f for each child metric, in arbitrary order.
 	ForEach(f func(metric ChildMetric))
 	Clear()
@@ -345,6 +358,22 @@ type UnorderedCacheWrapper struct {
 	cache *cache.UnorderedCache
 }
 
+func (ucw *UnorderedCacheWrapper) GetValue(key uint64) (ChildMetric, bool) {
+	value, ok := ucw.cache.Get(key)
+	if !ok {
+		return nil, false
+	}
+	return value.(ChildMetric), ok
+}
+
+func (ucw *UnorderedCacheWrapper) AddKey(key uint64, metric ChildMetric) error {
+	if _, ok := ucw.cache.Get(key); ok {
+		return errors.Newf("child %v already exists\n", metric.labelValues())
+	}
+	ucw.cache.Add(key, metric)
+	return nil
+}
+
 func (ucw *UnorderedCacheWrapper) Get(labelVals ...string) (ChildMetric, bool) {
 	hashKey := metricKey(labelVals...)
 	value, ok := ucw.cache.Get(hashKey)
@@ -384,6 +413,18 @@ type BtreeWrapper struct {
 	tree *btree.BTreeG[MetricItem]
 }
 
+func (b BtreeWrapper) GetValue(key uint64) (ChildMetric, bool) {
+	// GetValue method is not relevant for BtreeWrapper as it uses ChildMetric
+	// as an item in Btree. We are going to remove BtreeWrapper as ChildrenStorage.
+	panic("unimplemented")
+}
+
+func (b BtreeWrapper) AddKey(_ uint64, _ ChildMetric) error {
+	// AddKey method is not relevant for BtreeWrapper as it uses ChildMetric
+	// as an item in Btree. We are going to remove BtreeWrapper as ChildrenStorage.
+	panic("unimplemented")
+}
+
 func (b BtreeWrapper) Get(labelVals ...string) (ChildMetric, bool) {
 	key := labelValuesSlice(labelVals)
 	cm, ok := b.tree.Get(&key)

pkg/util/metric/label_slice_cache.go

@@ -0,0 +1,117 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package metric
+
+import (
+	"sync/atomic"
+
+	"github.com/cockroachdb/cockroach/pkg/util/cache"
+	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
+)
+
+// LabelSliceCacheKey is the hash key type for the cache.
+type LabelSliceCacheKey uint64
+
+// LabelSliceCacheValue is the value stored in the cache.
+type LabelSliceCacheValue struct {
+	LabelValues []string
+	Counter     atomic.Int64
+}
+
+// LabelSliceCache is a thread-safe cache mapping hash keys to label value/counter pairs.
+type LabelSliceCache struct {
+	mu struct {
+		syncutil.Mutex
+		cache *cache.UnorderedCache
+	}
+}
+
+// NewLabelSliceCache creates a new LabelSliceCache.
+func NewLabelSliceCache() *LabelSliceCache {
+	labelSliceCache := &LabelSliceCache{}
+	labelSliceCache.mu.cache = cache.NewUnorderedCache(cache.Config{})
+	return labelSliceCache
+}
+
+// Get returns the value for the given key, or nil if not present.
+func (lsc *LabelSliceCache) Get(key LabelSliceCacheKey) (*LabelSliceCacheValue, bool) {
+	lsc.mu.Lock()
+	defer lsc.mu.Unlock()
+	val, ok := lsc.mu.cache.Get(key)
+	if !ok {
+		return nil, false
+	}
+	return val.(*LabelSliceCacheValue), true
+}
+
+// Upsert adds or updates the value for the given key with reference counting.
+// This method implements the "add reference" part of the reference counting mechanism.
+// When a new metric with a specific label combination is created, this method performs
+// either:
+//  1. Add a new entry to the cache with a reference count of 1 (if the label combination
+//     is being used for the first time)
+//  2. Increment the existing reference count by 1 (if the label combination is already
+//     cached and being used by other metrics)
+//
+// This ensures that the cache tracks how many metrics are currently using each
+// label combination, enabling proper cleanup via DecrementAndDeleteIfZero when
+// metrics are no longer needed.
+func (lsc *LabelSliceCache) Upsert(key LabelSliceCacheKey, value *LabelSliceCacheValue) {
+	lsc.mu.Lock()
+	defer lsc.mu.Unlock()
+	val, ok := lsc.mu.cache.Get(key)
+
+	if !ok {
+		value.Counter.Store(1)
+		lsc.mu.cache.Add(key, value)
+	} else {
+		existingValue := val.(*LabelSliceCacheValue)
+		existingValue.Counter.Add(1)
+		lsc.mu.cache.Add(key, existingValue)
+	}
+}
+
+// Delete removes the value for the given key.
+func (lsc *LabelSliceCache) Delete(key LabelSliceCacheKey) {
+	lsc.mu.Lock()
+	defer lsc.mu.Unlock()
+	lsc.mu.cache.Del(key)
+}
+
+// DecrementAndDeleteIfZero decrements the reference counter for the given key by 1.
+// This method implements reference counting for cached label value combinations.
+// When metrics with specific label combinations are created, the cache counter
+// is incremented via Upsert. When those metrics are no longer needed or go out
+// of scope, this method should be called to decrement the reference count.
+//
+// The automatic deletion when the counter reaches zero is crucial for preventing
+// memory leaks in long-running processes where metrics with dynamic label values
+// might be created and destroyed frequently. Without this cleanup mechanism,
+// the cache would accumulate unused label combinations indefinitely.
+//
+// Returns true if the entry was deleted due to the counter reaching zero,
+// false if the entry still has references or didn't exist.
+func (lsc *LabelSliceCache) DecrementAndDeleteIfZero(key LabelSliceCacheKey) bool {
+	lsc.mu.Lock()
+	defer lsc.mu.Unlock()
+
+	val, ok := lsc.mu.cache.Get(key)
+	if !ok {
+		// Key doesn't exist, return 0 and false
+		return false
+	}
+
+	existingValue := val.(*LabelSliceCacheValue)
+	newCount := existingValue.Counter.Add(-1)
+
+	if newCount <= 0 {
+		// Remove the entry when counter reaches 0 or below
+		lsc.mu.cache.Del(key)
+		return true
+	}
+
+	return false
+}