Merge #153571

153571: util/metric: introduce HighCardinalityGauge in aggregated metrics  r=aa-joshi a=aa-joshi

This patch introduces `HighCardinalityGauge` metric which is similar to the
`HighCardinalityCounter` introduced in #153568. It relies on unordered cache
with LRU eviction as child storage. The parent values represents the
aggregation of all child metric values. The child metrics values are only
exported and aggregated values is persisted in CRDB. It relies on
LabelSliceCache to efficiently store label values at registry. The child metric
eviction policy is combination of max cache size of 5000 and minimum retention
time of 20 seconds. This guarantees that we would see the child metric values
at least in one scrape with default interval of 10 seconds before getting
evicted due to cache size. The child eviction won't impact the parent value.

Epic: CRDB-53398
Part of: CRDB-53832
Release note: None

Co-authored-by: Akshay Joshi <[email protected]>

Author: craig[bot]

pkg/util/metric/aggmetric/gauge.go

@@ -8,8 +8,10 @@ package aggmetric
 import (
 	"math"
 	"sync/atomic"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
 	"github.com/gogo/protobuf/proto"
 	io_prometheus_client "github.com/prometheus/client_model/go"
@@ -521,3 +523,212 @@ func (scg *SQLChildGauge) Inc(i int64) {
 func (scg *SQLChildGauge) Dec(i int64) {
 	scg.gauge.Dec(i)
 }
+
+// HighCardinalityGauge is similar to AggGauge but uses cache storage instead of B-tree,
+// allowing for automatic eviction of less frequently used child metrics.
+// This is useful when dealing with high cardinality metrics that might exceed resource limits.
+type HighCardinalityGauge struct {
+	g metric.Gauge
+	childSet
+	labelSliceCache *metric.LabelSliceCache
+}
+
+var _ metric.Iterable = (*HighCardinalityGauge)(nil)
+var _ metric.PrometheusEvictable = (*HighCardinalityGauge)(nil)
+
+// NewHighCardinalityGauge constructs a new HighCardinalityGauge that uses cache storage
+// with eviction for child metrics.
+func NewHighCardinalityGauge(
+	metadata metric.Metadata, childLabels ...string,
+) *HighCardinalityGauge {
+	g := &HighCardinalityGauge{g: *metric.NewGauge(metadata)}
+	g.initWithCacheStorageType(childLabels, metadata.Name)
+	return g
+}
+
+// GetName is part of the metric.Iterable interface.
+func (g *HighCardinalityGauge) GetName(useStaticLabels bool) string {
+	return g.g.GetName(useStaticLabels)
+}
+
+// GetHelp is part of the metric.Iterable interface.
+func (g *HighCardinalityGauge) GetHelp() string { return g.g.GetHelp() }
+
+// GetMeasurement is part of the metric.Iterable interface.
+func (g *HighCardinalityGauge) GetMeasurement() string { return g.g.GetMeasurement() }
+
+// GetUnit is part of the metric.Iterable interface.
+func (g *HighCardinalityGauge) GetUnit() metric.Unit { return g.g.GetUnit() }
+
+// GetMetadata is part of the metric.Iterable interface.
+func (g *HighCardinalityGauge) GetMetadata() metric.Metadata { return g.g.GetMetadata() }
+
+// Inspect is part of the metric.Iterable interface.
+func (g *HighCardinalityGauge) Inspect(f func(interface{})) { f(g) }
+
+// GetType is part of the metric.PrometheusExportable interface.
+func (g *HighCardinalityGauge) GetType() *io_prometheus_client.MetricType {
+	return g.g.GetType()
+}
+
+// GetLabels is part of the metric.PrometheusExportable interface.
+func (g *HighCardinalityGauge) GetLabels(useStaticLabels bool) []*io_prometheus_client.LabelPair {
+	return g.g.GetLabels(useStaticLabels)
+}
+
+// ToPrometheusMetric is part of the metric.PrometheusExportable interface.
+func (g *HighCardinalityGauge) ToPrometheusMetric() *io_prometheus_client.Metric {
+	return g.g.ToPrometheusMetric()
+}
+
+// Value returns the aggregate sum of all of its current children.
+func (g *HighCardinalityGauge) Value() int64 {
+	return g.g.Value()
+}
+
+// Inc increments the gauge value by i for the given label values. If a
+// gauge with the given label values doesn't exist yet, it creates a new
+// gauge and increments it. Inc increments parent metrics as well.
+func (g *HighCardinalityGauge) Inc(i int64, labelValues ...string) {
+	g.g.Inc(i)
+
+	childMetric := g.GetOrAddChild(labelValues...)
+
+	if childMetric != nil {
+		childMetric.Inc(i)
+	}
+}
+
+// Dec decrements the gauge value by i for the given label values. If a
+// gauge with the given label values doesn't exist yet, it creates a new
+// gauge and decrements it. Dec decrements parent metrics as well.
+func (g *HighCardinalityGauge) Dec(i int64, labelValues ...string) {
+	g.g.Dec(i)
+
+	childMetric := g.GetOrAddChild(labelValues...)
+
+	if childMetric != nil {
+		childMetric.Dec(i)
+	}
+}
+
+// Update sets the gauge value to val for the given label values. If a
+// gauge with the given label values doesn't exist yet, it creates a new
+// gauge and sets it. Update updates parent metrics as well.
+//
+// The parent metric value represents the sum of all children. The parent
+// metric is updated by the delta (new - old) to maintain the aggregate
+// sum of all children.
+// For example, if a child gauge changes from 10 to 25, the
+// parent is incremented by 15, preserving the total sum across all children.
+func (g *HighCardinalityGauge) Update(val int64, labelValues ...string) {
+	childMetric := g.GetOrAddChild(labelValues...)
+
+	if childMetric != nil {
+		old := childMetric.Value()
+		// Increment the parent by the delta of the child metric.
+		g.g.Inc(val - old)
+		childMetric.Update(val)
+	}
+}
+
+// Each is part of the metric.PrometheusIterable interface.
+func (g *HighCardinalityGauge) Each(
+	labels []*io_prometheus_client.LabelPair, f func(metric *io_prometheus_client.Metric),
+) {
+	g.EachWithLabels(labels, f, g.labelSliceCache)
+}
+
+// InitializeMetrics is part of the PrometheusEvictable interface.
+func (g *HighCardinalityGauge) InitializeMetrics(labelCache *metric.LabelSliceCache) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	g.labelSliceCache = labelCache
+}
+
+// GetOrAddChild returns the existing child gauge for the given label values,
+// or creates a new one if it doesn't exist. This is the preferred method for
+// cache-based storage to avoid panics on existing keys.
+func (g *HighCardinalityGauge) GetOrAddChild(labelVals ...string) *HighCardinalityChildGauge {
+
+	if len(labelVals) == 0 {
+		return nil
+	}
+
+	// Create a LabelSliceCacheKey from the tenantID.
+	key := metric.LabelSliceCacheKey(metricKey(labelVals...))
+
+	child := g.getOrAddWithLabelSliceCache(g.GetMetadata().Name, g.createHighCardinalityChildGauge, g.labelSliceCache, labelVals...)
+
+	g.labelSliceCache.Upsert(key, &metric.LabelSliceCacheValue{
+		LabelValues: labelVals,
+	})
+
+	return child.(*HighCardinalityChildGauge)
+}
+
+func (g *HighCardinalityGauge) createHighCardinalityChildGauge(
+	key uint64, cache *metric.LabelSliceCache,
+) LabelSliceCachedChildMetric {
+	return &HighCardinalityChildGauge{
+		LabelSliceCacheKey: metric.LabelSliceCacheKey(key),
+		LabelSliceCache:    cache,
+		createdAt:          timeutil.Now(),
+	}
+}
+
+// HighCardinalityChildGauge is a child of a HighCardinalityGauge. When metrics are
+// collected by prometheus, each of the children will appear with a distinct label,
+// however, when cockroach internally collects metrics, only the parent is collected.
+type HighCardinalityChildGauge struct {
+	metric.LabelSliceCacheKey
+	value metric.Gauge
+	*metric.LabelSliceCache
+	createdAt time.Time
+}
+
+func (g *HighCardinalityChildGauge) CreatedAt() time.Time {
+	return g.createdAt
+}
+
+func (g *HighCardinalityChildGauge) DecrementLabelSliceCacheReference() {
+	g.LabelSliceCache.DecrementAndDeleteIfZero(g.LabelSliceCacheKey)
+}
+
+// ToPrometheusMetric constructs a prometheus metric for this HighCardinalityChildGauge.
+func (g *HighCardinalityChildGauge) ToPrometheusMetric() *io_prometheus_client.Metric {
+	return &io_prometheus_client.Metric{
+		Gauge: &io_prometheus_client.Gauge{
+			Value: proto.Float64(float64(g.Value())),
+		},
+	}
+}
+
+func (g *HighCardinalityChildGauge) labelValues() []string {
+	lv, ok := g.LabelSliceCache.Get(g.LabelSliceCacheKey)
+	if !ok {
+		return nil
+	}
+	return lv.LabelValues
+}
+
+// Value returns the HighCardinalityChildGauge's current value.
+func (g *HighCardinalityChildGauge) Value() int64 {
+	return g.value.Value()
+}
+
+// Inc increments the HighCardinalityChildGauge's value.
+func (g *HighCardinalityChildGauge) Inc(i int64) {
+	g.value.Inc(i)
+}
+
+// Dec decrements the HighCardinalityChildGauge's value.
+func (g *HighCardinalityChildGauge) Dec(i int64) {
+	g.value.Dec(i)
+}
+
+// Update sets the HighCardinalityChildGauge's value.
+func (g *HighCardinalityChildGauge) Update(val int64) {
+	g.value.Update(val)
+}

pkg/util/metric/aggmetric/gauge_test.go

@@ -8,12 +8,14 @@ package aggmetric
 import (
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/echotest"
 	"github.com/cockroachdb/cockroach/pkg/util/cache"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
+	"github.com/stretchr/testify/require"
 )
 
 func TestSQLGaugeEviction(t *testing.T) {
@@ -80,3 +82,96 @@ func TestSQLGaugeMethods(t *testing.T) {
 	testFile := "SQLGauge.txt"
 	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
 }
+
+func TestHighCardinalityGauge(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+
+	const cacheSize = 10
+	r := metric.NewRegistry()
+	writePrometheusMetrics := WritePrometheusMetricsFunc(r)
+
+	g := NewHighCardinalityGauge(metric.Metadata{
+		Name: "foo_gauge",
+	}, "database", "application_name")
+	g.mu.children = &UnorderedCacheWrapper{
+		cache: initialiseCacheStorageForTesting(),
+	}
+	r.AddMetric(g)
+
+	// Initialize with a label slice cache to test eviction
+	labelSliceCache := metric.NewLabelSliceCache()
+	g.InitializeMetrics(labelSliceCache)
+
+	for i := 0; i < cacheSize+5; i++ {
+		g.Update(int64(i+1), "1", strconv.Itoa(i))
+	}
+
+	// wait more than cache eviction time to make sure that keys are not evicted based on only cache size.
+	time.Sleep(6 * time.Second)
+
+	testFile := "HighCardinalityGauge_pre_eviction.txt"
+	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
+
+	for i := 0; i < cacheSize+5; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("1", strconv.Itoa(i)))
+		labelSliceValue, ok := labelSliceCache.Get(metricKey)
+		require.True(t, ok, "missing labelSliceValue in label slice cache")
+		require.Equal(t, int64(1), labelSliceValue.Counter.Load(), "the reference count should be 1")
+		require.Equal(t, []string{"1", strconv.Itoa(i)}, labelSliceValue.LabelValues, "label values are mismatching")
+	}
+
+	for i := 0 + cacheSize; i < cacheSize+5; i++ {
+		g.Inc(5, "2", strconv.Itoa(i))
+	}
+
+	testFile = "HighCardinalityGauge_post_eviction.txt"
+	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
+
+	for i := 0; i < 5; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("1", strconv.Itoa(i)))
+		_, ok := labelSliceCache.Get(metricKey)
+		require.False(t, ok, "labelSliceValue should not be present.")
+	}
+
+	for i := 10; i < 15; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("1", strconv.Itoa(i)))
+		labelSliceValue, ok := labelSliceCache.Get(metricKey)
+		require.True(t, ok, "missing labelSliceValue in label slice cache")
+		require.Equal(t, int64(1), labelSliceValue.Counter.Load(), "the reference count should be 1")
+		require.Equal(t, []string{"1", strconv.Itoa(i)}, labelSliceValue.LabelValues, "label values are mismatching")
+	}
+
+	for i := 10; i < 15; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("2", strconv.Itoa(i)))
+		labelSliceValue, ok := labelSliceCache.Get(metricKey)
+		require.True(t, ok, "missing labelSliceValue in label slice cache")
+		require.Equal(t, int64(1), labelSliceValue.Counter.Load(), "the reference count should be 1")
+		require.Equal(t, []string{"2", strconv.Itoa(i)}, labelSliceValue.LabelValues, "label values are mismatching")
+	}
+}
+
+func TestHighCardinalityGaugeMethods(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	r := metric.NewRegistry()
+	writePrometheusMetrics := WritePrometheusMetricsFunc(r)
+
+	g := NewHighCardinalityGauge(metric.Metadata{
+		Name: "foo_gauge",
+	}, "database", "application_name")
+	g.mu.children = &UnorderedCacheWrapper{
+		cache: initialiseCacheStorageForTesting(),
+	}
+
+	r.AddMetric(g)
+
+	// Test Update operation
+	g.Update(10, "1", "1")
+	g.Update(20, "2", "2")
+
+	// Test Inc operation
+	g.Inc(5, "1", "1")
+	g.Dec(3, "2", "2")
+
+	testFile := "HighCardinalityGauge.txt"
+	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
+}

pkg/util/metric/aggmetric/testdata/HighCardinalityGauge.txt

@@ -0,0 +1,5 @@
+echo
+----
+foo_gauge 32
+foo_gauge{database="1",application_name="1"} 15
+foo_gauge{database="2",application_name="2"} 17

pkg/util/metric/aggmetric/testdata/HighCardinalityGauge_post_eviction.txt

@@ -0,0 +1,13 @@
+echo
+----
+foo_gauge 145
+foo_gauge{database="1",application_name="10"} 11
+foo_gauge{database="1",application_name="11"} 12
+foo_gauge{database="1",application_name="12"} 13
+foo_gauge{database="1",application_name="13"} 14
+foo_gauge{database="1",application_name="14"} 15
+foo_gauge{database="2",application_name="10"} 5
+foo_gauge{database="2",application_name="11"} 5
+foo_gauge{database="2",application_name="12"} 5
+foo_gauge{database="2",application_name="13"} 5
+foo_gauge{database="2",application_name="14"} 5

pkg/util/metric/aggmetric/testdata/HighCardinalityGauge_pre_eviction.txt

@@ -0,0 +1,18 @@
+echo
+----
+foo_gauge 120
+foo_gauge{database="1",application_name="0"} 1
+foo_gauge{database="1",application_name="1"} 2
+foo_gauge{database="1",application_name="10"} 11
+foo_gauge{database="1",application_name="11"} 12
+foo_gauge{database="1",application_name="12"} 13
+foo_gauge{database="1",application_name="13"} 14
+foo_gauge{database="1",application_name="14"} 15
+foo_gauge{database="1",application_name="2"} 3
+foo_gauge{database="1",application_name="3"} 4
+foo_gauge{database="1",application_name="4"} 5
+foo_gauge{database="1",application_name="5"} 6
+foo_gauge{database="1",application_name="6"} 7
+foo_gauge{database="1",application_name="7"} 8
+foo_gauge{database="1",application_name="8"} 9
+foo_gauge{database="1",application_name="9"} 10