util/metric: introduce HighCardinalityHistogram in aggregated metrics

This patch introduces `HighCardinalityHistogram` metric which is similar to the
`HighCardinalityCounter` introduced in #153568. It relies on
unordered cache with LRU eviction as child storage. The parent values
represents the aggregation of all child metric values. The child metrics values
are only exported and aggregated values is persisted in CRDB. It relies on
LabelSliceCache to efficiently store label values at registry. The child metric
eviction policy is combination of max cache size of 5000 and minimum retention
time of 20 seconds. This guarantees that we would see the child metric values
at least in one scrape with default interval of 10 seconds before getting
evicted due to cache size. The child eviction won't impact the parent value.

Epic: CRDB-53398
Part of: CRDB-53833
Release note: None

Author: aa-joshi

pkg/util/metric/aggmetric/BUILD.bazel

@@ -46,7 +46,6 @@ go_test(
         "//pkg/util/leaktest",
         "//pkg/util/metric",
         "//pkg/util/timeutil",
-        "@com_github_cockroachdb_crlib//testutils/require",
         "@com_github_prometheus_client_model//go",
         "@com_github_prometheus_common//expfmt",
         "@com_github_stretchr_testify//require",

pkg/util/metric/aggmetric/histogram.go

@@ -357,3 +357,220 @@ func (sch *SQLChildHistogram) RecordValue(v int64) {
 func (sch *SQLChildHistogram) Value() metric.HistogramSnapshot {
 	return sch.h.CumulativeSnapshot()
 }
+
+// HighCardinalityHistogram is similar to AggHistogram but uses cache storage instead of B-tree,
+// allowing for automatic eviction of less frequently used child metrics.
+// This is useful when dealing with high cardinality metrics that might exceed resource limits.
+type HighCardinalityHistogram struct {
+	h      metric.IHistogram
+	create func() metric.IHistogram
+	childSet
+	labelSliceCache *metric.LabelSliceCache
+	ticker          struct {
+		// We use a RWMutex, because we don't want child histograms to contend when
+		// recording values, unless we're rotating histograms for the parent & children.
+		// In this instance, the "writer" for the RWMutex is the ticker, and the "readers"
+		// are all the child histograms recording their values.
+		syncutil.RWMutex
+		*tick.Ticker
+	}
+}
+
+var _ metric.Iterable = (*HighCardinalityHistogram)(nil)
+var _ metric.PrometheusEvictable = (*HighCardinalityHistogram)(nil)
+var _ metric.WindowedHistogram = (*HighCardinalityHistogram)(nil)
+var _ metric.CumulativeHistogram = (*HighCardinalityHistogram)(nil)
+
+// NewHighCardinalityHistogram constructs a new HighCardinalityHistogram that uses cache storage
+// with eviction for child metrics.
+func NewHighCardinalityHistogram(
+	opts metric.HistogramOptions, childLabels ...string,
+) *HighCardinalityHistogram {
+	create := func() metric.IHistogram {
+		return metric.NewHistogram(opts)
+	}
+	h := &HighCardinalityHistogram{
+		h:      create(),
+		create: create,
+	}
+	h.ticker.Ticker = tick.NewTicker(
+		now(),
+		opts.Duration/metric.WindowedHistogramWrapNum,
+		func() {
+			// Atomically rotate the histogram window for the
+			// parent histogram, and all the child histograms.
+			h.h.Tick()
+			h.childSet.apply(func(childItem MetricItem) {
+				childHist, ok := childItem.(*HighCardinalityChildHistogram)
+				if !ok {
+					panic(errors.AssertionFailedf(
+						"unable to assert type of child for histogram %q when rotating histogram windows",
+						opts.Metadata.Name))
+				}
+				childHist.h.Tick()
+			})
+		})
+	h.initWithCacheStorageType(childLabels, opts.Metadata.Name)
+	return h
+}
+
+// GetName is part of the metric.Iterable interface.
+func (h *HighCardinalityHistogram) GetName(useStaticLabels bool) string {
+	return h.h.GetName(useStaticLabels)
+}
+
+// GetHelp is part of the metric.Iterable interface.
+func (h *HighCardinalityHistogram) GetHelp() string { return h.h.GetHelp() }
+
+// GetMeasurement is part of the metric.Iterable interface.
+func (h *HighCardinalityHistogram) GetMeasurement() string { return h.h.GetMeasurement() }
+
+// GetUnit is part of the metric.Iterable interface.
+func (h *HighCardinalityHistogram) GetUnit() metric.Unit { return h.h.GetUnit() }
+
+// GetMetadata is part of the metric.Iterable interface.
+func (h *HighCardinalityHistogram) GetMetadata() metric.Metadata { return h.h.GetMetadata() }
+
+// Inspect is part of the metric.Iterable interface.
+func (h *HighCardinalityHistogram) Inspect(f func(interface{})) {
+	func() {
+		h.ticker.Lock()
+		defer h.ticker.Unlock()
+		tick.MaybeTick(&h.ticker)
+	}()
+	f(h)
+}
+
+// CumulativeSnapshot is part of the metric.CumulativeHistogram interface.
+func (h *HighCardinalityHistogram) CumulativeSnapshot() metric.HistogramSnapshot {
+	return h.h.CumulativeSnapshot()
+}
+
+// WindowedSnapshot is part of the metric.WindowedHistogram interface.
+func (h *HighCardinalityHistogram) WindowedSnapshot() metric.HistogramSnapshot {
+	return h.h.WindowedSnapshot()
+}
+
+// GetType is part of the metric.PrometheusExportable interface.
+func (h *HighCardinalityHistogram) GetType() *prometheusgo.MetricType {
+	return h.h.GetType()
+}
+
+// GetLabels is part of the metric.PrometheusExportable interface.
+func (h *HighCardinalityHistogram) GetLabels(useStaticLabels bool) []*prometheusgo.LabelPair {
+	return h.h.GetLabels(useStaticLabels)
+}
+
+// ToPrometheusMetric is part of the metric.PrometheusExportable interface.
+func (h *HighCardinalityHistogram) ToPrometheusMetric() *prometheusgo.Metric {
+	return h.h.ToPrometheusMetric()
+}
+
+// RecordValue records the histogram value for the given label values. If a
+// histogram with the given label values doesn't exist yet, it creates a new
+// histogram and records against it. RecordValue records value in parent metrics as well.
+func (h *HighCardinalityHistogram) RecordValue(v int64, labelValues ...string) {
+	childMetric := h.GetOrAddChild(labelValues...)
+
+	h.ticker.RLock()
+	defer h.ticker.RUnlock()
+
+	h.h.RecordValue(v)
+	if childMetric != nil {
+		childMetric.RecordValue(v)
+	}
+}
+
+// Each is part of the metric.PrometheusIterable interface.
+func (h *HighCardinalityHistogram) Each(
+	labels []*prometheusgo.LabelPair, f func(metric *prometheusgo.Metric),
+) {
+	h.EachWithLabels(labels, f, h.labelSliceCache)
+}
+
+// InitializeMetrics is part of the PrometheusEvictable interface.
+func (h *HighCardinalityHistogram) InitializeMetrics(labelCache *metric.LabelSliceCache) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.labelSliceCache = labelCache
+}
+
+// GetOrAddChild returns the existing child histogram for the given label values,
+// or creates a new one if it doesn't exist. This is the preferred method for
+// cache-based storage to avoid panics on existing keys.
+func (h *HighCardinalityHistogram) GetOrAddChild(
+	labelVals ...string,
+) *HighCardinalityChildHistogram {
+	if len(labelVals) == 0 {
+		return nil
+	}
+
+	// Create a LabelSliceCacheKey from the labelVals.
+	key := metric.LabelSliceCacheKey(metricKey(labelVals...))
+
+	child := h.getOrAddWithLabelSliceCache(h.GetMetadata().Name, h.createHighCardinalityChildHistogram, h.labelSliceCache, labelVals...)
+
+	h.labelSliceCache.Upsert(key, &metric.LabelSliceCacheValue{
+		LabelValues: labelVals,
+	})
+
+	return child.(*HighCardinalityChildHistogram)
+}
+
+func (h *HighCardinalityHistogram) createHighCardinalityChildHistogram(
+	key uint64, cache *metric.LabelSliceCache,
+) LabelSliceCachedChildMetric {
+	return &HighCardinalityChildHistogram{
+		LabelSliceCacheKey: metric.LabelSliceCacheKey(key),
+		LabelSliceCache:    cache,
+		h:                  h.create(),
+		createdAt:          timeutil.Now(),
+	}
+}
+
+// HighCardinalityChildHistogram is a child of a HighCardinalityHistogram. When metrics are
+// collected by prometheus, each of the children will appear with a distinct label,
+// however, when cockroach internally collects metrics, only the parent is collected.
+type HighCardinalityChildHistogram struct {
+	metric.LabelSliceCacheKey
+	h metric.IHistogram
+	*metric.LabelSliceCache
+	createdAt time.Time
+}
+
+func (h *HighCardinalityChildHistogram) CreatedAt() time.Time {
+	return h.createdAt
+}
+
+func (h *HighCardinalityChildHistogram) DecrementLabelSliceCacheReference() {
+	h.LabelSliceCache.DecrementAndDeleteIfZero(h.LabelSliceCacheKey)
+}
+
+// ToPrometheusMetric constructs a prometheus metric for this HighCardinalityChildHistogram.
+func (h *HighCardinalityChildHistogram) ToPrometheusMetric() *prometheusgo.Metric {
+	return h.h.ToPrometheusMetric()
+}
+
+func (h *HighCardinalityChildHistogram) labelValues() []string {
+	lv, ok := h.LabelSliceCache.Get(h.LabelSliceCacheKey)
+	if !ok {
+		return nil
+	}
+	return lv.LabelValues
+}
+
+// RecordValue records the histogram value.
+func (h *HighCardinalityChildHistogram) RecordValue(v int64) {
+	h.h.RecordValue(v)
+}
+
+// CumulativeSnapshot returns the cumulative histogram snapshot.
+func (h *HighCardinalityChildHistogram) CumulativeSnapshot() metric.HistogramSnapshot {
+	return h.h.CumulativeSnapshot()
+}
+
+// WindowedSnapshot returns the windowed histogram snapshot.
+func (h *HighCardinalityChildHistogram) WindowedSnapshot() metric.HistogramSnapshot {
+	return h.h.WindowedSnapshot()
+}

pkg/util/metric/aggmetric/histogram_test.go

@@ -12,15 +12,16 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/echotest"
 	"github.com/cockroachdb/cockroach/pkg/util/cache"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
-	"github.com/cockroachdb/crlib/testutils/require"
 	"github.com/prometheus/common/expfmt"
+	"github.com/stretchr/testify/require"
 )
 
 func TestSQLHistogram(t *testing.T) {
@@ -86,3 +87,84 @@ func TestSQLHistogram(t *testing.T) {
 	}
 	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
 }
+
+func TestHighCardinalityHistogram(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	const cacheSize = 10
+	r := metric.NewRegistry()
+	writePrometheusMetrics := WritePrometheusMetricsFunc(r)
+
+	h := NewHighCardinalityHistogram(metric.HistogramOptions{
+		Metadata: metric.Metadata{
+			Name: "histo_gram",
+		},
+		Duration:     base.DefaultHistogramWindowInterval(),
+		MaxVal:       100,
+		SigFigs:      1,
+		BucketConfig: metric.Percent100Buckets,
+	}, "database", "application_name")
+
+	h.mu.children = &UnorderedCacheWrapper{
+		cache: initialiseCacheStorageForTesting(),
+	}
+	r.AddMetric(h)
+
+	// Initialize with a label slice cache to test eviction
+	labelSliceCache := metric.NewLabelSliceCache()
+	h.InitializeMetrics(labelSliceCache)
+
+	for i := 0; i < cacheSize+5; i++ {
+		h.RecordValue(int64(i+1), "1", strconv.Itoa(i))
+	}
+
+	// Wait more than cache eviction time to make sure that keys are not evicted based on only cache size.
+	time.Sleep(6 * time.Second)
+
+	testFile := "HighCardinalityHistogram_pre_eviction.txt"
+	if metric.HdrEnabled() {
+		testFile = "HighCardinalityHistogram_pre_eviction_hdr.txt"
+	}
+
+	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
+
+	for i := 0; i < cacheSize+5; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("1", strconv.Itoa(i)))
+		labelSliceValue, ok := labelSliceCache.Get(metricKey)
+		require.True(t, ok, "missing labelSliceValue in label slice cache")
+		require.Equal(t, int64(1), labelSliceValue.Counter.Load(), "the value should be 1")
+		require.Equal(t, []string{"1", strconv.Itoa(i)}, labelSliceValue.LabelValues, "label values are mismatching")
+
+	}
+
+	for i := 0 + cacheSize; i < cacheSize+5; i++ {
+		h.RecordValue(int64(i+10), "2", strconv.Itoa(i))
+	}
+
+	testFile = "HighCardinalityHistogram_post_eviction.txt"
+	if metric.HdrEnabled() {
+		testFile = "HighCardinalityHistogram_post_eviction_hdr.txt"
+	}
+	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
+
+	for i := 0; i < 5; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("1", strconv.Itoa(i)))
+		_, ok := labelSliceCache.Get(metricKey)
+		require.False(t, ok, "labelSliceValue should not be present.")
+	}
+
+	for i := 10; i < 15; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("1", strconv.Itoa(i)))
+		labelSliceValue, ok := labelSliceCache.Get(metricKey)
+		require.True(t, ok, "missing labelSliceValue in label slice cache")
+		require.Equal(t, int64(1), labelSliceValue.Counter.Load(), "the value should be 1")
+		require.Equal(t, []string{"1", strconv.Itoa(i)}, labelSliceValue.LabelValues, "label values are mismatching")
+	}
+
+	for i := 10; i < 15; i++ {
+		metricKey := metric.LabelSliceCacheKey(metricKey("2", strconv.Itoa(i)))
+		labelSliceValue, ok := labelSliceCache.Get(metricKey)
+		require.True(t, ok, "missing labelSliceValue in label slice cache")
+		require.Equal(t, int64(1), labelSliceValue.Counter.Load(), "the value should be 1")
+		require.Equal(t, []string{"2", strconv.Itoa(i)}, labelSliceValue.LabelValues, "label values are mismatching")
+	}
+}