metrics: add SQLGauge to support high cardinality metrics

This patch introduces `SQLGauge` which is an aggregation gauge of
`SQLChildGauge` metrics. SQLGauge supports combination of `database` and
`application_name` labels. The SQLChildGauge stores the value of a gauge for a
given combination of database and application name. SQLGauge internally uses
cache.UnorderedCache to store child metrics with default size of 5000.
SQLGauge will report to crdb-internal time series only the aggregate sum of all
its children, while its children are additionally exported to prometheus.
SQLGauge differs from AggGauge in that a SQLGauge creates child metrics
dynamically while AggGauge needs child creation up front. We have extracted out
dynamic child creation from AggGauge in SQLGauge.

Epic: CRDB-43153
Part of: CRDB-48489
Release note: None

Author: aa-joshi

pkg/util/metric/aggmetric/BUILD.bazel

@@ -46,7 +46,6 @@ go_test(
         "@com_github_cockroachdb_crlib//testutils/require",
         "@com_github_prometheus_client_model//go",
         "@com_github_prometheus_common//expfmt",
-        "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/util/metric/aggmetric/agg_metric.go

@@ -11,6 +11,7 @@ package aggmetric
 import (
 	"hash/fnv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/cockroachdb/cockroach/pkg/util/cache"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
@@ -168,6 +169,128 @@ func (cs *childSet) clear() {
 	cs.mu.children.Clear()
 }
 
+type SQLMetric struct {
+	labelConfig atomic.Uint64
+	mu          struct {
+		syncutil.Mutex
+		children ChildrenStorage
+	}
+}
+
+func NewSQLMetric(labelConfig uint64) *SQLMetric {
+	sm := &SQLMetric{}
+	sm.labelConfig.Store(labelConfig)
+	sm.mu.children = &UnorderedCacheWrapper{
+		cache: getCacheStorage(),
+	}
+	return sm
+}
+
+func (sm *SQLMetric) Each(
+	labels []*io_prometheus_client.LabelPair, f func(metric *io_prometheus_client.Metric),
+) {
+	sm.mu.Lock()
+	defer sm.mu.Unlock()
+
+	sm.mu.children.Do(func(e interface{}) {
+		cm := sm.mu.children.GetChildMetric(e)
+		pm := cm.ToPrometheusMetric()
+
+		childLabels := make([]*io_prometheus_client.LabelPair, 0, len(labels)+2)
+		childLabels = append(childLabels, labels...)
+		lvs := cm.labelValues()
+		dbLabel := dbLabel
+		appLabel := appLabel
+		switch sm.labelConfig.Load() {
+		case LabelConfigDB:
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &dbLabel,
+				Value: &lvs[0],
+			})
+		case LabelConfigApp:
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &appLabel,
+				Value: &lvs[0],
+			})
+		case LabelConfigAppAndDB:
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &dbLabel,
+				Value: &lvs[0],
+			})
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &appLabel,
+				Value: &lvs[1],
+			})
+		default:
+		}
+		pm.Label = childLabels
+		f(pm)
+	})
+}
+
+func (sm *SQLMetric) get(labelVals ...string) (ChildMetric, bool) {
+	return sm.mu.children.Get(labelVals...)
+}
+
+func (sm *SQLMetric) add(metric ChildMetric) {
+	sm.mu.children.Add(metric)
+}
+
+// getOrAddChild returns the child metric for the given label values. If the child
+// doesn't exist, it creates a new one and adds it to the collection.
+func (sm *SQLMetric) getOrAddChild(
+	metricType io_prometheus_client.MetricType, labelValues ...string,
+) ChildMetric {
+	sm.mu.Lock()
+	defer sm.mu.Unlock()
+
+	// If the child already exists, return it.
+	if child, ok := sm.get(labelValues...); ok {
+		return child
+	}
+
+	// Otherwise, create a new child, add and return it.
+	var child ChildMetric
+	switch metricType {
+	case io_prometheus_client.MetricType_COUNTER:
+		child = &SQLChildCounter{
+			labelValuesSlice: labelValuesSlice(labelValues),
+		}
+	case io_prometheus_client.MetricType_GAUGE:
+		child = &SQLChildGauge{
+			labelValuesSlice: labelValuesSlice(labelValues),
+		}
+	default:
+		panic(errors.AssertionFailedf("unrecognised metric type %v", metricType))
+	}
+
+	sm.add(child)
+	return child
+}
+
+// getChildByLabelConfig returns the child metric based on the label configuration.
+// It returns the child metric and a boolean indicating if the child was found.
+// If the label configuration is either LabelConfigDisabled or unrecognised, it returns
+// ChildMetric as nil and false.
+func (sm *SQLMetric) getChildByLabelConfig(
+	metricType io_prometheus_client.MetricType, db string, app string,
+) (ChildMetric, bool) {
+	var childMetric ChildMetric
+	switch sm.labelConfig.Load() {
+	case LabelConfigDB:
+		childMetric = sm.getOrAddChild(metricType, db)
+		return childMetric, true
+	case LabelConfigApp:
+		childMetric = sm.getOrAddChild(metricType, app)
+		return childMetric, true
+	case LabelConfigAppAndDB:
+		childMetric = sm.getOrAddChild(metricType, db, app)
+		return childMetric, true
+	default:
+		return nil, false
+	}
+}
+
 type MetricItem interface {
 	labelValuer
 }

pkg/util/metric/aggmetric/counter.go

@@ -9,7 +9,6 @@ import (
 	"sync/atomic"
 
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
-	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/gogo/protobuf/proto"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
@@ -252,54 +251,8 @@ func (g *CounterFloat64) UpdateIfHigher(i float64) {
 // a SQLCounter creates child metrics dynamically while AggCounter needs the
 // child creation up front.
 type SQLCounter struct {
-	g           metric.Counter
-	labelConfig atomic.Uint64
-	mu          struct {
-		syncutil.Mutex
-		children ChildrenStorage
-	}
-}
-
-func (c *SQLCounter) Each(
-	labels []*io_prometheus_client.LabelPair, f func(metric *io_prometheus_client.Metric),
-) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.mu.children.Do(func(e interface{}) {
-		cm := c.mu.children.GetChildMetric(e)
-		pm := cm.ToPrometheusMetric()
-
-		childLabels := make([]*io_prometheus_client.LabelPair, 0, len(labels)+2)
-		childLabels = append(childLabels, labels...)
-		lvs := cm.labelValues()
-		dbLabel := dbLabel
-		appLabel := appLabel
-		switch c.labelConfig.Load() {
-		case LabelConfigDB:
-			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
-				Name:  &dbLabel,
-				Value: &lvs[0],
-			})
-		case LabelConfigApp:
-			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
-				Name:  &appLabel,
-				Value: &lvs[0],
-			})
-		case LabelConfigAppAndDB:
-			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
-				Name:  &dbLabel,
-				Value: &lvs[0],
-			})
-			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
-				Name:  &appLabel,
-				Value: &lvs[1],
-			})
-		default:
-		}
-		pm.Label = childLabels
-		f(pm)
-	})
+	g metric.Counter
+	*SQLMetric
 }
 
 var _ metric.Iterable = (*SQLCounter)(nil)
@@ -311,32 +264,10 @@ func NewSQLCounter(metadata metric.Metadata) *SQLCounter {
 	c := &SQLCounter{
 		g: *metric.NewCounter(metadata),
 	}
-	c.mu.children = &UnorderedCacheWrapper{
-		cache: getCacheStorage(),
-	}
-	c.labelConfig.Store(LabelConfigDisabled)
+	c.SQLMetric = NewSQLMetric(LabelConfigDisabled)
 	return c
 }
 
-// getOrAddChild returns the child metric for the given label values. If the child
-// doesn't exist, it creates a new one and adds it to the collection.
-func (c *SQLCounter) getOrAddChild(labelValues ...string) ChildMetric {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	// If the child already exists, return it.
-	if child, ok := c.mu.children.Get(labelValues...); ok {
-		return child
-	}
-
-	// Otherwise, create a new child and return it.
-	child := &SQLChildCounter{
-		labelValuesSlice: labelValuesSlice(labelValues),
-	}
-	c.mu.children.Add(child)
-	return child
-}
-
 // GetType is part of the metric.PrometheusExportable interface.
 func (c *SQLCounter) GetType() *io_prometheus_client.MetricType {
 	return c.g.GetType()
@@ -389,17 +320,11 @@ func (c *SQLCounter) Inspect(f func(interface{})) {
 func (c *SQLCounter) Inc(i int64, db, app string) {
 	c.g.Inc(i)
 
-	var childMetric ChildMetric
-	switch c.labelConfig.Load() {
-	case LabelConfigDB:
-		childMetric = c.getOrAddChild(db)
-	case LabelConfigApp:
-		childMetric = c.getOrAddChild(app)
-	case LabelConfigAppAndDB:
-		childMetric = c.getOrAddChild(db, app)
-	default:
+	childMetric, isChildMetricEnabled := c.getChildByLabelConfig(*c.GetType(), db, app)
+	if !isChildMetricEnabled {
 		return
 	}
+
 	childMetric.(*SQLChildCounter).Inc(i)
 }