Merge #144377

144377: metrics: add SQLCounter to support high cardinality metrics r=aa-joshi a=aa-joshi

This patch introduces `SQLCounter` which is an aggregation counter of `SQLChildCounter` metrics. SQLCounter supports combination of `database` and `application_name` labels. The SQLChildCounter stores the value of a counter for a given combination of database and application name. SQLCounter internally uses cache.UnorderedCache to store child metrics with default size of 5000. SQLCounter will report to crdb-internal time series only the aggregate sum of all its children, while its children are additionally exported to prometheus. SQLCounter differs from AggCounter in that a SQLCounter creates child metrics dynamically while AggCounter needs child creation up front. We have extracted out dynamic child creation from AggCounter in SQLCounter.

Epic: CRDB-43153
Part of: CRDB-48489
Release note: None

Co-authored-by: Akshay Joshi <[email protected]>

Author: craig[bot]

pkg/util/metric/aggmetric/agg_metric.go

@@ -22,6 +22,18 @@ import (
 
 var delimiter = []byte{'_'}
 
+const (
+	dbLabel  = "database"
+	appLabel = "application_name"
+)
+
+const (
+	LabelConfigDisabled = iota
+	LabelConfigApp
+	LabelConfigDB
+	LabelConfigAppAndDB
+)
+
 // Builder is used to ease constructing metrics with the same labels.
 type Builder struct {
 	labels []string
@@ -78,22 +90,16 @@ func (cs *childSet) initWithBTreeStorageType(labels []string) {
 	}
 }
 
-func (cs *childSet) initWithCacheStorageType(labels []string) {
-	// cacheSize is the default number of children that can be stored in the cache.
-	// If the cache exceeds this size, the oldest children are evicted. This is
-	// specific to cache storage for children
+func getCacheStorage() *cache.UnorderedCache {
 	const cacheSize = 5000
-	cs.labels = labels
 	cacheStorage := cache.NewUnorderedCache(cache.Config{
 		Policy: cache.CacheLRU,
 		//TODO (aa-joshi) : make cacheSize configurable in the future
 		ShouldEvict: func(size int, key, value interface{}) bool {
 			return size > cacheSize
 		},
 	})
-	cs.mu.children = &UnorderedCacheWrapper{
-		cache: cacheStorage,
-	}
+	return cacheStorage
 }
 
 func (cs *childSet) Each(

pkg/util/metric/aggmetric/agg_metric_test.go

@@ -293,24 +293,23 @@ func TestAggMetricClear(t *testing.T) {
 	}, "tenant_id")
 	r.AddMetric(c)
 
-	d := NewCounter(metric.Metadata{
+	d := NewSQLCounter(metric.Metadata{
 		Name: "bar_counter",
-	}, "tenant_id")
-	d.initWithCacheStorageType([]string{"tenant_id"})
+	})
 	r.AddMetric(d)
-
+	d.labelConfig.Store(LabelConfigAppAndDB)
 	tenant2 := roachpb.MustMakeTenantID(2)
 	c1 := c.AddChild(tenant2.String())
 
 	t.Run("before clear", func(t *testing.T) {
 		c1.Inc(2)
-		d.Inc(2, "3")
+		d.Inc(2, "test-db", "test-app")
 		testFile := "aggMetric_pre_clear.txt"
 		echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
 	})
 
 	c.clear()
-	d.clear()
+	d.mu.children.Clear()
 
 	t.Run("post clear", func(t *testing.T) {
 		testFile := "aggMetric_post_clear.txt"

pkg/util/metric/aggmetric/counter.go

@@ -9,7 +9,7 @@ import (
 	"sync/atomic"
 
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
-	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/gogo/protobuf/proto"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
@@ -83,29 +83,6 @@ func (c *AggCounter) AddChild(labelVals ...string) *Counter {
 	return child
 }
 
-// Inc increments the counter value by i for the given label values. If a
-// counter with the given label values doesn't exist yet, it creates a new
-// counter and increments it. Panics if the number of label values doesn't
-// match the number of labels defined for this counter and if the storage type
-// is not StorageTypeCache.
-func (c *AggCounter) Inc(i int64, labelVals ...string) {
-	if len(c.labels) != len(labelVals) {
-		panic(errors.AssertionFailedf(
-			"cannot increment child with %d label values %v to a metric with %d labels %v",
-			len(labelVals), labelVals, len(c.labels), c.labels))
-	}
-
-	// If the child already exists, increment it.
-	if child, ok := c.get(labelVals...); ok {
-		child.(*Counter).Inc(i)
-		return
-	}
-
-	// Otherwise, create a new child and increment it.
-	child := c.AddChild(labelVals...)
-	child.Inc(i)
-}
-
 // RemoveChild removes a Gauge from this AggGauge. This method panics if a Gauge
 // does not exist for this set of labelVals.
 func (g *AggCounter) RemoveChild(labelVals ...string) {
@@ -267,3 +244,188 @@ func (g *CounterFloat64) Inc(i float64) {
 func (g *CounterFloat64) UpdateIfHigher(i float64) {
 	g.value.UpdateIfHigher(i)
 }
+
+// SQLCounter maintains a value as the sum of its children. The counter will
+// report to crdb-internal time series only the aggregate sum of all of its
+// children, while its children are additionally exported to prometheus via the
+// PrometheusIterable interface. SQLCounter differs from AggCounter in that
+// a SQLCounter creates child metrics dynamically while AggCounter needs the
+// child creation up front.
+type SQLCounter struct {
+	g           metric.Counter
+	labelConfig atomic.Uint64
+	mu          struct {
+		syncutil.Mutex
+		children ChildrenStorage
+	}
+}
+
+func (c *SQLCounter) Each(
+	labels []*io_prometheus_client.LabelPair, f func(metric *io_prometheus_client.Metric),
+) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.mu.children.Do(func(e interface{}) {
+		cm := c.mu.children.GetChildMetric(e)
+		pm := cm.ToPrometheusMetric()
+
+		childLabels := make([]*io_prometheus_client.LabelPair, 0, len(labels)+2)
+		childLabels = append(childLabels, labels...)
+		lvs := cm.labelValues()
+		dbLabel := dbLabel
+		appLabel := appLabel
+		switch c.labelConfig.Load() {
+		case LabelConfigDB:
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &dbLabel,
+				Value: &lvs[0],
+			})
+		case LabelConfigApp:
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &appLabel,
+				Value: &lvs[0],
+			})
+		case LabelConfigAppAndDB:
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &dbLabel,
+				Value: &lvs[0],
+			})
+			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
+				Name:  &appLabel,
+				Value: &lvs[1],
+			})
+		default:
+		}
+		pm.Label = childLabels
+		f(pm)
+	})
+}
+
+var _ metric.Iterable = (*SQLCounter)(nil)
+var _ metric.PrometheusIterable = (*SQLCounter)(nil)
+var _ metric.PrometheusExportable = (*SQLCounter)(nil)
+
+// NewSQLCounter constructs a new SQLCounter.
+func NewSQLCounter(metadata metric.Metadata) *SQLCounter {
+	c := &SQLCounter{
+		g: *metric.NewCounter(metadata),
+	}
+	c.mu.children = &UnorderedCacheWrapper{
+		cache: getCacheStorage(),
+	}
+	c.labelConfig.Store(LabelConfigDisabled)
+	return c
+}
+
+// getOrAddChild returns the child metric for the given label values. If the child
+// doesn't exist, it creates a new one and adds it to the collection.
+func (c *SQLCounter) getOrAddChild(labelValues ...string) ChildMetric {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If the child already exists, return it.
+	if child, ok := c.mu.children.Get(labelValues...); ok {
+		return child
+	}
+
+	// Otherwise, create a new child and return it.
+	child := &SQLChildCounter{
+		labelValuesSlice: labelValuesSlice(labelValues),
+	}
+	c.mu.children.Add(child)
+	return child
+}
+
+// GetType is part of the metric.PrometheusExportable interface.
+func (c *SQLCounter) GetType() *io_prometheus_client.MetricType {
+	return c.g.GetType()
+}
+
+// GetLabels is part of the metric.PrometheusExportable interface.
+func (c *SQLCounter) GetLabels(useStaticLabels bool) []*io_prometheus_client.LabelPair {
+	return c.g.GetLabels(useStaticLabels)
+}
+
+// ToPrometheusMetric is part of the metric.PrometheusExportable interface.
+func (c *SQLCounter) ToPrometheusMetric() *io_prometheus_client.Metric {
+	return c.g.ToPrometheusMetric()
+}
+
+// GetName is part of the metric.Iterable interface.
+func (c *SQLCounter) GetName(useStaticLabels bool) string {
+	return c.g.GetName(useStaticLabels)
+}
+
+// GetHelp is part of the metric.Iterable interface.
+func (c *SQLCounter) GetHelp() string {
+	return c.g.GetHelp()
+}
+
+// GetMeasurement is part of the metric.Iterable interface.
+func (c *SQLCounter) GetMeasurement() string {
+	return c.g.GetMeasurement()
+}
+
+// GetUnit is part of the metric.Iterable interface.
+func (c *SQLCounter) GetUnit() metric.Unit {
+	return c.g.GetUnit()
+}
+
+// GetMetadata is part of the metric.Iterable interface.
+func (c *SQLCounter) GetMetadata() metric.Metadata {
+	return c.g.GetMetadata()
+}
+
+// Inspect is part of the metric.Iterable interface.
+func (c *SQLCounter) Inspect(f func(interface{})) {
+	f(c)
+}
+
+// Inc increments the counter value by i for the given label values. If a
+// counter with the given label values doesn't exist yet, it creates a new
+// counter based on labelConfig and increments it. Inc increments parent metrics
+// irrespective of labelConfig.
+func (c *SQLCounter) Inc(i int64, db, app string) {
+	c.g.Inc(i)
+
+	var childMetric ChildMetric
+	switch c.labelConfig.Load() {
+	case LabelConfigDB:
+		childMetric = c.getOrAddChild(db)
+	case LabelConfigApp:
+		childMetric = c.getOrAddChild(app)
+	case LabelConfigAppAndDB:
+		childMetric = c.getOrAddChild(db, app)
+	default:
+		return
+	}
+	childMetric.(*SQLChildCounter).Inc(i)
+}
+
+// SQLChildCounter is a child of a SQLCounter. When metrics are collected by prometheus,
+// each of the children will appear with a distinct label, however, when cockroach
+// internally collects metrics, only the parent is collected.
+type SQLChildCounter struct {
+	labelValuesSlice
+	value metric.Counter
+}
+
+// ToPrometheusMetric constructs a prometheus metric for this Counter.
+func (s *SQLChildCounter) ToPrometheusMetric() *io_prometheus_client.Metric {
+	return &io_prometheus_client.Metric{
+		Counter: &io_prometheus_client.Counter{
+			Value: proto.Float64(float64(s.Value())),
+		},
+	}
+}
+
+// Value returns the SQLChildCounter's current value.
+func (s *SQLChildCounter) Value() int64 {
+	return s.value.Count()
+}
+
+// Inc increments the SQLChildCounter's value.
+func (s *SQLChildCounter) Inc(i int64) {
+	s.value.Inc(i)
+}

pkg/util/metric/aggmetric/counter_test.go

@@ -43,11 +43,11 @@ func TestAggCounter(t *testing.T) {
 		return strings.Join(lines, "\n")
 	}
 
-	c := NewCounter(metric.Metadata{
+	c := NewSQLCounter(metric.Metadata{
 		Name: "foo_counter",
-	}, "tenant_id", "counter_label")
+	})
+	c.labelConfig.Store(LabelConfigAppAndDB)
 	r.AddMetric(c)
-	c.initWithCacheStorageType([]string{"tenant_id", "counter_label"})
 	cacheStorage := cache.NewUnorderedCache(cache.Config{
 		Policy: cache.CacheLRU,
 		ShouldEvict: func(size int, key, value interface{}) bool {
@@ -62,13 +62,13 @@ func TestAggCounter(t *testing.T) {
 		c.Inc(1, "1", strconv.Itoa(i))
 	}
 
-	testFile := "aggCounter_pre_eviction.txt"
+	testFile := "SQLCounter_pre_eviction.txt"
 	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
 
 	for i := 0 + cacheSize; i < cacheSize+5; i++ {
 		c.Inc(1, "2", strconv.Itoa(i))
 	}
 
-	testFile = "aggCounter_post_eviction.txt"
+	testFile = "SQLCounter_post_eviction.txt"
 	echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
 }

pkg/util/metric/aggmetric/testdata/SQLCounter_post_eviction.txt

@@ -0,0 +1,13 @@
+echo
+----
+foo_counter 15
+foo_counter{database="1",application_name="5"} 1
+foo_counter{database="1",application_name="6"} 1
+foo_counter{database="1",application_name="7"} 1
+foo_counter{database="1",application_name="8"} 1
+foo_counter{database="1",application_name="9"} 1
+foo_counter{database="2",application_name="10"} 1
+foo_counter{database="2",application_name="11"} 1
+foo_counter{database="2",application_name="12"} 1
+foo_counter{database="2",application_name="13"} 1
+foo_counter{database="2",application_name="14"} 1

pkg/util/metric/aggmetric/testdata/SQLCounter_pre_eviction.txt

@@ -0,0 +1,13 @@
+echo
+----
+foo_counter 10
+foo_counter{database="1",application_name="0"} 1
+foo_counter{database="1",application_name="1"} 1
+foo_counter{database="1",application_name="2"} 1
+foo_counter{database="1",application_name="3"} 1
+foo_counter{database="1",application_name="4"} 1
+foo_counter{database="1",application_name="5"} 1
+foo_counter{database="1",application_name="6"} 1
+foo_counter{database="1",application_name="7"} 1
+foo_counter{database="1",application_name="8"} 1
+foo_counter{database="1",application_name="9"} 1

pkg/util/metric/aggmetric/testdata/aggCounter_post_eviction.txt

@@ -1,6 +1,6 @@
 echo
 ----
 bar_counter 2
-bar_counter{tenant_id="3"} 2
+bar_counter{database="test-db",application_name="test-app"} 2
 foo_counter 2
 foo_counter{tenant_id="2"} 2