metric: reinitialise sql child metrics on cluster settings change

This patch introduces two new cluster settings:
`sql.metrics.application_name.enabled` and `sql.metrics.database_name.enabled`.
These settings are used to configure `application_name` and `database` labels
in sql metrics. To support these labels changes in subsequent metric export,
sql metrics require ability to "reset" labels. This reset functionality needs
to be added to the sql server because cluster settings and labels are scoped to
the virtual cluster. We have extended existing metric registry functionality to
reinitialise metrics on cluster change. The metric registry was ideal place
because it tracks and manages metrics.

Epic: CRDB-43153
Part of: CRDB-48253
Release note (sql change): New cluster settings
`sql.metrics.application_name.enabled` and `sql.metrics.database_name.enabled`
with default value of `false` can be set to true, to display application and
database name on supported metrics respectively.

Author: aa-joshi

docs/generated/settings/settings-for-tenants.txt

@@ -323,6 +323,8 @@ sql.log.slow_query.internal_queries.enabled	boolean	false	when set to true, inte
 sql.log.slow_query.latency_threshold	duration	0s	when set to non-zero, log statements whose service latency exceeds the threshold to a secondary logger on each node	application
 sql.log.user_audit	string		user/role-based audit logging configuration. An enterprise license is required for this cluster setting to take effect.	application
 sql.log.user_audit.reduced_config.enabled	boolean	false	enables logic to compute a reduced audit configuration, computing the audit configuration only once at session start instead of at each SQL event. The tradeoff with the increase in performance (~5%), is that changes to the audit configuration (user role memberships/cluster setting) are not reflected within session. Users will need to start a new session to see these changes in their auditing behaviour.	application
+sql.metrics.application_name.enabled	boolean	false	when enabled, SQL metrics would export application name as and additional label as part of child metrics. The number of unique label combinations is limited to 5000 by default.	application
+sql.metrics.database_name.enabled	boolean	false	when enabled, SQL metrics would export database name as and additional label as part of child metrics. The number of unique label combinations is limited to 5000 by default.	application
 sql.metrics.index_usage_stats.enabled	boolean	true	collect per index usage statistics	application
 sql.metrics.max_mem_reported_stmt_fingerprints	integer	100000	the maximum number of reported statement fingerprints stored in memory	application
 sql.metrics.max_mem_reported_txn_fingerprints	integer	100000	the maximum number of reported transaction fingerprints stored in memory	application

docs/generated/settings/settings.html

@@ -278,6 +278,8 @@
 <tr><td><div id="setting-sql-log-slow-query-latency-threshold" class="anchored"><code>sql.log.slow_query.latency_threshold</code></div></td><td>duration</td><td><code>0s</code></td><td>when set to non-zero, log statements whose service latency exceeds the threshold to a secondary logger on each node</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-log-user-audit" class="anchored"><code>sql.log.user_audit</code></div></td><td>string</td><td><code></code></td><td>user/role-based audit logging configuration. An enterprise license is required for this cluster setting to take effect.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-log-user-audit-reduced-config-enabled" class="anchored"><code>sql.log.user_audit.reduced_config.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>enables logic to compute a reduced audit configuration, computing the audit configuration only once at session start instead of at each SQL event. The tradeoff with the increase in performance (~5%), is that changes to the audit configuration (user role memberships/cluster setting) are not reflected within session. Users will need to start a new session to see these changes in their auditing behaviour.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-sql-metrics-application-name-enabled" class="anchored"><code>sql.metrics.application_name.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>when enabled, SQL metrics would export application name as and additional label as part of child metrics. The number of unique label combinations is limited to 5000 by default.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-sql-metrics-database-name-enabled" class="anchored"><code>sql.metrics.database_name.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>when enabled, SQL metrics would export database name as and additional label as part of child metrics. The number of unique label combinations is limited to 5000 by default.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-metrics-index-usage-stats-enabled" class="anchored"><code>sql.metrics.index_usage_stats.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>collect per index usage statistics</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-metrics-max-mem-reported-stmt-fingerprints" class="anchored"><code>sql.metrics.max_mem_reported_stmt_fingerprints</code></div></td><td>integer</td><td><code>100000</code></td><td>the maximum number of reported statement fingerprints stored in memory</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-metrics-max-mem-reported-txn-fingerprints" class="anchored"><code>sql.metrics.max_mem_reported_txn_fingerprints</code></div></td><td>integer</td><td><code>100000</code></td><td>the maximum number of reported transaction fingerprints stored in memory</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/server/server_sql.go

@@ -1384,6 +1384,14 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
 	vmoduleSetting.SetOnChange(&cfg.Settings.SV, fn)
 	fn(ctx)
 
+	metric.AppNameLabelEnabled.SetOnChange(&cfg.Settings.SV, func(ctx context.Context) {
+		pgServer.SQLServer.UpdateLabelValueConfig(&cfg.Settings.SV, cfg.registry)
+	})
+
+	metric.DBNameLabelEnabled.SetOnChange(&cfg.Settings.SV, func(ctx context.Context) {
+		pgServer.SQLServer.UpdateLabelValueConfig(&cfg.Settings.SV, cfg.registry)
+	})
+
 	auditlogging.ConfigureRoleBasedAuditClusterSettings(ctx, execCfg.AuditConfig, execCfg.Settings, &execCfg.Settings.SV)
 
 	return &SQLServer{

pkg/sql/conn_executor.go

@@ -850,6 +850,11 @@ func (s *Server) GetBytesMonitor() *mon.BytesMonitor {
 	return s.pool
 }
 
+func (s *Server) UpdateLabelValueConfig(sv *settings.Values, registry *metric.Registry) {
+	registry.ReinitialiseChildMetrics(metric.DBNameLabelEnabled.Get(sv),
+		metric.AppNameLabelEnabled.Get(sv))
+}
+
 // SetupConn creates a connExecutor for the client connection.
 //
 // When this method returns there are no resources allocated yet that

pkg/util/metric/BUILD.bazel

@@ -22,6 +22,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/util/metric",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/settings",
         "//pkg/util/buildutil",
         "//pkg/util/envutil",
         "//pkg/util/log",

pkg/util/metric/aggmetric/agg_metric.go

@@ -28,13 +28,6 @@ const (
 	appLabel = "application_name"
 )
 
-const (
-	LabelConfigDisabled = iota
-	LabelConfigApp
-	LabelConfigDB
-	LabelConfigAppAndDB
-)
-
 // Builder is used to ease constructing metrics with the same labels.
 type Builder struct {
 	labels []string
@@ -189,9 +182,9 @@ type SQLMetric struct {
 	}
 }
 
-func NewSQLMetric(labelConfig uint64) *SQLMetric {
+func NewSQLMetric(labelConfig metric.LabelConfig) *SQLMetric {
 	sm := &SQLMetric{}
-	sm.labelConfig.Store(labelConfig)
+	sm.labelConfig.Store(uint64(labelConfig))
 	sm.mu.children = &UnorderedCacheWrapper{
 		cache: getCacheStorage(),
 	}
@@ -213,17 +206,17 @@ func (sm *SQLMetric) Each(
 		dbLabel := dbLabel
 		appLabel := appLabel
 		switch sm.labelConfig.Load() {
-		case LabelConfigDB:
+		case uint64(metric.LabelConfigDB):
 			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
 				Name:  &dbLabel,
 				Value: &lvs[0],
 			})
-		case LabelConfigApp:
+		case uint64(metric.LabelConfigApp):
 			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
 				Name:  &appLabel,
 				Value: &lvs[0],
 			})
-		case LabelConfigAppAndDB:
+		case uint64(metric.LabelConfigAppAndDB):
 			childLabels = append(childLabels, &io_prometheus_client.LabelPair{
 				Name:  &dbLabel,
 				Value: &lvs[0],
@@ -275,20 +268,29 @@ func (sm *SQLMetric) getChildByLabelConfig(
 ) (ChildMetric, bool) {
 	var childMetric ChildMetric
 	switch sm.labelConfig.Load() {
-	case LabelConfigDB:
+	case uint64(metric.LabelConfigDB):
 		childMetric = sm.getOrAddChild(f, db)
 		return childMetric, true
-	case LabelConfigApp:
+	case uint64(metric.LabelConfigApp):
 		childMetric = sm.getOrAddChild(f, app)
 		return childMetric, true
-	case LabelConfigAppAndDB:
+	case uint64(metric.LabelConfigAppAndDB):
 		childMetric = sm.getOrAddChild(f, db, app)
 		return childMetric, true
 	default:
 		return nil, false
 	}
 }
 
+// ReinitialiseChildMetrics clears the child metrics and
+// sets the label configuration.
+func (sm *SQLMetric) ReinitialiseChildMetrics(labelConfig metric.LabelConfig) {
+	sm.mu.Lock()
+	defer sm.mu.Unlock()
+	sm.mu.children.Clear()
+	sm.labelConfig.Store(uint64(labelConfig))
+}
+
 type MetricItem interface {
 	labelValuer
 }

pkg/util/metric/aggmetric/agg_metric_test.go

@@ -297,7 +297,7 @@ func TestAggMetricClear(t *testing.T) {
 		Name: "bar_counter",
 	})
 	r.AddMetric(d)
-	d.labelConfig.Store(LabelConfigAppAndDB)
+	d.labelConfig.Store(uint64(metric.LabelConfigAppAndDB))
 	tenant2 := roachpb.MustMakeTenantID(2)
 	c1 := c.AddChild(tenant2.String())
 
@@ -368,3 +368,53 @@ func WritePrometheusMetricsFunc(r *metric.Registry) func(t *testing.T) string {
 	}
 	return writePrometheusMetrics
 }
+
+func TestSQLMetricsReinitialise(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	r := metric.NewRegistry()
+	writePrometheusMetrics := WritePrometheusMetricsFunc(r)
+
+	counter := NewSQLCounter(metric.Metadata{Name: "test.counter"})
+	r.AddMetric(counter)
+
+	gauge := NewSQLGauge(metric.Metadata{Name: "test.gauge"})
+	r.AddMetric(gauge)
+
+	histogram := NewSQLHistogram(metric.HistogramOptions{
+		Metadata: metric.Metadata{
+			Name: "test.histogram",
+		},
+		Duration:     base.DefaultHistogramWindowInterval(),
+		MaxVal:       100,
+		SigFigs:      1,
+		BucketConfig: metric.Percent100Buckets,
+	})
+	r.AddMetric(histogram)
+
+	t.Run("before invoking reinitialise sql metrics", func(t *testing.T) {
+		counter.Inc(1, "test_db", "test_app")
+		gauge.Update(10, "test_db", "test_app")
+		histogram.RecordValue(10, "test_db", "test_app")
+
+		testFile := "sql_metric_pre_reinitialise_child_metrics.txt"
+		if metric.HdrEnabled() {
+			testFile = "sql_metric_pre_reinitialise_child_metrics_hdr.txt"
+		}
+		echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
+	})
+
+	r.ReinitialiseChildMetrics(true, true)
+
+	t.Run("after invoking reinitialise sql metrics", func(t *testing.T) {
+		counter.Inc(1, "test_db", "test_app")
+		gauge.Update(10, "test_db", "test_app")
+		histogram.RecordValue(10, "test_db", "test_app")
+
+		testFile := "sql_metric_post_reinitialise_child_metrics.txt"
+		if metric.HdrEnabled() {
+			testFile = "sql_metric_post_reinitialise_child_metrics_hdr.txt"
+		}
+		echotest.Require(t, writePrometheusMetrics(t), datapathutils.TestDataPath(t, testFile))
+	})
+
+}

pkg/util/metric/aggmetric/counter.go

@@ -256,15 +256,15 @@ type SQLCounter struct {
 }
 
 var _ metric.Iterable = (*SQLCounter)(nil)
-var _ metric.PrometheusIterable = (*SQLCounter)(nil)
+var _ metric.PrometheusReinitialisable = (*SQLCounter)(nil)
 var _ metric.PrometheusExportable = (*SQLCounter)(nil)
 
 // NewSQLCounter constructs a new SQLCounter.
 func NewSQLCounter(metadata metric.Metadata) *SQLCounter {
 	c := &SQLCounter{
 		g: *metric.NewCounter(metadata),
 	}
-	c.SQLMetric = NewSQLMetric(LabelConfigDisabled)
+	c.SQLMetric = NewSQLMetric(metric.LabelConfigDisabled)
 	return c
 }
 

pkg/util/metric/aggmetric/counter_test.go

@@ -46,7 +46,7 @@ func TestAggCounter(t *testing.T) {
 	c := NewSQLCounter(metric.Metadata{
 		Name: "foo_counter",
 	})
-	c.labelConfig.Store(LabelConfigAppAndDB)
+	c.labelConfig.Store(uint64(metric.LabelConfigAppAndDB))
 	r.AddMetric(c)
 	cacheStorage := cache.NewUnorderedCache(cache.Config{
 		Policy: cache.CacheLRU,

pkg/util/metric/aggmetric/gauge.go

@@ -371,14 +371,14 @@ type SQLGauge struct {
 }
 
 var _ metric.Iterable = (*SQLGauge)(nil)
-var _ metric.PrometheusIterable = (*SQLGauge)(nil)
+var _ metric.PrometheusReinitialisable = (*SQLGauge)(nil)
 var _ metric.PrometheusExportable = (*SQLGauge)(nil)
 
 func NewSQLGauge(metadata metric.Metadata) *SQLGauge {
 	g := &SQLGauge{
 		g: *metric.NewGauge(metadata),
 	}
-	g.SQLMetric = NewSQLMetric(LabelConfigDisabled)
+	g.SQLMetric = NewSQLMetric(metric.LabelConfigDisabled)
 	return g
 }