Merge #143511

craig[bot] · dhartunian · craig[bot] · commit 2733799ce366 · 2025-04-07T21:21:26.000Z
143511: metric: add static labels to metric metadata r=angles-n-daemons a=dhartunian Previously, it was not possible to emit metrics with static labels that were known at startup. Labels would need to be dynamically added after `Metadata` was constructed. This made it difficult to define groups of metrics with the same name but different labels. This commit adds two new fields to the `metric.Metadata` struct, which are meant to enable a backwards-compatible transition to labeled metrics from today's definitions which reify the label values into the metric name. The Prometheus export code is modified to accept a flag that governs whether static labels are requested. If so, the labeled name of the metric be used if available. A future PR will add a `/metrics` endpoint that is like `/_status/ vars` except that it will emit the static label version of metrics when they have those definitions available. Part of: #142570 Release note: None Co-authored-by: David Hartunian <davidh@cockroachlabs.com>
diff --git a/pkg/ccl/multitenantccl/tenantcostclient/tenant_side_test.go b/pkg/ccl/multitenantccl/tenantcostclient/tenant_side_test.go
@@ -647,7 +647,7 @@ func (ts *testState) metrics(*testing.T, *datadriven.TestData, cmdArgs) string {
 					if !ok {
 						return
 					}
-					promIter.Each(it.GetLabels(), func(m *prometheusgo.Metric) {
+					promIter.Each(it.GetLabels(false /* useStaticLabels */), func(m *prometheusgo.Metric) {
 						childMetrics += fmt.Sprintf("%s{", typ.GetName())
 						for i, l := range m.Label {
 							if i > 0 {
diff --git a/pkg/ccl/sqlproxyccl/server.go b/pkg/ccl/sqlproxyccl/server.go
@@ -149,7 +149,7 @@ func (s *Server) handleVars(w http.ResponseWriter, r *http.Request) {
 	contentType := expfmt.Negotiate(r.Header)
 	w.Header().Set(httputil.ContentTypeHeader, string(contentType))
 	scrape := func(pm *metric.PrometheusExporter) {
-		pm.ScrapeRegistry(s.metricsRegistry, true /* includeChildMetrics*/, true /* includeAggregateMetrics */)
+		pm.ScrapeRegistry(s.metricsRegistry, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 	}
 	if err := s.prometheusExporter.ScrapeAndPrintAsText(w, contentType, scrape); err != nil {
 		log.Errorf(r.Context(), "%v", err)
diff --git a/pkg/kv/kvserver/client_tenant_test.go b/pkg/kv/kvserver/client_tenant_test.go
@@ -99,7 +99,7 @@ func TestTenantsStorageMetricsOnSplit(t *testing.T) {
 		})
 		ex := metric.MakePrometheusExporter()
 		scrape := func(ex *metric.PrometheusExporter) {
-			ex.ScrapeRegistry(store.Registry(), true /* includeChildMetrics */, true)
+			ex.ScrapeRegistry(store.Registry(), metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 		}
 		var in bytes.Buffer
 		if err := ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape); err != nil {
diff --git a/pkg/server/load_endpoint.go b/pkg/server/load_endpoint.go
@@ -89,7 +89,7 @@ func newLoadEndpoint(
 
 // Exporter for the load vars that are provided only by the load handler.
 func (le *loadEndpoint) scrapeLoadVarsIntoPrometheus(pm *metric.PrometheusExporter) {
-	pm.ScrapeRegistry(le.registry, true, true)
+	pm.ScrapeRegistry(le.registry, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 }
 
 // Handler responsible for serving the instant values of selected
diff --git a/pkg/server/status/recorder.go b/pkg/server/status/recorder.go
@@ -346,15 +346,15 @@ func (mr *MetricsRecorder) ScrapeIntoPrometheus(pm *metric.PrometheusExporter) {
 	}
 	includeChildMetrics := ChildMetricsEnabled.Get(&mr.settings.SV)
 	includeAggregateMetrics := includeAggregateMetricsEnabled.Get(&mr.settings.SV)
-	pm.ScrapeRegistry(mr.mu.nodeRegistry, includeChildMetrics, includeAggregateMetrics)
-	pm.ScrapeRegistry(mr.mu.appRegistry, includeChildMetrics, includeAggregateMetrics)
-	pm.ScrapeRegistry(mr.mu.logRegistry, includeChildMetrics, includeAggregateMetrics)
-	pm.ScrapeRegistry(mr.mu.sysRegistry, includeChildMetrics, includeAggregateMetrics)
+	pm.ScrapeRegistry(mr.mu.nodeRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
+	pm.ScrapeRegistry(mr.mu.appRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
+	pm.ScrapeRegistry(mr.mu.logRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
+	pm.ScrapeRegistry(mr.mu.sysRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
 	for _, reg := range mr.mu.storeRegistries {
-		pm.ScrapeRegistry(reg, includeChildMetrics, includeAggregateMetrics)
+		pm.ScrapeRegistry(reg, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
 	}
 	for _, tenantRegistry := range mr.mu.tenantRegistries {
-		pm.ScrapeRegistry(tenantRegistry, includeChildMetrics, includeAggregateMetrics)
+		pm.ScrapeRegistry(tenantRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
 	}
 }
 
@@ -808,7 +808,7 @@ func (rr registryRecorder) recordChild(
 			return
 		}
 		m := prom.ToPrometheusMetric()
-		m.Label = append(labels, prom.GetLabels()...)
+		m.Label = append(labels, prom.GetLabels(false /* useStaticLabels */)...)
 
 		processChildMetric := func(metric *prometheusgo.Metric) {
 			found := false
diff --git a/pkg/sql/mvcc_statistics_update_job_test.go b/pkg/sql/mvcc_statistics_update_job_test.go
@@ -127,7 +127,7 @@ func TestTenantGlobalAggregatedLivebytes(t *testing.T) {
 			&in,
 			expfmt.FmtText,
 			func(ex *metric.PrometheusExporter) {
-				ex.ScrapeRegistry(r, true /* includeChildMetrics */, true)
+				ex.ScrapeRegistry(r, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 			},
 		)
 		require.NoError(t, err)
diff --git a/pkg/testutils/metrictestutils/metrics_text.go b/pkg/testutils/metrictestutils/metrics_text.go
@@ -21,7 +21,7 @@ import (
 func GetMetricsText(registry *metric.Registry, re *regexp.Regexp) (string, error) {
 	ex := metric.MakePrometheusExporter()
 	scrape := func(ex *metric.PrometheusExporter) {
-		ex.ScrapeRegistry(registry, true /* includeChildMetrics */, true)
+		ex.ScrapeRegistry(registry, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 	}
 	var in bytes.Buffer
 	if err := ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape); err != nil {
diff --git a/pkg/util/metric/BUILD.bazel b/pkg/util/metric/BUILD.bazel
@@ -62,12 +62,14 @@ go_test(
         "//pkg/util/buildutil",
         "//pkg/util/log",
         "//pkg/util/randutil",
+        "@com_github_gogo_protobuf//proto",
         "@com_github_kr_pretty//:pretty",
         "@com_github_prometheus_client_golang//prometheus",
         "@com_github_prometheus_client_model//go",
         "@com_github_prometheus_common//expfmt",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
+        "@org_golang_google_protobuf//proto",
     ],
 )
 
diff --git a/pkg/util/metric/aggmetric/agg_metric_test.go b/pkg/util/metric/aggmetric/agg_metric_test.go
@@ -35,7 +35,7 @@ func TestExcludeAggregateMetrics(t *testing.T) {
 	// Flipping the includeAggregateMetrics flag should have no effect.
 	testutils.RunTrueAndFalse(t, "includeChildMetrics=false,includeAggregateMetrics", func(t *testing.T, includeAggregateMetrics bool) {
 		pe := metric.MakePrometheusExporter()
-		pe.ScrapeRegistry(r, false, includeAggregateMetrics)
+		pe.ScrapeRegistry(r, metric.WithIncludeChildMetrics(false), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
 		families, err := pe.Gather()
 		require.NoError(t, err)
 		require.Equal(t, 1, len(families))
@@ -46,7 +46,7 @@ func TestExcludeAggregateMetrics(t *testing.T) {
 
 	testutils.RunTrueAndFalse(t, "includeChildMetrics=true,includeAggregateMetrics", func(t *testing.T, includeAggregateMetrics bool) {
 		pe := metric.MakePrometheusExporter()
-		pe.ScrapeRegistry(r, true, includeAggregateMetrics)
+		pe.ScrapeRegistry(r, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))
 		families, err := pe.Gather()
 		require.NoError(t, err)
 		require.Equal(t, 1, len(families))
@@ -355,7 +355,7 @@ func WritePrometheusMetricsFunc(r *metric.Registry) func(t *testing.T) string {
 		var in bytes.Buffer
 		ex := metric.MakePrometheusExporter()
 		scrape := func(ex *metric.PrometheusExporter) {
-			ex.ScrapeRegistry(r, true /* includeChildMetrics */, true)
+			ex.ScrapeRegistry(r, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 		}
 		require.NoError(t, ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape))
 		var lines []string
diff --git a/pkg/util/metric/aggmetric/counter.go b/pkg/util/metric/aggmetric/counter.go
@@ -37,6 +37,9 @@ func NewCounter(metadata metric.Metadata, childLabels ...string) *AggCounter {
 // GetName is part of the metric.Iterable interface.
 func (c *AggCounter) GetName() string { return c.g.GetName() }
 
+// GetLabeledName is part of the metric.Iterable interface.
+func (c *AggCounter) GetLabeledName() string { return c.g.GetLabeledName() }
+
 // GetHelp is part of the metric.Iterable interface.
 func (c *AggCounter) GetHelp() string { return c.g.GetHelp() }
 
@@ -58,8 +61,8 @@ func (c *AggCounter) GetType() *io_prometheus_client.MetricType {
 }
 
 // GetLabels is part of the metric.PrometheusExportable interface.
-func (c *AggCounter) GetLabels() []*io_prometheus_client.LabelPair {
-	return c.g.GetLabels()
+func (c *AggCounter) GetLabels(useStaticLabels bool) []*io_prometheus_client.LabelPair {
+	return c.g.GetLabels(useStaticLabels)
 }
 
 // ToPrometheusMetric is part of the metric.PrometheusExportable interface.
@@ -176,6 +179,9 @@ func NewCounterFloat64(metadata metric.Metadata, childLabels ...string) *AggCoun
 // GetName is part of the metric.Iterable interface.
 func (c *AggCounterFloat64) GetName() string { return c.g.GetName() }
 
+// GetLabeledName is part of the metric.Iterable interface.
+func (c *AggCounterFloat64) GetLabeledName() string { return c.g.GetLabeledName() }
+
 // GetHelp is part of the metric.Iterable interface.
 func (c *AggCounterFloat64) GetHelp() string { return c.g.GetHelp() }
 
@@ -197,8 +203,8 @@ func (c *AggCounterFloat64) GetType() *io_prometheus_client.MetricType {
 }
 
 // GetLabels is part of the metric.PrometheusExportable interface.
-func (c *AggCounterFloat64) GetLabels() []*io_prometheus_client.LabelPair {
-	return c.g.GetLabels()
+func (c *AggCounterFloat64) GetLabels(useStaticLabels bool) []*io_prometheus_client.LabelPair {
+	return c.g.GetLabels(useStaticLabels)
 }
 
 // ToPrometheusMetric is part of the metric.PrometheusExportable interface.
diff --git a/pkg/util/metric/aggmetric/counter_test.go b/pkg/util/metric/aggmetric/counter_test.go
@@ -30,7 +30,7 @@ func TestAggCounter(t *testing.T) {
 		var in bytes.Buffer
 		ex := metric.MakePrometheusExporter()
 		scrape := func(ex *metric.PrometheusExporter) {
-			ex.ScrapeRegistry(r, true /* includeChildMetrics */, true)
+			ex.ScrapeRegistry(r, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 		}
 		require.NoError(t, ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape))
 		var lines []string
diff --git a/pkg/util/metric/aggmetric/gauge.go b/pkg/util/metric/aggmetric/gauge.go
@@ -60,6 +60,9 @@ func NewFunctionalGauge(
 // GetName is part of the metric.Iterable interface.
 func (g *AggGauge) GetName() string { return g.g.GetName() }
 
+// GetLabeledName is part of the metric.Iterable interface.
+func (g *AggGauge) GetLabeledName() string { return g.g.GetLabeledName() }
+
 // GetHelp is part of the metric.Iterable interface.
 func (g *AggGauge) GetHelp() string { return g.g.GetHelp() }
 
@@ -81,8 +84,8 @@ func (g *AggGauge) GetType() *io_prometheus_client.MetricType {
 }
 
 // GetLabels is part of the metric.PrometheusExportable interface.
-func (g *AggGauge) GetLabels() []*io_prometheus_client.LabelPair {
-	return g.g.GetLabels()
+func (g *AggGauge) GetLabels(useStaticLabels bool) []*io_prometheus_client.LabelPair {
+	return g.g.GetLabels(useStaticLabels)
 }
 
 // ToPrometheusMetric is part of the metric.PrometheusExportable interface.
@@ -262,6 +265,9 @@ func NewGaugeFloat64(metadata metric.Metadata, childLabels ...string) *AggGaugeF
 // GetName is part of the metric.Iterable interface.
 func (g *AggGaugeFloat64) GetName() string { return g.g.GetName() }
 
+// GetLabeledName is part of the metric.Iterable interface.
+func (g *AggGaugeFloat64) GetLabeledName() string { return g.g.GetLabeledName() }
+
 // GetHelp is part of the metric.Iterable interface.
 func (g *AggGaugeFloat64) GetHelp() string { return g.g.GetHelp() }
 
@@ -283,8 +289,8 @@ func (g *AggGaugeFloat64) GetType() *io_prometheus_client.MetricType {
 }
 
 // GetLabels is part of the metric.PrometheusExportable interface.
-func (g *AggGaugeFloat64) GetLabels() []*io_prometheus_client.LabelPair {
-	return g.g.GetLabels()
+func (g *AggGaugeFloat64) GetLabels(useStaticLabels bool) []*io_prometheus_client.LabelPair {
+	return g.g.GetLabels(useStaticLabels)
 }
 
 // ToPrometheusMetric is part of the metric.PrometheusExportable interface.
diff --git a/pkg/util/metric/aggmetric/gauge_test.go b/pkg/util/metric/aggmetric/gauge_test.go
@@ -65,7 +65,7 @@ func TestAggGaugeMethods(t *testing.T) {
 		var in bytes.Buffer
 		ex := metric.MakePrometheusExporter()
 		scrape := func(ex *metric.PrometheusExporter) {
-			ex.ScrapeRegistry(r, true /* includeChildMetrics */, true)
+			ex.ScrapeRegistry(r, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 		}
 		require.NoError(t, ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape))
 		var lines []string
diff --git a/pkg/util/metric/aggmetric/histogram.go b/pkg/util/metric/aggmetric/histogram.go
@@ -94,6 +94,9 @@ func NewHistogram(opts metric.HistogramOptions, childLabels ...string) *AggHisto
 // GetName is part of the metric.Iterable interface.
 func (a *AggHistogram) GetName() string { return a.h.GetName() }
 
+// GetLabeledName is part of the metric.Iterable interface.
+func (a *AggHistogram) GetLabeledName() string { return a.h.GetLabeledName() }
+
 // GetHelp is part of the metric.Iterable interface.
 func (a *AggHistogram) GetHelp() string { return a.h.GetHelp() }
 
@@ -132,8 +135,8 @@ func (a *AggHistogram) GetType() *prometheusgo.MetricType {
 }
 
 // GetLabels is part of the metric.PrometheusExportable interface.
-func (a *AggHistogram) GetLabels() []*prometheusgo.LabelPair {
-	return a.h.GetLabels()
+func (a *AggHistogram) GetLabels(useStaticLabels bool) []*prometheusgo.LabelPair {
+	return a.h.GetLabels(useStaticLabels)
 }
 
 // ToPrometheusMetric is part of the metric.PrometheusExportable interface.
diff --git a/pkg/util/metric/aggmetric/histogram_test.go b/pkg/util/metric/aggmetric/histogram_test.go
@@ -31,7 +31,7 @@ func TestAggHistogram(t *testing.T) {
 		var in bytes.Buffer
 		ex := metric.MakePrometheusExporter()
 		scrape := func(ex *metric.PrometheusExporter) {
-			ex.ScrapeRegistry(r, true /* includeChildMetrics */, true)
+			ex.ScrapeRegistry(r, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))
 		}
 		require.NoError(t, ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape))
 		var lines []string
diff --git a/pkg/util/metric/metric.go b/pkg/util/metric/metric.go
@@ -60,12 +60,14 @@ type Iterable interface {
 type PrometheusCompatible interface {
 	// GetName is a method on Metadata
 	GetName() string
+	// GetLabeledName is a method on Metadata
+	GetLabeledName() string
 	// GetHelp is a method on Metadata
 	GetHelp() string
 	// GetType returns the prometheus type enum for this metric.
 	GetType() *prometheusgo.MetricType
 	// GetLabels is a method on Metadata
-	GetLabels() []*prometheusgo.LabelPair
+	GetLabels(useStaticLabels bool) []*prometheusgo.LabelPair
 }
 
 // PrometheusExportable is the standard interface for an individual metric
@@ -141,6 +143,17 @@ func (m *Metadata) GetName() string {
 	return m.Name
 }
 
+// GetLabeledName returns the metric's labeled name. This name is expected to be
+// repeated with different static label sets. Those labels can be accessed via
+// `GetLabels(true)`. If the metric does not have a labeled name, the metric's
+// name is returned.
+func (m *Metadata) GetLabeledName() string {
+	if m.LabeledName == "" {
+		return m.Name
+	}
+	return m.LabeledName
+}
+
 // GetHelp returns the metric's help string.
 func (m *Metadata) GetHelp() string {
 	return m.Help
@@ -159,12 +172,23 @@ func (m *Metadata) GetUnit() Unit {
 // GetLabels returns the metric's labels. For rationale behind the conversion
 // from metric.LabelPair to prometheusgo.LabelPair, see the LabelPair comment
 // in pkg/util/metric/metric.proto.
-func (m *Metadata) GetLabels() []*prometheusgo.LabelPair {
-	lps := make([]*prometheusgo.LabelPair, len(m.Labels))
+func (m *Metadata) GetLabels(useStaticLabels bool) []*prometheusgo.LabelPair {
 	// x satisfies the field XXX_unrecognized in prometheusgo.LabelPair.
 	var x []byte
+
+	var lps []*prometheusgo.LabelPair
+	numStaticLabels := 0
+	if useStaticLabels {
+		numStaticLabels = len(m.StaticLabels)
+		lps = make([]*prometheusgo.LabelPair, len(m.Labels)+numStaticLabels)
+		for i, v := range m.StaticLabels {
+			lps[i] = &prometheusgo.LabelPair{Name: v.Name, Value: v.Value, XXX_unrecognized: x}
+		}
+	} else {
+		lps = make([]*prometheusgo.LabelPair, len(m.Labels))
+	}
 	for i, v := range m.Labels {
-		lps[i] = &prometheusgo.LabelPair{Name: v.Name, Value: v.Value, XXX_unrecognized: x}
+		lps[i+numStaticLabels] = &prometheusgo.LabelPair{Name: v.Name, Value: v.Value, XXX_unrecognized: x}
 	}
 	return lps
 }
diff --git a/pkg/util/metric/metric.proto b/pkg/util/metric/metric.proto
@@ -50,10 +50,17 @@ enum Unit {
 // each metric object. It's used to export information about the
 // metric to Prometheus and for Admin UI charts.
 message Metadata {
+  // name is the name of the metric as if it was unlabeled. Set this to ensure
+  // it gets recorded in TSDB.
   required string name = 1 [(gogoproto.nullable) = false];
   required string help = 2 [(gogoproto.nullable) = false];
   required string measurement = 3 [(gogoproto.nullable) = false];
   required Unit unit = 4 [(gogoproto.nullable) = false];
   optional io.prometheus.client.MetricType metricType = 5 [(gogoproto.nullable) = false];
   repeated LabelPair labels = 6;
+
+  // if a labeled_name is provided, it will be output in the /metrics endpoint
+  // with the corresponding static labels.
+  required string labeled_name = 8 [(gogoproto.nullable) = false];
+  repeated LabelPair static_labels = 7;
 }
diff --git a/pkg/util/metric/metric_test.go b/pkg/util/metric/metric_test.go
diff --git a/pkg/util/metric/prometheus_exporter.go b/pkg/util/metric/prometheus_exporter.go
diff --git a/pkg/util/metric/prometheus_exporter_test.go b/pkg/util/metric/prometheus_exporter_test.go

Original file line number	Diff line number	Diff line change
`@@ -647,7 +647,7 @@ func (ts testState) metrics(testing.T, *datadriven.TestData, cmdArgs) string {`
`647`	`647`	`if !ok {`
`648`	`648`	`return`
`649`	`649`	`}`
`650`		`- promIter.Each(it.GetLabels(), func(m *prometheusgo.Metric) {`
	`650`	`+ promIter.Each(it.GetLabels(false /* useStaticLabels /), func(m prometheusgo.Metric) {`
`651`	`651`	`childMetrics += fmt.Sprintf("%s{", typ.GetName())`
`652`	`652`	`for i, l := range m.Label {`
`653`	`653`	`if i > 0 {`
Original file line number	Diff line number	Diff line change
`@@ -149,7 +149,7 @@ func (s Server) handleVars(w http.ResponseWriter, r http.Request) {`
`149`	`149`	`contentType := expfmt.Negotiate(r.Header)`
`150`	`150`	`w.Header().Set(httputil.ContentTypeHeader, string(contentType))`
`151`	`151`	`scrape := func(pm *metric.PrometheusExporter) {`
`152`		`- pm.ScrapeRegistry(s.metricsRegistry, true /* includeChildMetrics/, true / includeAggregateMetrics */)`
	`152`	`+ pm.ScrapeRegistry(s.metricsRegistry, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))`
`153`	`153`	`}`
`154`	`154`	`if err := s.prometheusExporter.ScrapeAndPrintAsText(w, contentType, scrape); err != nil {`
`155`	`155`	`log.Errorf(r.Context(), "%v", err)`
Original file line number	Diff line number	Diff line change
`@@ -99,7 +99,7 @@ func TestTenantsStorageMetricsOnSplit(t *testing.T) {`
`99`	`99`	`})`
`100`	`100`	`ex := metric.MakePrometheusExporter()`
`101`	`101`	`scrape := func(ex *metric.PrometheusExporter) {`
`102`		`- ex.ScrapeRegistry(store.Registry(), true /* includeChildMetrics */, true)`
	`102`	`+ ex.ScrapeRegistry(store.Registry(), metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))`
`103`	`103`	`}`
`104`	`104`	`var in bytes.Buffer`
`105`	`105`	`if err := ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape); err != nil {`
Original file line number	Diff line number	Diff line change
`@@ -89,7 +89,7 @@ func newLoadEndpoint(`
`89`	`89`
`90`	`90`	`// Exporter for the load vars that are provided only by the load handler.`
`91`	`91`	`func (le loadEndpoint) scrapeLoadVarsIntoPrometheus(pm metric.PrometheusExporter) {`
`92`		`- pm.ScrapeRegistry(le.registry, true, true)`
	`92`	`+ pm.ScrapeRegistry(le.registry, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))`
`93`	`93`	`}`
`94`	`94`
`95`	`95`	`// Handler responsible for serving the instant values of selected`
Original file line number	Diff line number	Diff line change
`@@ -346,15 +346,15 @@ func (mr MetricsRecorder) ScrapeIntoPrometheus(pm metric.PrometheusExporter) {`
`346`	`346`	`}`
`347`	`347`	`includeChildMetrics := ChildMetricsEnabled.Get(&mr.settings.SV)`
`348`	`348`	`includeAggregateMetrics := includeAggregateMetricsEnabled.Get(&mr.settings.SV)`
`349`		`- pm.ScrapeRegistry(mr.mu.nodeRegistry, includeChildMetrics, includeAggregateMetrics)`
`350`		`- pm.ScrapeRegistry(mr.mu.appRegistry, includeChildMetrics, includeAggregateMetrics)`
`351`		`- pm.ScrapeRegistry(mr.mu.logRegistry, includeChildMetrics, includeAggregateMetrics)`
`352`		`- pm.ScrapeRegistry(mr.mu.sysRegistry, includeChildMetrics, includeAggregateMetrics)`
	`349`	`+ pm.ScrapeRegistry(mr.mu.nodeRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))`
	`350`	`+ pm.ScrapeRegistry(mr.mu.appRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))`
	`351`	`+ pm.ScrapeRegistry(mr.mu.logRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))`
	`352`	`+ pm.ScrapeRegistry(mr.mu.sysRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))`
`353`	`353`	`for _, reg := range mr.mu.storeRegistries {`
`354`		`- pm.ScrapeRegistry(reg, includeChildMetrics, includeAggregateMetrics)`
	`354`	`+ pm.ScrapeRegistry(reg, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))`
`355`	`355`	`}`
`356`	`356`	`for _, tenantRegistry := range mr.mu.tenantRegistries {`
`357`		`- pm.ScrapeRegistry(tenantRegistry, includeChildMetrics, includeAggregateMetrics)`
	`357`	`+ pm.ScrapeRegistry(tenantRegistry, metric.WithIncludeChildMetrics(includeChildMetrics), metric.WithIncludeAggregateMetrics(includeAggregateMetrics))`
`358`	`358`	`}`
`359`	`359`	`}`
`360`	`360`
`@@ -808,7 +808,7 @@ func (rr registryRecorder) recordChild(`
`808`	`808`	`return`
`809`	`809`	`}`
`810`	`810`	`m := prom.ToPrometheusMetric()`
`811`		`- m.Label = append(labels, prom.GetLabels()...)`
	`811`	`+ m.Label = append(labels, prom.GetLabels(false /* useStaticLabels */)...)`
`812`	`812`
`813`	`813`	`processChildMetric := func(metric *prometheusgo.Metric) {`
`814`	`814`	`found := false`
Original file line number	Diff line number	Diff line change
`@@ -127,7 +127,7 @@ func TestTenantGlobalAggregatedLivebytes(t *testing.T) {`
`127`	`127`	`&in,`
`128`	`128`	`expfmt.FmtText,`
`129`	`129`	`func(ex *metric.PrometheusExporter) {`
`130`		`- ex.ScrapeRegistry(r, true /* includeChildMetrics */, true)`
	`130`	`+ ex.ScrapeRegistry(r, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))`
`131`	`131`	`},`
`132`	`132`	`)`
`133`	`133`	`require.NoError(t, err)`
Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@ import (`
`21`	`21`	`func GetMetricsText(registry metric.Registry, re regexp.Regexp) (string, error) {`
`22`	`22`	`ex := metric.MakePrometheusExporter()`
`23`	`23`	`scrape := func(ex *metric.PrometheusExporter) {`
`24`		`- ex.ScrapeRegistry(registry, true /* includeChildMetrics */, true)`
	`24`	`+ ex.ScrapeRegistry(registry, metric.WithIncludeChildMetrics(true), metric.WithIncludeAggregateMetrics(true))`
`25`	`25`	`}`
`26`	`26`	`var in bytes.Buffer`
`27`	`27`	`if err := ex.ScrapeAndPrintAsText(&in, expfmt.FmtText, scrape); err != nil {`