From 3c3bb21c5ccf8520f0ad1133c47b8dbe902e3ec8 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Thu, 14 Aug 2025 21:59:03 +0900 Subject: [PATCH 1/2] Add EnableTypeAndUnitLabels flag to add type and unit label Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 5 + docs/configuration/v1-guarantees.md | 1 + integration/e2ecortex/client.go | 7 +- integration/otlp_test.go | 69 +++++++++++- pkg/distributor/distributor.go | 8 +- pkg/util/push/otlp.go | 7 +- pkg/util/push/otlp_test.go | 117 ++++++++++++++++---- 8 files changed, 181 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f05aaf715f2..3b9a14e7961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718 * [CHANGE] Validate a tenantID when to use a single tenant resolver. #6727 +* [FEATURE] Distributor: Add an experimental `-distributor.otlp.enable-type-and-unit-labels` flag to add `__type__` and `__unit__` labels for OTLP metrics. #6969 * [FEATURE] Distributor: Add an experimental `-distributor.otlp.allow-delta-temporality` flag to ingest delta temporality otlp metrics. #6934 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index c4ef8305ed6..b461d98f0b3 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3265,6 +3265,11 @@ otlp: # EXPERIMENTAL: If true, delta temporality otlp metrics to be ingested. # CLI flag: -distributor.otlp.allow-delta-temporality [allow_delta_temporality: | default = false] + + # EXPERIMENTAL: If true, the '__type__' and '__unit__' labels are added for + # the OTLP metrics. + # CLI flag: -distributor.otlp.enable-type-and-unit-labels + [enable_type_and_unit_labels: | default = false] ``` ### `etcd_config` diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index 681086c6e98..8825b19e2a9 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -118,6 +118,7 @@ Currently experimental features are: - `alertmanager-sharding-ring.final-sleep` (duration) CLI flag - OTLP Receiver - Ingest delta temporality OTLP metrics (`-distributor.otlp.allow-delta-temporality=true`) + - Add `__type__` and `__unit__` labels (`-distributor.otlp.enable-type-and-unit-labels`) - Persistent tokens in the Ruler Ring: - `-ruler.ring.tokens-file-path` (path) CLI flag - Native Histograms diff --git a/integration/e2ecortex/client.go b/integration/e2ecortex/client.go index bc7fc9c482b..2b46d2262ec 100644 --- a/integration/e2ecortex/client.go +++ b/integration/e2ecortex/client.go @@ -270,7 +270,7 @@ func convertTimeseriesToMetrics(timeseries []prompb.TimeSeries, metadata []promp return metrics } -func otlpWriteRequest(name string, temporality pmetric.AggregationTemporality, labels ...prompb.Label) pmetricotlp.ExportRequest { +func otlpWriteRequest(name, unit string, temporality pmetric.AggregationTemporality, labels ...prompb.Label) pmetricotlp.ExportRequest { d := pmetric.NewMetrics() // Generate One Counter, One Gauge, One Histogram, One Exponential-Histogram @@ -292,6 +292,7 @@ func otlpWriteRequest(name string, temporality pmetric.AggregationTemporality, l // Generate One Counter counterMetric := scopeMetric.Metrics().AppendEmpty() counterMetric.SetName(name) + counterMetric.SetUnit(unit) counterMetric.SetDescription("test-counter-description") counterMetric.SetEmptySum() @@ -310,8 +311,8 @@ func otlpWriteRequest(name string, temporality pmetric.AggregationTemporality, l return pmetricotlp.NewExportRequestFromMetrics(d) } -func (c *Client) OTLPPushExemplar(name string, temporality pmetric.AggregationTemporality, labels ...prompb.Label) (*http.Response, error) { - data, err := otlpWriteRequest(name, temporality, labels...).MarshalProto() +func (c *Client) OTLPPushExemplar(name, unit string, temporality pmetric.AggregationTemporality, labels ...prompb.Label) (*http.Response, error) { + data, err := otlpWriteRequest(name, unit, temporality, labels...).MarshalProto() if err != nil { return nil, err } diff --git a/integration/otlp_test.go b/integration/otlp_test.go index a2a40351330..fe83c1852fa 100644 --- a/integration/otlp_test.go +++ b/integration/otlp_test.go @@ -150,7 +150,7 @@ func TestOTLPIngestExemplar(t *testing.T) { c, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), cortex.HTTPEndpoint(), "", "", "user-1") require.NoError(t, err) - res, err := c.OTLPPushExemplar("exemplar_1", pmetric.AggregationTemporalityCumulative) + res, err := c.OTLPPushExemplar("exemplar_1", "", pmetric.AggregationTemporalityCumulative) require.NoError(t, err) require.Equal(t, 200, res.StatusCode) @@ -242,15 +242,15 @@ func TestOTLPPromoteResourceAttributesPerTenant(t *testing.T) { {Name: "attr3", Value: "value"}, } - res, err := c1.OTLPPushExemplar("series_1", pmetric.AggregationTemporalityCumulative, labels...) + res, err := c1.OTLPPushExemplar("series_1", "", pmetric.AggregationTemporalityCumulative, labels...) require.NoError(t, err) require.Equal(t, 200, res.StatusCode) - res, err = c2.OTLPPushExemplar("series_1", pmetric.AggregationTemporalityCumulative, labels...) + res, err = c2.OTLPPushExemplar("series_1", "", pmetric.AggregationTemporalityCumulative, labels...) require.NoError(t, err) require.Equal(t, 200, res.StatusCode) - res, err = c3.OTLPPushExemplar("series_1", pmetric.AggregationTemporalityCumulative, labels...) + res, err = c3.OTLPPushExemplar("series_1", "", pmetric.AggregationTemporalityCumulative, labels...) require.NoError(t, err) require.Equal(t, 200, res.StatusCode) @@ -267,6 +267,65 @@ func TestOTLPPromoteResourceAttributesPerTenant(t *testing.T) { require.Equal(t, labelSet3, []string{"__name__", "attr1", "attr2", "attr3", "instance", "job"}) } +func TestOTLPEnableTypeAndUnitLabels(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + minio := e2edb.NewMinio(9000, bucketName) + require.NoError(t, s.StartAndWaitReady(minio)) + + // Configure the blocks storage to frequently compact TSDB head + // and ship blocks to the storage. + flags := mergeFlags(BlocksStorageFlags(), map[string]string{ + "-auth.enabled": "true", + + // OTLP + "-distributor.otlp.enable-type-and-unit-labels": "true", + + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + }) + + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) + + // start cortex and assert runtime-config is loaded correctly + cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, flags, "", 9009, 9095) + require.NoError(t, s.StartAndWaitReady(cortex)) + + c, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), cortex.HTTPEndpoint(), "", "", "user-1") + require.NoError(t, err) + + // Push some series to Cortex. + now := time.Now() + + labels := []prompb.Label{ + {Name: "service.name", Value: "test-service"}, + {Name: "attr1", Value: "value"}, + } + + res, err := c.OTLPPushExemplar("series_1", "seconds", pmetric.AggregationTemporalityCumulative, labels...) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + + value, err := c.Query("series_1_seconds", now) + require.NoError(t, err) + vector, ok := value.(model.Vector) + fmt.Println("vector", vector) + require.True(t, ok) + require.Equal(t, 1, len(vector)) + + metric := vector[0].Metric + require.Equal(t, model.LabelValue("seconds"), metric["__unit__"]) + require.Equal(t, model.LabelValue("gauge"), metric["__type__"]) +} + func TestOTLPPushDeltaTemporality(t *testing.T) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) @@ -310,7 +369,7 @@ func TestOTLPPushDeltaTemporality(t *testing.T) { {Name: "attr1", Value: "value"}, } - res, err := c.OTLPPushExemplar("series_1", pmetric.AggregationTemporalityDelta, labels...) + res, err := c.OTLPPushExemplar("series_1", "", pmetric.AggregationTemporalityDelta, labels...) require.NoError(t, err) require.Equal(t, 200, res.StatusCode) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index beaa738e751..80f4ec9025d 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -193,9 +193,10 @@ type InstanceLimits struct { } type OTLPConfig struct { - ConvertAllAttributes bool `yaml:"convert_all_attributes"` - DisableTargetInfo bool `yaml:"disable_target_info"` - AllowDeltaTemporality bool `yaml:"allow_delta_temporality"` + ConvertAllAttributes bool `yaml:"convert_all_attributes"` + DisableTargetInfo bool `yaml:"disable_target_info"` + AllowDeltaTemporality bool `yaml:"allow_delta_temporality"` + EnableTypeAndUnitLabels bool `yaml:"enable_type_and_unit_labels"` } // RegisterFlags adds the flags required to config this to the given FlagSet @@ -224,6 +225,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.OTLPConfig.ConvertAllAttributes, "distributor.otlp.convert-all-attributes", false, "If true, all resource attributes are converted to labels.") f.BoolVar(&cfg.OTLPConfig.DisableTargetInfo, "distributor.otlp.disable-target-info", false, "If true, a target_info metric is not ingested. (refer to: https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems)") f.BoolVar(&cfg.OTLPConfig.AllowDeltaTemporality, "distributor.otlp.allow-delta-temporality", false, "EXPERIMENTAL: If true, delta temporality otlp metrics to be ingested.") + f.BoolVar(&cfg.OTLPConfig.EnableTypeAndUnitLabels, "distributor.otlp.enable-type-and-unit-labels", false, "EXPERIMENTAL: If true, the '__type__' and '__unit__' labels are added for the OTLP metrics.") } // Validate config and returns error on failure diff --git a/pkg/util/push/otlp.go b/pkg/util/push/otlp.go index 83468120656..cd0d3059ab2 100644 --- a/pkg/util/push/otlp.go +++ b/pkg/util/push/otlp.go @@ -178,9 +178,10 @@ func decodeOTLPWriteRequest(ctx context.Context, r *http.Request, maxSize int) ( func convertToPromTS(ctx context.Context, pmetrics pmetric.Metrics, cfg distributor.OTLPConfig, overrides *validation.Overrides, userID string, logger log.Logger) ([]prompb.TimeSeries, []prompb.MetricMetadata, error) { promConverter := prometheusremotewrite.NewPrometheusConverter() settings := prometheusremotewrite.Settings{ - AddMetricSuffixes: true, - DisableTargetInfo: cfg.DisableTargetInfo, - AllowDeltaTemporality: cfg.AllowDeltaTemporality, + AddMetricSuffixes: true, + DisableTargetInfo: cfg.DisableTargetInfo, + AllowDeltaTemporality: cfg.AllowDeltaTemporality, + EnableTypeAndUnitLabels: cfg.EnableTypeAndUnitLabels, } var annots annotations.Annotations diff --git a/pkg/util/push/otlp_test.go b/pkg/util/push/otlp_test.go index 99932188fcd..c02b98d68ef 100644 --- a/pkg/util/push/otlp_test.go +++ b/pkg/util/push/otlp_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/go-kit/log" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,6 +28,80 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) +func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) { + logger := log.NewNopLogger() + ctx := context.Background() + ts := time.Now() + + tests := []struct { + description string + enableTypeAndUnitLabels bool + allowDeltaTemporality bool + otlpSeries pmetric.Metric + expectedLabels labels.Labels + expectedMetadata prompb.MetricMetadata + }{ + { + description: "[enableTypeAndUnitLabels: true], the '__type__' label should be attached when the type is the gauge", + enableTypeAndUnitLabels: true, + otlpSeries: createOtelSum("test", "seconds", pmetric.AggregationTemporalityCumulative, ts), + expectedLabels: labels.FromMap(map[string]string{ + "__name__": "test_seconds", + "__type__": "gauge", + "__unit__": "seconds", + "test_label": "test_value", + }), + expectedMetadata: createPromMetadata("test_seconds", "seconds", prompb.MetricMetadata_GAUGE), + }, + { + description: "[enableTypeAndUnitLabels: true], the '__type__' label should not be attached when the type is unknown", + enableTypeAndUnitLabels: true, + allowDeltaTemporality: true, + otlpSeries: createOtelSum("test", "seconds", pmetric.AggregationTemporalityDelta, ts), + expectedLabels: labels.FromMap(map[string]string{ + "__name__": "test_seconds", + "__unit__": "seconds", + "test_label": "test_value", + }), + expectedMetadata: createPromMetadata("test_seconds", "seconds", prompb.MetricMetadata_UNKNOWN), + }, + { + description: "[enableTypeAndUnitLabels: false]", + enableTypeAndUnitLabels: false, + otlpSeries: createOtelSum("test", "seconds", pmetric.AggregationTemporalityCumulative, ts), + expectedLabels: labels.FromMap(map[string]string{ + "__name__": "test_seconds", + "test_label": "test_value", + }), + expectedMetadata: createPromMetadata("test_seconds", "seconds", prompb.MetricMetadata_GAUGE), + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + cfg := distributor.OTLPConfig{ + EnableTypeAndUnitLabels: test.enableTypeAndUnitLabels, + AllowDeltaTemporality: test.allowDeltaTemporality, + } + metrics := pmetric.NewMetrics() + rm := metrics.ResourceMetrics().AppendEmpty() + sm := rm.ScopeMetrics().AppendEmpty() + + test.otlpSeries.CopyTo(sm.Metrics().AppendEmpty()) + + limits := validation.Limits{} + overrides := validation.NewOverrides(limits, nil) + promSeries, metadata, err := convertToPromTS(ctx, metrics, cfg, overrides, "user-1", logger) + require.NoError(t, err) + require.Equal(t, 1, len(promSeries)) + require.Equal(t, prompb.FromLabels(test.expectedLabels, nil), promSeries[0].Labels) + + require.Equal(t, 1, len(metadata)) + require.Equal(t, test.expectedMetadata, metadata[0]) + }) + } +} + func TestOTLP_AllowDeltaTemporality(t *testing.T) { logger := log.NewNopLogger() ctx := context.Background() @@ -44,24 +119,24 @@ func TestOTLP_AllowDeltaTemporality(t *testing.T) { description: "[allowDeltaTemporality: false] cumulative type should be converted", allowDeltaTemporality: false, otlpSeries: []pmetric.Metric{ - createOtelSum("test_1", pmetric.AggregationTemporalityCumulative, ts), - createOtelSum("test_2", pmetric.AggregationTemporalityCumulative, ts), + createOtelSum("test_1", "", pmetric.AggregationTemporalityCumulative, ts), + createOtelSum("test_2", "", pmetric.AggregationTemporalityCumulative, ts), }, expectedSeries: []prompb.TimeSeries{ createPromFloatSeries("test_1", ts), createPromFloatSeries("test_2", ts), }, expectedMetadata: []prompb.MetricMetadata{ - createPromMetadata("test_1", prompb.MetricMetadata_GAUGE), - createPromMetadata("test_2", prompb.MetricMetadata_GAUGE), + createPromMetadata("test_1", "", prompb.MetricMetadata_GAUGE), + createPromMetadata("test_2", "", prompb.MetricMetadata_GAUGE), }, }, { description: "[allowDeltaTemporality: false] delta type should not be converted", allowDeltaTemporality: false, otlpSeries: []pmetric.Metric{ - createOtelSum("test_1", pmetric.AggregationTemporalityDelta, ts), - createOtelSum("test_2", pmetric.AggregationTemporalityDelta, ts), + createOtelSum("test_1", "", pmetric.AggregationTemporalityDelta, ts), + createOtelSum("test_2", "", pmetric.AggregationTemporalityDelta, ts), }, expectedSeries: []prompb.TimeSeries{}, expectedMetadata: []prompb.MetricMetadata{}, @@ -71,30 +146,30 @@ func TestOTLP_AllowDeltaTemporality(t *testing.T) { description: "[allowDeltaTemporality: true] delta type should be converted", allowDeltaTemporality: true, otlpSeries: []pmetric.Metric{ - createOtelSum("test_1", pmetric.AggregationTemporalityDelta, ts), - createOtelSum("test_2", pmetric.AggregationTemporalityDelta, ts), + createOtelSum("test_1", "", pmetric.AggregationTemporalityDelta, ts), + createOtelSum("test_2", "", pmetric.AggregationTemporalityDelta, ts), }, expectedSeries: []prompb.TimeSeries{ createPromFloatSeries("test_1", ts), createPromFloatSeries("test_2", ts), }, expectedMetadata: []prompb.MetricMetadata{ - createPromMetadata("test_1", prompb.MetricMetadata_UNKNOWN), - createPromMetadata("test_2", prompb.MetricMetadata_UNKNOWN), + createPromMetadata("test_1", "", prompb.MetricMetadata_UNKNOWN), + createPromMetadata("test_2", "", prompb.MetricMetadata_UNKNOWN), }, }, { description: "[allowDeltaTemporality: false] mixed delta and cumulative, should be converted only for cumulative type", allowDeltaTemporality: false, otlpSeries: []pmetric.Metric{ - createOtelSum("test_1", pmetric.AggregationTemporalityDelta, ts), - createOtelSum("test_2", pmetric.AggregationTemporalityCumulative, ts), + createOtelSum("test_1", "", pmetric.AggregationTemporalityDelta, ts), + createOtelSum("test_2", "", pmetric.AggregationTemporalityCumulative, ts), }, expectedSeries: []prompb.TimeSeries{ createPromFloatSeries("test_2", ts), }, expectedMetadata: []prompb.MetricMetadata{ - createPromMetadata("test_2", prompb.MetricMetadata_GAUGE), + createPromMetadata("test_2", "", prompb.MetricMetadata_GAUGE), }, expectedErr: `invalid temporality and type combination for metric "test_1"`, }, @@ -110,8 +185,8 @@ func TestOTLP_AllowDeltaTemporality(t *testing.T) { createPromNativeHistogramSeries("test_2", prompb.Histogram_UNKNOWN, ts), }, expectedMetadata: []prompb.MetricMetadata{ - createPromMetadata("test_1", prompb.MetricMetadata_HISTOGRAM), - createPromMetadata("test_2", prompb.MetricMetadata_HISTOGRAM), + createPromMetadata("test_1", "", prompb.MetricMetadata_HISTOGRAM), + createPromMetadata("test_2", "", prompb.MetricMetadata_HISTOGRAM), }, }, { @@ -137,8 +212,8 @@ func TestOTLP_AllowDeltaTemporality(t *testing.T) { createPromNativeHistogramSeries("test_2", prompb.Histogram_GAUGE, ts), }, expectedMetadata: []prompb.MetricMetadata{ - createPromMetadata("test_1", prompb.MetricMetadata_UNKNOWN), - createPromMetadata("test_2", prompb.MetricMetadata_UNKNOWN), + createPromMetadata("test_1", "", prompb.MetricMetadata_UNKNOWN), + createPromMetadata("test_2", "", prompb.MetricMetadata_UNKNOWN), }, }, { @@ -152,7 +227,7 @@ func TestOTLP_AllowDeltaTemporality(t *testing.T) { createPromNativeHistogramSeries("test_2", prompb.Histogram_UNKNOWN, ts), }, expectedMetadata: []prompb.MetricMetadata{ - createPromMetadata("test_2", prompb.MetricMetadata_HISTOGRAM), + createPromMetadata("test_2", "", prompb.MetricMetadata_HISTOGRAM), }, expectedErr: `invalid temporality and type combination for metric "test_1"`, }, @@ -184,9 +259,10 @@ func TestOTLP_AllowDeltaTemporality(t *testing.T) { } } -func createPromMetadata(name string, metadataType prompb.MetricMetadata_MetricType) prompb.MetricMetadata { +func createPromMetadata(name, unit string, metadataType prompb.MetricMetadata_MetricType) prompb.MetricMetadata { return prompb.MetricMetadata{ Type: metadataType, + Unit: unit, MetricFamilyName: name, } } @@ -221,10 +297,11 @@ func createPromFloatSeries(name string, ts time.Time) prompb.TimeSeries { } // copied from: https://github.com/prometheus/prometheus/blob/v3.5.0/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go -func createOtelSum(name string, temporality pmetric.AggregationTemporality, ts time.Time) pmetric.Metric { +func createOtelSum(name, unit string, temporality pmetric.AggregationTemporality, ts time.Time) pmetric.Metric { metrics := pmetric.NewMetricSlice() m := metrics.AppendEmpty() m.SetName(name) + m.SetUnit(unit) sum := m.SetEmptySum() sum.SetAggregationTemporality(temporality) dp := sum.DataPoints().AppendEmpty() From 07622e40b9c8e32357f4471f3d891a127f1fd05c Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Thu, 14 Aug 2025 22:44:26 +0900 Subject: [PATCH 2/2] fix lint Signed-off-by: SungJin1212 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b9a14e7961..ec1979ef6f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718 * [CHANGE] Validate a tenantID when to use a single tenant resolver. #6727 -* [FEATURE] Distributor: Add an experimental `-distributor.otlp.enable-type-and-unit-labels` flag to add `__type__` and `__unit__` labels for OTLP metrics. #6969 +* [FEATURE] Distributor: Add an experimental `-distributor.otlp.enable-type-and-unit-labels` flag to add `__type__` and `__unit__` labels for OTLP metrics. #6969 * [FEATURE] Distributor: Add an experimental `-distributor.otlp.allow-delta-temporality` flag to ingest delta temporality otlp metrics. #6934 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526