Add span_name_sanitization as user-configurable override (#6411)

* Add span_name_sanitization as user-configurable override

This adds support for configuring span_name_sanitization through the
user-configurable overrides API, allowing tenants to enable the DRAIN
span name sanitizer without requiring operator configuration changes.

Changes:
- Add SpanNameSanitization field to LimitsMetricsGenerator
- Add MetricsGeneratorSpanNameSanitization() to overrides manager
- Add validation for span_name_sanitization in overrides API
- Add comprehensive tests for the new configuration option
- Add integration test for the overrides API endpoint

Co-authored-by: Cursor <cursoragent@cursor.com>

* Add PR link

* Add documentation for span_name_sanitization override

Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: Logiraptor

CHANGELOG.md

@@ -15,6 +15,7 @@
 * [ENHANCEMENT] Improved live store readiness check and added `readiness_target_lag` and `readiness_max_wait` config parameters. Live store will now - if `readiness_target_lag` is set - not report `/ready` until Kafka lag is brought under the specified value [#6238](https://github.com/grafana/tempo/pull/6238) [#6405](https://github.com/grafana/tempo/pull/6405) (@oleg-kozlyuk-grafana, @ruslan-mikhailov)
 * [ENHANCEMENT] Expose a new histogram metric to track the jobs per query distribution [#6343](https://github.com/grafana/tempo/pull/6343) (@javiermolinar)
 * [ENHANCEMENT] Do deep validation for filter policies in user configurable overrides API [#6407](https://github.com/grafana/tempo/pull/6407) (@electron0zero)
+* [ENHANCEMENT] Allow span_name_sanitization to be set via user-configurable overrides API [#6411](https://github.com/grafana/tempo/pull/6411) (@Logiraptor)
 * [ENHANCEMENT] Add `fail_on_high_lag` parameter to allow live-store to fail if it is lagged [#6363](https://github.com/grafana/tempo/pull/6363) (@ruslan-mikhailov)
 * [ENHANCEMENT] Add new metric for generator ring size: `tempo_distributor_metrics_generator_tenant_ring_size` [#5686](https://github.com/grafana/tempo/pull/5686) (@zalegrala)
 * [BUGFIX] Fix query-frontend unable to convert dedicated column blob option [#6377](https://github.com/grafana/tempo/pull/6377) (@stoewer)

cmd/tempo/app/overrides_validation.go

@@ -36,6 +36,12 @@ func (r *runtimeConfigValidator) Validate(config *overrides.Overrides) (warnings
 		}
 	}
 
+	if config.MetricsGenerator.SpanNameSanitization != "" {
+		if err := validation.ValidateSpanNameSanitization(config.MetricsGenerator.SpanNameSanitization); err != nil {
+			return warnings, err
+		}
+	}
+
 	if config.MetricsGenerator.NativeHistogramBucketFactor != 0 {
 		if err := validation.ValidateNativeHistogramBucketFactor(config.MetricsGenerator.NativeHistogramBucketFactor); err != nil {
 			return warnings, err
@@ -170,6 +176,12 @@ func (v *overridesValidator) Validate(limits *client.Limits) error {
 		}
 	}
 
+	if spanNameSanitization, ok := limits.GetMetricsGenerator().GetSpanNameSanitization(); ok {
+		if err := validation.ValidateSpanNameSanitization(spanNameSanitization); err != nil {
+			return err
+		}
+	}
+
 	if metricName, ok := limits.GetMetricsGenerator().GetProcessor().GetHostInfo().GetMetricName(); ok {
 		if err := validation.ValidateHostInfoMetricName(metricName); err != nil {
 			return err

cmd/tempo/app/overrides_validation_test.go

@@ -246,6 +246,35 @@ func Test_runtimeOverridesValidator(t *testing.T) {
 				backend.WarnTooManyColumns{Type: "string", Scope: "resource", Count: 21, MaxCount: 20},
 			},
 		},
+		{
+			name: "metrics_generator.span_name_sanitization empty (disabled)",
+			cfg:  Config{},
+			overrides: overrides.Overrides{MetricsGenerator: overrides.MetricsGeneratorOverrides{
+				SpanNameSanitization: "",
+			}},
+		},
+		{
+			name: "metrics_generator.span_name_sanitization dry_run",
+			cfg:  Config{},
+			overrides: overrides.Overrides{MetricsGenerator: overrides.MetricsGeneratorOverrides{
+				SpanNameSanitization: "dry_run",
+			}},
+		},
+		{
+			name: "metrics_generator.span_name_sanitization enabled",
+			cfg:  Config{},
+			overrides: overrides.Overrides{MetricsGenerator: overrides.MetricsGeneratorOverrides{
+				SpanNameSanitization: "enabled",
+			}},
+		},
+		{
+			name: "metrics_generator.span_name_sanitization invalid",
+			cfg:  Config{},
+			overrides: overrides.Overrides{MetricsGenerator: overrides.MetricsGeneratorOverrides{
+				SpanNameSanitization: "invalid",
+			}},
+			expErr: "span_name_sanitization \"invalid\" is not valid, valid values: [ dry_run enabled]",
+		},
 	}
 
 	for _, tc := range testCases {
@@ -788,6 +817,43 @@ func Test_overridesValidator(t *testing.T) {
 			},
 			expErr: `dimension_mapping "combined" produces label "combined" which collides with dimension_mapping "combined"`,
 		},
+		{
+			name: "metrics_generator.span_name_sanitization empty (disabled)",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: strPtr(""),
+				},
+			},
+		},
+		{
+			name: "metrics_generator.span_name_sanitization dry_run",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: strPtr("dry_run"),
+				},
+			},
+		},
+		{
+			name: "metrics_generator.span_name_sanitization enabled",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: strPtr("enabled"),
+				},
+			},
+		},
+		{
+			name: "metrics_generator.span_name_sanitization invalid",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: strPtr("invalid"),
+				},
+			},
+			expErr: "span_name_sanitization \"invalid\" is not valid, valid values: [ dry_run enabled]",
+		},
 	}
 
 	for _, tc := range testCases {

docs/sources/tempo/configuration/_index.md

@@ -1988,6 +1988,14 @@ overrides:
       # receiver must be configured to ingest native histograms.
       [generate_native_histograms: <classic|native|both> | default = classic]
 
+      # Enables span name sanitization using DRAIN clustering to reduce cardinality.
+      # Similar span names are clustered together (e.g., "GET /users/123" becomes "GET /users/<*>").
+      # Options:
+      #   - "" (empty string): Disabled (default)
+      #   - "dry_run": Produces a demand metric for the sanitized cardinality without applying changes
+      #   - "enabled": Applies DRAIN clustering to span names
+      [span_name_sanitization: <string> | default = ""]
+
       # Distributor -> metrics-generator forwarder related overrides
       forwarder:
         # Spans are stored in a queue in the distributor before being sent to the metrics-generators.

docs/sources/tempo/operations/manage-advanced-systems/user-configurable-overrides.md

@@ -72,6 +72,7 @@ metrics_generator:
   [native_histogram_max_bucket_number: <int> | default = 100]
   [native_histogram_bucket_factor: <float> | default = 1.1]
   [native_histogram_min_reset_duration: <duration> | default = 15m]
+  [span_name_sanitization: <string>]
 
   processor:
 

integration/api/overrides_api_test.go

@@ -678,6 +678,111 @@ func TestOverridesAPI_DELETE(t *testing.T) {
 	})
 }
 
+func TestOverridesAPI_SpanNameSanitization(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay:  configOverrides,
+		DeploymentMode: util.DeploymentModeSingleBinary,
+		Backends:       util.BackendObjectStorageS3,
+	}, func(h *util.TempoHarness) {
+		t.Run("sets span_name_sanitization via POST and verifies GET response", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-span-name-sanitization-1")
+
+			// Create overrides with span_name_sanitization set to "enabled"
+			limits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: stringPtr("enabled"),
+				},
+			}
+			setEtag, err := apiClient.SetOverrides(limits, "0")
+			require.NoError(t, err)
+			require.NotEmpty(t, setEtag)
+
+			// Verify GET response contains the value
+			returnedLimits, etag, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+			require.Equal(t, setEtag, etag)
+			spanNameSanitization, ok := returnedLimits.GetMetricsGenerator().GetSpanNameSanitization()
+			require.True(t, ok)
+			require.Equal(t, "enabled", spanNameSanitization)
+		})
+
+		t.Run("sets span_name_sanitization via PATCH and verifies GET response", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-span-name-sanitization-2")
+
+			// Create initial config
+			initialLimits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+				},
+			}
+			_, err := apiClient.SetOverrides(initialLimits, "0")
+			require.NoError(t, err)
+
+			// PATCH with span_name_sanitization set to "dry_run"
+			patch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: stringPtr("dry_run"),
+				},
+			}
+			returnedLimits, _, err := apiClient.PatchOverrides(patch)
+			require.NoError(t, err)
+
+			// Verify PATCH response contains the value
+			spanNameSanitization, ok := returnedLimits.GetMetricsGenerator().GetSpanNameSanitization()
+			require.True(t, ok)
+			require.Equal(t, "dry_run", spanNameSanitization)
+
+			// Verify GET response also contains the value
+			getLimits, _, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+			spanNameSanitization, ok = getLimits.GetMetricsGenerator().GetSpanNameSanitization()
+			require.True(t, ok)
+			require.Equal(t, "dry_run", spanNameSanitization)
+		})
+
+		t.Run("returns 400 for invalid span_name_sanitization value via POST", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-span-name-sanitization-invalid-1")
+
+			// Try to set invalid span_name_sanitization value
+			limits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: stringPtr("invalid"),
+				},
+			}
+			_, err := apiClient.SetOverrides(limits, "0")
+			require.Error(t, err)
+			require.ErrorContains(t, err, "400")
+			require.ErrorContains(t, err, "span_name_sanitization")
+			require.ErrorContains(t, err, "not valid")
+		})
+
+		t.Run("returns 400 for invalid span_name_sanitization value via PATCH", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-span-name-sanitization-invalid-2")
+
+			// Create initial config
+			initialLimits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+				},
+			}
+			_, err := apiClient.SetOverrides(initialLimits, "0")
+			require.NoError(t, err)
+
+			// Try to PATCH with invalid span_name_sanitization value
+			patch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					SpanNameSanitization: stringPtr("invalid"),
+				},
+			}
+			_, _, err = apiClient.PatchOverrides(patch)
+			require.Error(t, err)
+			require.ErrorContains(t, err, "400")
+			require.ErrorContains(t, err, "span_name_sanitization")
+			require.ErrorContains(t, err, "not valid")
+		})
+	})
+}
+
 // Helper functions for overrides API tests
 
 func printLimits(limits *client.Limits, version string) {
@@ -710,3 +815,7 @@ func keys(m map[string]struct{}) []string {
 	}
 	return keys
 }
+
+func stringPtr(s string) *string {
+	return &s
+}

modules/overrides/user_configurable_overrides.go

@@ -261,6 +261,13 @@ func (o *userConfigurableOverridesManager) MetricsGeneratorTraceIDLabelName(user
 	return o.Interface.MetricsGeneratorTraceIDLabelName(userID)
 }
 
+func (o *userConfigurableOverridesManager) MetricsGeneratorSpanNameSanitization(userID string) string {
+	if spanNameSanitization, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetSpanNameSanitization(); ok {
+		return spanNameSanitization
+	}
+	return o.Interface.MetricsGeneratorSpanNameSanitization(userID)
+}
+
 func (o *userConfigurableOverridesManager) MetricsGeneratorGenerateNativeHistograms(userID string) histograms.HistogramMethod {
 	if method, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetGenerateNativeHistograms(); ok {
 		return method

modules/overrides/user_configurable_overrides_test.go

@@ -209,6 +209,49 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 	assert.Equal(t, "custom_key", mgr.MetricsGeneratorProcessorSpanMetricsSpanMultiplierKey(tenant1))
 }
 
+func TestUserConfigOverridesManager_MetricsGeneratorSpanNameSanitization(t *testing.T) {
+	defaultLimits := Overrides{
+		MetricsGenerator: MetricsGeneratorOverrides{
+			SpanNameSanitization: "disabled",
+		},
+	}
+	_, mgr, cleanup := localUserConfigOverrides(t, defaultLimits, nil)
+	defer cleanup()
+
+	// Test fallback behavior - tenant without override should return default value
+	assert.Equal(t, "disabled", mgr.MetricsGeneratorSpanNameSanitization(tenant1))
+	assert.Equal(t, "disabled", mgr.MetricsGeneratorSpanNameSanitization(tenant2))
+
+	// Set user-configurable override for tenant1
+	mgr.tenantLimits[tenant1] = &userconfigurableoverrides.Limits{
+		MetricsGenerator: userconfigurableoverrides.LimitsMetricsGenerator{
+			SpanNameSanitization: strPtr("enabled"),
+		},
+	}
+
+	// Test tenant override behavior - tenant1 should return override value
+	assert.Equal(t, "enabled", mgr.MetricsGeneratorSpanNameSanitization(tenant1))
+
+	// Test fallback behavior - tenant2 without override should still return default value
+	assert.Equal(t, "disabled", mgr.MetricsGeneratorSpanNameSanitization(tenant2))
+
+	// Update override for tenant1
+	mgr.tenantLimits[tenant1] = &userconfigurableoverrides.Limits{
+		MetricsGenerator: userconfigurableoverrides.LimitsMetricsGenerator{
+			SpanNameSanitization: strPtr("strict"),
+		},
+	}
+
+	// Test updated override value
+	assert.Equal(t, "strict", mgr.MetricsGeneratorSpanNameSanitization(tenant1))
+
+	// Remove override for tenant1
+	delete(mgr.tenantLimits, tenant1)
+
+	// Test fallback behavior after removal - should return default value again
+	assert.Equal(t, "disabled", mgr.MetricsGeneratorSpanNameSanitization(tenant1))
+}
+
 func TestUserConfigOverridesManager_populateFromBackend(t *testing.T) {
 	defaultLimits := Overrides{
 		Forwarders: []string{"my-forwarder"},

modules/overrides/userconfigurable/api/limits.go

@@ -24,6 +24,13 @@ func limitsFromOverrides(overrides overrides.Interface, userID string) *client.L
 			NativeHistogramMaxBucketNumber:  uint32Ptr(overrides.MetricsGeneratorNativeHistogramMaxBucketNumber(userID)),
 			NativeHistogramBucketFactor:     floatPtr(overrides.MetricsGeneratorNativeHistogramBucketFactor(userID)),
 			NativeHistogramMinResetDuration: timePtr(overrides.MetricsGeneratorNativeHistogramMinResetDuration(userID)),
+			SpanNameSanitization: func() *string {
+				s := overrides.MetricsGeneratorSpanNameSanitization(userID)
+				if s == "" {
+					return nil
+				}
+				return &s
+			}(),
 			Processor: client.LimitsMetricsGeneratorProcessor{
 				ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
 					Dimensions:               strArrPtr(overrides.MetricsGeneratorProcessorServiceGraphsDimensions(userID)),

modules/overrides/userconfigurable/api/limits_test.go

@@ -30,6 +30,7 @@ func Test_limitsFromOverrides(t *testing.T) {
 				NativeHistogramMaxBucketNumber:  160,
 				NativeHistogramBucketFactor:     1.2,
 				NativeHistogramMinResetDuration: 10 * time.Minute,
+				SpanNameSanitization:            "enabled",
 				Processor: overrides.ProcessorOverrides{
 					ServiceGraphs: overrides.ServiceGraphsOverrides{
 						HistogramBuckets:         []float64{0.1, 0.2, 0.5},
@@ -91,6 +92,7 @@ func Test_limitsFromOverrides(t *testing.T) {
     "native_histogram_max_bucket_number": 160,
     "native_histogram_bucket_factor": 1.2,
     "native_histogram_min_reset_duration": "10m0s",
+    "span_name_sanitization": "enabled",
     "processor": {
       "service_graphs": {
         "dimensions": [