enhancement(tag_cardinality_limit transform): Add metric and tag name to tag_value_limit_exceeded_total metric

changelog.d/tag_cardinality_limit_metric_labels.enhancement.md

@@ -0,0 +1,5 @@
+# Added `internal_metrics.include_key_in_limit_metric` configuration option
+
+Added `internal_metrics` configuration section to the `tag_cardinality_limit` transform to better organize internal metrics configuration. The `internal_metrics.include_key_in_limit_metric` option controls whether to include extended labels (`metric_name`, `tag_key`) in the `tag_value_limit_exceeded_total` metric to help identify which specific metrics and tag keys are hitting the configured value limit. This option defaults to `false` to avoid high cardinality issues, and should only be enabled when needed for debugging.
+
+authors: kaarolch

src/internal_events/tag_cardinality_limit.rs

@@ -7,6 +7,7 @@ pub struct TagCardinalityLimitRejectingEvent<'a> {
     pub metric_name: &'a str,
     pub tag_key: &'a str,
     pub tag_value: &'a str,
+    pub include_extended_tags_in_limit_metric: bool,
 }
 
 impl InternalEvent for TagCardinalityLimitRejectingEvent<'_> {
@@ -17,7 +18,16 @@ impl InternalEvent for TagCardinalityLimitRejectingEvent<'_> {
             tag_key = self.tag_key,
             tag_value = self.tag_value,
         );
-        counter!("tag_value_limit_exceeded_total").increment(1);
+        if self.include_extended_tags_in_limit_metric {
+            counter!(
+                "tag_value_limit_exceeded_total",
+                "metric_name" => self.metric_name.to_string(),
+                "tag_key" => self.tag_key.to_string(),
+            )
+            .increment(1);
+        } else {
+            counter!("tag_value_limit_exceeded_total").increment(1);
+        }
 
         emit!(ComponentEventsDropped::<INTENTIONAL> {
             count: 1,
@@ -31,6 +41,7 @@ pub struct TagCardinalityLimitRejectingTag<'a> {
     pub metric_name: &'a str,
     pub tag_key: &'a str,
     pub tag_value: &'a str,
+    pub include_extended_tags_in_limit_metric: bool,
 }
 
 impl InternalEvent for TagCardinalityLimitRejectingTag<'_> {
@@ -41,7 +52,16 @@ impl InternalEvent for TagCardinalityLimitRejectingTag<'_> {
             tag_key = self.tag_key,
             tag_value = self.tag_value,
         );
-        counter!("tag_value_limit_exceeded_total").increment(1);
+        if self.include_extended_tags_in_limit_metric {
+            counter!(
+                "tag_value_limit_exceeded_total",
+                "metric_name" => self.metric_name.to_string(),
+                "tag_key" => self.tag_key.to_string(),
+            )
+            .increment(1);
+        } else {
+            counter!("tag_value_limit_exceeded_total").increment(1);
+        }
     }
 }
 

src/transforms/tag_cardinality_limit/config.rs

@@ -11,6 +11,23 @@ use crate::{
     transforms::{Transform, tag_cardinality_limit::TagCardinalityLimit},
 };
 
+/// Configuration of internal metrics for the TagCardinalityLimit transform.
+#[configurable_component]
+#[derive(Clone, Debug, PartialEq, Eq, Default)]
+#[serde(deny_unknown_fields)]
+pub struct TagCardinalityLimitInternalMetricsConfig {
+    /// Whether to include extended labels (metric_name, tag_key) in the `tag_value_limit_exceeded_total` metric.
+    ///
+    /// This can be useful for debugging, but should be used with caution as it can significantly
+    /// increase metric cardinality if metric names or tag keys are high cardinality.
+    ///
+    /// Note that this defaults to false because the extended tags have potentially unbounded cardinality.
+    /// Only set this to true if you know that the number of unique metric names and tag keys is bounded.
+    #[serde(default = "default_include_key_in_limit_metric")]
+    #[configurable(metadata(docs::human_name = "Include Key in Limit Metric"))]
+    pub include_key_in_limit_metric: bool,
+}
+
 /// Configuration for the `tag_cardinality_limit` transform.
 #[configurable_component(transform(
     "tag_cardinality_limit",
@@ -44,6 +61,10 @@ pub struct TagCardinalityLimitInnerConfig {
 
     #[serde(flatten)]
     pub mode: Mode,
+
+    #[configurable(derived)]
+    #[serde(default)]
+    pub internal_metrics: TagCardinalityLimitInternalMetricsConfig,
 }
 
 /// Controls the approach taken for tracking tag cardinality.
@@ -115,6 +136,10 @@ const fn default_value_limit() -> usize {
     500
 }
 
+const fn default_include_key_in_limit_metric() -> bool {
+    false
+}
+
 pub(crate) const fn default_cache_size() -> usize {
     5 * 1024 // 5KB
 }
@@ -126,6 +151,7 @@ impl GenerateConfig for TagCardinalityLimitConfig {
                 mode: Mode::Exact,
                 value_limit: default_value_limit(),
                 limit_exceeded_action: default_limit_exceeded_action(),
+                internal_metrics: TagCardinalityLimitInternalMetricsConfig::default(),
             },
             per_metric_limits: HashMap::default(),
         })

src/transforms/tag_cardinality_limit/mod.rs

@@ -151,10 +151,14 @@ impl TagCardinalityLimit {
 
                     for (key, value) in tags_map.iter_sets() {
                         if self.tag_limit_exceeded(metric_key.as_ref(), key, value) {
+                            let config = self.get_config_for_metric(metric_key.as_ref());
                             emit!(TagCardinalityLimitRejectingEvent {
                                 metric_name: &metric_name,
                                 tag_key: key,
                                 tag_value: &value.to_string(),
+                                include_extended_tags_in_limit_metric: config
+                                    .internal_metrics
+                                    .include_key_in_limit_metric,
                             });
                             return None;
                         }
@@ -164,6 +168,8 @@ impl TagCardinalityLimit {
                     }
                 }
                 LimitExceededAction::DropTag => {
+                    let config = self.get_config_for_metric(metric_key.as_ref());
+                    let include_extended_tags = config.internal_metrics.include_key_in_limit_metric;
                     tags_map.retain(|key, value| {
                         if self.try_accept_tag(metric_key.as_ref(), key, value) {
                             true
@@ -172,6 +178,7 @@ impl TagCardinalityLimit {
                                 metric_name: &metric_name,
                                 tag_key: key,
                                 tag_value: &value.to_string(),
+                                include_extended_tags_in_limit_metric: include_extended_tags,
                             });
                             false
                         }

src/transforms/tag_cardinality_limit/tests.rs

@@ -16,7 +16,9 @@ use crate::{
     event::{Event, Metric, MetricTags, metric, metric::TagValue},
     test_util::components::assert_transform_compliance,
     transforms::{
-        tag_cardinality_limit::config::{BloomFilterConfig, Mode, default_cache_size},
+        tag_cardinality_limit::config::{
+            BloomFilterConfig, Mode, TagCardinalityLimitInternalMetricsConfig, default_cache_size,
+        },
         test::create_topology,
     },
 };
@@ -53,6 +55,7 @@ fn make_transform_hashset(
             value_limit,
             limit_exceeded_action,
             mode: Mode::Exact,
+            internal_metrics: TagCardinalityLimitInternalMetricsConfig::default(),
         },
         per_metric_limits: HashMap::new(),
     }
@@ -69,6 +72,7 @@ fn make_transform_bloom(
             mode: Mode::Probabilistic(BloomFilterConfig {
                 cache_size_per_key: default_cache_size(),
             }),
+            internal_metrics: TagCardinalityLimitInternalMetricsConfig::default(),
         },
         per_metric_limits: HashMap::new(),
     }
@@ -84,6 +88,9 @@ const fn make_transform_hashset_with_per_metric_limits(
             value_limit,
             limit_exceeded_action,
             mode: Mode::Exact,
+            internal_metrics: TagCardinalityLimitInternalMetricsConfig {
+                include_key_in_limit_metric: false,
+            },
         },
         per_metric_limits,
     }
@@ -101,6 +108,9 @@ const fn make_transform_bloom_with_per_metric_limits(
             mode: Mode::Probabilistic(BloomFilterConfig {
                 cache_size_per_key: default_cache_size(),
             }),
+            internal_metrics: TagCardinalityLimitInternalMetricsConfig {
+                include_key_in_limit_metric: false,
+            },
         },
         per_metric_limits,
     }

website/cue/reference/components/sources/internal_metrics.cue

@@ -838,11 +838,29 @@ components: sources: internal_metrics: {
 		tag_value_limit_exceeded_total: {
 			description: """
 				The total number of events discarded because the tag has been rejected after
-				hitting the configured `value_limit`.
+				hitting the configured `value_limit`. When `internal_metrics.include_key_in_limit_metric`
+				is enabled in the `tag_cardinality_limit` transform, this metric includes
+				`metric_name` and `tag_key` labels. By default, this metric has no labels to
+				keep cardinality low.
 				"""
 			type:              "counter"
 			default_namespace: "vector"
-			tags:              _component_tags
+			tags: _component_tags & {
+				metric_name: {
+					description: """
+						The name of the metric whose tag value limit was exceeded.
+						Only present when `internal_metrics.include_key_in_limit_metric` is enabled.
+						"""
+					required:    false
+				}
+				tag_key: {
+					description: """
+						The key of the tag whose value limit was exceeded.
+						Only present when `internal_metrics.include_key_in_limit_metric` is enabled.
+						"""
+					required:    false
+				}
+			}
 		}
 		timestamp_parse_errors_total: {
 			description:       "The total number of errors encountered parsing [RFC 3339](\(urls.rfc_3339)) timestamps."

website/cue/reference/components/transforms/generated/tag_cardinality_limit.cue

@@ -12,6 +12,23 @@ generated: components: transforms: tag_cardinality_limit: configuration: {
 		required:      false
 		type: uint: default: 5120
 	}
+	internal_metrics: {
+		description: "Configuration of internal metrics for the TagCardinalityLimit transform."
+		required:    false
+		type: object: options: include_key_in_limit_metric: {
+			description: """
+				Whether to include extended labels (metric_name, tag_key) in the `tag_value_limit_exceeded_total` metric.
+
+				This can be useful for debugging, but should be used with caution as it can significantly
+				increase metric cardinality if metric names or tag keys are high cardinality.
+
+				Note that this defaults to false because the extended tags have potentially unbounded cardinality.
+				Only set this to true if you know that the number of unique metric names and tag keys is bounded.
+				"""
+			required: false
+			type: bool: default: false
+		}
+	}
 	limit_exceeded_action: {
 		description: """
 			Possible actions to take when an event arrives that would exceed the cardinality limit for one
@@ -64,6 +81,23 @@ generated: components: transforms: tag_cardinality_limit: configuration: {
 					required:      false
 					type: uint: default: 5120
 				}
+				internal_metrics: {
+					description: "Configuration of internal metrics for the TagCardinalityLimit transform."
+					required:    false
+					type: object: options: include_key_in_limit_metric: {
+						description: """
+																				Whether to include extended labels (metric_name, tag_key) in the `tag_value_limit_exceeded_total` metric.
+
+																				This can be useful for debugging, but should be used with caution as it can significantly
+																				increase metric cardinality if metric names or tag keys are high cardinality.
+
+																				Note that this defaults to false because the extended tags have potentially unbounded cardinality.
+																				Only set this to true if you know that the number of unique metric names and tag keys is bounded.
+																				"""
+						required: false
+						type: bool: default: false
+					}
+				}
 				limit_exceeded_action: {
 					description: """
 						Possible actions to take when an event arrives that would exceed the cardinality limit for one

website/cue/reference/components/transforms/tag_cardinality_limit.cue

@@ -187,7 +187,13 @@ components: transforms: tag_cardinality_limit: {
 	}
 
 	telemetry: metrics: {
-		tag_value_limit_exceeded_total: components.sources.internal_metrics.output.metrics.tag_value_limit_exceeded_total
+		tag_value_limit_exceeded_total: components.sources.internal_metrics.output.metrics.tag_value_limit_exceeded_total & {
+			description: """
+				The number of times a tag value limit was exceeded. When `internal_metrics.include_key_in_limit_metric` is enabled,
+				this metric includes `metric_name` and `tag_key` labels to help identify which specific metrics and tag keys
+				are hitting the configured value limit. By default, this metric has no labels to keep cardinality low.
+				"""
+		}
 		value_limit_reached_total:      components.sources.internal_metrics.output.metrics.value_limit_reached_total
 	}
 }