feat(helm v5): Make k8sattributes processor as default and deprecate k8s_tagger for k8s metadata enrichment (#4093)

Author: jagan2221

.changelog/4093.changed.txt

@@ -0,0 +1 @@
+feat(k8sattributesprocessor): Make k8sattributes processor as default and deprecate k8s_tagger for k8s metadata enrichment

deploy/helm/sumologic/README.md

@@ -115,6 +115,7 @@ processors:
     timeout: 10s
 
   ## Tags spans with K8S metadata, basing on the context IP
+{{ if .Values.otelcolInstrumentation.useSumoK8sProcessor }}
   k8s_tagger:
     ## When true, only IP is assigned and passed (so it could be tagged on another collector)
     passthrough: false
@@ -145,6 +146,57 @@ processors:
       labels:
         - tag_name: "k8s.pod.label.%s"
           key: "*"
+{{ else }}
+  k8sattributes:
+    wait_for_metadata: {{ .Values.otelcolInstrumentation.waitForMetadata }}
+    wait_for_metadata_timeout: {{ .Values.otelcolInstrumentation.waitForMetadataTimeout | quote }}
+    auth_type: "serviceAccount"
+    extract:
+      labels:
+        - key_regex: "(.*)"
+          tag_name: "k8s.pod.label.$$1"
+          from: pod
+        - key_regex: "(.*)"
+          tag_name: "k8s.namespace.label.$$1"
+          from: namespace
+      annotations:
+        - key_regex: "(.*)"
+          tag_name: "k8s.pod.annotation.$$1"
+          from: pod
+      metadata:
+        ## extract the following well-known metadata fields
+        - k8s.pod.name
+        - k8s.pod.uid
+        - k8s.deployment.name
+        - k8s.daemonset.name
+        - k8s.replicaset.name
+        - k8s.statefulset.name
+        - k8s.namespace.name
+        - k8s.node.name
+        - k8s.container.name
+        - service.name
+        - service.namespace
+    ## Has to be false to enrich metadata
+    passthrough: false
+    pod_association:
+    ## Mirrors the k8s_tagger default association order (pod_association.go)
+    ## https://github.com/SumoLogic/sumologic-otel-collector/blob/main/pkg/processor/k8sprocessor/pod_association.go#L44
+    ## k8sIPLabelName (k8s.pod.ip) → clientIPLabelName (ip) → connection → host.name (if IP)
+    ## We prepend k8s.pod.uid as the most explicit match, then follow k8s_tagger's order.
+    ## k8s.pod.ip and ip survive gateway hops (forwarded as resource attributes);
+    ## connection uses the TCP source IP and only works for direct pod→collector connections.
+      - sources:
+        - from: resource_attribute
+          name: k8s.pod.uid
+      - sources:
+        - from: resource_attribute
+          name: k8s.pod.ip
+      - sources:
+        - from: resource_attribute
+          name: ip
+      - sources:
+        - from: connection
+{{ end }}
 
   ## The memory_limiter processor is used to prevent out of memory situations on the collector.
   memory_limiter:
@@ -179,7 +231,7 @@ service:
   pipelines:
     traces:
       receivers: [jaeger, otlp, otlp/deprecated, zipkin]
-      processors: [memory_limiter, k8s_tagger, source, resource, batch]
+      processors: [memory_limiter, {{ if .Values.otelcolInstrumentation.useSumoK8sProcessor }}k8s_tagger{{ else }}k8sattributes{{ end }}, source, resource, batch]
       exporters:
 {{- if eq .Values.tracesGateway.enabled true }}
         - otlphttp/traces
@@ -191,7 +243,7 @@ service:
 {{- end }}
     metrics:
       receivers: [otlp, otlp/deprecated]
-      processors: [memory_limiter, k8s_tagger, source, resource, batch]
+      processors: [memory_limiter, {{ if .Values.otelcolInstrumentation.useSumoK8sProcessor }}k8s_tagger{{ else }}k8sattributes{{ end }}, source, resource, batch]
       exporters:
         - sumologic/metrics
 {{- if eq .Values.debug.instrumentation.otelcolInstrumentation.print true }}

deploy/helm/sumologic/conf/instrumentation/otelcol.instrumentation.conf.yaml

@@ -358,9 +358,21 @@ processors:
     ## Has to be false to enrich metadata
     passthrough: false
     pod_association:
+    ## Mirrors k8s_tagger's `build_hostname` association (pod_association.go):
+    ## build_hostname combined k8s.pod.name + k8s.namespace.name into a single
+    ## compound key "{podName}.{namespace}" for pod cache lookup.
+    ## k8sattributes two-source match achieves the same uniqueness guarantee:
+    ## both sources must match simultaneously, equivalent to the compound key.
+    ## k8s.pod.uid is prepended as a more precise fallback when set (e.g. via
+    ## the filelog receiver which parses the UID from the log file path).
       - sources:
         - from: resource_attribute
           name: k8s.pod.uid
+      - sources:
+        - from: resource_attribute
+          name: k8s.namespace.name
+        - from: resource_attribute
+          name: k8s.pod.name
 {{ end }}
 {{ end }}
 

deploy/helm/sumologic/conf/logs/otelcol/config.yaml

@@ -18,7 +18,9 @@ filter/drop_unnecessary_metrics:
       # drop histograms we've extracted sums and counts from, but don't want the full thing
       - (not IsMatch(name, {{ .Values.sumologic.metrics.allowHistogramRegex | quote }})) and (type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket"))
 {{- end }}
-
+      # Exclude k8sattributes pod related metrics which could cause unwanted noise
+      # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor/documentation.md#internal-telemetry
+      - IsMatch(name, "^otelcol[._](k8s[._]pod[._]association|otelsvc_k8s_(pod_added|pod_updated|pod_table_size|pod_deleted|ip_lookup_miss))$")
 # Prometheus receiver puts all labels in record-level attributes, and we need them in resource
 groupbyattrs:
   keys:
@@ -59,13 +61,19 @@ k8sattributes:
   wait_for_metadata_timeout: {{ .Values.metadata.metrics.waitForMetadataTimeout | quote }}
   auth_type: "serviceAccount"
   extract:
+    {{- if or .Values.metadata.metrics.extractPodLabels .Values.metadata.metrics.extractNodeLabels }}
     labels:
+      {{- if .Values.metadata.metrics.extractPodLabels }}
       - key_regex: "(.*)"
         tag_name: "pod_labels_$$1"
         from: pod
-      - key_regex: "(.*)" 
+      {{- end }}
+      {{- if .Values.metadata.metrics.extractNodeLabels }}
+      - key_regex: "(.*)"
         tag_name: "node_labels_$$1"
         from: node
+      {{- end }}
+    {{- end }}
     metadata:
       ## extract the following well-known metadata fields
       - k8s.pod.name
@@ -80,6 +88,13 @@ k8sattributes:
   ## Has to be false to enrich metadata
   passthrough: false
   pod_association:
+  ## Mirrors k8s_tagger's `build_hostname` association (pod_association.go):
+  ## build_hostname combined k8s.pod.name + k8s.namespace.name into a single
+  ## compound key "{podName}.{namespace}" for pod cache lookup.
+  ## k8sattributes two-source match achieves the same uniqueness guarantee:
+  ## both sources must match simultaneously, equivalent to the compound key.
+  ## Metrics from OtelOperator scraping always carry pod name + namespace
+  ## (set by the prometheus receiver from target labels), so no uid fallback needed.
     - sources:
       - from: resource_attribute
         name: k8s.pod.name

deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml

@@ -1420,6 +1420,14 @@ otelcolInstrumentation:
     ## Matching pods will be excluded from Sumo. The logs will still be sent to otelcol.
     excludePodRegex: ""
 
+  ## Use the sumologic k8s processor to add metadata to traces.
+  ## This is disabled by default, but can be enabled if you want to use the sumologic k8s_tagger processor.
+  useSumoK8sProcessor: false
+
+  ## Only used if useSumoK8sProcessor is set to false
+  waitForMetadata: true
+  waitForMetadataTimeout: 10s
+
   ## Option to turn autoscaling on for otelcol and specify params for HPA.
   ## Autoscaling needs metrics-server to access cpu metrics.
   autoscaling:
@@ -1655,13 +1663,27 @@ metadata:
     metricsLevel: normal
 
     ## Use the sumologic k8s processor to add metadata to the logs.
-    ## This is enabled by default, but can be disabled if you want to use the upstream processor.
-    useSumoK8sProcessor: true
+    ## This is disabled by default, but can be enabled if you want to use the sumologic k8s_tagger processor.
+    useSumoK8sProcessor: false
 
     ## Only used if useSumoK8sProcessor is set to false
     waitForMetadata: true
     waitForMetadataTimeout: 10s
 
+    ## Extract pod labels as resource attributes
+    ## Pod labels include: app, version, tier, component, etc.
+    ## Default: true
+    ## Only used if useSumoK8sProcessor is set to false
+    extractPodLabels: true
+
+    ## Extract node labels as resource attributes
+    ## WARNING: Node labels significantly increase metric cardinality
+    ## Cloud providers add 50-100 labels per node (instance-type, zone, etc.)
+    ## Example impact: 100 nodes × 50 labels = 5,000+ additional label combinations
+    ## Default: false
+    ## Only used if useSumoK8sProcessor is set to false
+    extractNodeLabels: false
+
     config:
       ## Directly alter the OT configuration. The value of this key should be a dictionary, that will
       ## be directly merged with the generated configuration, overriding existing values.
@@ -1772,9 +1794,9 @@ metadata:
     ## Metrics Level (basic, normal, detailed)
     metricsLevel: normal
 
-    ## Use the sumologic k8s processor to add metadata to the logs.
-    ## This is enabled by default, but can be disabled if you want to use the upstream processor.
-    useSumoK8sProcessor: true
+    ## Use the k8sattributes processor to add metadata to the logs.
+    ## This is disabled by default, but can be disabled if you want to use the sumologic k8s_tagger processor.
+    useSumoK8sProcessor: false
 
     ## Only used if useSumoK8sProcessor is set to false
     waitForMetadata: true

deploy/helm/sumologic/values.yaml

@@ -124,7 +124,7 @@ func TestMetadataMetricsOtelConfigExtraProcessors(t *testing.T) {
 		"metricstransform",
 		"groupbyattrs",
 		"resource",
-		"k8s_tagger",
+		"k8sattributes",
 		"source",
 		"sumologic",
 		"filter/1",

tests/helm/metrics_test.go

@@ -155,34 +155,53 @@ data:
         - _sourceName
         - _sourceHost
         - _collector
-      k8s_tagger:
+      k8sattributes:
+        auth_type: serviceAccount
         extract:
           annotations:
-          - key: '*'
-            tag_name: pod_annotations_%s
-          delimiter: _
+          - from: pod
+            key_regex: (.*)
+            tag_name: pod_annotations_$$1
+          - from: namespace
+            key_regex: (.*)
+            tag_name: namespace_annotations_$$1
+          - from: node
+            key_regex: (.*)
+            tag_name: node_annotations_$$1
           labels:
-          - key: '*'
-            tag_name: pod_labels_%s
+          - from: pod
+            key_regex: (.*)
+            tag_name: pod_labels_$$1
+          - from: namespace
+            key_regex: (.*)
+            tag_name: namespace_labels_$$1
+          - from: node
+            key_regex: (.*)
+            tag_name: node_labels_$$1
           metadata:
-          - daemonSetName
-          - deploymentName
-          - hostName
-          - namespace
-          - nodeName
-          - podName
-          - serviceName
-          - statefulSetName
-          namespace_annotations:
-          - key: '*'
-            tag_name: namespace_annotations_%s
-          namespace_labels:
-          - key: '*'
-            tag_name: namespace_labels_%s
-        owner_lookup_enabled: true
+          - k8s.pod.name
+          - k8s.pod.uid
+          - k8s.pod.hostname
+          - k8s.deployment.name
+          - k8s.daemonset.name
+          - k8s.replicaset.name
+          - k8s.statefulset.name
+          - k8s.namespace.name
+          - k8s.node.name
+          - service.name
+          - service.namespace
         passthrough: false
         pod_association:
-        - from: build_hostname
+        - sources:
+          - from: resource_attribute
+            name: k8s.pod.uid
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+          - from: resource_attribute
+            name: k8s.pod.name
+        wait_for_metadata: true
+        wait_for_metadata_timeout: 10s
       memory_limiter:
         check_interval: 5s
         limit_percentage: 90
@@ -298,7 +317,7 @@ data:
           - memory_limiter
           - filter/include_containers
           - groupbyattrs/containers
-          - k8s_tagger
+          - k8sattributes
           - resource/add_cluster
           - resource/set_empty_source_metadata
           - resource/containers_copy_node_to_host

tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml

@@ -164,34 +164,53 @@ data:
         - _sourceName
         - _sourceHost
         - _collector
-      k8s_tagger:
+      k8sattributes:
+        auth_type: serviceAccount
         extract:
           annotations:
-          - key: '*'
-            tag_name: pod_annotations_%s
-          delimiter: _
+          - from: pod
+            key_regex: (.*)
+            tag_name: pod_annotations_$$1
+          - from: namespace
+            key_regex: (.*)
+            tag_name: namespace_annotations_$$1
+          - from: node
+            key_regex: (.*)
+            tag_name: node_annotations_$$1
           labels:
-          - key: '*'
-            tag_name: pod_labels_%s
+          - from: pod
+            key_regex: (.*)
+            tag_name: pod_labels_$$1
+          - from: namespace
+            key_regex: (.*)
+            tag_name: namespace_labels_$$1
+          - from: node
+            key_regex: (.*)
+            tag_name: node_labels_$$1
           metadata:
-          - daemonSetName
-          - deploymentName
-          - hostName
-          - namespace
-          - nodeName
-          - podName
-          - serviceName
-          - statefulSetName
-          namespace_annotations:
-          - key: '*'
-            tag_name: namespace_annotations_%s
-          namespace_labels:
-          - key: '*'
-            tag_name: namespace_labels_%s
-        owner_lookup_enabled: true
+          - k8s.pod.name
+          - k8s.pod.uid
+          - k8s.pod.hostname
+          - k8s.deployment.name
+          - k8s.daemonset.name
+          - k8s.replicaset.name
+          - k8s.statefulset.name
+          - k8s.namespace.name
+          - k8s.node.name
+          - service.name
+          - service.namespace
         passthrough: false
         pod_association:
-        - from: build_hostname
+        - sources:
+          - from: resource_attribute
+            name: k8s.pod.uid
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+          - from: resource_attribute
+            name: k8s.pod.name
+        wait_for_metadata: true
+        wait_for_metadata_timeout: 10s
       memory_limiter:
         check_interval: 5s
         limit_percentage: 90
@@ -308,7 +327,7 @@ data:
           - memory_limiter
           - filter/include_containers
           - groupbyattrs/containers
-          - k8s_tagger
+          - k8sattributes
           - resource/add_cluster
           - resource/set_empty_source_metadata
           - resource/containers_copy_node_to_host