add http headers to spans

docs/config-schema.json

@@ -677,6 +677,10 @@
           "$ref": "#/$defs/ElasticsearchConfig",
           "description": "Elasticsearch payload extraction and parsing"
         },
+        "generic": {
+          "$ref": "#/$defs/HTTPGenericParsingConfig",
+          "description": "Generic HTTP header and payload extraction with policy-based rules"
+        },
         "graphql": {
           "$ref": "#/$defs/GraphQLConfig",
           "description": "GraphQL payload extraction and parsing"
@@ -692,6 +696,102 @@
       },
       "type": "object"
     },
+    "HTTPGenericParsingConfig": {
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "description": "Enable generic HTTP header and payload extraction",
+          "x-env-var": "OTEL_EBPF_HTTP_GENERIC_PARSING_ENABLED"
+        },
+        "policy": {
+          "$ref": "#/$defs/HTTPParsingPolicy",
+          "description": "Policy controls the default behavior and matching strategy"
+        },
+        "rules": {
+          "items": {
+            "$ref": "#/$defs/HTTPParsingRule"
+          },
+          "type": "array",
+          "description": "Rules is an ordered list of include/exclude/obfuscate rules. Rules are evaluated according to Policy.MatchOrder."
+        }
+      },
+      "type": "object",
+      "description": "HTTPGenericParsingConfig configures generic HTTP header and payload extraction."
+    },
+    "HTTPParsingMatch": {
+      "properties": {
+        "case_sensitive": {
+          "type": "boolean",
+          "description": "CaseSensitive controls whether matching is case-sensitive."
+        }
+      },
+      "type": "object",
+      "description": "HTTPParsingMatch defines matching criteria for an HTTP parsing rule. Regex patterns are compiled during YAML unmarshaling. When CaseSensitive is false (the default), patterns are automatically wrapped with (?i)."
+    },
+    "HTTPParsingPolicy": {
+      "properties": {
+        "default_action": {
+          "type": "string",
+          "enum": [
+            "exclude",
+            "include",
+            "obfuscate"
+          ],
+          "description": "DefaultAction specifies what to do when no rule matches: \"include\" or \"exclude\"",
+          "x-env-var": "OTEL_EBPF_HTTP_PARSING_DEFAULT_ACTION"
+        },
+        "match_order": {
+          "type": "string",
+          "enum": [
+            "first_match_wins"
+          ],
+          "description": "MatchOrder controls how rules are evaluated: \"first_match_wins\"",
+          "x-env-var": "OTEL_EBPF_HTTP_PARSING_MATCH_ORDER"
+        },
+        "obfuscation_string": {
+          "type": "string",
+          "description": "ObfuscationString is the replacement string used when a rule's action is \"obfuscate\"",
+          "x-env-var": "OTEL_EBPF_HTTP_PARSING_OBFUSCATION_STRING"
+        }
+      },
+      "type": "object",
+      "description": "HTTPParsingPolicy defines the default action and match strategy for generic parsing rules."
+    },
+    "HTTPParsingRule": {
+      "properties": {
+        "action": {
+          "type": "string",
+          "enum": [
+            "exclude",
+            "include",
+            "obfuscate"
+          ],
+          "description": "Action of the rule: \"include\", \"exclude\", or \"obfuscate\""
+        },
+        "match": {
+          "$ref": "#/$defs/HTTPParsingMatch",
+          "description": "Match defines the matching criteria for this rule"
+        },
+        "scope": {
+          "type": "string",
+          "enum": [
+            "both",
+            "request",
+            "response"
+          ],
+          "description": "Scope of the rule: \"request\", \"response\", or \"both\""
+        },
+        "type": {
+          "type": "string",
+          "enum": [
+            "headers"
+          ],
+          "description": "Type specifies what this rule matches against: \"headers\""
+        }
+      },
+      "type": "object",
+      "description": "HTTPParsingRule defines a single include/exclude/obfuscate rule for HTTP header and payload extraction."
+    },
     "HostIDConfig": {
       "properties": {
         "override": {

pkg/appolly/app/request/span.go

@@ -359,6 +359,12 @@ type Span struct {
 	AWS               *AWS           `json:"-"`
 	OpenAI            *OpenAI        `json:"-"`
 
+	// RequestHeaders stores extracted HTTP request headers based on generic parsing rules.
+	// Keys are canonical header names, values are the (possibly obfuscated) header values.
+	RequestHeaders map[string]string `json:"-"`
+	// ResponseHeaders stores extracted HTTP response headers based on generic parsing rules.
+	ResponseHeaders map[string]string `json:"-"`
+
 	// OverrideTraceName is set under some conditions, like spanmetrics reaching the maximum
 	// cardinality for trace names.
 	OverrideTraceName string `json:"-"`

pkg/config/payload_extraction.go

@@ -3,12 +3,18 @@
 
 package config // import "go.opentelemetry.io/obi/pkg/config"
 
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
 type PayloadExtraction struct {
 	HTTP HTTPConfig `yaml:"http"`
 }
 
 func (p PayloadExtraction) Enabled() bool {
-	return p.HTTP.GraphQL.Enabled || p.HTTP.Elasticsearch.Enabled || p.HTTP.AWS.Enabled || p.HTTP.SQLPP.Enabled || p.HTTP.OpenAI.Enabled
+	return p.HTTP.GraphQL.Enabled || p.HTTP.Elasticsearch.Enabled || p.HTTP.AWS.Enabled || p.HTTP.SQLPP.Enabled || p.HTTP.OpenAI.Enabled || p.HTTP.GenericParsing.Enabled
 }
 
 type HTTPConfig struct {
@@ -22,6 +28,8 @@
 	SQLPP SQLPPConfig `yaml:"sqlpp"`
 	// OpenAI payload extraction
 	OpenAI OpenAIConfig `yaml:"openai"`
+	// Generic HTTP header and payload extraction with policy-based rules
+	GenericParsing HTTPGenericParsingConfig `yaml:"generic"`
 }
 
 type GraphQLConfig struct {
@@ -51,3 +59,148 @@
 	// Enable OpenAI payload extraction and parsing
 	Enabled bool `yaml:"enabled" env:"OTEL_EBPF_HTTP_OPENAI_ENABLED" validate:"boolean"`
 }
+
+// HTTPGenericParsingConfig configures generic HTTP header and payload extraction.
+type HTTPGenericParsingConfig struct {
+	// Enable generic HTTP header and payload extraction
+	Enabled bool `yaml:"enabled" env:"OTEL_EBPF_HTTP_GENERIC_PARSING_ENABLED" validate:"boolean"`
+	// Policy controls the default behavior and matching strategy
+	Policy HTTPParsingPolicy `yaml:"policy"`
+	// Rules is an ordered list of include/exclude/obfuscate rules.
+	// Rules are evaluated according to Policy.MatchOrder.
+	Rules []HTTPParsingRule `yaml:"rules"`
+}
+
+// HTTPParsingPolicy defines the default action and match strategy for generic parsing rules.
+type HTTPParsingPolicy struct {
+	// DefaultAction specifies what to do when no rule matches: "include" or "exclude"
+	DefaultAction HTTPParsingAction `yaml:"default_action" env:"OTEL_EBPF_HTTP_PARSING_DEFAULT_ACTION"`
+	// MatchOrder controls how rules are evaluated: "first_match_wins"
+	MatchOrder HTTPParsingMatchOrder `yaml:"match_order" env:"OTEL_EBPF_HTTP_PARSING_MATCH_ORDER"`
+	// ObfuscationString is the replacement string used when a rule's action is "obfuscate"
+	ObfuscationString string `yaml:"obfuscation_string" env:"OTEL_EBPF_HTTP_PARSING_OBFUSCATION_STRING"`
+}
+
+// HTTPParsingRule defines a single include/exclude/obfuscate rule for HTTP header and payload extraction.
+type HTTPParsingRule struct {
+	// Action of the rule: "include", "exclude", or "obfuscate"
+	Action HTTPParsingAction `yaml:"action"`
+	// Type specifies what this rule matches against: "headers"
+	Type HTTPParsingRuleType `yaml:"type"`
+	// Scope of the rule: "request", "response", or "both"
+	Scope HTTPParsingScope `yaml:"scope"`
+	// Match defines the matching criteria for this rule
+	Match HTTPParsingMatch `yaml:"match"`
+}
+
+// HTTPParsingRuleType specifies the target of a parsing rule.
+type HTTPParsingRuleType string
+
+const (
+	HTTPParsingRuleTypeHeaders HTTPParsingRuleType = "headers"
+)
+
+func (t *HTTPParsingRuleType) UnmarshalText(text []byte) error {
+	str := HTTPParsingRuleType(strings.TrimSpace(strings.ToLower(string(text))))
+	switch str {
+	case HTTPParsingRuleTypeHeaders:
+		*t = str
+		return nil
+	default:
+		return fmt.Errorf("invalid parsing rule type: %q (valid: headers)", string(text))
+	}
+}
+
+// HTTPParsingMatch defines matching criteria for an HTTP parsing rule.
+// Regex patterns are compiled during YAML unmarshaling. When CaseSensitive
+// is false (the default), patterns are automatically wrapped with (?i).
+type HTTPParsingMatch struct {
+	// Regex is a list of compiled regular expressions to match against.
+	Regex []*regexp.Regexp `yaml:"-"`
+	// CaseSensitive controls whether matching is case-sensitive.
+	CaseSensitive bool `yaml:"case_sensitive"`
+}
+
+// UnmarshalYAML deserializes the match config and compiles regex patterns.
+func (m *HTTPParsingMatch) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Use a raw struct to capture the string patterns before compiling.
+	var raw struct {
+		Regex         []string `yaml:"regex"`
+		CaseSensitive bool     `yaml:"case_sensitive"`
+	}
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	m.CaseSensitive = raw.CaseSensitive
+	m.Regex = make([]*regexp.Regexp, 0, len(raw.Regex))
+	for _, pattern := range raw.Regex {
+		if !m.CaseSensitive {
+			pattern = "(?i)" + pattern
+		}
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return fmt.Errorf("invalid regex %q in parsing match: %w", pattern, err)
+		}
+		m.Regex = append(m.Regex, re)
+	}
+	return nil
+}
+
+// HTTPParsingAction represents the action for a generic parsing rule or default policy.
+type HTTPParsingAction string
+
+const (
+	HTTPParsingActionInclude   HTTPParsingAction = "include"
+	HTTPParsingActionExclude   HTTPParsingAction = "exclude"
+	HTTPParsingActionObfuscate HTTPParsingAction = "obfuscate"
+)
+
+func (a *HTTPParsingAction) UnmarshalText(text []byte) error {
+	str := HTTPParsingAction(strings.TrimSpace(strings.ToLower(string(text))))
+	switch str {
+	case HTTPParsingActionInclude, HTTPParsingActionExclude, HTTPParsingActionObfuscate:
+		*a = str
+		return nil
+	default:
+		return fmt.Errorf("invalid parsing action: %q (valid: include, exclude, obfuscate)", string(text))
+	}
+}
+
+// HTTPParsingAction represents the action for a http parsing rule or default policy.
+type HTTPParsingScope string
+
+const (
+	HTTPParsingScopeRequest  HTTPParsingScope = "request"
+	HTTPParsingScopeResponse HTTPParsingScope = "response"
+	HTTPParsingScopeBoth     HTTPParsingScope = "both"
+)
+
+func (a *HTTPParsingScope) UnmarshalText(text []byte) error {
+	str := HTTPParsingScope(strings.TrimSpace(strings.ToLower(string(text))))
+	switch str {
+	case HTTPParsingScopeRequest, HTTPParsingScopeResponse, HTTPParsingScopeBoth:
+		*a = str
+		return nil
+	default:
+		return fmt.Errorf("invalid parsing scope: %q (valid: include, exclude, obfuscate)", string(text))
+	}
+}
+
+// HTTPParsingMatchOrder controls how rules are evaluated.
+type HTTPParsingMatchOrder string
+
+const (
+	HTTPParsingMatchOrderFirstMatchWins HTTPParsingMatchOrder = "first_match_wins"
+)
+
+func (m *HTTPParsingMatchOrder) UnmarshalText(text []byte) error {
+	str := HTTPParsingMatchOrder(strings.TrimSpace(strings.ToLower(string(text))))
+	switch str {
+	case HTTPParsingMatchOrderFirstMatchWins:
+		*m = str
+		return nil
+	default:
+		return fmt.Errorf("invalid parsing match order: %q (valid: first_match_wins)", string(text))
+	}
+}