fix: gateway trace headers 5248 (#5256)

Co-authored-by: Copilot <[email protected]>

Author: kevwan

gateway/internal/headerprocessor.go

@@ -11,16 +11,40 @@ const (
 	metadataPrefix       = "gateway-"
 )
 
+// OpenTelemetry trace propagation headers that need to be forwarded to gRPC metadata.
+// These headers are used by the W3C Trace Context standard for distributed tracing.
+var traceHeaders = map[string]bool{
+	"traceparent": true,
+	"tracestate":  true,
+	"baggage":     true,
+}
+
 // ProcessHeaders builds the headers for the gateway from HTTP headers.
+// It forwards both custom metadata headers (with Grpc-Metadata- prefix)
+// and OpenTelemetry trace propagation headers (traceparent, tracestate, baggage)
+// to ensure distributed tracing works correctly across the gateway.
 func ProcessHeaders(header http.Header) []string {
 	var headers []string
 
 	for k, v := range header {
+		// Forward OpenTelemetry trace propagation headers
+		// These must be lowercase per gRPC metadata conventions
+		if lowerKey := strings.ToLower(k); traceHeaders[lowerKey] {
+			for _, vv := range v {
+				headers = append(headers, lowerKey+":"+vv)
+			}
+			continue
+		}
+
+		// Forward custom metadata headers with Grpc-Metadata- prefix
 		if !strings.HasPrefix(k, metadataHeaderPrefix) {
 			continue
 		}
 
-		key := fmt.Sprintf("%s%s", metadataPrefix, strings.TrimPrefix(k, metadataHeaderPrefix))
+		// gRPC metadata keys are case-insensitive and stored as lowercase,
+		// so we lowercase the key to match gRPC conventions
+		trimmedKey := strings.TrimPrefix(k, metadataHeaderPrefix)
+		key := strings.ToLower(fmt.Sprintf("%s%s", metadataPrefix, trimmedKey))
 		for _, vv := range v {
 			headers = append(headers, key+":"+vv)
 		}

gateway/internal/headerprocessor_test.go

@@ -18,5 +18,93 @@ func TestBuildHeadersWithValues(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", http.NoBody)
 	req.Header.Add("grpc-metadata-a", "b")
 	req.Header.Add("grpc-metadata-b", "b")
-	assert.ElementsMatch(t, []string{"gateway-A:b", "gateway-B:b"}, ProcessHeaders(req.Header))
+	assert.ElementsMatch(t, []string{"gateway-a:b", "gateway-b:b"}, ProcessHeaders(req.Header))
+}
+
+func TestProcessHeadersWithTraceContext(t *testing.T) {
+	req := httptest.NewRequest("GET", "/", http.NoBody)
+	req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
+	req.Header.Set("tracestate", "key1=value1,key2=value2")
+	req.Header.Set("baggage", "userId=alice,serverNode=DF:28")
+
+	headers := ProcessHeaders(req.Header)
+
+	assert.Len(t, headers, 3)
+	assert.Contains(t, headers, "traceparent:00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
+	assert.Contains(t, headers, "tracestate:key1=value1,key2=value2")
+	assert.Contains(t, headers, "baggage:userId=alice,serverNode=DF:28")
+}
+
+func TestProcessHeadersWithMixedHeaders(t *testing.T) {
+	req := httptest.NewRequest("GET", "/", http.NoBody)
+	req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
+	req.Header.Set("grpc-metadata-custom", "value1")
+	req.Header.Set("content-type", "application/json")
+	req.Header.Set("tracestate", "key1=value1")
+
+	headers := ProcessHeaders(req.Header)
+
+	// Should include trace headers and grpc-metadata headers, but not regular headers
+	assert.Len(t, headers, 3)
+	assert.Contains(t, headers, "traceparent:00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
+	assert.Contains(t, headers, "tracestate:key1=value1")
+	assert.Contains(t, headers, "gateway-custom:value1")
+}
+
+func TestProcessHeadersTraceparentCaseInsensitive(t *testing.T) {
+	tests := []struct {
+		name        string
+		headerKey   string
+		headerVal   string
+		expectedKey string
+	}{
+		{
+			name:        "lowercase traceparent",
+			headerKey:   "traceparent",
+			headerVal:   "00-trace-span-01",
+			expectedKey: "traceparent",
+		},
+		{
+			name:        "uppercase Traceparent",
+			headerKey:   "Traceparent",
+			headerVal:   "00-trace-span-01",
+			expectedKey: "traceparent",
+		},
+		{
+			name:        "mixed case TraceParent",
+			headerKey:   "TraceParent",
+			headerVal:   "00-trace-span-01",
+			expectedKey: "traceparent",
+		},
+		{
+			name:        "lowercase tracestate",
+			headerKey:   "tracestate",
+			headerVal:   "key=value",
+			expectedKey: "tracestate",
+		},
+		{
+			name:        "mixed case TraceState",
+			headerKey:   "TraceState",
+			headerVal:   "key=value",
+			expectedKey: "tracestate",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := httptest.NewRequest("GET", "/", http.NoBody)
+			req.Header.Set(tt.headerKey, tt.headerVal)
+
+			headers := ProcessHeaders(req.Header)
+
+			assert.Len(t, headers, 1)
+			assert.Contains(t, headers, tt.expectedKey+":"+tt.headerVal)
+		})
+	}
+}
+
+func TestProcessHeadersEmptyHeaders(t *testing.T) {
+	req := httptest.NewRequest("GET", "/", http.NoBody)
+	headers := ProcessHeaders(req.Header)
+	assert.Empty(t, headers)
 }