[Exporterhelper Exporter] Enable metadata population in exporterhelper when sending_queue is enabled

.chloggen/otlphttp_mergge_metadata.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: exporter/otlphttp
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Populate metadata into the OTLP HTTP exporter context
+
+# One or more tracking issues or pull requests related to the change
+issues: [14139]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/otlphttpexporter/config.go

@@ -63,6 +63,9 @@ type Config struct {
 
 	// The encoding to export telemetry (default: "proto")
 	Encoding EncodingType `mapstructure:"encoding"`
+
+	// MetadataKeys is a list of metadata keys that will be used to partition the datainto batches if sending_queue is enabled
+	MetadataKeys []string `mapstructure:"metadata_keys"`
 }
 
 var _ component.Config = (*Config)(nil)

exporter/otlphttpexporter/factory.go

@@ -89,14 +89,21 @@ func createTraces(
 		return nil, err
 	}
 
+	qbs := xexporterhelper.NewTracesQueueBatchSettings()
+	if len(oCfg.MetadataKeys) > 0 {
+		partitioner := metadataKeysPartitioner{keys: oCfg.MetadataKeys}
+		qbs.Partitioner = partitioner
+		qbs.MergeCtx = partitioner.MergeCtx
+	}
+
 	return exporterhelper.NewTraces(ctx, set, cfg,
 		oce.pushTraces,
 		exporterhelper.WithStart(oce.start),
 		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}),
 		// explicitly disable since we rely on http.Client timeout logic.
 		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
 		exporterhelper.WithRetry(oCfg.RetryConfig),
-		exporterhelper.WithQueue(oCfg.QueueConfig))
+		xexporterhelper.WithQueueBatch(oCfg.QueueConfig, qbs))
 }
 
 func createMetrics(
@@ -115,14 +122,21 @@ func createMetrics(
 		return nil, err
 	}
 
+	qbs := xexporterhelper.NewMetricsQueueBatchSettings()
+	if len(oCfg.MetadataKeys) > 0 {
+		partitioner := metadataKeysPartitioner{keys: oCfg.MetadataKeys}
+		qbs.Partitioner = partitioner
+		qbs.MergeCtx = partitioner.MergeCtx
+	}
+
 	return exporterhelper.NewMetrics(ctx, set, cfg,
 		oce.pushMetrics,
 		exporterhelper.WithStart(oce.start),
 		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}),
 		// explicitly disable since we rely on http.Client timeout logic.
 		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
 		exporterhelper.WithRetry(oCfg.RetryConfig),
-		exporterhelper.WithQueue(oCfg.QueueConfig))
+		xexporterhelper.WithQueueBatch(oCfg.QueueConfig, qbs))
 }
 
 func createLogs(
@@ -140,14 +154,21 @@ func createLogs(
 		return nil, err
 	}
 
+	qbs := xexporterhelper.NewLogsQueueBatchSettings()
+	if len(oCfg.MetadataKeys) > 0 {
+		partitioner := metadataKeysPartitioner{keys: oCfg.MetadataKeys}
+		qbs.Partitioner = partitioner
+		qbs.MergeCtx = partitioner.MergeCtx
+	}
+
 	return exporterhelper.NewLogs(ctx, set, cfg,
 		oce.pushLogs,
 		exporterhelper.WithStart(oce.start),
 		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}),
 		// explicitly disable since we rely on http.Client timeout logic.
 		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
 		exporterhelper.WithRetry(oCfg.RetryConfig),
-		exporterhelper.WithQueue(oCfg.QueueConfig))
+		xexporterhelper.WithQueueBatch(oCfg.QueueConfig, qbs))
 }
 
 func createProfiles(

exporter/otlphttpexporter/factory_test.go

@@ -17,6 +17,7 @@ import (
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/configopaque"
 	"go.opentelemetry.io/collector/config/configtls"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
 	"go.opentelemetry.io/collector/exporter/exportertest"
 	"go.opentelemetry.io/collector/exporter/xexporter"
 	"go.opentelemetry.io/collector/internal/testutil"
@@ -49,6 +50,16 @@ func TestCreateMetrics(t *testing.T) {
 	oexp, err := factory.CreateMetrics(context.Background(), set, cfg)
 	require.NoError(t, err)
 	require.NotNil(t, oexp)
+
+	// Test with MetadataKeys and sending_queue enabled to cover partitioner creation
+	cfgWithKeys := factory.CreateDefaultConfig().(*Config)
+	cfgWithKeys.ClientConfig.Endpoint = "http://" + testutil.GetAvailableLocalAddress(t)
+	cfgWithKeys.MetadataKeys = []string{"key1", "key2"}
+	cfgWithKeys.QueueConfig.Enabled = true
+
+	oexpWithKeys, err := factory.CreateMetrics(context.Background(), set, cfgWithKeys)
+	require.NoError(t, err)
+	require.NotNil(t, oexpWithKeys)
 }
 
 func clientConfig(endpoint string, headers configopaque.MapList, tlsSetting configtls.ClientConfig, compression configcompression.Type) confighttp.ClientConfig {
@@ -159,6 +170,14 @@ func TestCreateTraces(t *testing.T) {
 				ClientConfig: clientConfig(endpoint, nil, configtls.ClientConfig{}, configCompression),
 			},
 		},
+		{
+			name: "WithMetadataKeys",
+			config: &Config{
+				ClientConfig: clientConfig(endpoint, nil, configtls.ClientConfig{}, configCompression),
+				MetadataKeys: []string{"key1", "key2"},
+				QueueConfig:  exporterhelper.NewDefaultQueueConfig(),
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -197,6 +216,16 @@ func TestCreateLogs(t *testing.T) {
 	oexp, err := factory.CreateLogs(context.Background(), set, cfg)
 	require.NoError(t, err)
 	require.NotNil(t, oexp)
+
+	// Test with MetadataKeys and sending_queue enabled to cover partitioner creation
+	cfgWithKeys := factory.CreateDefaultConfig().(*Config)
+	cfgWithKeys.ClientConfig.Endpoint = "http://" + testutil.GetAvailableLocalAddress(t)
+	cfgWithKeys.MetadataKeys = []string{"key1", "key2"}
+	cfgWithKeys.QueueConfig.Enabled = true
+
+	oexpWithKeys, err := factory.CreateLogs(context.Background(), set, cfgWithKeys)
+	require.NoError(t, err)
+	require.NotNil(t, oexpWithKeys)
 }
 
 func TestCreateProfiles(t *testing.T) {

exporter/otlphttpexporter/go.mod

@@ -5,6 +5,7 @@ go 1.24.0
 require (
 	github.com/stretchr/testify v1.11.1
 	go.opentelemetry.io/collector v0.139.0
+	go.opentelemetry.io/collector/client v1.45.0
 	go.opentelemetry.io/collector/component v1.45.0
 	go.opentelemetry.io/collector/component/componenttest v0.139.0
 	go.opentelemetry.io/collector/config/configcompression v1.45.0
@@ -58,7 +59,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/collector/client v1.45.0 // indirect
 	go.opentelemetry.io/collector/config/configauth v1.45.0 // indirect
 	go.opentelemetry.io/collector/config/configmiddleware v1.45.0 // indirect
 	go.opentelemetry.io/collector/consumer/consumererror/xconsumererror v0.139.0 // indirect

exporter/otlphttpexporter/partitioner.go

@@ -0,0 +1,72 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package otlphttpexporter // import "go.opentelemetry.io/collector/exporter/otlphttpexporter"
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"slices"
+
+	"go.opentelemetry.io/collector/client"
+	"go.opentelemetry.io/collector/exporter/exporterhelper/xexporterhelper"
+)
+
+type metadataKeysPartitioner struct {
+	keys []string
+}
+
+func (p metadataKeysPartitioner) GetKey(
+	ctx context.Context,
+	_ xexporterhelper.Request,
+) string {
+	var kb bytes.Buffer
+	meta := client.FromContext(ctx).Metadata
+
+	var afterFirst bool
+	for _, k := range p.keys {
+		if values := meta.Get(k); len(values) != 0 {
+			if afterFirst {
+				kb.WriteByte(0)
+			}
+			kb.WriteString(k)
+			afterFirst = true
+			for _, val := range values {
+				kb.WriteByte(0)
+				kb.WriteString(val)
+			}
+		}
+	}
+	return kb.String()
+}
+
+func (p metadataKeysPartitioner) MergeCtx(
+	ctx1, ctx2 context.Context,
+) context.Context {
+	m1 := client.FromContext(ctx1).Metadata
+	m2 := client.FromContext(ctx2).Metadata
+
+	m := make(map[string][]string, len(p.keys))
+	for _, key := range p.keys {
+		v1 := m1.Get(key)
+		v2 := m2.Get(key)
+		if len(v1) == 0 && len(v2) == 0 {
+			continue
+		}
+
+		// Since the mergeCtx is based on partition key, we MUST have the same
+		// partition key-values in both the metadata. If they are not same then
+		// fail fast and dramatically.
+		if !slices.Equal(v1, v2) {
+			panic(fmt.Errorf(
+				"unexpected client metadata found when merging context for key %s", key,
+			))
+		}
+		m[key] = v1
+	}
+	return client.NewContext(
+		context.Background(),
+		client.Info{Metadata: client.NewMetadata(m)},
+	)
+}