Refactor target label handling in target allocator for improved performance (#4587)

* Refactor target relabelling

Manually build labels for targets

* Use scratchbuilder

Reuse the sorted label name buffer

Author: swiatekm

cmd/otel-allocator/internal/prehook/relabel.go

@@ -7,6 +7,7 @@ import (
 	"slices"
 
 	"github.com/go-logr/logr"
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
 
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/target"
@@ -36,19 +37,23 @@ func (tf *relabelConfigTargetFilter) Apply(targets []*target.Item) []*target.Ite
 		return targets
 	}
 
+	builder := labels.NewBuilder(labels.EmptyLabels())
 	writeIndex := 0
 	for _, tItem := range targets {
-		keepTarget := true
-		lset := tItem.Labels
-		for _, cfg := range tf.relabelCfg[tItem.JobName] {
-			lset, keepTarget = relabel.Process(lset, cfg)
-			if !keepTarget {
-				break // inner loop
-			}
-		}
+		builder.Reset(tItem.Labels)
+		keepTarget := relabel.ProcessBuilder(builder, tf.relabelCfg[tItem.JobName]...)
 
 		if keepTarget {
-			targets[writeIndex] = target.NewItem(tItem.JobName, tItem.TargetURL, tItem.Labels, tItem.CollectorName, target.WithRelabeledLabels(lset))
+			// Compute hash immediately while we have the builder, skipping meta labels.
+			// This avoids materializing the filtered labels.
+			hash := target.HashFromBuilder(builder, tItem.JobName)
+			targets[writeIndex] = target.NewItem(
+				tItem.JobName,
+				tItem.TargetURL,
+				tItem.Labels,
+				tItem.CollectorName,
+				target.WithHash(hash),
+			)
 			writeIndex++
 		}
 	}

cmd/otel-allocator/internal/prehook/relabel_test.go

@@ -244,7 +244,7 @@ func TestApply(t *testing.T) {
 	allocatorPrehook.SetConfig(relabelCfg)
 	remainingItems := allocatorPrehook.Apply(targets)
 	assert.Len(t, remainingItems, numRemaining)
-	assert.Equal(t, remainingItems, expectedTargetMap)
+	assert.Equal(t, expectedTargetMap, remainingItems)
 
 	// clear out relabelCfg to test with empty values
 	for key := range relabelCfg {
@@ -370,7 +370,15 @@ func MakeTargetFromProm(rCfgs []*relabel.Config, rawTarget *target.Item) (*targe
 		return nil, nil
 	}
 
-	newTarget := target.NewItem(rawTarget.JobName, rawTarget.TargetURL, rawTarget.Labels, rawTarget.CollectorName, target.WithRelabeledLabels(lset))
+	// Compute the hash from the builder, skipping meta labels
+	hash := target.HashFromBuilder(lb, rawTarget.JobName)
+	newTarget := target.NewItem(
+		rawTarget.JobName,
+		rawTarget.TargetURL,
+		rawTarget.Labels,
+		rawTarget.CollectorName,
+		target.WithHash(hash),
+	)
 	return newTarget, nil
 }
 

cmd/otel-allocator/internal/target/discovery.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	"hash"
 	"hash/fnv"
+	"maps"
+	"slices"
 	"sync"
 	"time"
 
@@ -26,6 +28,8 @@ import (
 	allocatorWatcher "github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/watcher"
 )
 
+const labelBuilderPreallocSize = 100
+
 type Discoverer struct {
 	log                         logr.Logger
 	manager                     *discovery.Manager
@@ -199,7 +203,11 @@ func (m *Discoverer) Reload() {
 
 // processTargetGroups processes the target groups and returns a map of targets.
 func (m *Discoverer) processTargetGroups(jobName string, groups []*targetgroup.Group, intoTargets []*Item) {
-	builder := labels.NewBuilder(labels.Labels{})
+	// the builder for group labels
+	groupBuilder := labels.NewScratchBuilder(labelBuilderPreallocSize)
+
+	// a slice for sorting target label names, we allocate it here to avoid doing it in the hot loop
+	targetLabelNames := make([]string, 0, labelBuilderPreallocSize)
 
 	begin := time.Now()
 	defer func() {
@@ -208,18 +216,32 @@ func (m *Discoverer) processTargetGroups(jobName string, groups []*targetgroup.G
 	var count float64 = 0
 	index := 0
 	for _, tg := range groups {
-		builder.Reset(labels.EmptyLabels())
+		groupBuilder.Reset()
 		for ln, lv := range tg.Labels {
-			builder.Set(string(ln), string(lv))
+			groupBuilder.Add(string(ln), string(lv))
 		}
-		groupLabels := builder.Labels()
+		groupBuilder.Sort()
 		for _, t := range tg.Targets {
 			count++
-			builder.Reset(groupLabels)
-			for ln, lv := range t {
-				builder.Set(string(ln), string(lv))
+			// ScratchBuilder is a struct containing a slice of labels. By assigning to a new variable, we get a copy
+			// of the struct, with a new slice pointing to the same underlying array. As long as we don't mutate the
+			// original slice and only append to it, we can avoid copying the group labels.
+			targetBuilder := groupBuilder
+			targetLabelNames = targetLabelNames[:0]
+
+			// We can't sort the whole builder slice, because that would modify the underlying groupBuilder. Instead,
+			// we sort the labels in a separate slice. As a result, the group labels and the target labels are sorted
+			// subslices of the builder slice, which is in itself not sorted. This is fine, as we don't care what the
+			// order of labels is - just that it's consistent, so the hash is always the same.
+			for ln := range maps.Keys(t) {
+				targetLabelNames = append(targetLabelNames, string(ln))
+			}
+			slices.Sort(targetLabelNames)
+			for _, ln := range targetLabelNames {
+				lv := t[model.LabelName(ln)]
+				targetBuilder.Add(ln, string(lv))
 			}
-			item := NewItem(jobName, string(t[model.AddressLabel]), builder.Labels(), "")
+			item := NewItem(jobName, string(t[model.AddressLabel]), targetBuilder.Labels(), "")
 			intoTargets[index] = item
 			index++
 		}

cmd/otel-allocator/internal/target/target.go

@@ -5,15 +5,26 @@ package target
 
 import (
 	"encoding/binary"
-	"slices"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
 )
 
+// seps is the separator used between label name/value pairs in hash computation.
+// This matches Prometheus's label hashing approach.
+var seps = []byte{'\xff'}
+
+// hasherPool is a pool of xxhash digesters for efficient hash computation.
+var hasherPool = sync.Pool{
+	New: func() any {
+		return xxhash.New()
+	},
+}
+
 // nodeLabels are labels that are used to identify the node on which the given
 // target is residing. To learn more about these labels, please refer to:
 // https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config
@@ -37,40 +48,53 @@ func (h ItemHash) String() string {
 
 // Item represents a target to be scraped.
 type Item struct {
-	JobName   string
-	TargetURL string
-	Labels    labels.Labels
-	// relabeledLabels contains the final labels after Prometheus relabeling processing.
-	relabeledLabels labels.Labels
-	CollectorName   string
-	hash            ItemHash
+	JobName       string
+	TargetURL     string
+	Labels        labels.Labels
+	CollectorName string
+	hash          ItemHash
 }
 
 type ItemOption func(*Item)
 
-func WithRelabeledLabels(lbs labels.Labels) ItemOption {
+// WithHash sets a precomputed hash on the item.
+// Use this when the hash has been computed during relabeling to avoid recomputation.
+func WithHash(hash ItemHash) ItemOption {
 	return func(i *Item) {
-		// In Prometheus, labels with the MetaLabelPrefix are discarded after relabeling, which means they are not used in hash calculation.
-		// For details, see https://github.com/prometheus/prometheus/blob/e6cfa720fbe6280153fab13090a483dbd40bece3/scrape/target.go#L534.
-		writeIndex := 0
-		relabeledLabels := make(labels.Labels, len(lbs))
-		for _, l := range lbs {
-			if !strings.HasPrefix(l.Name, model.MetaLabelPrefix) {
-				relabeledLabels[writeIndex] = l
-				writeIndex++
-			}
-		}
-		i.relabeledLabels = slices.Clip(relabeledLabels[:writeIndex])
+		i.hash = hash
 	}
 }
 
 func (t *Item) Hash() ItemHash {
 	if t.hash == 0 {
-		t.hash = ItemHash(LabelsHashWithJobName(t.relabeledLabels, t.JobName))
+		t.hash = ItemHash(LabelsHashWithJobName(t.Labels, t.JobName))
 	}
 	return t.hash
 }
 
+// HashFromBuilder computes a hash from a labels.Builder, skipping meta labels.
+// This is used during relabeling to compute the hash efficiently without materializing
+// the filtered labels.
+func HashFromBuilder(builder *labels.Builder, jobName string) ItemHash {
+	hash := hasherPool.Get().(*xxhash.Digest)
+	hash.Reset()
+	builder.Range(func(l labels.Label) {
+		// Skip meta labels - they are discarded after relabeling in Prometheus.
+		// For details, see https://github.com/prometheus/prometheus/blob/e6cfa720fbe6280153fab13090a483dbd40bece3/scrape/target.go#L534
+		if strings.HasPrefix(l.Name, model.MetaLabelPrefix) {
+			return
+		}
+		_, _ = hash.WriteString(l.Name)
+		_, _ = hash.Write(seps)
+		_, _ = hash.WriteString(l.Value)
+		_, _ = hash.Write(seps)
+	})
+	_, _ = hash.WriteString(jobName)
+	result := hash.Sum64()
+	hasherPool.Put(hash)
+	return ItemHash(result)
+}
+
 func (t *Item) GetNodeName() string {
 	relevantLabels := t.Labels.MatchLabels(true, relevantLabelNames...)
 	for _, label := range nodeLabels {
@@ -95,14 +119,12 @@ func (t *Item) GetEndpointSliceName() string {
 // NewItem Creates a new target item.
 // INVARIANTS:
 // * Item fields must not be modified after creation.
-func NewItem(jobName string, targetURL string, labels labels.Labels, collectorName string, opts ...ItemOption) *Item {
+func NewItem(jobName string, targetURL string, itemLabels labels.Labels, collectorName string, opts ...ItemOption) *Item {
 	item := &Item{
-		JobName:   jobName,
-		TargetURL: targetURL,
-		Labels:    labels,
-		// relabeledLabels defaults to original labels if WithRelabeledLabels is not specified.
-		relabeledLabels: labels,
-		CollectorName:   collectorName,
+		JobName:       jobName,
+		TargetURL:     targetURL,
+		Labels:        itemLabels,
+		CollectorName: collectorName,
 	}
 	for _, opt := range opts {
 		opt(item)
@@ -116,10 +138,13 @@ func NewItem(jobName string, targetURL string, labels labels.Labels, collectorNa
 // The scrape manager adds it later. Address is already included in the labels, so it is not needed here.
 func LabelsHashWithJobName(ls labels.Labels, jobName string) uint64 {
 	labelsHash := ls.Hash()
-	labelsHashBytes := make([]byte, 8)
-	_, _ = binary.Encode(labelsHashBytes, binary.LittleEndian, labelsHash) // nolint: errcheck // this can only fail if the buffer size is wrong
-	hash := xxhash.New()
-	_, _ = hash.Write(labelsHashBytes) // nolint: errcheck // xxhash.Write can't fail
-	_, _ = hash.Write([]byte(jobName)) // nolint: errcheck // xxhash.Write can't fail
-	return hash.Sum64()
+	var labelsHashBytes [8]byte
+	binary.LittleEndian.PutUint64(labelsHashBytes[:], labelsHash)
+	hash := hasherPool.Get().(*xxhash.Digest)
+	hash.Reset()
+	_, _ = hash.Write(labelsHashBytes[:]) // nolint: errcheck // xxhash.Write can't fail
+	_, _ = hash.WriteString(jobName)      // nolint: errcheck // xxhash.Write can't fail
+	result := hash.Sum64()
+	hasherPool.Put(hash)
+	return result
 }