Skip to content

[DNM] perf: use DeepCopyUpdate to eliminate Clone+DeepUpdate allocations #49762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
strawgate wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[DNM] perf: use DeepCopyUpdate to eliminate Clone+DeepUpdate allocations #49762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog/fragments/1774757000-deepcopyupdate-allocations.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
kind: enhancement
summary: Use single-pass DeepCopyUpdate to eliminate Clone+DeepUpdate allocations in add_fields, cloud/host/observer metadata, rename, and pipeline setup
component: libbeat
9 changes: 6 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ require (
github.com/elastic/go-freelru v0.16.0
github.com/elastic/go-quark v0.3.0
github.com/elastic/go-sfdc v0.0.0-20251207194532-c5aadd4a4e06
github.com/elastic/mito v1.25.1
github.com/elastic/mito v1.24.1
github.com/elastic/mock-es v0.0.0-20250530054253-8c3b6053f9b6
github.com/elastic/sarama v1.19.1-0.20260310070522-abae92ca1603
github.com/elastic/tk-btf v0.2.0
Expand All @@ -188,7 +188,7 @@ require (
github.com/go-resty/resty/v2 v2.17.1
github.com/gofrs/uuid/v5 v5.3.2
github.com/golang-jwt/jwt/v5 v5.3.0
github.com/google/cel-go v0.27.0
github.com/google/cel-go v0.26.1
github.com/googleapis/gax-go/v2 v2.15.0
github.com/gorilla/handlers v1.5.1
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
Expand Down Expand Up @@ -456,6 +456,7 @@ require (
github.com/sergi/go-diff v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
github.com/stoewer/go-strcase v1.3.1 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect
Expand Down Expand Up @@ -518,7 +519,7 @@ require (
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
gonum.org/v1/gonum v0.17.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
Expand Down Expand Up @@ -549,3 +550,5 @@ replace (
github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6
github.com/meraki/dashboard-api-go/v3 => github.com/tommyers-elastic/dashboard-api-go/v3 v3.0.0-20250616163611-a325b49669a4
)

replace github.com/elastic/elastic-agent-libs => github.com/strawgate/elastic-agent-libs v0.33.4-0.20260327142400-b15ccc340463
18 changes: 10 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,8 +377,6 @@ github.com/elastic/elastic-agent-autodiscover v0.10.2 h1:fzi+CIcK7FKUQlQHfKP+3Ix
github.com/elastic/elastic-agent-autodiscover v0.10.2/go.mod h1:qBoYxp3lX3qFRYjEgsOaROC+xL4ItG63GSvOg1SjjsQ=
github.com/elastic/elastic-agent-client/v7 v7.18.1 h1:WnM53JjaukeysrAuiTyrhDPmFxJG07ZAByc2TrkcKcs=
github.com/elastic/elastic-agent-client/v7 v7.18.1/go.mod h1:uDpSGZ+YCKgqgtkwCA0qjwX0gU/wmixDsVbPjY3GkPs=
github.com/elastic/elastic-agent-libs v0.33.3 h1:Gsq5FA29sUbbZVJbeLCKPyRkAxCrOhv3VtXvuG9Uu6k=
github.com/elastic/elastic-agent-libs v0.33.3/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
github.com/elastic/elastic-agent-system-metrics v0.14.3 h1:v867kcgCVguOX3AYIHEVn2RNracdH40FqqXiZq71pDU=
github.com/elastic/elastic-agent-system-metrics v0.14.3/go.mod h1:JNfnZrC0viAjlJRUzQKKuMpDlXgjXBn4WdWEXQF7jcA=
github.com/elastic/elastic-transport-go/v8 v8.8.0 h1:7k1Ua+qluFr6p1jfJjGDl97ssJS/P7cHNInzfxgBQAo=
Expand Down Expand Up @@ -427,8 +425,8 @@ github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 h1:VgOx6omXIMK
github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
github.com/elastic/gosigar v0.14.4 h1:7NRnWJDFjEKpOjnHhtzrPGZWr9EMrYFsLjF4q0Czosk=
github.com/elastic/gosigar v0.14.4/go.mod h1:tx91Eb3YgFk6y++h88fRAnxic3Si1ZDHooqnJU/hqo8=
github.com/elastic/mito v1.25.1 h1:k+YjP+TnS3GvMOUgjnm9ChEBSXLhn3SLZ15dsOGgGPQ=
github.com/elastic/mito v1.25.1/go.mod h1:ylq8Yi7G6bP049D0whoDbWaVdUA+fUcGxc67dtKxwWs=
github.com/elastic/mito v1.24.1 h1:sx7TlL1OSKvIAWouRgCeuSd66V40pYl5s5vfcTqjHUo=
github.com/elastic/mito v1.24.1/go.mod h1:h1V+8B62+DXsu0TstJkjsTh5ewJIDJlwzxPkP3HBM9s=
github.com/elastic/mock-es v0.0.0-20250530054253-8c3b6053f9b6 h1:JVNuBrmOoqLJgp9o68YBMnOrXCzQI3mCppW+suwRSlw=
github.com/elastic/mock-es v0.0.0-20250530054253-8c3b6053f9b6/go.mod h1:cXqWcLnmu5y4QveTb2hjk7rgzkHMuZsqeXtbJpNAcu0=
github.com/elastic/sarama v1.19.1-0.20260310070522-abae92ca1603 h1:QFDM5JuLch52FxjHizLP2tiuzfhulUdyOsUe/JuPhrQ=
Expand Down Expand Up @@ -604,8 +602,8 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
github.com/google/cel-go v0.27.0 h1:e7ih85+4qVrBuqQWTW4FKSqZYokVuc3HnhH5keboFTo=
github.com/google/cel-go v0.27.0/go.mod h1:tTJ11FWqnhw5KKpnWpvW9CJC3Y9GK4EIS0WXnBbebzw=
github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ=
github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM=
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
Expand Down Expand Up @@ -991,6 +989,10 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs=
github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
github.com/strawgate/elastic-agent-libs v0.33.4-0.20260327142400-b15ccc340463 h1:VEylPwQ1+jTlM/CY3WDHX34sexha16NWPF96rcGiTBk=
github.com/strawgate/elastic-agent-libs v0.33.4-0.20260327142400-b15ccc340463/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
Expand Down Expand Up @@ -1466,8 +1468,8 @@ google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuO
google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4=
google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
Expand Down
2 changes: 1 addition & 1 deletion heartbeat/eventext/eventext.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
// MergeEventFields merges the given mapstr.M into the given Event's Fields.
func MergeEventFields(e *beat.Event, merge mapstr.M) {
if e.Fields != nil {
e.Fields.DeepUpdate(merge.Clone())
e.Fields.DeepCopyUpdate(merge)
} else {
e.Fields = merge.Clone()
}
Expand Down
152 changes: 149 additions & 3 deletions libbeat/processors/actions/addfields/add_fields.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,24 @@ type addFields struct {
fields mapstr.M
shared bool
overwrite bool

// metaFields contains only the @metadata value when fields has @metadata
// but no @timestamp. This allows splitting the update into a fast-path
// Fields.DeepUpdate + a targeted Meta update, avoiding the overhead of
// event.deepUpdate's delete/defer pattern.
metaFields mapstr.M

// fieldsOnly contains the fields without @metadata/@timestamp keys.
// Used together with metaFields to avoid the generic deepUpdate path.
fieldsOnly mapstr.M

// singleKey is set when the fields map has exactly one top-level key
// wrapping an inner mapstr.M (e.g. {"elastic_agent": {"id": "...", ...}}).
// This is the dominant shape created by MakeFieldsProcessor/generateAddFieldsProcessor.
// When set, Run() clones only the inner map and builds a temporary wrapper,
// saving one map allocation per event vs cloning the entire tree.
singleKey string
singleKeyInner mapstr.M
}

// FieldsKey is the default target key for the add_fields processor.
Expand Down Expand Up @@ -58,25 +76,153 @@ func CreateAddFields(c *conf.C, _ *logp.Logger) (beat.Processor, error) {
// Set `shared` true if there is the chance of labels being changed/modified by
// subsequent processors.
func NewAddFields(fields mapstr.M, shared bool, overwrite bool) beat.Processor {
return &addFields{fields: fields, shared: shared, overwrite: overwrite}
_, hasTimestamp := fields[beat.TimestampFieldKey]
metaValue, hasMeta := fields[beat.MetadataFieldKey]

af := &addFields{
fields: fields,
shared: shared,
overwrite: overwrite,
}

// Pre-split fields with @metadata but no @timestamp for the optimized path.
if hasMeta && !hasTimestamp {
if metaMap, ok := metaValue.(mapstr.M); ok {
af.metaFields = metaMap
if len(fields) > 1 {
af.fieldsOnly = make(mapstr.M, len(fields)-1)
for k, v := range fields {
if k != beat.MetadataFieldKey {
af.fieldsOnly[k] = v
}
}
}
}
}

// Detect single-key wrapper shape: {"target": mapstr.M{...}}.
// This is the dominant pattern from MakeFieldsProcessor and elastic agent.
// When shared=true, we only need to clone the inner map, not the outer wrapper.
if shared && !hasTimestamp && !hasMeta && len(fields) == 1 {
for k, v := range fields {
if inner, ok := v.(mapstr.M); ok {
af.singleKey = k
af.singleKeyInner = inner
}
}
}

return af
}

func (af *addFields) Run(event *beat.Event) (*beat.Event, error) {
if event == nil || len(af.fields) == 0 {
return event, nil
}

// Single-key wrapper fast path: when fields have exactly one top-level key
// wrapping a nested mapstr.M (e.g. {"elastic_agent": {"id": "...", ...}}),
// clone only the inner map and build a temporary wrapper. This avoids
// cloning the outer map and bypasses event.deepUpdate's special key checks.
// This is the dominant shape from MakeFieldsProcessor and elastic agent.
if af.singleKeyInner != nil {
if event.Fields == nil {
event.Fields = mapstr.M{}
}
if af.shared && af.overwrite {
event.Fields.DeepCopyUpdate(mapstr.M{af.singleKey: af.singleKeyInner})
} else if af.shared {
// Skip no-overwrite merge entirely if the destination already
// has a map at this key with all the source keys present.
// This is the common case for the builtin processor (ecs, host,
// agent) running after agent-injected processors that already
// set these keys.
if dstVal, ok := event.Fields[af.singleKey]; ok {
if dstMap, ok := dstVal.(mapstr.M); ok {
allExist := true
for sk := range af.singleKeyInner {
if _, ok := dstMap[sk]; !ok {
allExist = false
break
}
}
if allExist {
return event, nil
}
}
}
event.Fields.DeepCopyUpdateNoOverwrite(mapstr.M{af.singleKey: af.singleKeyInner})
Comment on lines +140 to +154
Copy link
Copy Markdown

@coderabbitai coderabbitai bot Mar 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Don’t short-circuit recursive no-overwrite merges on top-level key presence.

Line 143 only checks whether the immediate child keys exist. That changes behavior for nested maps: if the destination already has host.os, this returns early even when nested leaves like host.os.version are still missing, while DeepCopyUpdateNoOverwrite would descend and add them. That can silently drop enrichment on partially populated objects.

Safe fix 
-			if dstVal, ok := event.Fields[af.singleKey]; ok {
-				if dstMap, ok := dstVal.(mapstr.M); ok {
-					allExist := true
-					for sk := range af.singleKeyInner {
-						if _, ok := dstMap[sk]; !ok {
-							allExist = false
-							break
-						}
-					}
-					if allExist {
-						return event, nil
-					}
-				}
-			}
 			event.Fields.DeepCopyUpdateNoOverwrite(mapstr.M{af.singleKey: af.singleKeyInner})
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if dstVal, ok := event.Fields[af.singleKey]; ok {
if dstMap, ok := dstVal.(mapstr.M); ok {
allExist := true
for sk := range af.singleKeyInner {
if _, ok := dstMap[sk]; !ok {
allExist = false
break
}
}
if allExist {
return event, nil
}
}
}
event.Fields.DeepCopyUpdateNoOverwrite(mapstr.M{af.singleKey: af.singleKeyInner})
event.Fields.DeepCopyUpdateNoOverwrite(mapstr.M{af.singleKey: af.singleKeyInner})
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@libbeat/processors/actions/addfields/add_fields.go` around lines 140 - 154,
The current check around af.singleKey/af.singleKeyInner short-circuits and
returns early when the top-level key exists but nested leaves may be missing;
remove or replace that shortcut so DeepCopyUpdateNoOverwrite always runs for
this case. Specifically, eliminate the early-return branch that inspects
event.Fields[af.singleKey] and returns when all immediate child keys exist, or
change it to perform a full recursive existence check against af.singleKeyInner
before returning; otherwise always call
event.Fields.DeepCopyUpdateNoOverwrite(mapstr.M{af.singleKey:
af.singleKeyInner}) so nested missing leaves (e.g., host.os.version) are merged
in.

} else if af.overwrite {
event.Fields.DeepUpdate(mapstr.M{af.singleKey: af.singleKeyInner})
} else {
event.Fields.DeepUpdateNoOverwrite(mapstr.M{af.singleKey: af.singleKeyInner})
}
return event, nil
}

// Metadata split path: when fields contain @metadata but no @timestamp,
// update event.Meta and event.Fields separately. This avoids cloning the
// outer {"@metadata": inner} wrapper and bypasses event.deepUpdate's
// delete/defer pattern for @metadata handling.
if af.metaFields != nil {
if event.Meta == nil {
event.Meta = mapstr.M{}
}
if af.shared && af.overwrite {
event.Meta.DeepCopyUpdate(af.metaFields)
} else if af.shared {
event.Meta.DeepCopyUpdateNoOverwrite(af.metaFields)
} else if af.overwrite {
event.Meta.DeepUpdate(af.metaFields)
} else {
event.Meta.DeepUpdateNoOverwrite(af.metaFields)
}
if len(af.fieldsOnly) > 0 {
if event.Fields == nil {
event.Fields = mapstr.M{}
}
if af.shared && af.overwrite {
event.Fields.DeepCopyUpdate(af.fieldsOnly)
} else if af.shared {
event.Fields.DeepCopyUpdateNoOverwrite(af.fieldsOnly)
} else if af.overwrite {
event.Fields.DeepUpdate(af.fieldsOnly)
} else {
event.Fields.DeepUpdateNoOverwrite(af.fieldsOnly)
}
}
return event, nil
}

// General path: handles @timestamp, @metadata, and regular fields.
if event.Fields == nil {
event.Fields = mapstr.M{}
}

_, hasTimestamp := af.fields[beat.TimestampFieldKey]
_, hasMeta := af.fields[beat.MetadataFieldKey]

if !hasTimestamp && !hasMeta && af.shared {
// No special keys — safe to merge directly into Fields.
if af.overwrite {
event.Fields.DeepCopyUpdate(af.fields)
} else {
event.Fields.DeepCopyUpdateNoOverwrite(af.fields)
}
return event, nil
}

// Slow path: has @timestamp or @metadata, needs event.DeepUpdate
// which handles those special keys.
fields := af.fields
if af.shared {
fields = fields.Clone()
}

if af.overwrite {
event.DeepUpdate(fields)
} else {
event.DeepUpdateNoOverwrite(fields)
}

return event, nil
}

Expand Down
Loading
Loading