Skip to content

Commit d5c9596

Browse files
strawgateclaude
strawgate
and
claude
committed
perf: use DeepCopyUpdate to eliminate Clone+DeepUpdate allocations
Replace the two-step Clone()+DeepUpdate() pattern with single-pass DeepCopyUpdate() across hot-path processors and pipeline setup. This creates fresh nested maps during the merge rather than cloning the entire source tree first, eliminating intermediate map allocations. Changes: - add_fields: detect single-key wrapper shape at init, clone only inner map at runtime. Split @metadata handling to bypass event.deepUpdate overhead. Skip no-overwrite merge when all destination keys exist. - add_cloud_metadata: replace full metadata.Clone() with per-value deep copy — strings/numbers returned as-is, only nested maps cloned. - add_host_metadata: lock-free fast path via atomic timestamp + mapstr.Pointer. Use DeepCopyUpdate for cached data merge. - add_observer_metadata: Clone()+DeepUpdate → DeepCopyUpdate. - rename: skip event.Clone() backup for single-field renames with different top-level keys (Put-first-then-Delete ordering). - publisher/processing: Clone()+DeepUpdate → DeepCopyUpdate (3 sites). - heartbeat/eventext: Clone()+DeepUpdate → DeepCopyUpdate. Depends on elastic/elastic-agent-libs#390 for DeepCopyUpdate/ DeepCopyUpdateNoOverwrite (temporarily pinned to fork). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 2d10f57 commit d5c9596

File tree

14 files changed

+1660
-68
lines changed

14 files changed

+1660
-68
lines changed

‎changelog/fragments/1774757000-deepcopyupdate-allocations.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
kind: enhancement
2+
summary: Use single-pass DeepCopyUpdate to eliminate Clone+DeepUpdate allocations in add_fields, cloud/host/observer metadata, rename, and pipeline setup
3+
component: libbeat

‎go.mod‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ require (
178178
github.com/elastic/go-freelru v0.16.0
179179
github.com/elastic/go-quark v0.3.0
180180
github.com/elastic/go-sfdc v0.0.0-20251207194532-c5aadd4a4e06
181-
github.com/elastic/mito v1.25.1
181+
github.com/elastic/mito v1.24.1
182182
github.com/elastic/mock-es v0.0.0-20250530054253-8c3b6053f9b6
183183
github.com/elastic/sarama v1.19.1-0.20260310070522-abae92ca1603
184184
github.com/elastic/tk-btf v0.2.0
@@ -188,7 +188,7 @@ require (
188188
github.com/go-resty/resty/v2 v2.17.1
189189
github.com/gofrs/uuid/v5 v5.3.2
190190
github.com/golang-jwt/jwt/v5 v5.3.0
191-
github.com/google/cel-go v0.27.0
191+
github.com/google/cel-go v0.26.1
192192
github.com/googleapis/gax-go/v2 v2.15.0
193193
github.com/gorilla/handlers v1.5.1
194194
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
@@ -456,6 +456,7 @@ require (
456456
github.com/sergi/go-diff v1.3.1 // indirect
457457
github.com/sirupsen/logrus v1.9.3 // indirect
458458
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
459+
github.com/stoewer/go-strcase v1.3.1 // indirect
459460
github.com/stretchr/objx v0.5.2 // indirect
460461
github.com/tklauser/numcpus v0.11.0 // indirect
461462
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect
@@ -518,7 +519,7 @@ require (
518519
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
519520
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
520521
gonum.org/v1/gonum v0.17.0 // indirect
521-
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
522+
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
522523
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
523524
k8s.io/klog/v2 v2.130.1 // indirect
524525
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
@@ -549,3 +550,5 @@ replace (
549550
github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6
550551
github.com/meraki/dashboard-api-go/v3 => github.com/tommyers-elastic/dashboard-api-go/v3 v3.0.0-20250616163611-a325b49669a4
551552
)
553+
554+
replace github.com/elastic/elastic-agent-libs => github.com/strawgate/elastic-agent-libs v0.33.4-0.20260327142400-b15ccc340463

‎go.sum‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -377,8 +377,6 @@ github.com/elastic/elastic-agent-autodiscover v0.10.2 h1:fzi+CIcK7FKUQlQHfKP+3Ix
377377
github.com/elastic/elastic-agent-autodiscover v0.10.2/go.mod h1:qBoYxp3lX3qFRYjEgsOaROC+xL4ItG63GSvOg1SjjsQ=
378378
github.com/elastic/elastic-agent-client/v7 v7.18.1 h1:WnM53JjaukeysrAuiTyrhDPmFxJG07ZAByc2TrkcKcs=
379379
github.com/elastic/elastic-agent-client/v7 v7.18.1/go.mod h1:uDpSGZ+YCKgqgtkwCA0qjwX0gU/wmixDsVbPjY3GkPs=
380-
github.com/elastic/elastic-agent-libs v0.33.3 h1:Gsq5FA29sUbbZVJbeLCKPyRkAxCrOhv3VtXvuG9Uu6k=
381-
github.com/elastic/elastic-agent-libs v0.33.3/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
382380
github.com/elastic/elastic-agent-system-metrics v0.14.3 h1:v867kcgCVguOX3AYIHEVn2RNracdH40FqqXiZq71pDU=
383381
github.com/elastic/elastic-agent-system-metrics v0.14.3/go.mod h1:JNfnZrC0viAjlJRUzQKKuMpDlXgjXBn4WdWEXQF7jcA=
384382
github.com/elastic/elastic-transport-go/v8 v8.8.0 h1:7k1Ua+qluFr6p1jfJjGDl97ssJS/P7cHNInzfxgBQAo=
@@ -427,8 +425,8 @@ github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 h1:VgOx6omXIMK
427425
github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
428426
github.com/elastic/gosigar v0.14.4 h1:7NRnWJDFjEKpOjnHhtzrPGZWr9EMrYFsLjF4q0Czosk=
429427
github.com/elastic/gosigar v0.14.4/go.mod h1:tx91Eb3YgFk6y++h88fRAnxic3Si1ZDHooqnJU/hqo8=
430-
github.com/elastic/mito v1.25.1 h1:k+YjP+TnS3GvMOUgjnm9ChEBSXLhn3SLZ15dsOGgGPQ=
431-
github.com/elastic/mito v1.25.1/go.mod h1:ylq8Yi7G6bP049D0whoDbWaVdUA+fUcGxc67dtKxwWs=
428+
github.com/elastic/mito v1.24.1 h1:sx7TlL1OSKvIAWouRgCeuSd66V40pYl5s5vfcTqjHUo=
429+
github.com/elastic/mito v1.24.1/go.mod h1:h1V+8B62+DXsu0TstJkjsTh5ewJIDJlwzxPkP3HBM9s=
432430
github.com/elastic/mock-es v0.0.0-20250530054253-8c3b6053f9b6 h1:JVNuBrmOoqLJgp9o68YBMnOrXCzQI3mCppW+suwRSlw=
433431
github.com/elastic/mock-es v0.0.0-20250530054253-8c3b6053f9b6/go.mod h1:cXqWcLnmu5y4QveTb2hjk7rgzkHMuZsqeXtbJpNAcu0=
434432
github.com/elastic/sarama v1.19.1-0.20260310070522-abae92ca1603 h1:QFDM5JuLch52FxjHizLP2tiuzfhulUdyOsUe/JuPhrQ=
@@ -604,8 +602,8 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
604602
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
605603
github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
606604
github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
607-
github.com/google/cel-go v0.27.0 h1:e7ih85+4qVrBuqQWTW4FKSqZYokVuc3HnhH5keboFTo=
608-
github.com/google/cel-go v0.27.0/go.mod h1:tTJ11FWqnhw5KKpnWpvW9CJC3Y9GK4EIS0WXnBbebzw=
605+
github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ=
606+
github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM=
609607
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
610608
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
611609
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
@@ -991,6 +989,10 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
991989
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
992990
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
993991
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
992+
github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs=
993+
github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
994+
github.com/strawgate/elastic-agent-libs v0.33.4-0.20260327142400-b15ccc340463 h1:VEylPwQ1+jTlM/CY3WDHX34sexha16NWPF96rcGiTBk=
995+
github.com/strawgate/elastic-agent-libs v0.33.4-0.20260327142400-b15ccc340463/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
994996
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
995997
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
996998
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -1466,8 +1468,8 @@ google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuO
14661468
google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
14671469
google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4=
14681470
google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng=
1469-
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
1470-
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
1471+
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ=
1472+
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
14711473
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
14721474
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
14731475
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

‎heartbeat/eventext/eventext.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525
// MergeEventFields merges the given mapstr.M into the given Event's Fields.
2626
func MergeEventFields(e *beat.Event, merge mapstr.M) {
2727
if e.Fields != nil {
28-
e.Fields.DeepUpdate(merge.Clone())
28+
e.Fields.DeepCopyUpdate(merge)
2929
} else {
3030
e.Fields = merge.Clone()
3131
}

‎libbeat/processors/actions/addfields/add_fields.go‎

Lines changed: 149 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,24 @@ type addFields struct {
3131
fields mapstr.M
3232
shared bool
3333
overwrite bool
34+
35+
// metaFields contains only the @metadata value when fields has @metadata
36+
// but no @timestamp. This allows splitting the update into a fast-path
37+
// Fields.DeepUpdate + a targeted Meta update, avoiding the overhead of
38+
// event.deepUpdate's delete/defer pattern.
39+
metaFields mapstr.M
40+
41+
// fieldsOnly contains the fields without @metadata/@timestamp keys.
42+
// Used together with metaFields to avoid the generic deepUpdate path.
43+
fieldsOnly mapstr.M
44+
45+
// singleKey is set when the fields map has exactly one top-level key
46+
// wrapping an inner mapstr.M (e.g. {"elastic_agent": {"id": "...", ...}}).
47+
// This is the dominant shape created by MakeFieldsProcessor/generateAddFieldsProcessor.
48+
// When set, Run() clones only the inner map and builds a temporary wrapper,
49+
// saving one map allocation per event vs cloning the entire tree.
50+
singleKey string
51+
singleKeyInner mapstr.M
3452
}
3553

3654
// FieldsKey is the default target key for the add_fields processor.
@@ -58,25 +76,153 @@ func CreateAddFields(c *conf.C, _ *logp.Logger) (beat.Processor, error) {
5876
// Set `shared` true if there is the chance of labels being changed/modified by
5977
// subsequent processors.
6078
func NewAddFields(fields mapstr.M, shared bool, overwrite bool) beat.Processor {
61-
return &addFields{fields: fields, shared: shared, overwrite: overwrite}
79+
_, hasTimestamp := fields[beat.TimestampFieldKey]
80+
metaValue, hasMeta := fields[beat.MetadataFieldKey]
81+
82+
af := &addFields{
83+
fields: fields,
84+
shared: shared,
85+
overwrite: overwrite,
86+
}
87+
88+
// Pre-split fields with @metadata but no @timestamp for the optimized path.
89+
if hasMeta && !hasTimestamp {
90+
if metaMap, ok := metaValue.(mapstr.M); ok {
91+
af.metaFields = metaMap
92+
if len(fields) > 1 {
93+
af.fieldsOnly = make(mapstr.M, len(fields)-1)
94+
for k, v := range fields {
95+
if k != beat.MetadataFieldKey {
96+
af.fieldsOnly[k] = v
97+
}
98+
}
99+
}
100+
}
101+
}
102+
103+
// Detect single-key wrapper shape: {"target": mapstr.M{...}}.
104+
// This is the dominant pattern from MakeFieldsProcessor and elastic agent.
105+
// When shared=true, we only need to clone the inner map, not the outer wrapper.
106+
if shared && !hasTimestamp && !hasMeta && len(fields) == 1 {
107+
for k, v := range fields {
108+
if inner, ok := v.(mapstr.M); ok {
109+
af.singleKey = k
110+
af.singleKeyInner = inner
111+
}
112+
}
113+
}
114+
115+
return af
62116
}
63117

64118
func (af *addFields) Run(event *beat.Event) (*beat.Event, error) {
65119
if event == nil || len(af.fields) == 0 {
66120
return event, nil
67121
}
68122

123+
// Single-key wrapper fast path: when fields have exactly one top-level key
124+
// wrapping a nested mapstr.M (e.g. {"elastic_agent": {"id": "...", ...}}),
125+
// clone only the inner map and build a temporary wrapper. This avoids
126+
// cloning the outer map and bypasses event.deepUpdate's special key checks.
127+
// This is the dominant shape from MakeFieldsProcessor and elastic agent.
128+
if af.singleKeyInner != nil {
129+
if event.Fields == nil {
130+
event.Fields = mapstr.M{}
131+
}
132+
if af.shared && af.overwrite {
133+
event.Fields.DeepCopyUpdate(mapstr.M{af.singleKey: af.singleKeyInner})
134+
} else if af.shared {
135+
// Skip no-overwrite merge entirely if the destination already
136+
// has a map at this key with all the source keys present.
137+
// This is the common case for the builtin processor (ecs, host,
138+
// agent) running after agent-injected processors that already
139+
// set these keys.
140+
if dstVal, ok := event.Fields[af.singleKey]; ok {
141+
if dstMap, ok := dstVal.(mapstr.M); ok {
142+
allExist := true
143+
for sk := range af.singleKeyInner {
144+
if _, ok := dstMap[sk]; !ok {
145+
allExist = false
146+
break
147+
}
148+
}
149+
if allExist {
150+
return event, nil
151+
}
152+
}
153+
}
154+
event.Fields.DeepCopyUpdateNoOverwrite(mapstr.M{af.singleKey: af.singleKeyInner})
155+
} else if af.overwrite {
156+
event.Fields.DeepUpdate(mapstr.M{af.singleKey: af.singleKeyInner})
157+
} else {
158+
event.Fields.DeepUpdateNoOverwrite(mapstr.M{af.singleKey: af.singleKeyInner})
159+
}
160+
return event, nil
161+
}
162+
163+
// Metadata split path: when fields contain @metadata but no @timestamp,
164+
// update event.Meta and event.Fields separately. This avoids cloning the
165+
// outer {"@metadata": inner} wrapper and bypasses event.deepUpdate's
166+
// delete/defer pattern for @metadata handling.
167+
if af.metaFields != nil {
168+
if event.Meta == nil {
169+
event.Meta = mapstr.M{}
170+
}
171+
if af.shared && af.overwrite {
172+
event.Meta.DeepCopyUpdate(af.metaFields)
173+
} else if af.shared {
174+
event.Meta.DeepCopyUpdateNoOverwrite(af.metaFields)
175+
} else if af.overwrite {
176+
event.Meta.DeepUpdate(af.metaFields)
177+
} else {
178+
event.Meta.DeepUpdateNoOverwrite(af.metaFields)
179+
}
180+
if len(af.fieldsOnly) > 0 {
181+
if event.Fields == nil {
182+
event.Fields = mapstr.M{}
183+
}
184+
if af.shared && af.overwrite {
185+
event.Fields.DeepCopyUpdate(af.fieldsOnly)
186+
} else if af.shared {
187+
event.Fields.DeepCopyUpdateNoOverwrite(af.fieldsOnly)
188+
} else if af.overwrite {
189+
event.Fields.DeepUpdate(af.fieldsOnly)
190+
} else {
191+
event.Fields.DeepUpdateNoOverwrite(af.fieldsOnly)
192+
}
193+
}
194+
return event, nil
195+
}
196+
197+
// General path: handles @timestamp, @metadata, and regular fields.
198+
if event.Fields == nil {
199+
event.Fields = mapstr.M{}
200+
}
201+
202+
_, hasTimestamp := af.fields[beat.TimestampFieldKey]
203+
_, hasMeta := af.fields[beat.MetadataFieldKey]
204+
205+
if !hasTimestamp && !hasMeta && af.shared {
206+
// No special keys — safe to merge directly into Fields.
207+
if af.overwrite {
208+
event.Fields.DeepCopyUpdate(af.fields)
209+
} else {
210+
event.Fields.DeepCopyUpdateNoOverwrite(af.fields)
211+
}
212+
return event, nil
213+
}
214+
215+
// Slow path: has @timestamp or @metadata, needs event.DeepUpdate
216+
// which handles those special keys.
69217
fields := af.fields
70218
if af.shared {
71219
fields = fields.Clone()
72220
}
73-
74221
if af.overwrite {
75222
event.DeepUpdate(fields)
76223
} else {
77224
event.DeepUpdateNoOverwrite(fields)
78225
}
79-
80226
return event, nil
81227
}
82228

0 commit comments

Comments
 (0)