Skip to content

Commit 1fa39fd

Browse files
loewenheimclaude
loewenheim
and
claude
authored
feat(eap): Track removed keys in EAP trimming processor (#5570)
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 415eb8d commit 1fa39fd

File tree

4 files changed

+200
-22
lines changed

4 files changed

+200
-22
lines changed

‎relay-event-normalization/src/eap/trimming.rs‎

Lines changed: 161 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::ops::Bound;
2+
13
use relay_event_schema::processor::{
24
self, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor, ValueType,
35
};
@@ -13,6 +15,24 @@ struct SizeState {
1315
size_remaining: Option<usize>,
1416
}
1517

18+
/// The action to take when deleting a value.
19+
#[derive(Debug, Clone, Copy)]
20+
enum DeleteAction {
21+
/// Delete the value without leaving a trace.
22+
Hard,
23+
/// Delete the value and leave a remark with the given rule ID.
24+
WithRemark(&'static str),
25+
}
26+
27+
impl From<DeleteAction> for ProcessingAction {
28+
fn from(action: DeleteAction) -> Self {
29+
match action {
30+
DeleteAction::Hard => ProcessingAction::DeleteValueHard,
31+
DeleteAction::WithRemark(rule_id) => ProcessingAction::DeleteValueWithRemark(rule_id),
32+
}
33+
}
34+
}
35+
1636
/// Processor for trimming EAP items (logs, V2 spans).
1737
///
1838
/// This primarily differs from the regular [`TrimmingProcessor`](crate::trimming::TrimmingProcessor)
@@ -27,13 +47,15 @@ struct SizeState {
2747
#[derive(Default)]
2848
pub struct TrimmingProcessor {
2949
size_state: Vec<SizeState>,
50+
removed_key_byte_budget: usize,
3051
}
3152

3253
impl TrimmingProcessor {
3354
/// Creates a new trimming processor.
34-
pub fn new() -> Self {
55+
pub fn new(removed_key_byte_budget: usize) -> Self {
3556
Self {
3657
size_state: Default::default(),
58+
removed_key_byte_budget,
3759
}
3860
}
3961

@@ -76,6 +98,21 @@ impl TrimmingProcessor {
7698
*remaining = remaining.saturating_sub(size);
7799
}
78100
}
101+
102+
/// Returns a [`DeleteAction`] for removing the given key.
103+
///
104+
/// If there is enough `removed_key_byte_budget` left to accomodate the key,
105+
/// this will be [`DeleteAction::WithRemark`] (which causes a remark to be left).
106+
/// Otherwise, it will be [`DeleteAction::Hard`] (the key is removed without a trace).
107+
fn delete_value(&mut self, key: Option<&str>) -> DeleteAction {
108+
let len = key.map_or(0, |key| key.len());
109+
if len <= self.removed_key_byte_budget {
110+
self.removed_key_byte_budget -= len;
111+
DeleteAction::WithRemark("trimmed")
112+
} else {
113+
DeleteAction::Hard
114+
}
115+
}
79116
}
80117

81118
impl Processor for TrimmingProcessor {
@@ -96,11 +133,12 @@ impl Processor for TrimmingProcessor {
96133
}
97134

98135
if state.attrs().trim {
136+
let key = state.keys().next();
99137
if self.remaining_size() == Some(0) {
100-
return Err(ProcessingAction::DeleteValueHard);
138+
return Err(self.delete_value(key).into());
101139
}
102140
if self.remaining_depth(state) == Some(0) {
103-
return Err(ProcessingAction::DeleteValueHard);
141+
return Err(self.delete_value(key).into());
104142
}
105143
}
106144
Ok(())
@@ -220,7 +258,20 @@ impl Processor for TrimmingProcessor {
220258
}
221259

222260
if let Some(split_index) = split_index {
223-
let _ = value.split_off(split_index);
261+
let mut i = split_index;
262+
263+
for item in &mut value[split_index..] {
264+
match self.delete_value(None) {
265+
DeleteAction::Hard => break,
266+
DeleteAction::WithRemark(rule_id) => {
267+
processor::delete_with_remark(item, rule_id)
268+
}
269+
}
270+
271+
i += 1;
272+
}
273+
274+
let _ = value.split_off(i);
224275
}
225276

226277
if value.len() != original_length {
@@ -266,6 +317,24 @@ impl Processor for TrimmingProcessor {
266317
}
267318

268319
if let Some(split_key) = split_key {
320+
let mut i = split_key.as_str();
321+
322+
// Morally this is just `range_mut(split_key.as_str()..)`, but that doesn't work for type
323+
// inference reasons.
324+
for (key, value) in value
325+
.range_mut::<str, _>((Bound::Included(split_key.as_str()), Bound::Unbounded))
326+
{
327+
i = key.as_str();
328+
329+
match self.delete_value(Some(key.as_ref())) {
330+
DeleteAction::Hard => break,
331+
DeleteAction::WithRemark(rule_id) => {
332+
processor::delete_with_remark(value, rule_id)
333+
}
334+
}
335+
}
336+
337+
let split_key = i.to_owned();
269338
let _ = value.split_off(&split_key);
270339
}
271340

@@ -312,7 +381,20 @@ impl Processor for TrimmingProcessor {
312381
}
313382

314383
if let Some(split_idx) = split_idx {
315-
let _ = sorted.split_off(split_idx);
384+
let mut i = split_idx;
385+
386+
for (key, value) in &mut sorted[split_idx..] {
387+
match self.delete_value(Some(key.as_ref())) {
388+
DeleteAction::Hard => break,
389+
DeleteAction::WithRemark(rule_id) => {
390+
processor::delete_with_remark(value, rule_id)
391+
}
392+
}
393+
394+
i += 1;
395+
}
396+
397+
let _ = sorted.split_off(i);
316398
}
317399

318400
attributes.0 = sorted.into_iter().collect();
@@ -370,7 +452,7 @@ mod tests {
370452
footer: Annotated::empty(),
371453
});
372454

373-
let mut processor = TrimmingProcessor::new();
455+
let mut processor = TrimmingProcessor::new(100);
374456

375457
let state = ProcessingState::new_root(Default::default(), []);
376458
processor::process_value(&mut value, &mut processor, &state).unwrap();
@@ -379,6 +461,7 @@ mod tests {
379461
{
380462
"body": "This is...",
381463
"attributes": {
464+
"attribute is very large and should be removed": null,
382465
"medium string": {
383466
"type": "string",
384467
"value": "This string..."
@@ -393,6 +476,16 @@ mod tests {
393476
"": {
394477
"len": 101
395478
},
479+
"attribute is very large and should be removed": {
480+
"": {
481+
"rem": [
482+
[
483+
"trimmed",
484+
"x"
485+
]
486+
]
487+
}
488+
},
396489
"medium string": {
397490
"value": {
398491
"": {
@@ -444,7 +537,7 @@ mod tests {
444537
footer: Annotated::empty(),
445538
});
446539

447-
let mut processor = TrimmingProcessor::new();
540+
let mut processor = TrimmingProcessor::new(100);
448541

449542
let state = ProcessingState::new_root(Default::default(), []);
450543
processor::process_value(&mut value, &mut processor, &state).unwrap();
@@ -519,7 +612,7 @@ mod tests {
519612
footer: Annotated::empty(),
520613
});
521614

522-
let mut processor = TrimmingProcessor::new();
615+
let mut processor = TrimmingProcessor::new(100);
523616

524617
let state = ProcessingState::new_root(Default::default(), []);
525618
processor::process_value(&mut value, &mut processor, &state).unwrap();
@@ -528,6 +621,7 @@ mod tests {
528621
{
529622
"body": "This is...",
530623
"attributes": {
624+
"attribute is very large and should be removed": null,
531625
"attribute with long name": {
532626
"type": "integer",
533627
"value": 71
@@ -541,6 +635,16 @@ mod tests {
541635
"attributes": {
542636
"": {
543637
"len": 91
638+
},
639+
"attribute is very large and should be removed": {
640+
"": {
641+
"rem": [
642+
[
643+
"trimmed",
644+
"x"
645+
]
646+
]
647+
}
544648
}
545649
},
546650
"body": {
@@ -577,7 +681,7 @@ mod tests {
577681
footer: Annotated::new("Hello World".to_owned()),
578682
});
579683

580-
let mut processor = TrimmingProcessor::new();
684+
let mut processor = TrimmingProcessor::new(100);
581685

582686
// The `body` takes up 5B, `other_number` 10B, the `"small"` attribute 13B, and the key "medium string" another 13B.
583687
// That leaves 9B for the string's value.
@@ -593,6 +697,7 @@ mod tests {
593697
"number": 0,
594698
"other_number": 0,
595699
"attributes": {
700+
"attribute is very large and should be removed": null,
596701
"medium string": {
597702
"type": "string",
598703
"value": "This s..."
@@ -602,11 +707,22 @@ mod tests {
602707
"value": 17
603708
}
604709
},
710+
"footer": null,
605711
"_meta": {
606712
"attributes": {
607713
"": {
608714
"len": 101
609715
},
716+
"attribute is very large and should be removed": {
717+
"": {
718+
"rem": [
719+
[
720+
"trimmed",
721+
"x"
722+
]
723+
]
724+
}
725+
},
610726
"medium string": {
611727
"value": {
612728
"": {
@@ -622,6 +738,16 @@ mod tests {
622738
}
623739
}
624740
}
741+
},
742+
"footer": {
743+
"": {
744+
"rem": [
745+
[
746+
"trimmed",
747+
"x"
748+
]
749+
]
750+
}
625751
}
626752
}
627753
}
@@ -655,7 +781,7 @@ mod tests {
655781
footer: Annotated::empty(),
656782
});
657783

658-
let mut processor = TrimmingProcessor::new();
784+
let mut processor = TrimmingProcessor::new(100);
659785
let state = ProcessingState::new_root(Default::default(), []);
660786
processor::process_value(&mut value, &mut processor, &state).unwrap();
661787

@@ -670,7 +796,8 @@ mod tests {
670796
"value": [
671797
"first string",
672798
"second string",
673-
"another..."
799+
"another...",
800+
null
674801
]
675802
}
676803
},
@@ -681,9 +808,6 @@ mod tests {
681808
},
682809
"array": {
683810
"value": {
684-
"": {
685-
"len": 4
686-
},
687811
"2": {
688812
"": {
689813
"rem": [
@@ -696,6 +820,16 @@ mod tests {
696820
],
697821
"len": 14
698822
}
823+
},
824+
"3": {
825+
"": {
826+
"rem": [
827+
[
828+
"trimmed",
829+
"x"
830+
]
831+
]
832+
}
699833
}
700834
}
701835
}
@@ -719,7 +853,7 @@ mod tests {
719853
footer: Annotated::new("Hello World".to_owned()), // 11B
720854
});
721855

722-
let mut processor = TrimmingProcessor::new();
856+
let mut processor = TrimmingProcessor::new(100);
723857
let state =
724858
ProcessingState::new_root(Some(Cow::Owned(FieldAttrs::default().max_bytes(30))), []);
725859
processor::process_value(&mut value, &mut processor, &state).unwrap();
@@ -732,13 +866,24 @@ mod tests {
732866
"a": {
733867
"type": "integer",
734868
"value": 1
735-
}
869+
},
870+
"this_key_is_exactly_35_chars_long!!": null
736871
},
737872
"footer": "Hello World",
738873
"_meta": {
739874
"attributes": {
740875
"": {
741876
"len": 45
877+
},
878+
"this_key_is_exactly_35_chars_long!!": {
879+
"": {
880+
"rem": [
881+
[
882+
"trimmed",
883+
"x"
884+
]
885+
]
886+
}
742887
}
743888
}
744889
}

0 commit comments

Comments
 (0)