Skip to content
Draft
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 29 additions & 14 deletions sdk/log/record.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ var logAttrDropped = sync.OnceFunc(func() {
global.Warn("limit reached: dropping log Record attributes")
})

var logKeyValuePairDropped = sync.OnceFunc(func() {
global.Warn("key duplication: dropping key-value pair")
})

// indexPool is a pool of index maps used for de-duplication.
var indexPool = sync.Pool{
New: func() any { return make(map[string]int) },
Expand Down Expand Up @@ -97,12 +101,10 @@ type Record struct {
}

func (r *Record) addDropped(n int) {
logAttrDropped()
r.dropped += n
}

func (r *Record) setDropped(n int) {
logAttrDropped()
r.dropped = n
}

Expand Down Expand Up @@ -192,11 +194,15 @@ func (r *Record) AddAttributes(attrs ...log.KeyValue) {
if n == 0 {
// Avoid the more complex duplicate map lookups below.
var drop int
attrs, drop = dedup(attrs)
r.setDropped(drop)
attrs = dedup(attrs)
r.setDropped(0)

attrs, drop = head(attrs, r.attributeCountLimit)

r.addDropped(drop)
if drop > 0 {
logAttrDropped()
}

r.addAttrs(attrs)
return
Expand All @@ -221,15 +227,15 @@ func (r *Record) AddAttributes(attrs ...log.KeyValue) {
// Last-value-wins for any duplicates in attrs.
idx, found := uIndex[a.Key]
if found {
r.addDropped(1)
logKeyValuePairDropped()
unique[idx] = a
continue
}

idx, found = rIndex[a.Key]
if found {
// New attrs overwrite any existing with the same key.
r.addDropped(1)
logKeyValuePairDropped()
if idx < 0 {
r.front[-(idx + 1)] = a
} else {
Expand All @@ -249,7 +255,12 @@ func (r *Record) AddAttributes(attrs ...log.KeyValue) {
// Do not use head(attrs, r.attributeCountLimit - n) here. If
// (r.attributeCountLimit - n) <= 0 attrs needs to be emptied.
last := max(0, r.attributeCountLimit-n)
r.addDropped(len(attrs) - last)
dropped := len(attrs) - last
r.addDropped(dropped)

if dropped > 0 {
logAttrDropped()
}
attrs = attrs[:last]
}

Expand Down Expand Up @@ -297,12 +308,16 @@ func (r *Record) addAttrs(attrs []log.KeyValue) {
// SetAttributes sets (and overrides) attributes to the log record.
func (r *Record) SetAttributes(attrs ...log.KeyValue) {
var drop int
attrs, drop = dedup(attrs)
r.setDropped(drop)
attrs = dedup(attrs)
r.setDropped(0)

attrs, drop = head(attrs, r.attributeCountLimit)
r.addDropped(drop)

if drop > 0 {
logAttrDropped()
}

r.nFront = 0
var i int
for i = 0; i < len(attrs) && r.nFront < len(r.front); i++ {
Expand All @@ -327,22 +342,22 @@ func head(kvs []log.KeyValue, n int) (out []log.KeyValue, dropped int) {
}

// dedup deduplicates kvs front-to-back with the last value saved.
func dedup(kvs []log.KeyValue) (unique []log.KeyValue, dropped int) {
func dedup(kvs []log.KeyValue) (unique []log.KeyValue) {
index := getIndex()
defer putIndex(index)

unique = kvs[:0] // Use the same underlying array as kvs.
for _, a := range kvs {
idx, found := index[a.Key]
if found {
dropped++
unique[idx] = a
logKeyValuePairDropped()
} else {
unique = append(unique, a)
index[a.Key] = len(unique) - 1
}
}
return unique, dropped
return unique
}

// AttributesLen returns the number of attributes in the log record.
Expand Down Expand Up @@ -428,8 +443,8 @@ func (r *Record) applyValueLimits(val log.Value) log.Value {
case log.KindMap:
// Deduplicate then truncate. Do not do at the same time to avoid
// wasted truncation operations.
kvs, dropped := dedup(val.AsMap())
r.addDropped(dropped)
kvs := dedup(val.AsMap())

for i := range kvs {
kvs[i] = r.applyAttrLimits(kvs[i])
}
Expand Down
6 changes: 3 additions & 3 deletions sdk/log/record_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,12 +204,12 @@ func TestRecordClone(t *testing.T) {
}

func TestRecordDroppedAttributes(t *testing.T) {
orig := logAttrDropped
t.Cleanup(func() { logAttrDropped = orig })
orig := logKeyValuePairDropped
t.Cleanup(func() { logKeyValuePairDropped = orig })

for i := 1; i < attributesInlineCount*5; i++ {
var called bool
logAttrDropped = func() { called = true }
logKeyValuePairDropped = func() { called = true }

r := new(Record)
r.attributeCountLimit = 1
Expand Down
Loading