cap log pattern cardinality

Author: def

cmd/logparser.go

@@ -23,7 +23,7 @@ func main() {
 
 	reader := bufio.NewReader(os.Stdin)
 	ch := make(chan logparser.LogEntry)
-	parser := logparser.NewParser(ch, nil, nil, time.Second)
+	parser := logparser.NewParser(ch, nil, nil, time.Second, 256)
 	t := time.Now()
 	for {
 		line, err := reader.ReadString('\n')

go.mod

@@ -1,6 +1,6 @@
 module github.com/coroot/logparser
 
-go 1.21
+go 1.24
 
 require github.com/stretchr/testify v1.8.4
 

parser.go

@@ -6,6 +6,10 @@ import (
 	"time"
 )
 
+var (
+	unclassifiedPatternLabel = "unclassified pattern (pattern limit reached)"
+)
+
 type LogEntry struct {
 	Timestamp time.Time
 	Content   string
@@ -22,8 +26,10 @@ type LogCounter struct {
 type Parser struct {
 	decoder Decoder
 
-	patterns map[patternKey]*patternStat
-	lock     sync.RWMutex
+	patterns              map[patternKey]*patternStat
+	patternsPerLevel      map[Level]int
+	patternsPerLevelLimit int
+	lock                  sync.RWMutex
 
 	multilineCollector *MultilineCollector
 
@@ -34,11 +40,13 @@ type Parser struct {
 
 type OnMsgCallbackF func(ts time.Time, level Level, patternHash string, msg string)
 
-func NewParser(ch <-chan LogEntry, decoder Decoder, onMsgCallback OnMsgCallbackF, multilineCollectorTimeout time.Duration) *Parser {
+func NewParser(ch <-chan LogEntry, decoder Decoder, onMsgCallback OnMsgCallbackF, multilineCollectorTimeout time.Duration, patternsPerLevelLimit int) *Parser {
 	p := &Parser{
-		decoder:  decoder,
-		patterns: map[patternKey]*patternStat{},
-		onMsgCb:  onMsgCallback,
+		decoder:               decoder,
+		patterns:              map[patternKey]*patternStat{},
+		patternsPerLevel:      map[Level]int{},
+		patternsPerLevelLimit: patternsPerLevelLimit,
+		onMsgCb:               onMsgCallback,
 	}
 	ctx, stop := context.WithCancel(context.Background())
 	p.stop = stop
@@ -96,26 +104,43 @@ func (p *Parser) inc(msg Message) {
 	}
 
 	pattern := NewPattern(msg.Content)
-	key := patternKey{level: msg.Level, hash: pattern.Hash()}
-	stat := p.patterns[key]
-	if stat == nil {
-		for k, ps := range p.patterns {
-			if k.level == msg.Level && ps.pattern.WeakEqual(pattern) {
-				stat = ps
-				break
-			}
-		}
-		if stat == nil {
-			stat = &patternStat{pattern: pattern, sample: msg.Content}
-			p.patterns[key] = stat
-		}
-	}
+	stat, key := p.getPatternStat(msg.Level, pattern, msg.Content)
 	if p.onMsgCb != nil {
 		p.onMsgCb(msg.Timestamp, msg.Level, key.hash, msg.Content)
 	}
 	stat.messages++
 }
 
+func (p *Parser) getPatternStat(level Level, pattern *Pattern, sample string) (*patternStat, patternKey) {
+	key := patternKey{level: level, hash: pattern.Hash()}
+	if stat := p.patterns[key]; stat != nil {
+		return stat, key
+	}
+	for k, ps := range p.patterns {
+		if k.level != level || ps.pattern == nil {
+			continue
+		}
+		if ps.pattern.WeakEqual(pattern) {
+			return ps, k
+		}
+	}
+
+	if p.patternsPerLevel[level] >= p.patternsPerLevelLimit {
+		fallbackKey := patternKey{level: level, hash: ""}
+		stat := p.patterns[fallbackKey]
+		if stat == nil {
+			stat = &patternStat{sample: unclassifiedPatternLabel}
+			p.patterns[fallbackKey] = stat
+		}
+		return stat, fallbackKey
+	}
+
+	stat := &patternStat{pattern: pattern, sample: sample}
+	p.patterns[key] = stat
+	p.patternsPerLevel[level]++
+	return stat, key
+}
+
 func (p *Parser) GetCounters() []LogCounter {
 	p.lock.RLock()
 	defer p.lock.RUnlock()

parser_test.go

@@ -0,0 +1,42 @@
+package logparser
+
+import (
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParserCardinalityLimit(t *testing.T) {
+	p := &Parser{
+		patterns:              map[patternKey]*patternStat{},
+		patternsPerLevel:      map[Level]int{},
+		patternsPerLevelLimit: 2,
+	}
+
+	msgs := []string{
+		"error alpha beta gamma",
+		"error delta epsilon zeta",
+		"error eta theta iota",
+	}
+	for _, m := range msgs {
+		p.inc(Message{Timestamp: time.Now(), Content: m, Level: LevelError})
+	}
+	assert.Equal(t, 2, p.patternsPerLevel[LevelError])
+
+	fallbackKey := patternKey{level: LevelError, hash: ""}
+	stat, ok := p.patterns[fallbackKey]
+	require.True(t, ok)
+	assert.Equal(t, 1, stat.messages)
+	assert.Equal(t, unclassifiedPatternLabel, stat.sample)
+
+	counters := p.GetCounters()
+	sort.Slice(counters, func(i, j int) bool { return counters[i].Sample < counters[j].Sample })
+
+	assert.Equal(t, 3, len(counters))
+	assert.Equal(t, msgs[0], counters[0].Sample)
+	assert.Equal(t, msgs[1], counters[1].Sample)
+	assert.Equal(t, unclassifiedPatternLabel, counters[2].Sample)
+}