perf(rule-engine, eventsource): Event existence checks against bitmask

Author: rabbitstack

internal/etw/consumer.go

@@ -74,22 +74,21 @@ func (c *Consumer) ProcessEvent(ev *etw.EventRecord) error {
 	if c.isClosing {
 		return nil
 	}
-	if event.IsCurrentProcDropped(ev.Header.ProcessID) {
+
+	if !c.config.EventSource.EventExists(ev.ID()) {
+		eventsUnknown.Add(1)
 		return nil
 	}
-	if c.config.EventSource.ExcludeEvent(ev.Header.ProviderID, ev.HookID()) {
-		eventsExcluded.Add(1)
+	if event.IsCurrentProcDropped(ev.Header.ProcessID) {
 		return nil
 	}
-
-	etype := event.NewFromEventRecord(ev)
-	if !etype.Exists() {
-		eventsUnknown.Add(1)
+	if c.config.EventSource.ExcludeEvent(ev.ID()) {
+		eventsExcluded.Add(1)
 		return nil
 	}
 
 	eventsProcessed.Add(1)
-	evt := event.New(c.sequencer.Get(), etype, ev)
+	evt := event.New(c.sequencer.Get(), ev)
 
 	// Dispatch each event to the processor chain.
 	// Processors may further augment the event with

internal/etw/source_test.go

@@ -123,6 +123,7 @@ func TestEventSourceStartTraces(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			tt.cfg.EventSource.Init()
 			evs := NewEventSource(psnap, hsnap, tt.cfg, nil)
 			require.NoError(t, evs.Open(tt.cfg))
 			defer evs.Close()
@@ -193,6 +194,7 @@ func TestEventSourceEnableFlagsDynamically(t *testing.T) {
 		Filters: &config.Filters{},
 	}
 
+	cfg.EventSource.Init()
 	evs := NewEventSource(psnap, hsnap, cfg, r)
 	require.NoError(t, evs.Open(cfg))
 	defer evs.Close()
@@ -277,6 +279,7 @@ func TestEventSourceEnableFlagsDynamicallyWithYaraEnabled(t *testing.T) {
 		},
 	}
 
+	cfg.EventSource.Init()
 	evs := NewEventSource(psnap, hsnap, cfg, r)
 	require.NoError(t, evs.Open(cfg))
 	defer evs.Close()
@@ -328,6 +331,7 @@ func TestEventSourceRundownEvents(t *testing.T) {
 		Filters:     &config.Filters{},
 	}
 
+	cfg.EventSource.Init()
 	evs := NewEventSource(psnap, hsnap, cfg, nil)
 
 	l := &MockListener{}
@@ -741,6 +745,7 @@ func TestEventSourceAllEvents(t *testing.T) {
 		StackEnrichment:      false,
 	}
 
+	evsConfig.Init()
 	cfg := &config.Config{EventSource: evsConfig, Filters: &config.Filters{}}
 	evs := NewEventSource(psnap, hsnap, cfg, nil)
 
@@ -1226,6 +1231,8 @@ func testCallstackEnrichment(t *testing.T, hsnap handle.Snapshotter, psnap ps.Sn
 		FlushTimer:           1,
 	}
 
+	evsConfig.Init()
+
 	cfg := &config.Config{
 		EventSource:              evsConfig,
 		Filters:                  &config.Filters{},

internal/etw/stackext_test.go

@@ -38,6 +38,9 @@ func TestStackExtensions(t *testing.T) {
 			FlushTimer:         time.Millisecond * 2300,
 		},
 	}
+
+	cfg.EventSource.Init()
+
 	exts := NewStackExtensions(cfg.EventSource)
 	assert.Len(t, exts.EventIds(), 0)
 

internal/etw/trace_test.go

@@ -36,6 +36,8 @@ func TestStartTrace(t *testing.T) {
 		},
 	}
 
+	cfg.EventSource.Init()
+
 	trace := NewKernelTrace(cfg)
 	require.NoError(t, trace.Start())
 	require.True(t, trace.IsStarted())

pkg/config/eventsource.go

@@ -23,7 +23,7 @@ package config
 
 import (
 	"github.com/rabbitstack/fibratus/pkg/event"
-	"golang.org/x/sys/windows"
+	"github.com/rabbitstack/fibratus/pkg/util/bitmask"
 	"runtime"
 	"time"
 
@@ -102,7 +102,8 @@ type EventSourceConfig struct {
 	// ExcludedImages are process image names that will be rejected if they generate a kernel event.
 	ExcludedImages []string `json:"blacklist.images" yaml:"blacklist.images"`
 
-	dropMasks event.EventsetMasks
+	dropMasks *bitmask.Bitmask
+	allMasks  *bitmask.Bitmask
 
 	excludedImages map[string]bool
 }
@@ -127,13 +128,21 @@ func (c *EventSourceConfig) initFromViper(v *viper.Viper) {
 	c.ExcludedEvents = v.GetStringSlice(excludedEvents)
 	c.ExcludedImages = v.GetStringSlice(excludedImages)
 
+	c.dropMasks = bitmask.New()
+	c.allMasks = bitmask.New()
+
 	c.excludedImages = make(map[string]bool)
 
 	for _, name := range c.ExcludedEvents {
 		if typ := event.NameToType(name); typ != event.UnknownType {
-			c.dropMasks.Set(typ)
+			c.dropMasks.Set(typ.ID())
 		}
 	}
+
+	for _, typ := range event.AllWithState() {
+		c.allMasks.Set(typ.ID())
+	}
+
 	for _, name := range c.ExcludedImages {
 		c.excludedImages[name] = true
 	}
@@ -142,35 +151,54 @@ func (c *EventSourceConfig) initFromViper(v *viper.Viper) {
 // Init is an exported method to allow initializing exclusion maps from external modules.
 func (c *EventSourceConfig) Init() {
 	c.excludedImages = make(map[string]bool)
+
+	if c.dropMasks == nil {
+		c.dropMasks = bitmask.New()
+	}
 	for _, name := range c.ExcludedEvents {
 		for _, typ := range event.NameToTypes(name) {
 			if typ != event.UnknownType {
-				c.dropMasks.Set(typ)
+				c.dropMasks.Set(typ.ID())
 			}
 		}
 	}
+
 	for _, name := range c.ExcludedImages {
 		c.excludedImages[name] = true
 	}
+
+	if c.allMasks == nil {
+		c.allMasks = bitmask.New()
+	}
+	for _, typ := range event.AllWithState() {
+		c.allMasks.Set(typ.ID())
+	}
 }
 
 // SetDropMask inserts the event mask in the bitset to
 // instruct the given event type should be dropped from
 // the event stream.
-func (c *EventSourceConfig) SetDropMask(Type event.Type) {
-	c.dropMasks.Set(Type)
+func (c *EventSourceConfig) SetDropMask(typ event.Type) {
+	c.dropMasks.Set(typ.ID())
 }
 
 // TestDropMask checks if the specified event type has
 // the drop mask in the bitset.
-func (c *EventSourceConfig) TestDropMask(Type event.Type) bool {
-	return c.dropMasks.Test(Type.GUID(), Type.HookID())
+func (c *EventSourceConfig) TestDropMask(typ event.Type) bool {
+	return c.dropMasks.IsSet(typ.ID())
+}
+
+// ExcludeEvent determines whether the supplied short
+// event ID exists in the bitset of excluded events.
+func (c *EventSourceConfig) ExcludeEvent(id uint) bool {
+	return c.dropMasks.IsSet(id)
 }
 
-// ExcludeEvent determines whether the supplied provider GUID
-// and the hook identifier are in the bitset of excluded events.
-func (c *EventSourceConfig) ExcludeEvent(guid windows.GUID, hookID uint16) bool {
-	return c.dropMasks.Test(guid, hookID)
+// EventExists determines if the provided event ID exists
+// in the internal event catalog by checking the event ID
+// bitmask.
+func (c *EventSourceConfig) EventExists(id uint) bool {
+	return c.allMasks.IsSet(id)
 }
 
 // ExcludeImage determines whether the process generating event is present in the

pkg/config/eventsource_test.go

@@ -55,8 +55,8 @@ func TestEventSourceConfig(t *testing.T) {
 	assert.False(t, c.EventSource.EnableImageEvents)
 	assert.False(t, c.EventSource.EnableFileIOEvents)
 
-	assert.True(t, c.EventSource.ExcludeEvent(event.CloseHandle.GUID(), event.CloseHandle.HookID()))
-	assert.False(t, c.EventSource.ExcludeEvent(event.CreateProcess.GUID(), event.CreateProcess.HookID()))
+	assert.True(t, c.EventSource.ExcludeEvent(event.CloseHandle.ID()))
+	assert.False(t, c.EventSource.ExcludeEvent(event.CreateProcess.ID()))
 
 	assert.True(t, c.EventSource.ExcludeImage(&pstypes.PS{Name: "svchost.exe"}))
 	assert.False(t, c.EventSource.ExcludeImage(&pstypes.PS{Name: "explorer.exe"}))

pkg/event/event_windows.go

@@ -47,13 +47,15 @@ var (
 
 // New constructs a fresh event instance with basic fields and parameters
 // from the raw ETW event record.
-func New(seq uint64, typ Type, evt *etw.EventRecord) *Event {
+func New(seq uint64, evt *etw.EventRecord) *Event {
 	var (
 		pid = evt.Header.ProcessID
 		tid = evt.Header.ThreadID
 		cpu = *(*uint8)(unsafe.Pointer(&evt.BufferContext.ProcessorIndex[0]))
 		ts  = filetime.ToEpoch(evt.Header.Timestamp)
+		typ = NewTypeFromEventRecord(evt)
 	)
+
 	e := &Event{
 		Seq:         seq,
 		PID:         pid,
@@ -68,8 +70,10 @@ func New(seq uint64, typ Type, evt *etw.EventRecord) *Event {
 		Metadata:    make(map[MetadataKey]any),
 		Host:        hostname.Get(),
 	}
+
 	e.produceParams(evt)
 	e.adjustPID()
+
 	return e
 }
 

pkg/event/metainfo_windows.go

@@ -216,12 +216,42 @@ var indexedEvents = []Info{
 // All returns all event types.
 func All() []Type {
 	s := make([]Type, 0, len(types))
-	for _, Type := range types {
-		s = append(s, Type)
+	for _, typ := range types {
+		s = append(s, typ)
 	}
 	return s
 }
 
+// AllWithState returns all event types +
+// event types used for state management.
+func AllWithState() []Type {
+	s := All()
+
+	s = append(s, ProcessRundown)
+	s = append(s, ThreadRundown)
+	s = append(s, ImageRundown)
+	s = append(s, FileRundown)
+	s = append(s, RegKCBRundown)
+	s = append(s, RegCreateKCB)
+	s = append(s, RegDeleteKCB)
+	s = append(s, FileOpEnd)
+	s = append(s, ReleaseFile)
+	s = append(s, MapFileRundown)
+	s = append(s, StackWalk)
+
+	return s
+}
+
+// MaxTypeID returns the maximum event type (hook id) value.
+func MaxTypeID() uint16 {
+	types := AllWithState()
+	ids := make([]uint16, len(types))
+	for i, t := range types {
+		ids[i] = t.HookID()
+	}
+	return slices.Max(ids)
+}
+
 // TypeToEventInfo maps the event type to the structure storing detailed information about the event.
 func TypeToEventInfo(typ Type) Info {
 	if info, ok := events[typ]; ok {

pkg/sys/etw/types.go

@@ -564,6 +564,24 @@ func (e *EventRecord) HookID() uint16 {
 	return e.Header.EventDescriptor.ID
 }
 
+// ID is an unsigned integer that uniquely
+// identifies the event. Handy for bitmask
+// operations.
+func (e *EventRecord) ID() uint {
+	d1 := e.Header.ProviderID.Data1
+	d2 := e.Header.ProviderID.Data2
+
+	id := uint(byte(d1>>24))<<56 |
+		uint(byte(d1>>16))<<48 |
+		uint(byte(d1>>8))<<40 |
+		uint(byte(d1))<<32 |
+		uint(byte(d2>>8))<<24 |
+		uint(byte(d2))<<16 |
+		uint(e.HookID())
+
+	return id
+}
+
 // ReadByte reads the byte from the buffer at the specified offset.
 func (e *EventRecord) ReadByte(offset uint16) byte {
 	if offset > e.BufferLen {

pkg/sys/etw/types_test.go

@@ -98,3 +98,8 @@ func TestReadBuffer(t *testing.T) {
 		tt.assertions(t, ev)
 	}
 }
+
+func TestID(t *testing.T) {
+	ev := &EventRecord{Header: EventHeader{ProviderID: ThreadpoolGUID, EventDescriptor: EventDescriptor{ID: 44}}}
+	assert.Equal(t, uint(14439051552138264620), ev.ID())
+}