perf(rule-engine): Use bitsets for determining if the event is evaluable by the expression

Author: rabbitstack

pkg/filter/ql/literal.go

@@ -278,10 +278,8 @@ type SequenceExpr struct {
 	// Alias represents the sequence expression alias.
 	Alias string
 
-	emasks event.EventsetMasks
-	cmasks event.CategoryMasks
-
-	types []event.Type
+	bitsets event.BitSets
+	types   []event.Type
 }
 
 func (e *SequenceExpr) init() {
@@ -335,28 +333,47 @@ func (e *SequenceExpr) walk() {
 
 	WalkFunc(e.Expr, walk)
 
+	uniqCats := make(map[event.Category]bool)
+
 	// initialize event type/category buckets for every such field
 	for name, values := range stringFields {
 		for _, v := range values {
 			switch name {
 			case fields.EvtName:
 				for _, typ := range event.NameToTypes(v) {
-					e.emasks.Set(typ)
+					if typ == event.UnknownType {
+						continue
+					}
 					e.types = append(e.types, typ)
+					uniqCats[event.TypeToEventInfo(typ).Category] = true
 				}
 			case fields.EvtCategory:
-				e.cmasks.Set(event.Category(v))
+				e.bitsets.SetCategoryBit(event.Category(v))
 			}
 		}
 	}
+
+	for _, t := range e.types {
+		switch len(uniqCats) {
+		case 0:
+			continue
+		case 1:
+			// happy path can use a single bitmask for all
+			// event types pertaining to the same category
+			e.bitsets.SetBit(event.TypeBitSet, t)
+		default:
+			// use map-backed bitmask for event identifiers
+			e.bitsets.SetBit(event.BitmaskBitSet, t)
+		}
+	}
 }
 
 // IsEvaluable determines if the expression should be evaluated by inspecting
 // the event type filter fields defined in the expression. We permit the expression
-// to be evaluated when the incoming event type or category pertains to the one
+// to be evaluated when the incoming event type, ID, or category pertains to the one
 // defined in the field literal.
 func (e *SequenceExpr) IsEvaluable(evt *event.Event) bool {
-	return e.emasks.Test(evt.Type.GUID(), evt.Type.HookID()) || e.cmasks.Test(evt.Category)
+	return e.bitsets.IsBitSet(evt)
 }
 
 // HasBoundFields determines if this sequence expression references any bound field.

pkg/filter/ql/literal_test.go

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2021-present by Nedim Sabic Sabic
+ * https://www.fibratus.io
+ * All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ql
+
+import (
+	"github.com/rabbitstack/fibratus/pkg/event"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestSequenceExprIsEvaluable(t *testing.T) {
+	var tests = []struct {
+		expr       string
+		evt        *event.Event
+		isEval     bool
+		assertions func(t *testing.T, sexpr *SequenceExpr)
+	}{
+		{"evt.name = 'CreateProcess'", &event.Event{Type: event.CreateProcess, Category: event.Process}, true,
+			func(t *testing.T, sexpr *SequenceExpr) {
+				assert.True(t, sexpr.bitsets.IsTypesInitialized())
+				assert.False(t, sexpr.bitsets.IsBitmaskInitialized())
+				assert.False(t, sexpr.bitsets.IsCategoryInitialized())
+			},
+		},
+		{"evt.name = 'CreateProcess'", &event.Event{Type: event.TerminateProcess, Category: event.Process}, false, nil},
+		{"evt.name = 'CreateProcess' or evt.name = 'TerminateThread'", &event.Event{Type: event.TerminateProcess, Category: event.Process}, false, nil},
+		{"evt.name = 'CreateProcess' or evt.category = 'object'", &event.Event{Type: event.TerminateProcess, Category: event.Process}, false, nil},
+		{"evt.name = 'CreateProcess' or evt.name = 'OpenProcess'", &event.Event{Type: event.OpenProcess, Category: event.Process}, true,
+			func(t *testing.T, sexpr *SequenceExpr) {
+				assert.True(t, sexpr.bitsets.IsTypesInitialized())
+				assert.False(t, sexpr.bitsets.IsBitmaskInitialized())
+				assert.False(t, sexpr.bitsets.IsCategoryInitialized())
+			},
+		},
+		{"evt.name = 'CreateProcess' or evt.name = 'CreateThread'", &event.Event{Type: event.CreateThread, Category: event.Thread}, true,
+			func(t *testing.T, sexpr *SequenceExpr) {
+				assert.False(t, sexpr.bitsets.IsTypesInitialized())
+				assert.True(t, sexpr.bitsets.IsBitmaskInitialized())
+				assert.False(t, sexpr.bitsets.IsCategoryInitialized())
+			},
+		},
+		{"evt.name = 'CreateProcess' or evt.category = 'registry'", &event.Event{Type: event.RegSetValue, Category: event.Registry}, true,
+			func(t *testing.T, sexpr *SequenceExpr) {
+				assert.True(t, sexpr.bitsets.IsTypesInitialized())
+				assert.False(t, sexpr.bitsets.IsBitmaskInitialized())
+				assert.True(t, sexpr.bitsets.IsCategoryInitialized())
+			},
+		},
+		{"evt.name = 'CreateProcess' or evt.name = 'OpenProcess' or evt.category = 'registry'", &event.Event{Type: event.OpenProcess, Category: event.Process}, true,
+			func(t *testing.T, sexpr *SequenceExpr) {
+				assert.True(t, sexpr.bitsets.IsTypesInitialized())
+				assert.False(t, sexpr.bitsets.IsBitmaskInitialized())
+				assert.True(t, sexpr.bitsets.IsCategoryInitialized())
+			},
+		},
+		{"evt.name = 'CreateProcess' or evt.name = 'SetThreadContext' or evt.category = 'registry'", &event.Event{Type: event.CreateProcess, Category: event.Process}, true,
+			func(t *testing.T, sexpr *SequenceExpr) {
+				assert.False(t, sexpr.bitsets.IsTypesInitialized())
+				assert.True(t, sexpr.bitsets.IsBitmaskInitialized())
+				assert.True(t, sexpr.bitsets.IsCategoryInitialized())
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.expr, func(t *testing.T) {
+			p := NewParser(tt.expr)
+			expr, err := p.ParseExpr()
+			require.NoError(t, err)
+
+			sexpr := &SequenceExpr{Expr: expr}
+			sexpr.init()
+			sexpr.walk()
+
+			assert.Equal(t, tt.isEval, sexpr.IsEvaluable(tt.evt))
+			if tt.assertions != nil {
+				tt.assertions(t, sexpr)
+			}
+		})
+	}
+}