refactor(rule-engine): Revamp rule engine

Refactor the rule engine code by moving it into a
separate package and segregating the main building
blocks into distinct source files. Along the way,
shift the state machine states from expression names
to simple integer indices representing sequence ids.

Author: rabbitstack

cmd/fibratus/app/rules/validate.go

@@ -24,6 +24,7 @@ import (
 	"github.com/rabbitstack/fibratus/internal/bootstrap"
 	"github.com/rabbitstack/fibratus/pkg/filter"
 	"github.com/rabbitstack/fibratus/pkg/filter/fields"
+	"github.com/rabbitstack/fibratus/pkg/rules"
 	"path/filepath"
 	"strings"
 )
@@ -91,7 +92,7 @@ func validateRules() error {
 		f := filter.New(rule.Condition, cfg)
 		err := f.Compile()
 		if err != nil {
-			return fmt.Errorf("%v %v", emoji.DisappointedFace, filter.ErrInvalidFilter(rule.Name, err))
+			return fmt.Errorf("%v %v", emoji.DisappointedFace, rules.ErrInvalidFilter(rule.Name, err))
 		}
 
 		w := warning{rule: rule.Name}

internal/bootstrap/bootstrap.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rabbitstack/fibratus/pkg/handle"
 	"github.com/rabbitstack/fibratus/pkg/kcap"
 	"github.com/rabbitstack/fibratus/pkg/ps"
+	"github.com/rabbitstack/fibratus/pkg/rules"
 	"github.com/rabbitstack/fibratus/pkg/symbolize"
 	"github.com/rabbitstack/fibratus/pkg/sys"
 	"github.com/rabbitstack/fibratus/pkg/util/multierror"
@@ -52,7 +53,7 @@ type App struct {
 	config     *config.Config
 	evs        *EventSourceControl
 	symbolizer *symbolize.Symbolizer
-	rules      *filter.Rules
+	engine     *rules.Engine
 	hsnap      handle.Snapshotter
 	psnap      ps.Snapshotter
 	filament   filament.Filament
@@ -134,34 +135,34 @@ func NewApp(cfg *config.Config, options ...Option) (*App, error) {
 	hsnap := handle.NewSnapshotter(cfg, opts.handleSnapshotFn)
 	psnap := ps.NewSnapshotter(hsnap, cfg)
 
-	var (
-		rules *filter.Rules
-		res   *config.RulesCompileResult
-	)
+	var engine *rules.Engine
+	var rs *config.RulesCompileResult
+
 	if cfg.Filters.Rules.Enabled && !cfg.ForwardMode && !cfg.IsCaptureSet() {
-		rules = filter.NewRules(psnap, cfg)
+		engine = rules.NewEngine(psnap, cfg)
 		var err error
-		res, err = rules.Compile()
+		rs, err = engine.Compile()
 		if err != nil {
 			return nil, err
 		}
-		if res != nil {
-			log.Infof("rules compile summary: %s", res)
+		if rs != nil {
+			log.Infof("rules compile summary: %s", rs)
 		}
 	} else {
 		log.Info("rule engine is disabled")
 	}
 
-	evs := NewEventSourceControl(psnap, hsnap, cfg, res)
+	evs := NewEventSourceControl(psnap, hsnap, cfg, rs)
 
 	app := &App{
 		config:  cfg,
 		evs:     evs,
-		rules:   rules,
+		engine:  engine,
 		hsnap:   hsnap,
 		psnap:   psnap,
 		signals: sigs,
 	}
+
 	return app, nil
 }
 
@@ -234,8 +235,8 @@ func (f *App) Run(args []string) error {
 			f.evs.RegisterEventListener(f.symbolizer)
 		}
 		// register rule engine
-		if f.rules != nil {
-			f.evs.RegisterEventListener(f.rules)
+		if f.engine != nil {
+			f.evs.RegisterEventListener(f.engine)
 		}
 		// register YARA scanner
 		if cfg.Yara.Enabled {

pkg/filter/_fixtures/sequence_gc.yml

@@ -53,7 +53,7 @@ type Filter interface {
 	// RunSequence runs a filter with sequence expressions. Sequence rules depend
 	// on the state machine transitions and partial matches to decide whether the
 	// rule is fired.
-	RunSequence(evt *kevent.Kevent, seqID uint16, partials map[uint16][]*kevent.Kevent, rawMatch bool) bool
+	RunSequence(evt *kevent.Kevent, seqID int, partials map[int][]*kevent.Kevent, rawMatch bool) bool
 	// GetStringFields returns field names mapped to their string values.
 	GetStringFields() map[fields.Field][]string
 	// GetFields returns all fields used in the filter expression.
@@ -87,7 +87,7 @@ type filter struct {
 	segments    []fields.Segment
 	boundFields []*ql.BoundFieldLiteral
 	// seqBoundFields contains per-sequence bound fields resolved from bound field literals
-	seqBoundFields map[uint16][]BoundField
+	seqBoundFields map[int][]BoundField
 	// stringFields contains filter field names mapped to their string values
 	stringFields map[fields.Field][]string
 	hasFunctions bool
@@ -182,22 +182,22 @@ func (f *filter) Compile() error {
 	return f.checkBoundRefs()
 }
 
-func (f *filter) Run(kevt *kevent.Kevent) bool {
+func (f *filter) Run(e *kevent.Kevent) bool {
 	if f.expr == nil {
 		return false
 	}
-	return ql.Eval(f.expr, f.mapValuer(kevt), f.hasFunctions)
+	return ql.Eval(f.expr, f.mapValuer(e), f.hasFunctions)
 }
 
-func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uint16][]*kevent.Kevent, rawMatch bool) bool {
+func (f *filter) RunSequence(e *kevent.Kevent, seqID int, partials map[int][]*kevent.Kevent, rawMatch bool) bool {
 	if f.seq == nil {
 		return false
 	}
-	nseqs := uint16(len(f.seq.Expressions))
+	nseqs := len(f.seq.Expressions)
 	if seqID > nseqs-1 {
 		return false
 	}
-	valuer := f.mapValuer(kevt)
+	valuer := f.mapValuer(e)
 	expr := f.seq.Expressions[seqID]
 
 	if rawMatch {
@@ -213,12 +213,12 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 		// aliases
 		p := make(map[string][]*kevent.Kevent)
 		nslots := len(partials[seqID])
-		for i := uint16(0); i < seqID; i++ {
+		for i := 0; i < seqID; i++ {
 			alias := f.seq.Expressions[i].Alias
 			if alias == "" {
 				continue
 			}
-			p[alias] = partials[i+1]
+			p[alias] = partials[i]
 			if len(p[alias]) > nslots {
 				nslots = len(p[alias])
 			}
@@ -292,8 +292,8 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 			match = ql.Eval(expr.Expr, valuer, f.hasFunctions)
 			if match {
 				// compute sequence key hash to tie the events
-				evt.AddMeta(kevent.RuleSequenceByKey, hashers.FnvUint64(hash))
-				kevt.AddMeta(kevent.RuleSequenceByKey, hashers.FnvUint64(hash))
+				evt.AddMeta(kevent.RuleSequenceLink, hashers.FnvUint64(hash))
+				e.AddMeta(kevent.RuleSequenceLink, hashers.FnvUint64(hash))
 				break
 			}
 		}
@@ -308,9 +308,9 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 			joins := make([]bool, seqID)
 			joinID := valuer[by.Value]
 		outer:
-			for i := uint16(0); i < seqID; i++ {
-				for _, p := range partials[i+1] {
-					if compareSeqJoin(joinID, p.SequenceBy()) {
+			for i := 0; i < seqID; i++ {
+				for _, p := range partials[i] {
+					if CompareSeqLink(joinID, p.SequenceLink()) {
 						joins[i] = true
 						continue outer
 					}
@@ -323,7 +323,7 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 
 		if match && by != nil {
 			if v := valuer[by.Value]; v != nil {
-				kevt.AddMeta(kevent.RuleSequenceByKey, v)
+				e.AddMeta(kevent.RuleSequenceLink, v)
 			}
 		}
 	}
@@ -456,7 +456,7 @@ func (f *filter) addSegment(segment *ql.BoundSegmentLiteral) {
 // addSeqBoundFields receives the sequence id and the list of bound field literals
 // and populates the list of bound fields containing the field structure convenient
 // for accessors.
-func (f *filter) addSeqBoundFields(seqID uint16, fields []*ql.BoundFieldLiteral) []BoundField {
+func (f *filter) addSeqBoundFields(seqID int, fields []*ql.BoundFieldLiteral) []BoundField {
 	flds := make([]BoundField, 0, len(fields))
 	for _, field := range fields {
 		flds = append(flds,
@@ -497,9 +497,9 @@ func (f *filter) checkBoundRefs() error {
 	return nil
 }
 
-// compareSeqJoin returns true if both values
+// CompareSeqLink returns true if both values
 // representing the sequence joins are equal.
-func compareSeqJoin(s1, s2 any) bool {
+func CompareSeqLink(s1, s2 any) bool {
 	if s1 == nil || s2 == nil {
 		return false
 	}

pkg/filter/_fixtures/sequence_rule_bound_fields.yml

@@ -99,7 +99,7 @@ func New(expr string, config *config.Config, options ...Option) Filter {
 		segments:       make([]fields.Segment, 0),
 		stringFields:   make(map[fields.Field][]string),
 		boundFields:    make([]*ql.BoundFieldLiteral, 0),
-		seqBoundFields: make(map[uint16][]BoundField),
+		seqBoundFields: make(map[int][]BoundField),
 	}
 }
 
@@ -129,7 +129,7 @@ func NewFromCLIWithAllAccessors(args []string) (Filter, error) {
 		segments:       make([]fields.Segment, 0),
 		stringFields:   make(map[fields.Field][]string),
 		boundFields:    make([]*ql.BoundFieldLiteral, 0),
-		seqBoundFields: make(map[uint16][]BoundField),
+		seqBoundFields: make(map[int][]BoundField),
 	}
 	if err := filter.Compile(); err != nil {
 		return nil, fmt.Errorf("bad filter:\n %v", err)