wip

Author: rabbitstack

internal/bootstrap/bootstrap.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rabbitstack/fibratus/pkg/handle"
 	"github.com/rabbitstack/fibratus/pkg/kcap"
 	"github.com/rabbitstack/fibratus/pkg/ps"
+	"github.com/rabbitstack/fibratus/pkg/rules"
 	"github.com/rabbitstack/fibratus/pkg/symbolize"
 	"github.com/rabbitstack/fibratus/pkg/sys"
 	"github.com/rabbitstack/fibratus/pkg/util/multierror"
@@ -52,7 +53,7 @@ type App struct {
 	config     *config.Config
 	evs        *EventSourceControl
 	symbolizer *symbolize.Symbolizer
-	rules      *filter.Rules
+	engine     *rules.Engine
 	hsnap      handle.Snapshotter
 	psnap      ps.Snapshotter
 	filament   filament.Filament
@@ -134,34 +135,34 @@ func NewApp(cfg *config.Config, options ...Option) (*App, error) {
 	hsnap := handle.NewSnapshotter(cfg, opts.handleSnapshotFn)
 	psnap := ps.NewSnapshotter(hsnap, cfg)
 
-	var (
-		rules *filter.Rules
-		res   *config.RulesCompileResult
-	)
+	var engine *rules.Engine
+	var rs *config.RulesCompileResult
+
 	if cfg.Filters.Rules.Enabled && !cfg.ForwardMode && !cfg.IsCaptureSet() {
-		rules = filter.NewRules(psnap, cfg)
+		engine = rules.NewEngine(psnap, cfg)
 		var err error
-		res, err = rules.Compile()
+		rs, err = engine.Compile()
 		if err != nil {
 			return nil, err
 		}
-		if res != nil {
-			log.Infof("rules compile summary: %s", res)
+		if rs != nil {
+			log.Infof("rules compile summary: %s", rs)
 		}
 	} else {
 		log.Info("rule engine is disabled")
 	}
 
-	evs := NewEventSourceControl(psnap, hsnap, cfg, res)
+	evs := NewEventSourceControl(psnap, hsnap, cfg, rs)
 
 	app := &App{
 		config:  cfg,
 		evs:     evs,
-		rules:   rules,
+		engine:  engine,
 		hsnap:   hsnap,
 		psnap:   psnap,
 		signals: sigs,
 	}
+
 	return app, nil
 }
 
@@ -234,8 +235,8 @@ func (f *App) Run(args []string) error {
 			f.evs.RegisterEventListener(f.symbolizer)
 		}
 		// register rule engine
-		if f.rules != nil {
-			f.evs.RegisterEventListener(f.rules)
+		if f.engine != nil {
+			f.evs.RegisterEventListener(f.engine)
 		}
 		// register YARA scanner
 		if cfg.Yara.Enabled {

pkg/filter/_fixtures/sequence_rule_bound_fields.yml

@@ -53,7 +53,7 @@ type Filter interface {
 	// RunSequence runs a filter with sequence expressions. Sequence rules depend
 	// on the state machine transitions and partial matches to decide whether the
 	// rule is fired.
-	RunSequence(evt *kevent.Kevent, seqID uint16, partials map[uint16][]*kevent.Kevent, rawMatch bool) bool
+	RunSequence(evt *kevent.Kevent, seqID int, partials map[int][]*kevent.Kevent, rawMatch bool) bool
 	// GetStringFields returns field names mapped to their string values.
 	GetStringFields() map[fields.Field][]string
 	// GetFields returns all fields used in the filter expression.
@@ -87,7 +87,7 @@ type filter struct {
 	segments    []fields.Segment
 	boundFields []*ql.BoundFieldLiteral
 	// seqBoundFields contains per-sequence bound fields resolved from bound field literals
-	seqBoundFields map[uint16][]BoundField
+	seqBoundFields map[int][]BoundField
 	// stringFields contains filter field names mapped to their string values
 	stringFields map[fields.Field][]string
 	hasFunctions bool
@@ -182,22 +182,22 @@ func (f *filter) Compile() error {
 	return f.checkBoundRefs()
 }
 
-func (f *filter) Run(kevt *kevent.Kevent) bool {
+func (f *filter) Run(e *kevent.Kevent) bool {
 	if f.expr == nil {
 		return false
 	}
-	return ql.Eval(f.expr, f.mapValuer(kevt), f.hasFunctions)
+	return ql.Eval(f.expr, f.mapValuer(e), f.hasFunctions)
 }
 
-func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uint16][]*kevent.Kevent, rawMatch bool) bool {
+func (f *filter) RunSequence(e *kevent.Kevent, seqID int, partials map[int][]*kevent.Kevent, rawMatch bool) bool {
 	if f.seq == nil {
 		return false
 	}
-	nseqs := uint16(len(f.seq.Expressions))
+	nseqs := len(f.seq.Expressions)
 	if seqID > nseqs-1 {
 		return false
 	}
-	valuer := f.mapValuer(kevt)
+	valuer := f.mapValuer(e)
 	expr := f.seq.Expressions[seqID]
 
 	if rawMatch {
@@ -213,12 +213,12 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 		// aliases
 		p := make(map[string][]*kevent.Kevent)
 		nslots := len(partials[seqID])
-		for i := uint16(0); i < seqID; i++ {
+		for i := 0; i < seqID; i++ {
 			alias := f.seq.Expressions[i].Alias
 			if alias == "" {
 				continue
 			}
-			p[alias] = partials[i+1]
+			p[alias] = partials[i]
 			if len(p[alias]) > nslots {
 				nslots = len(p[alias])
 			}
@@ -292,8 +292,8 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 			match = ql.Eval(expr.Expr, valuer, f.hasFunctions)
 			if match {
 				// compute sequence key hash to tie the events
-				evt.AddMeta(kevent.RuleSequenceByKey, hashers.FnvUint64(hash))
-				kevt.AddMeta(kevent.RuleSequenceByKey, hashers.FnvUint64(hash))
+				evt.AddMeta(kevent.RuleSequenceLink, hashers.FnvUint64(hash))
+				e.AddMeta(kevent.RuleSequenceLink, hashers.FnvUint64(hash))
 				break
 			}
 		}
@@ -308,9 +308,9 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 			joins := make([]bool, seqID)
 			joinID := valuer[by.Value]
 		outer:
-			for i := uint16(0); i < seqID; i++ {
-				for _, p := range partials[i+1] {
-					if compareSeqJoin(joinID, p.SequenceBy()) {
+			for i := 0; i < seqID; i++ {
+				for _, p := range partials[i] {
+					if CompareSeqLink(joinID, p.SequenceLink()) {
 						joins[i] = true
 						continue outer
 					}
@@ -323,7 +323,7 @@ func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uin
 
 		if match && by != nil {
 			if v := valuer[by.Value]; v != nil {
-				kevt.AddMeta(kevent.RuleSequenceByKey, v)
+				e.AddMeta(kevent.RuleSequenceLink, v)
 			}
 		}
 	}
@@ -456,7 +456,7 @@ func (f *filter) addSegment(segment *ql.BoundSegmentLiteral) {
 // addSeqBoundFields receives the sequence id and the list of bound field literals
 // and populates the list of bound fields containing the field structure convenient
 // for accessors.
-func (f *filter) addSeqBoundFields(seqID uint16, fields []*ql.BoundFieldLiteral) []BoundField {
+func (f *filter) addSeqBoundFields(seqID int, fields []*ql.BoundFieldLiteral) []BoundField {
 	flds := make([]BoundField, 0, len(fields))
 	for _, field := range fields {
 		flds = append(flds,
@@ -497,9 +497,9 @@ func (f *filter) checkBoundRefs() error {
 	return nil
 }
 
-// compareSeqJoin returns true if both values
+// CompareSeqLink returns true if both values
 // representing the sequence joins are equal.
-func compareSeqJoin(s1, s2 any) bool {
+func CompareSeqLink(s1, s2 any) bool {
 	if s1 == nil || s2 == nil {
 		return false
 	}

pkg/filter/_fixtures/sequence_rule_bound_fields_with_functions.yml

@@ -99,7 +99,7 @@ func New(expr string, config *config.Config, options ...Option) Filter {
 		segments:       make([]fields.Segment, 0),
 		stringFields:   make(map[fields.Field][]string),
 		boundFields:    make([]*ql.BoundFieldLiteral, 0),
-		seqBoundFields: make(map[uint16][]BoundField),
+		seqBoundFields: make(map[int][]BoundField),
 	}
 }
 
@@ -129,7 +129,7 @@ func NewFromCLIWithAllAccessors(args []string) (Filter, error) {
 		segments:       make([]fields.Segment, 0),
 		stringFields:   make(map[fields.Field][]string),
 		boundFields:    make([]*ql.BoundFieldLiteral, 0),
-		seqBoundFields: make(map[uint16][]BoundField),
+		seqBoundFields: make(map[int][]BoundField),
 	}
 	if err := filter.Compile(); err != nil {
 		return nil, fmt.Errorf("bad filter:\n %v", err)

pkg/filter/filter.go

@@ -47,13 +47,13 @@ const (
 )
 
 var (
-	filterMatches = expvar.NewMap("filter.matches")
-	filtersCount  = expvar.NewInt("filter.filters.count")
+	filterMatches = expvar.NewMap("filter.matches1")
+	filtersCount  = expvar.NewInt("filter.filters.count1")
 
-	matchTransitionErrors = expvar.NewInt("sequence.match.transition.errors")
-	partialsPerSequence   = expvar.NewMap("sequence.partials.count")
-	partialExpirations    = expvar.NewMap("sequence.partial.expirations")
-	partialBreaches       = expvar.NewMap("sequence.partial.breaches")
+	matchTransitionErrors = expvar.NewInt("sequence.match.transition.errors1")
+	partialsPerSequence   = expvar.NewMap("sequence.partials.count1")
+	partialExpirations    = expvar.NewMap("sequence.partial.expirations1")
+	partialBreaches       = expvar.NewMap("sequence.partial.breaches1")
 
 	ErrInvalidFilter = func(rule string, err error) error {
 		return fmt.Errorf("syntax error in rule %q: \n%v", rule, err)
@@ -283,8 +283,8 @@ func (s *sequenceState) addPartial(rule string, kevt *kevent.Kevent, outOfOrder
 		}
 	}
 	if outOfOrder {
-		kevt.AddMeta(kevent.RuleExpressionKey, rule)
-		kevt.AddMeta(kevent.RuleSequenceOutOfOrderKey, true)
+		kevt.AddMeta(kevent.RuleSequenceID, rule)
+		kevt.AddMeta(kevent.RuleSequenceOOOKey, true)
 	}
 	log.Debugf("adding partial to slot [%d] for rule %q: %s", i, rule, kevt)
 	partialsPerSequence.Add(s.name, 1)
@@ -464,7 +464,7 @@ func (f compiledFilter) run(kevt *kevent.Kevent, i int, rawMatch, lock bool) boo
 			f.ss.mu.RLock()
 			defer f.ss.mu.RUnlock()
 		}
-		return f.filter.RunSequence(kevt, uint16(i), f.ss.partials, rawMatch)
+		return f.filter.RunSequence(kevt, i, nil, rawMatch)
 	}
 	return f.filter.Run(kevt)
 }
@@ -790,7 +790,7 @@ func (r *Rules) runSequence(kevt *kevent.Kevent, f *compiledFilter) bool {
 		for i := uint16(1); i < nseqs+1; i++ {
 			for _, outer := range f.ss.partials[i] {
 				for _, inner := range f.ss.partials[i+1] {
-					if compareSeqJoin(outer.SequenceBy(), inner.SequenceBy()) {
+					if CompareSeqLink(outer.SequenceLink(), inner.SequenceLink()) {
 						setMatch(i, outer)
 						setMatch(i+1, inner)
 					}
@@ -806,19 +806,19 @@ func (r *Rules) matchUnorderedPartials(f *compiledFilter) {
 	defer f.ss.mu.Unlock()
 	for n, partials := range f.ss.partials {
 		for _, partial := range partials {
-			if !partial.ContainsMeta(kevent.RuleSequenceOutOfOrderKey) {
+			if !partial.ContainsMeta(kevent.RuleSequenceOOOKey) {
 				continue
 			}
 			matches := f.run(partial, int(n)-1, false, false)
-			rule := partial.GetMetaAsString(kevent.RuleExpressionKey)
+			rule := partial.GetMetaAsString(kevent.RuleSequenceID)
 			// transition the state machine
 			if matches {
 				err := f.ss.matchTransition(rule, partial)
 				if err != nil {
 					matchTransitionErrors.Add(1)
 					log.Warnf("out of order match transition failure: %v", err)
 				}
-				partial.RemoveMeta(kevent.RuleSequenceOutOfOrderKey)
+				partial.RemoveMeta(kevent.RuleSequenceOOOKey)
 			}
 		}
 	}

pkg/filter/filter_windows.go

@@ -793,7 +793,7 @@ func TestSequenceOutOfOrder(t *testing.T) {
 
 	require.False(t, wrapProcessEvent(e2, rules.ProcessEvent))
 	assert.Len(t, ss.partials[2], 1)
-	assert.True(t, ss.partials[2][0].ContainsMeta(kevent.RuleSequenceOutOfOrderKey))
+	assert.True(t, ss.partials[2][0].ContainsMeta(kevent.RuleSequenceOOOKey))
 
 	require.True(t, wrapProcessEvent(e1, rules.ProcessEvent))
 }

pkg/filter/rules.go

@@ -44,15 +44,13 @@ const (
 	YaraMatchesKey MetadataKey = "yara.matches"
 	// RuleNameKey identifies the rule that was triggered by the event
 	RuleNameKey MetadataKey = "rule.name"
-	// RuleGroupKey identifies the group to which the triggered rule pertains
-	RuleGroupKey MetadataKey = "rule.group"
-	// RuleSequenceByKey represents the join field value in sequence rules
-	RuleSequenceByKey MetadataKey = "rule.seq.by"
-	// RuleExpressionKey represents the rule filter expression
-	RuleExpressionKey MetadataKey = "rule.expr"
-	// RuleSequenceOutOfOrderKey the presence of this metadata key indicates the
+	// RuleSequenceLink represents the join link value in sequence rules
+	RuleSequenceLink MetadataKey = "rule.seq.link"
+	// RuleSequenceID represents the sequence id tied to the expression
+	RuleSequenceID MetadataKey = "rule.seq.id"
+	// RuleSequenceOOOKey the presence of this metadata key indicates the
 	// event in the partials list arrived out of order and requires reevaluation
-	RuleSequenceOutOfOrderKey MetadataKey = "rule.seq.outoforder"
+	RuleSequenceOOOKey MetadataKey = "rule.seq.ooo"
 )
 
 func (key MetadataKey) String() string { return string(key) }
@@ -267,6 +265,18 @@ func (e *Kevent) GetMetaAsString(k MetadataKey) string {
 	return ""
 }
 
+// GetMetaAsInt returns the metadata as integer value.
+func (e *Kevent) GetMetaAsInt(k MetadataKey) int {
+	e.mmux.RLock()
+	defer e.mmux.RUnlock()
+	if v, ok := e.Metadata[k]; ok {
+		if n, ok := v.(int); ok {
+			return n
+		}
+	}
+	return 0
+}
+
 // ContainsMeta returns true if the metadata contains the specified key.
 func (e *Kevent) ContainsMeta(k MetadataKey) bool {
 	e.mmux.RLock()
@@ -308,9 +318,9 @@ func (e *Kevent) GetFlagsAsSlice(name string) []string {
 	return strings.Split(e.GetParamAsString(name), "|")
 }
 
-// SequenceBy returns the BY statement join field from event metadata.
-func (e *Kevent) SequenceBy() any {
+// SequenceLink returns the sequence link value from event metadata.
+func (e *Kevent) SequenceLink() any {
 	e.mmux.RLock()
 	defer e.mmux.RUnlock()
-	return e.Metadata[RuleSequenceByKey]
+	return e.Metadata[RuleSequenceLink]
 }