feat(rule-engine): Match all strategy

The match all rule engine strategy permits
a single event to trigger multiple rules.

Author: rabbitstack

configs/fibratus.yml

@@ -131,6 +131,10 @@ filament:
 # For local file system rule paths, it is possible to use the glob expression to load the
 # rules from different directory locations.
 filters:
+  # Indicates if the rule engine match all strategy is enabled. When the match all strategy
+  # is enabled, a single event can trigger multiple rules.
+  match-all: true
+
   rules:
     # Indicates if the rule engine is enabled and rules loaded
     enabled: true

pkg/config/config_windows.go

@@ -380,6 +380,7 @@ func (c *Config) addFlags() {
 		c.flags.StringSlice(rulesFromPaths, []string{filepath.Join(dir, "*")}, "Comma-separated list of rules files")
 		c.flags.StringSlice(macrosFromPaths, []string{filepath.Join(dir, "Macros", "*")}, "Comma-separated list of macro files")
 		c.flags.StringSlice(rulesFromURLs, []string{}, "Comma-separated list of rules URL resources")
+		c.flags.Bool(matchAll, true, "Indicates if the match all strategy is enabled for the rule engine. If the match all strategy is enabled, a single event can trigger multiple rules")
 	}
 	if c.opts.capture {
 		c.flags.StringP(kcapFile, "o", "", "The path of the output kcap file")

pkg/config/filters.go

@@ -102,10 +102,13 @@ func (f FilterConfig) HasLabel(l string) bool { return f.Labels[l] != "" }
 
 // Filters contains references to rule and macro definitions.
 type Filters struct {
-	Rules   Rules  `json:"rules" yaml:"rules"`
-	Macros  Macros `json:"macros" yaml:"macros"`
-	macros  map[string]*Macro
-	filters []*FilterConfig
+	Rules  Rules  `json:"rules" yaml:"rules"`
+	Macros Macros `json:"macros" yaml:"macros"`
+	// MatchAll indicates if the match all strategy is enabled for the rule engine.
+	// If the match all strategy is enabled, a single event can trigger multiple rules.
+	MatchAll bool `json:"match-all" yaml:"match-all"`
+	macros   map[string]*Macro
+	filters  []*FilterConfig
 }
 
 // FiltersWithMacros builds the filter config with the map of
@@ -241,13 +244,15 @@ const (
 	rulesFromPaths  = "filters.rules.from-paths"
 	rulesFromURLs   = "filters.rules.from-urls"
 	macrosFromPaths = "filters.macros.from-paths"
+	matchAll        = "filters.match-all"
 )
 
 func (f *Filters) initFromViper(v *viper.Viper) {
 	f.Rules.Enabled = v.GetBool(rulesEnabled)
 	f.Rules.FromPaths = v.GetStringSlice(rulesFromPaths)
 	f.Rules.FromURLs = v.GetStringSlice(rulesFromURLs)
 	f.Macros.FromPaths = v.GetStringSlice(macrosFromPaths)
+	f.MatchAll = v.GetBool(matchAll)
 }
 
 func (f Filters) HasMacros() bool           { return len(f.macros) > 0 }

pkg/config/filters_test.go

@@ -37,6 +37,7 @@ func TestLoadRulesFromPaths(t *testing.T) {
 			},
 		},
 		Macros{FromPaths: nil},
+		false,
 		map[string]*Macro{},
 		[]*FilterConfig{},
 	}
@@ -78,6 +79,7 @@ func TestLoadRulesFromPathsWithTemplate(t *testing.T) {
 			},
 		},
 		Macros{FromPaths: nil},
+		false,
 		map[string]*Macro{},
 		[]*FilterConfig{},
 	}
@@ -116,6 +118,7 @@ func TestLoadGroupsFromURLs(t *testing.T) {
 			},
 		},
 		Macros{FromPaths: nil},
+		false,
 		map[string]*Macro{},
 		[]*FilterConfig{},
 	}

pkg/config/schema_windows.go

@@ -146,6 +146,7 @@ var schema = `
 		"filters": {
 			"type": "object",
 			"properties": {
+				"match-all": {"type": "boolean"},
 				"rules": {
 					"type": "object",
 					"properties": {
@@ -510,7 +511,7 @@ var rulesSchema = `
 		"id": 					{"type": "string", "minLength": 36, "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"},
 		"version": 				{"type": "string", "minLength": 5, "pattern": "^([0-9]+.)([0-9]+.)([0-9]+)$"},
 		"name": 				{"type": "string", "minLength": 3},
-        "description":  		{"type": "string"},
+		"description":  		{"type": "string"},
 		"output": 				{"type": "string", "minLength": 5},
 		"notes": 				{"type": "string"},
 		"severity":  			{"type": "string", "enum": ["low", "medium", "high", "critical"]},

pkg/rules/engine.go

@@ -54,7 +54,7 @@ var (
 // the collection of compiled filters that are derived
 // from the loaded ruleset.
 type Engine struct {
-	filters map[uint32][]*compiledFilter
+	filters compiledFilters
 	config  *config.Config
 	psnap   ps.Snapshotter
 
@@ -120,6 +120,29 @@ type compiledFilter struct {
 	ss     *sequenceState
 }
 
+type compiledFilters map[uint32][]*compiledFilter
+
+// collect collects all compiled filters for a
+// particular event type or category. If no filters
+// are found, the event is not asserted against the
+// ruleset.
+func (filters compiledFilters) collect(hashCache *hashCache, e *kevent.Kevent) []*compiledFilter {
+	h := hashCache.typeHash(e)
+	if h == 0 {
+		h = hashCache.addTypeHash(e)
+	}
+
+	if !hashCache.lookupCategory {
+		return filters[h]
+	}
+
+	c := hashCache.categoryHash(e)
+	if c == 0 {
+		c = hashCache.addCategoryHash(e)
+	}
+	return append(filters[h], filters[c]...)
+}
+
 func newCompiledFilter(f filter.Filter, c *config.FilterConfig, ss *sequenceState) *compiledFilter {
 	return &compiledFilter{filter: f, config: c, ss: ss}
 }
@@ -231,13 +254,14 @@ func (e *Engine) RegisterMatchFunc(fn RuleMatchFunc) {
 func (*Engine) CanEnqueue() bool { return true }
 
 // ProcessEvent processes the system event against compiled filters.
-// Filter is the internal lingo to designate the rule condition. The
-// filters can be simple direct-event matchers or sequence states that
+// Filter is the internal lingo that designates a rule condition.
+// Filters can be simple direct-event matchers or sequence states that
 // track an ordered series of events over a short period of time.
 func (e *Engine) ProcessEvent(evt *kevent.Kevent) (bool, error) {
 	if len(e.filters) == 0 {
 		return true, nil
 	}
+	var matches bool
 	if evt.IsTerminateProcess() {
 		// expire all sequences if the
 		// process referenced in any
@@ -246,45 +270,30 @@ func (e *Engine) ProcessEvent(evt *kevent.Kevent) (bool, error) {
 			seq.expire(evt)
 		}
 	}
-	filters := e.findFilters(evt)
+	filters := e.filters.collect(e.hashCache, evt)
 	for _, f := range filters {
 		match := f.run(evt)
-		if match {
-			if f.isSequence() {
-				e.appendMatch(f.config, f.ss.events()...)
-				f.ss.clearLocked()
-			} else {
-				e.appendMatch(f.config, evt)
-			}
-			err := e.processActions()
-			if err != nil {
-				log.Errorf("unable to execute rule action: %v", err)
-			}
-			return true, nil
+		if !match {
+			continue
 		}
-	}
-	return false, nil
-}
-
-// findFilters collects all compiled filters for a
-// particular event type or category. If no filters
-// are found, the event is not asserted against the
-// ruleset.
-func (e *Engine) findFilters(evt *kevent.Kevent) []*compiledFilter {
-	h := e.hashCache.typeHash(evt)
-	if h == 0 {
-		h = e.hashCache.addTypeHash(evt)
-	}
-	switch {
-	case !e.hashCache.lookupCategory:
-		return e.filters[h]
-	default:
-		c := e.hashCache.categoryHash(evt)
-		if c == 0 {
-			c = e.hashCache.addCategoryHash(evt)
+		if f.isSequence() {
+			e.appendMatch(f.config, f.ss.events()...)
+			f.ss.clearLocked()
+		} else {
+			e.appendMatch(f.config, evt)
+		}
+		err := e.processActions()
+		if err != nil {
+			log.Errorf("unable to execute rule action: %v", err)
+		}
+		switch {
+		case e.config.Filters.MatchAll:
+			matches = true
+		default:
+			return true, nil
 		}
-		return append(e.filters[h], e.filters[c]...)
 	}
+	return matches, nil
 }
 
 // processActions executes rule actions
@@ -310,6 +319,7 @@ func (e *Engine) processActions() error {
 		if err != nil {
 			return err
 		}
+
 		for _, act := range actions {
 			switch act.(type) {
 			case config.KillAction:

pkg/rules/engine_test.go

@@ -158,7 +158,7 @@ func TestCompileIndexableFilters(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.evt.Type.String(), func(t *testing.T) {
-			assert.Len(t, e.findFilters(tt.evt), tt.wants)
+			assert.Len(t, e.filters.collect(e.hashCache, tt.evt), tt.wants)
 		})
 	}
 
@@ -333,8 +333,9 @@ func TestRunSimpleAndSequenceRules(t *testing.T) {
 	log.SetLevel(log.DebugLevel)
 
 	expectedMatches := make(map[string][]uint64)
-
-	e := NewEngine(new(ps.SnapshotterMock), newConfig("_fixtures/simple_and_sequence_rules/*.yml"))
+	c := newConfig("_fixtures/simple_and_sequence_rules/*.yml")
+	c.Filters.MatchAll = true
+	e := NewEngine(new(ps.SnapshotterMock), c)
 	e.RegisterMatchFunc(func(f *config.FilterConfig, evts ...*kevent.Kevent) {
 		ids := make([]uint64, 0)
 		for _, evt := range evts {