feat(yara): Revamp Yara scanner

The Yara scanner is revamped to perform file and
memory scanning triggered by multiple signals. Aside
from the basic process creation and image loading, the
scan is initiated when the PE file is dropped in the file
system, or when the ADS (Alternate Data Stream) is created.
Memory scan is triggered under suspicious memory allocation or section mapping. Lastly, when the
registry binary value is set, the scan is also performed
on the binary blob.

Author: rabbitstack

pkg/yara/_fixtures/yara-test.dll internal/etw/_fixtures/yara-test.dllpkg/yara/_fixtures/yara-test.dll renamed to internal/etw/_fixtures/yara-test.dll

@@ -256,7 +256,6 @@ func TestEventSourceEnableFlagsDynamicallyWithYaraEnabled(t *testing.T) {
 		HasNetworkEvents:  true,
 		HasFileEvents:     false,
 		HasThreadEvents:   false,
-		HasVAMapEvents:    true,
 		HasAuditAPIEvents: true,
 		UsedEvents: []ktypes.Ktype{
 			ktypes.CreateProcess,
@@ -275,11 +274,15 @@ func TestEventSourceEnableFlagsDynamicallyWithYaraEnabled(t *testing.T) {
 			EnableImageKevents:    true,
 			EnableFileIOKevents:   true,
 			EnableAuditAPIEvents:  true,
+			EnableVAMapKevents:    false,
+			EnableMemKevents:      true,
 		},
 		Filters: &config.Filters{},
 		Yara: yara.Config{
-			Enabled:   true,
-			SkipFiles: false,
+			Enabled:    true,
+			SkipFiles:  false,
+			SkipMmaps:  true,
+			SkipAllocs: false,
 		},
 	}
 
@@ -292,8 +295,15 @@ func TestEventSourceEnableFlagsDynamicallyWithYaraEnabled(t *testing.T) {
 	// rules compile result doesn't have file events
 	// but Yara file scanning is enabled
 	require.True(t, flags&etw.FileIO != 0)
+	// VAMap events are not in the ruleset and VaMap is disabled
+	require.False(t, flags&etw.VaMap != 0)
+	// VirtualAlloc is not present in the ruleset, but Yara
+	// alloc scanning is enabled
+	require.True(t, flags&etw.VirtualAlloc != 0)
 
 	require.False(t, cfg.Kstream.TestDropMask(ktypes.CreateFile))
+	require.True(t, cfg.Kstream.TestDropMask(ktypes.MapViewFile))
+	require.False(t, cfg.Kstream.TestDropMask(ktypes.VirtualAlloc))
 }
 
 func TestEventSourceRundownEvents(t *testing.T) {
@@ -486,7 +496,7 @@ func TestEventSourceAllEvents(t *testing.T) {
 				var sec windows.Handle
 				var offset uintptr
 				var baseViewAddr uintptr
-				dll := "../../pkg/yara/_fixtures/yara-test.dll"
+				dll := "_fixtures/yara-test.dll"
 				f, err := os.Open(dll)
 				if err != nil {
 					return err
@@ -529,7 +539,7 @@ func TestEventSourceAllEvents(t *testing.T) {
 				return e.CurrentPid() && e.Type == ktypes.MapViewFile &&
 					e.GetParamAsString(kparams.MemProtect) == "EXECUTE_READWRITE|READONLY" &&
 					e.GetParamAsString(kparams.FileViewSectionType) == "IMAGE" &&
-					strings.Contains(e.GetParamAsString(kparams.FileName), "pkg\\yara\\_fixtures\\yara-test.dll")
+					strings.Contains(e.GetParamAsString(kparams.FileName), "_fixtures\\yara-test.dll")
 			},
 			false,
 		},

internal/etw/source_test.go

@@ -223,13 +223,13 @@ yara:
     strings:
       - string: "rule test : tag1 { meta: author = \"Hilko Bengen\" strings: $a = \"abc\" fullword condition: $a }"
         namespace: default
-  alert-via: slack
-  alert-template:
-    title: ""
-    text: ""
+  alert-template: ""
   fastscan: true
   scan-timeout: 20s
   skip-files: true
+  skip-mmaps: false
+  skip-allocs: false
+  skip-registry: false
   excluded-files:
     - kernel32.dll
   excluded-procs:

pkg/config/_fixtures/fibratus.yml

@@ -484,17 +484,12 @@ var schema = `
 						"additionalProperties": false
 					}]
 				},
-				"alert-via":		{"type": "string", "enum": ["slack", "mail", "systray"]},
-				"alert-template":   {
-						"type": 		"object",
-						"properties": {
-							"text":	 	{"type": "string"},
-							"title": 	{"type": "string"}
-						},
-						"additionalProperties": false
-				},
+				"alert-template":	{"type": "string"},
 				"fastscan":			{"type": "boolean"},
 				"skip-files":		{"type": "boolean"},
+				"skip-allocs":		{"type": "boolean"},
+				"skip-mmaps":		{"type": "boolean"},
+				"skip-registry":	{"type": "boolean"},
 				"scan-timeout":		{"type": "string", "minLength": 2, "pattern": "[0-9]+s"},
 				"excluded-files":	{"type": "array", "items": [{"type": "string", "minLength": 1}]},
 				"excluded-procs":	{"type": "array", "items": [{"type": "string", "minLength": 1}]}

pkg/config/schema_windows.go

@@ -211,17 +211,24 @@ var MemProtectionFlags = []ParamFlag{
 	{"WRITECOMBINE", windows.PAGE_WRITECOMBINE},
 }
 
+const (
+	// SectionRX designates section read/execute protection
+	SectionRX = 0x30000
+	// SectionRWX designates section read/write/execute protection
+	SectionRWX = 0x60000
+)
+
 // ViewProtectionFlags describes section protection flags. These
 // have different values than the memory protection flags as they
 // are reported by the kernel.
 var ViewProtectionFlags = []ParamFlag{
-	{"EXECUTE_READWRITE", 0x60000},
+	{"EXECUTE_READWRITE", SectionRWX},
 	{"EXECUTE_WRITECOPY", 0x70000},
 	{"NOCACHE", 0x80000},
 	{"WRITECOMBINE", 0x90000},
 	{"READONLY", 0x10000},
 	{"EXECUTE", 0x20000},
-	{"EXECUTE_READ", 0x30000},
+	{"EXECUTE_READ", SectionRX},
 	{"READWRITE", 0x40000},
 	{"WRITECOPY", 0x50000},
 }

pkg/kevent/flags.go

@@ -1,21 +1,12 @@
-rule Notepad : notepad
+rule Np : T1 T2
 {
 	meta:
-		severity = "Normal"
+		severity = 50
 		date = "2016-07"
+		threat_name = "Notepad.Shell"
+		id = "babf9101-1e6e-4268-a530-e99e2c905b0d"
 	strings:
-		$c0 = "Notepad" fullword ascii
+		$c1 = "Notepad" fullword ascii
 	condition:
-		$c0
+		$c1
 }
-
-rule NotepadCompany
-{
-	meta:
-		severity = "Normal"
-		date = "2016-07"
-	strings:
-		$c0 = "Microsoft" fullword ascii
-	condition:
-		$c0
-}

pkg/yara/_fixtures/rules/dll.yar

@@ -0,0 +1,12 @@
+rule Regedit : T1
+{
+	meta:
+		severity = 50
+		date = "2016-07"
+		threat_name = "Regedit"
+		id = "1abf9101-1e6e-4268-a530-e99e2c905b0d"
+	strings:
+		$c1 = "Regedit" nocase fullword ascii
+	condition:
+		$c1
+}

pkg/yara/_fixtures/rules/notepad.yar

@@ -19,24 +19,37 @@
 package config
 
 import (
+	"bytes"
+	"fmt"
 	"github.com/mitchellh/mapstructure"
+	"github.com/rabbitstack/fibratus/pkg/kevent"
+	"github.com/rabbitstack/fibratus/pkg/kevent/kparams"
+	"github.com/rabbitstack/fibratus/pkg/kevent/ktypes"
+	"github.com/rabbitstack/fibratus/pkg/util/wildcard"
+	ytypes "github.com/rabbitstack/fibratus/pkg/yara/types"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
-	"path/filepath"
 	"strings"
+	"text/template"
 	"time"
 )
 
 const (
-	enabled            = "yara.enabled"
-	alertVia           = "yara.alert-via"
-	alertTextTemplate  = "yara.alert-template.text"
-	alertTitleTemplate = "yara.alert-template.title"
-	fastScanMode       = "yara.fastscan"
-	scanTimeout        = "yara.scan-timeout"
-	skipFiles          = "yara.skip-files"
-	excludedProcesses  = "yara.excluded-procs"
-	excludedFiles      = "yara.excluded-files"
+	enabled           = "yara.enabled"
+	alertTemplate     = "yara.alert-template"
+	fastScanMode      = "yara.fastscan"
+	scanTimeout       = "yara.scan-timeout"
+	skipFiles         = "yara.skip-files"
+	skipAllocs        = "yara.skip-allocs"
+	skipMmaps         = "yara.skip-mmaps"
+	skipRegistry      = "yara.skip-registry"
+	excludedProcesses = "yara.excluded-procs"
+	excludedFiles     = "yara.excluded-files"
+)
+
+const (
+	FileThreatAlertTitle   = "File Threat Detected"
+	MemoryThreatAlertTitle = "Memory Threat Detected"
 )
 
 // RulePath contains the rule path information.
@@ -59,39 +72,44 @@ type Rule struct {
 	Strings []RuleString `json:"yara.rule.strings" yaml:"yara.rule.strings" mapstructure:"strings"`
 }
 
-// Config stores YARA watcher specific configuration.
+// Config stores YARA scanner specific configuration.
 type Config struct {
 	// Enabled indicates if YARA watcher is enabled.
 	Enabled bool `json:"yara.enabled" yaml:"yara.enabled"`
 	// Rule contains rule-specific settings.
 	Rule Rule `json:"yara.rule" yaml:"yara.rule" mapstructure:"rule"`
-	// AlertVia defines which alert sender is used to emit the alert on rule matches.
-	AlertVia string `json:"yara.alert-via" yaml:"yara.alert-via"`
-	// AlertTemplate defines the template that is used to render the text of the alert.
-	AlertTextTemplate string `json:"yara.alert-text-template" yaml:"yara.alert-text-template"`
-	// AlertTitle represents the template for the alert title
-	AlertTitleTemplate string `json:"yara.alert-title-template" yaml:"yara.alert-title-template"`
+	// AlertTemplate represents the template for the alert title
+	AlertTemplate string `json:"yara.alert-template" yaml:"yara.alert-template"`
 	// FastScanMode avoids multiple matches of the same string when not necessary.
 	FastScanMode bool `json:"yara.fastscan" yaml:"yara.fastscan"`
 	// ScanTimeout sets the timeout for the scanner. If the timeout is reached, the scan operation is cancelled.
 	ScanTimeout time.Duration `json:"yara.scan-timeout" yaml:"yara.scan-timeout"`
-	// SkipFiles indicates whether file scanning is disabled
+	// SkipFiles indicates whether file scanning is disabled.
 	SkipFiles bool `json:"yara.skip-files" yaml:"yara.skip-files"`
-	// ExcludedProcesses contains the list of the process' image names that shouldn't be scanned
+	// SkipAllocs indicates whether scanning on suspicious memory allocations is disabled.
+	SkipAllocs bool `json:"yara.skip-allocs" yaml:"yara.skip-allocs"`
+	// SkipMmaps indicates whether scanning on suspicious mappings of sections is disabled.
+	SkipMmaps bool `json:"yara.skip-mmaps" yaml:"yara.skip-mmaps"`
+	// SkipRegistry indicates whether registry value scanning is disabled.
+	SkipRegistry bool `json:"yara.skip-registry" yaml:"yara.skip-registry"`
+	// ExcludedProcesses contains the list of the comma-separated process image paths that shouldn't be scanned.
+	// Wildcard matching is possible.
 	ExcludedProcesses []string `json:"yara.excluded-procs" yaml:"yara.excluded-procs"`
-	// ExcludedProcesses contains the list of the file names that shouldn't be scanned
+	// ExcludedProcesses contains the list of the comma-separated file paths that shouldn't be scanned.
+	// Wildcard matching is possible.
 	ExcludedFiles []string `json:"yara.excluded-files" yaml:"yara.excluded-files"`
 }
 
 // InitFromViper initializes Yara config from Viper.
 func (c *Config) InitFromViper(v *viper.Viper) {
 	c.Enabled = v.GetBool(enabled)
-	c.AlertVia = v.GetString(alertVia)
-	c.AlertTextTemplate = v.GetString(alertTextTemplate)
-	c.AlertTitleTemplate = v.GetString(alertTitleTemplate)
+	c.AlertTemplate = v.GetString(alertTemplate)
 	c.FastScanMode = v.GetBool(fastScanMode)
 	c.ScanTimeout = v.GetDuration(scanTimeout)
 	c.SkipFiles = v.GetBool(skipFiles)
+	c.SkipAllocs = v.GetBool(skipAllocs)
+	c.SkipMmaps = v.GetBool(skipMmaps)
+	c.SkipRegistry = v.GetBool(skipRegistry)
 	c.ExcludedFiles = v.GetStringSlice(excludedFiles)
 	c.ExcludedProcesses = v.GetStringSlice(excludedProcesses)
 
@@ -111,36 +129,81 @@ func (c *Config) InitFromViper(v *viper.Viper) {
 // AddFlags registers persistent flags.
 func AddFlags(flags *pflag.FlagSet) {
 	flags.Bool(enabled, false, "Specifies if Yara scanner is enabled")
-	flags.String(alertVia, "mail", "Defines which alert sender is used to emit the alert on rule matches")
-	flags.String(alertTextTemplate, "", "Defines the template that is used to render the text of the alert")
-	flags.String(alertTitleTemplate, "", "Defines the template that is used to render the title of the alert")
+	flags.String(alertTemplate, "", "Defines the template that is used to render the alert. By default only the threat/rule name is rendered")
 	flags.Bool(fastScanMode, true, "Avoids multiple matches of the same string when not necessary")
 	flags.Duration(scanTimeout, time.Second*10, "Specifies the timeout for the scanner. If the timeout is reached, the scan operation is cancelled")
-	flags.Bool(skipFiles, true, "Indicates whether file scanning is disabled")
-	flags.StringSlice(excludedFiles, []string{}, "Contains the list of the comma-separated file names that shouldn't be scanned")
-	flags.StringSlice(excludedProcesses, []string{}, "Contains the list of the comma-separated process' image names that shouldn't be scanned")
+	flags.Bool(skipFiles, false, "Indicates whether file scanning is disabled")
+	flags.Bool(skipAllocs, false, "Indicates whether scanning on suspicious memory allocations is disabled")
+	flags.Bool(skipMmaps, false, "Indicates whether scanning on suspicious mappings of sections is disabled")
+	flags.Bool(skipRegistry, false, "Indicates whether registry value scanning is disabled")
+	flags.StringSlice(excludedFiles, []string{}, "Contains the list of the comma-separated file paths that shouldn't be scanned. Wildcard matching is possible")
+	flags.StringSlice(excludedProcesses, []string{}, "Contains the list of the comma-separated process image paths that shouldn't be scanned. Wildcard matching is possible")
 }
 
-// ShouldSkipProcess determines whether the specified process name is rejected by the scanner.
-func (c Config) ShouldSkipProcess(ps string) bool {
-	for _, proc := range c.ExcludedProcesses {
-		if strings.EqualFold(proc, ps) {
+// ShouldSkipProcess determines whether the specified full process image path is rejected by the scanner.
+// Wildcard matching is possible.
+func (c Config) ShouldSkipProcess(proc string) bool {
+	for _, p := range c.ExcludedProcesses {
+		if wildcard.Match(strings.ToLower(p), strings.ToLower(proc)) {
 			return true
 		}
 	}
 	return false
 }
 
-// ShouldSkipFile determines whether the specified file name is rejected by the scanner.
+// ShouldSkipFile determines whether the specified full file path is rejected by the scanner.
 func (c Config) ShouldSkipFile(file string) bool {
 	for _, f := range c.ExcludedFiles {
-		if strings.EqualFold(f, filepath.Base(file)) {
+		if wildcard.Match(strings.ToLower(f), strings.ToLower(file)) {
 			return true
 		}
 	}
 	return false
 }
 
+// AlertTitle returns the brief alert title depending on
+// whether the process scan took place or a file/registry
+// key was scanned.
+func (c Config) AlertTitle(e *kevent.Kevent) string {
+	if (e.Category == ktypes.File && e.Kparams.Contains(kparams.FileName)) || e.Category == ktypes.Registry {
+		return FileThreatAlertTitle
+	}
+	return MemoryThreatAlertTitle
+}
+
+// AlertText returns the short alert text if the Go template is
+// not specified. On the contrary, the provided Go template is
+// parsed and executing yielding the alert text.
+func (c Config) AlertText(e *kevent.Kevent, match ytypes.MatchRule) (string, error) {
+	if c.AlertTemplate == "" {
+		threat := match.ThreatName()
+		if threat == "" {
+			threat = match.Rule
+		}
+		return fmt.Sprintf("Threat detected %s", threat), nil
+	}
+
+	var writer bytes.Buffer
+	var data = struct {
+		Match ytypes.MatchRule
+		Event *kevent.Kevent
+	}{
+		match,
+		e,
+	}
+
+	tmpl, err := template.New("yara").Parse(c.AlertTemplate)
+	if err != nil {
+		return "", fmt.Errorf("yara alert template syntax error: %v", err)
+	}
+	err = tmpl.Execute(&writer, data)
+	if err != nil {
+		return "", fmt.Errorf("couldn't execute yara alert template: %v", err)
+	}
+
+	return writer.String(), nil
+}
+
 func decode(input, output interface{}) error {
 	var decoderConfig = &mapstructure.DecoderConfig{
 		Metadata:         nil,