chore: Raw event parsing and performance optimizations (#126)

* raw event parsing

* reading UTF16 encoded strings

* introduce version handling in raw event parsers, refactor trace controller

* remove ignored params map

* introduce Kparams constructor with variadic args

* use the regular `if` conditional operator for version handling

* implement event schema version checking

* add ktype descriptions, include raw event parsing test

* fix failing tests, disable filament build flag

* fix tests and lint warnings

* more fixing/polishing

* check if param is hex before coercing

* add filament build flags for the cpython code

Author: rabbitstack

.editorconfig

@@ -11,7 +11,7 @@ indent_size = 2
 
 [{go.mod,go.sum,*.go}]
 indent_style = tab
-indent_size = 8
+indent_size = 4
 
 [*.md]
 indent_size = 4

.github/workflows/master.yml

@@ -168,7 +168,7 @@ jobs:
            export PKG_CONFIG_PATH=$(pwd)/pkg-config
            ./make.bat test
         env:
-          TAGS: kcap,filament,yara,yara_static
+          TAGS: kcap,yara,yara_static
 
   lint:
     runs-on: windows-latest

.github/workflows/pr.yml

@@ -121,7 +121,7 @@ jobs:
            export PKG_CONFIG_PATH=$(pwd)/pkg-config
            ./make.bat test
         env:
-          TAGS: kcap,filament,yara,yara_static
+          TAGS: kcap,yara,yara_static
 
   lint:
     runs-on: windows-latest

.github/workflows/release.yml

@@ -170,4 +170,3 @@ jobs:
              build/fibratus-${{ steps.get_version.outputs.VERSION }}-slim-amd64.msi
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

cmd/fibratus/app/capture_windows.go

@@ -95,7 +95,7 @@ func capture(cmd *cobra.Command, args []string) error {
 		kstreamc.SetFilter(kfilter)
 	}
 
-	err = kstreamc.OpenKstream()
+	err = kstreamc.OpenKstream(ktracec.Traces())
 	if err != nil {
 		return multierror.Wrap(err, ktracec.CloseKtrace())
 	}

cmd/fibratus/app/control_service_windows.go

@@ -230,7 +230,7 @@ func (s *fsvc) run() error {
 	psnap := ps.NewSnapshotter(hsnap, svcConfig)
 	consumer = kstream.NewConsumer(ctrl, psnap, hsnap, svcConfig)
 	// open the kernel event stream, start processing events and forwarding to outputs
-	err = consumer.OpenKstream()
+	err = consumer.OpenKstream(ctrl.Traces())
 	if err != nil {
 		return err
 	}

cmd/fibratus/app/run_windows.go

@@ -117,7 +117,7 @@ func run(cmd *cobra.Command, args []string) error {
 		if f.Filter() != nil {
 			kstreamc.SetFilter(f.Filter())
 		}
-		err = kstreamc.OpenKstream()
+		err = kstreamc.OpenKstream(ktracec.Traces())
 		if err != nil {
 			return multierror.Wrap(err, ktracec.CloseKtrace())
 		}
@@ -134,7 +134,7 @@ func run(cmd *cobra.Command, args []string) error {
 			}
 		}()
 	} else {
-		err = kstreamc.OpenKstream()
+		err = kstreamc.OpenKstream(ktracec.Traces())
 		if err != nil {
 			return multierror.Wrap(err, ktracec.CloseKtrace())
 		}

configs/fibratus.yml

@@ -175,6 +175,9 @@ kstream:
   # collected by Kernel Logger provider
   #enable-handle: false
 
+  # Determines if raw event buffer parsing is used instead of TDH (Trace Data Helper) API
+  # raw-event-parsing: true
+
   # Determines which events are dropped either by the event name or the process' image
   # name that triggered the event.
   blacklist:

configs/rules/default/default.yml

@@ -378,4 +378,3 @@
               '\\Start',
               'CurrentVersion\\Run'
            )
-

pkg/config/config_windows.go

@@ -341,6 +341,7 @@ func (c *Config) addFlags() {
 		c.flags.Duration(flushInterval, defaultFlushInterval, "Specifies how often the trace buffers are forcibly flushed")
 		c.flags.StringSlice(excludedEvents, []string{}, "A list of symbolical kernel event names that will be dropped from the kernel event stream. By default all events are accepted")
 		c.flags.StringSlice(excludedImages, []string{"System"}, "A list of image names that will be dropped from the kernel event stream. Image names are case insensitive")
+		c.flags.Bool(rawEventParsing, true, "Determines if raw event buffer parsing is used instead of TDH (Trace Data Helper) API")
 
 		c.flags.Bool(serializeThreads, false, "Indicates if threads are serialized as part of the process state")
 		c.flags.Bool(serializeImages, false, "Indicates if images are serialized as part of the process state")