m365_defender,sentinel_one_cloud_funnel: improve command line split script performance (#13335)

Investigation in the crowdstrike fdr data stream has shown that this
processor can be significantly improved in terms of performance by
reducing string allocation while tokenising the command line. This
change replays the changes in that data stream in m365_defender and
sentinel_one_cloud_funnel.

ref: #13325

Author: efd6

packages/m365_defender/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.1.0"
+  changes:
+    - description: Improve performance of event ingest pipeline.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13335
 - version: "3.0.1"
   changes:
     - description: Fix `event.category` value for the AdvancedHunting-AlertInfo event.

packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_alert.yml

@@ -369,19 +369,19 @@ processors:
             return b;
         }
 
-        // readNextArg splits command line string cmd into next
-        // argument and command line remainder.
-        def readNextArg(String cmd) {
+        // readNextArg splits command line string into next
+        // argument and command line remainder offset.
+        def readNextArg(String line, int offset) {
             def b = new StringBuilder();
             boolean inquote;
             int nslash;
-            for (; cmd.length() > 0; cmd = cmd.substring(1)) {
-                def c = cmd.charAt(0);
+            for (; offset < line.length(); offset++) {
+                def c = line.charAt(offset);
                 if (c == (char)' ' || c == (char)0x09) {
                     if (!inquote) {
                         return [
                             "arg":  appendBSBytes(b, nslash).toString(),
-                            "rest": cmd.substring(1)
+                            "offset": offset+1
                         ];
                     }
                 } else if (c == (char)'"') {
@@ -390,9 +390,9 @@ processors:
                         // use "Prior to 2008" rule from
                         // http://daviddeley.com/autohotkey/parameters/parameters.htm
                         // section 5.2 to deal with double double quotes
-                        if (inquote && cmd.length() > 1 && cmd.charAt(1) == (char)'"') {
+                        if (inquote && offset+1 < line.length() && line.charAt(offset+1) == (char)'"') {
                             b.append(c);
-                            cmd = cmd.substring(1);
+                            offset++;
                         }
                         inquote = !inquote;
                     } else {
@@ -409,24 +409,28 @@ processors:
                 b.append(c);
             }
             return [
-                "arg":  appendBSBytes(b, nslash).toString(),
-                "rest": ''
+                "arg":  appendBSBytes(b, nslash).toString(), 
+                "offset": line.length()
             ];
         }
 
         // commandLineToArgv splits a command line into individual argument
         // strings, following the Windows conventions documented
         // at http://daviddeley.com/autohotkey/parameters/parameters.htm#WINARGV
         // Original implementation found at: https://github.com/golang/go/commit/39c8d2b7faed06b0e91a1ad7906231f53aab45d1
-        def commandLineToArgv(String cmd) {
+        def commandLineToArgv(String line) {
             def args = new ArrayList();
-            while (cmd.length() > 0) {
-                if (cmd.charAt(0) == (char)' ' || cmd.charAt(0) == (char)0x09) {
-                    cmd = cmd.substring(1);
+            for (int i = 0; i < line.length();) {
+                if (line.charAt(i) == (char)' ' || line.charAt(i) == (char)0x09) {
+                    i++;
+                    continue;
+                }
+                def next = readNextArg(line, i);
+                i = next.offset;
+                if (next.arg == '') {
+                    // Empty strings will be removed later so don't bother adding them.
                     continue;
                 }
-                def next = readNextArg(cmd);
-                cmd = next.rest;
                 args.add(next.arg);
             }
             return args;

packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml

@@ -2628,19 +2628,19 @@ processors:
             return b;
         }
 
-        // readNextArg splits command line string cmd into next
-        // argument and command line remainder.
-        def readNextArg(String cmd) {
+        // readNextArg splits command line string into next
+        // argument and command line remainder offset.
+        def readNextArg(String line, int offset) {
             def b = new StringBuilder();
             boolean inquote;
             int nslash;
-            for (; cmd.length() > 0; cmd = cmd.substring(1)) {
-                def c = cmd.charAt(0);
+            for (; offset < line.length(); offset++) {
+                def c = line.charAt(offset);
                 if (c == (char)' ' || c == (char)0x09) {
                     if (!inquote) {
                         return [
                             "arg":  appendBSBytes(b, nslash).toString(),
-                            "rest": cmd.substring(1)
+                            "offset": offset+1
                         ];
                     }
                 } else if (c == (char)'"') {
@@ -2649,9 +2649,9 @@ processors:
                         // use "Prior to 2008" rule from
                         // http://daviddeley.com/autohotkey/parameters/parameters.htm
                         // section 5.2 to deal with double double quotes
-                        if (inquote && cmd.length() > 1 && cmd.charAt(1) == (char)'"') {
+                        if (inquote && offset+1 < line.length() && line.charAt(offset+1) == (char)'"') {
                             b.append(c);
-                            cmd = cmd.substring(1);
+                            offset++;
                         }
                         inquote = !inquote;
                     } else {
@@ -2668,24 +2668,28 @@ processors:
                 b.append(c);
             }
             return [
-                "arg":  appendBSBytes(b, nslash).toString(),
-                "rest": ''
+                "arg":  appendBSBytes(b, nslash).toString(), 
+                "offset": line.length()
             ];
         }
 
         // commandLineToArgv splits a command line into individual argument
         // strings, following the Windows conventions documented
         // at http://daviddeley.com/autohotkey/parameters/parameters.htm#WINARGV
         // Original implementation found at: https://github.com/golang/go/commit/39c8d2b7faed06b0e91a1ad7906231f53aab45d1
-        def commandLineToArgv(String cmd) {
+        def commandLineToArgv(String line) {
             def args = new ArrayList();
-            while (cmd.length() > 0) {
-                if (cmd.charAt(0) == (char)' ' || cmd.charAt(0) == (char)0x09) {
-                    cmd = cmd.substring(1);
+            for (int i = 0; i < line.length();) {
+                if (line.charAt(i) == (char)' ' || line.charAt(i) == (char)0x09) {
+                    i++;
+                    continue;
+                }
+                def next = readNextArg(line, i);
+                i = next.offset;
+                if (next.arg == '') {
+                    // Empty strings will be removed later so don't bother adding them.
                     continue;
                 }
-                def next = readNextArg(cmd);
-                cmd = next.rest;
                 args.add(next.arg);
             }
             return args;

packages/m365_defender/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.2.3"
 name: m365_defender
 title: Microsoft M365 Defender
-version: "3.0.1"
+version: "3.1.0"
 description: Collect logs from Microsoft M365 Defender with Elastic Agent.
 categories:
   - "security"

packages/sentinel_one_cloud_funnel/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.11.0"
+  changes:
+    - description: Improve performance of event ingest pipeline.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13335
 - version: "1.10.0"
   changes:
     - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.

packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml

@@ -2775,19 +2775,19 @@ processors:
             return b;
         }
 
-        // readNextArg splits command line string cmd into next
-        // argument and command line remainder.
-        def readNextArg(String cmd) {
+        // readNextArg splits command line string into next
+        // argument and command line remainder offset.
+        def readNextArg(String line, int offset) {
             def b = new StringBuilder();
             boolean inquote;
             int nslash;
-            for (; cmd.length() > 0; cmd = cmd.substring(1)) {
-                def c = cmd.charAt(0);
+            for (; offset < line.length(); offset++) {
+                def c = line.charAt(offset);
                 if (c == (char)' ' || c == (char)0x09) {
                     if (!inquote) {
                         return [
                             "arg":  appendBSBytes(b, nslash).toString(),
-                            "rest": cmd.substring(1)
+                            "offset": offset+1
                         ];
                     }
                 } else if (c == (char)'"') {
@@ -2796,9 +2796,9 @@ processors:
                         // use "Prior to 2008" rule from
                         // http://daviddeley.com/autohotkey/parameters/parameters.htm
                         // section 5.2 to deal with double double quotes
-                        if (inquote && cmd.length() > 1 && cmd.charAt(1) == (char)'"') {
+                        if (inquote && offset+1 < line.length() && line.charAt(offset+1) == (char)'"') {
                             b.append(c);
-                            cmd = cmd.substring(1);
+                            offset++;
                         }
                         inquote = !inquote;
                     } else {
@@ -2816,23 +2816,27 @@ processors:
             }
             return [
                 "arg":  appendBSBytes(b, nslash).toString(), 
-                "rest": ''
+                "offset": line.length()
             ];
         }
 
         // commandLineToArgv splits a command line into individual argument
         // strings, following the Windows conventions documented
         // at http://daviddeley.com/autohotkey/parameters/parameters.htm#WINARGV
         // Original implementation found at: https://github.com/golang/go/commit/39c8d2b7faed06b0e91a1ad7906231f53aab45d1
-        def commandLineToArgv(String cmd) {
+        def commandLineToArgv(String line) {
             def args = new ArrayList();
-            while (cmd.length() > 0) {
-                if (cmd.charAt(0) == (char)' ' || cmd.charAt(0) == (char)0x09) {
-                    cmd = cmd.substring(1);
+            for (int i = 0; i < line.length();) {
+                if (line.charAt(i) == (char)' ' || line.charAt(i) == (char)0x09) {
+                    i++;
+                    continue;
+                }
+                def next = readNextArg(line, i);
+                i = next.offset;
+                if (next.arg == '') {
+                    // Empty strings will be removed later so don't bother adding them.
                     continue;
                 }
-                def next = readNextArg(cmd);
-                cmd = next.rest;
                 args.add(next.arg);
             }
             return args;

packages/sentinel_one_cloud_funnel/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: sentinel_one_cloud_funnel
 title: SentinelOne Cloud Funnel
-version: "1.10.0"
+version: "1.11.0"
 description: Collect logs from SentinelOne Cloud Funnel with Elastic Agent.
 type: integration
 categories: ["security", "edr_xdr"]