Merge pull request #528 from lostsnow/feature/fluent-multi-line-parser

add fluent multi line parser

Author: lostsnow

dongtai-agent/src/main/java/io/dongtai/iast/agent/LogCollector.java

@@ -22,6 +22,15 @@ public static void extractFluent() {
         }
         try {
             if (!isMacOs() && !isWindows()) {
+                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
+                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
+                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
+                FileUtils.confReplace(FLUENT_FILE_CONF);
+
+                String multiParserFile = IastProperties.getInstance().getTmpDir() + "parsers_multiline.conf";
+                FileUtils.getResourceToFile("bin/parsers_multiline.conf", multiParserFile);
+                FileUtils.confReplace(multiParserFile);
+
                 FLUENT_FILE = IastProperties.getInstance().getTmpDir() + "fluent";
                 File f = new File(FLUENT_FILE);
                 if (f.exists()) {
@@ -34,10 +43,6 @@ public static void extractFluent() {
                     FileUtils.getResourceToFile("bin/fluent", FLUENT_FILE);
                 }
 
-                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
-                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
-                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
-                FileUtils.confReplace(FLUENT_FILE_CONF);
                 if (!(new File(FLUENT_FILE)).setExecutable(true)) {
                     DongTaiLog.warn(ErrorCode.FLUENT_SET_EXECUTABLE_FAILED, FLUENT_FILE);
                 }

dongtai-agent/src/main/resources/bin/fluent.conf

@@ -3,6 +3,7 @@
   Daemon       OFF
   Log_Level    error
   HTTP_Server  Off
+  parsers_file parsers_multiline.conf
 [INPUT]
   Name         tail
   Path         ${LOG_PATH}
@@ -13,6 +14,7 @@
   Buffer_Max_Size   16MB
   Skip_Long_Lines   On
   Read_from_Head    true
+  multiline.parser multiline-regex-test
 [FILTER]
   Name         record_modifier
   Match        *

dongtai-agent/src/main/resources/bin/parsers_multiline.conf

@@ -0,0 +1,6 @@
+[MULTILINE_PARSER]
+    name          multiline-regex-test
+    type          regex
+    flush_timeout 1000
+    rule      "start_state"   "/(\d+\d+\-\d+\-\d+ \d+\:\d+\:\d+)(.*)/"  "cont"
+    rule      "cont"          "/(^\s+at.*|^Caused.*|^\s+\.\.\..*)/"     "cont"