Merge pull request #29 from wpoely86/cleanup

bring config file for test in sync with production

Author: lexming

tests/logstash_7.6.2.conf

@@ -2,9 +2,9 @@ input {
     stdin { } # logstash stops when input is closed
 }
 
-filter {
+filter
+{
     grok {
-
         add_field => {
             "received_at" => "%{@timestamp}"
             "received_from" => "%{@source_host}"
@@ -15,9 +15,36 @@ filter {
         patterns_dir => [ "/tmp/logpatterns-groktest" ]
 
         match => {
-            # RSYSLOGCUSTOM always last (and no PREFIX)!
-            "message" => ["%{RSYSLOGCUSTOMHEADER}%{PIXIU_ALARM}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_BASH}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_NGINX}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_TOMCAT}", "%{RSYSLOGPREFIX}%{PIXIU_APACHE}", "%{RSYSLOGPREFIX}%{SU_MSG}", "%{RSYSLOGPREFIX}%{SUDO_MSG}", "%{RSYSLOGPREFIX}%{REFRAME_MSG}", "%{RSYSLOGPREFIX}%{BASH_MSG}", "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}", "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}", "%{RSYSLOGPREFIX}%{DHCPD_MSG}", "%{RSYSLOGPREFIX}%{SSH_MSG}", "%{RSYSLOGPREFIX}%{MODULECMD_MSG}", "%{RSYSLOGPREFIX}%{LMOD_MSG}", "%{RSYSLOGPREFIX}%{NFS_MSG}", "%{RSYSLOGPREFIX}%{CEPH_MSG}", "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}", "%{RSYSLOGPREFIX}%{JUBE_MSG}", "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}", "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", "%{RSYSLOGPREFIX}%{APACHE_MSG}", "%{RSYSLOGCUSTOM}"]
+            "message" => [
+                "%{RSYSLOGCUSTOMHEADER}%{PIXIU_ALARM}",
+                "%{RSYSLOGCUSTOMHEADER}%{PIXIU_BASH}",
+                "%{RSYSLOGCUSTOMHEADER}%{PIXIU_NGINX}",
+                "%{RSYSLOGCUSTOMHEADER}%{PIXIU_TOMCAT}",
+                "%{RSYSLOGPREFIX}%{PIXIU_APACHE}",
+                "%{RSYSLOGPREFIX}%{SU_MSG}",
+                "%{RSYSLOGPREFIX}%{SUDO_MSG}",
+                "%{RSYSLOGPREFIX}%{REFRAME_MSG}",
+                "%{RSYSLOGPREFIX}%{BASH_MSG}",
+                "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}",
+                "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}",
+                "%{RSYSLOGPREFIX}%{DHCPD_MSG}",
+                "%{RSYSLOGPREFIX}%{SSH_MSG}",
+                "%{RSYSLOGPREFIX}%{MODULECMD_MSG}",
+                "%{RSYSLOGPREFIX}%{LMOD_MSG}",
+                "%{RSYSLOGPREFIX}%{NFS_MSG}",
+                "%{RSYSLOGPREFIX}%{CEPH_MSG}",
+                "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}",
+                "%{RSYSLOGPREFIX}%{JUBE_MSG}",
+                "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}",
+                "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}",
+                "%{RSYSLOGPREFIX}%{QUATTOR_MSG}",
+                "%{RSYSLOGPREFIX}%{SNOOPY_MSG}",
+                "%{RSYSLOGPREFIX}%{APACHE_MSG}",
+                # RSYSLOGCUSTOM always last (and no PREFIX)!
+                "%{RSYSLOGCUSTOM}"
+            ]
         }
+        id => "main_grok"
     }
 
     kv {
@@ -32,28 +59,26 @@ filter {
         mutate {
             replace => {
                 "@source_host" => "%{syslog_hostname}"
-                "@message" => "%{syslog_message}"
             }
         }
     }
 
-    if "_grokparsefailure" not in [tags] {
-      mutate {
-        # in the future remove the field 'message' too
-        remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA", "int" ]
-        convert => { "success" => "boolean" }
-
-        # we need MB converted to MiB for bytes2human
-        gsub => [ "used_space", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib",
-                  "quota_space_limit", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib" ]
-      }
+    if ([syslog_message]) {
+        mutate {
+            replace => {
+                "@message" => "%{syslog_message}"
+            }
+        }
     }
 
-    if [jube_id] {
-        if ("_grokparsefailure" not in [tags]) {
-            mutate {
-                convert => { "success" => "boolean" }
-            }
+    if ("_grokparsefailure" not in [tags]) {
+        mutate {
+            remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA", "int" ]
+            convert => { "success" => "boolean" }
+
+            # we need MB converted to MiB for bytes2human
+            gsub => [ "used_space", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib",
+                      "quota_space_limit", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib" ]
         }
     }