Use specific version of fluent-bit

Setup parsing for request, access, and error logs
Tag log files

Author: ilanRosenbaum

charts/templates/configmap.yaml

@@ -19,20 +19,119 @@ metadata:
 data:
   fluent-bit.conf: |
       [SERVICE]
-        Flush  5
-        Log_Level  info
-        Daemon  off
+        Flush 5
+        Log_Level info
+        Parsers_File parsers.conf
+        Daemon off
+
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/*Error*
+        Read_from_head true
+        Tag kube.marklogic.logs.error
+        Path_Key path
       
       [INPUT]
-        Name  tail
-        Path  {{ .Values.persistence.mountPath }}/Logs/*
-        Tag  marklogic-logs
-        Path_Key  path
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/*Access*
+        Read_from_head true
+        tag kube.marklogic.logs.access
+        Path_Key path
+      
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/*Request*
+        Read_from_head true
+        tag kube.marklogic.logs.request
+        Path_Key path
+      
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/CrashLog.txt
+        Read_from_head true
+        tag kube.marklogic.logs.crash
+        Path_Key path
+      
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/AuditLog.txt
+        Read_from_head true
+        tag kube.marklogic.logs.audit
+        Path_Key path
+      
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.request
+        Add tag kube.marklogic.logs.request
+      
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.audit
+        Add tag kube.marklogic.logs.audit
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.crash
+        Add tag kube.marklogic.logs.crash
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.access
+        Add tag kube.marklogic.logs.access
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.error
+        Add tag kube.marklogic.logs.error
+
+      [FILTER]
+        Name parser
+        Match kube.marklogic.logs.error
+        Key_Name log
+        Parser error_parser
+        Reserve_Data On
+
+      [FILTER]
+        Name parser
+        Match kube.marklogic.logs.access
+        Key_Name log
+        Parser access_parser
+        Reserve_Data On
+      
+      [FILTER]
+        Name parser
+        Match kube.marklogic.logs.request
+        Key_Name log
+        Parser json_parser
+        Reserve_Data On
 
       [OUTPUT]
-        Name  es
-        Match  *
-        Host  {{ .Values.logging.host }}
-        Port  {{ .Values.logging.port }}
-        HTTP_User  {{ .Values.logging.username }}
-        HTTP_Passwd  {{ .Values.logging.password }}
+        Name es
+        Match *
+        Host {{ .Values.logging.host }}
+        Port {{ .Values.logging.port }}
+        HTTP_User {{ .Values.logging.username }}
+        HTTP_Passwd {{ .Values.logging.password }}
+  parsers.conf: |
+      [PARSER]
+        Name error_parser
+        Format regex
+        Regex ^(?<date_time>(.+?)(?=[a-zA-Z]))(?<log_level>(.+?)(?=:))(.+?)(?=[a-zA-Z])(?<log>.*)
+        Time_Key date_time 
+        Time_Format %Y-%m-%d %H:%M:%S.%L
+        Time_Keep On
+      
+      [PARSER]
+        Name access_parser
+        Format regex
+        Regex ^(?<host>[^ ]*)(.+?)(?<=\- )(?<user>(.+?)(?=\[))(.+?)(?<=\[)(?<date_time>(.+?)(?=\]))(.+?)(?<=")(?<request>[^\ ]+[^\"]+)(.+?)(?=\d)(?<response_code>[^\ ]*)(.+?)(?=\d|-)(?<response_obj_size>[^\ ]*)(.+?)(?=")(?<request_info>.*)
+        Time_Key date_time 
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+        Time_Keep On
+      
+      [PARSER]
+        Name json_parser
+        Format json
+        Time_Key time
+        Time_Format %Y-%m-%dT%H:%M:%S%z
+        Time_Keep On

charts/templates/statefulset.yaml

@@ -98,7 +98,7 @@ spec:
           {{- end }}
         {{- if .Values.logging.enabled }}
         - name: fluent-bit
-          image: fluent/fluent-bit:latest
+          image: fluent/fluent-bit:1.9.7
           imagePullPolicy: IfNotPresent
           volumeMounts:
             - name: datadir
@@ -109,6 +109,9 @@ spec:
             - name: {{ .Values.logging.configMapName }}
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
+            - name: {{ .Values.logging.configMapName }}
+              mountPath: /fluent-bit/etc/parsers.conf
+              subPath: parsers.conf
         {{- end }}
       dnsConfig:
         searches: