Fluent Bit sidecar archatecture. Complete POC

Author: ilanRosenbaum

charts/templates/configmap.yaml

@@ -14,56 +14,25 @@ data:
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: fluentd-config
+  name: {{ .Values.logging.configMapName }}
   namespace: {{ .Release.Namespace }}
-  labels:
-    kubernetes.io/cluster-service: "true"
 data:
-  fluent.conf: |
-      <match fluent.**>
-        @type null
-      </match>
-
-      <source>
-        @type tail
-        @id tail_marklogic_logs
-        format none
-        path /var/opt/MarkLogic/Logs/*
-        pos_file /tmp/log/ml-logs.log.pos
-        path_key log_file
-        tag marklogic.logs
-        read_from_head true
-        <parse>
-          @type none
-          time_key @timestamp
-          time_format %Y-%m-%dT%H:%M:%S.%N%z
-          keep_time_key true
-        </parse>
-      </source>
-
-      <filter marklogic.logs>
-        @type record_transformer
-        enable_ruby true
-        <record>
-          # overwrite full path to file name
-          log_file ${File.basename(record["log_file"])}
-        </record>
-      </filter>
-
-      <match marklogic.logs>
-        @type elasticsearch
-        host elasticsearch.default.svc.cluster.local
-        port 9200
-        user admin
-        password admin
-        index_name fluentd.${tag}
-        include_timestamp true
-        utc_index false
-        time_key_format "%Y-%m-%dT%H:%M:%S.%N%z"
-        time_key time
-        logstash_format true
-        logstash_prefix marklogic.logs
-        path_key out/${log_file}
-        flush_interval 1s
-      </match>
+  fluent-bit.conf: |
+      [SERVICE]
+        Flush  5
+        Log_Level  info
+        Daemon  off
+      
+      [INPUT]
+        Name  tail
+        Path  {{ .Values.persistence.mountPath }}/Logs/*
+        Tag  marklogic-logs
+        Path_Key  path
 
+      [OUTPUT]
+        Name  es
+        Match  *
+        Host  {{ .Values.logging.host }}
+        Port  {{ .Values.logging.port }}
+        HTTP_User  {{ .Values.logging.username }}
+        HTTP_Passwd  {{ .Values.logging.password }}

charts/templates/statefulset.yaml

@@ -96,23 +96,20 @@ spec:
           {{- with .Values.resources }}
           resources: {{- toYaml . | nindent 12 }}
           {{- end }}
-        - name: fluentd
-          env:
-          - name: FLUENT_UID
-            value: root
-          - name: FLUENT_CONF
-            value: fluent.conf
-          - name: FLUENTD_ARGS
-            value: -c /fluentd/etc/fluent.conf
-          image: saravak/fluentd:elastic
+        {{- if .Values.logging.enabled }}
+        - name: fluent-bit
+          image: fluent/fluent-bit:latest
+          imagePullPolicy: IfNotPresent
           volumeMounts:
             - name: datadir
               mountPath: {{ .Values.persistence.mountPath }}
             {{- if .Values.extraVolumeMounts }}
               {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
             {{- end }}
-            - name:  fdconf
-              mountPath:  /fluentd/etc/
+            - name: {{ .Values.logging.configMapName }}
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+        {{- end }}
       dnsConfig:
         searches:
           - {{ include "marklogic.headlessURL" . }}
@@ -121,9 +118,11 @@ spec:
         - name: {{ include "marklogic.fullname" . }}-registry
       {{- end }}
       volumes:
-        - name: fdconf
+        {{- if .Values.logging.enabled }}
+        - name: {{ .Values.logging.configMapName }}
           configMap:
-            name: fluentd-config
+            name: {{ .Values.logging.configMapName }}
+        {{- end }}
       {{- if or (not .Values.persistence.enabled) (.Values.extraVolumes) }}
         {{- if not .Values.persistence.enabled }}
         - name: datadir

charts/values.yaml

@@ -110,3 +110,10 @@ startupProbe:
   failureThreshold: 30
   successThreshold: 1
 
+logging:
+  enabled: true
+  username: admin
+  password: admin
+  configMapName: fluent-bit-config
+  host: elasticsearch.default.svc.cluster.local
+  port: 9200