fluentbit and helm (#163)

* fluentbit and helm

* wip1

* wip2

* final - ready for CR

* pre commit

Author: EliMoshkovich

charts/pdp/templates/deployment.yaml

@@ -26,6 +26,18 @@ spec:
                 secretKeyRef:
                   name: permitio-pdp-secret
                   key: ApiKey
+            {{- if .Values.pdp.logs_forwarder.enabled }}
+            - name: PDP_OPA_DECISION_LOG_CONSOLE
+              value: "true"
+            - name: OPAL_LOG_TO_FILE
+              value: "true"
+            - name: OPAL_LOG_FILE_PATH
+              value: "/tmp/pdp.log"
+            {{- end }}
+            {{- if .Values.pdp.debug_mode }}
+            - name: PDP_DEBUG
+              value: "true"
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /health
@@ -38,4 +50,32 @@ spec:
               port: 7000
             initialDelaySeconds: 10
             periodSeconds: 10
+          {{- if .Values.pdp.logs_forwarder.enabled }}
+          volumeMounts:
+            - name: logs
+              mountPath: /tmp/
+          {{- end }}
           resources: {}
+        {{- if .Values.pdp.logs_forwarder.enabled }}
+        - name: fluentbit
+          image: fluent/fluent-bit:3.1.4
+          volumeMounts:
+            - name: fluent-bit-config
+              mountPath: /fluent-bit/etc
+              readOnly: true
+            - name: logs
+              mountPath: /tmp/
+          env:
+            - name: APP_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['app']
+        {{- end }}
+      {{- if .Values.pdp.logs_forwarder.enabled }}
+      volumes:
+        - name: fluent-bit-config
+          configMap:
+            name: fluentbit-config
+        - name: logs
+          emptyDir: {}
+      {{- end }}

charts/pdp/templates/logs-forwarder-cm.yaml

@@ -0,0 +1,70 @@
+{{ if .Values.pdp.logs_forwarder.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentbit-config
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush         1
+        Log_Level     {{ if .Values.pdp.logs_forwarder.debug_mode }}debug{{ else }}info{{ end }}
+        HTTP_Server   On
+        HTTP_Listen   0.0.0.0
+        HTTP_Port     2020
+
+    [INPUT]
+        Name          tail
+        Path          /tmp/pdp.log
+        Tag           kube.*
+        DB            /var/log/flb_kube.db
+        Mem_Buf_Limit 5MB
+        Skip_Long_Lines On
+
+    [FILTER]
+        Name          grep
+        Match         kube.*
+        Regex         log "Decision Log.*{.*}"
+
+    [FILTER]
+        Name          lua
+        Match         kube.*
+        script        /fluent-bit/etc/filter.lua
+        call          process_log
+
+    {{- if eq .Values.pdp.logs_forwarder.type "stdout" }}
+    [OUTPUT]
+        Name          stdout
+        Match         *
+    {{- else if eq .Values.pdp.logs_forwarder.type "elasticsearch" }}
+    [OUTPUT]
+        Name          es
+        Match         *
+        Host          {{ .Values.pdp.logs_forwarder.elasticsearch.host }}
+        Index         {{ .Values.pdp.logs_forwarder.elasticsearch.index }}
+        Port          {{ .Values.pdp.logs_forwarder.elasticsearch.port }}
+        tls           On
+        tls.verify    Off
+        Cloud_Auth    {{ .Values.pdp.logs_forwarder.elasticsearch.cloud_auth }}
+        Suppress_Type_Name On
+        Retry_Limit     False
+    {{- end }}
+
+  filter.lua: |
+    function process_log(tag, timestamp, record)
+        local log_message = record["log"]
+
+        if log_message then
+            log_message = log_message:gsub('\\\\"', '\\"')
+            log_message = log_message:gsub('\\"', '"')
+            log_message = log_message:gsub('\\\\', '\\')
+        end
+
+        local json_data = string.match(log_message, '{"decision_id":.*}')
+
+        if json_data then
+            return 1, timestamp, { log = json_data }
+        else
+            return -1, timestamp, record
+        end
+    end
+{{- end }}

charts/pdp/values.yaml

@@ -6,3 +6,13 @@ pdp:
     repository: permitio/pdp-v2
     tag: latest
     pullPolicy: Always
+  logs_forwarder:
+    enabled: false
+    debug_mode: false
+    type: "stdout"  # 'stdout' or 'elasticsearch'
+    elasticsearch:
+      host: "<elasticsearch host>"
+      cloud_auth: "<{user}:{password}>"
+      port: 443
+      index: "<elasticsearch index>"
+  debug_mode: false