fluentbit ES ingest pipeline

Author: codeyourweb

config/fluentbit_server/add_timestamp.lua

@@ -0,0 +1,9 @@
+function add_timestamp(tag, timestamp, record)
+    if record["@timestamp"] == nil then
+        local seconds = timestamp[1]
+        local formatted_time = os.date("!%Y-%m-%dT%H:%M:%S", seconds) .. string.format(".%03dZ", timestamp[2]/1000000)
+        record["@timestamp"] = formatted_time
+    end
+
+    return 1, timestamp, record
+end

config/fluentbit_server/fluent-bit.conf

@@ -8,7 +8,6 @@
     HTTP_Port    2020
     Health_Check On
 
-# HTTP INPUT FOR LOGS
 [INPUT]
     Name         http
     Listen       0.0.0.0
@@ -21,16 +20,33 @@
     Key_Name     log
     Parser       json_parser
     Preserve_Key On
+    
+[FILTER]
+    Name         lua
+    Match        http.logs
+    Script       /fluent-bit/etc/add_timestamp.lua
+    Call         add_timestamp
+
+[FILTER]
+    Name         lua
+    Match        http.logs
+    Script       /fluent-bit/etc/set_target_index.lua
+    Call         set_target_index
 
 [OUTPUT]
     Name         es
     Match        http.logs
-    Host         sentinel-kit-db-elasticsearch
+    Host         sentinel-kit-db-elasticsearch-es01
     Port         9200
-    Index        essai
-    Logstash_Format Off
-    Retry_Limit False
-    Type  _doc
-    Time_Key @timestamp
+    Logstash_Format     On
+    Logstash_Prefix_Key target_index 
+    Logstash_DateFormat %Y.%m.%d
+    Type         _doc
+    Time_Key     @timestamp
     Replace_Dots On
-    Suppress_Type_Name On
+    Suppress_Type_Name On
+    Retry_Limit  False
+    TLS           On
+    TLS.Verify    Off
+    HTTP_User     elastic
+    HTTP_Passwd   ${ELASTIC_PASSWORD}

config/fluentbit_server/set_target_index.lua

@@ -0,0 +1,10 @@
+function set_target_index(tag, timestamp, record)
+    local exit_code = 0
+
+    if record["target_index"] == nil then
+        record["target_index"] = "undefined-index"
+        exit_code = 1
+    end
+
+    return exit_code, timestamp, record
+end

docker-compose.yml

@@ -54,10 +54,14 @@ services:
     container_name: sentinel-kit-server-fluentbit
     image: fluent/fluent-bit:latest
     restart: on-failure
+    environment:
+      - ELASTIC_PASSWORD=${ELASTICSEARCH_PASSWORD}
     volumes:
-      - ./config/fluentbit_server/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
-      - ./config/fluentbit_server/parsers.conf:/fluent-bit/etc/parsers.conf
+      - ./config/fluentbit_server:/fluent-bit/etc
       - ./data/log_ingest_data:/var/log:ro
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
     networks:
       - sentinel-kit-network
     depends_on: