add fluentbit

Author: c-neto

fluentbit/docker-compose.yml

@@ -0,0 +1,31 @@
+version: '3.8'
+
+services:
+  log-generator:
+    container_name: log-generator
+    user: root
+    build:
+      context: log-generator
+    volumes:
+      - logs:/logs
+
+  fluentbit:
+    container_name: fluentbit
+    image: fluent/fluent-bit:latest
+    user: root
+    volumes:
+      - logs:/logs
+      - ./fluentbit/fluent-bit.conf:/fluent-bit/fluent-bit.conf
+    command:
+      - "fluent-bit"
+      - "-c"
+      - "/fluent-bit/fluent-bit.conf"
+
+  http-server:
+    container_name: http-server
+    user: root
+    build:
+      context: http-server
+
+volumes:
+  logs:

fluentbit/fluentbit/fluent-bit.conf

@@ -0,0 +1,22 @@
+# >>> https://docs.fluentbit.io/manual/administration/buffering-and-storage
+[SERVICE]
+    Flush           1
+    Log_Level       info
+    storage.path    /logs/fluent-bit-storage
+    storage.sync    full
+
+# >>> https://docs.fluentbit.io/manual/pipeline/tail
+[INPUT]
+    Name            tail
+    Path            /logs/*.log
+    storage.type    filesystem
+
+# >>> https://docs.fluentbit.io/manual/pipeline/outputs
+[OUTPUT]
+    Name            http
+    Match           *
+    Host            http-server
+    Port            5000
+    URI             /logs
+    Format          json
+    Retry_Limit     False

fluentbit/http-server/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9-slim
+
+ENV PYTHONUNBUFFERED=1
+WORKDIR /app
+COPY app.py /app
+RUN pip install flask
+
+CMD ["python", "app.py"]

fluentbit/http-server/app.py

@@ -0,0 +1,18 @@
+from flask import Flask, request
+import logging
+import sys
+
+app = Flask(__name__)
+
+_log = logging.getLogger('werkzeug')
+_log.setLevel(logging.ERROR)
+
+@app.route('/logs', methods=['POST'])
+def receive_logs():
+    log_batch = request.get_json()
+    logs_raw = [log['log'] for log in log_batch]
+    print(logs_raw)
+    return logs_raw, 200
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000, debug=True)

fluentbit/log-generator/Dockerfile

@@ -0,0 +1,4 @@
+FROM fedora:38
+COPY run.sh /opt/run.sh
+RUN chmod +x /opt/run.sh
+CMD /opt/run.sh

fluentbit/log-generator/run.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Define the log file path
+LOG_FILE="/logs/logfile.log"
+
+# Ensure the log directory exists
+mkdir -p "$(dirname "$LOG_FILE")"
+
+# Initialize the counter
+i=0
+
+# Infinite loop to generate log messages
+while true; do
+    LOG_MESSAGE="#$i"
+    # Append the log message to the log file
+    echo "$LOG_MESSAGE" >> "$LOG_FILE"
+    echo "$LOG_MESSAGE"
+
+    # Increment the counter
+    ((i++))
+    
+    # Sleep for 1 second
+    sleep 1
+done