Switch fluentbit for vectordev

Author: mrnicegyu11

services/logging/docker-compose.aws.yml

@@ -15,7 +15,7 @@ services:
         constraints:
           - node.labels.logging==true
 
-  fluentbit:
+  vector:
     deploy:
       placement:
         constraints:

services/logging/docker-compose.dalco.yml

@@ -17,7 +17,7 @@ services:
         constraints:
           - node.labels.logging==true
 
-  fluentbit:
+  vector:
     deploy:
       placement:
         constraints:

services/logging/docker-compose.local.yml

@@ -14,7 +14,7 @@ services:
       placement:
         constraints: []
 
-  fluentbit:
+  vector:
     deploy:
       placement:
         constraints: []

services/logging/docker-compose.master.yml

@@ -17,7 +17,7 @@ services:
         constraints:
           - node.labels.logging==true
 
-  fluentbit:
+  0.49.X-debian:
     deploy:
       placement:
         constraints:

services/logging/docker-compose.yml.j2

@@ -99,41 +99,26 @@ services:
         - traefik.http.middlewares.graylog_replace_regex.replacepathregex.regex=^/graylog/?(.*)$$
         - traefik.http.middlewares.graylog_replace_regex.replacepathregex.replacement=/$${1}
         - traefik.http.routers.graylog.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, graylog_replace_regex
-  fluentbit:
-    image: cr.fluentbit.io/fluent/fluent-bit:4.0.9
-    configs:
-      - source: fluentbit_config
-        target: /fluent-bit/etc/fluent-bit.conf
-    environment:
-      - GRAYLOG_HOST=graylog
-      - GRAYLOG_PORT=12201
-      - LOKI_HOST=loki
+  vector:
+    image: timberio/vector:0.49.X-debian
     ports:
-      - "12201:12201/udp"
+      - "12201:12201/udp"  # GELF input
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      - VECTOR_CONFIG=/etc/vector/vector.yaml
+      - VECTOR_LOG=info
+    configs:
+      - source: vector_config
+        target: /etc/vector/vector.yaml
     deploy:
-      restart_policy:
-        condition: on-failure
       resources:
         limits:
-          cpus: '1.0'
-          memory: 1G
-        reservations:
-          cpus: '0.5'
+          cpu: "1.0"
           memory: 512M
-      update_config:
-        parallelism: 1
-        delay: 10s
-        order: start-first
-      labels:
-        - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
-        - traefik.http.services.fluentbit.loadbalancer.server.port=2020
-        - traefik.http.routers.fluentbit.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/fluentbit`)
-        - traefik.http.routers.fluentbit.entrypoints=https
-        - traefik.http.routers.fluentbit.tls=true
-        - traefik.http.middlewares.fluentbit_replace_regex.replacepathregex.regex=^/fluentbit/?(.*)$$
-        - traefik.http.middlewares.fluentbit_replace_regex.replacepathregex.replacement=/$${1}
-        - traefik.http.routers.fluentbit.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, fluentbit_replace_regex
+        reservations:
+          memory: 256M
+      labels: []
     networks:
       - monitoring
       - graylog
@@ -197,9 +182,9 @@ configs:
   graylog_config:
     name: ${STACK_NAME}_graylog_config_{{ "./data/contentpacks/osparc-custom-content-pack-v2.json" | sha256file | substring(0,10) }}
     file: ./data/contentpacks/osparc-custom-content-pack-v2.json
-  fluentbit_config:
-    name: ${STACK_NAME}_fluentbit_config_{{ "./fluentbit/fluentbit.conf" | sha256file | substring(0,10) }}
-    file: ./fluentbit/fluentbit.conf
+  vector_config:
+    name: ${STACK_NAME}_vector_config_{{ "./vector.yaml" | sha256file | substring(0,10) }}
+    file: ./vector.yaml
   loki_config:
     name: ${STACK_NAME}_loki_config_{{ "./loki.yaml" | sha256file | substring(0,10) }}
     file: ./loki.yaml

services/logging/fluentbit/fluentbit.conf

@@ -0,0 +1,94 @@
+# Vector configuration to replace Fluent Bit
+# Ingests GELF logs from Docker daemon and forwards to Loki and Graylog
+
+sources:
+  # Receive GELF messages from Docker containers via UDP
+  docker_gelf:
+    type: socket
+    address: "0.0.0.0:12201"
+    mode: udp
+    decoding:
+      codec: gelf
+
+transforms:
+  # Process and enrich the logs
+  process_logs:
+    type: remap
+    inputs: ["docker_gelf"]
+    source: |
+      if !exists(.host) {
+        .host = get_hostname!()
+      }
+
+      # Map short_message to message for Loki compatibility
+      if exists(.short_message) {
+        .message = .short_message
+      }
+
+      # Handle container name - GELF uses _container_name (with underscore prefix)
+      if exists(._container_name) {
+        .container_name = ._container_name
+      } else {
+        .container_name = "unknown"
+      }
+
+      # Handle container ID
+      if exists(._container_id) {
+        .container_id = ._container_id
+      } else {
+        .container_id = "unknown"
+      }
+
+      # Handle image name
+      if exists(._image_name) {
+        .image_name = ._image_name
+      } else {
+        .image_name = "unknown"
+      }
+
+      # Add processing metadata
+      .processed_by = "vector"
+
+sinks:
+  # Send to Loki
+  loki:
+    type: loki
+    inputs: ["process_logs"]
+    endpoint: "http://loki:3100"
+    encoding:
+      codec: json
+    labels:
+      job: "docker"
+      source: "vector"
+      # Simple field references - defaults are set in transform above
+      host: "{{ host }}"
+      container_name: "{{ container_name }}"
+    # Remove label fields from the log line to avoid duplication
+    remove_label_fields: true
+    healthcheck:
+      enabled: true
+
+  # Send to Graylog via GELF over UDP (not TCP with framing)
+  graylog:
+    type: socket
+    inputs: ["process_logs"]
+    address: "logging_graylog:12201"
+    mode: udp
+    encoding:
+      codec: gelf
+    healthcheck:
+      enabled: true
+
+  # Temporary: Output to console for debugging
+  #console_debug:
+  #  type: console
+  #  inputs: ["process_logs"]
+  #  encoding:
+  #    codec: json
+
+# Global configuration
+api:
+  enabled: true
+  address: "0.0.0.0:8686"
+
+data_dir: "/var/lib/vector"