Fluentd fixes

Author: mrnicegyu11

services/logging/docker-compose.local.yml

@@ -13,3 +13,8 @@ services:
     deploy:
       placement:
         constraints: []
+
+  fluentd:
+    deploy:
+      placement:
+        constraints: []

services/logging/docker-compose.yml.j2

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   # MongoDB: https://hub.docker.com/_/mongo/
   mongodb:
@@ -102,7 +101,7 @@ services:
         - traefik.http.middlewares.graylog_replace_regex.replacepathregex.replacement=/$${1}
         - traefik.http.routers.graylog.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, graylog_replace_regex
   fluentd:
-    image: itisfoundation/fluentd:v1.16.8-1.0
+    image: itisfoundation/fluentd:v1.16.9-1.0
     configs:
       - source: fluentd_config
         target: /fluentd/etc/fluent.conf

services/logging/fluentd/Dockerfile

@@ -1,4 +1,4 @@
-FROM fluent/fluentd:v1.16.8-1.0
+FROM fluent/fluentd:v1.16.9-1.0
 
 USER root
 

services/logging/fluentd/Makefile

@@ -0,0 +1,9 @@
+.DEFAULT_GOAL := help
+STACK_NAME = $(notdir $(shell pwd)/..)
+REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
+
+# TARGETS --------------------------------------------------
+include ${REPO_BASE_DIR}/scripts/common.Makefile
+
+build:
+	@docker build -t itisfoundation/fluentd:v1.16.9-1.0 .

services/logging/fluentd/fluent.conf

@@ -13,9 +13,9 @@
   metrics_path metrics
 </source>
 
-#<system>
-#  log_level debug
-#</system>
+<system>
+  workers 1
+</system>
 
 <source>
   @type prometheus_output_monitor
@@ -49,7 +49,7 @@
   # Output to Graylog using GELF
   <store>
     @type gelf
-    host graylog
+    host logging_graylog
     port 12201
     protocol udp
     add_msec_time true
@@ -59,10 +59,14 @@
       path /fluentd/buffer/graylog
       flush_thread_count 8
       flush_interval 5s
-      retry_forever true
-      retry_max_interval 30
+      retry_forever false
+      retry_timeout 1h
+      retry_max_times 30
+      retry_randomize true
       chunk_limit_size 8M
-      total_limit_size 1G
+      total_limit_size 2G
+      overflow_action drop_oldest_chunk
+      flush_mode interval
     </buffer>
     <secondary>
       @type file
@@ -88,10 +92,12 @@
       path /fluentd/buffer/loki
       flush_thread_count 8
       flush_interval 5s
-      retry_forever true
+      retry_forever false
       retry_max_interval 30
+      retry_max_times 30
+      retry_randomize true
       chunk_limit_size 8M
-      total_limit_size 1G
+      total_limit_size 2G
     </buffer>
     <secondary>
       @type file