WIP: use fluentd to collect logs

Author: willcl-ark

src/services/fluentd.py

@@ -0,0 +1,24 @@
+from .base_service import BaseService
+
+FLUENT_IP = "100.102.108.117"
+FLUENT_CONF = "fluent.conf"
+
+class Fluentd(BaseService):
+    PORT = 24224
+
+    def __init__(self, docker_network, config_dir):
+        super().__init__(docker_network, config_dir)
+        self.service = {
+            "image": "fluent/fluentd:v1.16-debian-1", # Debian version is recommended officially since it has jemalloc support.
+            "container_name": f"{self.docker_network}_fluentd",
+            "ports": [f"{self.PORT}:{self.PORT}"],
+            "volumes": [
+                f"{self.config_dir / FLUENT_CONF}:/fluentd/etc/{FLUENT_CONF}"
+            ],
+            "command": ["/bin/sh", "-c", f"sleep 10 && fluentd -c /fluentd/etc/{FLUENT_CONF}"],
+            "networks": {
+                self.docker_network: {
+                    "ipv4_address": f"{FLUENT_IP}",
+                }
+            },
+        }

src/templates/fluent.conf

@@ -0,0 +1,9 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<match *>
+  @type stdout
+</match>

src/warnet/tank.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from docker.api import service
 from docker.models.containers import Container
+from services.fluentd import FLUENT_IP
 from templates import TEMPLATES
 from warnet.utils import (
     exponential_backoff,
@@ -241,22 +242,30 @@ def add_services(self, services):
                 },
                 "labels": {"warnet": "tank"},
                 "privileged": True,
+                # "depends_on": ["fluentd"],
+                # "logging": {
+                #     "driver": "fluentd",
+                #     "options": {
+                #         "fluentd-address": f"{FLUENT_IP}:24224",
+                #         "tag": "{{.Name}}"
+                #     }
+                # }
             }
         )
 
         # Add the prometheus data exporter in a neighboring container
-        # services[self.exporter_name] = {
-        #     "image": "jvstein/bitcoin-prometheus-exporter",
-        #     "container_name": self.exporter_name,
-        #     "environment": {
-        #         "BITCOIN_RPC_HOST": self.bitcoind_name,
-        #         "BITCOIN_RPC_PORT": self.rpc_port,
-        #         "BITCOIN_RPC_USER": self.rpc_user,
-        #         "BITCOIN_RPC_PASSWORD": self.rpc_password,
-        #     },
-        #     "ports": [f"{8335 + self.index}:9332"],
-        #     "networks": [self.docker_network],
-        # }
+        services[self.exporter_name] = {
+            "image": "jvstein/bitcoin-prometheus-exporter",
+            "container_name": self.exporter_name,
+            "environment": {
+                "BITCOIN_RPC_HOST": self.container_name,
+                "BITCOIN_RPC_PORT": self.rpc_port,
+                "BITCOIN_RPC_USER": self.rpc_user,
+                "BITCOIN_RPC_PASSWORD": self.rpc_password,
+            },
+            "ports": [f"{8335 + self.index}:9332"],
+            "networks": [self.docker_network],
+        }
 
     def add_scrapers(self, scrapers):
         scrapers.append(

src/warnet/warnet.py

@@ -17,6 +17,7 @@
 from services.grafana import Grafana
 from services.tor import Tor
 from services.fork_observer import ForkObserver
+from services.fluentd import FLUENT_CONF, Fluentd, FLUENT_IP
 from services.dns_seed import DnsSeed, ZONE_FILE_NAME, DNS_SEED_NAME
 from warnet.tank import Tank
 from warnet.utils import parse_bitcoin_conf, gen_config_dir, bubble_exception_str
@@ -43,6 +44,7 @@ def __init__(self, config_dir):
             f"copying config {TEMPLATES / FO_CONF_NAME} to {self.fork_observer_config}"
         )
         shutil.copy(TEMPLATES / FO_CONF_NAME, self.fork_observer_config)
+        shutil.copy(TEMPLATES / FLUENT_CONF, self.config_dir)
 
     def __str__(self) -> str:
         tanks_str = ",\n".join([str(tank) for tank in self.tanks])
@@ -266,6 +268,7 @@ def write_docker_compose(self, dns=True):
             Grafana(self.docker_network),
             Tor(self.docker_network, TEMPLATES),
             ForkObserver(self.docker_network, self.fork_observer_config),
+            # Fluentd(self.docker_network, self.config_dir),
         ]
         if dns:
             services.append(DnsSeed(self.docker_network, TEMPLATES, self.config_dir))