add promtail (#321)

Co-authored-by: Aly <16789036+aly-obol@users.noreply.github.com>

Author: apham0001

.env.sample.mainnet

@@ -140,3 +140,14 @@ MEVBOOST_RELAYS=https://0xa15b52576bcbf1072f4a011c0f99f9fb6c66f3e1ff321f11f461d1
 # Grafana Loki docker container image version, e.g. `latest` or `2.8.2`.
 # See available tags https://hub.docker.com/r/grafana/loki/tags.
 #LOKI_VERSION=
+
+# Uncomment these if you have log exporting with Promtail
+# and want to disable log export on a particular container.
+
+# NETHERMIND_PROMTAIL_MONITORED=false
+# LIGHTHOUSE_PROMTAIL_MONITORED=false
+# CHARON_PROMTAIL_MONITORED=false
+# LODESTAR_PROMTAIL_MONITORED=false
+# MEV_BOOST_PROMTAIL_MONITORED=false
+# EJECTOR_PROMTAIL_MONITORED=false
+# DV_EXIT_PROMTAIL_MONITORED=false

docker-compose.yml

@@ -16,6 +16,8 @@ services:
       - ${NETHERMIND_PORT_P2P:-30303}:30303/udp # P2P UDP
       - ${NETHERMIND_IP_HTTP:-127.0.0.1}:${NETHERMIND_PORT_HTTP:-8545}:8545 # JSON-RPC
       - ${NETHERMIND_IP_ENGINE:-127.0.0.1}:${NETHERMIND_PORT_ENGINE:-8551}:8551 # ENGINE-API
+    labels:
+      - "promtail-monitored=${NETHERMIND_PROMTAIL_MONITORED:-true}"
     command: |
       --config=${NETWORK}
       --datadir=data
@@ -48,6 +50,8 @@ services:
     ports:
       - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/tcp # P2P TCP
       - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/udp # P2P UDP
+    labels:
+      - "promtail-monitored=${LIGHTHOUSE_PROMTAIL_MONITORED:-true}"
     command: |
       lighthouse bn
       --network=${NETWORK}
@@ -97,6 +101,8 @@ services:
       - CHARON_NICKNAME=${CHARON_NICKNAME:-}
     ports:
       - ${CHARON_PORT_P2P_TCP:-3610}:${CHARON_PORT_P2P_TCP:-3610}/tcp # P2P TCP libp2p
+    labels:
+      - "promtail-monitored=${CHARON_PROMTAIL_MONITORED:-true}"
     networks: [dvnode]
     volumes:
       - .charon:/opt/charon/.charon
@@ -120,6 +126,8 @@ services:
       NETWORK: ${NETWORK}
       BUILDER_API_ENABLED: ${BUILDER_API_ENABLED:-false}
       BUILDER_SELECTION: ${BUILDER_SELECTION:-builderalways}
+    labels:
+      - "promtail-monitored=${LODESTAR_PROMTAIL_MONITORED:-true}"
     volumes:
       - ./lodestar/run.sh:/opt/lodestar/run.sh
       - .charon/validator_keys:/home/charon/validator_keys
@@ -139,6 +147,8 @@ services:
       -addr=0.0.0.0:18550
       -relay-check
       -relays=${MEVBOOST_RELAYS}
+    labels:
+      - "promtail-monitored=${MEV_BOOST_PROMTAIL_MONITORED:-true}"
     networks: [dvnode]
     restart: unless-stopped
 

logging.yml

@@ -0,0 +1,18 @@
+services:
+
+  promtail:
+    image: grafana/promtail:${PROMTAIL_VERSION:-2.8.2}
+    environment:
+      CHARON_LOKI_ADDRESSES: ${CHARON_LOKI_ADDRESSES}
+      CLUSTER_NAME: ${CLUSTER_NAME}
+      CLUSTER_PEER: ${CLUSTER_PEER}
+    command: -config.file=/etc/promtail/config.yml
+    volumes:
+      - ./promtail:/etc/promtail
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks: [dvnode]
+    entrypoint: /etc/promtail/run.sh
+    restart: unless-stopped
+
+networks:
+  dvnode:

promtail/config.yml.example

@@ -0,0 +1,45 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: $CHARON_LOKI_ADDRESSES
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+    relabel_configs:      
+      - source_labels:
+          [__meta_docker_container_label_promtail_monitored]
+        regex: "true"
+        action: keep
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        replacement: '$1'
+        target_label: 'container'
+      - source_labels: ['container']
+        regex: '.*charon.*'
+        replacement: 'charon'
+        target_label: 'job'
+      - source_labels: ['container']
+        regex: '.*nethermind.*'
+        replacement: 'nethermind'
+        target_label: 'job'
+      - source_labels: ['container']
+        regex: '.*lodestar.*'
+        replacement: 'lodestar'
+        target_label: 'job'
+      - source_labels: ['container']
+        regex: '.*lighthouse.*'
+        replacement: 'lighthouse'
+        target_label: 'job'
+      - target_label: 'cluster_name'
+        replacement: $CLUSTER_NAME
+      - target_label: 'cluster_peer'
+        replacement: $CLUSTER_PEER
+    pipeline_stages:
+      - docker: {}

promtail/run.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+if [ -z "$CHARON_LOKI_ADDRESSES" ]; then
+  echo "Error: \$CHARON_LOKI_ADDRESSES variable is empty" >&2
+  exit 1
+fi
+
+if [ -z "$CLUSTER_NAME" ]; then
+  echo "Error: \$CLUSTER_NAME variable is empty" >&2
+  exit 1
+fi
+
+if [ -z "$CLUSTER_PEER" ]; then
+  echo "Error: \$CLUSTER_PEER variable is empty" >&2
+  exit 1
+fi
+
+# Process the template file once
+sed -e "s|\$CHARON_LOKI_ADDRESSES|${CHARON_LOKI_ADDRESSES}|g" \
+    -e "s|\$CLUSTER_NAME|${CLUSTER_NAME}|g" \
+    -e "s|\$CLUSTER_PEER|${CLUSTER_PEER}|g" \
+    /etc/promtail/config.yml.example > /etc/promtail/config.yml
+
+# Start Promtail with the generated config
+/usr/bin/promtail \
+  -config.file=/etc/promtail/config.yml