Add ALERT_DISCORD_IDS label and env variable

.gitignore

@@ -16,3 +16,4 @@ teku/validator/
 # Teku directory for storing logs
 teku/logs/
 commit-boost/config.toml
+prometheus/prometheus.yml

docker-compose.yml

@@ -198,6 +198,7 @@ services:
       - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro
       - ./grafana/dashboards:/etc/dashboards
       - ./data/grafana:/var/lib/grafana
+    entrypoint: /etc/prometheus/run.sh
     restart: unless-stopped
 
   loki:

prometheus/prometheus.yml

@@ -21,6 +21,9 @@ scrape_configs:
   - job_name: "charon"
     static_configs:
       - targets: ["charon:3620"]
+    relabel_configs:
+      - target_label: alert_discord_ids
+        replacement: "$ALERT_DISCORD_IDS"
   - job_name: "lodestar"
     static_configs:
       - targets: [ "lodestar:5064" ]

prometheus/prometheus.yml.example

@@ -0,0 +1,29 @@
+global:
+  scrape_interval: 30s # Set the scrape interval to every 30 seconds.
+  evaluation_interval: 30s # Evaluate rules every 30 seconds.
+
+remote_write:
+  - url: https://vm.monitoring.gcp.obol.tech/write
+    authorization:
+      credentials: $PROM_REMOTE_WRITE_TOKEN
+    write_relabel_configs:
+      - source_labels: [job]
+        regex: "charon|nethermind|lighthouse|lodestar"
+        action: keep # Keeps charon metrics and drop metrics from other containers.
+
+scrape_configs:
+  - job_name: "nethermind"
+    static_configs:
+      - targets: ["nethermind:8008"]
+  - job_name: "lighthouse"
+    static_configs:
+      - targets: ["lighthouse:5054"]
+  - job_name: "charon"
+    static_configs:
+      - targets: ["charon:3620"]
+  - job_name: "lodestar"
+    static_configs:
+      - targets: [ "lodestar:5064" ]
+  - job_name: "validator-ejector"
+    static_configs:
+      - targets: [ "validator-ejector:8989" ]

prometheus/run.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+if [ -z "$SERVICE_OWNER" ]
+then
+  echo "\$SERVICE_OWNER variable is empty" >&2
+  exit 1
+fi
+
+if [ -z "$PROM_REMOTE_WRITE_TOKEN" ]
+then
+  echo "\$PROM_REMOTE_WRITE_TOKEN variable is empty" >&2
+  exit 1
+fi
+
+sed -e "s|\$PROM_REMOTE_WRITE_TOKEN|${PROM_REMOTE_WRITE_TOKEN}|g" \
+    -e "s|\$SERVICE_OWNER|${SERVICE_OWNER}|g" \
+    -e "s|\$ALERT_DISCORD_IDS|${ALERT_DISCORD_IDS}|g" \
+    /etc/prometheus/prometheus.yml.example > /etc/prometheus/prometheus.yml
+
+/bin/prometheus \
+  --config.file=/etc/prometheus/prometheus.yml