feat: start Grafana and Prometheus when --metrics is passed

.gitignore

@@ -22,3 +22,6 @@ hosts.yml
 tmp/
 
 ansible-deployment*.log
+
+# Generated Prometheus config (created by generate-prometheus-config.sh)
+metrics/prometheus/prometheus.yml

README.md

@@ -65,11 +65,23 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --generateGenesis --popupT
 
 ### Enabling metrics
 
+The `--metrics` flag starts a **Prometheus + Grafana** monitoring stack alongside the devnet nodes. Prometheus scrapes all node metrics endpoints and Grafana provides pre-built dashboards for monitoring consensus health.
+
 ```sh
-# Start all nodes with metrics enabled
+# Start all nodes with metrics stack
 NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --metrics
 ```
 
+Once running, services are available at:
+- **Grafana:** http://localhost:3000 (no login required)
+- **Prometheus:** http://localhost:9090
+
+Grafana is started with the two pre-provisioned dashboards from [leanMetrics](https://github.com/leanEthereum/leanMetrics):
+- **Lean Ethereum Client Interop Dashboard**: for seeing a general overview of all clients
+- **Lean Ethereum Client Dashboard**: for viewing metrics for a single client
+
+> **Note:** The `--metrics` flag only affects local deployments. When using Ansible deployment mode, this flag is ignored. Metrics ports are always exposed by clients regardless of this flag.
+
 ## Args
 
 1. `NETWORK_DIR` is an env to specify the network directory. Should have a `genesis` directory with genesis config. A `data` folder will be created inside this `NETWORK_DIR` if not already there.
@@ -119,7 +131,14 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --metrics
    - If not provided, defaults to `latest` for zeam, ream, and lantern, and `dd67521` for qlean
    - The script will automatically pull the specified Docker images before running containers
    - Example: `--tag devnet0` or `--tag devnet1`
-11. `--metrics` enables metrics collection on all nodes. When specified, each client will activate its metrics endpoint according to its implementation. Metrics ports are configured per node in `validator-config.yaml`.
+11. `--metrics` starts a Prometheus + Grafana metrics stack alongside the devnet (local deployments only). When specified:
+    - Generates `metrics/prometheus/prometheus.yml` from `validator-config.yaml` with scrape targets for all configured nodes
+    - Starts Prometheus (http://localhost:9090) and Grafana (http://localhost:3000) via Docker Compose
+    - Grafana is pre-provisioned with Lean Ethereum dashboards (no login required)
+    - On `--stop --metrics`, the metrics stack is also torn down
+    - On Ctrl+C cleanup, the metrics stack is stopped automatically
+
+    Note: Client metrics endpoints are always enabled regardless of this flag.
 
 ### Clients supported
 

generate-prometheus-config.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Generate prometheus.yml from validator-config.yaml
+# Reads all validators and creates scrape targets using host.docker.internal
+#
+# Usage: ./generate-prometheus-config.sh <validator-config.yaml> <output-dir>
+# Example: ./generate-prometheus-config.sh local-devnet/genesis/validator-config.yaml metrics/prometheus
+
+set -e
+
+validator_config="$1"
+output_dir="$2"
+
+if [ -z "$validator_config" ] || [ -z "$output_dir" ]; then
+    echo "Usage: $0 <validator-config.yaml> <output-dir>"
+    exit 1
+fi
+
+if [ ! -f "$validator_config" ]; then
+    echo "Error: Validator config not found at $validator_config"
+    exit 1
+fi
+
+if ! command -v yq &> /dev/null; then
+    echo "Error: yq is required but not installed."
+    exit 1
+fi
+
+mkdir -p "$output_dir"
+
+# Extract node names and metrics ports
+node_count=$(yq eval '.validators | length' "$validator_config")
+
+# Build scrape configs
+scrape_configs=""
+for ((i=0; i<node_count; i++)); do
+    name=$(yq eval ".validators[$i].name" "$validator_config")
+    port=$(yq eval ".validators[$i].metricsPort" "$validator_config")
+
+    if [ -z "$name" ] || [ "$name" == "null" ] || [ -z "$port" ] || [ "$port" == "null" ]; then
+        echo "Warning: Skipping validator $i (missing name or metricsPort)"
+        continue
+    fi
+
+    scrape_configs="${scrape_configs}
+  - job_name: \"${name}\"
+    static_configs:
+      - targets: [\"host.docker.internal:${port}\"]
+        labels:
+          client: \"${name}\"
+          instance: \"local\""
+done
+
+# Write prometheus.yml
+cat > "$output_dir/prometheus.yml" << EOF
+# Auto-generated by generate-prometheus-config.sh from validator-config.yaml
+# Do not edit manually — changes will be overwritten on next devnet start.
+
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: "lean-devnet-metrics"
+
+scrape_configs:
+${scrape_configs}
+
+  # Prometheus self-monitoring
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+EOF
+
+echo "Generated $output_dir/prometheus.yml with $node_count scrape targets"

metrics/docker-compose-metrics.yaml

@@ -0,0 +1,51 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: lean-prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/etc/prometheus/console_libraries"
+      - "--web.console.templates=/etc/prometheus/consoles"
+      - "--storage.tsdb.retention.time=30d"
+      - "--web.enable-lifecycle"
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    networks:
+      - metrics-network
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: lean-grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    networks:
+      - metrics-network
+    restart: unless-stopped
+    depends_on:
+      - prometheus
+
+volumes:
+  prometheus-data:
+    driver: local
+  grafana-data:
+    driver: local
+
+networks:
+  metrics-network:
+    driver: bridge