Add 'cardano-nodes' monitoring

jpraynaud · jpraynaud · commit f190cf78b9be · 2023-07-04T12:35:03.000+02:00
diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-auth.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-auth.yaml
@@ -17,7 +17,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-aggregator/cardano/config/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-aggregator/cardano/db:/db
       - ../data/${NETWORK}/mithril-aggregator/cardano/ipc:/ipc
     environment:
diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-noauth.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-noauth.yaml
@@ -17,7 +17,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-aggregator/cardano/config/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-aggregator/cardano/db:/db
       - ../data/${NETWORK}/mithril-aggregator/cardano/ipc:/ipc
     environment:
diff --git a/mithril-infra/assets/docker/docker-compose-monitoring.yaml b/mithril-infra/assets/docker/docker-compose-monitoring.yaml
@@ -6,14 +6,15 @@ services:
     container_name: prometheus
     profiles:
       - tools
+      - monitoring
       - all
     ports:
       - "9090:9090"
     command:
       - --config.file=/etc/prometheus/prometheus.yml
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ./prometheus/targets.json:/etc/prometheus/targets.json:ro
+      - ./prometheus/cardano.json:/etc/prometheus/cardano.json:ro
     labels:
       - 'traefik.enable=true'
       - 'traefik.http.routers.prometheus.rule=Host(`${PROMETHEUS_HOST}`)'
@@ -25,6 +26,10 @@ services:
   node-exporter:
     image: prom/node-exporter:latest
     container_name: node-exporter
+    profiles:
+      - tools
+      - monitoring
+      - all
     restart: unless-stopped
     volumes:
       - /proc:/host/proc:ro
@@ -37,10 +42,11 @@ services:
       - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
 
   promtail:
-    image: grafana/promtail:1.4.1
+    image: grafana/promtail:latest
     container_name: promtail
     profiles:
       - tools
+      - monitoring
       - all
     volumes:
       - /var/lib/docker/containers:/var/lib/docker/containers
diff --git a/mithril-infra/assets/docker/docker-compose-signer-unverified-norelay.yaml b/mithril-infra/assets/docker/docker-compose-signer-unverified-norelay.yaml
@@ -5,7 +5,7 @@ version: "3.9"
 services:
   cardano-node-signer:
     image: cardano-node/${CARDANO_IMAGE_ID}-modified
-    container_name: cardano-node-signer-${SIGNER_ID}
+    container_name: cardano-node-full-signer-${SIGNER_ID}
     restart: always
     build:
       context: .
@@ -19,7 +19,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/full/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/passive:/db
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc
     environment:
diff --git a/mithril-infra/assets/docker/docker-compose-signer-unverified.yaml b/mithril-infra/assets/docker/docker-compose-signer-unverified.yaml
@@ -5,7 +5,7 @@ version: "3.9"
 services:
   cardano-node-signer:
     image: cardano-node/${CARDANO_IMAGE_ID}-modified
-    container_name: cardano-node-signer-${SIGNER_ID}
+    container_name: cardano-node-full-signer-${SIGNER_ID}
     restart: always
     build:
       context: .
@@ -19,7 +19,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/full/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/passive:/db
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc
     environment:
diff --git a/mithril-infra/assets/docker/docker-compose-signer-verified-norelay.yaml b/mithril-infra/assets/docker/docker-compose-signer-verified-norelay.yaml
@@ -19,7 +19,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/relay/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/relay:/db
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/pool:/pool
@@ -58,7 +58,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/block-producer/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/block-producer:/db
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/pool:/pool
diff --git a/mithril-infra/assets/docker/docker-compose-signer-verified.yaml b/mithril-infra/assets/docker/docker-compose-signer-verified.yaml
@@ -19,7 +19,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/relay/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/relay:/db
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/pool:/pool
@@ -58,7 +58,7 @@ services:
     logging:
       driver: "${LOGGING_DRIVER}"
     volumes:
-      - ./cardano-configurations/network/${NETWORK}:/config
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/block-producer/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/block-producer:/db
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/pool:/pool
diff --git a/mithril-infra/assets/docker/prometheus/cardano.json b/mithril-infra/assets/docker/prometheus/cardano.json
@@ -0,0 +1 @@
+[]
diff --git a/mithril-infra/assets/docker/prometheus/prometheus.yml b/mithril-infra/assets/docker/prometheus/prometheus.yml
@@ -2,18 +2,22 @@ global:
   scrape_interval:     15s 
   evaluation_interval: 15s
 
-rule_files:
-  # - "first_rules.yml"
-  # - "second_rules.yml"
-
 scrape_configs:
   - job_name: 'prometheus'
     static_configs:
     - targets: ['prometheus:9090']
 
-  - job_name: 'targets-discovery'
+  - job_name: 'node-exporter'
+    static_configs:
+    - targets: ['node-exporter:9100']
+
+  - job_name: 'reverse-proxy'
+    static_configs:
+    - targets: ['reverse-proxy:8080']
+
+  - job_name: 'cardano-nodes'
     file_sd_configs:
     - files:
-      - 'targets.json'
+      - 'cardano.json'
 
 
diff --git a/mithril-infra/assets/docker/prometheus/targets.json b/mithril-infra/assets/docker/prometheus/targets.json
diff --git a/mithril-infra/mithril.aggregator.tf b/mithril-infra/mithril.aggregator.tf
@@ -21,10 +21,20 @@ resource "null_resource" "mithril_aggregator" {
   provisioner "remote-exec" {
     inline = [
       "mkdir -p /home/curry/data/${var.cardano_network}",
+      "mkdir -p /home/curry/data/${var.cardano_network}/mithril-aggregator/cardano/config",
       "mkdir -p /home/curry/data/${var.cardano_network}/mithril-aggregator/cardano/db",
       "mkdir -p /home/curry/data/${var.cardano_network}/mithril-aggregator/cardano/ipc",
       "mkdir -p /home/curry/data/${var.cardano_network}/mithril-aggregator/mithril/stores",
-      "mkdir -p /home/curry/data/${var.cardano_network}/mithril-aggregator/mithril/snapshots"
+      "mkdir -p /home/curry/data/${var.cardano_network}/mithril-aggregator/mithril/snapshots",
+      <<-EOT
+set -e
+# Setup cardano node configuration
+AGGREGATOR_CONFIG_DIRECTORY=/home/curry/data/${var.cardano_network}/mithril-aggregator/cardano/config
+cp -R /home/curry/docker/cardano-configurations/network/${var.cardano_network} $AGGREGATOR_CONFIG_DIRECTORY
+cat $AGGREGATOR_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json | jq ".hasPrometheus[0] |= \"cardano-node-aggregator\"" > $AGGREGATOR_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json.new
+rm -f $AGGREGATOR_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json
+mv $AGGREGATOR_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json.new $AGGREGATOR_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json
+EOT
     ]
   }
 
diff --git a/mithril-infra/mithril.monitoring.tf b/mithril-infra/mithril.monitoring.tf
@@ -21,6 +21,23 @@ resource "null_resource" "mithril_monitoring" {
     host        = google_compute_address.mithril-external-address.address
   }
 
+  provisioner "remote-exec" {
+    inline = [
+      <<-EOT
+# Setup prometheus targets configuration for Cardano nodes
+CARDANO_NODES=$(docker ps --format='{{.Names}}:12798,' | grep "cardano-node" | sort | tr -d '\n\t\r ' | sed 's/.$//')
+cat /home/curry/docker/prometheus/cardano.json | jq --arg CARDANO_NODES "$CARDANO_NODES" '. += [{
+    "labels": {
+        "job": "cardano-nodes"
+    },
+    "targets": $CARDANO_NODES
+}]' | jq '. | map(try(.targets |= split(",")) // .)' > /home/curry/docker/prometheus/cardano.json.new
+rm -f /home/curry/docker/prometheus/cardano.json
+mv /home/curry/docker/prometheus/cardano.json.new docker/prometheus/cardano.json
+EOT
+    ]
+  }
+
   provisioner "remote-exec" {
     inline = [
       "export PROMETHEUS_HOST=${local.prometheus_host}",
diff --git a/mithril-infra/mithril.signer.tf b/mithril-infra/mithril.signer.tf
@@ -28,6 +28,7 @@ resource "null_resource" "mithril_signer" {
   provisioner "remote-exec" {
     inline = [
       "mkdir -p /home/curry/data/${var.cardano_network}",
+      "mkdir -p /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/config",
       "mkdir -p /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/db",
       "mkdir -p /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/db/passive",
       "mkdir -p /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/db/block-producer",
@@ -39,6 +40,7 @@ resource "null_resource" "mithril_signer" {
       "mkdir -p /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/mithril/snapshots",
       "echo -n ${local.mithril_signers_relay_cardano_port[each.key]} > /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/pool/port",
       <<-EOT
+# Setup cardano node topology
 cat > /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/pool/topology-block-producer.json << EOF
 {
   "Producers": [
@@ -51,6 +53,19 @@ cat > /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano
 }
 EOF
 cat /home/curry/docker/cardano-configurations/network/${var.cardano_network}/cardano-node/topology.json | jq '.Producers[1] |= . + { "addr": "${google_compute_address.mithril-external-address.address}", "port": ${local.mithril_signers_block_producer_cardano_port[each.key]}, "valency": 1}' > /home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/pool/topology-relay.json
+EOT
+      ,
+      <<-EOT
+# Setup cardano node configuration
+SIGNER_TYPES="full relay block-producer"
+for SIGNER_TYPE in $SIGNER_TYPES; do
+  SIGNER_TYPE_CONFIG_DIRECTORY=/home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/cardano/config/$SIGNER_TYPE
+  mkdir -p $SIGNER_TYPE_CONFIG_DIRECTORY
+  cp -R /home/curry/docker/cardano-configurations/network/${var.cardano_network} $SIGNER_TYPE_CONFIG_DIRECTORY
+  cat $SIGNER_TYPE_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json | jq ".hasPrometheus[0] |= \"cardano-node-$SIGNER_TYPE-signer-${each.key}\"" > $SIGNER_TYPE_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json.new
+  rm -f $SIGNER_TYPE_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json
+  mv $SIGNER_TYPE_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json.new $SIGNER_TYPE_CONFIG_DIRECTORY/${var.cardano_network}/cardano-node/config.json
+done
 EOT
     ]
   }
