feat: setup otlp prometheus exporter with correct invocation in node (#1087)

Co-authored-by: Andreas Penzkofer <[email protected]>
Co-authored-by: Mikhail Zabaluev <[email protected]>
Co-authored-by: Liam Monninger <[email protected]>
Co-authored-by: Andy Golay <[email protected]>
Co-authored-by: musitdev <[email protected]>
Co-authored-by: Richard Melkonian <[email protected]>
Co-authored-by: primata <[email protected]>
Co-authored-by: Richard Melkonian <[email protected]>
Co-authored-by: Radu Popa <[email protected]>
Co-authored-by: Liam Monninger <[email protected]>

Author: 9 people

.github/workflows/checks-all.yml

@@ -0,0 +1,78 @@
+services:
+  prometheus:
+    image: prom/prometheus:v2.49.1
+    container_name: movement-prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:9090/-/healthy || exit 1
+      interval: 5s
+      timeout: 3s
+      retries: 3
+
+  grafana:
+    image: grafana/grafana:10.2.3
+    container_name: movement-grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+    depends_on:
+      - prometheus
+
+  movement-full-node:
+    environment:
+      - MOVEMENT_METRICS_ADDR=127.0.0.1:9464
+      - MOVEMENT_OTLP=http://otel-collector:4317
+    ports:
+      - "9464:9464"
+    depends_on:
+      - prometheus:
+          condition: service_healthy
+      - otel-collector:
+          condition: service_healthy
+
+  movement-celestia-da-light-node:
+    environment:
+      - MOVEMENT_METRICS_ADDR=127.0.0.1:9464
+      - MOVEMENT_OTLP=http://otel-collector:4317
+    ports:
+      - "9465:9464"
+    depends_on:
+      - prometheus:
+          condition: service_healthy
+      - otel-collector:
+          condition: service_healthy
+
+  otel-collector:
+    image: otel/opentelemetry-collector:0.96.0
+    container_name: movement-otel-collector
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP
+      - "8888:8888" # Prometheus metrics exposed by the collector
+      - "8889:8889" # Prometheus exporter metrics
+      - "13133:13133" # Health check extension
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:13133 || exit 1
+      interval: 5s
+      timeout: 3s
+      retries: 3
+
+volumes:
+  prometheus_data:

Cargo.lock

@@ -0,0 +1,37 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 127.0.0.1:4317
+      http:
+        endpoint: 127.0.0.1:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
+
+exporters:
+  prometheus:
+    endpoint: "127.0.0.1:8889"
+    namespace: "movement"
+    const_labels:
+      label1: value1
+  logging:
+    loglevel: debug
+
+extensions:
+  health_check:
+    endpoint: 127.0.0.1:13133
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheus, logging]

docker/compose/movement-full-node/docker-compose.telemetry.yml

@@ -0,0 +1,10 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: "movement-full-node"
+    static_configs:
+      - targets: ["127.0.0.1:9464"]
+    metrics_path: /metrics
+    scheme: http

docker/compose/movement-full-node/otel-collector-config.yaml

@@ -12,8 +12,6 @@ use tokio::sync::RwLock;
 use tracing::info;
 use url::Url;
 
-const TIMING_LOG_ENV: &str = "SUZUKA_TIMING_LOG";
-
 pub fn get_movement_config(
 	dot_movement: &DotMovement,
 ) -> Result<movement_config::Config, anyhow::Error> {
@@ -394,14 +392,11 @@ pub async fn basic_coin_transfers(
 
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
-	let tracing_config = movement_tracing::Config {
-		timing_log_path: std::env::var_os(TIMING_LOG_ENV).map(Into::into),
-	};
+	let dot_movement = DotMovement::try_from_env().context("Failed to get .movement path")?;
+	let config = get_movement_config(&dot_movement)?;
+	let tracing_config = movement_tracing::Config::default();
 	let _guard = movement_tracing::init_tracing_subscriber(tracing_config);
 
-	// Get the lead dot movement from the environment.
-	let dot_movement = DotMovement::try_from_env()?;
-
 	// Get the follower count from the first argument.
 	let follower_count = std::env::args()
 		.nth(1)

docker/compose/movement-full-node/prometheus.yml

@@ -2,18 +2,18 @@
 
 use clap::*;
 use movement_full_node::MovementFullNode;
-const TIMING_LOG_ENV: &str = "SUZUKA_TIMING_LOG";
-use std::env;
+use std::time::Duration;
 
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
-	let tracing_config =
-		movement_tracing::Config { timing_log_path: env::var_os(TIMING_LOG_ENV).map(Into::into) };
-	let _guard = movement_tracing::init_tracing_subscriber(tracing_config);
+    let tracing_config = movement_tracing::Config::default();
+    let _guard = movement_tracing::init_telemetry(tracing_config).await;
+    
+    tokio::time::sleep(Duration::from_secs(1)).await;
 
-	let suzuka_util = MovementFullNode::parse();
+    let suzuka_util = MovementFullNode::parse();
+    let result = suzuka_util.execute().await;
 
-	suzuka_util.execute().await?;
-
-	Ok(())
+    result
 }
+

networks/movement/movement-client/src/bin/e2e/followers_consistent.rs

@@ -0,0 +1,26 @@
+version: "3"
+
+environment:
+
+processes:
+  prometheus:
+    is_daemon: true
+    command: |
+      mkdir -p ${PWD}/tmp/prometheus
+      cp ${PWD}/docker/compose/movement-full-node/prometheus.yml ${PWD}/tmp/prometheus/prometheus.yml
+      sed -i.bak 's/movement-full-node:9464/127.0.0.1:9464/g' ${PWD}/tmp/prometheus/prometheus.yml
+      prometheus --config.file=${PWD}/tmp/prometheus/prometheus.yml --storage.tsdb.path=${PWD}/tmp/prometheus/data --web.listen-address=0.0.0.0:9091 --web.enable-lifecycle
+    shutdown:
+      command: |
+        pkill -f "prometheus --config.file"
+    readiness_probe:
+      initial_delay_seconds: 3
+      exec:
+        command: curl -s http://127.0.0.1:9091/-/healthy || exit 1
+
+  movement-full-node:
+    depends_on:
+      prometheus:
+        condition: process_started
+    environment:
+      - MOVEMENT_METRICS_ADDR=127.0.0.1:9464

networks/movement/movement-full-node/src/main.rs

@@ -0,0 +1 @@
+build

process-compose/movement-full-node/process-compose.telemetry.yml

@@ -0,0 +1,8 @@
+[package]
+name = "Minter"
+version = "0.0.0"
+
+[dependencies.AptosFramework]
+git = "https://github.com/movementlabsxyz/aptos-core.git"
+rev = "movement"
+subdir = "aptos-move/framework/aptos-framework"