[telemetry] support prometheus sink

Author: rustielin

Cargo.lock

@@ -35,6 +35,7 @@ gcp-bigquery-client = { workspace = true }
 jsonwebtoken = { workspace = true }
 once_cell = { workspace = true }
 prometheus = { workspace = true }
+prost = { workspace = true }
 rand = { workspace = true }
 rand_core = { workspace = true }
 reqwest = { workspace = true }
@@ -43,6 +44,7 @@ reqwest-retry = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
+snap = "1.1"
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }

crates/aptos-telemetry-service/Cargo.toml

@@ -46,6 +46,26 @@ else
         echo -e "${YELLOW}No node process found${NC}"
     fi
 fi
+
+# Stop telemetry service
+if [ -f "$TEST_DIR/telemetry.pid" ]; then
+    TELEMETRY_PID=$(cat "$TEST_DIR/telemetry.pid")
+    if ps -p $TELEMETRY_PID > /dev/null 2>&1; then
+        kill $TELEMETRY_PID
+        echo -e "${GREEN}✓ Telemetry service stopped (PID: $TELEMETRY_PID)${NC}"
+    else
+        echo -e "${YELLOW}Telemetry process not running${NC}"
+    fi
+    rm "$TEST_DIR/telemetry.pid"
+else
+    # Try to find and kill any telemetry service process on port 8082
+    if lsof -Pi :8082 -sTCP:LISTEN -t >/dev/null 2>&1; then
+        kill $(lsof -t -i:8082) 2>/dev/null || true
+        echo -e "${GREEN}✓ Stopped process on port 8082${NC}"
+    else
+        echo -e "${YELLOW}No telemetry service process found${NC}"
+    fi
+fi
 echo ""
 
 # Ask about data removal

crates/aptos-telemetry-service/e2e-test/cleanup.sh

@@ -1,9 +1,15 @@
 # Docker Compose setup for E2E testing of Aptos Telemetry Service
-# Runs VictoriaMetrics and Loki locally for telemetry data ingestion testing
+#
+# Architecture:
+# - VictoriaMetrics: Primary metrics backend (accepts Prometheus text format via /api/v1/import/prometheus)
+# - Prometheus: Secondary metrics store (accepts remote write via /api/v1/write)
+# - Loki: Log aggregation backend
+# - Grafana: Visualization (queries all backends)
 version: '3.8'
 
 services:
-  # VictoriaMetrics - Time-series database for metrics
+  # VictoriaMetrics - Primary metrics backend for telemetry ingestion
+  # Accepts Prometheus text format via /api/v1/import/prometheus (simpler than protobuf)
   victoria-metrics:
     image: victoriametrics/victoria-metrics:latest
     container_name: telemetry-victoria-metrics
@@ -23,6 +29,28 @@ services:
       timeout: 5s
       retries: 3
 
+  # Prometheus - Secondary metrics backend for remote write
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: telemetry-prometheus
+    ports:
+      - "9090:9090"  # HTTP API for queries and remote write
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+      - prometheus-data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=30d'
+      - '--web.enable-remote-write-receiver'  # Enable receiving remote writes
+      - '--web.enable-lifecycle'  # Enable /-/reload endpoint
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
   # Loki - Log aggregation system
   loki:
     image: grafana/loki:3.0.0
@@ -56,11 +84,14 @@ services:
     restart: unless-stopped
     depends_on:
       - victoria-metrics
+      - prometheus
       - loki
 
 volumes:
   victoria-data:
     driver: local
+  prometheus-data:
+    driver: local
   loki-data:
     driver: local
   grafana-data:

crates/aptos-telemetry-service/e2e-test/docker-compose.yaml

@@ -1,10 +1,16 @@
 # Grafana datasources for E2E testing
-# Auto-configures VictoriaMetrics and Loki datasources
+#
+# Architecture:
+# - VictoriaMetrics (default): Receives telemetry metrics via text format import
+# - Prometheus: Scrapes VictoriaMetrics, provides alternative PromQL endpoint
+# - Loki: Receives telemetry logs
+#
+# Both VictoriaMetrics and Prometheus support PromQL, so queries work on either.
 
 apiVersion: 1
 
 datasources:
-  # VictoriaMetrics (Prometheus-compatible) datasource
+  # VictoriaMetrics - Primary metrics backend (receives telemetry via push)
   - name: VictoriaMetrics
     type: prometheus
     access: proxy
@@ -15,7 +21,19 @@ datasources:
       timeInterval: 10s
     editable: true
 
-  # Loki datasource
+  # Prometheus - Secondary metrics backend (scrapes VictoriaMetrics)
+  # Demonstrates multi-backend querying with same PromQL syntax
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: false
+    jsonData:
+      httpMethod: POST
+      timeInterval: 15s
+    editable: true
+
+  # Loki datasource - log aggregation
   - name: Loki
     type: loki
     access: proxy