feat: vector axiom scraping[

Author: izadoesdev

infra/ingest/env.example

@@ -14,6 +14,14 @@ REDPANDA_PASSWORD=super_secret_password
 VECTOR_LOG=warn
 VECTOR_KAFKA_USER=vector-agent
 VECTOR_KAFKA_PASSWORD=super_secret_password
+    
+REDPANDA_METRICS_ENDPOINT=http://redpanda:9644/public_metrics
+REDPANDA_CLUSTER_NAME=databuddy-prod
+ENVIRONMENT=production
+
+# Axiom Configuration
+AXIOM_DATASET=redpanda
+AXIOM_TOKEN=your-axiom-ingest-token
 
 # Console Configuration
-CONSOLE_JWT_SIGNING_KEY=databuddy-console-jwt-secret-key-change-in-production-min-32-chars
+CONSOLE_JWT_SIGNING_KEY=databuddy-console-jwt-secret-kenvy-change-in-production-min-32-chars

infra/ingest/vector.yaml

@@ -20,6 +20,13 @@ sources:
     decoding:
       codec: json
 
+  redpanda_metrics:
+    type: prometheus_scrape
+    endpoints:
+      - "${REDPANDA_METRICS_ENDPOINT}"
+    scrape_interval_secs: 15
+    scrape_timeout_secs: 5
+
 transforms:
   route_analytics:
     type: route
@@ -34,6 +41,14 @@ transforms:
       email_events: '.topic == "analytics-email-events"'
       blocked_traffic: '.topic == "analytics-blocked-traffic"'
 
+  redpanda_metrics_transform:
+    type: remap
+    inputs:
+      - redpanda_metrics
+    source: |
+      .tags.cluster = "${REDPANDA_CLUSTER_NAME}"
+      .tags.env = "${ENVIRONMENT}"
+
 sinks:
   clickhouse_events:
     type: clickhouse
@@ -182,3 +197,11 @@ sinks:
     path: /var/log/vector/dead_letters.log
     encoding:
       codec: json
+
+  axiom_redpanda_metrics:
+    type: axiom
+    inputs:
+      - redpanda_metrics_transform
+    dataset: "${AXIOM_DATASET}"
+    token: "${AXIOM_TOKEN}"
+    compression: gzip