Run Grafana alloy to replace exporters

Author: alhendrickson

observability/build-docker.sh

@@ -0,0 +1,8 @@
+
+docker build -t cogstacksystems/cogstack-observability-prometheus:latest -f prometheus/Dockerfile.prometheus ./prometheus
+
+docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest -f prometheus/Dockerfile.blackbox ./prometheus
+
+docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana
+
+docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik

observability/examples/full/docker-compose.yml

@@ -5,6 +5,8 @@ services:
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
+    ports:
+      - "9090:9090"
     volumes:
       - ./prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
@@ -29,6 +31,30 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  alloy:
+    image: grafana/alloy:latest
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.ui-path-prefix=/alloy
+      - /etc/alloy/config.alloy
+    ports:
+      - "12345:12345"
+    volumes:
+      - ./grafana-alloy/config.alloy:/etc/alloy/config.alloy
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.cadvisor.rule=PathPrefix(`/alloy`)"
+    environment:
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write
+    networks:
+      - observability
 networks:
   observability:
     driver: bridge

observability/examples/full/grafana-alloy/config.alloy

@@ -0,0 +1,36 @@
+logging {
+  level  = "debug"
+  format = "logfmt"
+}
+
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("PROMETHEUS_URL")
+  }
+}
+
+prometheus.scrape "exporter" {
+
+    scrape_interval = "15s"
+
+    targets    = array.concat(
+        prometheus.exporter.self.alloy.targets,
+        prometheus.exporter.cadvisor.local_cadvisor.targets,
+        prometheus.exporter.unix.local_node_exporter.targets,
+    )
+    forward_to = [prometheus.remote_write.default.receiver]
+}
+
+// Alloys internal metrics
+prometheus.exporter.self "alloy" {
+}
+
+// CAdvisor
+prometheus.exporter.cadvisor "local_cadvisor" {
+  docker_host = "unix:///var/run/docker.sock"
+  storage_duration = "5m"
+}
+
+// Node exporter
+prometheus.exporter.unix "local_node_exporter" {
+}

observability/prometheus/Dockerfile.prometheus

@@ -10,5 +10,6 @@ CMD [ \
     "--storage.tsdb.path=/prometheus", \
     "--storage.tsdb.retention.time=30d", \
     "--web.external-url=/prometheus", \
-    "--web.route-prefix=/prometheus" \
+    "--web.route-prefix=/prometheus", \
+    "--web.enable-remote-write-receiver" \
     ]