Add default labels to alloy scraped metrics

Author: alhendrickson

observability/examples/alloy/docker-compose.yml

@@ -8,7 +8,7 @@ services:
     ports:
       - "9090:9090"
     volumes:
-      - ${BASE_PATH-.}/prometheus:/etc/prometheus/cogstack/site/
+      - ${BASE_DIR-.}/prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
     networks:
       - observability
@@ -38,6 +38,11 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  blackbox-exporter:
+    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
+    restart: unless-stopped
+    networks:
+      - observability
   alloy:
     image: grafana/alloy:latest
     command:
@@ -49,7 +54,7 @@ services:
     ports:
       - "12345:12345"
     volumes:
-      - ${BASE_PATH-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
+      - ${BASE_DIR-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
       # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
@@ -59,7 +64,9 @@ services:
       - "traefik.enable=true"
       - "traefik.http.routers.cadvisor.rule=PathPrefix(`/alloy`)"
     environment:
-      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write
+      - PROMETHEUS_URL=${PROMETHEUS_URL-http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write}
+      - ALLOY_HOSTNAME=my-custom-host
+      - ALLOY_IP_ADDRESS=my-custom-ip
     networks:
       - observability
 networks:

observability/examples/alloy/grafana-alloy/config.alloy

@@ -7,12 +7,14 @@ prometheus.remote_write "default" {
   endpoint {
     url = sys.env("PROMETHEUS_URL")
   }
+  external_labels = {
+    host = sys.env("ALLOY_HOSTNAME"),
+    ip_address = sys.env("ALLOY_IP_ADDRESS"),
+  }
 }
 
 prometheus.scrape "exporter" {
-
     scrape_interval = "15s"
-
     targets    = array.concat(
         prometheus.exporter.self.alloy.targets,
         prometheus.exporter.cadvisor.local_cadvisor.targets,

observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml

@@ -0,0 +1,6 @@
+# Example of probe targets
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-services

observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml

@@ -0,0 +1,8 @@
+groups:
+  - name:  slo-target-rules
+    rules:
+      # What SLO am I targeting
+      - record: slo_target_over_30_days
+        expr: 0.95 # We target 95% uptime over 30 days
+        labels:
+          job: "probe-external-demo-apps" #Job here must match the job in the probe targets

observability/examples/full/docker-compose.yml

@@ -29,6 +29,11 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  blackbox-exporter:
+    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
+    restart: unless-stopped
+    networks:
+      - observability
 networks:
   observability:
     driver: bridge