Skip to content

Commit 3f1c43a

Browse files
alhendricksonalhendrickson
authored
Setup Grafana Alloy to simplify Exporter usage (#6)
* Run Grafana alloy to replace exporters * Make sample folder using alloy * Add default labels to alloy scraped metrics * Add stub for making alloy the default
1 parent 7cd422f commit 3f1c43a

File tree

9 files changed

+184
-1
lines changed

9 files changed

+184
-1
lines changed

‎observability/build-docker.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
docker build -t cogstacksystems/cogstack-observability-prometheus:latest -f prometheus/Dockerfile.prometheus ./prometheus
3+
4+
docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest -f prometheus/Dockerfile.blackbox ./prometheus
5+
6+
docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana
7+
8+
docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik

‎observability/examples/alloy/docker-compose.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Observability main stack. Prometheus and Grafana.
2+
# Depends on docker-compose.exporters.yml for the network
3+
name: "cogstack-observability"
4+
services:
5+
prometheus:
6+
image: cogstacksystems/cogstack-observability-prometheus:latest
7+
restart: unless-stopped
8+
ports:
9+
- "9090:9090"
10+
volumes:
11+
- ${BASE_DIR-.}/prometheus:/etc/prometheus/cogstack/site/
12+
- prometheus-data:/prometheus
13+
networks:
14+
- observability
15+
command:
16+
- "--config.file=/etc/prometheus/cogstack/defaults/prometheus.yml"
17+
- "--storage.tsdb.path=/prometheus"
18+
- "--storage.tsdb.retention.time=30d"
19+
- "--web.external-url=/prometheus"
20+
- "--web.route-prefix=/prometheus"
21+
- "--web.enable-remote-write-receiver"
22+
grafana:
23+
image: cogstacksystems/cogstack-observability-grafana:latest
24+
restart: unless-stopped
25+
volumes:
26+
- grafana-data:/var/lib/grafana
27+
networks:
28+
- observability
29+
environment:
30+
- GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in
31+
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
32+
traefik:
33+
image: cogstacksystems/cogstack-observability-traefik:latest
34+
networks:
35+
- observability
36+
restart: unless-stopped
37+
ports:
38+
- "80:80"
39+
volumes:
40+
- /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events
41+
blackbox-exporter:
42+
image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
43+
restart: unless-stopped
44+
networks:
45+
- observability
46+
alloy:
47+
image: grafana/alloy:latest
48+
command:
49+
- run
50+
- --server.http.listen-addr=0.0.0.0:12345
51+
- --storage.path=/var/lib/alloy/data
52+
- --server.http.ui-path-prefix=/alloy
53+
- /etc/alloy
54+
ports:
55+
- "12345:12345"
56+
volumes:
57+
- ${BASE_DIR-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
58+
# CAdvisor
59+
- /:/rootfs:ro
60+
- /var/run:/var/run:rw
61+
- /sys:/sys:ro
62+
- /var/lib/docker/:/var/lib/docker:ro
63+
labels:
64+
- "traefik.enable=true"
65+
- "traefik.http.routers.alloy.rule=PathPrefix(`/alloy`)"
66+
environment:
67+
- PROMETHEUS_URL=${PROMETHEUS_URL-http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write}
68+
- ALLOY_HOSTNAME=${ALLOY_HOSTNAME} # Used to add a label to metrics
69+
- ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS} # Used to add a label to metrics
70+
networks:
71+
- observability
72+
networks:
73+
observability:
74+
driver: bridge
75+
76+
volumes:
77+
prometheus-data:
78+
grafana-data:

‎observability/examples/alloy/grafana-alloy/config.alloy‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
logging {
2+
level = "debug"
3+
format = "logfmt"
4+
}
5+
6+
prometheus.remote_write "default" {
7+
endpoint {
8+
url = sys.env("PROMETHEUS_URL")
9+
}
10+
external_labels = {
11+
host = sys.env("ALLOY_HOSTNAME"),
12+
ip_address = sys.env("ALLOY_IP_ADDRESS"),
13+
}
14+
}
15+
16+
prometheus.scrape "exporter" {
17+
scrape_interval = "15s"
18+
targets = array.concat(
19+
prometheus.exporter.self.alloy.targets,
20+
prometheus.exporter.cadvisor.local_cadvisor.targets,
21+
prometheus.exporter.unix.local_node_exporter.targets,
22+
)
23+
forward_to = [prometheus.remote_write.default.receiver]
24+
}
25+
26+
// Alloys internal metrics
27+
prometheus.exporter.self "alloy" {
28+
}
29+
30+
// CAdvisor
31+
prometheus.exporter.cadvisor "local_cadvisor" {
32+
docker_host = "unix:///var/run/docker.sock"
33+
storage_duration = "5m"
34+
}
35+
36+
// Node exporter
37+
prometheus.exporter.unix "local_node_exporter" {
38+
}

‎observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Example of probe targets
2+
- targets:
3+
- https://cogstack.org
4+
labels:
5+
name: cogstack-homepage
6+
job: probe-services

‎observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
groups:
2+
- name: slo-target-rules
3+
rules:
4+
# What SLO am I targeting
5+
- record: slo_target_over_30_days
6+
expr: 0.95 # We target 95% uptime over 30 days
7+
labels:
8+
job: "probe-external-demo-apps" #Job here must match the job in the probe targets

‎observability/examples/full/docker-compose.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ services:
2929
- "80:80"
3030
volumes:
3131
- /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events
32+
blackbox-exporter:
33+
image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
34+
restart: unless-stopped
35+
networks:
36+
- observability
3237
networks:
3338
observability:
3439
driver: bridge

‎observability/grafana/Dockerfile.alloy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# TODO

‎observability/grafana/alloy/default-config.alloy‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
logging {
2+
level = "debug"
3+
format = "logfmt"
4+
}
5+
6+
prometheus.remote_write "default" {
7+
endpoint {
8+
url = sys.env("PROMETHEUS_URL")
9+
}
10+
external_labels = {
11+
host = sys.env("ALLOY_HOSTNAME"),
12+
ip_address = sys.env("ALLOY_IP_ADDRESS"),
13+
}
14+
}
15+
16+
prometheus.scrape "exporter" {
17+
scrape_interval = "15s"
18+
targets = array.concat(
19+
prometheus.exporter.self.alloy.targets,
20+
prometheus.exporter.cadvisor.local_cadvisor.targets,
21+
prometheus.exporter.unix.local_node_exporter.targets,
22+
)
23+
forward_to = [prometheus.remote_write.default.receiver]
24+
}
25+
26+
// Alloys internal metrics
27+
prometheus.exporter.self "alloy" {
28+
}
29+
30+
// CAdvisor
31+
prometheus.exporter.cadvisor "local_cadvisor" {
32+
docker_host = "unix:///var/run/docker.sock"
33+
storage_duration = "5m"
34+
}
35+
36+
// Node exporter
37+
prometheus.exporter.unix "local_node_exporter" {
38+
}

‎observability/prometheus/Dockerfile.prometheus‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@ CMD [ \
1010
"--storage.tsdb.path=/prometheus", \
1111
"--storage.tsdb.retention.time=30d", \
1212
"--web.external-url=/prometheus", \
13-
"--web.route-prefix=/prometheus" \
13+
"--web.route-prefix=/prometheus", \
14+
"--web.enable-remote-write-receiver" \
1415
]

0 commit comments

Comments
 (0)