Skip to content

Commit 0388411

Browse files
committed
wip process exporter and sd-bridge
1 parent b4e2763 commit 0388411

File tree

3 files changed

+142
-1
lines changed

3 files changed

+142
-1
lines changed

framework/observability/compose/conf/prometheus.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,8 @@ scrape_configs:
6565
regex: "^([^|]+)\\|.*"
6666
target_label: groupname
6767
replacement: "$1"
68-
action: replace
68+
action: replace
69+
- job_name: container-sd
70+
file_sd_configs:
71+
- files: [ "/etc/prometheus/targets/merged.json" ]
72+
refresh_interval: 15s

framework/observability/compose/docker-compose.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ services:
4545
volumes:
4646
- /var/run/docker.sock:/var/run/docker.sock
4747
- ./conf/prometheus.yml:/etc/prometheus/prometheus.yml
48+
- sd-targets:/etc/prometheus/targets
4849
ports:
4950
- '9099:9090'
5051

@@ -149,13 +150,29 @@ services:
149150
ports:
150151
- "9256:9256"
151152

153+
sd-bridge:
154+
image: alpine:3.20
155+
command: [ "/bin/sh","-c","apk add --no-cache bash curl jq docker-cli && exec bash scripts/sd-bridge.sh" ]
156+
volumes:
157+
- /var/run/docker.sock:/var/run/docker.sock:ro
158+
- sd-targets:/out
159+
- ./scripts:/scripts:ro
160+
environment:
161+
LABEL_MATCH: "prom_sd=true"
162+
DISCOVERY_PATH: "/discovery"
163+
DISCOVERY_PORT: "6688"
164+
DISCOVERY_SCHEME: "http"
165+
OUT: "/out/merged.json"
166+
SLEEP: "15"
167+
152168
volumes:
153169
loki_data:
154170
grafana_data:
155171
grafana_home:
156172
grafana_logs:
157173
grafana_plugins:
158174
tempo_data:
175+
sd-targets:
159176

160177
networks:
161178
default:
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#!/usr/bin/env bash
2+
# sd-bridge.sh
3+
# Discover Docker containers by label, pull each container's discovery JSON, merge, de-duplicate,
4+
# and write a single Prometheus file_sd JSON. Optionally serve the same JSON over HTTP.
5+
6+
set -Eeuo pipefail
7+
8+
# ---------- Configuration via env vars ----------
9+
LABEL_MATCH="${LABEL_MATCH:-prom_sd=true}" # filter for worker containers
10+
DEFAULT_PATH="${DISCOVERY_PATH:-/discovery}"
11+
DEFAULT_PORT="${DISCOVERY_PORT:-6688}"
12+
DEFAULT_SCHEME="${DISCOVERY_SCHEME:-http}"
13+
PREFER_NETWORK="${NETWORK_NAME:-}" # optional Docker network to prefer for IP
14+
OUT="${OUT:-/out/merged.json}" # file_sd output
15+
SLEEP="${SLEEP:-15}" # seconds between scans
16+
REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-5}" # curl timeout seconds
17+
18+
# Optional lightweight HTTP serving of OUT for http_sd
19+
SERVE_ADDR="${SERVE_ADDR:-}" # example: ":8080" to serve /targets
20+
SERVE_PATH="${SERVE_PATH:-/targets}" # URL path
21+
22+
# ---------- Helpers ----------
23+
log(){ printf '[sd-bridge] %s\n' "$*" >&2; }
24+
get_ip(){
25+
local cid="$1"; local net="$2"
26+
if [[ -n "$net" ]]; then
27+
docker inspect "$cid" | jq -r --arg n "$net" '.[0].NetworkSettings.Networks[$n].IPAddress // empty'
28+
else
29+
docker inspect "$cid" | jq -r '.[0].NetworkSettings.Networks | to_entries[0].value.IPAddress // empty'
30+
fi
31+
}
32+
get_label(){
33+
local cid="$1" key="$2"
34+
docker inspect "$cid" | jq -r --arg k "$key" '.[0].Config.Labels[$k] // empty'
35+
}
36+
merge_and_dedupe(){
37+
# stdin: many JSON arrays of target groups
38+
# out: one array, grouped by identical labels with unique sorted targets
39+
jq -s '
40+
add // []
41+
| map({targets: (.targets // []), labels: (.labels // {})})
42+
| sort_by(.labels)
43+
| group_by(.labels)
44+
| map({labels: (.[0].labels), targets: ([.[].targets[]] | unique | sort)})
45+
'
46+
}
47+
atomic_write(){
48+
local path="$1"; local tmp="$1.tmp"
49+
cat > "$tmp" && mv "$tmp" "$path"
50+
}
51+
52+
# ---------- Optional HTTP server ----------
53+
serve_http(){
54+
# Serves OUT at SERVE_PATH. Requires busybox httpd or python3 in the container.
55+
if command -v busybox >/dev/null 2>&1; then
56+
log "serving http_sd at ${SERVE_ADDR}${SERVE_PATH} using busybox httpd"
57+
# Busybox serves a directory. Symlink requested path to OUT.
58+
local root; root="$(dirname "$OUT")"; mkdir -p "$root"
59+
# Keep a symlink named targets.json and rewrite on change
60+
ln -sf "$(basename "$OUT")" "$root/targets.json"
61+
exec busybox httpd -f -p "${SERVE_ADDR#:}" -h "$root"
62+
elif command -v python3 >/dev/null 2>&1; then
63+
log "serving http_sd at ${SERVE_ADDR}${SERVE_PATH} using python http.server"
64+
cd "$(dirname "$OUT")" && exec python3 -m http.server "${SERVE_ADDR#:}"
65+
else
66+
log "no http server available in image; install busybox or python3"
67+
sleep infinity
68+
fi
69+
}
70+
71+
# Ensure output dir and initial file
72+
mkdir -p "$(dirname "$OUT")"
73+
echo '[]' | atomic_write "$OUT"
74+
75+
# If SERVE_ADDR is set, background a tiny polling loop that keeps a shadow file named targets.json
76+
if [[ -n "$SERVE_ADDR" ]]; then
77+
# Run server in background subshell so main loop continues to update OUT
78+
serve_http &
79+
fi
80+
81+
# ---------- Main loop ----------
82+
while true; do
83+
mapfile -t cids < <(docker ps -q --filter "label=$LABEL_MATCH" || true)
84+
if (( ${#cids[@]} == 0 )); then
85+
echo '[]' | atomic_write "$OUT"
86+
log "no matching containers; wrote empty array"
87+
sleep "$SLEEP"; continue
88+
fi
89+
90+
files=()
91+
for cid in "${cids[@]}"; do
92+
ip="$(get_ip "$cid" "$PREFER_NETWORK")"
93+
if [[ -z "$ip" ]]; then log "skip ${cid:0:12}: no IP"; continue; fi
94+
95+
# Per container overrides via labels
96+
path="$(get_label "$cid" prom_sd_path)"; path="${path:-$DEFAULT_PATH}"
97+
port="$(get_label "$cid" prom_sd_port)"; port="${port:-$DEFAULT_PORT}"
98+
scheme="$(get_label "$cid" prom_sd_scheme)"; scheme="${scheme:-$DEFAULT_SCHEME}"
99+
100+
url="${scheme}://${ip}:${port}${path}"
101+
f="$(mktemp)"; files+=("$f")
102+
if curl -fsSL --max-time "$REQUEST_TIMEOUT" "$url" | jq '.' > "$f" 2>/dev/null; then
103+
log "ok ${url}"
104+
else
105+
log "fail ${url}; using []"
106+
echo '[]' > "$f"
107+
fi
108+
done
109+
110+
if (( ${#files[@]} > 0 )); then
111+
cat "${files[@]}" | merge_and_dedupe | atomic_write "$OUT"
112+
rm -f "${files[@]}"
113+
log "merged ${#files[@]} lists into $(wc -c < "$OUT") bytes"
114+
else
115+
echo '[]' | atomic_write "$OUT"
116+
fi
117+
118+
sleep "$SLEEP"
119+
120+
done

0 commit comments

Comments
 (0)