Skip to content

Commit 7c34ab4

Browse files
YuryHrytsukYuryHrytsuk
authored
Monitoring: rolling config update (#1022)
* Monitoring: rolling config update Make sure monitoring stack is automatically deployed on config content changes Related Issue(s): * #984 * Fix docker compose target dependencies * Monitoring: refactor makefile Makefile is too complex, contains lots of duplications. This commit attempts to make it easier. * Add missing deps and remove stale files * Remove unused dependencies * Add .gitignore * Remove unused targets * Fix '${TEMP_COMPOSE}-local' target It was missing docker compose file that defines prometheus/prometheus.yml config
1 parent 3e60c63 commit 7c34ab4

10 files changed

+133
-105
lines changed

‎services/monitoring/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
docker-compose.aws.yml
2+
docker-compose.dalco.yml
3+
docker-compose.public.yml
4+
docker-compose.master.yml

‎services/monitoring/Makefile‎

Lines changed: 70 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
.DEFAULT_GOAL := help
22

3-
43
# STACK_NAME defaults to name of the current directory. Should not to be changed if you follow GitOps operating procedures.
54
STACK_NAME = $(notdir $(CURDIR))
65
TEMP_COMPOSE=.stack.${STACK_NAME}.yaml
@@ -19,91 +18,106 @@ define create-s3-bucket
1918
# bucket is available in S3
2019
endef
2120

22-
.PHONY: up
23-
up: .init .env config.prometheus ${TEMP_COMPOSE} ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
24-
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} $(STACK_NAME)
25-
$(MAKE) grafana-import
21+
#
22+
# up targets
23+
#
2624

2725
.PHONY: up-local
28-
up-local: .init .env config.prometheus.simcore ${TEMP_COMPOSE}-local ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
26+
up-local: ${TEMP_COMPOSE}-local ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
2927
@$(create-s3-bucket)
3028
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local $(STACK_NAME)
3129
$(MAKE) grafana-import
3230

33-
.PHONY: up-letsencrypt-http
34-
up-letsencrypt-http: .init .env config.monitoring config.prometheus ${TEMP_COMPOSE}-letsencrypt-http ## Deploys or updates current stack "$(STACK_NAME)" using let's encrypt http challenge
35-
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME}
36-
$(MAKE) grafana-import
37-
38-
.PHONY: up-letsencrypt-dns
39-
up-letsencrypt-dns: .init .env config.monitoring config.prometheus ${TEMP_COMPOSE}-letsencrypt-dns ## Deploys or updates current stack "$(STACK_NAME)" using let's encrypt dns challenge
40-
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME}
41-
$(MAKE) grafana-import
42-
4331
.PHONY: up-dalco
44-
up-dalco: .init .env config.monitoring config.prometheus.ceph.simcore ${TEMP_COMPOSE}-dalco ## Deploys monitoring stack for Dalco Cluster
32+
up-dalco: ${TEMP_COMPOSE}-dalco ## Deploys monitoring stack for Dalco Cluster
4533
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME}
4634
$(MAKE) grafana-import
4735

4836
.PHONY: up-public
49-
up-public: .init .env config.monitoring config.prometheus ${TEMP_COMPOSE}-public ## Deploys monitoring stack for Public Cluster
37+
up-public: ${TEMP_COMPOSE}-public ## Deploys monitoring stack for Public Cluster
5038
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-public ${STACK_NAME}
5139
$(MAKE) grafana-import
5240

5341
.PHONY: up-aws
54-
up-aws: .init .env config.monitoring config.prometheus.simcore.aws ${TEMP_COMPOSE}-aws ## Deploys or updates current stack "$(STACK_NAME)" in AWS
42+
up-aws: ${TEMP_COMPOSE}-aws ## Deploys or updates current stack "$(STACK_NAME)" in AWS
5543
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME}
5644
$(MAKE) grafana-import
5745

5846
.PHONY: up-master
59-
up-master: .init .env config.monitoring config.prometheus.ceph.simcore ${TEMP_COMPOSE}-master ## Deploys monitoring stack for Master Cluster
47+
up-master: ${TEMP_COMPOSE}-master ## Deploys monitoring stack for Master Cluster
6048
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
6149
$(MAKE) grafana-import
6250

63-
${TEMP_COMPOSE}: docker-compose.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
64-
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
51+
#
52+
# docker compose targets
53+
#
54+
55+
COMMON_COMPOSE_DEPENDENCIES: .env .venv
6556

66-
${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.http.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
67-
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.http.yml > $@
57+
docker-compose.yml: docker-compose.yml.j2 \
58+
prometheus/prometheus-federation.yml \
59+
prometheus/prometheus.rules.yml \
60+
node-exporter/docker-entrypoint.sh \
61+
pgsql_query_exporter_config.yaml \
62+
smokeping_prober_config.yaml \
63+
tempo_config.yaml \
64+
alertmanager/config.yml \
65+
grafana/config.monitoring \
66+
$(COMMON_COMPOSE_DEPENDENCIES)
67+
# generating $@
68+
@$(call jinja,$<,.env,$@)
6869

69-
${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
70-
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
70+
docker-compose.dalco.yml: docker-compose.dalco.yml.j2 config.prometheus.ceph.simcore $(COMMON_COMPOSE_DEPENDENCIES)
71+
# generating $@
72+
@$(call jinja,$<,.env,$@)
7173

72-
${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
74+
docker-compose.public.yml: docker-compose.public.yml.j2 config.prometheus $(COMMON_COMPOSE_DEPENDENCIES)
75+
# generating $@
76+
@$(call jinja,$<,.env,$@)
77+
78+
docker-compose.aws.yml: docker-compose.aws.yml.j2 config.prometheus.simcore.aws $(COMMON_COMPOSE_DEPENDENCIES)
79+
# generating $@
80+
@$(call jinja,$<,.env,$@)
81+
82+
docker-compose.master.yml: docker-compose.master.yml.j2 config.prometheus.ceph.simcore $(COMMON_COMPOSE_DEPENDENCIES)
83+
# generating $@
84+
@$(call jinja,$<,.env,$@)
85+
86+
#
87+
# temp compose targets
88+
#
89+
90+
TEMP_COMPOSE_COMMON_DEPENDENCIES := docker-compose.yml .env
91+
92+
${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
7393
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@
7494

75-
${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
95+
${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
7696
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.public.yml > $@
7797

78-
${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
98+
${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
7999
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.aws.yml > $@
80100

81-
${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
101+
${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
82102
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
83103

84-
${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
85-
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
104+
${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.master.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
105+
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
86106

87-
docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
88-
$(call jinja,$<,.env,$@)
89-
90-
.PHONY: grafana-import
91-
grafana-import: grafana/assets ## Imports the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
92-
@pushd ${REPO_BASE_DIR}/services/monitoring/grafana && \
93-
$(MAKE) terraform-plan && \
94-
$(MAKE) terraform-apply; \
95-
popd > /dev/null
107+
#
108+
# artifacts targets
109+
#
96110

97111
.PHONY: config.grafana.dashboards
98112
config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
99113
$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
100114

101-
.PHONY: config.monitoring
102-
config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
115+
.PHONY: grafana/config.monitoring
116+
grafana/config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
103117
@set -o allexport; \
104118
source $(REPO_CONFIG_LOCATION); \
105119
set +o allexport; \
106-
envsubst < $< > grafana/$@
120+
envsubst < $< > $@
107121

108122
.PHONY: config.prometheus.simcore
109123
config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv
@@ -124,7 +138,7 @@ config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} .venv
124138
mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
125139

126140
.PHONY: config.prometheus.ceph.simcore
127-
config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .venv
141+
config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env .venv
128142
@set -o allexport; \
129143
source $<; \
130144
set +o allexport; \
@@ -151,6 +165,17 @@ smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCA
151165
tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
152166
$(call jinja, $<, .env, $@);
153167

168+
#
169+
# grafana targets
170+
#
171+
154172
.PHONY: grafana/assets
155173
grafana/assets: ${REPO_CONFIG_LOCATION}
156174
@$(MAKE_C) grafana assets
175+
176+
.PHONY: grafana-import
177+
grafana-import: grafana/assets ## Imports the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
178+
@pushd ${REPO_BASE_DIR}/services/monitoring/grafana && \
179+
$(MAKE) terraform-plan && \
180+
$(MAKE) terraform-apply; \
181+
popd > /dev/null

‎services/monitoring/docker-compose.aws.yml‎ renamed to ‎services/monitoring/docker-compose.aws.yml.j2‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,8 @@ services:
2929
placement:
3030
constraints:
3131
- node.labels.grafana==true
32+
33+
configs:
34+
prometheus_config:
35+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
36+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.public.yml‎ renamed to ‎services/monitoring/docker-compose.dalco.yml.j2‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
version: '3.7'
22
services:
3+
34
cadvisor-exporter:
45
volumes:
56
- /:/rootfs:ro
@@ -26,3 +27,8 @@ services:
2627
placement:
2728
constraints:
2829
- node.labels.prometheus==true
30+
31+
configs:
32+
prometheus_config:
33+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
34+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.letsencrypt.dns.yml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎services/monitoring/docker-compose.letsencrypt.http.yml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎services/monitoring/docker-compose.master.yml‎ renamed to ‎services/monitoring/docker-compose.master.yml.j2‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,8 @@ services:
1515
placement:
1616
constraints:
1717
- node.labels.prometheus==true
18+
19+
configs:
20+
prometheus_config:
21+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
22+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.dalco.yml‎ renamed to ‎services/monitoring/docker-compose.public.yml.j2‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
version: '3.7'
22
services:
3-
43
cadvisor-exporter:
54
volumes:
65
- /:/rootfs:ro
@@ -27,3 +26,8 @@ services:
2726
placement:
2827
constraints:
2928
- node.labels.prometheus==true
29+
30+
configs:
31+
prometheus_config:
32+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
33+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.yml.j2‎

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,5 @@
11
version: "3.7"
22

3-
volumes:
4-
prometheus_data: {}
5-
grafana_data: {}
6-
alertmanager_data: {}
7-
prometheus_data_federation: {}
8-
9-
networks:
10-
monitored:
11-
name: ${MONITORED_NETWORK}
12-
external: true
13-
public:
14-
external: true
15-
name: ${PUBLIC_NETWORK}
16-
17-
configs:
18-
alertmanager_config:
19-
file: ./alertmanager/config.yml
20-
tempo_config:
21-
file: ./tempo_config.yaml
22-
node_exporter_entrypoint:
23-
file: ./node-exporter/docker-entrypoint.sh
24-
prometheus_config:
25-
file: ./prometheus/prometheus.yml
26-
prometheus_config_federation:
27-
file: ./prometheus/prometheus-federation.yml
28-
prometheus_rules:
29-
file: ./prometheus/prometheus.rules.yml
30-
pgsql_query_exporter_config:
31-
file: ./pgsql_query_exporter_config.yaml
32-
smokeping_prober_config:
33-
file: ./smokeping_prober_config.yaml
343
services:
354
cadvisor-exporter:
365
image: gcr.io/cadvisor/cadvisor:v0.49.1
@@ -424,3 +393,40 @@ services:
424393
limits:
425394
memory: 2000M
426395
cpus: "2.0"
396+
397+
configs:
398+
alertmanager_config:
399+
name: ${STACK_NAME}_alertmanager_config_{{ "./alertmanager/config.yml" | sha256file | substring(0,10) }}
400+
file: ./alertmanager/config.yml
401+
tempo_config:
402+
name: ${STACK_NAME}_tempo_config_{{ "./tempo_config.yaml" | sha256file | substring(0,10) }}
403+
file: ./tempo_config.yaml
404+
node_exporter_entrypoint:
405+
name: ${STACK_NAME}_node_exporter_entrypoint_{{ "./node-exporter/docker-entrypoint.sh" | sha256file | substring(0,10) }}
406+
file: ./node-exporter/docker-entrypoint.sh
407+
prometheus_config_federation:
408+
name: ${STACK_NAME}_prometheus_config_federation_{{ "./prometheus/prometheus-federation.yml" | sha256file | substring(0,10) }}
409+
file: ./prometheus/prometheus-federation.yml
410+
prometheus_rules:
411+
name: ${STACK_NAME}_prometheus_rules_{{ "./prometheus/prometheus.rules.yml" | sha256file | substring(0,10) }}
412+
file: ./prometheus/prometheus.rules.yml
413+
pgsql_query_exporter_config:
414+
name: ${STACK_NAME}_pgsql_query_exporter_config_{{ "./pgsql_query_exporter_config.yaml" | sha256file | substring(0,10) }}
415+
file: ./pgsql_query_exporter_config.yaml
416+
smokeping_prober_config:
417+
name: ${STACK_NAME}_smokeping_prober_config_{{ "./smokeping_prober_config.yaml" | sha256file | substring(0,10) }}
418+
file: ./smokeping_prober_config.yaml
419+
420+
volumes:
421+
prometheus_data: {}
422+
grafana_data: {}
423+
alertmanager_data: {}
424+
prometheus_data_federation: {}
425+
426+
networks:
427+
monitored:
428+
name: ${MONITORED_NETWORK}
429+
external: true
430+
public:
431+
external: true
432+
name: ${PUBLIC_NETWORK}

‎services/monitoring/template.env‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
STACK_NAME=${STACK_NAME}
12
MONITORING_DOMAIN=${MONITORING_DOMAIN}
23
MONITORING_PROMETHEUS_RETENTION=${MONITORING_PROMETHEUS_RETENTION}
34
MONITORING_PROMETHEUS_FEDERATION_RETENTION=${MONITORING_PROMETHEUS_FEDERATION_RETENTION}

0 commit comments

Comments
 (0)