Skip to content

Commit 1542df5

Browse files
YuryHrytsukYuryHrytsuk
committed
Monitoring: refactor makefile
Makefile is too complex, contains lots of duplications. This commit attempts to make it easier.
1 parent d689607 commit 1542df5

File tree

6 files changed

+94
-52
lines changed

6 files changed

+94
-52
lines changed

‎services/monitoring/Makefile‎

Lines changed: 73 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
.DEFAULT_GOAL := help
22

3-
43
# STACK_NAME defaults to name of the current directory. Should not to be changed if you follow GitOps operating procedures.
54
STACK_NAME = $(notdir $(CURDIR))
65
TEMP_COMPOSE=.stack.${STACK_NAME}.yaml
@@ -19,99 +18,114 @@ define create-s3-bucket
1918
# bucket is available in S3
2019
endef
2120

21+
#
22+
# up targets
23+
#
24+
2225
.PHONY: up
23-
up: .init .env config.prometheus ${TEMP_COMPOSE} ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
26+
up: ${TEMP_COMPOSE} ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
2427
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} $(STACK_NAME)
2528
$(MAKE) grafana-import
2629

2730
.PHONY: up-local
28-
up-local: .init .env config.prometheus.simcore ${TEMP_COMPOSE}-local ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
31+
up-local: ${TEMP_COMPOSE}-local ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
2932
@$(create-s3-bucket)
3033
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local $(STACK_NAME)
3134
$(MAKE) grafana-import
3235

33-
.PHONY: up-letsencrypt-http
34-
up-letsencrypt-http: .init .env config.monitoring config.prometheus ${TEMP_COMPOSE}-letsencrypt-http ## Deploys or updates current stack "$(STACK_NAME)" using let's encrypt http challenge
35-
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME}
36-
$(MAKE) grafana-import
37-
38-
.PHONY: up-letsencrypt-dns
39-
up-letsencrypt-dns: .init .env config.monitoring config.prometheus ${TEMP_COMPOSE}-letsencrypt-dns ## Deploys or updates current stack "$(STACK_NAME)" using let's encrypt dns challenge
40-
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME}
41-
$(MAKE) grafana-import
42-
4336
.PHONY: up-dalco
44-
up-dalco: .init .env config.monitoring config.prometheus.ceph.simcore ${TEMP_COMPOSE}-dalco ## Deploys monitoring stack for Dalco Cluster
37+
up-dalco: ${TEMP_COMPOSE}-dalco ## Deploys monitoring stack for Dalco Cluster
4538
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME}
4639
$(MAKE) grafana-import
4740

4841
.PHONY: up-public
49-
up-public: .init .env config.monitoring config.prometheus ${TEMP_COMPOSE}-public ## Deploys monitoring stack for Public Cluster
42+
up-public: ${TEMP_COMPOSE}-public ## Deploys monitoring stack for Public Cluster
5043
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-public ${STACK_NAME}
5144
$(MAKE) grafana-import
5245

5346
.PHONY: up-aws
54-
up-aws: .init .env config.monitoring config.prometheus.simcore.aws ${TEMP_COMPOSE}-aws ## Deploys or updates current stack "$(STACK_NAME)" in AWS
47+
up-aws: ${TEMP_COMPOSE}-aws ## Deploys or updates current stack "$(STACK_NAME)" in AWS
5548
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME}
5649
$(MAKE) grafana-import
5750

5851
.PHONY: up-master
59-
up-master: .init .env config.monitoring config.prometheus.ceph.simcore ${TEMP_COMPOSE}-master ## Deploys monitoring stack for Master Cluster
52+
up-master: ${TEMP_COMPOSE}-master ## Deploys monitoring stack for Master Cluster
6053
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
6154
$(MAKE) grafana-import
6255

63-
${TEMP_COMPOSE}: docker-compose.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
64-
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
56+
#
57+
# docker compose targets
58+
#
6559

66-
${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.http.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
67-
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.http.yml > $@
60+
COMMON_COMPOSE_DEPENDENCIES: .env .venv
6861

69-
${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
70-
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
62+
docker-compose.yml: docker-compose.yml.j2 \
63+
prometheus/prometheus-federation.yml \
64+
prometheus/prometheus.rules.yml \
65+
node-exporter/docker-entrypoint.sh \
66+
pgsql_query_exporter_config.yaml \
67+
smokeping_prober_config.yaml \
68+
tempo_config.yaml \
69+
alertmanager/config.yml \
70+
grafana/config.monitoring \
71+
$(COMMON_COMPOSE_DEPENDENCIES)
72+
# generating $@
73+
@$(call jinja,$<,.env,$@)
74+
75+
docker-compose.dalco.yml: docker-compose.dalco.yml.j2 config.prometheus.ceph.simcore $(COMMON_COMPOSE_DEPENDENCIES)
76+
# generating $@
77+
@$(call jinja,$<,.env,$@)
78+
79+
docker-compose.public.yml: docker-compose.public.yml.j2 config.prometheus $(COMMON_COMPOSE_DEPENDENCIES)
80+
# generating $@
81+
@$(call jinja,$<,.env,$@)
82+
83+
docker-compose.aws.yml: docker-compose.aws.yml.j2 config.prometheus.simcore.aws $(COMMON_COMPOSE_DEPENDENCIES)
84+
# generating $@
85+
@$(call jinja,$<,.env,$@)
86+
87+
docker-compose.master.yml: docker-compose.master.yml.j2 config.prometheus.ceph.simcore $(COMMON_COMPOSE_DEPENDENCIES)
88+
# generating $@
89+
@$(call jinja,$<,.env,$@)
90+
91+
#
92+
# temp compose targets
93+
#
94+
95+
TEMP_COMPOSE_COMMON_DEPENDENCIES := docker-compose.yml .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
96+
97+
${TEMP_COMPOSE}: config.prometheus $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
98+
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
7199

72-
${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
100+
${TEMP_COMPOSE}-dalco: docker-compose.dalco.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
73101
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@
74102

75-
${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
103+
${TEMP_COMPOSE}-public: docker-compose.public.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
76104
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.public.yml > $@
77105

78-
${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
106+
${TEMP_COMPOSE}-aws: docker-compose.aws.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
79107
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.aws.yml > $@
80108

81-
${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
109+
${TEMP_COMPOSE}-master: docker-compose.master.yml $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
82110
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
83111

84-
${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
112+
${TEMP_COMPOSE}-local: docker-compose.letsencrypt.dns.yml config.prometheus.simcore $(TEMP_COMPOSE_COMMON_DEPENDENCIES)
85113
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
86114

87-
docker-compose.yml: docker-compose.yml.j2 .env .venv \
88-
prometheus/prometheus.yml \
89-
prometheus/prometheus-federation.yml \
90-
prometheus/prometheus.rules.yml \
91-
node-exporter/docker-entrypoint.sh \
92-
pgsql_query_exporter_config.yaml \
93-
smokeping_prober_config.yaml \
94-
tempo_config.yaml \
95-
alertmanager/config.yml
96-
$(call jinja,$<,.env,$@)
97-
98-
.PHONY: grafana-import
99-
grafana-import: grafana/assets ## Imports the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
100-
@pushd ${REPO_BASE_DIR}/services/monitoring/grafana && \
101-
$(MAKE) terraform-plan && \
102-
$(MAKE) terraform-apply; \
103-
popd > /dev/null
115+
#
116+
# artifacts targets
117+
#
104118

105119
.PHONY: config.grafana.dashboards
106120
config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
107121
$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
108122

109-
.PHONY: config.monitoring
110-
config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
123+
.PHONY: grafana/config.monitoring
124+
grafana/config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
111125
@set -o allexport; \
112126
source $(REPO_CONFIG_LOCATION); \
113127
set +o allexport; \
114-
envsubst < $< > grafana/$@
128+
envsubst < $< > $@
115129

116130
.PHONY: config.prometheus.simcore
117131
config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv
@@ -159,6 +173,17 @@ smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCA
159173
tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
160174
$(call jinja, $<, .env, $@);
161175

176+
#
177+
# grafana targets
178+
#
179+
162180
.PHONY: grafana/assets
163181
grafana/assets: ${REPO_CONFIG_LOCATION}
164182
@$(MAKE_C) grafana assets
183+
184+
.PHONY: grafana-import
185+
grafana-import: grafana/assets ## Imports the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
186+
@pushd ${REPO_BASE_DIR}/services/monitoring/grafana && \
187+
$(MAKE) terraform-plan && \
188+
$(MAKE) terraform-apply; \
189+
popd > /dev/null

‎services/monitoring/docker-compose.aws.yml‎ renamed to ‎services/monitoring/docker-compose.aws.yml.j2‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,8 @@ services:
2929
placement:
3030
constraints:
3131
- node.labels.grafana==true
32+
33+
configs:
34+
prometheus_config:
35+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
36+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.public.yml‎ renamed to ‎services/monitoring/docker-compose.dalco.yml.j2‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
version: '3.7'
22
services:
3+
34
cadvisor-exporter:
45
volumes:
56
- /:/rootfs:ro
@@ -26,3 +27,8 @@ services:
2627
placement:
2728
constraints:
2829
- node.labels.prometheus==true
30+
31+
configs:
32+
prometheus_config:
33+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
34+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.master.yml‎ renamed to ‎services/monitoring/docker-compose.master.yml.j2‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,8 @@ services:
1515
placement:
1616
constraints:
1717
- node.labels.prometheus==true
18+
19+
configs:
20+
prometheus_config:
21+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
22+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.dalco.yml‎ renamed to ‎services/monitoring/docker-compose.public.yml.j2‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
version: '3.7'
22
services:
3-
43
cadvisor-exporter:
54
volumes:
65
- /:/rootfs:ro
@@ -27,3 +26,8 @@ services:
2726
placement:
2827
constraints:
2928
- node.labels.prometheus==true
29+
30+
configs:
31+
prometheus_config:
32+
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
33+
file: ./prometheus/prometheus.yml

‎services/monitoring/docker-compose.yml.j2‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -404,9 +404,6 @@ configs:
404404
node_exporter_entrypoint:
405405
name: ${STACK_NAME}_node_exporter_entrypoint_{{ "./node-exporter/docker-entrypoint.sh" | sha256file | substring(0,10) }}
406406
file: ./node-exporter/docker-entrypoint.sh
407-
prometheus_config:
408-
name: ${STACK_NAME}_prometheus_config_{{ "./prometheus/prometheus.yml" | sha256file | substring(0,10) }}
409-
file: ./prometheus/prometheus.yml
410407
prometheus_config_federation:
411408
name: ${STACK_NAME}_prometheus_config_federation_{{ "./prometheus/prometheus-federation.yml" | sha256file | substring(0,10) }}
412409
file: ./prometheus/prometheus-federation.yml

0 commit comments

Comments
 (0)