Skip to content

Commit 1369e50

Browse files
mrnicegyu11mrnicegyu11
committed
init work
1 parent cd22e09 commit 1369e50

File tree

8 files changed

+158
-9
lines changed

8 files changed

+158
-9
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ docs/_build
129129
/services/monitoring/pgsql_query_exporter_config.yaml
130130
/services/monitoring/docker-compose.yml
131131
/services/monitoring/smokeping_prober_config.yaml
132-
132+
services/monitoring/tempo_config.yaml
133133

134134
# Simcore: Contains location of repo.config file on the machine and of the whole config directory
135135
.config.location

‎services/minio/template.env‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,7 @@
22
MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
33
MINIO_SECRET_KEY=${S3_SECRET_KEY}
44
STORAGE_DOMAIN=${STORAGE_DOMAIN}
5+
TEMPO_S3_BUCKET=${TEMPO_S3_BUCKET}
6+
MONITORING_DOMAIN=${MONITORING_DOMAIN}
7+
PUBLIC_NETWORK=${PUBLIC_NETWORK}
8+
MONITORED_NETWORK=${MONITORED_NETWORK}

‎services/monitoring/Makefile‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,28 +49,28 @@ up-master: .init .env config.monitoring config.prometheus.ceph.simcore ${TEMP_C
4949
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
5050
$(MAKE) grafana-import
5151

52-
${TEMP_COMPOSE}: docker-compose.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
52+
${TEMP_COMPOSE}: docker-compose.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
5353
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
5454

55-
${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.http.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
55+
${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.http.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
5656
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.http.yml > $@
5757

58-
${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
58+
${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
5959
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
6060

61-
${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
61+
${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
6262
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@
6363

64-
${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
64+
${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
6565
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.public.yml > $@
6666

67-
${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
67+
${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
6868
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.aws.yml > $@
6969

70-
${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
70+
${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
7171
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
7272

73-
${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
73+
${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
7474
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
7575

7676
docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
@@ -161,6 +161,9 @@ pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CON
161161
smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
162162
$(call jinja, $<, $@);
163163

164+
tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
165+
$(call jinja, $<, $@);
166+
164167
.PHONY: grafana/assets
165168
grafana/assets: ${REPO_CONFIG_LOCATION}
166169
@if [ ! -d "$(shell dirname ${REPO_CONFIG_LOCATION})/assets/grafana" ]; then \

‎services/monitoring/docker-compose.yml.j2‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ networks:
1717
configs:
1818
alertmanager_config:
1919
file: ./alertmanager/config.yml
20+
tempo_config:
21+
file: ./tempo_config.yaml
2022
node_exporter_entrypoint:
2123
file: ./node-exporter/docker-entrypoint.sh
2224
prometheus_config:
@@ -396,3 +398,75 @@ services:
396398
reservations:
397399
memory: 32M
398400
cpus: "0.1"{% endfor %}
401+
tempo1:
402+
image: grafana/tempo:2.6.1
403+
command: "-target=scalable-single-binary -config.file=/etc/tempo.yaml"
404+
configs:
405+
- source: tempo_config
406+
target: /etc/tempo.yaml
407+
networks:
408+
- monitored
409+
deploy:
410+
labels:
411+
- traefik.enable=true
412+
- traefik.docker.network=${PUBLIC_NETWORK}
413+
- traefik.http.services.tempo.loadbalancer.server.port=9095
414+
- traefik.http.routers.tempo.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/tempo`)
415+
- traefik.http.routers.tempo.priority=10
416+
- traefik.http.routers.tempo.entrypoints=https
417+
- traefik.http.routers.tempo.tls=true
418+
- traefik.http.middlewares.tempo_replace_regex.replacepathregex.regex=^/tempo/?(.*)$$
419+
- traefik.http.middlewares.tempo_replace_regex.replacepathregex.replacement=/$${1}
420+
- traefik.http.routers.tempo.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, tempo_replace_regex
421+
resources:
422+
limits:
423+
memory: 2000M
424+
cpus: "2.0"
425+
tempo2:
426+
image: grafana/tempo:2.6.1
427+
command: "-target=scalable-single-binary -config.file=/etc/tempo.yaml"
428+
configs:
429+
- source: tempo_config
430+
target: /etc/tempo.yaml
431+
networks:
432+
- monitored
433+
deploy:
434+
labels:
435+
- traefik.enable=true
436+
- traefik.docker.network=${PUBLIC_NETWORK}
437+
- traefik.http.services.tempo.loadbalancer.server.port=9095
438+
- traefik.http.routers.tempo.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/tempo`)
439+
- traefik.http.routers.tempo.priority=10
440+
- traefik.http.routers.tempo.entrypoints=https
441+
- traefik.http.routers.tempo.tls=true
442+
- traefik.http.middlewares.tempo_replace_regex.replacepathregex.regex=^/tempo/?(.*)$$
443+
- traefik.http.middlewares.tempo_replace_regex.replacepathregex.replacement=/$${1}
444+
- traefik.http.routers.tempo.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, tempo_replace_regex
445+
resources:
446+
limits:
447+
memory: 2000M
448+
cpus: "2.0"
449+
tempo3:
450+
image: grafana/tempo:2.6.1
451+
command: "-target=scalable-single-binary -config.file=/etc/tempo.yaml"
452+
configs:
453+
- source: tempo_config
454+
target: /etc/tempo.yaml
455+
networks:
456+
- monitored
457+
deploy:
458+
labels:
459+
- traefik.enable=true
460+
- traefik.docker.network=${PUBLIC_NETWORK}
461+
- traefik.http.services.tempo.loadbalancer.server.port=9095
462+
- traefik.http.routers.tempo.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/tempo`)
463+
- traefik.http.routers.tempo.priority=10
464+
- traefik.http.routers.tempo.entrypoints=https
465+
- traefik.http.routers.tempo.tls=true
466+
- traefik.http.middlewares.tempo_replace_regex.replacepathregex.regex=^/tempo/?(.*)$$
467+
- traefik.http.middlewares.tempo_replace_regex.replacepathregex.replacement=/$${1}
468+
- traefik.http.routers.tempo.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, tempo_replace_regex
469+
resources:
470+
limits:
471+
memory: 2000M
472+
cpus: "2.0"

‎services/monitoring/template.env‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,8 @@ MONITORING_PROMETHEUS_PGSQL_GID_MONITORED=${MONITORING_PROMETHEUS_PGSQL_GID_MONI
2222
MONITORING_PROMETHEUS_SMOKEPING_TARGETS=${MONITORING_PROMETHEUS_SMOKEPING_TARGETS}
2323
PUBLIC_NETWORK=${PUBLIC_NETWORK}
2424
MONITORED_NETWORK=${MONITORED_NETWORK}
25+
TEMPO_S3_BUCKET=${TEMPO_S3_BUCKET}
26+
STORAGE_DOMAIN=${STORAGE_DOMAIN}
27+
S3_REGION=${S3_REGION}
28+
S3_ACCESS_KEY=${S3_ACCESS_KEY}
29+
S3_SECRET_KEY=${S3_SECRET_KEY}

‎services/monitoring/tempo_config.yaml.j2‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
server:
2+
http_listen_port: 3200
3+
4+
distributor:
5+
receivers: # this configuration will listen on all ports and protocols that tempo is capable of.
6+
otlp:
7+
protocols:
8+
http:
9+
grpc:
10+
11+
#ingester:
12+
# max_block_duration: 5m # cut the headblock when this much time passes. this should probably be left alone normally
13+
14+
compactor:
15+
compaction:
16+
block_retention: 96h # overall Tempo trace retention.
17+
18+
memberlist:
19+
abort_if_cluster_join_fails: false
20+
bind_port: 7946
21+
join_members:
22+
- tempo1:7946
23+
- tempo2:7946
24+
- tempo3:7946
25+
26+
metrics_generator:
27+
registry:
28+
external_labels:
29+
source: tempo
30+
cluster: {{ MACHINE_FQDN }}
31+
storage:
32+
path: /var/tempo/generator/wal
33+
remote_write:
34+
- url: http://prometheuscatchall:9090/api/v1/write
35+
36+
storage:
37+
trace:
38+
backend: s3 # backend configuration to use
39+
wal:
40+
path: /var/tempo/wal # where to store the wal locally
41+
s3:
42+
bucket: {{ TEMPO_S3_BUCKET }} # how to store data in s3
43+
endpoint: {{STORAGE_DOMAIN}}
44+
region: {{S3_REGION}}
45+
access_key: {{S3_ACCESS_KEY}}
46+
secret_key: {{S3_SECRET_KEY}}
47+
insecure: false
48+
tls_insecure_skip_verify: true
49+
# For using AWS, select the appropriate regional endpoint and region
50+
# endpoint: s3.dualstack.us-west-2.amazonaws.com
51+
# region: us-west-2
52+
53+
#querier:
54+
# frontend_worker:
55+
# frontend_address: {{MONITORING_DOMAIN}}/tempo:9095
56+
57+
overrides:
58+
defaults:
59+
metrics_generator:
60+
processors: ['service-graphs', 'span-metrics']

‎services/registry/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ define create-s3-bucket
1717
source .env; \
1818
echo Creating bucket "$${S3_BUCKET}";\
1919
${REPO_BASE_DIR}/scripts/create-s3-bucket.bash "$${S3_BUCKET}" && \
20+
echo Creating bucket "$${TEMPO_S3_BUCKET}";\
21+
${REPO_BASE_DIR}/scripts/create-s3-bucket.bash "$${TEMPO_S3_BUCKET}" && \
2022
set +o allexport; \
2123
# bucket is available in S3
2224
endef

‎services/traefik/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
docker-compose.letsencrypt.dns.yml
22
docker-compose.yml
33
traefik_dynamic_config.yml
4+
tempo_config.yaml

0 commit comments

Comments
 (0)