Merge branch 'main' into traefik-rolling-config-updates

YuryHrytsuk · web-flow · commit ee9f18bdf89d · 2025-04-14T10:31:42.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -129,7 +129,7 @@ docs/_build
 /services/monitoring/pgsql_query_exporter_config.yaml
 /services/monitoring/docker-compose.yml
 /services/monitoring/smokeping_prober_config.yaml
-
+services/monitoring/tempo_config.yaml
 
 # Simcore: Contains location of repo.config file on the machine and of the whole config directory
 .config.location
diff --git a/Makefile b/Makefile
@@ -71,7 +71,6 @@ down-maintenance: ## Stop the maintenance mode
 	fi \
 	,)
 
-
 # Misc: info & clean
 .PHONY: info info-vars info-local
 info: ## Displays some important info
diff --git a/services/admin-panels/Makefile b/services/admin-panels/Makefile
@@ -13,42 +13,42 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 # Helpers --------------------------------------------------
 define custom-jinja
-	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3)
+	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \
+	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
 endef
 
 .PHONY: .data.json
 .data.json:
 	@$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json
 
-
 .PHONY: docker-compose.yml
-docker-compose.yml: docker-compose.yml.j2 .venv .data.json
+docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py
 	$(call custom-jinja, $<, .data.json, tmp.yml)
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@
 	@rm tmp.yml
 
 .PHONY: up
-up: .init .env jupyter_server_config.py ${TEMP_COMPOSE}  ## Deploys jaeger stack
+up: .init ${TEMP_COMPOSE} prune-docker-stack-configs ## Deploys jaeger stack
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
 
 .PHONY: up-letsencrypt-http
-up-letsencrypt-http: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-http  ## Deploys jaeger stack using let's encrypt http challenge
+up-letsencrypt-http: .init ${TEMP_COMPOSE}-letsencrypt-http prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt http challenge
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME}
 
 .PHONY: up-letsencrypt-dns
-up-letsencrypt-dns: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-dns  ## Deploys jaeger stack using let's encrypt dns challenge
+up-letsencrypt-dns: .init ${TEMP_COMPOSE}-letsencrypt-dns prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt dns challenge
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME}
 
 .PHONY: up-dalco ## Deploys jaeger stack for Dalco Cluster
-up-dalco: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-dalco
+up-dalco: .init ${TEMP_COMPOSE}-dalco prune-docker-stack-configs
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME}
 
 .PHONY: up-aws
-up-aws: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-aws  ## Deploys jaeger stack in aws
+up-aws: .init ${TEMP_COMPOSE}-aws prune-docker-stack-configs ## Deploys jaeger stack in aws
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME}
 
 .PHONY: up-master
-up-master:  .init .env jupyter_server_config.py ${TEMP_COMPOSE}-master
+up-master:  .init ${TEMP_COMPOSE}-master prune-docker-stack-configs
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
 
 .PHONY: up-local
@@ -85,7 +85,6 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml .env
 ${TEMP_COMPOSE}: docker-compose.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $<  > $@
 
-
 .PHONY: jupyter_server_config.py
 jupyter_server_config.py: jupyter_server_config.py.template
 	@set -o allexport; \
diff --git a/services/admin-panels/docker-compose.yml.j2 b/services/admin-panels/docker-compose.yml.j2
@@ -1,9 +1,11 @@
 version: "3.7"
 configs:
   adminpanel-jupyter-server-config:
+    name: ${STACK_NAME}_adminpanel-jupyter-server-config_{{ "./jupyter_server_config.py" | sha256file | substring(0,10) }}
     file: ./jupyter_server_config.py
 {% for item in contents %}
   {{ item.name }}:
+    name: {% raw %}${STACK_NAME}{% endraw %}_{{item.name}}_{{ ("./data/" ~ item.name) | sha256file | substring(0,10) }}
     file: ./data/{{ item.name }}{% endfor %}
 services:
   adminpanels:
diff --git a/services/admin-panels/template.env b/services/admin-panels/template.env
@@ -1,4 +1,6 @@
 # This contains all relevant secrets so that the admin panels may use them to access the services
+STACK_NAME=${STACK_NAME}
+
 MONITORING_DOMAIN=${MONITORING_DOMAIN}
 ADMINPANELS_DOMAIN=${ADMINPANELS_DOMAIN}
 DEPLOYMENT_FQDNS=${DEPLOYMENT_FQDNS}
diff --git a/services/graylog/scripts/configure.py b/services/graylog/scripts/configure.py
@@ -292,7 +292,7 @@ def configure_syslog_capture(_session: requests.Session, _headers: dict) -> None
         if len([i for i in r2["inputs"] if i["title"] == "Syslog"]) == 0:
             raw_data = (
                 '{"title":"Syslog","type":"org.graylog2.inputs.syslog.udp.SyslogUDPInput","configuration":{"bind_address":"0.0.0.0","port":'
-                + GRAYLOG_SYSLOG_CAPTURE_PORT
+                + str(GRAYLOG_SYSLOG_CAPTURE_PORT)
                 + ',"recv_buffer_size":262144,"number_worker_threads":8,"override_source":null,"force_rdns":false,"allow_override_date":true,"store_full_message":true,"expand_structured_data":false},"global":true,"node":"'
                 + node_uuid
                 + '"}'
diff --git a/services/jaeger/opentelemetry-collector-config.yaml b/services/jaeger/opentelemetry-collector-config.yaml
@@ -8,11 +8,15 @@ receivers:
 exporters:
   otlphttp:
     endpoint: ${TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT}  # Adjust to your Jaeger endpoint
+  otlp:
+    endpoint: http://tempo:4317
+    tls:
+      insecure: true
 service:
   pipelines:
     traces:
       receivers: [otlp]
-      exporters: [otlphttp]
+      exporters: [otlphttp,otlp]
       processors: [batch,probabilistic_sampler,filter/drop_healthcheck]
   telemetry:
     logs:
diff --git a/services/monitoring/Makefile b/services/monitoring/Makefile
@@ -9,13 +9,24 @@ REPO_BASE_DIR := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))../..)
 # TARGETS --------------------------------------------------
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
+define create-s3-bucket
+	# ensure bucket is available in S3...
+	@set -o allexport; \
+	source .env; \
+	echo Creating bucket "$${TEMPO_S3_BUCKET}";\
+	${REPO_BASE_DIR}/scripts/create-s3-bucket.bash "$${TEMPO_S3_BUCKET}" && \
+	set +o allexport; \
+	# bucket is available in S3
+endef
+
 .PHONY: up
 up: .init .env config.prometheus ${TEMP_COMPOSE} ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} $(STACK_NAME)
 	$(MAKE) grafana-import
 
 .PHONY: up-local
 up-local: .init .env config.prometheus.simcore ${TEMP_COMPOSE}-local ## Deploys or updates current stack "$(STACK_NAME)". If MONITORED_NETWORK is not specified, it will create an attachable network
+	@$(create-s3-bucket)
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local $(STACK_NAME)
 	$(MAKE) grafana-import
 
@@ -49,28 +60,28 @@ up-master: .init .env config.monitoring  config.prometheus.ceph.simcore ${TEMP_C
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
 	$(MAKE) grafana-import
 
-${TEMP_COMPOSE}: docker-compose.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}: docker-compose.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
-${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.http.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.http.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.http.yml > $@
 
-${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
 
-${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@
 
-${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.public.yml > $@
 
-${TEMP_COMPOSE}-aws:  docker-compose.yml docker-compose.aws.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}-aws:  docker-compose.yml docker-compose.aws.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.aws.yml > $@
 
-${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
 
-${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml
+${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml config.monitoring .env pgsql_query_exporter_config.yaml smokeping_prober_config.yaml tempo_config.yaml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
 
 docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
@@ -137,6 +148,9 @@ pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CON
 smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
 	$(call jinja, $<, .env, $@);
 
+tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
+	$(call jinja, $<, .env, $@);
+
 .PHONY: grafana/assets
 grafana/assets: ${REPO_CONFIG_LOCATION}
 	@$(MAKE_C) grafana assets
diff --git a/services/monitoring/docker-compose.yml.j2 b/services/monitoring/docker-compose.yml.j2
@@ -17,6 +17,8 @@ networks:
 configs:
   alertmanager_config:
     file: ./alertmanager/config.yml
+  tempo_config:
+    file: ./tempo_config.yaml
   node_exporter_entrypoint:
     file: ./node-exporter/docker-entrypoint.sh
   prometheus_config:
@@ -398,3 +400,27 @@ services:
         reservations:
           memory: 32M
           cpus: "0.1"
+  tempo:
+    image: grafana/tempo:2.6.1
+    command: "-target=scalable-single-binary -config.file=/etc/tempo.yaml"
+    configs:
+      - source: tempo_config
+        target: /etc/tempo.yaml
+    networks:
+      - monitored
+    deploy:
+      labels:
+        - traefik.enable=true
+        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.http.services.tempo.loadbalancer.server.port=9095
+        - traefik.http.routers.tempo.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/tempo`)
+        - traefik.http.routers.tempo.priority=10
+        - traefik.http.routers.tempo.entrypoints=https
+        - traefik.http.routers.tempo.tls=true
+        - traefik.http.middlewares.tempo_replace_regex.replacepathregex.regex=^/tempo/?(.*)$$
+        - traefik.http.middlewares.tempo_replace_regex.replacepathregex.replacement=/$${1}
+        - traefik.http.routers.tempo.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, tempo_replace_regex
+      resources:
+        limits:
+          memory: 2000M
+          cpus: "2.0"
diff --git a/services/monitoring/grafana/terraform/datasources.tf b/services/monitoring/grafana/terraform/datasources.tf
@@ -15,3 +15,11 @@ resource "grafana_data_source" "prometheuscatchall" {
   is_default         = false
   uid                = "RmZEr52nz"
 }
+
+resource "grafana_data_source" "tempo" {
+  type               = "tempo"
+  name               = "tempo"
+  url                = var.TEMPO_URL
+  basic_auth_enabled = false
+  is_default         = false
+}
diff --git a/services/monitoring/grafana/terraform/main.tf.j2 b/services/monitoring/grafana/terraform/main.tf.j2
@@ -17,10 +17,10 @@ terraform {
     skip_credentials_validation = true
     skip_requesting_account_id  = true
     skip_metadata_api_check     = true
-    skip_region_validation = true
-    skip_s3_checksum = true
+    skip_region_validation      = true
+    skip_s3_checksum            = true
     use_path_style              = true
-    endpoints = {
+    endpoints                   = {
       s3 = "{{ GRAFANA_TERRAFORM_STATE_BACKEND_S3_ENDPOINT }}"
     }
     {% endif %}
diff --git a/services/monitoring/grafana/terraform/variables.tf b/services/monitoring/grafana/terraform/variables.tf
@@ -2,6 +2,10 @@ variable "GRAFANA_URL" {
   description = "grafana_url"
   sensitive   = false
 }
+variable "TEMPO_URL" {
+  description = "tempo_url"
+  sensitive   = false
+}
 variable "GRAFANA_AUTH" {
   description = "Username:Password"
   sensitive   = true
diff --git a/services/monitoring/template.env b/services/monitoring/template.env
@@ -21,3 +21,9 @@ MONITORING_PROMETHEUS_PGSQL_GID_MONITORED=${MONITORING_PROMETHEUS_PGSQL_GID_MONI
 MONITORING_PROMETHEUS_SMOKEPING_TARGETS=${MONITORING_PROMETHEUS_SMOKEPING_TARGETS}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
 MONITORED_NETWORK=${MONITORED_NETWORK}
+TEMPO_S3_BUCKET=${TEMPO_S3_BUCKET}
+STORAGE_DOMAIN=${STORAGE_DOMAIN}
+S3_REGION=${S3_REGION}
+S3_ACCESS_KEY=${S3_ACCESS_KEY}
+S3_SECRET_KEY=${S3_SECRET_KEY}
+TF_VAR_PROMETHEUS_CATCHALL_URL=${TF_VAR_PROMETHEUS_CATCHALL_URL}
diff --git a/services/monitoring/tempo_config.yaml.j2 b/services/monitoring/tempo_config.yaml.j2
@@ -0,0 +1,52 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+#ingester:
+#  max_block_duration: 5m               # cut the headblock when this much time passes. this should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 96h             # overall Tempo trace retention.
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: {{ MACHINE_FQDN }}
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: {{ TF_VAR_PROMETHEUS_CATCHALL_URL }}/api/v1/write
+
+storage:
+  trace:
+    backend: s3                        # backend configuration to use
+    wal:
+      path: /var/tempo/wal             # where to store the wal locally
+    s3:
+      bucket: {{ TEMPO_S3_BUCKET }}                    # how to store data in s3
+      endpoint: {{STORAGE_DOMAIN}}
+      region: {{S3_REGION}}
+      access_key: {{S3_ACCESS_KEY}}
+      secret_key:  {{S3_SECRET_KEY}}
+      insecure: false
+      tls_insecure_skip_verify: true
+      # For using AWS, select the appropriate regional endpoint and region
+      # endpoint: s3.dualstack.us-west-2.amazonaws.com
+      # region: us-west-2
+
+querier:
+  frontend_worker:
+    frontend_address: localhost:9095
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: ['service-graphs', 'span-metrics']
diff --git a/services/redis-commander/.gitignore b/services/redis-commander/.gitignore
@@ -1 +1,2 @@
 config.json
+docker-compose.yml
diff --git a/services/redis-commander/Makefile b/services/redis-commander/Makefile
@@ -10,15 +10,15 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 .PHONY: up ## Deploys redis stack
-up: .init .env ${TEMP_COMPOSE}
+up: .init ${TEMP_COMPOSE} prune-docker-stack-configs
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
 
 .PHONY: up-letsencrypt-http ## Deploys redis stack using let's encrypt http challenge
-up-letsencrypt-http: .init .env ${TEMP_COMPOSE}-letsencrypt-http
+up-letsencrypt-http: .init ${TEMP_COMPOSE}-letsencrypt-http
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME}
 
 .PHONY: up-letsencrypt-dns ## Deploys redis stack using let's encrypt dns challenge
-up-letsencrypt-dns: .init .env ${TEMP_COMPOSE}-letsencrypt-dns
+up-letsencrypt-dns: .init ${TEMP_COMPOSE}-letsencrypt-dns
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME}
 
 .PHONY: up-dalco ## Deploys redis-commander stack for Dalco Cluster
@@ -36,7 +36,10 @@ up-local: up
 .PHONY: up-aws ## Deploys redis-commander stack in aws
 up-aws: up
 
-docker-compose.yml: config.json
+docker-compose.yml: .env config.json
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
+	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
+	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 
 .PHONY: ${TEMP_COMPOSE}
 ${TEMP_COMPOSE}: docker-compose.yml .env
diff --git a/services/redis-commander/docker-compose.yml.j2 b/services/redis-commander/docker-compose.yml.j2
@@ -14,6 +14,11 @@ services:
     environment:
       - URL_PREFIX=/redis
     deploy:
+      update_config:
+        order: start-first
+        delay: 10s
+        failure_action: rollback
+        parallelism: 1
       placement:
         constraints:
           - node.labels.ops == true
@@ -37,6 +42,7 @@ services:
 configs:
   redis_commander_config:
     file: ./config.json
+    name: ${STACK_NAME}_config_{{ "./config.json" | sha256file | substring(0,10) }}
 
 networks:
   public:
diff --git a/services/redis-commander/template.env b/services/redis-commander/template.env
@@ -1,3 +1,5 @@
+STACK_NAME=${STACK_NAME}
+
 MONITORING_DOMAIN=${MONITORING_DOMAIN}
 PREFIX_STACK_NAME=${PREFIX_STACK_NAME}
 REDIS_HOST=${REDIS_HOST}
diff --git a/services/registry/Makefile b/services/registry/Makefile
diff --git a/services/registry/docker-compose.yml.j2 b/services/registry/docker-compose.yml.j2
diff --git a/services/registry/template.env b/services/registry/template.env
