Merge branch 'ITISFoundation:main' into main

mrnicegyu11 · web-flow · commit cdef57f26b82 · 2025-05-21T11:28:15.000+02:00
diff --git a/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl b/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl
@@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-replicaCount: 1
+replicaCount: {{ requiredEnv "RESOURCE_USAGE_TRACKER_KUBERNETES_REPLICAS" }}
 
 image:
   repository: '{{  .Values | get "DOCKER_REGISTRY" "itisfoundation" }}/resource-usage-tracker'
@@ -104,14 +104,14 @@ env:
   - name: POSTGRES_DB
     value: {{ requiredEnv "POSTGRES_DB" }}
   - name: POSTGRES_ENDPOINT
-    value: {{ requiredEnv "POSTGRES_ENDPOINT" }}
+    value: {{ requiredEnv "POSTGRES_EXTERNAL_ENDPOINT" }}
   - name: POSTGRES_HOST
-    value: {{ requiredEnv "POSTGRES_HOST" }}
+    value: {{ requiredEnv "POSTGRES_EXTERNAL_HOST" }}
   - name: POSTGRES_PASSWORD
     value: {{ requiredEnv "POSTGRES_PASSWORD" }}
     sensitive: true
   - name: POSTGRES_PORT
-    value: {{ requiredEnv "POSTGRES_PORT" }}
+    value: {{ requiredEnv "POSTGRES_EXTERNAL_PORT" }}
   - name: POSTGRES_USER
     value: {{ requiredEnv "POSTGRES_USER" }}
   - name: PROMETHEUS_URL
@@ -122,18 +122,18 @@ env:
     value: {{ requiredEnv "RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD" }}
     sensitive: true
   - name: RABBIT_HOST
-    value: {{ requiredEnv "RABBIT_HOST" }}
+    value: {{ requiredEnv "RABBIT_EXTERNAL_HOST" }}
   - name: RABBIT_PASSWORD
     value: {{ requiredEnv "RABBIT_PASSWORD" }}
     sensitive: true
   - name: RABBIT_PORT
-    value: {{ requiredEnv "RABBIT_PORT" }}
+    value: {{ requiredEnv "RABBIT_EXTERNAL_PORT" }}
   - name: RABBIT_SECURE
     value: {{ requiredEnv "RABBIT_SECURE" }}
   - name: RABBIT_USER
     value: {{ requiredEnv "RABBIT_USER" }}
   - name: REDIS_USER
-    value: {{ requiredEnv "REDIS_USER" }}
+    value: {{ env "REDIS_USER" }}
   - name: REDIS_HOST
     value: {{ requiredEnv "REDIS_EXTERNAL_HOST" }}
   - name: REDIS_PORT
diff --git a/services/admin-panels/docker-compose.yml.j2 b/services/admin-panels/docker-compose.yml.j2
@@ -87,7 +87,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.adminpanels.loadbalancer.server.port=8888
         - traefik.http.routers.adminpanels.rule=Host(`${ADMINPANELS_DOMAIN}`)
         - traefik.http.routers.adminpanels.entrypoints=https
diff --git a/services/appmotion_gateway/docker-compose.yml.j2 b/services/appmotion_gateway/docker-compose.yml.j2
@@ -13,7 +13,7 @@ services:
             replicas: 0
             labels:
                 - traefik.enable=true
-                - traefik.docker.network=${PUBLIC_NETWORK}
+                - traefik.swarm.network=${PUBLIC_NETWORK}
                 - traefik.http.services.adminer_appmotion_gateway.loadbalancer.server.port=8080
                 - traefik.http.routers.adminer_appmotion_gateway.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/adminer/appmotion_gateway`)
                 - traefik.http.routers.adminer_appmotion_gateway.entrypoints=https
@@ -75,7 +75,7 @@ services:
             replicas: 1
             labels:
                 - traefik.enable=true
-                - traefik.docker.network=${PUBLIC_NETWORK}
+                - traefik.swarm.network=${PUBLIC_NETWORK}
                 - traefik.http.routers.appmotion_gateway.rule=${DEPLOYMENT_FQDNS_APPMOTION_CAPTURE_TRAEFIK_RULE}
                 - traefik.http.routers.appmotion_gateway.entrypoints=https
                 - traefik.http.routers.appmotion_gateway.tls=true
diff --git a/services/filestash/docker-compose.yml.j2 b/services/filestash/docker-compose.yml.j2
@@ -13,7 +13,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.filestash.loadbalancer.server.port=8334
         - traefik.http.routers.filestash.rule=Host(`${FILESTASH_DOMAIN}`)
         - traefik.http.routers.filestash.entrypoints=https
diff --git a/services/graylog/docker-compose.yml.j2 b/services/graylog/docker-compose.yml.j2
@@ -88,7 +88,7 @@ services:
 
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.graylog.loadbalancer.server.port=9000
         - traefik.http.routers.graylog.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/graylog`)
diff --git a/services/jaeger/docker-compose.yml.j2 b/services/jaeger/docker-compose.yml.j2
@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   jaeger:
-    image: jaegertracing/all-in-one:1.60.0
+    image: jaegertracing/all-in-one:1.68.0
     command: >
       --query.base-path=/jaeger
     init: true
@@ -17,7 +17,7 @@ services:
           - node.labels.ops==true
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.jaeger.loadbalancer.server.port=16686
         - traefik.http.routers.jaeger.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/jaeger`)
         - traefik.http.routers.jaeger.entrypoints=https
@@ -33,7 +33,7 @@ services:
           memory: 64M
           cpus: "0.1"
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.105.0
+    image: otel/opentelemetry-collector-contrib:0.123.0
     configs:
       - source: opentelemetry-collector-config
         target: /etc/otel/config.yaml
diff --git a/services/maintenance-page/docker-compose.yml.j2 b/services/maintenance-page/docker-compose.yml.j2
@@ -26,7 +26,7 @@ services:
           - node.labels.ops==true
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`,`{subhost:[a-zA-Z0-9-]+}.services.{{j2item}}`) && PathPrefix(`/`))
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.tls=true
@@ -50,7 +50,7 @@ services:
           - node.labels.ops==true
       labels:
           - traefik.enable=true
-          - traefik.docker.network=${PUBLIC_NETWORK}
+          - traefik.swarm.network=${PUBLIC_NETWORK}
           - traefik.http.routers.nginx_api.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
           - traefik.http.routers.nginx_api.tls=true
           - traefik.http.routers.nginx_api.rule=${DEPLOYMENT_API_DOMAIN_CAPTURE_TRAEFIK_RULE}
diff --git a/services/minio/docker-compose.yaml b/services/minio/docker-compose.yaml
@@ -22,7 +22,7 @@ services:
         window: 60s
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access without path (necessary for minio client it does not like /path)
         - traefik.http.services.minio9000.loadbalancer.server.port=9000
         - traefik.http.services.minio9000.loadbalancer.healthcheck.path=/minio/health/ready
diff --git a/services/monitoring/docker-compose.yml.j2 b/services/monitoring/docker-compose.yml.j2
@@ -25,7 +25,7 @@ services:
           cpus: "0.1"
   prometheuscatchall:
     hostname: "{% raw %}{{.Service.Name}}{% endraw %}"
-    image: prom/prometheus:v2.54.0
+    image: prom/prometheus:v2.55.1
     volumes:
       - prometheus_data:/prometheus
       - /var/run/docker.sock:/var/run/docker.sock:ro
@@ -44,6 +44,7 @@ services:
       - "--web.external-url=https://${MONITORING_DOMAIN}/prometheus/"
       - "--web.route-prefix=/"
       - "--storage.tsdb.allow-overlapping-blocks" # via https://jessicagreben.medium.com/prometheus-fill-in-data-for-new-recording-rules-30a14ccb8467
+      - "--enable-feature=exemplar-storage"
       #- "--web.enable-admin-api" This allows messing with prometheus using its API from the CLI. Disabled for security reasons by default.
     networks:
       - monitored
@@ -52,7 +53,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.prometheuscatchall.loadbalancer.server.port=${MONITORING_PROMETHEUS_PORT}
         - traefik.http.routers.prometheuscatchall.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/prometheus`)
@@ -71,7 +72,7 @@ services:
           cpus: "0.2"
   prometheusfederation:
     hostname: "{% raw %}{{.Service.Name}}{% endraw %}"
-    image: prom/prometheus:v2.54.0
+    image: prom/prometheus:v2.55.1
     volumes:
       - prometheus_data_federation:/prometheus
       - /var/run/docker.sock:/var/run/docker.sock:ro
@@ -98,7 +99,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.prometheusfederation.loadbalancer.server.port=${MONITORING_PROMETHEUS_PORT}
         - traefik.http.routers.prometheusfederation.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/prometheusfederation`)
@@ -218,7 +219,7 @@ services:
           cpus: "0.1"
 
   grafana:
-    image: grafana/grafana-oss:11.2.4
+    image: grafana/grafana-oss:12.0.0
     volumes:
       - grafana_data:/var/lib/grafana
     env_file:
@@ -232,7 +233,7 @@ services:
       #  condition: on-failure
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.grafana.loadbalancer.server.port=3000
         - traefik.http.routers.grafana.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/grafana`)
@@ -370,7 +371,7 @@ services:
           memory: 32M
           cpus: "0.1"
   tempo:
-    image: grafana/tempo:2.6.1
+    image: grafana/tempo:2.7.2
     command: "-target=scalable-single-binary -config.file=/etc/tempo.yaml"
     configs:
       - source: tempo_config
@@ -380,7 +381,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.tempo.loadbalancer.server.port=9095
         - traefik.http.routers.tempo.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/tempo`)
         - traefik.http.routers.tempo.priority=10
diff --git a/services/monitoring/grafana/Makefile b/services/monitoring/grafana/Makefile
@@ -27,7 +27,7 @@ terraform/plan.cache:
 	@exit 1
 
 .PHONY: terraform-plan
-terraform-plan: $(REPO_CONFIG_LOCATION) $(TF_STATE_FILE) ensure-grafana-online ## terraform plan
+terraform-plan: $(REPO_CONFIG_LOCATION) $(TF_STATE_FILE) ensure-grafana-online assets ## terraform plan
 	# terraform plan
 	@set -a; source $<; set +a; \
 	terraform -chdir=./terraform plan -out=plan.cache
diff --git a/services/monitoring/grafana/terraform/datasources.tf b/services/monitoring/grafana/terraform/datasources.tf
@@ -14,6 +14,16 @@ resource "grafana_data_source" "prometheuscatchall" {
   basic_auth_enabled = false
   is_default         = false
   uid                = "RmZEr52nz"
+
+  json_data_encoded = jsonencode({
+    exemplarTraceIdDestinations = [
+      {
+        datasourceUid = "tempo"
+        name          = "TraceID"
+      }
+    ]
+  })
+
 }
 
 resource "grafana_data_source" "tempo" {
@@ -23,3 +33,22 @@ resource "grafana_data_source" "tempo" {
   basic_auth_enabled = false
   is_default         = false
 }
+
+resource "grafana_data_source" "cloudwatch" {
+  # This resource is only created if the AWS Deployments
+  count = var.IS_AWS_DEPLOYMENT ? 1 : 0
+
+  type = "cloudwatch"
+  name = "cloudwatch"
+  uid  = "fem2inr5v64n4c"
+
+  json_data_encoded = jsonencode({
+    defaultRegion = var.AWS_DEFAULT_REGION
+    authType      = "keys"
+  })
+
+  secure_json_data_encoded = jsonencode({
+    accessKey = var.GRAFANA_CLOUDWATCH_DATASOURCE_USER_ACCESS_KEY
+    secretKey = var.GRAFANA_CLOUDWATCH_DATASOURCE_USER_SECRET_KEY
+  })
+}
diff --git a/services/monitoring/grafana/terraform/variables.tf b/services/monitoring/grafana/terraform/variables.tf
@@ -18,3 +18,26 @@ variable "PROMETHEUS_CATCHALL_URL" {
   description = "Prometheus Catchall URL"
   sensitive   = false
 }
+
+# to be used to create resource only for AWS / non AWS Deployments
+# this is not syncronised / validated with dashboards imported from folders
+variable "IS_AWS_DEPLOYMENT" {
+  description = "Is AWS Deployment"
+  type        = bool
+  default     = false
+}
+
+variable "AWS_DEFAULT_REGION" {
+  description = "AWS Region"
+  sensitive   = false
+}
+
+variable "GRAFANA_CLOUDWATCH_DATASOURCE_USER_ACCESS_KEY" {
+  description = "AWS Grafana Cloudwatch User Access Key"
+  sensitive   = true
+}
+
+variable "GRAFANA_CLOUDWATCH_DATASOURCE_USER_SECRET_KEY" {
+  description = "AWS Grafana Cloudwatch User Secret Key"
+  sensitive   = true
+}
diff --git a/services/monitoring/prometheus/prometheus-simcore.yml b/services/monitoring/prometheus/prometheus-simcore.yml
@@ -92,7 +92,7 @@ scrape_configs:
       - names:
           - "tasks.production_rabbit"
           - "tasks.staging_rabbit"
-          - "task.master_rabbit"
+          - "tasks.master_rabbit"
         type: "A"
         port: 15692
       - names:
diff --git a/services/monitoring/tempo_config.yaml.j2 b/services/monitoring/tempo_config.yaml.j2
@@ -6,7 +6,9 @@ distributor:
     otlp:
       protocols:
         http:
+          endpoint: "0.0.0.0:4318"  # Explicitly bind to all interfaces https://github.com/grafana/tempo/issues/4465
         grpc:
+          endpoint: "0.0.0.0:4317"  # Explicitly bind to all interfaces https://github.com/grafana/tempo/issues/4465
 
 #ingester:
 #  max_block_duration: 5m               # cut the headblock when this much time passes. this should probably be left alone normally
@@ -38,13 +40,10 @@ storage:
       secret_key:  {{S3_SECRET_KEY}}
       insecure: false
       tls_insecure_skip_verify: true
-      # For using AWS, select the appropriate regional endpoint and region
-      # endpoint: s3.dualstack.us-west-2.amazonaws.com
-      # region: us-west-2
 
 querier:
   frontend_worker:
-    frontend_address: localhost:9095
+    frontend_address: 0.0.0.0:9095
 
 overrides:
   defaults:
diff --git a/services/portainer/docker-compose.yml b/services/portainer/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   agent:
-    image: portainer/agent:2.20.3 # Needs to match portainer-ce version
+    image: portainer/agent:2.27.6 # Needs to match portainer-ce version
     init: true
     environment:
       # REQUIRED: Should be equal to the service name prefixed by "tasks." when
@@ -26,7 +26,7 @@ services:
           memory: 50M
 
   portainer:
-    image: portainer/portainer-ce:2.20.3
+    image: portainer/portainer-ce:2.27.6
     init: true
     command: >
       -H tcp://tasks.agent:9001 --tlsskipverify
diff --git a/services/redis-commander/docker-compose.yml.j2 b/services/redis-commander/docker-compose.yml.j2
@@ -24,7 +24,7 @@ services:
           - node.labels.ops == true
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.redis.loadbalancer.server.port=8081
         - traefik.http.routers.redis.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/redis`)
         - traefik.http.routers.redis.entrypoints=https
diff --git a/services/registry/docker-compose.yml.j2 b/services/registry/docker-compose.yml.j2
@@ -53,7 +53,7 @@ services:
           cpus: '0.1'
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.registry.loadbalancer.server.port=5000
         - traefik.http.routers.registry.rule=Host(`${REGISTRY_DOMAIN}`)
diff --git a/services/simcore/Makefile b/services/simcore/Makefile
@@ -15,7 +15,7 @@ $(SIMCORE_REPO_DIR):
 	$(error $@ repo not found. Please clone this repo manually)
 
 .PHONY: stack_with_prefix.yml
-stack_with_prefix.yml: $(SIMCORE_REPO_DIR) $(REPO_CONFIG_LOCATION)
+stack_with_prefix.yml: $(SIMCORE_REPO_DIR) $(REPO_CONFIG_LOCATION) ## generate stack file with prefix
 	# generating $@
 	@$(REPO_BASE_DIR)/scripts/deployments/prepare_simcore_stack.bash
 	@mv $(REPO_BASE_DIR)/scripts/deployments/stack_with_prefix.yml $@
diff --git a/services/simcore/docker-compose.deploy.master.yml b/services/simcore/docker-compose.deploy.master.yml
diff --git a/services/simcore/docker-compose.yml.j2 b/services/simcore/docker-compose.yml.j2
diff --git a/services/traefik/docker-compose.master.yml b/services/traefik/docker-compose.master.yml
diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2
diff --git a/services/traefik/template.env b/services/traefik/template.env
diff --git a/services/vendors/docker-compose.yml.j2 b/services/vendors/docker-compose.yml.j2
