Merge remote-tracking branch 'upstream/main' into 2025/add/fluentd

mrnicegyu11 · mrnicegyu11 · commit 57947e3128a1 · 2025-05-21T11:58:36.000+02:00
diff --git a/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl b/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl
@@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-replicaCount: 1
+replicaCount: {{ requiredEnv "RESOURCE_USAGE_TRACKER_KUBERNETES_REPLICAS" }}
 
 image:
   repository: '{{  .Values | get "DOCKER_REGISTRY" "itisfoundation" }}/resource-usage-tracker'
@@ -104,14 +104,14 @@ env:
   - name: POSTGRES_DB
     value: {{ requiredEnv "POSTGRES_DB" }}
   - name: POSTGRES_ENDPOINT
-    value: {{ requiredEnv "POSTGRES_ENDPOINT" }}
+    value: {{ requiredEnv "POSTGRES_EXTERNAL_ENDPOINT" }}
   - name: POSTGRES_HOST
-    value: {{ requiredEnv "POSTGRES_HOST" }}
+    value: {{ requiredEnv "POSTGRES_EXTERNAL_HOST" }}
   - name: POSTGRES_PASSWORD
     value: {{ requiredEnv "POSTGRES_PASSWORD" }}
     sensitive: true
   - name: POSTGRES_PORT
-    value: {{ requiredEnv "POSTGRES_PORT" }}
+    value: {{ requiredEnv "POSTGRES_EXTERNAL_PORT" }}
   - name: POSTGRES_USER
     value: {{ requiredEnv "POSTGRES_USER" }}
   - name: PROMETHEUS_URL
@@ -122,18 +122,18 @@ env:
     value: {{ requiredEnv "RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD" }}
     sensitive: true
   - name: RABBIT_HOST
-    value: {{ requiredEnv "RABBIT_HOST" }}
+    value: {{ requiredEnv "RABBIT_EXTERNAL_HOST" }}
   - name: RABBIT_PASSWORD
     value: {{ requiredEnv "RABBIT_PASSWORD" }}
     sensitive: true
   - name: RABBIT_PORT
-    value: {{ requiredEnv "RABBIT_PORT" }}
+    value: {{ requiredEnv "RABBIT_EXTERNAL_PORT" }}
   - name: RABBIT_SECURE
     value: {{ requiredEnv "RABBIT_SECURE" }}
   - name: RABBIT_USER
     value: {{ requiredEnv "RABBIT_USER" }}
   - name: REDIS_USER
-    value: {{ requiredEnv "REDIS_USER" }}
+    value: {{ env "REDIS_USER" }}
   - name: REDIS_HOST
     value: {{ requiredEnv "REDIS_EXTERNAL_HOST" }}
   - name: REDIS_PORT
diff --git a/services/admin-panels/docker-compose.yml.j2 b/services/admin-panels/docker-compose.yml.j2
@@ -87,7 +87,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.adminpanels.loadbalancer.server.port=8888
         - traefik.http.routers.adminpanels.rule=Host(`${ADMINPANELS_DOMAIN}`)
         - traefik.http.routers.adminpanels.entrypoints=https
diff --git a/services/appmotion_gateway/docker-compose.yml.j2 b/services/appmotion_gateway/docker-compose.yml.j2
@@ -13,7 +13,7 @@ services:
             replicas: 0
             labels:
                 - traefik.enable=true
-                - traefik.docker.network=${PUBLIC_NETWORK}
+                - traefik.swarm.network=${PUBLIC_NETWORK}
                 - traefik.http.services.adminer_appmotion_gateway.loadbalancer.server.port=8080
                 - traefik.http.routers.adminer_appmotion_gateway.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/adminer/appmotion_gateway`)
                 - traefik.http.routers.adminer_appmotion_gateway.entrypoints=https
@@ -75,7 +75,7 @@ services:
             replicas: 1
             labels:
                 - traefik.enable=true
-                - traefik.docker.network=${PUBLIC_NETWORK}
+                - traefik.swarm.network=${PUBLIC_NETWORK}
                 - traefik.http.routers.appmotion_gateway.rule=${DEPLOYMENT_FQDNS_APPMOTION_CAPTURE_TRAEFIK_RULE}
                 - traefik.http.routers.appmotion_gateway.entrypoints=https
                 - traefik.http.routers.appmotion_gateway.tls=true
diff --git a/services/filestash/docker-compose.yml.j2 b/services/filestash/docker-compose.yml.j2
@@ -13,7 +13,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.filestash.loadbalancer.server.port=8334
         - traefik.http.routers.filestash.rule=Host(`${FILESTASH_DOMAIN}`)
         - traefik.http.routers.filestash.entrypoints=https
diff --git a/services/graylog/docker-compose.yml.j2 b/services/graylog/docker-compose.yml.j2
@@ -48,7 +48,7 @@ services:
           cpus: "0.1"
   # Graylog: https://hub.docker.com/r/graylog/graylog/
   graylog:
-    image: graylog/graylog:6.2.2
+    image: graylog/graylog:6.0.5
     init: true
     # user: "1000:1001"
     configs:
@@ -88,7 +88,7 @@ services:
 
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.graylog.loadbalancer.server.port=9000
         - traefik.http.routers.graylog.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/graylog`)
diff --git a/services/jaeger/docker-compose.yml.j2 b/services/jaeger/docker-compose.yml.j2
@@ -17,7 +17,7 @@ services:
           - node.labels.ops==true
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.jaeger.loadbalancer.server.port=16686
         - traefik.http.routers.jaeger.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/jaeger`)
         - traefik.http.routers.jaeger.entrypoints=https
diff --git a/services/maintenance-page/docker-compose.yml.j2 b/services/maintenance-page/docker-compose.yml.j2
@@ -26,7 +26,7 @@ services:
           - node.labels.ops==true
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`,`{subhost:[a-zA-Z0-9-]+}.services.{{j2item}}`) && PathPrefix(`/`))
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.tls=true
@@ -50,7 +50,7 @@ services:
           - node.labels.ops==true
       labels:
           - traefik.enable=true
-          - traefik.docker.network=${PUBLIC_NETWORK}
+          - traefik.swarm.network=${PUBLIC_NETWORK}
           - traefik.http.routers.nginx_api.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
           - traefik.http.routers.nginx_api.tls=true
           - traefik.http.routers.nginx_api.rule=${DEPLOYMENT_API_DOMAIN_CAPTURE_TRAEFIK_RULE}
diff --git a/services/minio/docker-compose.yaml b/services/minio/docker-compose.yaml
@@ -22,7 +22,7 @@ services:
         window: 60s
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access without path (necessary for minio client it does not like /path)
         - traefik.http.services.minio9000.loadbalancer.server.port=9000
         - traefik.http.services.minio9000.loadbalancer.healthcheck.path=/minio/health/ready
diff --git a/services/monitoring/docker-compose.yml.j2 b/services/monitoring/docker-compose.yml.j2
@@ -44,6 +44,7 @@ services:
       - "--web.external-url=https://${MONITORING_DOMAIN}/prometheus/"
       - "--web.route-prefix=/"
       - "--storage.tsdb.allow-overlapping-blocks" # via https://jessicagreben.medium.com/prometheus-fill-in-data-for-new-recording-rules-30a14ccb8467
+      - "--enable-feature=exemplar-storage"
       #- "--web.enable-admin-api" This allows messing with prometheus using its API from the CLI. Disabled for security reasons by default.
     networks:
       - monitored
@@ -52,7 +53,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.prometheuscatchall.loadbalancer.server.port=${MONITORING_PROMETHEUS_PORT}
         - traefik.http.routers.prometheuscatchall.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/prometheus`)
@@ -98,7 +99,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.prometheusfederation.loadbalancer.server.port=${MONITORING_PROMETHEUS_PORT}
         - traefik.http.routers.prometheusfederation.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/prometheusfederation`)
@@ -232,7 +233,7 @@ services:
       #  condition: on-failure
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.grafana.loadbalancer.server.port=3000
         - traefik.http.routers.grafana.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/grafana`)
@@ -380,7 +381,7 @@ services:
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.tempo.loadbalancer.server.port=9095
         - traefik.http.routers.tempo.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/tempo`)
         - traefik.http.routers.tempo.priority=10
diff --git a/services/monitoring/grafana/Makefile b/services/monitoring/grafana/Makefile
@@ -27,7 +27,7 @@ terraform/plan.cache:
 	@exit 1
 
 .PHONY: terraform-plan
-terraform-plan: $(REPO_CONFIG_LOCATION) $(TF_STATE_FILE) ensure-grafana-online ## terraform plan
+terraform-plan: $(REPO_CONFIG_LOCATION) $(TF_STATE_FILE) ensure-grafana-online assets ## terraform plan
 	# terraform plan
 	@set -a; source $<; set +a; \
 	terraform -chdir=./terraform plan -out=plan.cache
diff --git a/services/monitoring/grafana/terraform/datasources.tf b/services/monitoring/grafana/terraform/datasources.tf
@@ -14,6 +14,16 @@ resource "grafana_data_source" "prometheuscatchall" {
   basic_auth_enabled = false
   is_default         = false
   uid                = "RmZEr52nz"
+
+  json_data_encoded = jsonencode({
+    exemplarTraceIdDestinations = [
+      {
+        datasourceUid = "tempo"
+        name          = "TraceID"
+      }
+    ]
+  })
+
 }
 
 resource "grafana_data_source" "tempo" {
@@ -31,3 +41,21 @@ resource "grafana_data_source" "loki" {
   basic_auth_enabled = false
   is_default         = false
 }
+resource "grafana_data_source" "cloudwatch" {
+  # This resource is only created if the AWS Deployments
+  count = var.IS_AWS_DEPLOYMENT ? 1 : 0
+
+  type = "cloudwatch"
+  name = "cloudwatch"
+  uid  = "fem2inr5v64n4c"
+
+  json_data_encoded = jsonencode({
+    defaultRegion = var.AWS_DEFAULT_REGION
+    authType      = "keys"
+  })
+
+  secure_json_data_encoded = jsonencode({
+    accessKey = var.GRAFANA_CLOUDWATCH_DATASOURCE_USER_ACCESS_KEY
+    secretKey = var.GRAFANA_CLOUDWATCH_DATASOURCE_USER_SECRET_KEY
+  })
+}
diff --git a/services/monitoring/grafana/terraform/variables.tf b/services/monitoring/grafana/terraform/variables.tf
@@ -18,3 +18,26 @@ variable "PROMETHEUS_CATCHALL_URL" {
   description = "Prometheus Catchall URL"
   sensitive   = false
 }
+
+# to be used to create resource only for AWS / non AWS Deployments
+# this is not syncronised / validated with dashboards imported from folders
+variable "IS_AWS_DEPLOYMENT" {
+  description = "Is AWS Deployment"
+  type        = bool
+  default     = false
+}
+
+variable "AWS_DEFAULT_REGION" {
+  description = "AWS Region"
+  sensitive   = false
+}
+
+variable "GRAFANA_CLOUDWATCH_DATASOURCE_USER_ACCESS_KEY" {
+  description = "AWS Grafana Cloudwatch User Access Key"
+  sensitive   = true
+}
+
+variable "GRAFANA_CLOUDWATCH_DATASOURCE_USER_SECRET_KEY" {
+  description = "AWS Grafana Cloudwatch User Secret Key"
+  sensitive   = true
+}
diff --git a/services/monitoring/prometheus/prometheus-simcore.yml b/services/monitoring/prometheus/prometheus-simcore.yml
@@ -92,7 +92,7 @@ scrape_configs:
       - names:
           - "tasks.production_rabbit"
           - "tasks.staging_rabbit"
-          - "task.master_rabbit"
+          - "tasks.master_rabbit"
         type: "A"
         port: 15692
       - names:
diff --git a/services/redis-commander/docker-compose.yml.j2 b/services/redis-commander/docker-compose.yml.j2
@@ -24,7 +24,7 @@ services:
           - node.labels.ops == true
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.redis.loadbalancer.server.port=8081
         - traefik.http.routers.redis.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/redis`)
         - traefik.http.routers.redis.entrypoints=https
diff --git a/services/registry/docker-compose.yml.j2 b/services/registry/docker-compose.yml.j2
@@ -53,7 +53,7 @@ services:
           cpus: '0.1'
       labels:
         - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.swarm.network=${PUBLIC_NETWORK}
         # direct access through port
         - traefik.http.services.registry.loadbalancer.server.port=5000
         - traefik.http.routers.registry.rule=Host(`${REGISTRY_DOMAIN}`)
diff --git a/services/simcore/Makefile b/services/simcore/Makefile
@@ -15,7 +15,7 @@ $(SIMCORE_REPO_DIR):
 	$(error $@ repo not found. Please clone this repo manually)
 
 .PHONY: stack_with_prefix.yml
-stack_with_prefix.yml: $(SIMCORE_REPO_DIR) $(REPO_CONFIG_LOCATION)
+stack_with_prefix.yml: $(SIMCORE_REPO_DIR) $(REPO_CONFIG_LOCATION) ## generate stack file with prefix
 	# generating $@
 	@$(REPO_BASE_DIR)/scripts/deployments/prepare_simcore_stack.bash
 	@mv $(REPO_BASE_DIR)/scripts/deployments/stack_with_prefix.yml $@
diff --git a/services/simcore/docker-compose.deploy.master.yml b/services/simcore/docker-compose.deploy.master.yml
@@ -35,6 +35,7 @@ services:
       - "--entryPoints.http.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.http.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.postgres.address=:5432"
+      - "--entryPoints.redis.address=:${REDIS_EXTERNAL_PORT}"
       - "--entryPoints.simcore_api.address=:10081"
       - "--entryPoints.simcore_api.forwardedHeaders.insecure"
       - "--entryPoints.simcore_api.transport.respondingTimeouts.idleTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
@@ -59,17 +60,24 @@ services:
           memory: 2048M
           cpus: '2.000'
       labels:
-        # oSparc postgres
+        # postgres
         - traefik.tcp.routers.${SWARM_STACK_NAME}_postgresRoute.entrypoints=postgres
         - traefik.tcp.routers.${SWARM_STACK_NAME}_postgresRoute.tls=false
         - traefik.tcp.routers.${SWARM_STACK_NAME}_postgresRoute.service=${SWARM_STACK_NAME}_postgresRoute
         - traefik.tcp.services.${SWARM_STACK_NAME}_postgresRoute.loadbalancer.server.port=5432
-        - "traefik.tcp.routers.${SWARM_STACK_NAME}_postgresRoute.rule=ClientIP(`195.176.8.0/24`) || ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
+        - "traefik.tcp.routers.${SWARM_STACK_NAME}_postgresRoute.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
+        # redis
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.entrypoints=redis
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.tls=false
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.service=${SWARM_STACK_NAME}_redisRoute
+        - traefik.tcp.services.${SWARM_STACK_NAME}_redisRoute.loadbalancer.server.port=${REDIS_EXTERNAL_PORT}
+        - "traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.rule=HostSNI(`*`)"
+
   clusters-keeper:
     deploy:
       replicas: 0
+
   payments:
-    # test prior to a big bang release Sept 2023. Can be deleted after (YH, 09 2023)
     deploy:
       replicas: 1
 
@@ -80,3 +88,28 @@ services:
   wb-api-server:
     deploy:
       replicas: 3
+
+  redis:
+    networks:
+      - public
+    deploy:
+      labels:
+        - traefik.enable=true
+        - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}
+        - traefik.swarm.network=${SWARM_STACK_NAME}_default
+        - "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis
+        - traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT}
+
+  rabbit:
+    # rabbit is already exposed via ops traefik
+    # adding one more route to this configuration
+    deploy:
+      labels:
+        - "traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.entrypoints=rabbitmq
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.tls=false
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.service=${SWARM_STACK_NAME}_rabbitmq
+        - traefik.tcp.services.${SWARM_STACK_NAME}_rabbitmq.loadbalancer.server.port=${RABBIT_PORT}
diff --git a/services/simcore/docker-compose.yml.j2 b/services/simcore/docker-compose.yml.j2
diff --git a/services/traefik/docker-compose.master.yml b/services/traefik/docker-compose.master.yml
diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2
diff --git a/services/traefik/template.env b/services/traefik/template.env
diff --git a/services/vendors/docker-compose.yml.j2 b/services/vendors/docker-compose.yml.j2
