Merge branch 'main' into 2025/refactor/pgBackup

Author: mrnicegyu11

.github/PULL_REQUEST_TEMPLATE.md

@@ -19,10 +19,11 @@
 - [ ] Service is restartable
 - [ ] Service restart is zero-downtime
 - [ ] Service has >1 replicas in PROD
-- [ ] Service has docker heathlcheck enabled
+- [ ] Service has docker healthcheck enabled
 - [ ] Service is monitored (via prometheus and grafana)
 - [ ] Service is not bound to one specific node (e.g. via files or volumes)
 - [ ] Relevant OPS E2E Test are added
+- [ ] Grafana dashboards updated accordingly
 
 If exposed via traefik
 - [ ] Service's Public URL is included in maintenance mode

charts/Makefile

@@ -50,9 +50,12 @@ helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile con
 	fi
 
 .PHONY: configure-local-hosts
-configure-local-hosts: ## Adds local hosts entries for the machine
-	@echo "Adding $(MACHINE_FQDN) hosts to /etc/hosts ..."
-	@grep -q '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' /etc/hosts || echo '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' | sudo tee -a /etc/hosts
+configure-local-hosts: $(REPO_CONFIG_LOCATION) ## Adds local hosts entries for the machine
+	# "Updating /etc/hosts with k8s $(MACHINE_FQDN) hosts ..."
+	@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
+	grep -q "127.0.0.1 $$K8S_MONITORING_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_MONITORING_FQDN" | sudo tee -a /etc/hosts
+	@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
+	grep -q "127.0.0.1 $$K8S_PRIVATE_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_PRIVATE_FQDN" | sudo tee -a /etc/hosts
 
 .PHONY: helmfile-diff
 helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile

scripts/deployments/deploy_everything_locally.bash

@@ -258,4 +258,9 @@ if [ "$start_simcore" -eq 0 ]; then
     pushd "${service_dir}"
     call_make "." up-"$stack_target"
     popd
+    log_info "starting vendor services..."
+    service_dir="${repo_basedir}"/services/vendors
+    pushd "${service_dir}"
+    call_make "." up-"$stack_target"
+    popd
 fi

services/maintenance-page/docker-compose.yml.j2

@@ -28,6 +28,7 @@ services:
         - traefik.enable=true
         - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
+        - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_CHATBOT_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`) && PathPrefix(`/`))
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`) && PathPrefix(`/`))
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.tls=true
         - traefik.http.services.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.loadbalancer.server.port=80

services/simcore/docker-compose.yml.j2

@@ -337,6 +337,34 @@ services:
           cpus: "1.0"
           memory: "512M"
 
+  wb-auth:
+    networks:
+      - monitored  # traces
+      - public  # public service use auth
+    deploy:
+      replicas: ${WB_AUTH_REPLICAS}
+      update_config:
+        parallelism: 2
+        order: start-first
+        failure_action: rollback
+        delay: 10s
+      restart_policy:
+        condition: any
+        delay: 5s
+        max_attempts: 3
+        window: 120s
+      placement:
+        constraints:
+          - node.labels.simcore==true
+      resources:
+        reservations:
+          cpus: "0.1"
+          memory: "256M"
+        limits:
+          cpus: "1"
+          memory: "1G"
+      # healthcheck: defined in image
+
   storage:
     environment:
       - S3_ENDPOINT=${S3_ENDPOINT}

services/traefik/docker-compose.yml.j2

@@ -131,7 +131,7 @@ services:
         - traefik.http.middlewares.ops_ratelimit.ratelimit.sourcecriterion.ipstrategy.depth=1
         # Platform user auth: Use this middleware to enforce only authenticated users
         # https://doc.traefik.io/traefik/middlewares/http/forwardauth
-        - traefik.http.middlewares.authenticated_platform_user.forwardauth.address=http://${WEBSERVER_HOST}:${WEBSERVER_PORT}/v0/auth:check
+        - traefik.http.middlewares.authenticated_platform_user.forwardauth.address=http://${WB_AUTH_WEBSERVER_HOST}:${WB_AUTH_WEBSERVER_PORT}/v0/auth:check
         - traefik.http.middlewares.authenticated_platform_user.forwardauth.trustForwardHeader=true
         - traefik.http.middlewares.authenticated_platform_user.forwardauth.authResponseHeaders=Set-Cookie,osparc-sc2
         #

services/traefik/template.env

@@ -34,8 +34,8 @@ DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE='${DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFI
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
 MONITORED_NETWORK=${MONITORED_NETWORK}
 
-WEBSERVER_HOST=${WEBSERVER_HOST}
-WEBSERVER_PORT=${WEBSERVER_PORT}
+WB_AUTH_WEBSERVER_HOST=${WB_AUTH_WEBSERVER_HOST}
+WB_AUTH_WEBSERVER_PORT=${WB_AUTH_WEBSERVER_PORT}
 
 TRAEFIK_DOMAINS_REDIRECT_FROM=${TRAEFIK_DOMAINS_REDIRECT_FROM}
 TRAEFIK_DOMAINS_REDIRECT_TO=${TRAEFIK_DOMAINS_REDIRECT_TO}

services/vendors/docker-compose.yml.j2

@@ -5,7 +5,7 @@ services:
     image: ${VENDOR_MANUAL_IMAGE}
     init: true
 {%- raw %}
-    hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
+    hostname: "v-manual-{{.Node.Hostname}}-{{.Task.Slot}}"
 {%- endraw %}
     deploy:
       replicas: ${VENDOR_MANUAL_REPLICAS}
@@ -30,11 +30,45 @@ services:
         - traefik.http.services.vendor_manual.loadbalancer.server.port=${VENDOR_MANUAL_PORT}
         - traefik.http.routers.vendor_manual.entrypoints=https
         - traefik.http.routers.vendor_manual.tls=true
-        - traefik.http.routers.vendor_manual.rule={{ generate_vendors_manual_traefik_rule(VENDOR_MANUAL_PRODUCTS, VENDOR_MANUAL_SUBDOMAIN_PREFIX) }}
+        - traefik.http.routers.vendor_manual.rule={{ generate_vendors_traefik_rule(VENDOR_MANUAL_PRODUCTS, VENDOR_MANUAL_SUBDOMAIN_PREFIX) }}
         - traefik.http.routers.vendor_manual.middlewares=ops_gzip@swarm, authenticated_platform_user@swarm
     networks:
       - public
-
+  chat:
+    image: ${VENDOR_CHATBOT_IMAGE}
+    init: true
+{%- raw %}
+    hostname: "v-chat-{{.Node.Hostname}}-{{.Task.Slot}}"
+{%- endraw %}
+    deploy:
+      replicas: ${VENDOR_CHATBOT_REPLICAS}
+      placement:
+        constraints:
+          - node.labels.simcore==true
+      resources:
+        limits:
+          cpus: "1.0"
+          memory: 2.5G
+        reservations:
+          cpus: "0.1"
+          memory: 512M
+      update_config:
+        parallelism: 1
+        order: start-first
+        failure_action: continue
+        delay: 10s
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=${PUBLIC_NETWORK}
+        - traefik.http.services.vendor_chat.loadbalancer.server.port=${VENDOR_CHATBOT_PORT}
+        - traefik.http.routers.vendor_chat.entrypoints=https
+        - traefik.http.routers.vendor_chat.tls=true
+        - traefik.http.routers.vendor_chat.rule={{ generate_vendors_traefik_rule(VENDOR_CHATBOT_PRODUCTS, VENDOR_CHATBOT_SUBDOMAIN_PREFIX) }}
+        - traefik.http.routers.vendor_chat.middlewares=authenticated_platform_user@swarm
+        - prometheus-job=vendor-chat
+        - prometheus-port=8000
+    networks:
+      - public
 networks:
   public:
     external: true

services/vendors/j2cli_customization.py

@@ -1,11 +1,9 @@
-def _generate_vendors_manual_traefik_rule(domains: str, subdomain_prefix: str) -> str:
+def _generate_vendors_traefik_rule(domains: str, subdomain_prefix: str) -> str:
     domain_list = domains.strip().strip(",").split(",")
     domains = [f"{subdomain_prefix}.{domain}" for domain in domain_list]
     return " || ".join(f"Host(`{d}`)" for d in domains)
 
 
 def j2_environment(env):
-    env.globals.update(
-        generate_vendors_manual_traefik_rule=_generate_vendors_manual_traefik_rule
-    )
+    env.globals.update(generate_vendors_traefik_rule=_generate_vendors_traefik_rule)
     return env

services/vendors/template.env

@@ -3,4 +3,10 @@ VENDOR_MANUAL_REPLICAS=${VENDOR_MANUAL_REPLICAS}
 VENDOR_MANUAL_SUBDOMAIN_PREFIX=${VENDOR_MANUAL_SUBDOMAIN_PREFIX}
 VENDOR_MANUAL_PRODUCTS=${VENDOR_MANUAL_PRODUCTS}
 VENDOR_MANUAL_PORT=${VENDOR_MANUAL_PORT}
+VENDOR_CHATBOT_IMAGE=${VENDOR_CHATBOT_IMAGE}
+VENDOR_CHATBOT_REPLICAS=${VENDOR_CHATBOT_REPLICAS}
+VENDOR_CHATBOT_SUBDOMAIN_PREFIX=${VENDOR_CHATBOT_SUBDOMAIN_PREFIX}
+VENDOR_CHATBOT_PRODUCTS=${VENDOR_CHATBOT_PRODUCTS}
+VENDOR_CHATBOT_PORT=${VENDOR_CHATBOT_PORT}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
+OTEL_EXPORTER_OTLP_ENDPOINT=${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}:${TRACING_OPENTELEMETRY_COLLECTOR_PORT}