Add fallback simcore traefik routes (#950)

YuryHrytsuk · web-flow · commit 7a94f0dc2c1f · 2025-06-25T11:40:31.000+02:00
* Draft files

* update

* minor fixes

* Add traefik placeholder service and fallback routers

Traefik placeholder will be used to define low priority traeifk
configuration that will kick in once main docker serviec is not heathy
(its configuration is removed)

* Remove nginx config

This shall be done in separate PR. First we introduce proper 503 status
codes and then we sync about serving some content

* Remove nginx config leftovers

Check first commit for explanations

* Remove stale comments

* Invitations: add traefik healthcheck

* ops traefik placeholder: add healthcheck

* Healthcheck retries 3

* Define replicas via ENV
diff --git a/services/simcore/.gitignore b/services/simcore/.gitignore
@@ -1,5 +1,6 @@
 .env
 docker-compose.deploy.yml
+docker-compose.yml
 dask-sidecar/**
 assets/
 docker-compose.yml
diff --git a/services/simcore/docker-compose.deploy.local.yml b/services/simcore/docker-compose.deploy.local.yml
@@ -109,9 +109,7 @@ services:
         - traefik.tcp.services.${SWARM_STACK_NAME}_postgresRoute.loadbalancer.server.port=5432
         - "traefik.tcp.routers.${SWARM_STACK_NAME}_postgresRoute.rule=ClientIP(`195.176.8.0/24`) || ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
       replicas: 1
-  traefik_api:
-    deploy:
-      replicas: 1
+
   webserver:
     deploy:
       replicas: 1
diff --git a/services/simcore/docker-compose.yml.j2 b/services/simcore/docker-compose.yml.j2
@@ -186,9 +186,13 @@ services:
         # internal traefik
         - traefik.enable=true
         - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}
+        # NOTE: keep in sync with fallback router (rule and entrypoint)
         - traefik.http.routers.${SWARM_STACK_NAME}_invitations.rule=(${DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS})
         - traefik.http.routers.${SWARM_STACK_NAME}_invitations.entrypoints=http
         - traefik.http.services.${SWARM_STACK_NAME}_invitations.loadbalancer.server.port=8000
+        - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.path=/
+        - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.interval=2000ms
+        - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.timeout=1000ms
         - traefik.http.routers.${SWARM_STACK_NAME}_invitations_swagger.rule=(${DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS}) && PathPrefix(`/dev/doc`)
         - traefik.http.routers.${SWARM_STACK_NAME}_invitations_swagger.entrypoints=http
         - traefik.http.middlewares.${SWARM_STACK_NAME_NO_HYPHEN}_invitations_swagger_auth.basicauth.users=${TRAEFIK_USER}:${TRAEFIK_PASSWORD}
@@ -270,7 +274,6 @@ services:
         <<: *webserver_resources
     extra_hosts: []
 
-
   wb-db-event-listener:
     hostname: "{% raw %}{{.Service.Name}}{% endraw %}"
     environment:
@@ -770,9 +773,11 @@ services:
         - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http.tls=true
         - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http.middlewares=ops_gzip@swarm, ops_sslheader@swarm, ops_ratelimit@swarm
         - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http.service=${SWARM_STACK_NAME}_simcore_http
+        # Note: keep in sync with fallback router (rule and entrypoint)
         - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http.rule=((${DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL}) && PathPrefix(`/`)) || ( (PathPrefix(`/dashboard`) || PathPrefix(`/api`) || PathPrefix(`/doc`) ) && Host(`traefikdashboard.${MACHINE_FQDN}`))
         - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http.priority=3
         # oSparc publicAPI
+        # Note: keep in sync with fallback router (rule and entrypoint)
         - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api.rule=(${DEPLOYMENT_API_DOMAIN_CAPTURE_TRAEFIK_RULE}) && PathPrefix(`/`)
         - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api.entrypoints=https
         - traefik.http.services.${SWARM_STACK_NAME}_simcore_api.loadbalancer.server.port=10081
@@ -813,31 +818,103 @@ services:
         reservations:
           memory: 128M
           cpus: '0.1'
-  traefik_api:
-    # NOTE: this is a trick to allow to access the internal traefik REST API
-    # A comment
-    # list router like so: curl https://domain/api/http/routers | jq
+
+  # use to define fallback routes for simcore traefik
+  # if docker healthcheck fails, simcore traefik configurarion is
+  # removed from ops traeifik https://github.com/traefik/traefik/issues/7842
+  #
+  # use fallback routes to return proper 503 (instead of 404)
+  # this service must be running at all times
+  ops-traefik-configuration-placeholder:
     image: busybox:1.35.0
-    command: sleep 900000d
+    command: sleep infinity
+    networks:
+      - public
+    deploy:
+      replicas: ${OPS_TRAEFIK_CONFIGURATION_PLACEHOLDER_REPLICAS}
+      placement:
+        constraints:
+          - node.labels.traefik==true
+      resources:
+        limits:
+          memory: 16M
+          cpus: '0.1'
+        reservations:
+          memory: 8M
+          cpus: '0.1'
+
+      labels:
+        # external traefik
+        - traefik.enable=true
+
+        ### oSparc web (fallback low priority rule)
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http_fallback.entrypoints=https
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http_fallback.tls=true
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http_fallback.middlewares=ops_gzip@swarm, ops_sslheader@swarm, ops_ratelimit@swarm
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http_fallback.service=${SWARM_STACK_NAME}_simcore_http_fallback
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http_fallback.rule=((${DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL}) && PathPrefix(`/`)) || ( (PathPrefix(`/dashboard`) || PathPrefix(`/api`) || PathPrefix(`/doc`) ) && Host(`traefikdashboard.${MACHINE_FQDN}`))
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_http_fallback.priority=1
+        # always return 503
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_http_fallback.loadbalancer.server.port=0
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_http_fallback.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_http_fallback.loadbalancer.healthcheck.interval=10s
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_http_fallback.loadbalancer.healthcheck.timeout=1ms
+
+        ### oSparc publicAPI (fallback low priority rule)
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api_fallback.rule=(${DEPLOYMENT_API_DOMAIN_CAPTURE_TRAEFIK_RULE}) && PathPrefix(`/`)
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api_fallback.priority=1
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api_fallback.entrypoints=https
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api_fallback.tls=true
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api_fallback.middlewares=ops_gzip@swarm, ops_ratelimit@swarm
+        - traefik.http.routers.${SWARM_STACK_NAME}_simcore_api_fallback.service=${SWARM_STACK_NAME}_simcore_api_fallback
+        # always return 503
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_api_fallback.loadbalancer.server.port=0
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_api_fallback.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_api_fallback.loadbalancer.healthcheck.interval=10s
+        - traefik.http.services.${SWARM_STACK_NAME}_simcore_api_fallback.loadbalancer.healthcheck.timeout=1ms
+    healthcheck:
+      test: command -v sleep
+      interval: 10s
+      timeout: 1s
+      start_period: 1s
+      retries: 3
+
+  traefik-configuration-placeholder: # simcore traefik with `io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}` label
+    image: busybox:1.35.0
+    command: sleep infinity
     networks:
       - default
     deploy:
+      replicas: ${SIMCORE_TRAEFIK_CONFIGURATION_PLACEHOLDER_REPLICAS}
       labels:
         # route to internal traefik
+        - traefik.enable=true
         - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}
+
         # traefik UI
-        - traefik.enable=true
         - traefik.http.routers.${SWARM_STACK_NAME}_traefik_api.service=api@internal
         - traefik.http.routers.${SWARM_STACK_NAME}_traefik_api.rule=(PathPrefix(`/dashboard`) || PathPrefix(`/api`) ) && Host(`traefikdashboard.${MACHINE_FQDN}`)
         - traefik.http.routers.${SWARM_STACK_NAME}_traefik_api.entrypoints=http
-        - traefik.http.routers.${SWARM_STACK_NAME}_traefik_api.priority=6
         - traefik.http.routers.${SWARM_STACK_NAME}_traefik_api.middlewares=${SWARM_STACK_NAME}_auth@swarm, ${SWARM_STACK_NAME}_whitelist_ips@swarm
         - traefik.http.services.${SWARM_STACK_NAME}_traefik_api.loadbalancer.server.port=8080
+
         # Middlewares
         # basic authentication
         - traefik.http.middlewares.${SWARM_STACK_NAME}_auth.basicauth.users=${TRAEFIK_USER}:${TRAEFIK_PASSWORD}
         # OPS IP Whitelist
         - traefik.http.middlewares.${SWARM_STACK_NAME}_whitelist_ips.ipallowlist.sourcerange=${TRAEFIK_IPWHITELIST_SOURCERANGE}
+
+        ### Fallback for invitations
+        - traefik.http.routers.${SWARM_STACK_NAME}_invitations_fallback.rule=${DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS}
+        - traefik.http.routers.${SWARM_STACK_NAME}_invitations_fallback.service=${SWARM_STACK_NAME}_invitations_fallback
+        - traefik.http.routers.${SWARM_STACK_NAME}_invitations_fallback.entrypoints=http
+        - traefik.http.routers.${SWARM_STACK_NAME}_invitations_fallback.priority=1
+        # always fail and return 503 via unhealthy loadbalancer healthcheck
+        - traefik.http.services.${SWARM_STACK_NAME}_invitations_fallback.loadbalancer.server.port=0
+        - traefik.http.services.${SWARM_STACK_NAME}_invitations_fallback.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503
+        - traefik.http.services.${SWARM_STACK_NAME}_invitations_fallback.loadbalancer.healthcheck.interval=10s
+        - traefik.http.services.${SWARM_STACK_NAME}_invitations_fallback.loadbalancer.healthcheck.timeout=1ms
+
       update_config:
         parallelism: 2
         order: start-first
@@ -858,6 +935,7 @@ services:
         reservations:
           memory: 8M
           cpus: '0.1'
+
   whoami:
     image: "containous/whoami:v1.5.0"
     networks:
@@ -896,6 +974,7 @@ services:
         reservations:
           memory: 8M
           cpus: '0.1'
+
   payments:
     deploy:
       placement:
diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2
@@ -194,6 +194,7 @@ services:
     networks:
       public: null
       monitored: null
+
   whoami:
     image: "containous/whoami"
     deploy:
@@ -220,15 +221,15 @@ services:
     networks:
       - public
 
+configs:
+  traefik_dynamic_config.yml:
+    name: {{ STACK_NAME }}_traefik_dynamic_config_{{ "./traefik_dynamic_config.yml" | sha256file | substring(0,10) }}
+    file: ./traefik_dynamic_config.yml
+
 networks:
   public:
     external: true
     name: ${PUBLIC_NETWORK}
   monitored:
     name: ${MONITORED_NETWORK}
     external: true
-
-configs:
-  traefik_dynamic_config.yml:
-    name: ${STACK_NAME}_traefik_dynamic_config_{{ "./traefik_dynamic_config.yml" | sha256file | substring(0,10) }}
-    file: ./traefik_dynamic_config.yml
