master: switch to standalone (cluster) rabbit ⚠️ 🚨

.pre-commit-config.yaml

@@ -122,6 +122,7 @@ repos:
         always_run: true
         language: script
         files: '^(.*\/Makefile.*)|(.*\.deploy_everything_locally.bash)|(.*\/services/.*\/.*\.((sh)|(bash)))$'
+        require_serial: true
       - id: helm-update-dependencies
         name: Helm Dependency Update
         description: Make sure all Chart.lock files are up-to-date

charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl

@@ -120,12 +120,12 @@ env:
     value: {{ requiredEnv "RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD" }}
     sensitive: true
   - name: RABBIT_HOST
-    value: {{ requiredEnv "RABBIT_EXTERNAL_HOST" }}
+    value: {{ requiredEnv "RABBIT_HOST" }}
   - name: RABBIT_PASSWORD
     value: {{ requiredEnv "RABBIT_PASSWORD" }}
     sensitive: true
   - name: RABBIT_PORT
-    value: {{ requiredEnv "RABBIT_EXTERNAL_PORT" }}
+    value: {{ requiredEnv "RABBIT_PORT" }}
   - name: RABBIT_SECURE
     value: {{ requiredEnv "RABBIT_SECURE" }}
   - name: RABBIT_USER

services/rabbit/configs/haproxy.cfg.j2

@@ -25,8 +25,9 @@ defaults
     option tcplog
 
     timeout connect 5s
-    timeout client 30s
-    timeout server 30s
+    # must be kept bigger than rabbitmq heartbeat (default 60) https://www.rabbitmq.com/docs/heartbeats
+    timeout client 120s
+    timeout server 120s
 
 frontend rabbit
     bind *:{{ RABBIT_PORT }}

services/rabbit/docker-compose.loadbalancer.yml.j2

@@ -2,6 +2,11 @@ services:
   loadbalancer:
     image: haproxy:3.2
     deploy:
+      placement:
+        constraints:
+          - node.labels.ops == true
+        preferences:
+          - spread: node.id
       update_config:
         order: start-first
         parallelism: 1
@@ -23,6 +28,21 @@ services:
         reservations:
           cpus: "0.1"
           memory: "128M"
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=${PUBLIC_NETWORK}
+        - traefik.http.services.rabbit_dashboard.loadbalancer.server.port=${RABBIT_MANAGEMENT_PORT}
+        - traefik.http.routers.rabbit_dashboard.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/rabbit`)
+        - traefik.http.routers.rabbit_dashboard.entrypoints=https
+        - traefik.http.routers.rabbit_dashboard.tls=true
+        - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.regex=^/rabbit/(.*)$$
+        - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.replacement=/$${1}
+        - traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm
+        - traefik.tcp.routers.rabbit.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)
+        - traefik.tcp.routers.rabbit.entrypoints=rabbitmq
+        - traefik.tcp.routers.rabbit.tls=false
+        - traefik.tcp.routers.rabbit.service=rabbit
+        - traefik.tcp.services.rabbit.loadbalancer.server.port=${RABBIT_PORT}
     healthcheck: # https://stackoverflow.com/a/76513320/12124525
       test: bash -c 'echo "" > /dev/tcp/127.0.0.1/32087 || exit 1'
       start_period: 5s
@@ -31,6 +51,7 @@ services:
       interval: 10s
     networks:
       - rabbit
+      - public
     configs:
       - source: haproxy.cfg
         target: /usr/local/etc/haproxy/haproxy.cfg
@@ -39,6 +60,9 @@ networks:
   rabbit:
     name: ${RABBIT_NETWORK}
     external: true
+  public:
+    name: ${PUBLIC_NETWORK}
+    external: true
 
 configs:
   haproxy.cfg:

services/rabbit/template.env

@@ -3,6 +3,7 @@ RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT=${RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_
 
 RABBIT_USER=${RABBIT_USER}
 RABBIT_PASSWORD=${RABBIT_PASSWORD}
+RABBIT_HOST=${RABBIT_HOST}
 RABBIT_PORT=${RABBIT_PORT}
 RABBIT_MANAGEMENT_PORT=${RABBIT_MANAGEMENT_PORT}
 
@@ -11,3 +12,4 @@ RABBIT_LB_REPLICAS=${RABBIT_LB_REPLICAS}
 RABBIT_NETWORK=${RABBIT_NETWORK}
 
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
+MONITORING_DOMAIN=${MONITORING_DOMAIN}

services/simcore/docker-compose.deploy.master.yml

@@ -1,20 +1,60 @@
 services:
-  autoscaling:
-    deploy:
-      replicas: 0
   agent:
     volumes:
       - /docker/volumes/:/docker/volumes/
+
+  autoscaling:
+    deploy:
+      replicas: 0
+
+  clusters-keeper:
+    deploy:
+      replicas: 0
+
+  dask-sidecar:
+    environment:
+      - SIDECAR_LOGLEVEL=INFO
+
+  payments:
+    deploy:
+      replicas: 1
+
   postgres:
     labels:
       - "pgbackup.enable=true"
     deploy:
       placement:
         constraints:
           - node.labels.postgres==true
-  dask-sidecar:
-    environment:
-      - SIDECAR_LOGLEVEL=INFO
+
+  wb-api-server:
+    deploy:
+      replicas: 3
+
+  webserver:
+    deploy:
+      replicas: 3
+
+  redis:
+    networks:
+      - public
+    deploy:
+      labels:
+        - traefik.enable=true
+        - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}
+        - traefik.swarm.network=${SWARM_STACK_NAME}_default
+        - "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false
+        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis
+        - traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT}
+
+  rabbit:
+    # rabbit is already exposed via ops traefik
+    # adding one more route to this configuration
+    deploy:
+      replicas: 0  # use standalone (cluster) rabbit stack
+
   traefik:
     command:
       - "--api=true"
@@ -70,44 +110,3 @@ services:
         - traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.service=${SWARM_STACK_NAME}_redisRoute
         - traefik.tcp.services.${SWARM_STACK_NAME}_redisRoute.loadbalancer.server.port=${REDIS_EXTERNAL_PORT}
         - "traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.rule=HostSNI(`*`)"
-
-  clusters-keeper:
-    deploy:
-      replicas: 0
-
-  payments:
-    deploy:
-      replicas: 1
-
-  webserver:
-    deploy:
-      replicas: 3
-
-  wb-api-server:
-    deploy:
-      replicas: 3
-
-  redis:
-    networks:
-      - public
-    deploy:
-      labels:
-        - traefik.enable=true
-        - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}
-        - traefik.swarm.network=${SWARM_STACK_NAME}_default
-        - "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
-        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis
-        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false
-        - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis
-        - traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT}
-
-  rabbit:
-    # rabbit is already exposed via ops traefik
-    # adding one more route to this configuration
-    deploy:
-      labels:
-        - "traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
-        - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.entrypoints=rabbitmq
-        - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.tls=false
-        - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.service=${SWARM_STACK_NAME}_rabbitmq
-        - traefik.tcp.services.${SWARM_STACK_NAME}_rabbitmq.loadbalancer.server.port=${RABBIT_PORT}

services/traefik/docker-compose.yml.j2

@@ -24,8 +24,8 @@ services:
         mode: host
       {% endif %}
       {% if OPS_TRAEFIK_EXPOSE_RABBITMQ|lower == "true" %}
-      - target: 5672
-        published: 5672
+      - target: ${RABBIT_PORT}
+        published: ${RABBIT_PORT}
         mode: host
       {% endif %}
       {% if OPS_TRAEFIK_EXPOSE_REDIS|lower == "true" %}

services/traefik/template.env

@@ -46,6 +46,8 @@ TRACING_OPENTELEMETRY_COLLECTOR_PORT=${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
 OPS_TRAEFIK_ACCESSLOG_ENABLED=${OPS_TRAEFIK_ACCESSLOG_ENABLED}
 OPS_TRAEFIK_REPLICAS=${OPS_TRAEFIK_REPLICAS}
 
+RABBIT_PORT=${RABBIT_PORT}
+
 ################ TRAEFIK OPTIONS
 # Attention: Traefik normalizes the environment variable key-value pairs by lowercasing them.
 # Core Configuration
@@ -89,7 +91,7 @@ TRAEFIK_PING_ENTRYPOINT=ping
 TRAEFIK_PING=true
 TRAEFIK_ENTRYPOINTS_METRICS_ADDRESS=:8082
 TRAEFIK_ENTRYPOINTS_POSTGRES_ADDRESS=:5432
-TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:5672
+TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:${RABBIT_PORT}
 TRAEFIK_ENTRYPOINTS_REDIS_ADDRESS=:${REDIS_EXTERNAL_PORT}
 # Providers
 TRAEFIK_PROVIDERS_SWARM_ENDPOINT=unix:///var/run/docker.sock