diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96a21e018..ced525556 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -122,6 +122,7 @@ repos: always_run: true language: script files: '^(.*\/Makefile.*)|(.*\.deploy_everything_locally.bash)|(.*\/services/.*\/.*\.((sh)|(bash)))$' + require_serial: true - id: helm-update-dependencies name: Helm Dependency Update description: Make sure all Chart.lock files are up-to-date diff --git a/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl b/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl index dbd26cc9b..75db405e6 100644 --- a/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl +++ b/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl @@ -120,12 +120,12 @@ env: value: {{ requiredEnv "RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD" }} sensitive: true - name: RABBIT_HOST - value: {{ requiredEnv "RABBIT_EXTERNAL_HOST" }} + value: {{ requiredEnv "RABBIT_HOST" }} - name: RABBIT_PASSWORD value: {{ requiredEnv "RABBIT_PASSWORD" }} sensitive: true - name: RABBIT_PORT - value: {{ requiredEnv "RABBIT_EXTERNAL_PORT" }} + value: {{ requiredEnv "RABBIT_PORT" }} - name: RABBIT_SECURE value: {{ requiredEnv "RABBIT_SECURE" }} - name: RABBIT_USER diff --git a/services/rabbit/configs/haproxy.cfg.j2 b/services/rabbit/configs/haproxy.cfg.j2 index 1619e7ce1..40102e174 100644 --- a/services/rabbit/configs/haproxy.cfg.j2 +++ b/services/rabbit/configs/haproxy.cfg.j2 @@ -25,8 +25,9 @@ defaults option tcplog timeout connect 5s - timeout client 30s - timeout server 30s + # must be kept bigger than rabbitmq heartbeat (default 60) https://www.rabbitmq.com/docs/heartbeats + timeout client 120s + timeout server 120s frontend rabbit bind *:{{ RABBIT_PORT }} diff --git a/services/rabbit/docker-compose.loadbalancer.yml.j2 b/services/rabbit/docker-compose.loadbalancer.yml.j2 index 242a25851..942a73d84 100644 --- a/services/rabbit/docker-compose.loadbalancer.yml.j2 +++ b/services/rabbit/docker-compose.loadbalancer.yml.j2 @@ -2,6 +2,11 @@ services: loadbalancer: image: haproxy:3.2 deploy: + placement: + constraints: + - node.labels.ops == true + preferences: + - spread: node.id update_config: order: start-first parallelism: 1 @@ -23,6 +28,21 @@ services: reservations: cpus: "0.1" memory: "128M" + labels: + - traefik.enable=true + - traefik.swarm.network=${PUBLIC_NETWORK} + - traefik.http.services.rabbit_dashboard.loadbalancer.server.port=${RABBIT_MANAGEMENT_PORT} + - traefik.http.routers.rabbit_dashboard.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/rabbit`) + - traefik.http.routers.rabbit_dashboard.entrypoints=https + - traefik.http.routers.rabbit_dashboard.tls=true + - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.regex=^/rabbit/(.*)$$ + - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.replacement=/$${1} + - traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm + - traefik.tcp.routers.rabbit.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`) + - traefik.tcp.routers.rabbit.entrypoints=rabbitmq + - traefik.tcp.routers.rabbit.tls=false + - traefik.tcp.routers.rabbit.service=rabbit + - traefik.tcp.services.rabbit.loadbalancer.server.port=${RABBIT_PORT} healthcheck: # https://stackoverflow.com/a/76513320/12124525 test: bash -c 'echo "" > /dev/tcp/127.0.0.1/32087 || exit 1' start_period: 5s @@ -31,6 +51,7 @@ services: interval: 10s networks: - rabbit + - public configs: - source: haproxy.cfg target: /usr/local/etc/haproxy/haproxy.cfg @@ -39,6 +60,9 @@ networks: rabbit: name: ${RABBIT_NETWORK} external: true + public: + name: ${PUBLIC_NETWORK} + external: true configs: haproxy.cfg: diff --git a/services/rabbit/template.env b/services/rabbit/template.env index 370ae73fa..45093877a 100644 --- a/services/rabbit/template.env +++ b/services/rabbit/template.env @@ -3,6 +3,7 @@ RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT=${RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_ RABBIT_USER=${RABBIT_USER} RABBIT_PASSWORD=${RABBIT_PASSWORD} +RABBIT_HOST=${RABBIT_HOST} RABBIT_PORT=${RABBIT_PORT} RABBIT_MANAGEMENT_PORT=${RABBIT_MANAGEMENT_PORT} @@ -11,3 +12,4 @@ RABBIT_LB_REPLICAS=${RABBIT_LB_REPLICAS} RABBIT_NETWORK=${RABBIT_NETWORK} PUBLIC_NETWORK=${PUBLIC_NETWORK} +MONITORING_DOMAIN=${MONITORING_DOMAIN} diff --git a/services/simcore/docker-compose.deploy.master.yml b/services/simcore/docker-compose.deploy.master.yml index 99dd7516d..57f8250a2 100644 --- a/services/simcore/docker-compose.deploy.master.yml +++ b/services/simcore/docker-compose.deploy.master.yml @@ -1,10 +1,24 @@ services: - autoscaling: - deploy: - replicas: 0 agent: volumes: - /docker/volumes/:/docker/volumes/ + + autoscaling: + deploy: + replicas: 0 + + clusters-keeper: + deploy: + replicas: 0 + + dask-sidecar: + environment: + - SIDECAR_LOGLEVEL=INFO + + payments: + deploy: + replicas: 1 + postgres: labels: - "pgbackup.enable=true" @@ -12,9 +26,35 @@ services: placement: constraints: - node.labels.postgres==true - dask-sidecar: - environment: - - SIDECAR_LOGLEVEL=INFO + + wb-api-server: + deploy: + replicas: 3 + + webserver: + deploy: + replicas: 3 + + redis: + networks: + - public + deploy: + labels: + - traefik.enable=true + - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} + - traefik.swarm.network=${SWARM_STACK_NAME}_default + - "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)" + - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis + - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false + - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis + - traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT} + + rabbit: + # rabbit is already exposed via ops traefik + # adding one more route to this configuration + deploy: + replicas: 0 # use standalone (cluster) rabbit stack + traefik: command: - "--api=true" @@ -70,44 +110,3 @@ services: - traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.service=${SWARM_STACK_NAME}_redisRoute - traefik.tcp.services.${SWARM_STACK_NAME}_redisRoute.loadbalancer.server.port=${REDIS_EXTERNAL_PORT} - "traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.rule=HostSNI(`*`)" - - clusters-keeper: - deploy: - replicas: 0 - - payments: - deploy: - replicas: 1 - - webserver: - deploy: - replicas: 3 - - wb-api-server: - deploy: - replicas: 3 - - redis: - networks: - - public - deploy: - labels: - - traefik.enable=true - - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - - traefik.swarm.network=${SWARM_STACK_NAME}_default - - "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)" - - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis - - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false - - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis - - traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT} - - rabbit: - # rabbit is already exposed via ops traefik - # adding one more route to this configuration - deploy: - labels: - - "traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)" - - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.entrypoints=rabbitmq - - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.tls=false - - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.service=${SWARM_STACK_NAME}_rabbitmq - - traefik.tcp.services.${SWARM_STACK_NAME}_rabbitmq.loadbalancer.server.port=${RABBIT_PORT} diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2 index b21faaca8..0f2ae13d0 100644 --- a/services/traefik/docker-compose.yml.j2 +++ b/services/traefik/docker-compose.yml.j2 @@ -24,8 +24,8 @@ services: mode: host {% endif %} {% if OPS_TRAEFIK_EXPOSE_RABBITMQ|lower == "true" %} - - target: 5672 - published: 5672 + - target: ${RABBIT_PORT} + published: ${RABBIT_PORT} mode: host {% endif %} {% if OPS_TRAEFIK_EXPOSE_REDIS|lower == "true" %} diff --git a/services/traefik/template.env b/services/traefik/template.env index c05daf02a..80a622627 100644 --- a/services/traefik/template.env +++ b/services/traefik/template.env @@ -46,6 +46,8 @@ TRACING_OPENTELEMETRY_COLLECTOR_PORT=${TRACING_OPENTELEMETRY_COLLECTOR_PORT} OPS_TRAEFIK_ACCESSLOG_ENABLED=${OPS_TRAEFIK_ACCESSLOG_ENABLED} OPS_TRAEFIK_REPLICAS=${OPS_TRAEFIK_REPLICAS} +RABBIT_PORT=${RABBIT_PORT} + ################ TRAEFIK OPTIONS # Attention: Traefik normalizes the environment variable key-value pairs by lowercasing them. # Core Configuration @@ -89,7 +91,7 @@ TRAEFIK_PING_ENTRYPOINT=ping TRAEFIK_PING=true TRAEFIK_ENTRYPOINTS_METRICS_ADDRESS=:8082 TRAEFIK_ENTRYPOINTS_POSTGRES_ADDRESS=:5432 -TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:5672 +TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:${RABBIT_PORT} TRAEFIK_ENTRYPOINTS_REDIS_ADDRESS=:${REDIS_EXTERNAL_PORT} # Providers TRAEFIK_PROVIDERS_SWARM_ENDPOINT=unix:///var/run/docker.sock