Skip to content

Commit 8e9514a

Browse files
YuryHrytsukYuryHrytsuk
authored
master: switch to standalone (cluster) rabbit ⚠️ 🚨 (ITISFoundation#1212)
* master: switch to standalone (cluster) rabbit Related issue/s * ITISFoundation#1176 Related PR/s * ITISFoundation#1179 * configuration https://git.speag.com/oSparc/osparc-ops-deployment-configuration/-/merge_requests/1542 * Delete compose * fix pre-commit hook * fix pre-commit hook * Add missing public network * Expose rabbit outside docker swarm * Use client ip private ips as a rule * Remove rabbit network from simcore * Fix haproxy timeout issues * rabbit lb: add placement constraints & preferences
1 parent 67a3919 commit 8e9514a

File tree

8 files changed

+83
-54
lines changed

8 files changed

+83
-54
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ repos:
122122
always_run: true
123123
language: script
124124
files: '^(.*\/Makefile.*)|(.*\.deploy_everything_locally.bash)|(.*\/services/.*\/.*\.((sh)|(bash)))$'
125+
require_serial: true
125126
- id: helm-update-dependencies
126127
name: Helm Dependency Update
127128
description: Make sure all Chart.lock files are up-to-date

‎charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,12 @@ env:
120120
value: {{ requiredEnv "RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD" }}
121121
sensitive: true
122122
- name: RABBIT_HOST
123-
value: {{ requiredEnv "RABBIT_EXTERNAL_HOST" }}
123+
value: {{ requiredEnv "RABBIT_HOST" }}
124124
- name: RABBIT_PASSWORD
125125
value: {{ requiredEnv "RABBIT_PASSWORD" }}
126126
sensitive: true
127127
- name: RABBIT_PORT
128-
value: {{ requiredEnv "RABBIT_EXTERNAL_PORT" }}
128+
value: {{ requiredEnv "RABBIT_PORT" }}
129129
- name: RABBIT_SECURE
130130
value: {{ requiredEnv "RABBIT_SECURE" }}
131131
- name: RABBIT_USER

‎services/rabbit/configs/haproxy.cfg.j2‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@ defaults
2525
option tcplog
2626

2727
timeout connect 5s
28-
timeout client 30s
29-
timeout server 30s
28+
# must be kept bigger than rabbitmq heartbeat (default 60) https://www.rabbitmq.com/docs/heartbeats
29+
timeout client 120s
30+
timeout server 120s
3031

3132
frontend rabbit
3233
bind *:{{ RABBIT_PORT }}

‎services/rabbit/docker-compose.loadbalancer.yml.j2‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ services:
22
loadbalancer:
33
image: haproxy:3.2
44
deploy:
5+
placement:
6+
constraints:
7+
- node.labels.ops == true
8+
preferences:
9+
- spread: node.id
510
update_config:
611
order: start-first
712
parallelism: 1
@@ -23,6 +28,21 @@ services:
2328
reservations:
2429
cpus: "0.1"
2530
memory: "128M"
31+
labels:
32+
- traefik.enable=true
33+
- traefik.swarm.network=${PUBLIC_NETWORK}
34+
- traefik.http.services.rabbit_dashboard.loadbalancer.server.port=${RABBIT_MANAGEMENT_PORT}
35+
- traefik.http.routers.rabbit_dashboard.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/rabbit`)
36+
- traefik.http.routers.rabbit_dashboard.entrypoints=https
37+
- traefik.http.routers.rabbit_dashboard.tls=true
38+
- traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.regex=^/rabbit/(.*)$$
39+
- traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.replacement=/$${1}
40+
- traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm
41+
- traefik.tcp.routers.rabbit.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)
42+
- traefik.tcp.routers.rabbit.entrypoints=rabbitmq
43+
- traefik.tcp.routers.rabbit.tls=false
44+
- traefik.tcp.routers.rabbit.service=rabbit
45+
- traefik.tcp.services.rabbit.loadbalancer.server.port=${RABBIT_PORT}
2646
healthcheck: # https://stackoverflow.com/a/76513320/12124525
2747
test: bash -c 'echo "" > /dev/tcp/127.0.0.1/32087 || exit 1'
2848
start_period: 5s
@@ -31,6 +51,7 @@ services:
3151
interval: 10s
3252
networks:
3353
- rabbit
54+
- public
3455
configs:
3556
- source: haproxy.cfg
3657
target: /usr/local/etc/haproxy/haproxy.cfg
@@ -39,6 +60,9 @@ networks:
3960
rabbit:
4061
name: ${RABBIT_NETWORK}
4162
external: true
63+
public:
64+
name: ${PUBLIC_NETWORK}
65+
external: true
4266

4367
configs:
4468
haproxy.cfg:

‎services/rabbit/template.env‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT=${RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_
33

44
RABBIT_USER=${RABBIT_USER}
55
RABBIT_PASSWORD=${RABBIT_PASSWORD}
6+
RABBIT_HOST=${RABBIT_HOST}
67
RABBIT_PORT=${RABBIT_PORT}
78
RABBIT_MANAGEMENT_PORT=${RABBIT_MANAGEMENT_PORT}
89

@@ -11,3 +12,4 @@ RABBIT_LB_REPLICAS=${RABBIT_LB_REPLICAS}
1112
RABBIT_NETWORK=${RABBIT_NETWORK}
1213

1314
PUBLIC_NETWORK=${PUBLIC_NETWORK}
15+
MONITORING_DOMAIN=${MONITORING_DOMAIN}

‎services/simcore/docker-compose.deploy.master.yml‎

Lines changed: 46 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,60 @@
11
services:
2-
autoscaling:
3-
deploy:
4-
replicas: 0
52
agent:
63
volumes:
74
- /docker/volumes/:/docker/volumes/
5+
6+
autoscaling:
7+
deploy:
8+
replicas: 0
9+
10+
clusters-keeper:
11+
deploy:
12+
replicas: 0
13+
14+
dask-sidecar:
15+
environment:
16+
- SIDECAR_LOGLEVEL=INFO
17+
18+
payments:
19+
deploy:
20+
replicas: 1
21+
822
postgres:
923
labels:
1024
- "pgbackup.enable=true"
1125
deploy:
1226
placement:
1327
constraints:
1428
- node.labels.postgres==true
15-
dask-sidecar:
16-
environment:
17-
- SIDECAR_LOGLEVEL=INFO
29+
30+
wb-api-server:
31+
deploy:
32+
replicas: 3
33+
34+
webserver:
35+
deploy:
36+
replicas: 3
37+
38+
redis:
39+
networks:
40+
- public
41+
deploy:
42+
labels:
43+
- traefik.enable=true
44+
- io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}
45+
- traefik.swarm.network=${SWARM_STACK_NAME}_default
46+
- "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
47+
- traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis
48+
- traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false
49+
- traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis
50+
- traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT}
51+
52+
rabbit:
53+
# rabbit is already exposed via ops traefik
54+
# adding one more route to this configuration
55+
deploy:
56+
replicas: 0 # use standalone (cluster) rabbit stack
57+
1858
traefik:
1959
command:
2060
- "--api=true"
@@ -70,44 +110,3 @@ services:
70110
- traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.service=${SWARM_STACK_NAME}_redisRoute
71111
- traefik.tcp.services.${SWARM_STACK_NAME}_redisRoute.loadbalancer.server.port=${REDIS_EXTERNAL_PORT}
72112
- "traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.rule=HostSNI(`*`)"
73-
74-
clusters-keeper:
75-
deploy:
76-
replicas: 0
77-
78-
payments:
79-
deploy:
80-
replicas: 1
81-
82-
webserver:
83-
deploy:
84-
replicas: 3
85-
86-
wb-api-server:
87-
deploy:
88-
replicas: 3
89-
90-
redis:
91-
networks:
92-
- public
93-
deploy:
94-
labels:
95-
- traefik.enable=true
96-
- io.simcore.zone=${TRAEFIK_SIMCORE_ZONE}
97-
- traefik.swarm.network=${SWARM_STACK_NAME}_default
98-
- "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
99-
- traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis
100-
- traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false
101-
- traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis
102-
- traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT}
103-
104-
rabbit:
105-
# rabbit is already exposed via ops traefik
106-
# adding one more route to this configuration
107-
deploy:
108-
labels:
109-
- "traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)"
110-
- traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.entrypoints=rabbitmq
111-
- traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.tls=false
112-
- traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.service=${SWARM_STACK_NAME}_rabbitmq
113-
- traefik.tcp.services.${SWARM_STACK_NAME}_rabbitmq.loadbalancer.server.port=${RABBIT_PORT}

‎services/traefik/docker-compose.yml.j2‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ services:
2424
mode: host
2525
{% endif %}
2626
{% if OPS_TRAEFIK_EXPOSE_RABBITMQ|lower == "true" %}
27-
- target: 5672
28-
published: 5672
27+
- target: ${RABBIT_PORT}
28+
published: ${RABBIT_PORT}
2929
mode: host
3030
{% endif %}
3131
{% if OPS_TRAEFIK_EXPOSE_REDIS|lower == "true" %}

‎services/traefik/template.env‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ TRACING_OPENTELEMETRY_COLLECTOR_PORT=${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
4646
OPS_TRAEFIK_ACCESSLOG_ENABLED=${OPS_TRAEFIK_ACCESSLOG_ENABLED}
4747
OPS_TRAEFIK_REPLICAS=${OPS_TRAEFIK_REPLICAS}
4848

49+
RABBIT_PORT=${RABBIT_PORT}
50+
4951
################ TRAEFIK OPTIONS
5052
# Attention: Traefik normalizes the environment variable key-value pairs by lowercasing them.
5153
# Core Configuration
@@ -89,7 +91,7 @@ TRAEFIK_PING_ENTRYPOINT=ping
8991
TRAEFIK_PING=true
9092
TRAEFIK_ENTRYPOINTS_METRICS_ADDRESS=:8082
9193
TRAEFIK_ENTRYPOINTS_POSTGRES_ADDRESS=:5432
92-
TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:5672
94+
TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:${RABBIT_PORT}
9395
TRAEFIK_ENTRYPOINTS_REDIS_ADDRESS=:${REDIS_EXTERNAL_PORT}
9496
# Providers
9597
TRAEFIK_PROVIDERS_SWARM_ENDPOINT=unix:///var/run/docker.sock

0 commit comments

Comments
 (0)