Skip to content

Commit 4fdde13

Browse files
YuryHrytsukYuryHrytsuk
authored
Add (ha) rabbit cluster (but not use it) ⚠️ (#1179)
* add-ha-rabbit * Add ha rabbit * Document erlang cookie rotation * Add ha proxy * Further configuration * Document autoscaling (not supported) * More configurable parameters * minor improvements * Add resource limits/reservations * Add haproxy resources * Document side effect of haproxy round robin * Add healthcheck for haproxy * Update readme * Removing volumes * Robust volume clean up + haproxy extra configuration * Simplification * Add confirmation dialogue * Unification * Minor clean up * update gitignore * fixes after clean up * clean up * clean up * Deploy rabbit only if necessary * clean up * Document cluster update behaviour. Architecture must be changed * Switch from services to stacks * fixes * improvements * minor fixes * update * update * Improvements * improvements * improvements * fixes and improvements * remove leftovers * Improve doc * fixes * Improve README * remove lines * Clean up * Fix readme header * Improve node index validation * remove TODOs from compose file * remove unecessary headers
1 parent 70840bd commit 4fdde13

File tree

10 files changed

+447
-0
lines changed

10 files changed

+447
-0
lines changed

‎services/rabbit/.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
*.yml
2+
*.env
3+
!template*.env
4+
!erlang.cookie.secret.template
5+
rabbitmq.conf
6+
haproxy.cfg

‎services/rabbit/.operations.Makefile‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#
2+
# Variables
3+
#
4+
5+
LOAD_BALANCER_STACK_NAME := rabbit-loadbalancer
6+
7+
MAKEFLAGS += --no-print-directory
8+
9+
#
10+
# Helpers
11+
#
12+
13+
define create_node_stack_name
14+
rabbit-node0$(1)
15+
endef
16+
17+
validate-NODE_COUNT: guard-NODE_COUNT
18+
@if ! echo "$(NODE_COUNT)" | grep --quiet --extended-regexp '^[1-9]$$'; then \
19+
echo NODE_COUNT must be a positive single digit integer; \
20+
exit 1; \
21+
fi
22+
23+
validate-node-ix0%: .env
24+
@if ! echo "$*" | grep --quiet --extended-regexp '^[1-9]$$'; then \
25+
echo "Node index $* must be a positive single digit integer"; \
26+
exit 1; \
27+
fi
28+
29+
@set -o allexport; . $<; set +o allexport; \
30+
if [ "$*" -lt 1 ] || [ "$*" -gt "$$RABBIT_CLUSTER_NODE_COUNT" ]; then \
31+
echo "Node index $* is out of range 1..$$RABBIT_CLUSTER_NODE_COUNT"; \
32+
exit 1; \
33+
fi
34+
35+
#
36+
# Cluster level
37+
#
38+
39+
### Note: up operation is called by CI automatically
40+
### it must NOT deploy stacks if they are already running
41+
### to avoid breaking existing cluster (stopping all nodes at once)
42+
up: start-cluster
43+
44+
down: stop-cluster
45+
46+
start-cluster: start-all-nodes start-loadbalancer
47+
48+
update-cluster stop-cluster:
49+
@$(error This operation may break cluster. Check README for details.)
50+
51+
#
52+
# Load Balancer
53+
#
54+
55+
start-loadbalancer: .stack.loadbalancer.yml
56+
@docker stack deploy --with-registry-auth --prune --compose-file $< $(LOAD_BALANCER_STACK_NAME)
57+
58+
update-loadbalancer: start-loadbalancer
59+
60+
stop-loadbalancer:
61+
@docker stack rm $(LOAD_BALANCER_STACK_NAME)
62+
63+
#
64+
# Rabbit all Nodes together
65+
#
66+
67+
.start-all-nodes: validate-NODE_COUNT
68+
@i=1; \
69+
while [ $$i -le $(NODE_COUNT) ]; do \
70+
$(MAKE) start-node0$$i; \
71+
i=$$((i + 1)); \
72+
done
73+
74+
start-all-nodes: .env
75+
@source $<; \
76+
$(MAKE) .start-all-nodes NODE_COUNT=$$RABBIT_CLUSTER_NODE_COUNT
77+
78+
update-all-nodes:
79+
@$(error Updating all nodes at the same time may break the cluster \
80+
as it may restart (i.e. stop) all nodes at the same time. \
81+
Update one node at a time)
82+
83+
stop-all-nodes:
84+
@$(error Stopping all nodes at the same time breaks the cluster. \
85+
Update one node at a time. \
86+
Read more at https://groups.google.com/g/rabbitmq-users/c/owvanX2iSqA/m/ZAyRDhRfCQAJ)
87+
88+
#
89+
# Rabbit Node level
90+
#
91+
92+
start-node0%: validate-node-ix0% .stack.node0%.yml
93+
@STACK_NAME=$(call create_node_stack_name,$*); \
94+
if docker stack ls --format '{{.Name}}' | grep --silent "$$STACK_NAME"; then \
95+
echo "Rabbit Node $* is already running, skipping"; \
96+
else \
97+
echo "Starting Rabbit Node $* ..."; \
98+
docker stack deploy --with-registry-auth --prune --compose-file $(word 2,$^) $(call create_node_stack_name,$*); \
99+
fi
100+
101+
update-node0%: validate-node-ix0% .stack.node0%.yml
102+
@docker stack deploy --detach=false --with-registry-auth --prune --compose-file $(word 2,$^) $(call create_node_stack_name,$*)
103+
104+
stop-node0%: validate-node-ix0%
105+
@docker stack rm --detach=false $(call create_node_stack_name,$*)

‎services/rabbit/Makefile‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
2+
3+
include ${REPO_BASE_DIR}/scripts/common-services.Makefile
4+
# common-services.Makefile should be included first as common.Makefile
5+
# relies on STACK_NAME var which is defined in common-services.Makefile
6+
include ${REPO_BASE_DIR}/scripts/common.Makefile
7+
8+
#
9+
# Operations
10+
#
11+
12+
include ${REPO_BASE_DIR}/services/rabbit/.operations.Makefile
13+
14+
#
15+
# Docker compose files
16+
#
17+
18+
### Load Balancer
19+
docker-compose.loadbalancer.yml: docker-compose.loadbalancer.yml.j2 \
20+
.env \
21+
configs/rabbitmq.conf \
22+
configs/erlang.cookie.secret \
23+
configs/haproxy.cfg \
24+
venv \
25+
$(VENV_BIN)/j2
26+
@$(call jinja, $<, .env, $@)
27+
28+
.stack.loadbalancer.yml: docker-compose.loadbalancer.yml .env
29+
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
30+
31+
### Node
32+
33+
.PRECIOUS: node0%.env
34+
node0%.env: .env
35+
envsubst < $< > $@; \
36+
echo NODE_INDEX=$* >> $@
37+
38+
.PRECIOUS: docker-compose.node0%.yml
39+
docker-compose.node0%.yml: docker-compose.node0x.yml.j2 \
40+
node0%.env \
41+
configs/rabbitmq.conf \
42+
configs/erlang.cookie.secret \
43+
configs/haproxy.cfg \
44+
venv \
45+
$(VENV_BIN)/j2
46+
@$(call jinja, $<, node0$*.env, $@)
47+
48+
.PRECIOUS: .stack.node0%.yml
49+
.stack.node0%.yml: docker-compose.node0%.yml node0%.env
50+
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e node0$*.env $< > $@
51+
52+
#
53+
# Config / Secret files
54+
#
55+
56+
configs/erlang.cookie.secret: configs/erlang.cookie.secret.template .env
57+
@set -a; source .env; set +a; \
58+
envsubst < $< > $@
59+
60+
configs/rabbitmq.conf: configs/rabbitmq.conf.j2 .env venv
61+
# generate $@
62+
@$(call jinja, $<, .env, $@)
63+
64+
configs/haproxy.cfg: configs/haproxy.cfg.j2 .env venv
65+
# generate $@
66+
@$(call jinja, $<, .env, $@)

‎services/rabbit/README.md‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
## Starting a cluster
2+
3+
Make sure all nodes have joined the cluster before using it. Otherwise, number of replicas in quorum queues might be affected. Say, you have a cluster of 3 nodes. You connect to cluster before the 3rd node join it. Your quorum queue would end up with only 2 replicas and will be broken once, 1 node (of 2 nodes holding the replicas of the queue) goes down.
4+
5+
## Updating a cluster
6+
7+
Perform update one node at a time. Never update all nodes at the same time (this may break cluster)! Follow instructions from official documentation https://www.rabbitmq.com/docs/upgrade#rolling-upgrade.
8+
9+
## Graceful shutdown
10+
11+
Shutdown nodes one by one gracefully. Wait until the nodes is stopped and leaves the cluster. Then remove next node. When starting cluster, start nodes **in the reverse order**! For example, if you shutdown node01, then node02 and lastly node03, first start node03 then node02 and finally node01.
12+
13+
If all Nodes were shutdown simultaneously, then you will see mnesia tables errors in node's logs. Restarting node solves the issue. Documentation also mentions force_boot CLI command in this case (see https://www.rabbitmq.com/docs/man/rabbitmqctl.8#force_boot)
14+
15+
## How to add / remove nodes
16+
17+
The only supported way, is to completely shutdown the cluster (docker stack and most likely rabbit node volumes) and start brand new.
18+
19+
With manual effort this can be done on the running cluster, by adding 1 more rabbit node manually (as a separate docker stack or new service) and manually executing rabbitmqctl commands (some hints can be found here https://www.rabbitmq.com/docs/clustering#creating)
20+
21+
## Updating rabbitmq.conf / advanced.config (zero-downtime)
22+
23+
We do not support this automated (except starting from scratch with empty volumes). But manually this can be achieved in case needed. `rabbitmq.conf` and `advanced.config` changes take effect after a node restart. This can be performed with zero-downtime when RabbitMQ is clustered (have multiple nodes). This can be achieved by stopping and starting rabbitmq nodes one by one
24+
* `docker exec -it <container-id> bash`
25+
* (inside container) `rabbitmqctl stop_app` and wait some time until node is stopped (can be seen in management ui)
26+
* (inside container) `rabbitmqctl start_app`
27+
28+
Source: https://www.rabbitmq.com/docs/next/configure#config-changes-effects
29+
30+
## Enable node Maintenance mode
31+
32+
1. Get inside container's shell (`docker exec -it <container-id> bash`)
33+
2. (Inside container) execute `rabbitmq-upgrade drain`
34+
35+
Source: https://www.rabbitmq.com/docs/upgrade#maintenance-mode
36+
37+
## Troubleshooting
38+
mnesia errors after all rabbit nodes (docker services) restart:
39+
* https://stackoverflow.com/questions/60407082/rabbit-mq-error-while-waiting-for-mnesia-tables
40+
41+
official documentation mentioning restart scenarios
42+
* https://www.rabbitmq.com/docs/clustering#restarting-schema-sync
43+
44+
all (3) cluster nodes go down simultaneosuly, cluster is broken:
45+
* https://groups.google.com/g/rabbitmq-users/c/owvanX2iSqA
46+
47+
## Autoscaling
48+
49+
Not supported at the moment.

‎services/rabbit/configs/erlang.cookie.secret.template‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
${RABBIT_ERLANG_COOKIE}

‎services/rabbit/configs/haproxy.cfg.j2‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{% set NODE_IXS = range(1, (RABBIT_CLUSTER_NODE_COUNT | int) + 1) -%}
2+
3+
global
4+
log stdout format raw local0
5+
6+
# haproxy by default resolves server hostname only once
7+
# this breaks if container restarts. By using resolvers
8+
# we tell haproxy to re-resolve the hostname (so container
9+
# restarts are handled properly)
10+
resolvers dockerdns
11+
nameserver dns1 127.0.0.11:53
12+
resolve_retries 3
13+
timeout resolve 1s
14+
timeout retry 1s
15+
hold other 10s
16+
hold refused 10s
17+
hold nx 10s
18+
hold timeout 10s
19+
hold valid 10s
20+
hold obsolete 10s
21+
22+
defaults
23+
log global
24+
mode tcp
25+
option tcplog
26+
27+
timeout connect 5s
28+
timeout client 30s
29+
timeout server 30s
30+
31+
frontend rabbit
32+
bind *:{{ RABBIT_PORT }}
33+
default_backend rabbit_backends
34+
35+
frontend rabbit_dashboard
36+
bind *:{{ RABBIT_MANAGEMENT_PORT }}
37+
default_backend rabbit_dashboard_backends
38+
39+
frontend health
40+
mode http
41+
bind 127.0.0.1:32087
42+
http-request return status 200 if { src 127.0.0.0/8 }
43+
44+
backend rabbit_backends
45+
# side effect of roundrobin is connection should be evenly distributed
46+
# thus rabbit queue leader replica shall be also evenly distributed
47+
# (https://www.rabbitmq.com/docs/4.0/clustering#replica-placement)
48+
# if algorithm below is changed, consider adjusting rabbit configuration
49+
# as stated in documentation link above
50+
balance roundrobin
51+
52+
# init-addrs libc,none - start even if there aren’t any backend servers running
53+
{% for ix in NODE_IXS %}
54+
server rabbit0{{ ix }} rabbit-node0{{ ix }}_rabbit0{{ ix }}:{{ RABBIT_PORT }} check resolvers dockerdns init-addr libc,none inter 5s rise 2 fall 3 send-proxy
55+
{%- endfor %}
56+
57+
backend rabbit_dashboard_backends
58+
mode http
59+
balance roundrobin
60+
61+
{% for ix in NODE_IXS %}
62+
server rabbit0{{ ix }} rabbit-node0{{ ix }}_rabbit0{{ ix }}:{{ RABBIT_MANAGEMENT_PORT }} check resolvers dockerdns init-addr libc,none inter 5s rise 2 fall 3
63+
{%- endfor %}
64+
# keep new line in the end to avoid "Missing LF on last line" error

‎services/rabbit/configs/rabbitmq.conf.j2‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{% set NODE_IXS = range(1, (RABBIT_CLUSTER_NODE_COUNT | int) + 1) -%}
2+
3+
# https://www.rabbitmq.com/docs/cluster-formation#peer-discovery-configuring-mechanism
4+
cluster_formation.peer_discovery_backend = classic_config
5+
6+
{% for ix in NODE_IXS %}
7+
cluster_formation.classic_config.nodes.{{ ix }} = rabbit@rabbit-node0{{ ix }}_rabbit0{{ ix }}
8+
{%- endfor %}
9+
10+
## Sets the initial quorum queue replica count for newly declared quorum queues.
11+
## This value can be overridden using the 'x-quorum-initial-group-size' queue argument
12+
## at declaration time.
13+
# https://www.rabbitmq.com/docs/quorum-queues#quorum-requirements
14+
quorum_queue.initial_cluster_size = {{ RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT }}
15+
16+
# Extract proper client ip when behind a proxy (e.g. haproxy)
17+
# https://www.rabbitmq.com/docs/networking#proxy-protocol
18+
# WARNING: this forces clients to use a proxy (direct access to nodes does not work)
19+
proxy_protocol = true

‎services/rabbit/docker-compose.loadbalancer.yml.j2‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
services:
2+
loadbalancer:
3+
image: haproxy:3.2
4+
deploy:
5+
update_config:
6+
order: start-first
7+
parallelism: 1
8+
delay: 30s
9+
failure_action: rollback
10+
# https://discourse.haproxy.org/t/haproxy-high-availability-configuration/11983
11+
replicas: ${RABBIT_LB_REPLICAS}
12+
# necessary to preserve client ip
13+
# otherwise we see overlay rabbit network lb ip
14+
# (rabbitmq management dashboard connection section)
15+
endpoint_mode: dnsrr
16+
resources:
17+
limits:
18+
# https://help.hcl-software.com/digital-experience/dx-95-doc-archive/CF203/platform/kubernetes/haproxy-migration/haproxy-configuration.html
19+
cpus: "1"
20+
memory: "2G"
21+
# according to local observations and link below
22+
# https://github.com/haproxytech/helm-charts/blob/haproxy-1.24.0/haproxy/values.yaml#L403
23+
reservations:
24+
cpus: "0.1"
25+
memory: "128M"
26+
healthcheck: # https://stackoverflow.com/a/76513320/12124525
27+
test: bash -c 'echo "" > /dev/tcp/127.0.0.1/32087 || exit 1'
28+
start_period: 5s
29+
timeout: 2s
30+
retries: 2
31+
interval: 10s
32+
networks:
33+
- rabbit
34+
configs:
35+
- source: haproxy.cfg
36+
target: /usr/local/etc/haproxy/haproxy.cfg
37+
38+
networks:
39+
rabbit:
40+
name: ${RABBIT_NETWORK}
41+
external: true
42+
43+
configs:
44+
haproxy.cfg:
45+
file: ./configs/haproxy.cfg
46+
name: rabbit_haproxy_conf_{{ "./configs/haproxy.cfg" | sha256file | substring(0,10) }}

0 commit comments

Comments
 (0)