Skip to content

Commit e16f1c1

Browse files
committed
Use 3-node cluster in test suite
Instead of 2-node cluster. This is a more realistic scenario.
1 parent dfcba03 commit e16f1c1

18 files changed

+272
-408
lines changed

.github/workflows/test-rabbitmq-alphas.yml

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,17 +44,19 @@ jobs:
4444
- name: Test with NIO
4545
run: |
4646
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
47-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
47+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
48+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
49+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4850
-Dmaven.javadoc.skip=true \
49-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
5051
--no-transfer-progress
5152
- name: Test with blocking IO
5253
run: |
5354
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
54-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
55-
-Dmaven.javadoc.skip=true \
56-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
57-
--no-transfer-progress
55+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
56+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
57+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
58+
-Dmaven.javadoc.skip=true \
59+
--no-transfer-progress
5860
- name: Stop broker A
5961
run: docker stop rabbitmq && docker rm rabbitmq
6062
- name: Stop broker B

.github/workflows/test-supported-java-versions-5.x.yml

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,17 +45,19 @@ jobs:
4545
run: |
4646
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
4747
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
48-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
48+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
49+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4950
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
5051
--no-transfer-progress \
5152
-Dnet.bytebuddy.experimental=true
5253
- name: Test with blocking IO
5354
run: |
5455
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
55-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
56-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
57-
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
58-
--no-transfer-progress \
59-
-Dnet.bytebuddy.experimental=true
56+
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
57+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
58+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
59+
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
60+
--no-transfer-progress \
61+
-Dnet.bytebuddy.experimental=true
6062
- name: Stop broker
6163
run: docker stop rabbitmq && docker rm rabbitmq

.github/workflows/test-supported-java-versions-main.yml

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,19 @@ jobs:
4343
run: |
4444
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
4545
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
46-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
46+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
47+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4748
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
4849
--no-transfer-progress \
4950
-Dnet.bytebuddy.experimental=true
5051
- name: Test with blocking IO
5152
run: |
5253
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
53-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
54-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
55-
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
56-
--no-transfer-progress \
57-
-Dnet.bytebuddy.experimental=true
54+
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
55+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
56+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
57+
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
58+
--no-transfer-progress \
59+
-Dnet.bytebuddy.experimental=true
5860
- name: Stop broker
5961
run: docker stop rabbitmq && docker rm rabbitmq

.github/workflows/test.yml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,21 @@ jobs:
4141
run: make deps
4242
- name: Test with NIO
4343
run: |
44-
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
45-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
44+
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
45+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
46+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
47+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4648
-Dmaven.javadoc.skip=true \
47-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
4849
--no-transfer-progress
4950
- name: Test with blocking IO
5051
run: |
5152
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
52-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
53-
-Dmaven.javadoc.skip=true \
54-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
55-
--no-transfer-progress
53+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
54+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
55+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
56+
-Dmaven.javadoc.skip=true \
57+
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
58+
--no-transfer-progress
5659
- name: Stop broker A
5760
run: docker stop rabbitmq && docker rm rabbitmq
5861
- name: Stop broker B
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config
2+
cluster_formation.classic_config.nodes.1 = rabbit@node0
3+
cluster_formation.classic_config.nodes.2 = rabbit@node1
4+
cluster_formation.classic_config.nodes.3 = rabbit@node2
5+
loopback_users = none
6+
7+
listeners.ssl.default = 5671
8+
9+
ssl_options.cacertfile = /etc/rabbitmq/tls/ca_certificate.pem
10+
ssl_options.certfile = /etc/rabbitmq/tls/server_acogoluegnes-nuc_certificate.pem
11+
ssl_options.keyfile = /etc/rabbitmq/tls/server_acogoluegnes-nuc_key.pem
12+
ssl_options.verify = verify_peer
13+
ssl_options.fail_if_no_peer_cert = false
14+
ssl_options.honor_cipher_order = true
15+
16+
auth_mechanisms.1 = PLAIN
17+
auth_mechanisms.2 = ANONYMOUS
18+
auth_mechanisms.3 = AMQPLAIN
19+
auth_mechanisms.4 = EXTERNAL
20+
auth_mechanisms.5 = RABBIT-CR-DEMO

ci/cluster/docker-compose.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
services:
2+
node0:
3+
environment:
4+
- RABBITMQ_ERLANG_COOKIE='secret_cookie'
5+
networks:
6+
- rabbitmq-cluster
7+
hostname: node0
8+
container_name: rabbitmq0
9+
image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
10+
pull_policy: always
11+
ports:
12+
- "5672:5672"
13+
- "5671:5671"
14+
tty: true
15+
volumes:
16+
- ./configuration/:/etc/rabbitmq/
17+
- ../../rabbitmq-configuration/tls/:/etc/rabbitmq/tls/
18+
node1:
19+
environment:
20+
- RABBITMQ_ERLANG_COOKIE='secret_cookie'
21+
networks:
22+
- rabbitmq-cluster
23+
hostname: node1
24+
container_name: rabbitmq1
25+
image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
26+
pull_policy: always
27+
ports:
28+
- "5673:5672"
29+
tty: true
30+
volumes:
31+
- ./configuration/:/etc/rabbitmq/
32+
- ../../rabbitmq-configuration/tls/:/etc/rabbitmq/tls/
33+
node2:
34+
environment:
35+
- RABBITMQ_ERLANG_COOKIE='secret_cookie'
36+
networks:
37+
- rabbitmq-cluster
38+
hostname: node2
39+
container_name: rabbitmq2
40+
image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
41+
pull_policy: always
42+
ports:
43+
- "5674:5672"
44+
tty: true
45+
volumes:
46+
- ./configuration/:/etc/rabbitmq/
47+
- ../../rabbitmq-configuration/tls/:/etc/rabbitmq/tls/
48+
networks:
49+
rabbitmq-cluster:

ci/start-broker.sh

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,33 @@ wait_for_message() {
1212
done
1313
}
1414

15-
make -C "${PWD}"/tls-gen/basic
15+
rm -rf rabbitmq-configuration
16+
mkdir -p rabbitmq-configuration/tls
1617

17-
mv tls-gen/basic/result/server_$(hostname -s)_certificate.pem tls-gen/basic/result/server_certificate.pem
18-
mv tls-gen/basic/result/server_$(hostname -s)_key.pem tls-gen/basic/result/server_key.pem
19-
mv tls-gen/basic/server_$(hostname -s) tls-gen/basic/server
20-
mv tls-gen/basic/client_$(hostname -s) tls-gen/basic/client
18+
make -C "${PWD}"/tls-gen/basic
2119

2220
rm -rf rabbitmq-configuration
2321
mkdir -p rabbitmq-configuration/tls
24-
25-
cp -R "${PWD}"/tls-gen/basic/* rabbitmq-configuration/tls
26-
chmod -R o+r rabbitmq-configuration/tls/*
27-
chmod -R g+r rabbitmq-configuration/tls/*
28-
./mvnw -q clean resources:testResources -Dtest-tls-certs.dir=/etc/rabbitmq/tls
29-
cp target/test-classes/[email protected] rabbitmq-configuration/rabbitmq.config
22+
cp -R "${PWD}"/tls-gen/basic/result/* rabbitmq-configuration/tls
23+
chmod o+r rabbitmq-configuration/tls/*
24+
chmod g+r rabbitmq-configuration/tls/*
25+
26+
echo "loopback_users = none
27+
28+
listeners.ssl.default = 5671
29+
30+
ssl_options.cacertfile = /etc/rabbitmq/tls/ca_certificate.pem
31+
ssl_options.certfile = /etc/rabbitmq/tls/server_$(hostname)_certificate.pem
32+
ssl_options.keyfile = /etc/rabbitmq/tls/server_$(hostname)_key.pem
33+
ssl_options.verify = verify_peer
34+
ssl_options.fail_if_no_peer_cert = false
35+
ssl_options.honor_cipher_order = true
36+
37+
auth_mechanisms.1 = PLAIN
38+
auth_mechanisms.2 = ANONYMOUS
39+
auth_mechanisms.3 = AMQPLAIN
40+
auth_mechanisms.4 = EXTERNAL
41+
auth_mechanisms.5 = RABBIT-CR-DEMO" >> rabbitmq-configuration/rabbitmq.conf
3042

3143
echo "Running RabbitMQ ${RABBITMQ_IMAGE}"
3244

ci/start-cluster.sh

Lines changed: 18 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,84 +1,38 @@
11
#!/usr/bin/env bash
22

3-
LOCAL_SCRIPT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
4-
5-
RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.0}
3+
export RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.0}
64

75
wait_for_message() {
86
while ! docker logs "$1" | grep -q "$2";
97
do
10-
sleep 5
11-
echo "Waiting 5 seconds for $1 to start..."
8+
sleep 2
9+
echo "Waiting 2 seconds for $1 to start..."
1210
done
1311
}
1412

15-
make -C "${PWD}"/tls-gen/basic
16-
17-
mv tls-gen/basic/result/server_$(hostname -s)_certificate.pem tls-gen/basic/result/server_certificate.pem
18-
mv tls-gen/basic/result/server_$(hostname -s)_key.pem tls-gen/basic/result/server_key.pem
19-
mv tls-gen/basic/server_$(hostname -s) tls-gen/basic/server
20-
mv tls-gen/basic/client_$(hostname -s) tls-gen/basic/client
21-
2213
rm -rf rabbitmq-configuration
2314
mkdir -p rabbitmq-configuration/tls
2415

25-
cp -R "${PWD}"/tls-gen/basic/* rabbitmq-configuration/tls
26-
chmod -R o+r rabbitmq-configuration/tls/*
27-
chmod -R g+r rabbitmq-configuration/tls/*
28-
./mvnw -q clean resources:testResources -Dtest-tls-certs.dir=/etc/rabbitmq/tls
29-
cp target/test-classes/[email protected] rabbitmq-configuration/[email protected]
30-
cp target/test-classes/[email protected] rabbitmq-configuration/[email protected]
31-
32-
echo "Running RabbitMQ ${RABBITMQ_IMAGE}"
33-
34-
docker rm -f rabbitmq 2>/dev/null || echo "rabbitmq was not running"
35-
docker run -d --name rabbitmq \
36-
--network host \
37-
-v "${PWD}"/rabbitmq-configuration:/etc/rabbitmq \
38-
--env RABBITMQ_CONFIG_FILE=/etc/rabbitmq/[email protected] \
39-
--env RABBITMQ_NODENAME=rabbit@$(hostname) \
40-
--env RABBITMQ_NODE_PORT=5672 \
41-
--env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-setcookie do-not-do-this-in-production" \
42-
"${RABBITMQ_IMAGE}"
43-
44-
# for CLI commands to share the same cookie
45-
docker exec rabbitmq bash -c "echo 'do-not-do-this-in-production' > /var/lib/rabbitmq/.erlang.cookie"
46-
docker exec rabbitmq chmod 0600 /var/lib/rabbitmq/.erlang.cookie
47-
48-
wait_for_message rabbitmq "completed with"
49-
50-
docker run -d --name hare \
51-
--network host \
52-
-v "${PWD}"/rabbitmq-configuration:/etc/rabbitmq \
53-
--env RABBITMQ_CONFIG_FILE=/etc/rabbitmq/[email protected] \
54-
--env RABBITMQ_NODENAME=hare@$(hostname) \
55-
--env RABBITMQ_NODE_PORT=5673 \
56-
--env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-setcookie do-not-do-this-in-production" \
57-
"${RABBITMQ_IMAGE}"
58-
59-
# for CLI commands to share the same cookie
60-
docker exec hare bash -c "echo 'do-not-do-this-in-production' > /var/lib/rabbitmq/.erlang.cookie"
61-
docker exec hare chmod 0600 /var/lib/rabbitmq/.erlang.cookie
16+
make -C "${PWD}"/tls-gen/basic
6217

63-
wait_for_message hare "completed with"
18+
rm -rf rabbitmq-configuration
19+
mkdir -p rabbitmq-configuration/tls
20+
cp -R "${PWD}"/tls-gen/basic/result/* rabbitmq-configuration/tls
21+
chmod o+r rabbitmq-configuration/tls/*
22+
chmod g+r rabbitmq-configuration/tls/*
6423

65-
docker exec hare rabbitmqctl --node hare@$(hostname) status
6624

67-
docker exec rabbitmq rabbitmq-diagnostics --node rabbit@$(hostname) is_running
68-
docker exec hare rabbitmq-diagnostics --node hare@$(hostname) is_running
25+
docker compose --file ci/cluster/docker-compose.yml down
26+
docker compose --file ci/cluster/docker-compose.yml up --detach
6927

70-
docker exec hare rabbitmqctl --node hare@$(hostname) stop_app
71-
docker exec hare rabbitmqctl --node hare@$(hostname) join_cluster rabbit@$(hostname)
72-
docker exec hare rabbitmqctl --node hare@$(hostname) start_app
28+
wait_for_message rabbitmq0 "completed with"
7329

74-
sleep 10
30+
docker exec rabbitmq0 rabbitmqctl await_online_nodes 3
7531

76-
docker exec hare rabbitmqctl --node hare@$(hostname) await_startup
32+
docker exec rabbitmq0 rabbitmqctl enable_feature_flag --opt-in khepri_db
33+
docker exec rabbitmq1 rabbitmqctl enable_feature_flag --opt-in khepri_db
34+
docker exec rabbitmq2 rabbitmqctl enable_feature_flag --opt-in khepri_db
7735

78-
docker exec hare rabbitmqctl --node hare@$(hostname) enable_feature_flag --opt-in khepri_db
79-
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) enable_feature_flag --opt-in khepri_db
36+
docker exec rabbitmq0 rabbitmqctl cluster_status
8037

81-
docker exec rabbitmq rabbitmq-diagnostics --node rabbit@$(hostname) erlang_version
82-
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) version
83-
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) status
84-
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) cluster_status
38+
docker compose --file ci/cluster/docker-compose.yml ps

0 commit comments

Comments
 (0)