Skip to content

Commit 42752c8

Browse files
dumbbelldumbbell
committed
cluster_minority_SUITE
1 parent 0cdc596 commit 42752c8

File tree

1 file changed

+155
-43
lines changed

1 file changed

+155
-43
lines changed

‎deps/rabbit/test/cluster_minority_SUITE.erl‎

Lines changed: 155 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@ groups() ->
4242
delete_policy,
4343
export_definitions
4444
]},
45-
{cluster_operation_add, [], [add_node]},
46-
{cluster_operation_remove, [], [remove_node]},
45+
{cluster_operation, [], [add_node_when_seed_node_is_leader,
46+
add_node_when_seed_node_is_follower,
47+
remove_node_when_seed_node_is_leader,
48+
remove_node_when_seed_node_is_follower]},
4749
{feature_flags, [], [enable_feature_flag]}
4850
].
4951

@@ -127,26 +129,44 @@ init_per_group(Group, Config0) when Group == client_operations;
127129
partition_5_node_cluster(Config1),
128130
Config1
129131
end;
130-
init_per_group(Group, Config0) ->
132+
init_per_group(_Group, Config0) ->
131133
Config = rabbit_ct_helpers:set_config(Config0, [{rmq_nodes_count, 5},
132-
{rmq_nodename_suffix, Group},
133134
{rmq_nodes_clustered, false},
134135
{tcp_ports_base},
135136
{net_ticktime, 5}]),
136137
Config1 = rabbit_ct_helpers:merge_app_env(
137-
Config, {rabbit, [{forced_feature_flags_on_init, []}]}),
138-
rabbit_ct_helpers:run_steps(Config1,
139-
rabbit_ct_broker_helpers:setup_steps() ++
140-
rabbit_ct_client_helpers:setup_steps()).
138+
Config, {rabbit, [{forced_feature_flags_on_init, []},
139+
{khepri_leader_wait_retry_timeout, 30000}]}),
140+
Config1.
141141

142142
end_per_group(_, Config) ->
143-
rabbit_ct_helpers:run_steps(Config,
144-
rabbit_ct_client_helpers:teardown_steps() ++
145-
rabbit_ct_broker_helpers:teardown_steps()).
146-
143+
Config.
144+
145+
init_per_testcase(Testcase, Config)
146+
when Testcase =:= add_node_when_seed_node_is_leader orelse
147+
Testcase =:= add_node_when_seed_node_is_follower orelse
148+
Testcase =:= remove_node_when_seed_node_is_leader orelse
149+
Testcase =:= remove_node_when_seed_node_is_follower ->
150+
rabbit_ct_helpers:testcase_started(Config, Testcase),
151+
Config1 = rabbit_ct_helpers:set_config(
152+
Config, [{rmq_nodename_suffix, Testcase}]),
153+
rabbit_ct_helpers:run_steps(
154+
Config1,
155+
rabbit_ct_broker_helpers:setup_steps() ++
156+
rabbit_ct_client_helpers:setup_steps());
147157
init_per_testcase(Testcase, Config) ->
148158
rabbit_ct_helpers:testcase_started(Config, Testcase).
149159

160+
end_per_testcase(Testcase, Config)
161+
when Testcase =:= add_node_when_seed_node_is_leader orelse
162+
Testcase =:= add_node_when_seed_node_is_follower orelse
163+
Testcase =:= remove_node_when_seed_node_is_leader orelse
164+
Testcase =:= remove_node_when_seed_node_is_follower ->
165+
rabbit_ct_helpers:run_steps(
166+
Config,
167+
rabbit_ct_client_helpers:teardown_steps() ++
168+
rabbit_ct_broker_helpers:teardown_steps()),
169+
rabbit_ct_helpers:testcase_finished(Config, Testcase);
150170
end_per_testcase(Testcase, Config) ->
151171
rabbit_ct_helpers:testcase_finished(Config, Testcase).
152172

@@ -271,53 +291,145 @@ set_policy(Config) ->
271291
delete_policy(Config) ->
272292
?assertError(_, rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"policy-to-delete">>)).
273293

274-
add_node(Config) ->
275-
[A, B, C, D, _E] = rabbit_ct_broker_helpers:get_node_configs(
294+
add_node_when_seed_node_is_leader(Config) ->
295+
[A, B, C, _D, E] = rabbit_ct_broker_helpers:get_node_configs(
276296
Config, nodename),
277297

278298
%% Three node cluster: A, B, C
279-
ok = rabbit_control_helper:command(stop_app, B),
280-
ok = rabbit_control_helper:command(join_cluster, B, [atom_to_list(A)], []),
281-
rabbit_control_helper:command(start_app, B),
299+
Cluster = [A, B, C],
300+
Config1 = rabbit_ct_broker_helpers:cluster_nodes(Config, Cluster),
282301

283-
ok = rabbit_control_helper:command(stop_app, C),
284-
ok = rabbit_control_helper:command(join_cluster, C, [atom_to_list(A)], []),
285-
rabbit_control_helper:command(start_app, C),
302+
AMember = {rabbit_khepri:get_store_id(), A},
303+
_ = ra:transfer_leadership(AMember, AMember),
286304

287305
%% Minority partition: A
306+
partition_3_node_cluster(Config1),
307+
308+
Pong = ra:ping(AMember, 10000),
309+
ct:pal("Member A state: ~0p", [Pong]),
310+
case Pong of
311+
{pong, State} when State =/= follower andalso State =/= candidate ->
312+
Ret = rabbit_control_helper:command(
313+
join_cluster, E, [atom_to_list(A)], []),
314+
?assertMatch({error, _, _}, Ret),
315+
{error, _, Msg} = Ret,
316+
?assertEqual(
317+
match,
318+
re:run(
319+
Msg, "\\{:rabbit, \\{\\{:error, :timeout\\}",
320+
[{capture, none}]));
321+
Ret ->
322+
ct:pal("A is not the expected leader: ~p", [Ret]),
323+
{skip, "Node A was not elected leader"}
324+
end.
325+
326+
add_node_when_seed_node_is_follower(Config) ->
327+
[A, B, C, _D, E] = rabbit_ct_broker_helpers:get_node_configs(
328+
Config, nodename),
329+
330+
%% Three node cluster: A, B, C
288331
Cluster = [A, B, C],
289-
partition_3_node_cluster(Config),
290-
291-
ok = rabbit_control_helper:command(stop_app, D),
292-
%% The command is appended to the log, but it will be dropped once the connectivity
293-
%% is restored
294-
?assertMatch(ok,
295-
rabbit_control_helper:command(join_cluster, D, [atom_to_list(A)], [])),
296-
timer:sleep(10000),
297-
join_3_node_cluster(Config),
298-
clustering_utils:assert_cluster_status({Cluster, Cluster}, Cluster).
299-
300-
remove_node(Config) ->
332+
Config1 = rabbit_ct_broker_helpers:cluster_nodes(Config, Cluster),
333+
334+
CMember = {rabbit_khepri:get_store_id(), C},
335+
ra:transfer_leadership(CMember, CMember),
336+
337+
%% Minority partition: A
338+
partition_3_node_cluster(Config1),
339+
340+
AMember = {rabbit_khepri:get_store_id(), A},
341+
Pong = ra:ping(AMember, 10000),
342+
ct:pal("Member A state: ~0p", [Pong]),
343+
case Pong of
344+
{pong, State}
345+
when State =:= follower orelse State =:= pre_vote ->
346+
Ret = rabbit_control_helper:command(
347+
join_cluster, E, [atom_to_list(A)], []),
348+
?assertMatch({error, _, _}, Ret),
349+
{error, _, Msg} = Ret,
350+
?assertEqual(
351+
match,
352+
re:run(
353+
Msg, "Khepri cluster could be in minority",
354+
[{capture, none}]));
355+
{pong, await_condition} ->
356+
Ret = rabbit_control_helper:command(
357+
join_cluster, E, [atom_to_list(A)], []),
358+
?assertMatch({error, _, _}, Ret),
359+
{error, _, Msg} = Ret,
360+
?assertEqual(
361+
match,
362+
re:run(
363+
Msg, "\\{:rabbit, \\{\\{:error, :timeout\\}",
364+
[{capture, none}]));
365+
Ret ->
366+
ct:pal("A is not the expected follower: ~p", [Ret]),
367+
{skip, "Node A was not a follower"}
368+
end.
369+
370+
remove_node_when_seed_node_is_leader(Config) ->
301371
[A, B, C | _] = rabbit_ct_broker_helpers:get_node_configs(
302372
Config, nodename),
303373

304374
%% Three node cluster: A, B, C
305-
ok = rabbit_control_helper:command(stop_app, B),
306-
ok = rabbit_control_helper:command(join_cluster, B, [atom_to_list(A)], []),
307-
rabbit_control_helper:command(start_app, B),
375+
Cluster = [A, B, C],
376+
Config1 = rabbit_ct_broker_helpers:cluster_nodes(Config, Cluster),
308377

309-
ok = rabbit_control_helper:command(stop_app, C),
310-
ok = rabbit_control_helper:command(join_cluster, C, [atom_to_list(A)], []),
311-
rabbit_control_helper:command(start_app, C),
378+
AMember = {rabbit_khepri:get_store_id(), A},
379+
ra:transfer_leadership(AMember, AMember),
312380

313381
%% Minority partition: A
314-
partition_3_node_cluster(Config),
382+
partition_3_node_cluster(Config1),
383+
384+
Pong = ra:ping(AMember, 10000),
385+
ct:pal("Member A state: ~0p", [Pong]),
386+
case Pong of
387+
{pong, leader} ->
388+
Ret = rabbit_control_helper:command(
389+
forget_cluster_node, A, [atom_to_list(B)], []),
390+
?assertEqual(ok, Ret);
391+
Ret ->
392+
ct:pal("A is not the expected leader: ~p", [Ret]),
393+
{skip, "Node A was not a leader"}
394+
end.
395+
396+
remove_node_when_seed_node_is_follower(Config) ->
397+
[A, B, C | _] = rabbit_ct_broker_helpers:get_node_configs(
398+
Config, nodename),
399+
400+
%% Three node cluster: A, B, C
315401
Cluster = [A, B, C],
402+
Config1 = rabbit_ct_broker_helpers:cluster_nodes(Config, Cluster),
403+
404+
CMember = {rabbit_khepri:get_store_id(), C},
405+
ra:transfer_leadership(CMember, CMember),
316406

317-
ok = rabbit_control_helper:command(forget_cluster_node, A, [atom_to_list(B)], []),
318-
timer:sleep(10000),
319-
join_3_node_cluster(Config),
320-
clustering_utils:assert_cluster_status({Cluster, Cluster}, Cluster).
407+
%% Minority partition: A
408+
partition_3_node_cluster(Config1),
409+
410+
AMember = {rabbit_khepri:get_store_id(), A},
411+
Pong = ra:ping(AMember, 10000),
412+
ct:pal("Member A state: ~0p", [Pong]),
413+
case Pong of
414+
{pong, State}
415+
when State =:= follower orelse State =:= pre_vote ->
416+
Ret = rabbit_control_helper:command(
417+
forget_cluster_node, A, [atom_to_list(B)], []),
418+
?assertMatch({error, _, _}, Ret),
419+
{error, _, Msg} = Ret,
420+
?assertEqual(
421+
match,
422+
re:run(
423+
Msg, "Khepri cluster could be in minority",
424+
[{capture, none}]));
425+
{pong, await_condition} ->
426+
Ret = rabbit_control_helper:command(
427+
forget_cluster_node, A, [atom_to_list(B)], []),
428+
?assertMatch(ok, Ret);
429+
Ret ->
430+
ct:pal("A is not the expected leader: ~p", [Ret]),
431+
{skip, "Node A was not a leader"}
432+
end.
321433

322434
enable_feature_flag(Config) ->
323435
[A | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),

0 commit comments

Comments
 (0)