Merge pull request #8646 from rabbitmq/mergify/bp/v3.11.x/pr-8644

Test resilience (backport #8643) (backport #8644)

Author: michaelklishin

deps/rabbit/app.bzl

@@ -1086,7 +1086,7 @@ def test_suite_beam_files(name = "test_suite_beam_files"):
         outs = ["test/per_vhost_msg_store_SUITE.beam"],
         app_name = "rabbit",
         erlc_opts = "//:test_erlc_opts",
-        deps = ["//deps/amqp_client:erlang_app"],
+        deps = ["//deps/amqp_client:erlang_app", "//deps/rabbitmq_ct_helpers:erlang_app"],
     )
     erlang_bytecode(
         name = "per_vhost_queue_limit_SUITE_beam_files",

deps/rabbit/test/per_vhost_msg_store_SUITE.erl

@@ -9,6 +9,7 @@
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("amqp_client/include/amqp_client.hrl").
+-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl").
 
 -compile(export_all).
 
@@ -109,23 +110,16 @@ storage_deleted_on_vhost_delete(Config) ->
     Vhost1 = ?config(vhost1, Config),
     Channel1 = ?config(channel1, Config),
     Queue1 = declare_durable_queue(Channel1),
-    FolderSize = get_global_folder_size(Config),
+    FolderSize = get_folder_size(Vhost1, Config),
 
     publish_persistent_messages(index, Channel1, Queue1),
     publish_persistent_messages(store, Channel1, Queue1),
-    FolderSizeAfterPublish = get_global_folder_size(Config),
-
-    %% Total storage size increased
-    true = (FolderSize < FolderSizeAfterPublish),
+    ?awaitMatch(true, get_folder_size(Vhost1, Config) > FolderSize, 30000),
 
     ok = rabbit_ct_broker_helpers:delete_vhost(Config, Vhost1),
 
-    %% Total memory reduced
-    FolderSizeAfterDelete = get_global_folder_size(Config),
-    true = (FolderSizeAfterPublish > FolderSizeAfterDelete),
-
     %% There is no Vhost1 folder
-    0 = get_folder_size(Vhost1, Config).
+    ?awaitMatch(0, get_folder_size(Vhost1, Config), 30000).
 
 
 single_vhost_storage_delete_is_safe(Config) ->
@@ -192,10 +186,6 @@ folder_size(Dir) ->
     filelib:fold_files(Dir, ".*", true,
                        fun(F,Acc) -> filelib:file_size(F) + Acc end, 0).
 
-get_global_folder_size(Config) ->
-    BaseDir = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_mnesia, dir, []),
-    folder_size(BaseDir).
-
 vhost_dir(Vhost, Config) ->
     rabbit_ct_broker_helpers:rpc(Config, 0,
                                  rabbit_vhost, msg_store_dir_path, [Vhost]).

deps/rabbit/test/quorum_queue_SUITE.erl

@@ -2656,6 +2656,13 @@ message_ttl(Config) ->
     ok.
 
 message_ttl_policy(Config) ->
+    %% Using ttl is very difficul to guarantee 100% test rate success, unless
+    %% using really high ttl values. Previously, this test used 1s and 3s ttl,
+    %% but expected to see first the messages in the queue and then the messages
+    %% gone. A slow CI run, would fail the first assertion as the messages would
+    %% have been dropped when the message count was performed. It happened
+    %% from time to time making this test case flaky.
+
     [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 
     Ch = rabbit_ct_client_helpers:open_channel(Config, Server),
@@ -2670,21 +2677,21 @@ message_ttl_policy(Config) ->
     RaName = binary_to_atom(<<VHost/binary, "_", QQ/binary>>, utf8),
 
     QueryFun = fun rabbit_fifo:overview/1,
-    {ok, {_, Overview}, _} = rpc:call(Server, ra, local_query, [RaName, QueryFun]),
-    ?assertMatch(#{config := #{msg_ttl := 1000}}, Overview),
-    %% wait for policy?
+    ?awaitMatch({ok, {_, #{config := #{msg_ttl := 1000}}}, _},
+                rpc:call(Server, ra, local_query, [RaName, QueryFun]),
+                ?DEFAULT_AWAIT),
     Msg1 = <<"msg1">>,
     Msg2 = <<"msg11">>,
 
     publish(Ch, QQ, Msg1),
     publish(Ch, QQ, Msg2),
-    wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]),
     wait_for_messages(Config, [[QQ, <<"0">>, <<"0">>, <<"0">>]]),
+
     ok = rabbit_ct_broker_helpers:set_policy(Config, 0, <<"msg-ttl">>,
                                              QQ, <<"queues">>,
-                                             [{<<"message-ttl">>, 3000}]),
+                                             [{<<"message-ttl">>, 10000}]),
     {ok, {_, Overview2}, _} = rpc:call(Server, ra, local_query, [RaName, QueryFun]),
-    ?assertMatch(#{config := #{msg_ttl := 3000}}, Overview2),
+    ?assertMatch(#{config := #{msg_ttl := 10000}}, Overview2),
     publish(Ch, QQ, Msg1),
     wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]),
     wait_for_messages(Config, [[QQ, <<"0">>, <<"0">>, <<"0">>]]),

deps/rabbitmq_recent_history_exchange/app.bzl

@@ -75,5 +75,5 @@ def test_suite_beam_files(name = "test_suite_beam_files"):
         hdrs = ["include/rabbit_recent_history.hrl"],
         app_name = "rabbitmq_recent_history_exchange",
         erlc_opts = "//:test_erlc_opts",
-        deps = ["//deps/amqp_client:erlang_app"],
+        deps = ["//deps/amqp_client:erlang_app", "//deps/rabbitmq_ct_helpers:erlang_app"],
     )

deps/rabbitmq_recent_history_exchange/test/system_SUITE.erl

@@ -12,6 +12,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("amqp_client/include/amqp_client.hrl").
 -include("rabbit_recent_history.hrl").
+-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl").
 
 all() ->
     [
@@ -148,13 +149,13 @@ e2e_test(Config) ->
                          routing_key = <<"">>
                         }),
 
-    #'queue.declare_ok'{message_count = Count, queue = Q} =
-        amqp_channel:call(Chan, #'queue.declare' {
-                                   passive   = true,
-                                   queue     = Q
-                                  }),
-
-    ?assertEqual(MsgCount, Count),
+    %% Wait for all messages to be queued.
+    ?awaitMatch(#'queue.declare_ok'{message_count = MsgCount, queue = Q},
+                amqp_channel:call(Chan, #'queue.declare' {
+                                           passive   = true,
+                                           queue     = Q
+                                          }),
+                30000),
 
     amqp_channel:call(Chan, #'exchange.delete' { exchange = make_exchange_name(Config, "1") }),
     amqp_channel:call(Chan, #'exchange.delete' { exchange = make_exchange_name(Config, "2") }),