Follow-up: finish 'rabbitmqctl clear_auth_backend_cache' plus add tests (backport #13325) (backport #13337)

deps/rabbitmq_auth_backend_cache/src/rabbit_auth_backend_cache.erl

@@ -13,7 +13,7 @@
 
 -export([user_login_authentication/2, user_login_authorization/2,
          check_vhost_access/3, check_resource_access/4, check_topic_access/4,
-         expiry_timestamp/1]).
+         expiry_timestamp/1, clear_cache_cluster_wide/0, clear_cache/0]).
 
 %% API
 
@@ -66,6 +66,17 @@ expiry_timestamp(_) -> never.
 %% Implementation
 %%
 
+clear_cache_cluster_wide() ->
+    Nodes = rabbit_nodes:list_running(),
+    rabbit_log:warning("Clearing auth_backend_cache in all nodes : ~p", [Nodes]),
+    rabbit_misc:append_rpc_all_nodes(Nodes, ?MODULE, clear_cache, []).
+
+clear_cache() ->
+    {ok, AuthCache} = application:get_env(rabbitmq_auth_backend_cache,
+                                          cache_module),
+    rabbit_log:warning("Clearing auth_backend_cache"),
+    AuthCache:clear().
+
 with_cache(BackendType, {F, A}, Fun) ->
     {ok, AuthCache} = application:get_env(rabbitmq_auth_backend_cache,
                                           cache_module),
@@ -105,3 +116,5 @@ should_cache(Result, Fun) ->
         {refusal, true} -> true;
         _               -> false
     end.
+
+

deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache.erl

@@ -15,6 +15,8 @@
 
 -callback delete(term()) -> ok.
 
+-callback clear() -> ok.
+
 expiration(TTL) ->
     erlang:system_time(milli_seconds) + TTL.
 

deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_dict.erl

@@ -15,7 +15,7 @@
 -include("rabbit_auth_backend_cache.hrl").
 
 -export([start_link/0,
-         get/1, put/3, delete/1]).
+         get/1, put/3, delete/1, clear/0]).
 
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
@@ -28,6 +28,8 @@ put(Key, Value, TTL) -> gen_server:cast(?MODULE, {put, Key, Value, TTL}).
 
 delete(Key) -> gen_server:call(?MODULE, {delete, Key}, ?CACHE_OPERATION_TIMEOUT).
 
+clear() -> gen_server:cast(?MODULE, clear).
+
 init(_Args) -> {ok, nostate}.
 
 handle_call({get, Key}, _From, nostate) ->
@@ -40,6 +42,10 @@ handle_call({delete, Key}, _From, nostate) ->
     do_delete(Key),
     {reply, ok, nostate}.
 
+handle_cast(clear, nostate) ->
+    _ = erlang:erase(),
+    {noreply, nostate};
+
 handle_cast({put, Key, Value, TTL}, nostate) ->
     erlang:put({items, Key}, Value),
     {ok, TRef} = timer:apply_after(TTL, rabbit_auth_cache_dict, delete, [Key]),

deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets.erl

@@ -15,7 +15,7 @@
 -behaviour(rabbit_auth_cache).
 
 -export([start_link/0,
-         get/1, put/3, delete/1]).
+         get/1, put/3, delete/1, clear/0]).
 
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
@@ -36,6 +36,8 @@ put(Key, Value, TTL) ->
 
 delete(Key) -> gen_server:call(?MODULE, {delete, Key}, ?CACHE_OPERATION_TIMEOUT).
 
+clear() -> gen_server:cast(?MODULE, clear).
+
 init([]) ->
     {ok, #state{cache = ets:new(?MODULE, [set, private]),
                 timers = ets:new(auth_cache_ets_timers, [set, private])}}.
@@ -53,6 +55,10 @@ handle_call({delete, Key}, _From, State = #state{cache = Table, timers = Timers}
     do_delete(Key, Table, Timers),
     {reply, ok, State}.
 
+handle_cast(clear, State = #state{cache = Table}) ->
+    ets:delete_all_objects(Table),
+    {noreply, State};
+
 handle_cast({put, Key, Value, TTL, Expiration},
             State = #state{cache = Table, timers = Timers}) ->
     do_delete(Key, Table, Timers),

deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented.erl

@@ -10,7 +10,7 @@
 -behaviour(rabbit_auth_cache).
 
 -export([start_link/1,
-         get/1, put/3, delete/1]).
+         get/1, put/3, delete/1, clear/0]).
 -export([gc/0]).
 
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -43,6 +43,11 @@ delete(Key) ->
          || Table <- gen_server:call(?MODULE, get_segment_tables, ?CACHE_OPERATION_TIMEOUT)],
     ok.
 
+clear() -> 
+    _ = [ets:delete_all_objects(Table)
+         || Table <- gen_server:call(?MODULE, get_segment_tables, ?CACHE_OPERATION_TIMEOUT)],
+    ok.
+
 gc() ->
     case whereis(?MODULE) of
         undefined -> ok;

deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented_stateless.erl

@@ -12,7 +12,7 @@
 -include("rabbit_auth_backend_cache.hrl").
 
 -export([start_link/1,
-         get/1, put/3, delete/1]).
+         get/1, put/3, delete/1, clear/0]).
 -export([gc/0]).
 
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -47,6 +47,11 @@ delete(Key) ->
          || Table <- get_all_segment_tables()],
     ok.
 
+clear() ->
+    _ = [ets:delete_all_objects(Table)
+         || Table <- get_all_segment_tables()],
+    ok.
+
 gc() ->
     case whereis(?MODULE) of
         undefined -> ok;

deps/rabbitmq_auth_backend_cache/test/rabbit_auth_cache_SUITE.erl

@@ -19,7 +19,14 @@ all() ->
     ].
 
 groups() ->
-    CommonTests = [get_empty, get_put, get_expired, put_replace, get_deleted, random_timing],
+    CommonTests = [
+        get_empty, 
+        get_put, 
+        get_expired, 
+        put_replace, 
+        get_deleted, 
+        random_timing, 
+        clear],
     [
     {rabbit_auth_cache_dict, [sequence], CommonTests},
     {rabbit_auth_cache_ets, [sequence], CommonTests},
@@ -153,6 +160,20 @@ get_deleted(Config) ->
     AuthCacheModule:delete(Key),
     {error, not_found} = AuthCacheModule:get(Key).
 
+clear(Config) ->
+    AuthCacheModule = ?config(auth_cache_module, Config),
+    Key1 = some_key1,
+    Key2 = some_key2,
+    TTL = ?config(current_ttl, Config),
+    {error, not_found} = AuthCacheModule:get(Key1),
+    {error, not_found} = AuthCacheModule:get(Key2),
+    ok = AuthCacheModule:put(Key1, some_value, TTL),
+    ok = AuthCacheModule:put(Key2, some_value, TTL),
+    {ok, some_value} = AuthCacheModule:get(Key1),
+    {ok, some_value} = AuthCacheModule:get(Key2),
+    AuthCacheModule:clear(),
+    {error, not_found} = AuthCacheModule:get(Key1),
+    {error, not_found} = AuthCacheModule:get(Key2).
 
 random_timing(Config) ->
     random_timing(Config, 15000, 1000).

deps/rabbitmq_auth_backend_cache/test/rabbit_auth_clear_cache_command_SUITE.erl

@@ -0,0 +1,112 @@
+%% This Source Code Form is subject to the terms of the Mozilla Public
+%% License, v. 2.0. If a copy of the MPL was not distributed with this
+%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
+%%
+%% Copyright (c) 2007-2025 Broadcom. All Rights Reserved. The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. All rights reserved.
+%%
+
+-module(rabbit_auth_clear_cache_command_SUITE).
+
+-include_lib("stdlib/include/assert.hrl").
+
+-compile(export_all).
+
+-define(CLEAR_CACHE_CMD, 'Elixir.RabbitMQ.CLI.Ctl.Commands.ClearAuthBackendCacheCommand').
+
+all() ->
+    [
+      {group, non_parallel_tests},
+      {group, cluster_size_2}
+    ].
+
+groups() ->
+    [
+     {non_parallel_tests, [], [
+                               clear_cache
+                              ]},
+     {cluster_size_2, [], [
+                               clear_cache
+                              ]}
+    ].
+
+%% -------------------------------------------------------------------
+%% Testsuite setup/teardown.
+%% -------------------------------------------------------------------
+
+init_per_suite(Config) ->
+    rabbit_ct_helpers:log_environment(),
+    rabbit_ct_helpers:run_setup_steps(Config).
+
+
+setup_env(Config, Nodename) ->
+    rpc(Config, Nodename, application, set_env, 
+        [rabbit, auth_backends, [rabbit_auth_backend_cache]]),
+    Config.
+
+end_per_suite(Config) ->
+    rabbit_ct_helpers:run_teardown_steps(Config).
+
+init_per_group(cluster_size_2, Config) ->
+    case rabbit_ct_helpers:is_mixed_versions() of 
+        true -> {skip, "cluster size 2 isn't mixed versions compatible"};
+        false -> init_per_multinode_group(cluster_size_2, Config, 2)
+    end;
+init_per_group(Group, Config) ->
+    init_per_multinode_group(Group, Config, 1).
+
+init_per_multinode_group(_Group, Config, NodeCount) ->
+    Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"),
+    Config1 = rabbit_ct_helpers:set_config(Config, [
+                                                    {rmq_nodes_count, NodeCount},
+                                                    {rmq_nodename_suffix, Suffix}
+      ]),
+    rabbit_ct_helpers:run_steps(Config1,
+    rabbit_ct_broker_helpers:setup_steps() ++ 
+    rabbit_ct_client_helpers:setup_steps()).
+
+end_per_group(_Group, Config) ->
+    rabbit_ct_helpers:run_steps(Config,
+      rabbit_ct_client_helpers:teardown_steps() ++
+      rabbit_ct_broker_helpers:teardown_steps()).
+
+init_per_testcase(Testcase, Config) ->
+    rabbit_ct_helpers:testcase_started(Config, Testcase).
+
+end_per_testcase(Testcase, Config) ->
+    rabbit_ct_helpers:testcase_finished(Config, Testcase).
+
+%% -------------------------------------------------------------------
+%% Testcases.
+%% -------------------------------------------------------------------
+
+
+clear_cache(Config) ->
+    F = user_login_authentication,
+    A = [<<"guest">>, [{password, <<"guest">>}]],
+    Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    [ setup_env(Config, Nodename) || Nodename <- Nodes],
+
+    [ ok = ensure_cache_entries(Config, Node, {F, A}) || Node <- Nodes],
+    ?CLEAR_CACHE_CMD:run([], #{node => lists:last(Nodes)}),
+    [ rabbit_ct_helpers:await_condition_with_retries(fun () -> 
+            case has_cache_entry(Config, Node, {F, A}) of 
+                {error, not_found} -> true;
+                _ -> false
+            end 
+        end, 20) || Node <- Nodes].
+
+ensure_cache_entries(Config, Nodename, {F, A}) ->
+    {ok, AuthRespOk} = rpc(Config, Nodename, rabbit_auth_backend_internal, F, A),
+    {ok, AuthRespOk} = rpc(Config, Nodename, rabbit_auth_backend_cache, F, A),
+    ok = has_cache_entry(Config, Nodename, {F, A}).
+
+rpc(Config, N, M, F, A) ->
+    rabbit_ct_broker_helpers:rpc(Config, N, M, F, A).
+
+has_cache_entry(Config, Node, {F, A}) ->
+    {ok, AuthCache} = rpc(Config, Node, application, get_env, 
+        [rabbitmq_auth_backend_cache, cache_module]),
+    case rpc(Config, Node, AuthCache, get, [{F, A}]) of
+        {ok, _} -> ok;
+        {error, not_found} = E -> E
+    end.