From f5ef64ad06a084998394ccd86f1c295472bcd548 Mon Sep 17 00:00:00 2001 From: Simon Unge Date: Wed, 23 Oct 2024 20:57:08 +0000 Subject: [PATCH 1/5] Add cluster tag config that is exposed via HTTP /api/overview and CTL cluster_status --- deps/rabbit/priv/schema/rabbit.schema | 14 +++++++ .../ctl/commands/cluster_status_command.ex | 38 +++++++++++++++++-- .../src/rabbit_mgmt_wm_overview.erl | 4 ++ .../test/rabbit_mgmt_http_SUITE.erl | 19 +++++++++- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/deps/rabbit/priv/schema/rabbit.schema b/deps/rabbit/priv/schema/rabbit.schema index e930ddbf0fcd..f6ccf7037bb2 100644 --- a/deps/rabbit/priv/schema/rabbit.schema +++ b/deps/rabbit/priv/schema/rabbit.schema @@ -2688,6 +2688,20 @@ fun(Conf) -> end end}. +{mapping, "cluster_tags.$tag", "rabbit.cluster_tags", [ + {datatype, [binary]} +]}. + +{translation, "rabbit.cluster_tags", +fun(Conf) -> + case cuttlefish:conf_get("cluster_tags", Conf, undefined) of + none -> []; + _ -> + Settings = cuttlefish_variable:filter_by_prefix("cluster_tags", Conf), + [ {list_to_binary(K), V} || {[_, K], V} <- Settings] + end +end}. + % =============================== % Validators % =============================== diff --git a/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex b/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex index bc5ca76ca0bc..95494883cbba 100644 --- a/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex +++ b/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex @@ -33,7 +33,7 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do use RabbitMQ.CLI.Core.RequiresRabbitAppRunning def run([], %{node: node_name, timeout: timeout} = opts) do - status = + status0 = case :rabbit_misc.rpc_call(node_name, :rabbit_db_cluster, :cli_cluster_status, []) do {:badrpc, {:EXIT, {:undef, _}}} -> :rabbit_misc.rpc_call(node_name, :rabbit_mnesia, :status, []) @@ -45,11 +45,13 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do status end - case status do + case status0 do {:badrpc, _} = err -> err - status -> + status0 -> + tags = cluster_tags(node_name, timeout) + status = status0 ++ [{:cluster_tags, tags}] case :rabbit_misc.rpc_call(node_name, :rabbit_nodes, :list_running, []) do {:badrpc, _} = err -> err @@ -122,7 +124,6 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do def output(result, %{node: node_name}) when is_list(result) do m = result_map(result) - total_cores = Enum.reduce(m[:cpu_cores], 0, fn {_, val}, acc -> acc + val end) cluster_name_section = [ @@ -131,6 +132,15 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do "Total CPU cores available cluster-wide: #{total_cores}" ] + cluster_tag_section = + [ + "\n#{bright("Cluster Tags")}\n" + ] ++ + case m[:cluster_tags] do + [] -> ["(none)"] + tags -> cluster_tag_lines(tags) + end + disk_nodes_section = [ "\n#{bright("Disk Nodes")}\n" @@ -210,6 +220,7 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do lines = cluster_name_section ++ + cluster_tag_section ++ disk_nodes_section ++ ram_nodes_section ++ running_nodes_section ++ @@ -260,6 +271,7 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do # {rabbit@warp10,[{resource_limit,memory,rabbit@warp10}]}]}] %{ cluster_name: Keyword.get(result, :cluster_name), + cluster_tags: result |> Keyword.get(:cluster_tags, []), disk_nodes: result |> Keyword.get(:nodes, []) |> Keyword.get(:disc, []), ram_nodes: result |> Keyword.get(:nodes, []) |> Keyword.get(:ram, []), running_nodes: result |> Keyword.get(:running_nodes, []) |> Enum.map(&to_string/1), @@ -383,6 +395,18 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do {node, result} end + defp cluster_tags(node, timeout) do + case :rabbit_misc.rpc_call( + node, + :application, + :get_env, + [:rabbit, :cluster_tags], + timeout) do + {:ok, tags} -> tags + _ -> [] + end + end + defp node_lines(nodes) do Enum.map(nodes, &to_string/1) |> Enum.sort() end @@ -413,4 +437,10 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do defp maintenance_lines(mapping) do Enum.map(mapping, fn {node, status} -> "Node: #{node}, status: #{status}" end) end + + defp cluster_tag_lines(mapping) do + Enum.map(mapping, fn {key, value} -> + "#{key}: #{value}" + end) + end end diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl b/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl index 24ab67ce8f49..46d0e5299ea1 100644 --- a/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl +++ b/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl @@ -47,6 +47,7 @@ to_json(ReqData, Context = #context{user = User = #user{tags = Tags}}) -> {product_name, list_to_binary(rabbit:product_name())}, {rabbitmq_version, list_to_binary(rabbit:base_product_version())}, {cluster_name, rabbit_nodes:cluster_name()}, + {cluster_tags, cluster_tags()}, {erlang_version, erlang_version()}, {erlang_full_version, erlang_full_version()}, {release_series_support_status, rabbit_release_series:readable_support_status()}, @@ -182,3 +183,6 @@ transform_retention_intervals([{MaxAgeInSeconds, _}|Rest], Acc) -> 0 end, transform_retention_intervals(Rest, [AccVal|Acc]). + +cluster_tags() -> + application:get_env(rabbit, cluster_tags, []). diff --git a/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl b/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl index eb9387975490..d70be1736e63 100644 --- a/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl +++ b/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl @@ -204,7 +204,8 @@ all_tests() -> [ connections_amqp, amqp_sessions, amqpl_sessions, - enable_plugin_amqp + enable_plugin_amqp, + cluster_tags_test ]. %% ------------------------------------------------------------------- @@ -285,6 +286,11 @@ init_per_testcase(Testcase = disabled_qq_replica_opers_test, Config) -> rabbit_ct_broker_helpers:rpc_all(Config, application, set_env, [rabbitmq_management, restrictions, Restrictions]), rabbit_ct_helpers:testcase_started(Config, Testcase); +init_per_testcase(Testcase = cluster_tags_test, Config) -> + Tags = [{<<"az">>, <<"us-east-3">>}, {<<"region">>,<<"us-east">>}, {<<"environment">>,<<"production">>}], + rabbit_ct_broker_helpers:rpc_all(Config, + application, set_env, [rabbit, cluster_tags, Tags]), + rabbit_ct_helpers:testcase_started(Config, Testcase); init_per_testcase(queues_detailed_test, Config) -> IsEnabled = rabbit_ct_broker_helpers:is_feature_flag_enabled( Config, detailed_queues_endpoint), @@ -351,6 +357,9 @@ end_per_testcase0(disabled_operator_policy_test, Config) -> end_per_testcase0(disabled_qq_replica_opers_test, Config) -> rpc(Config, application, unset_env, [rabbitmq_management, restrictions]), Config; +end_per_testcase0(cluster_tags_test, Config) -> + rpc(Config, application, unset_env, [rabbit, cluster_tags]), + Config; end_per_testcase0(Testcase, Config) when Testcase == list_deprecated_features_test; Testcase == list_used_deprecated_features_test -> @@ -4083,6 +4092,14 @@ list_used_deprecated_features_test(Config) -> ?assertEqual(list_to_binary(Desc), maps:get(desc, Feature)), ?assertEqual(list_to_binary(DocUrl), maps:get(doc_url, Feature)). +cluster_tags_test(Config) -> + Overview = http_get(Config, "/overview"), + Tags = maps:get(cluster_tags, Overview), + ExpectedTags = #{az => <<"us-east-3">>,environment => <<"production">>, + region => <<"us-east">>}, + ?assertEqual(ExpectedTags, Tags), + passed. + %% ------------------------------------------------------------------- %% Helpers. %% ------------------------------------------------------------------- From eeea517da5edb476228c39a72a0a19d11f3f8ee7 Mon Sep 17 00:00:00 2001 From: Simon Unge Date: Wed, 6 Nov 2024 19:39:37 +0000 Subject: [PATCH 2/5] Store tags in global parameters --- deps/rabbit/src/rabbit.erl | 28 ++++++++++++++++++- .../ctl/commands/cluster_status_command.ex | 10 +++---- .../src/rabbit_mgmt_wm_overview.erl | 6 +++- .../test/rabbit_mgmt_http_SUITE.erl | 9 ++++-- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/deps/rabbit/src/rabbit.erl b/deps/rabbit/src/rabbit.erl index f6f6fa364278..5517b4c74e6f 100644 --- a/deps/rabbit/src/rabbit.erl +++ b/deps/rabbit/src/rabbit.erl @@ -11,6 +11,8 @@ -include_lib("kernel/include/logger.hrl"). -include_lib("rabbit_common/include/logging.hrl"). +-feature(maybe_expr, enable). + -behaviour(application). -export([start/0, boot/0, stop/0, @@ -38,7 +40,7 @@ %%--------------------------------------------------------------------------- %% Boot steps. --export([maybe_insert_default_data/0, boot_delegate/0, recover/0, +-export([maybe_set_cluster_tags/0, maybe_insert_default_data/0, boot_delegate/0, recover/0, pg_local_amqp_session/0, pg_local_amqp_connection/0]). @@ -208,6 +210,12 @@ {requires, recovery}, {enables, routing_ready}]}). + +-rabbit_boot_step({cluster_tags, + [{description, "Set cluster tags"}, + {mfa, {?MODULE, maybe_set_cluster_tags, []}}, + {requires, core_initialized}]}). + -rabbit_boot_step({routing_ready, [{description, "message delivery logic ready"}, {requires, [core_initialized, recovery]}]}). @@ -1138,6 +1146,24 @@ pg_local_amqp_connection() -> pg_local_scope(Prefix) -> list_to_atom(io_lib:format("~s_~s", [Prefix, node()])). + +-spec maybe_set_cluster_tags() -> 'ok'. + +maybe_set_cluster_tags() -> + maybe + not_found ?= rabbit_runtime_parameters:lookup_global(cluster_tags), + Tags = application:get_env(rabbit, cluster_tags, []), + false ?= Tags == [], + ?LOG_INFO("Setting cluster tags...", + #{domain => ?RMQLOG_DOMAIN_GLOBAL}), + rabbit_runtime_parameters:set_global(cluster_tags, Tags, <<"internal_user">>) + else + _ -> + % Cluster tags are either already set (Other node, earlier start, CLI) + % Do nothing? + ok + end. + -spec maybe_insert_default_data() -> 'ok'. maybe_insert_default_data() -> diff --git a/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex b/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex index 95494883cbba..70bc8f3de5bc 100644 --- a/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex +++ b/deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/cluster_status_command.ex @@ -398,12 +398,12 @@ defmodule RabbitMQ.CLI.Ctl.Commands.ClusterStatusCommand do defp cluster_tags(node, timeout) do case :rabbit_misc.rpc_call( node, - :application, - :get_env, - [:rabbit, :cluster_tags], + :rabbit_runtime_parameters, + :value_global, + [:cluster_tags], timeout) do - {:ok, tags} -> tags - _ -> [] + :not_found -> [] + tags -> tags end end diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl b/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl index 46d0e5299ea1..817211200f70 100644 --- a/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl +++ b/deps/rabbitmq_management/src/rabbit_mgmt_wm_overview.erl @@ -185,4 +185,8 @@ transform_retention_intervals([{MaxAgeInSeconds, _}|Rest], Acc) -> transform_retention_intervals(Rest, [AccVal|Acc]). cluster_tags() -> - application:get_env(rabbit, cluster_tags, []). + case rabbit_runtime_parameters:value_global(cluster_tags) of + not_found -> + []; + Tags -> Tags + end. diff --git a/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl b/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl index d70be1736e63..845d19f2e885 100644 --- a/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl +++ b/deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl @@ -288,8 +288,9 @@ init_per_testcase(Testcase = disabled_qq_replica_opers_test, Config) -> rabbit_ct_helpers:testcase_started(Config, Testcase); init_per_testcase(Testcase = cluster_tags_test, Config) -> Tags = [{<<"az">>, <<"us-east-3">>}, {<<"region">>,<<"us-east">>}, {<<"environment">>,<<"production">>}], - rabbit_ct_broker_helpers:rpc_all(Config, - application, set_env, [rabbit, cluster_tags, Tags]), + rpc( + Config, rabbit_runtime_parameters, set_global, + [cluster_tags, Tags, none]), rabbit_ct_helpers:testcase_started(Config, Testcase); init_per_testcase(queues_detailed_test, Config) -> IsEnabled = rabbit_ct_broker_helpers:is_feature_flag_enabled( @@ -358,7 +359,9 @@ end_per_testcase0(disabled_qq_replica_opers_test, Config) -> rpc(Config, application, unset_env, [rabbitmq_management, restrictions]), Config; end_per_testcase0(cluster_tags_test, Config) -> - rpc(Config, application, unset_env, [rabbit, cluster_tags]), + rpc( + Config, rabbit_runtime_parameters, clear_global, + [cluster_tags, none]), Config; end_per_testcase0(Testcase, Config) when Testcase == list_deprecated_features_test; From e5d805ea6d46720e1059e9c224fcc91e99d83d20 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Sun, 10 Nov 2024 14:30:51 -0500 Subject: [PATCH 3/5] Cluster tags: set unconditionally Otherwise once set, it would not be possible to change them by updating rabbitmq.conf --- deps/rabbit/src/rabbit.erl | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/deps/rabbit/src/rabbit.erl b/deps/rabbit/src/rabbit.erl index 5517b4c74e6f..d3525a337a54 100644 --- a/deps/rabbit/src/rabbit.erl +++ b/deps/rabbit/src/rabbit.erl @@ -1150,18 +1150,13 @@ pg_local_scope(Prefix) -> -spec maybe_set_cluster_tags() -> 'ok'. maybe_set_cluster_tags() -> - maybe - not_found ?= rabbit_runtime_parameters:lookup_global(cluster_tags), - Tags = application:get_env(rabbit, cluster_tags, []), - false ?= Tags == [], - ?LOG_INFO("Setting cluster tags...", - #{domain => ?RMQLOG_DOMAIN_GLOBAL}), - rabbit_runtime_parameters:set_global(cluster_tags, Tags, <<"internal_user">>) - else - _ -> - % Cluster tags are either already set (Other node, earlier start, CLI) - % Do nothing? - ok + Tags = application:get_env(rabbit, cluster_tags, []), + case Tags of + [] -> ok; + Value -> + ?LOG_DEBUG("Seeding cluster tags from application environment key...", + #{domain => ?RMQLOG_DOMAIN_GLOBAL}), + rabbit_runtime_parameters:set_global(cluster_tags, Value, <<"internal_user">>) end. -spec maybe_insert_default_data() -> 'ok'. From 9e649aefc0e1f982a8c573a28085ee5b90e134b7 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Sun, 10 Nov 2024 14:35:14 -0500 Subject: [PATCH 4/5] We no longer use 'maybe' in this module --- deps/rabbit/src/rabbit.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/deps/rabbit/src/rabbit.erl b/deps/rabbit/src/rabbit.erl index d3525a337a54..31166206bd5f 100644 --- a/deps/rabbit/src/rabbit.erl +++ b/deps/rabbit/src/rabbit.erl @@ -11,8 +11,6 @@ -include_lib("kernel/include/logger.hrl"). -include_lib("rabbit_common/include/logging.hrl"). --feature(maybe_expr, enable). - -behaviour(application). -export([start/0, boot/0, stop/0, From 7c66fba0c3c462db2ca789a1e3aa3fecc0a501fd Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Sun, 10 Nov 2024 14:38:34 -0500 Subject: [PATCH 5/5] Make it possible to clear cluster_tags via rabbitmq.conf --- deps/rabbit/src/rabbit.erl | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/deps/rabbit/src/rabbit.erl b/deps/rabbit/src/rabbit.erl index 31166206bd5f..4a8c0b62d467 100644 --- a/deps/rabbit/src/rabbit.erl +++ b/deps/rabbit/src/rabbit.erl @@ -38,7 +38,7 @@ %%--------------------------------------------------------------------------- %% Boot steps. --export([maybe_set_cluster_tags/0, maybe_insert_default_data/0, boot_delegate/0, recover/0, +-export([update_cluster_tags/0, maybe_insert_default_data/0, boot_delegate/0, recover/0, pg_local_amqp_session/0, pg_local_amqp_connection/0]). @@ -211,7 +211,7 @@ -rabbit_boot_step({cluster_tags, [{description, "Set cluster tags"}, - {mfa, {?MODULE, maybe_set_cluster_tags, []}}, + {mfa, {?MODULE, update_cluster_tags, []}}, {requires, core_initialized}]}). -rabbit_boot_step({routing_ready, @@ -1145,17 +1145,13 @@ pg_local_scope(Prefix) -> list_to_atom(io_lib:format("~s_~s", [Prefix, node()])). --spec maybe_set_cluster_tags() -> 'ok'. +-spec update_cluster_tags() -> 'ok'. -maybe_set_cluster_tags() -> +update_cluster_tags() -> Tags = application:get_env(rabbit, cluster_tags, []), - case Tags of - [] -> ok; - Value -> - ?LOG_DEBUG("Seeding cluster tags from application environment key...", + ?LOG_DEBUG("Seeding cluster tags from application environment key...", #{domain => ?RMQLOG_DOMAIN_GLOBAL}), - rabbit_runtime_parameters:set_global(cluster_tags, Value, <<"internal_user">>) - end. + rabbit_runtime_parameters:set_global(cluster_tags, Tags, <<"internal_user">>). -spec maybe_insert_default_data() -> 'ok'.