Merge pull request #3317 from rabbitmq/mergify/bp/v3.9.x/pr-3311

Definition import: allow for arbitrary (and pluggable) sources (backport #3311)

Author: michaelklishin

deps/rabbit/priv/schema/rabbit.schema

@@ -121,18 +121,139 @@ end}.
 %% Definition import
 %%
 
-%% Load definitions from a JSON file or directory of files. See
+%% Original key for definition loading from a JSON file or directory of files. See
 %% https://www.rabbitmq.com/management.html#load-definitions
-%%
-%% {load_definitions, "/path/to/schema.json"},
-%% {load_definitions, "/path/to/schemas"},
 {mapping, "load_definitions", "rabbit.load_definitions",
     [{datatype, string},
      {validators, ["file_accessible"]}]}.
 
+%% Newer syntax for definition loading from a JSON file or directory of files. See
+%% https://www.rabbitmq.com/management.html#load-definitions
+{mapping, "definitions.local.path", "rabbit.definitions.local_path",
+    [{datatype, string},
+     {validators, ["file_accessible"]}]}.
+
+%% Extensive mechanism for loading definitions from a remote source
+{mapping, "definitions.import_backend", "rabbit.definitions.import_backend", [
+    {datatype, atom}
+]}.
+
+{translation, "rabbit.definitions.import_backend",
+fun(Conf) ->
+    case cuttlefish:conf_get("definitions.import_backend", Conf, rabbit_definitions_import_local_filesystem) of
+        %% short aliases for known backends
+        local_filesystem -> rabbit_definitions_import_local_filesystem;
+        local            -> rabbit_definitions_import_local_filesystem;
+        https            -> rabbit_definitions_import_https;
+        http             -> rabbit_definitions_import_https;
+        %% accept both rabbitmq_ and rabbit_ (typical core module prefix)
+        rabbitmq_definitions_import_local_filesystem -> rabbit_definitions_import_local_filesystem;
+        rabbitmq_definitions_import_local_filesystem -> rabbit_definitions_import_https;
+        %% any other value is used as is
+        Module           -> Module
+    end
+end}.
+
+%% Load definitions from a remote URL over HTTPS. See
+%% https://www.rabbitmq.com/management.html#load-definitions
+{mapping, "definitions.https.url", "rabbit.definitions.url",
+    [{datatype, string}]}.
+
+%% Client-side TLS settings used by e.g. HTTPS definition loading mechanism.
+%% These can be reused by other clients.
+
+{mapping, "definitions.tls.verify", "rabbit.definitions.ssl_options.verify", [
+    {datatype, {enum, [verify_peer, verify_none]}}]}.
+
+{mapping, "definitions.tls.fail_if_no_peer_cert", "rabbit.definitions.ssl_options.fail_if_no_peer_cert", [
+    {datatype, {enum, [true, false]}}]}.
+
+{mapping, "definitions.tls.cacertfile", "rabbit.definitions.ssl_options.cacertfile",
+    [{datatype, string}, {validators, ["file_accessible"]}]}.
+
+{mapping, "definitions.tls.certfile", "rabbit.definitions.ssl_options.certfile",
+    [{datatype, string}, {validators, ["file_accessible"]}]}.
+
+{mapping, "definitions.tls.cacerts.$name", "rabbit.definitions.ssl_options.cacerts",
+    [{datatype, string}]}.
+
+{translation, "rabbit.definitions.ssl_options.cacerts",
+fun(Conf) ->
+    Settings = cuttlefish_variable:filter_by_prefix("definitions.tls.cacerts", Conf),
+    [ list_to_binary(V) || {_, V} <- Settings ]
+end}.
+
+{mapping, "definitions.tls.cert", "rabbit.definitions.ssl_options.cert",
+    [{datatype, string}]}.
+
+{translation, "rabbit.definitions.ssl_options.cert",
+fun(Conf) ->
+    list_to_binary(cuttlefish:conf_get("definitions.tls.cert", Conf))
+end}.
+
+{mapping, "definitions.tls.reuse_session", "rabbit.definitions.ssl_options.reuse_session",
+    [{datatype, {enum, [true, false]}}]}.
+
+{mapping, "definitions.tls.crl_check", "rabbit.definitions.ssl_options.crl_check",
+    [{datatype, [{enum, [true, false, peer, best_effort]}]}]}.
+
+{mapping, "definitions.tls.depth", "rabbit.definitions.ssl_options.depth",
+    [{datatype, integer}, {validators, ["byte"]}]}.
+
+{mapping, "definitions.tls.dh", "rabbit.definitions.ssl_options.dh",
+    [{datatype, string}]}.
+
+{translation, "rabbit.definitions.ssl_options.dh",
+fun(Conf) ->
+    list_to_binary(cuttlefish:conf_get("definitions.tls.dh", Conf))
+end}.
+
+{translation, "rabbit.definitions.ssl_options.key",
+fun(Conf) ->
+    case cuttlefish_variable:filter_by_prefix("definitions.tls.key", Conf) of
+        [{[_,_,Key], Val}|_] -> {list_to_atom(Key), list_to_binary(Val)};
+        _ -> cuttlefish:unset()
+    end
+end}.
+
+{mapping, "definitions.tls.keyfile", "rabbit.definitions.ssl_options.keyfile",
+    [{datatype, string}, {validators, ["file_accessible"]}]}.
+
+{mapping, "definitions.tls.log_alert", "rabbit.definitions.ssl_options.log_alert",
+    [{datatype, {enum, [true, false]}}]}.
+
+{mapping, "definitions.tls.password", "rabbit.definitions.ssl_options.password",
+    [{datatype, string}]}.
+
+{mapping, "definitions.tls.secure_renegotiate", "rabbit.definitions.ssl_options.secure_renegotiate",
+    [{datatype, {enum, [true, false]}}]}.
+
+{mapping, "definitions.tls.reuse_sessions", "rabbit.definitions.ssl_options.reuse_sessions",
+    [{datatype, {enum, [true, false]}}]}.
+
+{mapping, "definitions.tls.versions.$version", "rabbit.definitions.ssl_options.versions",
+    [{datatype, atom}]}.
+
+{translation, "rabbit.definitions.ssl_options.versions",
+fun(Conf) ->
+    Settings = cuttlefish_variable:filter_by_prefix("definitions.tls.versions", Conf),
+    [V || {_, V} <- Settings]
+end}.
+
+{mapping, "definitions.tls.ciphers.$cipher", "rabbit.definitions.ssl_options.ciphers",
+    [{datatype, string}]}.
+
+{translation, "rabbit.definitions.ssl_options.ciphers",
+fun(Conf) ->
+    Settings = cuttlefish_variable:filter_by_prefix("definitions.tls.ciphers", Conf),
+    lists:reverse([V || {_, V} <- Settings])
+end}.
+
+{mapping, "definitions.tls.log_level", "rabbit.definitions.ssl_options.log_level",
+    [{datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug]}}]}.
+
 %%
-%% Security / AAA
-%% ==============
+%% Seed User, Authentication, Access Control
 %%
 
 %% The default "guest" user is only permitted to access the server
@@ -254,13 +375,16 @@ end}.
 fun(Conf) ->
     case cuttlefish_variable:filter_by_prefix("ssl_options.key", Conf) of
         [{[_,_,Key], Val}|_] -> {list_to_atom(Key), list_to_binary(Val)};
-        _ -> undefined
+        _ -> cuttlefish:unset()
     end
 end}.
 
 {mapping, "ssl_options.keyfile", "rabbit.ssl_options.keyfile",
     [{datatype, string}, {validators, ["file_accessible"]}]}.
 
+{mapping, "ssl_options.log_level", "rabbit.ssl_options.log_level",
+    [{datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug]}}]}.
+
 {mapping, "ssl_options.log_alert", "rabbit.ssl_options.log_alert",
     [{datatype, {enum, [true, false]}}]}.
 

deps/rabbit/src/rabbit_definitions.erl

@@ -10,8 +10,13 @@
 
 -export([boot/0]).
 %% automatic import on boot
--export([maybe_load_definitions/0, maybe_load_definitions/2, maybe_load_definitions_from/2,
-         has_configured_definitions_to_load/0]).
+-export([
+    maybe_load_definitions/0,
+    maybe_load_definitions/2,
+    maybe_load_definitions_from/2,
+
+    has_configured_definitions_to_load/0
+]).
 %% import
 -export([import_raw/1, import_raw/2, import_parsed/1, import_parsed/2,
          apply_defs/2, apply_defs/3, apply_defs/4, apply_defs/5]).
@@ -25,7 +30,7 @@
 ]).
 -export([decode/1, decode/2, args/1]).
 
--import(rabbit_misc, [pget/2]).
+-import(rabbit_misc, [pget/2, pget/3]).
 -import(rabbit_data_coercion, [to_binary/1]).
 
 %%
@@ -59,9 +64,10 @@ boot() ->
     rabbit_sup:start_supervisor_child(definition_import_pool_sup, worker_pool_sup, [PoolSize, ?IMPORT_WORK_POOL]).
 
 maybe_load_definitions() ->
-    %% Note that management.load_definitions is handled in the plugin for backwards compatibility.
-    %% This executes the "core" version of load_definitions.
-    maybe_load_definitions(rabbit, load_definitions).
+    %% Classic source: local file or data directory
+    maybe_load_definitions_from_local_filesystem(rabbit, load_definitions),
+    %% Extensible sources
+    maybe_load_definitions_from_pluggable_source(rabbit, definitions).
 
 -spec import_raw(Body :: binary() | iolist()) -> ok | {error, term()}.
 import_raw(Body) ->
@@ -126,63 +132,76 @@ all_definitions() ->
         exchanges         => Xs
     }.
 
+-spec has_configured_definitions_to_load() -> boolean().
+has_configured_definitions_to_load() ->
+    has_configured_definitions_to_load_via_classic_option() or has_configured_definitions_to_load_via_modern_option().
+
+%% Retained for backwards compatibility, implicitly assumes the local filesystem source
+maybe_load_definitions(App, Key) ->
+    maybe_load_definitions_from_local_filesystem(App, Key).
+
+maybe_load_definitions_from(IsDir, Path) ->
+    rabbit_definitions_import_local_filesystem:load(IsDir, Path).
+
 %%
 %% Implementation
 %%
 
--spec has_configured_definitions_to_load() -> boolean().
-has_configured_definitions_to_load() ->
+-spec has_configured_definitions_to_load_via_modern_option() -> boolean().
+has_configured_definitions_to_load_via_modern_option() ->
+    case application:get_env(rabbit, definitions) of
+        undefined  -> false;
+        {ok, none} -> false;
+        {ok, []}   -> false;
+        {ok, _Options} -> true
+    end.
+
+has_configured_definitions_to_load_via_classic_option() ->
     case application:get_env(rabbit, load_definitions) of
         undefined   -> false;
         {ok, none}  -> false;
         {ok, _Path} -> true
     end.
 
-maybe_load_definitions(App, Key) ->
+maybe_load_definitions_from_local_filesystem(App, Key) ->
     case application:get_env(App, Key) of
-        undefined  ->
-            rabbit_log:debug("No definition file configured to import via load_definitions"),
-            ok;
-        {ok, none} ->
-            rabbit_log:debug("No definition file configured to import via load_definitions"),
-            ok;
-        {ok, FileOrDir} ->
-            rabbit_log:debug("Will import definitions file from load_definitions"),
-            IsDir = filelib:is_dir(FileOrDir),
-            maybe_load_definitions_from(IsDir, FileOrDir)
+        undefined  -> ok;
+        {ok, none} -> ok;
+        {ok, Path} ->
+            IsDir = filelib:is_dir(Path),
+            rabbit_definitions_import_local_filesystem:load(IsDir, Path)
     end.
 
-maybe_load_definitions_from(true, Dir) ->
-    rabbit_log:info("Applying definitions from directory ~s", [Dir]),
-    load_definitions_from_files(file:list_dir(Dir), Dir);
-maybe_load_definitions_from(false, File) ->
-    load_definitions_from_file(File).
-
-load_definitions_from_files({ok, Filenames0}, Dir) ->
-    Filenames1 = lists:sort(Filenames0),
-    Filenames2 = [filename:join(Dir, F) || F <- Filenames1],
-    load_definitions_from_filenames(Filenames2);
-load_definitions_from_files({error, E}, Dir) ->
-    rabbit_log:error("Could not read definitions from directory ~s, Error: ~p", [Dir, E]),
-    {error, {could_not_read_defs, E}}.
-
-load_definitions_from_filenames([]) ->
-    ok;
-load_definitions_from_filenames([File|Rest]) ->
-    case load_definitions_from_file(File) of
-        ok         -> load_definitions_from_filenames(Rest);
-        {error, E} -> {error, {failed_to_import_definitions, File, E}}
+maybe_load_definitions_from_pluggable_source(App, Key) ->
+    case application:get_env(App, Key) of
+        undefined  -> ok;
+        {ok, none} -> ok;
+        {ok, []}   -> ok;
+        {ok, Proplist} ->
+            case pget(import_backend, Proplist, undefined) of
+                undefined  ->
+                    {error, "definition import source is configured but definitions.import_backend is not set"};
+                ModOrAlias ->
+                    Mod = normalize_backend_module(ModOrAlias),
+                    rabbit_log:debug("Will use module ~s to import definitions", [Mod]),
+                    Mod:load(Proplist)
+            end
     end.
 
-load_definitions_from_file(File) ->
-    case file:read_file(File) of
-        {ok, Body} ->
-            rabbit_log:info("Applying definitions from file at '~s'", [File]),
-            import_raw(Body);
-        {error, E} ->
-            rabbit_log:error("Could not read definitions from file at '~s', error: ~p", [File, E]),
-            {error, {could_not_read_defs, {File, E}}}
-    end.
+normalize_backend_module(local_filesystem) ->
+    rabbit_definitions_import_local_filesystem;
+normalize_backend_module(local) ->
+    rabbit_definitions_import_local_filesystem;
+normalize_backend_module(https) ->
+    rabbit_definitions_import_https;
+normalize_backend_module(http)  ->
+    rabbit_definitions_import_https;
+normalize_backend_module(rabbitmq_definitions_import_local_filesystem) ->
+    rabbit_definitions_import_local_filesystem;
+normalize_backend_module(rabbitmq_definitions_import_https) ->
+    rabbit_definitions_import_https;
+normalize_backend_module(Other) ->
+    Other.
 
 decode(Keys, Body) ->
     case decode(Body) of