Merge pull request #14709 from rabbitmq/mergify/bp/v4.2.x/pr-14664

Comprehensive tests for the `rabbitmq_aws` Cuttlefish schema (backport #14664)

Author: michaelklishin

deps/rabbitmq_aws/.editorconfig

@@ -7,9 +7,11 @@ define PROJECT_ENV
 []
 endef
 
+BUILD_DEPS = rabbit
+TEST_DEPS = meck rabbitmq_ct_helpers rabbitmq_ct_client_helpers
 LOCAL_DEPS = crypto inets ssl xmerl public_key
-BUILD_DEPS = rabbit_common
-TEST_DEPS = meck rabbit rabbitmq_ct_helpers rabbitmq_ct_client_helpers
+
+PLT_APPS = rabbit
 
 DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk
 DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk

deps/rabbitmq_aws/Makefile

@@ -5,13 +5,13 @@ A fork of [gmr/httpc-aws](https://github.com/gmr/httpc-aws) for use in building
 ## Supported Erlang Versions
 
 [Same as RabbitMQ](http://www.rabbitmq.com/which-erlang.html)
- 
+
 ## Configuration
 
 Configuration for *rabbitmq-aws* is can be provided in multiple ways. It is designed
 to behave similarly to the [AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html)
 with respect to providing region and configuration information. Additionally it
-has two methods, ``rabbitmq_aws:set_region/1`` and ``rabbitmq_aws:set_credentials/2``
+has two methods, `rabbitmq_aws:set_region/1` and `rabbitmq_aws:set_credentials/2`
 to allow for application specific configuration, bypassing the automatic configuration
 behavior.
 
@@ -40,36 +40,36 @@ and [adds defenses against additional vulnerabilities](https://aws.amazon.com/bl
 AWS recommends adopting IMDSv2 and disabling IMDSv1 [by configuring the Instance Metadata Service on the EC2 instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html).
 
 By default *rabbitmq-aws* will attempt to use IMDSv2 first and will fallback to use IMDSv1 if calls to IMDSv2 fail. This behavior can be overridden
-by setting the ``aws.prefer_imdsv2`` setting to ``false``.
+by setting the `aws.prefer_imdsv2` setting to `false`.
 
 ### Environment Variables
 
-As with the AWS CLI, the following environment variables can be used to provide 
+As with the AWS CLI, the following environment variables can be used to provide
 configuration or to impact configuration behavior:
 
- - ``AWS_DEFAULT_PROFILE``
- - ``AWS_DEFAULT_REGION``
- - ``AWS_CONFIG_FILE``
- - ``AWS_SHARED_CREDENTIALS_FILE``
- - ``AWS_ACCESS_KEY_ID``
- - ``AWS_SECRET_ACCESS_KEY``
- 
+ - `AWS_DEFAULT_PROFILE`
+ - `AWS_DEFAULT_REGION`
+ - `AWS_CONFIG_FILE`
+ - `AWS_SHARED_CREDENTIALS_FILE`
+ - `AWS_ACCESS_KEY_ID`
+ - `AWS_SECRET_ACCESS_KEY`
+
 ## API Functions
- 
-  Method                                | Description
- ---------------------------------------|--------------------------------------------------------------------------------------------
- ``rabbitmq_aws:set_region/1``          | Manually specify the AWS region to make requests to.
- ``rabbitmq_aws:set_credentials/2``     | Manually specify the request credentials to use.
- ``rabbitmq_aws:refresh_credentials/0`` | Refresh the credentials from the environment, filesystem, or EC2 Instance Metadata Service.
- ``rabbitmq_aws:ensure_imdsv2_token_valid/0`` | Make sure EC2 IMDSv2 token is active and valid.
- ``rabbitmq_aws:api_get_request/2``           | Perform an AWS service API request.
- ``rabbitmq_aws:get/2``                 | Perform a GET request to the API specifying the service and request path.
- ``rabbitmq_aws:get/3``                 | Perform a GET request specifying the service, path, and headers.
- ``rabbitmq_aws:post/4``                | Perform a POST request specifying the service, path, headers, and body.
- ``rabbitmq_aws:request/5``             | Perform a request specifying the service, method, path, headers, and body.
- ``rabbitmq_aws:request/6``             | Perform a request specifying the service, method, path, headers, body, and ``httpc:http_options().``
- ``rabbitmq_aws:request/7``             | Perform a request specifying the service, method, path, headers, body,  ``httpc:http_options()``, and override the API endpoint. 
- 
+
+  Method                                    | Description
+ -------------------------------------------|--------------------------------------------------------------------------------------------
+ `rabbitmq_aws:set_region/1`                | Manually specify the AWS region to make requests to.
+ `rabbitmq_aws:set_credentials/2`           | Manually specify the request credentials to use.
+ `rabbitmq_aws:refresh_credentials/0`       | Refresh the credentials from the environment, filesystem, or EC2 Instance Metadata Service.
+ `rabbitmq_aws:ensure_imdsv2_token_valid/0` | Make sure EC2 IMDSv2 token is active and valid.
+ `rabbitmq_aws:get/2`                       | Perform a GET request to the API specifying the service and request path.
+ `rabbitmq_aws:get/3`                       | Perform a GET request specifying the service, path, and headers.
+ `rabbitmq_aws:post/4`                      | Perform a POST request specifying the service, path, headers, and body.
+ `rabbitmq_aws:request/5`                   | Perform a request specifying the service, method, path, headers, and body.
+ `rabbitmq_aws:request/6`                   | Perform a request specifying the service, method, path, headers, body, and `httpc:http_options().`
+ `rabbitmq_aws:request/7`                   | Perform a request specifying the service, method, path, headers, body,  `httpc:http_options()`, and override the API endpoint.
+ `rabbitmq_aws:api_get_request/2`           | Perform an AWS service API request with retries.
+ `rabbitmq_aws:api_post_request/2`          | Perform an AWS service API request with retries.
 
 ## Example Usage
 
@@ -80,8 +80,7 @@ you're using the EC2 Instance Metadata Service for credentials:
 application:start(rabbitmq_aws).
 {ok, {Headers, Response}} = rabbitmq_aws:get("ec2","/?Action=DescribeTags&Version=2015-10-01").
 ```
-
-To configure credentials, invoke ``rabbitmq_aws:set_credentials/2``:
+To configure credentials, invoke `rabbitmq_aws:set_credentials/2`:
 
 ```erlang
 application:start(rabbitmq_aws).
@@ -90,8 +89,8 @@ rabbitmq_aws:set_credentials("AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMP
 
 RequestHeaders = [{"Content-Type", "application/x-amz-json-1.0"},
                   {"X-Amz-Target", "DynamoDB_20120810.ListTables"}],
-                  
-{ok, {Headers, Response}} = rabbitmq_aws:post("dynamodb", "/", 
+
+{ok, {Headers, Response}} = rabbitmq_aws:post("dynamodb", "/",
                                               "{\"Limit\": 20}",
                                               RequestHeaders).
 ```

deps/rabbitmq_aws/README.md

@@ -15,5 +15,5 @@
 %% When false, EC2 IMDSv1 will be used first and no attempt will be made to use EC2 IMDSv2.
 %% See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html.
 
-{mapping, "aws.prefer_imdsv2", "rabbit.aws_prefer_imdsv2",
+{mapping, "aws.prefer_imdsv2", "rabbitmq_aws.aws_prefer_imdsv2",
     [{datatype, {enum, [true, false]}}]}.

deps/rabbitmq_aws/priv/schema/rabbitmq_aws.schema

@@ -15,10 +15,12 @@
     refresh_credentials/0,
     request/5, request/6, request/7,
     set_credentials/2,
+    set_credentials/3,
     has_credentials/0,
     set_region/1,
     ensure_imdsv2_token_valid/0,
-    api_get_request/2
+    api_get_request/2,
+    api_post_request/4
 ]).
 
 %% gen-server exports
@@ -158,6 +160,12 @@ set_credentials(NewState) ->
 set_credentials(AccessKey, SecretAccessKey) ->
     gen_server:call(rabbitmq_aws, {set_credentials, AccessKey, SecretAccessKey}).
 
+-spec set_credentials(access_key(), secret_access_key(), security_token()) -> ok.
+%% @doc Manually set the access credentials with session token for requests.
+%% @end
+set_credentials(AccessKey, SecretAccessKey, SessionToken) ->
+    gen_server:call(rabbitmq_aws, {set_credentials, AccessKey, SecretAccessKey, SessionToken}).
+
 -spec set_region(Region :: string()) -> ok.
 %% @doc Manually set the AWS region to perform API requests to.
 %% @end
@@ -224,6 +232,14 @@ handle_msg({set_credentials, AccessKey, SecretAccessKey}, State) ->
         expiration = undefined,
         error = undefined
     }};
+handle_msg({set_credentials, AccessKey, SecretAccessKey, SessionToken}, State) ->
+    {reply, ok, State#state{
+        access_key = AccessKey,
+        secret_access_key = SecretAccessKey,
+        security_token = SessionToken,
+        expiration = undefined,
+        error = undefined
+    }};
 handle_msg({set_credentials, NewState}, State) ->
     {reply, ok, State#state{
         access_key = NewState#state.access_key,
@@ -607,8 +623,10 @@ ensure_credentials_valid() ->
     case has_credentials(State) of
         true ->
             case expired_credentials(State#state.expiration) of
-                true -> refresh_credentials(State);
-                _ -> ok
+                true ->
+                    refresh_credentials(State);
+                _ ->
+                    ok
             end;
         _ ->
             refresh_credentials(State)
@@ -618,19 +636,42 @@ ensure_credentials_valid() ->
 %% @doc Invoke an API call to an AWS service.
 %% @end
 api_get_request(Service, Path) ->
-    ?LOG_DEBUG("Invoking AWS request {Service: ~tp; Path: ~tp}...", [Service, Path]),
-    api_get_request_with_retries(Service, Path, ?MAX_RETRIES, ?LINEAR_BACK_OFF_MILLIS).
+    ?LOG_DEBUG("invoking AWS get request {Service: ~tp; Path: ~tp}...", [Service, Path]),
+    api_request_with_retries(Service, get, Path, "", [],
+                             ?MAX_RETRIES, ?LINEAR_BACK_OFF_MILLIS).
 
--spec api_get_request_with_retries(string(), path(), integer(), integer()) ->
+-spec api_post_request(
+    Service :: string(),
+    Path :: path(),
+    Body :: body(),
+    Headers :: headers()
+) -> result().
+%% @doc Perform a HTTP Post request to the AWS API for the specified service. The
+%%      response will automatically be decoded if it is either in JSON or XML
+%%      format.
+%% @end
+api_post_request(Service, Path, Body, Headers) ->
+    ?LOG_DEBUG("invoking AWS post request {Service: ~tp; Path: ~tp}...", [Service, Path]),
+    api_request_with_retries(Service, post, Path, Body, Headers,
+                             ?MAX_RETRIES, ?LINEAR_BACK_OFF_MILLIS).
+
+-spec api_request_with_retries(
+        Service :: string(),
+        Method :: method(),
+        Path :: path(),
+        Body :: body(),
+        Headers :: headers(),
+        Retries :: integer(),
+        WaitTime :: integer()) ->
     {'ok', list()} | {'error', term()}.
 %% @doc Invoke an API call to an AWS service with retries.
 %% @end
-api_get_request_with_retries(_, _, 0, _) ->
-    ?LOG_WARNING("Request to AWS service has failed after ~b retries", [?MAX_RETRIES]),
+api_request_with_retries(_, _, _, _, _, 0, _) ->
+    ?LOG_ERROR("Request to AWS service has failed after ~b retries", [?MAX_RETRIES]),
     {error, "AWS service is unavailable"};
-api_get_request_with_retries(Service, Path, Retries, WaitTimeBetweenRetries) ->
-    ensure_credentials_valid(),
-    case get(Service, Path) of
+api_request_with_retries(Service, Method, Path, Body, Headers, Retries, WaitTime) ->
+    ok = ensure_credentials_valid(),
+    case request(Service, Method, Path, Body, Headers) of
         {ok, {_Headers, Payload}} ->
             ?LOG_DEBUG("AWS request: ~ts~nResponse: ~tp", [Path, Payload]),
             {ok, Payload};
@@ -645,6 +686,6 @@ api_get_request_with_retries(Service, Path, Retries, WaitTimeBetweenRetries) ->
                     ok
             end,
             ?LOG_WARNING("Will retry AWS request, remaining retries: ~b", [Retries]),
-            timer:sleep(WaitTimeBetweenRetries),
-            api_get_request_with_retries(Service, Path, Retries - 1, WaitTimeBetweenRetries)
+            timer:sleep(WaitTime),
+            api_request_with_retries(Service, Method, Path, Body, Headers, Retries - 1, WaitTime)
     end.

deps/rabbitmq_aws/src/rabbitmq_aws.erl

@@ -134,7 +134,8 @@ credentials(Profile) ->
     lookup_credentials(
         Profile,
         os:getenv("AWS_ACCESS_KEY_ID"),
-        os:getenv("AWS_SECRET_ACCESS_KEY")
+        os:getenv("AWS_SECRET_ACCESS_KEY"),
+        os:getenv("AWS_SESSION_TOKEN")
     ).
 
 -spec region() -> {ok, string()}.
@@ -452,43 +453,53 @@ instance_id_url() ->
 -spec lookup_credentials(
     Profile :: string(),
     AccessKey :: string() | false,
-    SecretKey :: string() | false
+    SecretKey :: string() | false,
+    SessionToken :: string() | false
 ) ->
     security_credentials().
 %% @doc Return the access key and secret access key if they are set in
 %%      environment variables, otherwise lookup the credentials from the config
 %%      file for the specified profile.
 %% @end
-lookup_credentials(Profile, false, _) ->
+lookup_credentials(Profile, false, _, _) ->
     lookup_credentials_from_config(
         Profile,
         value(Profile, aws_access_key_id),
-        value(Profile, aws_secret_access_key)
+        value(Profile, aws_secret_access_key),
+        value(Profile, aws_session_token)
     );
-lookup_credentials(Profile, _, false) ->
+lookup_credentials(Profile, _, false, _) ->
     lookup_credentials_from_config(
         Profile,
         value(Profile, aws_access_key_id),
-        value(Profile, aws_secret_access_key)
+        value(Profile, aws_secret_access_key),
+        value(Profile, aws_session_token)
     );
-lookup_credentials(_, AccessKey, SecretKey) ->
-    {ok, AccessKey, SecretKey, undefined, undefined}.
+lookup_credentials(_, AccessKey, SecretKey, SessionToken) ->
+    case SessionToken of
+        false -> {ok, AccessKey, SecretKey, undefined, undefined};
+        SessionToken -> {ok, AccessKey, SecretKey, undefined, SessionToken}
+    end.
 
 -spec lookup_credentials_from_config(
     Profile :: string(),
     access_key() | {error, Reason :: atom()},
-    secret_access_key() | {error, Reason :: atom()}
+    secret_access_key() | {error, Reason :: atom()},
+    security_token() | {error, Reason :: atom()}
 ) ->
     security_credentials().
 %% @doc Return the access key and secret access key if they are set in
 %%      for the specified profile in the config file, if it exists. If it does
 %%      not exist or the profile is not set or the values are not set in the
 %%      profile, look up the values in the shared credentials file
 %% @end
-lookup_credentials_from_config(Profile, {error, _}, _) ->
+lookup_credentials_from_config(Profile, {error, _}, _, _) ->
     lookup_credentials_from_file(Profile, credentials_file_data());
-lookup_credentials_from_config(_, AccessKey, SecretKey) ->
-    {ok, AccessKey, SecretKey, undefined, undefined}.
+lookup_credentials_from_config(_, AccessKey, SecretKey, SessionToken) ->
+    case SessionToken of
+        {error, _} -> {ok, AccessKey, SecretKey, undefined, undefined};
+        SessionToken -> {ok, AccessKey, SecretKey, undefined, SessionToken}
+    end.
 
 -spec lookup_credentials_from_file(
     Profile :: string(),
@@ -518,22 +529,24 @@ lookup_credentials_from_section(undefined) ->
 lookup_credentials_from_section(Credentials) ->
     AccessKey = proplists:get_value(aws_access_key_id, Credentials, undefined),
     SecretKey = proplists:get_value(aws_secret_access_key, Credentials, undefined),
-    lookup_credentials_from_proplist(AccessKey, SecretKey).
+    SessionToken = proplists:get_value(aws_session_token, Credentials, undefined),
+    lookup_credentials_from_proplist(AccessKey, SecretKey, SessionToken).
 
 -spec lookup_credentials_from_proplist(
     AccessKey :: access_key(),
-    SecretAccessKey :: secret_access_key()
+    SecretAccessKey :: secret_access_key(),
+    SessionToken :: security_token()
 ) ->
     security_credentials().
 %% @doc Process the contents of the Credentials proplists checking if the
 %%      access key and secret access key are both set.
 %% @end
-lookup_credentials_from_proplist(undefined, _) ->
+lookup_credentials_from_proplist(undefined, _, _) ->
     lookup_credentials_from_instance_metadata();
-lookup_credentials_from_proplist(_, undefined) ->
+lookup_credentials_from_proplist(_, undefined, _) ->
     lookup_credentials_from_instance_metadata();
-lookup_credentials_from_proplist(AccessKey, SecretKey) ->
-    {ok, AccessKey, SecretKey, undefined, undefined}.
+lookup_credentials_from_proplist(AccessKey, SecretKey, SessionToken) ->
+    {ok, AccessKey, SecretKey, undefined, SessionToken}.
 
 -spec lookup_credentials_from_instance_metadata() ->
     security_credentials().
@@ -773,7 +786,7 @@ load_imdsv2_token() ->
 %% @doc Return headers used for instance metadata service requests.
 %% @end
 instance_metadata_request_headers() ->
-    case application:get_env(rabbit, aws_prefer_imdsv2) of
+    case application:get_env(rabbitmq_aws, aws_prefer_imdsv2) of
         {ok, false} ->
             [];
         %% undefined or {ok, true}

deps/rabbitmq_aws/src/rabbitmq_aws_config.erl

@@ -30,7 +30,7 @@ headers(Request) ->
     {_, Host, _} = URI#uri.authority,
     Headers = append_headers(
         RequestTimestamp,
-        length(Request#request.body),
+        get_content_length(Request),
         PayloadHash,
         Host,
         Request#request.security_token,
@@ -313,3 +313,9 @@ string_to_sign(RequestTimestamp, RequestDate, Region, Service, RequestHash) ->
 %% @end
 sort_headers(Headers) ->
     lists:sort(fun({A, _}, {B, _}) -> string:to_lower(A) =< string:to_lower(B) end, Headers).
+
+-spec get_content_length(Request :: #request{}) -> non_neg_integer().
+get_content_length(#request{body = Body}) when is_binary(Body) ->
+    byte_size(Body);
+get_content_length(#request{body = Body}) when is_list(Body) ->
+    length(Body).