Adds support for AssumeRoleWebIdentityCredentials

We wanted to run this plugin on EKS. Instead of acting like kiam or
kube2iam, EKS creates a web token identity file within a container
that allows the application to assume roles -- but using the newer
credential provider.

This commit adds a web_identity_credentials config section and handler
so we can use the identity file within EKS.

Signed-off-by: Jack Wink <[email protected]>

Author: mothershipper

README.md

@@ -93,6 +93,51 @@ is set to 3600 seconds.
 A unique identifier that is used by third parties when assuming roles in
 their customers' accounts.
 
+### web_identity_credentials
+
+Similar to the assume_role_credentials, but for usage in EKS.
+
+    <match *>
+      @type s3
+
+      <web_identity_credentials>
+        role_arn          ROLE_ARN
+        role_session_name ROLE_SESSION_NAME
+        web_identity_token_file AWS_WEB_IDENTITY_TOKEN_FILE
+      </web_identity_credentials>
+    </match>
+
+See also:
+
+*   [Using IAM Roles - AWS Identity and Access
+    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
+*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
+
+**role_arn (required)**
+
+The Amazon Resource Name (ARN) of the role to assume.
+
+**role_session_name (required)**
+
+An identifier for the assumed role session.
+
+**web_identity_token_file (required)**
+
+The absolute path to the file on disk containing the OIDC token
+
+**policy**
+
+An IAM policy in JSON format.
+
+**duration_seconds**
+
+The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds.
+
+
 ### instance_profile_credentials
 
 Retrieve temporary security credentials via HTTP request. This is useful on

lib/fluent/plugin/in_s3.rb

@@ -42,6 +42,20 @@ def initialize
       desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
       config_param :external_id, :string, default: nil
     end
+    # See the following link for additional params that could be added:
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
+    config_section :web_identity_credentials, multi: false do
+      desc "The Amazon Resource Name (ARN) of the role to assume"
+      config_param :role_arn, :string # required
+      desc "An identifier for the assumed role session"
+      config_param :role_session_name, :string #required
+      desc "The absolute path to the file on disk containing the OIDC token"
+      config_param :web_identity_token_file, :string, default: nil #required
+      desc "An IAM policy in JSON format"
+      config_param :policy, :string, default: nil
+      desc "The duration, in seconds, of the role session (900-43200)"
+      config_param :duration_seconds, :integer, default: nil
+    end
     config_section :instance_profile_credentials, multi: false do
       desc "Number of times to retry when retrieving credentials"
       config_param :retries, :integer, default: nil
@@ -202,6 +216,19 @@ def setup_credentials
           credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
         end
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      when @web_identity_credentials
+        c = @web_identity_credentials
+        credentials_options[:role_arn] = c.role_arn
+        credentials_options[:role_session_name] = c.role_session_name
+        credentials_options[:web_identity_token_file] = c.web_identity_token_file
+        credentials_options[:policy] = c.policy if c.policy
+        # TODO: Validation or clamping? If we go above 12 hours, the request will fail...
+        #       Is it better behavior to hard fail, or just provide the max session duration?
+        credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+        if @s3_region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
+        end
+        options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       when @instance_profile_credentials
         c = @instance_profile_credentials
         credentials_options[:retries] = c.retries if c.retries

lib/fluent/plugin/out_s3.rb

@@ -40,6 +40,20 @@ def initialize
       desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
       config_param :external_id, :string, default: nil, secret: true
     end
+    # See the following link for additional params that could be added:
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
+    config_section :web_identity_credentials, multi: false do
+      desc "The Amazon Resource Name (ARN) of the role to assume"
+      config_param :role_arn, :string # required
+      desc "An identifier for the assumed role session"
+      config_param :role_session_name, :string #required
+      desc "The absolute path to the file on disk containing the OIDC token"
+      config_param :web_identity_token_file, :string, default: nil #required
+      desc "An IAM policy in JSON format"
+      config_param :policy, :string, default: nil
+      desc "The duration, in seconds, of the role session (900-43200)"
+      config_param :duration_seconds, :integer, default: nil
+    end
     config_section :instance_profile_credentials, multi: false do
       desc "Number of times to retry when retrieving credentials"
       config_param :retries, :integer, default: nil
@@ -175,7 +189,7 @@ def configure(conf)
       end
 
       unless @index_format =~ /^%(0\d*)?[dxX]$/
-        raise Fluent::ConfigError, "index_format parameter should follow `%[flags][width]type`. `0` is the only supported flag, and is mandatory if width is specified. `d`, `x` and `X` are supported types" 
+        raise Fluent::ConfigError, "index_format parameter should follow `%[flags][width]type`. `0` is the only supported flag, and is mandatory if width is specified. `d`, `x` and `X` are supported types"
       end
 
       if @reduced_redundancy
@@ -460,6 +474,19 @@ def setup_credentials
           credentials_options[:client] = Aws::STS::Client.new(region: @s3_region)
         end
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      when @web_identity_credentials
+        c = @web_identity_credentials
+        credentials_options[:role_arn] = c.role_arn
+        credentials_options[:role_session_name] = c.role_session_name
+        credentials_options[:web_identity_token_file] = c.web_identity_token_file
+        credentials_options[:policy] = c.policy if c.policy
+        # TODO: Validation or clamping? If we go above 12 hours, the request will fail...
+        #       Is it better behavior to hard fail, or just provide the max session duration?
+        credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+        if @s3_region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
+        end
+        options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       when @instance_profile_credentials
         c = @instance_profile_credentials
         credentials_options[:retries] = c.retries if c.retries

test/test_out_s3.rb

@@ -567,6 +567,32 @@ def test_assume_role_credentials_with_region
     assert_equal(expected_credentials, credentials)
   end
 
+  def test_web_identity_credentials
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    mock(Aws::AssumeRoleWebIdentityCredentials).new(
+      role_arn: "test_arn",
+      role_session_name: "test_session",
+      web_identity_token_file: "test_file",
+      client: anything
+    ){
+      expected_credentials
+    }
+
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <web_identity_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        web_identity_token_file test_file
+      </web_identity_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
   def test_instance_profile_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     mock(Aws::InstanceProfileCredentials).new({}).returns(expected_credentials)