S3 Express (#2959)

Author: mullermp

benchmark/gems/s3.rb

@@ -50,6 +50,19 @@ def operation_benchmarks
               client.put_object(req)
             end
           },
+          put_object_multipart_large: {
+            n: 150,
+            setup: proc do |client|
+              {multipart_threshold: 5 * 1024 * 1024}
+            end,
+            test: proc do |client, req|
+              resource = Aws::S3::Resource.new(client: client)
+              object = resource.bucket('bucket').object('key')
+              tempfile = Tempfile.new('put_object_multipart_large')
+              tempfile << '.' * 1024*1024*10
+              object.upload_file(tempfile, **req)
+            end
+          },
           head_object: {
             setup: proc do |client|
               {bucket: 'bucket', key: 'key'}
@@ -58,6 +71,55 @@ def operation_benchmarks
               client.head_object(req)
             end
           },
+          s3_express_get_object_small: {
+            setup: proc do |client|
+              client.stub_responses(:get_object, [{body: "." * 128}])
+              {bucket: 'bucket--usw2-az2-d-s3', key: 'key'}
+            end,
+            test: proc do |client, req|
+              client.get_object(req)
+            end
+          },
+          s3_express_get_object_large: {
+            n: 150,
+            setup: proc do |client|
+              client.stub_responses(:get_object, [{body: "." * 1024*1024*10}]) # 10 MB
+              {bucket: 'bucket--usw2-az2-d-s3', key: 'key'}
+            end,
+            test: proc do |client, req|
+              client.get_object(req)
+            end
+          },
+          s3_express_put_object_small: {
+            setup: proc do |client|
+              {bucket: 'bucket--usw2-az2-d-s3', key: 'key', body: '.' * 128}
+            end,
+            test: proc do |client, req|
+              client.put_object(req)
+            end
+          },
+          s3_express_put_object_large: {
+            n: 150,
+            setup: proc do |client|
+              {bucket: 'bucket--usw2-az2-d-s3', key: 'key', body: "." * 1024*1024*10}
+            end,
+            test: proc do |client, req|
+              client.put_object(req)
+            end
+          },
+          s3_express_put_object_multipart_large: {
+            n: 150,
+            setup: proc do |client|
+              {multipart_threshold: 5 * 1024 * 1024}
+            end,
+            test: proc do |client, req|
+              resource = Aws::S3::Resource.new(client: client)
+              object = resource.bucket('bucket--usw2-az2-d-s3').object('key')
+              tempfile = Tempfile.new('s3_express_put_object_multipart_large')
+              tempfile << '.' * 1024*1024*10
+              object.upload_file(tempfile, **req)
+            end
+          }
         }
       end
 

build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/spec/endpoint_provider_spec_class.rb

@@ -90,6 +90,10 @@ def expect_auth?
               !@expect['endpoint']['properties']['authSchemes'].empty?
           end
 
+          def s3_express_auth?
+            expect_auth? && expected_auth['name'] == 'sigv4-s3express'
+          end
+
           def expected_headers
             @expect['endpoint']['headers'].map { |k,v| Param.new(k, v.join(",")) }
           end

build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache

@@ -29,16 +29,17 @@ module {{module_name}}
       # @api private
       class Handler < Seahorse::Client::Handler
         def call(context)
-          # If endpoint was discovered, do not resolve or apply the endpoint.
           unless context[:discovered_endpoint]
             params = parameters_for_operation(context)
             endpoint = context.config.endpoint_provider.resolve_endpoint(params)
 
             context.http_request.endpoint = endpoint.url
             apply_endpoint_headers(context, endpoint.headers)
+
+            context[:endpoint_params] = params
+            context[:endpoint_properties] = endpoint.properties
           end
 
-          context[:endpoint_params] = params
           context[:auth_scheme] =
             Aws::Endpoints.resolve_auth_scheme(context, endpoint)
 

build_tools/aws-sdk-code-generator/templates/spec/endpoint_provider_spec_class.mustache

@@ -50,6 +50,16 @@ module {{module_name}}
         {{/expect_error?}}
         {{^expect_error?}}
         {{#expect_auth?}}
+        {{#s3_express_auth?}}
+        client.stub_responses(:create_session, credentials: {
+          access_key_id: 's3-akid',
+          secret_access_key: 's3-secret',
+          session_token: 's3-session',
+          expiration: Time.now + 60 * 5
+        })
+        expect_auth({"name"=>"sigv4", "signingName"=>"s3express"})
+        EXPRESS_CREDENTIALS_CACHE.clear
+        {{/s3_express_auth?}}
         expect_auth({{{expected_auth}}})
         {{/expect_auth?}}
         resp = client.{{operation_name}}(

build_tools/services.rb

@@ -13,7 +13,7 @@ class ServiceEnumerator
     MINIMUM_CORE_VERSION = "3.188.0"
 
     # Minimum `aws-sdk-core` version for new S3 gem builds
-    MINIMUM_CORE_VERSION_S3 = "3.188.0"
+    MINIMUM_CORE_VERSION_S3 = "3.189.0"
 
     EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration"
 

gems/aws-sdk-core/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - Support S3 Express authentication.
+
 3.188.0 (2023-11-22)
 ------------------
 

gems/aws-sdk-core/aws-sdk-core.gemspec

@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency('jmespath', '~> 1', '>= 1.6.1') # necessary for secure jmespath JSON parsing
   spec.add_dependency('aws-partitions', '~> 1', '>= 1.651.0') # necessary for new endpoint resolution
-  spec.add_dependency('aws-sigv4', '~> 1.5') # necessary for making Aws::STS, SSO, SSOOIDC API calls
-  spec.add_dependency('aws-eventstream', '~> 1', '>= 1.0.2') # necessary for binary eventstream
+  spec.add_dependency('aws-sigv4', '~> 1.8') # necessary for s3 express auth
+  spec.add_dependency('aws-eventstream', '~> 1', '>= 1.3.0') # necessary for binary eventstream
 
   spec.metadata = {
     'source_code_uri' => 'https://github.com/aws/aws-sdk-ruby/tree/version-3/gems/aws-sdk-core',

gems/aws-sdk-core/lib/aws-sdk-core/endpoints.rb

@@ -53,7 +53,7 @@ def default_auth_scheme(context)
       end
 
       def merge_signing_defaults(auth_scheme, config)
-        if %w[sigv4 sigv4a].include?(auth_scheme['name'])
+        if %w[sigv4 sigv4a sigv4-s3express].include?(auth_scheme['name'])
           auth_scheme['signingName'] ||= sigv4_name(config)
           if auth_scheme['name'] == 'sigv4a'
             auth_scheme['signingRegionSet'] ||= ['*']

gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb

@@ -117,7 +117,8 @@ class ChecksumHandler < Seahorse::Client::Handler
 
         def call(context)
           if should_calculate_request_checksum?(context)
-            request_algorithm_input = ChecksumAlgorithm.request_algorithm_selection(context)
+            request_algorithm_input = ChecksumAlgorithm.request_algorithm_selection(context) ||
+                                      context[:default_request_checksum_algorithm]
             context[:checksum_algorithms] = request_algorithm_input
 
             request_checksum_property = {
@@ -140,7 +141,8 @@ def call(context)
 
         def should_calculate_request_checksum?(context)
           context.operation.http_checksum &&
-            ChecksumAlgorithm.request_algorithm_selection(context)
+            (ChecksumAlgorithm.request_algorithm_selection(context) ||
+              context[:default_request_checksum_algorithm])
         end
 
         def should_verify_response_checksum?(context)

gems/aws-sdk-core/lib/aws-sdk-core/plugins/http_checksum.rb

@@ -12,7 +12,8 @@ class Handler < Seahorse::Client::Handler
 
         def call(context)
           if checksum_required?(context) &&
-             !context[:checksum_algorithms] # skip in favor of flexible checksum
+             !context[:checksum_algorithms] && # skip in favor of flexible checksum
+             !context[:s3_express_endpoint] # s3 express endpoints do not support md5
             body = context.http_request.body
             context.http_request.headers['Content-Md5'] ||= md5(body)
           end