Reduce initial memory usage by auto-loading bundled gems (#3083)

Author: alextwoods

gems/aws-sdk-core/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Issue - Reduce initial memory usage by auto-loading bundled gems (STS, SSO, SSOOIDC).
+
 3.201.4 (2024-08-08)
 ------------------
 

gems/aws-sdk-core/lib/aws-sdk-core.rb

@@ -109,15 +109,7 @@
 # defaults
 require_relative 'aws-defaults'
 
-# plugins
-# loaded through building STS or SSO ..
-
-# aws-sdk-sts is included to support Aws::AssumeRoleCredentials
-require_relative 'aws-sdk-sts'
-
-# aws-sdk-sso is included to support Aws::SSOCredentials
-require_relative 'aws-sdk-sso'
-require_relative 'aws-sdk-ssooidc'
+# plugins - loaded through service clients as needed.
 
 module Aws
 
@@ -205,3 +197,14 @@ def eager_autoload!(*args)
 
   end
 end
+
+# Autoload bundled service gems used in credential providers
+# need to ensure that the files are the local files from aws-sdk-core
+# and not the installed, legacy/dummy service gems.
+
+# aws-sdk-sts is included to support Aws::AssumeRoleCredentials
+Aws.autoload :STS, File.join(__dir__, 'aws-sdk-sts.rb')
+
+# aws-sdk-sso is included to support Aws::SSOCredentials
+Aws.autoload :SSO, File.join(__dir__, 'aws-sdk-sso.rb')
+Aws.autoload :SSOOIDC, File.join(__dir__, 'aws-sdk-ssooidc.rb')

gems/aws-sdk-resources/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Issue - Remove autoloads for service gems bundled in `aws-sdk-core` (STS, SSO, SSOOIDC).
+
 3.200.0 (2024-08-01)
 ------------------
 

gems/aws-sdk-resources/bin/aws-v3.rb

@@ -106,11 +106,13 @@ def env_bool key, default
 # The Aws namespace used to check autoload requires aws-sdk-resources.
 require 'aws-sdk-resources'
 
-# Finally load all of the gems. Core is loaded a second time because of STS.
+# Finally load all of the gems.
+# Don't update the load path for gems bundled in core.
+core_gems = [:STS, :SSO, :SSOOIDC]
 if File.directory?(File.expand_path('../../../../build_tools', __FILE__))
   gems = []
   Aws.constants.each do |const_name|
-    if Aws.autoload?(const_name)
+    if Aws.autoload?(const_name) && !core_gems.include?(const_name)
       gems << "aws-sdk-#{const_name.downcase}"
     end
   end

tasks/build.rake

@@ -28,6 +28,7 @@ end
 # It is need to provide session credentials and assume role support.
 # Only building source, but not gemspecs, version file, etc.
 task 'build:aws-sdk-sts' do
+  Aws::STS if Aws.autoload?(:STS) # force autoload from core
   sts = BuildTools::Services.service('sts')
   generator = AwsSdkCodeGenerator::CodeBuilder.new(
     aws_sdk_core_lib_path: $CORE_LIB,
@@ -48,6 +49,7 @@ end
 # It is need to provide SSO Credentials.
 # Only building source, but not gemspecs, version file, etc.
 task 'build:aws-sdk-sso' do
+  Aws::SSO if Aws.autoload?(:SSO) # force autoload from core
   sso = BuildTools::Services.service('sso')
   generator = AwsSdkCodeGenerator::CodeBuilder.new(
     aws_sdk_core_lib_path: $CORE_LIB,
@@ -66,6 +68,7 @@ end
 # Aws::SSOOIDC is generated directly into the `aws-sdk-core` gem.
 # Only building source, but not gemspecs, version file, etc.
 task 'build:aws-sdk-ssooidc' do
+  Aws::SSOOIDC if Aws.autoload?(:SSOOIDC) # force autoload from core
   ssooidc = BuildTools::Services.service('ssooidc')
   generator = AwsSdkCodeGenerator::CodeBuilder.new(
     aws_sdk_core_lib_path: $CORE_LIB,

tasks/update-aws-sdk-dependencies.rake

@@ -19,11 +19,13 @@ task 'update-aws-sdk-dependencies' do
   )
 
   # update the module autoloads
+  # Remove service gems bundled in core (eg, sts) that have gem_name of aws-sdk-core.
+  autoload_services = BuildTools::Services.select { |svc| svc.gem_name != 'aws-sdk-core' }
   BuildTools.replace_lines(
     filename: "#{$GEMS_DIR}/aws-sdk-resources/lib/aws-sdk-resources.rb",
     start: /# service gems/,
     stop: /# end service gems/,
-    new_lines: BuildTools::Services.map { |service|
+    new_lines: autoload_services.map { |service|
       "  autoload :#{service.name}, '#{service.gem_name}'\n"
     }
   )