s3: migrate to aws-sdk-go-v2

.github/workflows/go-getter.yml

@@ -12,8 +12,7 @@ jobs:
     strategy:
       matrix:
         go-version:
-          - 1.18
-          - 1.19
+          - 1.21
     permissions:
       id-token: write
       contents: read
@@ -81,7 +80,7 @@ jobs:
 
       # Save coverage report parts
       - name: Upload and save artifacts
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: linux test results
           path: linux_cov.part
@@ -91,8 +90,7 @@ jobs:
     strategy:
       matrix:
         go-version:
-          - 1.18
-          - 1.19
+          - 1.21
     permissions:
       id-token: write
       contents: read
@@ -150,7 +148,7 @@ jobs:
 
       # Save coverage report parts
       - name: Upload and save artifacts
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: windows test results
           path: win_cov.part

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
         with:
-          go-version: '^1.15'
+          go-version: '^1.21'
       - name: Setup signore
         uses: hashicorp/setup-signore@v1
         with:

get_s3.go

@@ -12,12 +12,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 )
 
 // S3Getter is a Getter implementation that will download a module from
@@ -48,17 +46,17 @@ func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
 	}
 
 	// Create client config
-	client, err := g.newS3Client(region, u, creds)
+	client, err := g.newS3Client(ctx, region, u, creds)
 	if err != nil {
 		return 0, err
 	}
 
 	// List the object(s) at the given prefix
-	req := &s3.ListObjectsInput{
+	req := &s3.ListObjectsV2Input{
 		Bucket: aws.String(bucket),
 		Prefix: aws.String(path),
 	}
-	resp, err := client.ListObjectsWithContext(ctx, req)
+	resp, err := client.ListObjectsV2(ctx, req)
 	if err != nil {
 		return 0, err
 	}
@@ -113,34 +111,35 @@ func (g *S3Getter) Get(dst string, u *url.URL) error {
 		return err
 	}
 
-	client, err := g.newS3Client(region, u, creds)
+	client, err := g.newS3Client(ctx, region, u, creds)
 	if err != nil {
 		return err
 	}
 
 	// List files in path, keep listing until no more objects are found
-	lastMarker := ""
+	continuationToken := ""
 	hasMore := true
 	for hasMore {
-		req := &s3.ListObjectsInput{
+		req := &s3.ListObjectsV2Input{
 			Bucket: aws.String(bucket),
 			Prefix: aws.String(path),
 		}
-		if lastMarker != "" {
-			req.Marker = aws.String(lastMarker)
+
+		if continuationToken != "" {
+			req.ContinuationToken = aws.String(continuationToken)
 		}
 
-		resp, err := client.ListObjectsWithContext(ctx, req)
+		resp, err := client.ListObjectsV2(ctx, req)
 		if err != nil {
 			return err
 		}
 
-		hasMore = aws.BoolValue(resp.IsTruncated)
+		hasMore = aws.ToBool(resp.IsTruncated)
 
 		// Get each object storing each file relative to the destination path
 		for _, object := range resp.Contents {
-			lastMarker = aws.StringValue(object.Key)
-			objPath := aws.StringValue(object.Key)
+			continuationToken = aws.ToString(resp.NextContinuationToken)
+			objPath := aws.ToString(object.Key)
 
 			// If the key ends with a backslash assume it is a directory and ignore
 			if strings.HasSuffix(objPath, "/") {
@@ -177,15 +176,15 @@ func (g *S3Getter) GetFile(dst string, u *url.URL) error {
 		return err
 	}
 
-	client, err := g.newS3Client(region, u, creds)
+	client, err := g.newS3Client(ctx, region, u, creds)
 	if err != nil {
 		return err
 	}
 
 	return g.getObject(ctx, client, dst, bucket, path, version)
 }
 
-func (g *S3Getter) getObject(ctx context.Context, client *s3.S3, dst, bucket, key, version string) error {
+func (g *S3Getter) getObject(ctx context.Context, client *s3.Client, dst, bucket, key, version string) error {
 	req := &s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
@@ -194,7 +193,7 @@ func (g *S3Getter) getObject(ctx context.Context, client *s3.S3, dst, bucket, ke
 		req.VersionId = aws.String(version)
 	}
 
-	resp, err := client.GetObjectWithContext(ctx, req)
+	resp, err := client.GetObject(ctx, req)
 	if err != nil {
 		return err
 	}
@@ -216,39 +215,7 @@ func (g *S3Getter) getObject(ctx context.Context, client *s3.S3, dst, bucket, ke
 	return copyReader(dst, body, 0666, g.client.umask(), 0)
 }
 
-func (g *S3Getter) getAWSConfig(region string, url *url.URL, creds *credentials.Credentials) *aws.Config {
-	conf := &aws.Config{}
-	metadataURLOverride := os.Getenv("AWS_METADATA_URL")
-	if creds == nil && metadataURLOverride != "" {
-		creds = credentials.NewChainCredentials(
-			[]credentials.Provider{
-				&credentials.EnvProvider{},
-				&credentials.SharedCredentialsProvider{Filename: "", Profile: ""},
-				&ec2rolecreds.EC2RoleProvider{
-					Client: ec2metadata.New(session.New(&aws.Config{
-						Endpoint: aws.String(metadataURLOverride),
-					})),
-				},
-			})
-	}
-
-	if creds != nil {
-		conf.Endpoint = &url.Host
-		conf.S3ForcePathStyle = aws.Bool(true)
-		if url.Scheme == "http" {
-			conf.DisableSSL = aws.Bool(true)
-		}
-	}
-
-	conf.Credentials = creds
-	if region != "" {
-		conf.Region = aws.String(region)
-	}
-
-	return conf.WithCredentialsChainVerboseErrors(true)
-}
-
-func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, creds *credentials.Credentials, err error) {
+func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, creds *credentials.StaticCredentialsProvider, err error) {
 	// This just check whether we are dealing with S3 or
 	// any other S3 compliant service. S3 has a predictable
 	// url as others do not
@@ -329,34 +296,57 @@ func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, c
 	_, hasAwsSecret := u.Query()["aws_access_key_secret"]
 	_, hasAwsToken := u.Query()["aws_access_token"]
 	if hasAwsId || hasAwsSecret || hasAwsToken {
-		creds = credentials.NewStaticCredentials(
+		credentialProvider := credentials.NewStaticCredentialsProvider(
 			u.Query().Get("aws_access_key_id"),
 			u.Query().Get("aws_access_key_secret"),
 			u.Query().Get("aws_access_token"),
 		)
+		creds = &credentialProvider
 	}
 
 	return
 }
 
-func (g *S3Getter) newS3Client(
-	region string, url *url.URL, creds *credentials.Credentials,
-) (*s3.S3, error) {
-	var sess *session.Session
-
-	if profile := url.Query().Get("aws_profile"); profile != "" {
-		var err error
-		sess, err = session.NewSessionWithOptions(session.Options{
-			Profile:           profile,
-			SharedConfigState: session.SharedConfigEnable,
-		})
+func (g *S3Getter) newS3Client(ctx context.Context, region string, url *url.URL, creds *credentials.StaticCredentialsProvider) (*s3.Client, error) {
+	var cfg aws.Config
+	var err error
+
+	// We first check if the AWS_METADATA_URL is set, if it is we use it to load the config.
+	if creds == nil && os.Getenv("AWS_METADATA_URL") != "" {
+		cfg, err = config.LoadDefaultConfig(
+			ctx,
+			config.WithRegion(region),
+			config.WithEC2IMDSEndpoint(os.Getenv("AWS_METADATA_URL")),
+		)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to load AWS config with metadata URL override: %w", err)
 		}
-	} else {
-		config := g.getAWSConfig(region, url, creds)
-		sess = session.New(config)
+		return s3.NewFromConfig(cfg), nil
+	}
+
+	// Otherwise, we load the config with the default settings.
+	cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region))
+	if err != nil {
+		return nil, fmt.Errorf("failed to load AWS config: %w", err)
+	}
+
+	// If a credential provider is provided, override the default credentials;
+	// otherwise, let the SDK automatically resolve them.
+	if creds != nil {
+		cfg.Credentials = creds
 	}
 
-	return s3.New(sess), nil
+	// Create the client
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		if creds != nil {
+			o.UsePathStyle = true
+			o.BaseEndpoint = &url.Host
+
+			if url.Scheme == "http" {
+				o.EndpointOptions.DisableHTTPS = true
+			}
+		}
+	})
+
+	return client, nil
 }

go.mod

@@ -1,27 +1,64 @@
 module github.com/hashicorp/go-getter
 
+go 1.21
+
 require (
 	cloud.google.com/go/storage v1.29.0
 	github.com/aws/aws-sdk-go v1.44.122
+	github.com/aws/aws-sdk-go-v2 v1.36.1
+	github.com/aws/aws-sdk-go-v2/config v1.29.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.59
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
 	github.com/cheggaaa/pb v1.0.27
-	github.com/fatih/color v1.7.0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-safetemp v1.0.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/klauspost/compress v1.15.11
-	github.com/mattn/go-colorable v0.0.9 // indirect
-	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/go-testing-interface v1.14.1
 	github.com/ulikunitz/xz v0.5.10
-	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/oauth2 v0.7.0
 	google.golang.org/api v0.114.0
+)
+
+require (
+	cloud.google.com/go v0.110.0 // indirect
+	cloud.google.com/go/compute v1.19.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v0.13.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
+	github.com/fatih/color v1.7.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.1 // indirect
+	github.com/mattn/go-colorable v0.0.9 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-runewidth v0.0.4 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/grpc v1.56.3 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect
 )
-
-go 1.13