Add support for trailing checksum (#1600)

Updates the SDK adding support Amazon S3 for trailing checksums computed
with the aws-chunked content-encoding format. Adds a new internal
service module, `checksum` to the SDK that API models decorated with the
`httpChecksum` smithy trail will use.

Co-authored-by: skotambkar <[email protected]>

Author: jasdel

Makefile

@@ -1,5 +1,6 @@
 # Lint rules to ignore
 LINTIGNORESINGLEFIGHT='internal/sync/singleflight/singleflight.go:.+error should be the last type'
+LINT_IGNORE_S3MANAGER_INPUT='feature/s3/manager/upload.go:.+struct field SSEKMSKeyId should be SSEKMSKeyID'
 
 UNIT_TEST_TAGS=
 BUILD_TAGS=-tags "example,codegen,integration,ec2env,perftest"
@@ -447,6 +448,7 @@ lint:
 	@echo "go lint SDK and vendor packages"
 	@lint=`golint ./...`; \
 	dolint=`echo "$$lint" | grep -E -v \
+	-e ${LINT_IGNORE_S3MANAGER_INPUT} \
 	-e ${LINTIGNORESINGLEFIGHT}`; \
 	echo "$$dolint"; \
 	if [ "$$dolint" != "" ]; then exit 1; fi

codegen/sdk-codegen/aws-models/s3.2006-03-01.json

@@ -47,6 +47,8 @@ public class AwsGoDependency {
     public static final GoDependency SERVICE_INTERNAL_ENDPOINT_DISCOVERY = awsModuleDep("service/internal/endpoint-discovery",
             null, Versions.SERVICE_INTERNAL_ENDPOINT_DISCOVERY, "internalEndpointDiscovery");
     public static final GoDependency AWS_DEFAULTS = aws("aws/defaults");
+    public static final GoDependency SERVICE_INTERNAL_CHECKSUM = awsModuleDep("service/internal/checksum",
+            null, Versions.SERVICE_INTERNAL_CHECKSUM, "internalChecksum");
 
     public static final GoDependency REGEXP = SmithyGoDependency.stdlib("regexp");
 
@@ -99,5 +101,6 @@ private static final class Versions {
         private static final String SERVICE_INTERNAL_ENDPOINT_DISCOVERY = "v0.0.0-00010101000000-000000000000";
         private static final String INTERNAL_ENDPOINTS_V2 = "v2.0.0-00010101000000-000000000000";
         private static final String AWS_PROTOCOL_EVENTSTREAM = "v0.0.0-00010101000000-000000000000";
+        private static final String SERVICE_INTERNAL_CHECKSUM = "v0.0.0-00010101000000-000000000000";
     }
 }

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsGoDependency.java

@@ -1,6 +1,7 @@
 software.amazon.smithy.aws.go.codegen.AddProtocols
 software.amazon.smithy.aws.go.codegen.ClientResolvedDefaultsMode
 software.amazon.smithy.aws.go.codegen.AddAwsConfigFields
+software.amazon.smithy.aws.go.codegen.AwsHttpChecksumGenerator
 software.amazon.smithy.aws.go.codegen.RegisterServiceMetadataMiddleware
 software.amazon.smithy.aws.go.codegen.AwsEventStreamIntegration
 software.amazon.smithy.aws.go.codegen.AssembleMiddlewareStack

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java

@@ -25,6 +25,8 @@ replace github.com/aws/aws-sdk-go-v2/internal/ini => ../../../../internal/ini/
 
 replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../../service/internal/accept-encoding/
 
+replace github.com/aws/aws-sdk-go-v2/service/internal/checksum => ../../../../service/internal/checksum/
+
 replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../../service/internal/presigned-url/
 
 replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../../service/internal/s3shared/

codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration

@@ -27,6 +27,8 @@ replace github.com/aws/aws-sdk-go-v2/internal/ini => ../../../../internal/ini/
 
 replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../../service/internal/accept-encoding/
 
+replace github.com/aws/aws-sdk-go-v2/service/internal/checksum => ../../../../service/internal/checksum/
+
 replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../../service/internal/presigned-url/
 
 replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../../service/internal/s3shared/

example/service/s3/listObjects/go.mod

@@ -348,16 +348,16 @@ func (d *downloader) downloadRange(rng string) {
 
 // downloadChunk downloads the chunk from s3
 func (d *downloader) downloadChunk(chunk dlchunk) error {
-	in := &s3.GetObjectInput{}
-	awsutil.Copy(in, d.in)
+	var params s3.GetObjectInput
+	awsutil.Copy(&params, d.in)
 
 	// Get the next byte range of data
-	in.Range = aws.String(chunk.ByteRange())
+	params.Range = aws.String(chunk.ByteRange())
 
 	var n int64
 	var err error
 	for retry := 0; retry <= d.partBodyMaxRetries; retry++ {
-		n, err = d.tryDownloadChunk(in, &chunk)
+		n, err = d.tryDownloadChunk(&params, &chunk)
 		if err == nil {
 			break
 		}
@@ -374,23 +374,24 @@ func (d *downloader) downloadChunk(chunk dlchunk) error {
 
 		chunk.cur = 0
 
-		d.cfg.Logger.Logf(logging.Debug, "object part body download interrupted %s, err, %v, retrying attempt %d",
-			aws.ToString(in.Key), err, retry)
+		d.cfg.Logger.Logf(logging.Debug,
+			"object part body download interrupted %s, err, %v, retrying attempt %d",
+			aws.ToString(params.Key), err, retry)
 	}
 
 	d.incrWritten(n)
 
 	return err
 }
 
-func (d *downloader) tryDownloadChunk(in *s3.GetObjectInput, w io.Writer) (int64, error) {
+func (d *downloader) tryDownloadChunk(params *s3.GetObjectInput, w io.Writer) (int64, error) {
 	cleanup := func() {}
 	if d.cfg.BufferProvider != nil {
 		w, cleanup = d.cfg.BufferProvider.GetReadFrom(w)
 	}
 	defer cleanup()
 
-	resp, err := d.cfg.S3.GetObject(d.ctx, in, d.cfg.ClientOptions...)
+	resp, err := d.cfg.S3.GetObject(d.ctx, params, d.cfg.ClientOptions...)
 	if err != nil {
 		return 0, err
 	}

example/service/s3/usingPrivateLink/go.mod

@@ -28,6 +28,8 @@ replace github.com/aws/aws-sdk-go-v2/internal/ini => ../../../internal/ini/
 
 replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../service/internal/accept-encoding/
 
+replace github.com/aws/aws-sdk-go-v2/service/internal/checksum => ../../../service/internal/checksum/
+
 replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../service/internal/presigned-url/
 
 replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../service/internal/s3shared/

feature/s3/manager/download.go

@@ -17,6 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager/internal/integration"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 var integConfig aws.Config
@@ -92,7 +93,11 @@ func (d dlwriter) WriteAt(p []byte, pos int64) (n int, err error) {
 
 func validate(t *testing.T, key string, md5value string) {
 	mgr := manager.NewDownloader(client)
-	params := &s3.GetObjectInput{Bucket: bucketName, Key: &key}
+	params := &s3.GetObjectInput{
+		Bucket:       bucketName,
+		Key:          &key,
+		ChecksumMode: s3types.ChecksumModeEnabled,
+	}
 
 	w := newDLWriter(1024 * 1024 * 20)
 	n, err := mgr.Download(context.Background(), w, params)