diff --git a/feature/s3/manager/upload.go b/feature/s3/manager/upload.go index 93133aee585..f55c0414d44 100644 --- a/feature/s3/manager/upload.go +++ b/feature/s3/manager/upload.go @@ -245,6 +245,21 @@ type Uploader struct { // Defines the buffer strategy used when uploading a part BufferProvider ReadSeekerWriteToProvider + // RequestChecksumCalculation determines when request checksum calculation is performed + // for multipart uploads. + // + // There are two possible values for this setting: + // + // 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated + // if the operation supports it, regardless of whether the user sets an algorithm in the request. + // + // 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user + // explicitly sets a checksum algorithm in the request. This preserves backwards compatibility + // for applications that don't want automatic checksum calculation. + // + // Note: S3 Express buckets always require CRC32 checksums regardless of this setting. + RequestChecksumCalculation aws.RequestChecksumCalculation + // partPool allows for the re-usage of streaming payload part buffers between upload calls partPool byteSlicePool } @@ -274,12 +289,13 @@ type Uploader struct { // }) func NewUploader(client UploadAPIClient, options ...func(*Uploader)) *Uploader { u := &Uploader{ - S3: client, - PartSize: DefaultUploadPartSize, - Concurrency: DefaultUploadConcurrency, - LeavePartsOnError: false, - MaxUploadParts: MaxUploadParts, - BufferProvider: defaultUploadBufferProvider(), + S3: client, + PartSize: DefaultUploadPartSize, + Concurrency: DefaultUploadConcurrency, + LeavePartsOnError: false, + MaxUploadParts: MaxUploadParts, + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + BufferProvider: defaultUploadBufferProvider(), } for _, option := range options { @@ -776,7 +792,9 @@ func (u *multiuploader) initChecksumAlgorithm() { case u.in.ChecksumSHA256 != nil: u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 default: - u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32 + if u.cfg.RequestChecksumCalculation != aws.RequestChecksumCalculationWhenRequired { + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32 + } } } diff --git a/feature/s3/manager/upload_test.go b/feature/s3/manager/upload_test.go index ecbafb755d7..6aa2beba187 100644 --- a/feature/s3/manager/upload_test.go +++ b/feature/s3/manager/upload_test.go @@ -1013,6 +1013,94 @@ func TestUploaderValidARN(t *testing.T) { } } +// TestUploadRequestChecksumCalculation tests that checksum behavior respects +// the RequestChecksumCalculation setting for backwards compatibility. +func TestUploadRequestChecksumCalculation(t *testing.T) { + testCases := []struct { + name string + requestChecksumCalculation aws.RequestChecksumCalculation + inputChecksumAlgorithm types.ChecksumAlgorithm + expectedChecksumAlgorithm types.ChecksumAlgorithm + description string + }{ + { + name: "WhenRequired_NoDefault", + requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + inputChecksumAlgorithm: "", + expectedChecksumAlgorithm: "", + description: "When RequestChecksumCalculationWhenRequired is set, no default checksum should be applied (backwards compatibility)", + }, + { + name: "WhenSupported_HasDefault", + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + inputChecksumAlgorithm: "", + expectedChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + description: "When RequestChecksumCalculationWhenSupported is set, default CRC32 checksum should be applied", + }, + { + name: "WhenRequired_ExplicitPreserved", + requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + inputChecksumAlgorithm: types.ChecksumAlgorithmSha256, + expectedChecksumAlgorithm: types.ChecksumAlgorithmSha256, + description: "Explicit checksums should always be preserved regardless of RequestChecksumCalculation setting", + }, + { + name: "WhenSupported_ExplicitPreserved", + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + inputChecksumAlgorithm: types.ChecksumAlgorithmSha1, + expectedChecksumAlgorithm: types.ChecksumAlgorithmSha1, + description: "Explicit checksums should override defaults even with WhenSupported", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + c, invocations, args := s3testing.NewUploadLoggingClient(nil) + + mgr := manager.NewUploader(c, func(u *manager.Uploader) { + u.RequestChecksumCalculation = tc.requestChecksumCalculation + }) + + input := &s3.PutObjectInput{ + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), + Body: bytes.NewReader(buf12MB), + } + if tc.inputChecksumAlgorithm != "" { + input.ChecksumAlgorithm = tc.inputChecksumAlgorithm + } + + resp, err := mgr.Upload(context.Background(), input) + if err != nil { + t.Errorf("Expected no error but received %v", err) + } + + expectedOps := []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "UploadPart", "CompleteMultipartUpload"} + if diff := cmpDiff(expectedOps, *invocations); len(diff) > 0 { + t.Error(diff) + } + + if resp.UploadID != "UPLOAD-ID" { + t.Errorf("expect %q, got %q", "UPLOAD-ID", resp.UploadID) + } + + cmu := (*args)[0].(*s3.CreateMultipartUploadInput) + if cmu.ChecksumAlgorithm != tc.expectedChecksumAlgorithm { + t.Errorf("%s: Expected checksum algorithm %v in CreateMultipartUpload, but got %v", + tc.description, tc.expectedChecksumAlgorithm, cmu.ChecksumAlgorithm) + } + + for i := 1; i <= 3; i++ { + uploadPart := (*args)[i].(*s3.UploadPartInput) + if uploadPart.ChecksumAlgorithm != tc.expectedChecksumAlgorithm { + t.Errorf("%s: Expected checksum algorithm %v in UploadPart %d, but got %v", + tc.description, tc.expectedChecksumAlgorithm, i, uploadPart.ChecksumAlgorithm) + } + } + }) + } +} + type mockS3UploadServer struct { *http.ServeMux