Skip to content

Commit a6b9cb3

Browse files
stanhustanhu
committed
feature(s3/manager): add option to control default checksums
As announced in #2960, AWS SDK for Go v2 service/s3 v1.73.0 shipped a change (#2808) that adopted new default integrity protections, automatically calculating CRC32 checksums for operations like PutObject and UploadPart. While it is possible to revert to the previous behavior by setting `AWS_REQUEST_CHECKSUM_CALCULATION=when_required`, this config setting did not apply to the S3 Manager multipart uploader, which always enabled CRC32 checksums by default regardless of the global setting. This commit adds a new `RequestChecksumCalculation` field to the Uploader struct that allows users to control checksum behavior: - `RequestChecksumCalculationWhenSupported` (default): Always calculates CRC32 checksums for multipart uploads - `RequestChecksumCalculationWhenRequired`: Only calculates checksums when explicitly set by the user, preserving backwards compatibility For example: ```go uploader := manager.NewUploader(client, func(u *manager.Uploader) { u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired }) ``` S3 Express One Zone buckets always require CRC32 checksums regardless of this setting, as mandated by the S3 Express service requirements. The uploader automatically detects S3 Express buckets (names ending with `--x-s3`) and applies CRC32 checksums unconditionally. Fixes #3007 Signed-off-by: Stan Hu <[email protected]>
1 parent 4f215b9 commit a6b9cb3

File tree

2 files changed

+185
-7
lines changed

2 files changed

+185
-7
lines changed

‎feature/s3/manager/upload.go

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"io"
99
"net/http"
1010
"sort"
11+
"strings"
1112
"sync"
1213

1314
"github.com/aws/aws-sdk-go-v2/aws"
@@ -245,6 +246,21 @@ type Uploader struct {
245246
// Defines the buffer strategy used when uploading a part
246247
BufferProvider ReadSeekerWriteToProvider
247248

249+
// RequestChecksumCalculation determines when request checksum calculation is performed
250+
// for multipart uploads.
251+
//
252+
// There are two possible values for this setting:
253+
//
254+
// 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated
255+
// if the operation supports it, regardless of whether the user sets an algorithm in the request.
256+
//
257+
// 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user
258+
// explicitly sets a checksum algorithm in the request. This preserves backwards compatibility
259+
// for applications that don't want automatic checksum calculation.
260+
//
261+
// Note: S3 Express buckets always require CRC32 checksums regardless of this setting.
262+
RequestChecksumCalculation aws.RequestChecksumCalculation
263+
248264
// partPool allows for the re-usage of streaming payload part buffers between upload calls
249265
partPool byteSlicePool
250266
}
@@ -274,12 +290,13 @@ type Uploader struct {
274290
// })
275291
func NewUploader(client UploadAPIClient, options ...func(*Uploader)) *Uploader {
276292
u := &Uploader{
277-
S3: client,
278-
PartSize: DefaultUploadPartSize,
279-
Concurrency: DefaultUploadConcurrency,
280-
LeavePartsOnError: false,
281-
MaxUploadParts: MaxUploadParts,
282-
BufferProvider: defaultUploadBufferProvider(),
293+
S3: client,
294+
PartSize: DefaultUploadPartSize,
295+
Concurrency: DefaultUploadConcurrency,
296+
LeavePartsOnError: false,
297+
MaxUploadParts: MaxUploadParts,
298+
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported,
299+
BufferProvider: defaultUploadBufferProvider(),
283300
}
284301

285302
for _, option := range options {
@@ -776,10 +793,23 @@ func (u *multiuploader) initChecksumAlgorithm() {
776793
case u.in.ChecksumSHA256 != nil:
777794
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha256
778795
default:
779-
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32
796+
if u.isS3ExpressBucket() {
797+
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32
798+
return
799+
}
800+
801+
if u.cfg.RequestChecksumCalculation != aws.RequestChecksumCalculationWhenRequired {
802+
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32
803+
}
780804
}
781805
}
782806

807+
// isS3ExpressBucket returns true if the bucket has the S3 Express suffix.
808+
func (u *multiuploader) isS3ExpressBucket() bool {
809+
bucketName := aws.ToString(u.in.Bucket)
810+
return strings.HasSuffix(bucketName, "--x-s3")
811+
}
812+
783813
// geterr is a thread-safe getter for the error object
784814
func (u *multiuploader) geterr() error {
785815
u.m.Lock()

‎feature/s3/manager/upload_test.go

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,6 +1013,154 @@ func TestUploaderValidARN(t *testing.T) {
10131013
}
10141014
}
10151015

1016+
// TestUploadRequestChecksumCalculation tests that checksum behavior respects
1017+
// the RequestChecksumCalculation setting for backwards compatibility.
1018+
func TestUploadRequestChecksumCalculation(t *testing.T) {
1019+
testCases := []struct {
1020+
name string
1021+
requestChecksumCalculation aws.RequestChecksumCalculation
1022+
inputChecksumAlgorithm types.ChecksumAlgorithm
1023+
expectedChecksumAlgorithm types.ChecksumAlgorithm
1024+
description string
1025+
}{
1026+
{
1027+
name: "WhenRequired_NoDefault",
1028+
requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
1029+
inputChecksumAlgorithm: "",
1030+
expectedChecksumAlgorithm: "",
1031+
description: "When RequestChecksumCalculationWhenRequired is set, no default checksum should be applied (backwards compatibility)",
1032+
},
1033+
{
1034+
name: "WhenSupported_HasDefault",
1035+
requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported,
1036+
inputChecksumAlgorithm: "",
1037+
expectedChecksumAlgorithm: types.ChecksumAlgorithmCrc32,
1038+
description: "When RequestChecksumCalculationWhenSupported is set, default CRC32 checksum should be applied",
1039+
},
1040+
{
1041+
name: "WhenRequired_ExplicitPreserved",
1042+
requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
1043+
inputChecksumAlgorithm: types.ChecksumAlgorithmSha256,
1044+
expectedChecksumAlgorithm: types.ChecksumAlgorithmSha256,
1045+
description: "Explicit checksums should always be preserved regardless of RequestChecksumCalculation setting",
1046+
},
1047+
{
1048+
name: "WhenSupported_ExplicitPreserved",
1049+
requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported,
1050+
inputChecksumAlgorithm: types.ChecksumAlgorithmSha1,
1051+
expectedChecksumAlgorithm: types.ChecksumAlgorithmSha1,
1052+
description: "Explicit checksums should override defaults even with WhenSupported",
1053+
},
1054+
}
1055+
1056+
for _, tc := range testCases {
1057+
t.Run(tc.name, func(t *testing.T) {
1058+
c, invocations, args := s3testing.NewUploadLoggingClient(nil)
1059+
1060+
mgr := manager.NewUploader(c, func(u *manager.Uploader) {
1061+
u.RequestChecksumCalculation = tc.requestChecksumCalculation
1062+
})
1063+
1064+
input := &s3.PutObjectInput{
1065+
Bucket: aws.String("Bucket"),
1066+
Key: aws.String("Key"),
1067+
Body: bytes.NewReader(buf12MB),
1068+
}
1069+
if tc.inputChecksumAlgorithm != "" {
1070+
input.ChecksumAlgorithm = tc.inputChecksumAlgorithm
1071+
}
1072+
1073+
resp, err := mgr.Upload(context.Background(), input)
1074+
if err != nil {
1075+
t.Errorf("Expected no error but received %v", err)
1076+
}
1077+
1078+
expectedOps := []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "UploadPart", "CompleteMultipartUpload"}
1079+
if diff := cmpDiff(expectedOps, *invocations); len(diff) > 0 {
1080+
t.Error(diff)
1081+
}
1082+
1083+
if resp.UploadID != "UPLOAD-ID" {
1084+
t.Errorf("expect %q, got %q", "UPLOAD-ID", resp.UploadID)
1085+
}
1086+
1087+
cmu := (*args)[0].(*s3.CreateMultipartUploadInput)
1088+
if cmu.ChecksumAlgorithm != tc.expectedChecksumAlgorithm {
1089+
t.Errorf("%s: Expected checksum algorithm %v in CreateMultipartUpload, but got %v",
1090+
tc.description, tc.expectedChecksumAlgorithm, cmu.ChecksumAlgorithm)
1091+
}
1092+
1093+
for i := 1; i <= 3; i++ {
1094+
uploadPart := (*args)[i].(*s3.UploadPartInput)
1095+
if uploadPart.ChecksumAlgorithm != tc.expectedChecksumAlgorithm {
1096+
t.Errorf("%s: Expected checksum algorithm %v in UploadPart %d, but got %v",
1097+
tc.description, tc.expectedChecksumAlgorithm, i, uploadPart.ChecksumAlgorithm)
1098+
}
1099+
}
1100+
})
1101+
}
1102+
}
1103+
1104+
// TestUploadS3ExpressAlwaysRequiresChecksum tests that S3 Express buckets
1105+
// always get CRC32 checksums regardless of RequestChecksumCalculation setting.
1106+
func TestUploadS3ExpressAlwaysRequiresChecksum(t *testing.T) {
1107+
testCases := []struct {
1108+
name string
1109+
requestChecksumCalculation aws.RequestChecksumCalculation
1110+
description string
1111+
}{
1112+
{
1113+
name: "S3Express_WhenRequired_StillGetsCRC32",
1114+
requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
1115+
description: "S3 Express buckets should get CRC32 even with RequestChecksumCalculationWhenRequired",
1116+
},
1117+
{
1118+
name: "S3Express_WhenSupported_GetsCRC32",
1119+
requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported,
1120+
description: "S3 Express buckets should get CRC32 with RequestChecksumCalculationWhenSupported",
1121+
},
1122+
}
1123+
1124+
for _, tc := range testCases {
1125+
t.Run(tc.name, func(t *testing.T) {
1126+
c, invocations, args := s3testing.NewUploadLoggingClient(nil)
1127+
1128+
mgr := manager.NewUploader(c, func(u *manager.Uploader) {
1129+
u.RequestChecksumCalculation = tc.requestChecksumCalculation
1130+
})
1131+
1132+
_, err := mgr.Upload(context.Background(), &s3.PutObjectInput{
1133+
Bucket: aws.String("my-express-bucket--x-s3"),
1134+
Key: aws.String("Key"),
1135+
Body: bytes.NewReader(buf12MB),
1136+
})
1137+
1138+
if err != nil {
1139+
t.Errorf("Expected no error but received %v", err)
1140+
}
1141+
1142+
expectedOps := []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "UploadPart", "CompleteMultipartUpload"}
1143+
if diff := cmpDiff(expectedOps, *invocations); len(diff) > 0 {
1144+
t.Error(diff)
1145+
}
1146+
1147+
cmu := (*args)[0].(*s3.CreateMultipartUploadInput)
1148+
if cmu.ChecksumAlgorithm != types.ChecksumAlgorithmCrc32 {
1149+
t.Errorf("%s: Expected CRC32 checksum for S3 Express bucket, but got %v",
1150+
tc.description, cmu.ChecksumAlgorithm)
1151+
}
1152+
1153+
for i := 1; i <= 3; i++ {
1154+
uploadPart := (*args)[i].(*s3.UploadPartInput)
1155+
if uploadPart.ChecksumAlgorithm != types.ChecksumAlgorithmCrc32 {
1156+
t.Errorf("%s: Expected CRC32 checksum in UploadPart %d for S3 Express bucket, but got %v",
1157+
tc.description, i, uploadPart.ChecksumAlgorithm)
1158+
}
1159+
}
1160+
})
1161+
}
1162+
}
1163+
10161164
type mockS3UploadServer struct {
10171165
*http.ServeMux
10181166

0 commit comments

Comments
 (0)