Skip to content

Commit f1d2dbc

Browse files
stanhustanhu
authored
blob/s3blob: make it possible to configure the default integrity protection (#3634)
1 parent eff724a commit f1d2dbc

File tree

18 files changed

+554
-317
lines changed

18 files changed

+554
-317
lines changed

‎aws/aws.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,45 @@ import (
2020
"fmt"
2121
"net/url"
2222
"strconv"
23+
"strings"
2324

2425
"github.com/aws/aws-sdk-go-v2/aws"
2526
"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
2627
"github.com/aws/aws-sdk-go-v2/aws/retry"
2728
"github.com/aws/aws-sdk-go-v2/config"
2829
)
2930

31+
const (
32+
requestChecksumCalculationParamKey = "request_checksum_calculation"
33+
responseChecksumValidationParamKey = "response_checksum_validation"
34+
)
35+
36+
// parseRequestChecksumCalculation parses request checksum calculation mode values.
37+
// Supports AWS SDK documented values: "when_supported", "when_required".
38+
func parseRequestChecksumCalculation(value string) (aws.RequestChecksumCalculation, error) {
39+
switch strings.ToLower(value) {
40+
case "when_supported":
41+
return aws.RequestChecksumCalculationWhenSupported, nil
42+
case "when_required":
43+
return aws.RequestChecksumCalculationWhenRequired, nil
44+
default:
45+
return aws.RequestChecksumCalculationWhenSupported, fmt.Errorf("invalid value for %q: %q. Valid values are: when_supported, when_required", requestChecksumCalculationParamKey, value)
46+
}
47+
}
48+
49+
// parseResponseChecksumValidation parses response checksum validation mode values.
50+
// Supports AWS SDK documented values: "when_supported", "when_required".
51+
func parseResponseChecksumValidation(value string) (aws.ResponseChecksumValidation, error) {
52+
switch strings.ToLower(value) {
53+
case "when_supported":
54+
return aws.ResponseChecksumValidationWhenSupported, nil
55+
case "when_required":
56+
return aws.ResponseChecksumValidationWhenRequired, nil
57+
default:
58+
return aws.ResponseChecksumValidationWhenSupported, fmt.Errorf("invalid value for %q: %q. Valid values are: when_supported, when_required", responseChecksumValidationParamKey, value)
59+
}
60+
}
61+
3062
// NewDefaultV2Config returns a aws.Config for AWS SDK v2, using the default options.
3163
func NewDefaultV2Config(ctx context.Context) (aws.Config, error) {
3264
return config.LoadDefaultConfig(ctx)
@@ -53,6 +85,8 @@ func NewDefaultV2Config(ctx context.Context) (aws.Config, error) {
5385
// - rate_limiter_capacity: A integer value configures the capacity of a token bucket used
5486
// in client-side rate limits. If no value is set, the client-side rate limiting is disabled.
5587
// See https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/retries-timeouts/#client-side-rate-limiting.
88+
// - request_checksum_calculation: Request checksum calculation mode (when_supported, when_required)
89+
// - response_checksum_validation: Response checksum validation mode (when_supported, when_required)
5690
func V2ConfigFromURLParams(ctx context.Context, q url.Values) (aws.Config, error) {
5791
var endpoint string
5892
var hostnameImmutable bool
@@ -103,6 +137,20 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (aws.Config, error
103137
if anon {
104138
opts = append(opts, config.WithCredentialsProvider(aws.AnonymousCredentials{}))
105139
}
140+
case requestChecksumCalculationParamKey:
141+
value, err := parseRequestChecksumCalculation(value)
142+
if err != nil {
143+
return aws.Config{}, err
144+
}
145+
146+
opts = append(opts, config.WithRequestChecksumCalculation(value))
147+
case responseChecksumValidationParamKey:
148+
value, err := parseResponseChecksumValidation(value)
149+
if err != nil {
150+
return aws.Config{}, err
151+
}
152+
153+
opts = append(opts, config.WithResponseChecksumValidation(value))
106154
case "awssdk":
107155
// ignore, should be handled before this
108156
default:

‎aws/aws_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"context"
1919
"net/url"
2020
"reflect"
21+
"strings"
2122
"testing"
2223

2324
"github.com/aws/aws-sdk-go-v2/aws"
@@ -73,6 +74,54 @@ func TestV2ConfigFromURLParams(t *testing.T) {
7374
name: "Rate limit capacity",
7475
query: url.Values{"rate_limiter_capacity": {"500"}},
7576
},
77+
{
78+
name: "Request checksum calculation when_supported",
79+
query: url.Values{"request_checksum_calculation": {"when_supported"}},
80+
},
81+
{
82+
name: "Request checksum calculation when_required",
83+
query: url.Values{"request_checksum_calculation": {"when_required"}},
84+
},
85+
{
86+
name: "Response checksum validation when_supported",
87+
query: url.Values{"response_checksum_validation": {"when_supported"}},
88+
},
89+
{
90+
name: "Response checksum validation when_required",
91+
query: url.Values{"response_checksum_validation": {"when_required"}},
92+
},
93+
{
94+
name: "Both checksum parameters",
95+
query: url.Values{"request_checksum_calculation": {"when_required"}, "response_checksum_validation": {"when_supported"}},
96+
},
97+
{
98+
name: "Invalid request checksum value",
99+
query: url.Values{"request_checksum_calculation": {"invalid"}},
100+
wantErr: true,
101+
},
102+
{
103+
name: "Invalid response checksum value",
104+
query: url.Values{"response_checksum_validation": {"invalid"}},
105+
wantErr: true,
106+
},
107+
{
108+
name: "Empty request checksum value",
109+
query: url.Values{"request_checksum_calculation": {""}},
110+
wantErr: true,
111+
},
112+
{
113+
name: "Empty response checksum value",
114+
query: url.Values{"response_checksum_validation": {""}},
115+
wantErr: true,
116+
},
117+
{
118+
name: "Uppercase request checksum",
119+
query: url.Values{"request_checksum_calculation": {"WHEN_SUPPORTED"}},
120+
},
121+
{
122+
name: "Mixed case response checksum",
123+
query: url.Values{"response_checksum_validation": {"When_Required"}},
124+
},
76125
// Can't test "profile", since AWS validates that the profile exists.
77126
}
78127

@@ -90,6 +139,35 @@ func TestV2ConfigFromURLParams(t *testing.T) {
90139
t.Errorf("got region %q, want %q", got.Region, test.wantRegion)
91140
}
92141

142+
// Check checksum configuration based on query parameters
143+
if test.query.Has("request_checksum_calculation") {
144+
expectedValue := test.query.Get("request_checksum_calculation")
145+
var expectedChecksum aws.RequestChecksumCalculation
146+
switch strings.ToLower(expectedValue) {
147+
case "when_supported":
148+
expectedChecksum = aws.RequestChecksumCalculationWhenSupported
149+
case "when_required":
150+
expectedChecksum = aws.RequestChecksumCalculationWhenRequired
151+
}
152+
if got.RequestChecksumCalculation != expectedChecksum {
153+
t.Errorf("got RequestChecksumCalculation %v, want %v", got.RequestChecksumCalculation, expectedChecksum)
154+
}
155+
}
156+
157+
if test.query.Has("response_checksum_validation") {
158+
expectedValue := test.query.Get("response_checksum_validation")
159+
var expectedChecksum aws.ResponseChecksumValidation
160+
switch strings.ToLower(expectedValue) {
161+
case "when_supported":
162+
expectedChecksum = aws.ResponseChecksumValidationWhenSupported
163+
case "when_required":
164+
expectedChecksum = aws.ResponseChecksumValidationWhenRequired
165+
}
166+
if got.ResponseChecksumValidation != expectedChecksum {
167+
t.Errorf("got ResponseChecksumValidation %v, want %v", got.ResponseChecksumValidation, expectedChecksum)
168+
}
169+
}
170+
93171
if test.wantEndpoint != nil {
94172
if got.EndpointResolverWithOptions == nil {
95173
t.Fatalf("expected an EndpointResolverWithOptions, got nil")

‎blob/s3blob/s3blob.go‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -209,10 +209,15 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
209209
}
210210
client := s3.NewFromConfig(cfg, opts...)
211211

212+
// The S3 upload manager doesn't use the config or options to set the
213+
// request checksum calculation. We need to set it explicitly:
214+
// https://github.com/aws/aws-sdk-go-v2/pull/3151
215+
o.Options.RequestChecksumCalculation = cfg.RequestChecksumCalculation
216+
212217
return OpenBucket(ctx, client, u.Host, &o.Options)
213218
}
214219

215-
// Options sets options for constructing a *blob.Bucket backed by fileblob.
220+
// Options sets options for constructing a *blob.Bucket backed by S3.
216221
type Options struct {
217222
// UseLegacyList forces the use of ListObjects instead of ListObjectsV2.
218223
// Some S3-compatible services (like CEPH) do not currently support
@@ -228,6 +233,11 @@ type Options struct {
228233
// This is required when a bucket policy enforces the use of a specific
229234
// KMS key for uploads
230235
KMSEncryptionID string
236+
237+
// RequestChecksumCalculation configures the default integrity protection for
238+
// requests. This may need to be set to when_required to preserve compatibility for
239+
// third-party S3 providers: https://github.com/aws/aws-sdk-go-v2/discussions/2960.
240+
RequestChecksumCalculation aws.RequestChecksumCalculation
231241
}
232242

233243
// openBucket returns an S3 Bucket.
@@ -242,11 +252,12 @@ func openBucket(ctx context.Context, client *s3.Client, bucketName string, opts
242252
return nil, errors.New("s3blob.OpenBucket: client is required")
243253
}
244254
return &bucket{
245-
name: bucketName,
246-
client: client,
247-
useLegacyList: opts.UseLegacyList,
248-
kmsKeyId: opts.KMSEncryptionID,
249-
encryptionType: opts.EncryptionType,
255+
name: bucketName,
256+
client: client,
257+
useLegacyList: opts.UseLegacyList,
258+
kmsKeyId: opts.KMSEncryptionID,
259+
encryptionType: opts.EncryptionType,
260+
requestChecksumCalculation: opts.RequestChecksumCalculation,
250261
}, nil
251262
}
252263

@@ -382,8 +393,9 @@ type bucket struct {
382393
client *s3.Client
383394
useLegacyList bool
384395

385-
encryptionType types.ServerSideEncryption
386-
kmsKeyId string
396+
encryptionType types.ServerSideEncryption
397+
kmsKeyId string
398+
requestChecksumCalculation aws.RequestChecksumCalculation
387399
}
388400

389401
func (b *bucket) Close() error {
@@ -733,6 +745,8 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key, contentType string, op
733745
if opts.MaxConcurrency != 0 {
734746
u.Concurrency = opts.MaxConcurrency
735747
}
748+
749+
u.RequestChecksumCalculation = b.requestChecksumCalculation
736750
})
737751
md := make(map[string]string, len(opts.Metadata))
738752
for k, v := range opts.Metadata {

‎blob/s3blob/s3blob_test.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"errors"
2020
"fmt"
2121
"net/http"
22+
"net/url"
2223
"testing"
2324

2425
"github.com/aws/aws-sdk-go-v2/aws"
@@ -27,6 +28,7 @@ import (
2728
"github.com/aws/aws-sdk-go-v2/service/s3"
2829
"github.com/aws/aws-sdk-go-v2/service/s3/types"
2930
"github.com/aws/smithy-go"
31+
gcaws "gocloud.dev/aws"
3032
"gocloud.dev/blob"
3133
"gocloud.dev/blob/driver"
3234
"gocloud.dev/blob/drivertest"
@@ -324,6 +326,26 @@ func TestOpenBucketFromURL(t *testing.T) {
324326
{"s3://mybucket?fips=true", false},
325327
// OK, use anonymous.
326328
{"s3://mybucket?anonymous=true", false},
329+
// OK, use request checksum calculation when_supported
330+
{"s3://mybucket?request_checksum_calculation=when_supported", false},
331+
// OK, use request checksum calculation when_required
332+
{"s3://mybucket?request_checksum_calculation=when_required", false},
333+
// OK, use response checksum validation when_supported
334+
{"s3://mybucket?response_checksum_validation=when_supported", false},
335+
// OK, use response checksum validation when_required
336+
{"s3://mybucket?response_checksum_validation=when_required", false},
337+
// OK, use both checksum parameters
338+
{"s3://mybucket?request_checksum_calculation=when_required&response_checksum_validation=when_supported", false},
339+
// OK, case insensitive checksum parameters
340+
{"s3://mybucket?request_checksum_calculation=WHEN_SUPPORTED&response_checksum_validation=When_Required", false},
341+
// Invalid request checksum value
342+
{"s3://mybucket?request_checksum_calculation=invalid", true},
343+
// Invalid response checksum value
344+
{"s3://mybucket?response_checksum_validation=invalid", true},
345+
// Empty request checksum value
346+
{"s3://mybucket?request_checksum_calculation=", true},
347+
// Empty response checksum value
348+
{"s3://mybucket?response_checksum_validation=", true},
327349
// Invalid accelerate
328350
{"s3://mybucket?accelerate=bogus", true},
329351
// Invalid FIPS
@@ -354,6 +376,81 @@ func TestOpenBucketFromURL(t *testing.T) {
354376
}
355377
}
356378

379+
func TestChecksumConfigurationPassthrough(t *testing.T) {
380+
// Test that checksum configuration from URL parameters is properly passed through
381+
// to the S3 bucket options
382+
tests := []struct {
383+
name string
384+
url string
385+
wantRequestChecksumCalculation aws.RequestChecksumCalculation
386+
wantErr bool
387+
}{
388+
{
389+
name: "request checksum when_supported",
390+
url: "s3://mybucket?request_checksum_calculation=when_supported",
391+
wantRequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported,
392+
},
393+
{
394+
name: "request checksum when_required",
395+
url: "s3://mybucket?request_checksum_calculation=when_required",
396+
wantRequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
397+
},
398+
{
399+
name: "case insensitive",
400+
url: "s3://mybucket?request_checksum_calculation=WHEN_SUPPORTED",
401+
wantRequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported,
402+
},
403+
{
404+
name: "invalid value",
405+
url: "s3://mybucket?request_checksum_calculation=invalid",
406+
wantErr: true,
407+
},
408+
}
409+
410+
ctx := context.Background()
411+
for _, test := range tests {
412+
t.Run(test.name, func(t *testing.T) {
413+
// Parse the URL to extract the bucket name and query parameters
414+
u, err := url.Parse(test.url)
415+
if err != nil {
416+
t.Fatalf("failed to parse URL: %v", err)
417+
}
418+
419+
// Create a mock AWS config with the query parameters
420+
cfg, err := awscfg.LoadDefaultConfig(ctx)
421+
if err != nil {
422+
t.Fatalf("failed to load AWS config: %v", err)
423+
}
424+
425+
// Apply URL parameters to the config
426+
cfg, err = gcaws.V2ConfigFromURLParams(ctx, u.Query())
427+
if (err != nil) != test.wantErr {
428+
t.Errorf("got err %v want error %v", err, test.wantErr)
429+
return
430+
}
431+
if err != nil {
432+
return
433+
}
434+
435+
// Create URLOpener to test the integration
436+
opener := &URLOpener{}
437+
bucket, err := opener.OpenBucketURL(ctx, u)
438+
if err != nil {
439+
t.Fatalf("failed to open bucket: %v", err)
440+
}
441+
defer bucket.Close()
442+
443+
// Verify that the checksum configuration was applied
444+
// We can't directly access the internal bucket struct, but we can verify
445+
// that the configuration was applied to the AWS config
446+
if cfg.RequestChecksumCalculation != test.wantRequestChecksumCalculation {
447+
t.Errorf("got RequestChecksumCalculation %v, want %v",
448+
cfg.RequestChecksumCalculation, test.wantRequestChecksumCalculation)
449+
}
450+
})
451+
}
452+
}
453+
357454
func TestToServerSideEncryptionType(t *testing.T) {
358455
tests := []struct {
359456
value string

0 commit comments

Comments
 (0)