Skip to content

Commit ea88d01

Browse files
F21F21
authored
Implement presign for s3 UploadPart (#1232)
1 parent 8c4dbc7 commit ea88d01

File tree

4 files changed

+350
-2
lines changed

4 files changed

+350
-2
lines changed

‎codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration {
8383
private static final Map<ShapeId, Set<ShapeId>> presignedClientMap = MapUtils.of(
8484
ShapeId.from("com.amazonaws.s3#AmazonS3"), SetUtils.of(
8585
ShapeId.from("com.amazonaws.s3#GetObject"),
86-
ShapeId.from("com.amazonaws.s3#PutObject")
86+
ShapeId.from("com.amazonaws.s3#PutObject"),
87+
ShapeId.from("com.amazonaws.s3#UploadPart")
8788
),
8889
ShapeId.from("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"), SetUtils.of(
8990
ShapeId.from("com.amazonaws.sts#GetCallerIdentity"))

‎service/internal/integrationtest/s3/presign_test.go‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,131 @@ func TestInteg_PresignURL(t *testing.T) {
123123
}
124124
}
125125

126+
func TestInteg_MultipartPresignURL(t *testing.T) {
127+
cases := map[string]struct {
128+
key string
129+
body io.Reader
130+
expires time.Duration
131+
sha256Header string
132+
expectedSignedHeader http.Header
133+
}{
134+
"standard": {
135+
body: bytes.NewReader([]byte("Hello-world")),
136+
expectedSignedHeader: http.Header{},
137+
},
138+
"special characters": {
139+
key: "some_value_(1).foo",
140+
},
141+
"nil-body": {
142+
expectedSignedHeader: http.Header{},
143+
},
144+
"empty-body": {
145+
body: bytes.NewReader([]byte("")),
146+
expectedSignedHeader: http.Header{},
147+
},
148+
}
149+
150+
for name, c := range cases {
151+
t.Run(name, func(t *testing.T) {
152+
key := c.key
153+
if len(key) == 0 {
154+
key = integrationtest.UniqueID()
155+
}
156+
157+
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
158+
defer cancelFn()
159+
160+
cfg, err := integrationtest.LoadConfigWithDefaultRegion("us-west-2")
161+
if err != nil {
162+
t.Fatalf("failed to load config, %v", err)
163+
}
164+
165+
client := s3.NewFromConfig(cfg)
166+
167+
multipartUpload, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
168+
Bucket: &setupMetadata.Buckets.Source.Name,
169+
Key: &key,
170+
})
171+
172+
if err != nil {
173+
t.Fatalf("error creating multipart upload: %v", err)
174+
}
175+
176+
// construct an upload part object
177+
uploadPartInput := &s3.UploadPartInput{
178+
Bucket: &setupMetadata.Buckets.Source.Name,
179+
Key: &key,
180+
PartNumber: 1,
181+
UploadId: multipartUpload.UploadId,
182+
Body: c.body,
183+
}
184+
185+
presignerClient := s3.NewPresignClient(client, func(options *s3.PresignOptions) {
186+
options.Expires = 600 * time.Second
187+
})
188+
189+
presignRequest, err := presignerClient.PresignUploadPart(ctx, uploadPartInput)
190+
if err != nil {
191+
t.Fatalf("expect no error, got %v", err)
192+
}
193+
194+
for k, v := range c.expectedSignedHeader {
195+
value := presignRequest.SignedHeader[k]
196+
if len(value) == 0 {
197+
t.Fatalf("expected %v header to be present in presigned url, got %v", k, presignRequest.SignedHeader)
198+
}
199+
200+
if diff := cmp.Diff(v, value); len(diff) != 0 {
201+
t.Fatalf("expected %v header value to be %v got %v", k, v, value)
202+
}
203+
}
204+
205+
resp, err := sendHTTPRequest(presignRequest, uploadPartInput.Body)
206+
if err != nil {
207+
t.Errorf("expect no error while sending HTTP request using presigned url, got %v", err)
208+
}
209+
210+
defer resp.Body.Close()
211+
212+
if resp.StatusCode != http.StatusOK {
213+
t.Fatalf("failed to upload part, %d:%s", resp.StatusCode, resp.Status)
214+
}
215+
216+
_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
217+
Bucket: &setupMetadata.Buckets.Source.Name,
218+
Key: &key,
219+
UploadId: multipartUpload.UploadId,
220+
})
221+
222+
if err != nil {
223+
t.Fatalf("error completing multipart upload: %v", err)
224+
}
225+
226+
// construct a get object
227+
getObjectInput := &s3.GetObjectInput{
228+
Bucket: &setupMetadata.Buckets.Source.Name,
229+
Key: &key,
230+
}
231+
232+
presignRequest, err = presignerClient.PresignGetObject(ctx, getObjectInput)
233+
if err != nil {
234+
t.Errorf("expect no error, got %v", err)
235+
}
236+
237+
resp, err = sendHTTPRequest(presignRequest, nil)
238+
if err != nil {
239+
t.Errorf("expect no error while sending HTTP request using presigned url, got %v", err)
240+
}
241+
242+
defer resp.Body.Close()
243+
244+
if resp.StatusCode != http.StatusOK {
245+
t.Fatalf("failed to get S3 object, %d:%s", resp.StatusCode, resp.Status)
246+
}
247+
})
248+
}
249+
}
250+
126251
func sendHTTPRequest(presignRequest *v4.PresignedHTTPRequest, body io.Reader) (*http.Response, error) {
127252
// create a http request
128253
req, err := http.NewRequest(presignRequest.Method, presignRequest.URL, nil)

‎service/s3/api_op_UploadPart.go‎

Lines changed: 37 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎service/s3/internal/customizations/presign_test.go‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,3 +179,189 @@ func TestPutObject_PresignURL(t *testing.T) {
179179
})
180180
}
181181
}
182+
183+
func TestUploadPart_PresignURL(t *testing.T) {
184+
cases := map[string]struct {
185+
input s3.UploadPartInput
186+
options s3.PresignOptions
187+
expectPresignedURLHost string
188+
expectRequestURIQuery []string
189+
expectSignedHeader http.Header
190+
expectMethod string
191+
expectError string
192+
}{
193+
"standard case": {
194+
input: s3.UploadPartInput{
195+
Bucket: aws.String("mock-bucket"),
196+
Key: aws.String("mockkey"),
197+
PartNumber: 1,
198+
UploadId: aws.String("123456"),
199+
Body: strings.NewReader("hello-world"),
200+
},
201+
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
202+
expectRequestURIQuery: []string{
203+
"X-Amz-Expires=900",
204+
"X-Amz-Credential",
205+
"X-Amz-Date",
206+
"partNumber=1",
207+
"uploadId=123456",
208+
"x-id=UploadPart",
209+
"X-Amz-Signature",
210+
},
211+
expectMethod: "PUT",
212+
expectSignedHeader: http.Header{
213+
"Content-Length": []string{"11"},
214+
"Content-Type": []string{"application/octet-stream"},
215+
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
216+
},
217+
},
218+
"seekable payload": {
219+
input: s3.UploadPartInput{
220+
Bucket: aws.String("mock-bucket"),
221+
Key: aws.String("mockkey"),
222+
PartNumber: 1,
223+
UploadId: aws.String("123456"),
224+
Body: bytes.NewReader([]byte("hello-world")),
225+
},
226+
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
227+
expectRequestURIQuery: []string{
228+
"X-Amz-Expires=900",
229+
"X-Amz-Credential",
230+
"X-Amz-Date",
231+
"partNumber=1",
232+
"uploadId=123456",
233+
"x-id=UploadPart",
234+
"X-Amz-Signature",
235+
},
236+
expectMethod: "PUT",
237+
expectSignedHeader: http.Header{
238+
"Content-Length": []string{"11"},
239+
"Content-Type": []string{"application/octet-stream"},
240+
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
241+
},
242+
},
243+
"unseekable payload": {
244+
// unseekable payload succeeds as we disable content sha256 computation for streaming input
245+
input: s3.UploadPartInput{
246+
Bucket: aws.String("mock-bucket"),
247+
Key: aws.String("mockkey"),
248+
PartNumber: 1,
249+
UploadId: aws.String("123456"),
250+
Body: bytes.NewBuffer([]byte(`hello-world`)),
251+
},
252+
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
253+
expectRequestURIQuery: []string{
254+
"X-Amz-Expires=900",
255+
"X-Amz-Credential",
256+
"X-Amz-Date",
257+
"partNumber=1",
258+
"uploadId=123456",
259+
"x-id=UploadPart",
260+
"X-Amz-Signature",
261+
},
262+
expectMethod: "PUT",
263+
expectSignedHeader: http.Header{
264+
"Content-Length": []string{"11"},
265+
"Content-Type": []string{"application/octet-stream"},
266+
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
267+
},
268+
},
269+
"empty body": {
270+
input: s3.UploadPartInput{
271+
Bucket: aws.String("mock-bucket"),
272+
Key: aws.String("mockkey"),
273+
PartNumber: 1,
274+
UploadId: aws.String("123456"),
275+
Body: bytes.NewReader([]byte(``)),
276+
},
277+
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
278+
expectRequestURIQuery: []string{
279+
"X-Amz-Expires=900",
280+
"X-Amz-Credential",
281+
"X-Amz-Date",
282+
"partNumber=1",
283+
"uploadId=123456",
284+
"x-id=UploadPart",
285+
"X-Amz-Signature",
286+
},
287+
expectMethod: "PUT",
288+
expectSignedHeader: http.Header{
289+
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
290+
},
291+
},
292+
"nil body": {
293+
input: s3.UploadPartInput{
294+
Bucket: aws.String("mock-bucket"),
295+
Key: aws.String("mockkey"),
296+
PartNumber: 1,
297+
UploadId: aws.String("123456"),
298+
},
299+
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
300+
expectRequestURIQuery: []string{
301+
"X-Amz-Expires=900",
302+
"X-Amz-Credential",
303+
"X-Amz-Date",
304+
"partNumber=1",
305+
"uploadId=123456",
306+
"x-id=UploadPart",
307+
"X-Amz-Signature",
308+
},
309+
expectMethod: "PUT",
310+
expectSignedHeader: http.Header{
311+
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
312+
},
313+
},
314+
}
315+
316+
for name, c := range cases {
317+
t.Run(name, func(t *testing.T) {
318+
ctx := context.Background()
319+
cfg := aws.Config{
320+
Region: "us-west-2",
321+
Credentials: unit.StubCredentialsProvider{},
322+
Retryer: func() aws.Retryer {
323+
return aws.NopRetryer{}
324+
},
325+
}
326+
presignClient := s3.NewPresignClient(s3.NewFromConfig(cfg), func(options *s3.PresignOptions) {
327+
options = &c.options
328+
})
329+
330+
req, err := presignClient.PresignUploadPart(ctx, &c.input)
331+
if err != nil {
332+
if len(c.expectError) == 0 {
333+
t.Fatalf("expected no error, got %v", err)
334+
}
335+
// if expect error, match error and skip rest
336+
if e, a := c.expectError, err.Error(); !strings.Contains(a, e) {
337+
t.Fatalf("expected error to be %s, got %s", e, a)
338+
}
339+
} else {
340+
if len(c.expectError) != 0 {
341+
t.Fatalf("expected error to be %v, got none", c.expectError)
342+
}
343+
}
344+
345+
if e, a := c.expectPresignedURLHost, req.URL; !strings.Contains(a, e) {
346+
t.Fatalf("expected presigned url to contain host %s, got %s", e, a)
347+
}
348+
349+
if len(c.expectRequestURIQuery) != 0 {
350+
for _, label := range c.expectRequestURIQuery {
351+
if e, a := label, req.URL; !strings.Contains(a, e) {
352+
t.Fatalf("expected presigned url to contain %v label in url: %v", label, req.URL)
353+
}
354+
}
355+
}
356+
357+
if e, a := c.expectSignedHeader, req.SignedHeader; len(cmp.Diff(e, a)) != 0 {
358+
t.Fatalf("expected signed header to be %s, got %s, \n diff : %s", e, a, cmp.Diff(e, a))
359+
}
360+
361+
if e, a := c.expectMethod, req.Method; !strings.EqualFold(e, a) {
362+
t.Fatalf("expected presigning Method to be %s, got %s", e, a)
363+
}
364+
365+
})
366+
}
367+
}

0 commit comments

Comments
 (0)