Skip to content

Commit e9d282d

Browse files
jencymaryjosephjencymaryjoseph
authored
Merge branch 'feature/master/pre-signed-url-getobject' into feature/multipartSubscriber
2 parents aa302be + 9e6bf61 commit e9d282d

File tree

4 files changed

+198
-2
lines changed

4 files changed

+198
-2
lines changed

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.internal.multipart;
17+
18+
import java.util.concurrent.CompletableFuture;
19+
import software.amazon.awssdk.annotations.SdkInternalApi;
20+
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
21+
import software.amazon.awssdk.services.s3.S3AsyncClient;
22+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
23+
import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension;
24+
import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest;
25+
import software.amazon.awssdk.utils.Validate;
26+
27+
/**
28+
* An {@link AsyncPresignedUrlExtension} that automatically converts presigned URL downloads
29+
* to multipart downloads.
30+
*/
31+
@SdkInternalApi
32+
public class MultipartAsyncPresignedUrlExtension implements AsyncPresignedUrlExtension {
33+
private final PresignedUrlDownloadHelper downloadHelper;
34+
35+
public MultipartAsyncPresignedUrlExtension(
36+
S3AsyncClient s3AsyncClient,
37+
AsyncPresignedUrlExtension asyncPresignedUrlExtension,
38+
long bufferSizeInBytes,
39+
long partSizeInBytes) {
40+
Validate.paramNotNull(s3AsyncClient, "s3AsyncClient");
41+
Validate.paramNotNull(asyncPresignedUrlExtension, "asyncPresignedUrlExtension");
42+
this.downloadHelper = new PresignedUrlDownloadHelper(
43+
s3AsyncClient,
44+
asyncPresignedUrlExtension,
45+
bufferSizeInBytes,
46+
partSizeInBytes);
47+
}
48+
49+
@Override
50+
public <ReturnT> CompletableFuture<ReturnT> getObject(
51+
PresignedUrlDownloadRequest presignedUrlDownloadRequest,
52+
AsyncResponseTransformer<GetObjectResponse, ReturnT> asyncResponseTransformer) {
53+
Validate.paramNotNull(presignedUrlDownloadRequest, "presignedUrlDownloadRequest");
54+
Validate.paramNotNull(asyncResponseTransformer, "asyncResponseTransformer");
55+
return downloadHelper.downloadObject(presignedUrlDownloadRequest, asyncResponseTransformer);
56+
}
57+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient {
5353
private final CopyObjectHelper copyObjectHelper;
5454
private final DownloadObjectHelper downloadObjectHelper;
5555
private final boolean checksumEnabled;
56+
private final long apiCallBufferSize;
57+
private final long minPartSizeInBytes;
5658

5759
private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration,
5860
boolean checksumEnabled) {
@@ -63,6 +65,8 @@ private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration mu
6365
long minPartSizeInBytes = resolver.minimalPartSizeInBytes();
6466
long threshold = resolver.thresholdInBytes();
6567
long apiCallBufferSize = resolver.apiCallBufferSize();
68+
this.apiCallBufferSize = apiCallBufferSize;
69+
this.minPartSizeInBytes = minPartSizeInBytes;
6670
mpuHelper = new UploadObjectHelper(delegate, resolver);
6771
copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold);
6872
downloadObjectHelper = new DownloadObjectHelper(delegate, apiCallBufferSize);
@@ -114,7 +118,11 @@ protected <T extends S3Request, ReturnT> CompletableFuture<ReturnT> invokeOperat
114118

115119
@Override
116120
public AsyncPresignedUrlExtension presignedUrlExtension() {
117-
// TODO: Implement presigned URL extension support for multipart client
118-
throw new UnsupportedOperationException("Presigned URL extension is not supported for multipart client");
121+
AsyncPresignedUrlExtension delegateExtension = ((S3AsyncClient) delegate()).presignedUrlExtension();
122+
return new MultipartAsyncPresignedUrlExtension(
123+
(S3AsyncClient) delegate(),
124+
delegateExtension,
125+
apiCallBufferSize,
126+
minPartSizeInBytes);
119127
}
120128
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.internal.multipart;
17+
18+
import java.util.concurrent.CompletableFuture;
19+
import software.amazon.awssdk.annotations.SdkInternalApi;
20+
import software.amazon.awssdk.core.SplittingTransformerConfiguration;
21+
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
22+
import software.amazon.awssdk.services.s3.S3AsyncClient;
23+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
24+
import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension;
25+
import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest;
26+
import software.amazon.awssdk.utils.Logger;
27+
import software.amazon.awssdk.utils.Validate;
28+
29+
@SdkInternalApi
30+
public class PresignedUrlDownloadHelper {
31+
private static final Logger log = Logger.loggerFor(PresignedUrlDownloadHelper.class);
32+
33+
private final S3AsyncClient s3AsyncClient;
34+
private final AsyncPresignedUrlExtension asyncPresignedUrlExtension;
35+
private final long bufferSizeInBytes;
36+
private final long configuredPartSizeInBytes;
37+
38+
public PresignedUrlDownloadHelper(S3AsyncClient s3AsyncClient,
39+
AsyncPresignedUrlExtension asyncPresignedUrlExtension,
40+
long bufferSizeInBytes,
41+
long configuredPartSizeInBytes) {
42+
this.s3AsyncClient = Validate.paramNotNull(s3AsyncClient, "s3AsyncClient");
43+
this.asyncPresignedUrlExtension = Validate.paramNotNull(asyncPresignedUrlExtension, "asyncPresignedUrlExtension");
44+
this.bufferSizeInBytes = Validate.isPositive(bufferSizeInBytes, "bufferSizeInBytes");
45+
this.configuredPartSizeInBytes = Validate.isPositive(configuredPartSizeInBytes, "configuredPartSizeInBytes");
46+
}
47+
48+
public <T> CompletableFuture<T> downloadObject(
49+
PresignedUrlDownloadRequest presignedRequest,
50+
AsyncResponseTransformer<GetObjectResponse, T> asyncResponseTransformer) {
51+
52+
Validate.paramNotNull(presignedRequest, "presignedRequest");
53+
Validate.paramNotNull(asyncResponseTransformer, "asyncResponseTransformer");
54+
55+
if (presignedRequest.range() != null) {
56+
log.debug(() -> "Using single part download because presigned URL request range is included in the request. range = "
57+
+ presignedRequest.range());
58+
return asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer);
59+
}
60+
61+
SplittingTransformerConfiguration splittingConfig = SplittingTransformerConfiguration.builder()
62+
.bufferSizeInBytes(bufferSizeInBytes)
63+
.build();
64+
AsyncResponseTransformer.SplitResult<GetObjectResponse, T> split =
65+
asyncResponseTransformer.split(splittingConfig);
66+
// TODO: PresignedUrlMultipartDownloaderSubscriber needs to be implemented in next PR
67+
// PresignedUrlMultipartDownloaderSubscriber subscriber =
68+
// new PresignedUrlMultipartDownloaderSubscriber(
69+
// s3AsyncClient,
70+
// presignedRequest,
71+
// configuredPartSizeInBytes);
72+
//
73+
// split.publisher().subscribe(subscriber);
74+
// return split.resultFuture();
75+
throw new UnsupportedOperationException("Multipart presigned URL download not yet implemented - TODO in next PR");
76+
}
77+
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,17 @@
1515

1616
package software.amazon.awssdk.services.s3.internal.multipart;
1717

18+
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
1819
import static org.mockito.ArgumentMatchers.any;
1920
import static org.mockito.ArgumentMatchers.eq;
2021
import static org.mockito.Mockito.mock;
2122
import static org.mockito.Mockito.never;
2223
import static org.mockito.Mockito.times;
2324
import static org.mockito.Mockito.verify;
25+
import static org.mockito.Mockito.when;
2426

27+
import java.net.MalformedURLException;
28+
import java.net.URL;
2529
import org.junit.jupiter.api.Test;
2630
import software.amazon.awssdk.core.ResponseBytes;
2731
import software.amazon.awssdk.core.SplittingTransformerConfiguration;
@@ -30,6 +34,8 @@
3034
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
3135
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
3236
import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
37+
import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension;
38+
import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest;
3339

3440
class MultipartS3AsyncClientTest {
3541

@@ -64,4 +70,52 @@ void partManuallySpecified_shouldBypassMultipart() {
6470
verify(mockTransformer, never()).split(any(SplittingTransformerConfiguration.class));
6571
verify(mockDelegate, times(1)).getObject(any(GetObjectRequest.class), eq(mockTransformer));
6672
}
73+
74+
@Test
75+
void presignedUrlExtension_rangeSpecified_shouldBypassMultipart() throws MalformedURLException {
76+
S3AsyncClient mockDelegate = mock(S3AsyncClient.class);
77+
AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class);
78+
AsyncResponseTransformer<GetObjectResponse, String> mockTransformer = mock(AsyncResponseTransformer.class);
79+
PresignedUrlDownloadRequest req = PresignedUrlDownloadRequest.builder()
80+
.presignedUrl(new URL("https://s3.amazonaws.com/bucket/key?signature=abc"))
81+
.range("bytes=0-1023")
82+
.build();
83+
when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension);
84+
S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true);
85+
s3AsyncClient.presignedUrlExtension().getObject(req, mockTransformer);
86+
verify(mockTransformer, never()).split(any(SplittingTransformerConfiguration.class));
87+
verify(mockDelegateExtension, times(1)).getObject(eq(req), eq(mockTransformer));
88+
}
89+
90+
// TODO: Enable this test once PresignedUrlMultipartDownloaderSubscriber is implemented
91+
// // Currently fails because multipart presigned URL download throws UnsupportedOperationException
92+
// @Test
93+
// void presignedUrlExtension_noRange_shouldUseMultipart() throws MalformedURLException {
94+
// S3AsyncClient mockDelegate = mock(S3AsyncClient.class);
95+
// AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class);
96+
// AsyncResponseTransformer<GetObjectResponse, String> mockTransformer = mock(AsyncResponseTransformer.class);
97+
// AsyncResponseTransformer.SplitResult<GetObjectResponse, String> mockSplitResult = mock(AsyncResponseTransformer.SplitResult.class);
98+
// PresignedUrlDownloadRequest req = PresignedUrlDownloadRequest.builder()
99+
// .presignedUrl(new URL("https://s3.amazonaws.com/bucket/key?signature=abc"))
100+
// .build();
101+
// when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension);
102+
// when(mockTransformer.split(any(SplittingTransformerConfiguration.class))).thenReturn(mockSplitResult);
103+
// when(mockSplitResult.publisher()).thenReturn(mock(software.amazon.awssdk.core.async.SdkPublisher.class));
104+
// S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true);
105+
// s3AsyncClient.presignedUrlExtension().getObject(req, mockTransformer);
106+
// verify(mockTransformer, times(1)).split(any(SplittingTransformerConfiguration.class));
107+
// verify(mockDelegateExtension, never()).getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class));
108+
// }
109+
110+
@Test
111+
void presignedUrlExtension_shouldReturnMultipartExtension() {
112+
S3AsyncClient mockDelegate = mock(S3AsyncClient.class);
113+
AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class);
114+
when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension);
115+
116+
S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true);
117+
AsyncPresignedUrlExtension extension = s3AsyncClient.presignedUrlExtension();
118+
119+
assertThat(extension).isInstanceOf(MultipartAsyncPresignedUrlExtension.class);
120+
}
67121
}

0 commit comments

Comments
 (0)