Skip to content

Commit 592f9a4

Browse files
jencymaryjosephjencymaryjoseph
committed
PresignedUrl multipart, progress tracking support with transfermanager
1 parent 6466fcb commit 592f9a4

File tree

8 files changed

+539
-34
lines changed

8 files changed

+539
-34
lines changed

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,13 @@ public class S3IntegrationTestBase extends AwsTestBase {
5555
protected static S3Client s3;
5656

5757
protected static S3AsyncClient s3Async;
58+
protected static S3AsyncClient s3multiAsync;
5859

5960
protected static S3AsyncClient s3CrtAsync;
6061

6162
protected static S3TransferManager tmCrt;
6263
protected static S3TransferManager tmJava;
64+
protected static S3TransferManager tmMultipartJava;
6365

6466
/**
6567
* Loads the AWS account info for the integration tests and creates an S3
@@ -71,6 +73,8 @@ public static void setUpForAllIntegTests() throws Exception {
7173
System.setProperty("aws.crt.debugnative", "true");
7274
s3 = s3ClientBuilder().build();
7375
s3Async = s3AsyncClientBuilder().build();
76+
s3multiAsync = s3AsyncClientBuilder().multipartEnabled(true).build();
77+
7478
s3CrtAsync = S3CrtAsyncClient.builder()
7579
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
7680
.region(DEFAULT_REGION)
@@ -81,15 +85,21 @@ public static void setUpForAllIntegTests() throws Exception {
8185
tmJava = S3TransferManager.builder()
8286
.s3Client(s3Async)
8387
.build();
88+
tmMultipartJava = S3TransferManager.builder()
89+
.s3Client(s3multiAsync)
90+
.build();
8491

8592
}
8693

8794
@AfterAll
8895
public static void cleanUpForAllIntegTests() {
8996
s3.close();
9097
s3Async.close();
98+
s3multiAsync.close();
9199
s3CrtAsync.close();
92100
tmCrt.close();
101+
tmJava.close();
102+
tmMultipartJava.close();
93103
CrtResource.waitForNoResources();
94104
}
95105

@@ -181,4 +191,9 @@ static Stream<Arguments> transferManagers() {
181191
Arguments.of(tmJava));
182192
}
183193

194+
static Stream<Arguments> javaTransferManagerOnly() {
195+
return Stream.of(
196+
Arguments.of(tmJava),
197+
Arguments.of(tmMultipartJava));
198+
}
184199
}

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
20+
21+
import java.io.File;
22+
import java.io.IOException;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.time.Duration;
26+
import org.junit.jupiter.api.AfterAll;
27+
import org.junit.jupiter.api.BeforeAll;
28+
import org.junit.jupiter.params.ParameterizedTest;
29+
import org.junit.jupiter.params.provider.MethodSource;;
30+
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
31+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
32+
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
33+
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
34+
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
35+
import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest;
36+
import software.amazon.awssdk.testutils.RandomTempFile;
37+
import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
38+
import software.amazon.awssdk.transfer.s3.model.FileDownload;
39+
import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest;
40+
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;
41+
import software.amazon.awssdk.utils.Md5Utils;
42+
43+
public class S3TransferManagerPresignedUrlDownloadIntegrationTest extends S3IntegrationTestBase {
44+
private static final String BUCKET = temporaryBucketName(S3TransferManagerPresignedUrlDownloadIntegrationTest.class);
45+
private static final String SMALL_KEY = "small-key";
46+
private static final String LARGE_KEY = "large-key";
47+
private static final int SMALL_OBJ_SIZE = 5 * 1024 * 1024;
48+
private static final int LARGE_OBJ_SIZE = 16 * 1024 * 1024;
49+
50+
private static File smallFile;
51+
private static File largeFile;
52+
private static S3Presigner presigner;
53+
54+
@BeforeAll
55+
public static void setup() throws IOException {
56+
createBucket(BUCKET);
57+
smallFile = new RandomTempFile(SMALL_OBJ_SIZE);
58+
largeFile = new RandomTempFile(LARGE_OBJ_SIZE);
59+
s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SMALL_KEY).build(), smallFile.toPath());
60+
s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(LARGE_KEY).build(), largeFile.toPath());
61+
presigner = S3Presigner.builder()
62+
.region(DEFAULT_REGION)
63+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
64+
.build();
65+
}
66+
67+
@AfterAll
68+
public static void cleanup() {
69+
if (presigner != null) {
70+
presigner.close();
71+
}
72+
deleteBucketAndAllContents(BUCKET);
73+
}
74+
75+
@ParameterizedTest
76+
@MethodSource("javaTransferManagerOnly")
77+
void downloadFileWithPresignedUrl_smallFile_downloadedCorrectly(S3TransferManager tm) throws Exception {
78+
PresignedGetObjectRequest presignedRequest = createPresignedRequest(SMALL_KEY);
79+
Path downloadPath = RandomTempFile.randomUncreatedFile().toPath();
80+
PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder()
81+
.presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder()
82+
.presignedUrl(presignedRequest.url())
83+
.build())
84+
.destination(downloadPath)
85+
.addTransferListener(LoggingTransferListener.create())
86+
.build();
87+
88+
FileDownload download = tm.downloadFileWithPresignedUrl(request);
89+
CompletedFileDownload completed = download.completionFuture().join();
90+
91+
assertThat(Files.exists(downloadPath)).isTrue();
92+
assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(smallFile));
93+
assertThat(completed.response().responseMetadata().requestId()).isNotNull();
94+
}
95+
96+
@ParameterizedTest
97+
@MethodSource("javaTransferManagerOnly")
98+
void downloadFileWithPresignedUrl_largeFile_downloadedCorrectly(S3TransferManager tm) throws Exception {
99+
PresignedGetObjectRequest presignedRequest = createPresignedRequest(LARGE_KEY);
100+
Path downloadPath = RandomTempFile.randomUncreatedFile().toPath();
101+
PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder()
102+
.presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder()
103+
.presignedUrl(presignedRequest.url())
104+
.build())
105+
.destination(downloadPath)
106+
.addTransferListener(LoggingTransferListener.create())
107+
.build();
108+
109+
FileDownload download = tm.downloadFileWithPresignedUrl(request);
110+
CompletedFileDownload completed = download.completionFuture().join();
111+
112+
assertThat(Files.exists(downloadPath)).isTrue();
113+
assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(largeFile));
114+
assertThat(completed.response().responseMetadata().requestId()).isNotNull();
115+
}
116+
117+
private static PresignedGetObjectRequest createPresignedRequest(String key) {
118+
return presigner.presignGetObject(GetObjectPresignRequest.builder()
119+
.signatureDuration(Duration.ofMinutes(10))
120+
.getObjectRequest(GetObjectRequest.builder()
121+
.bucket(BUCKET)
122+
.key(key)
123+
.build())
124+
.build());
125+
}
126+
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
import software.amazon.awssdk.transfer.s3.model.DownloadRequest;
4242
import software.amazon.awssdk.transfer.s3.model.FileDownload;
4343
import software.amazon.awssdk.transfer.s3.model.FileUpload;
44+
import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest;
45+
import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest;
4446
import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
4547
import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload;
4648
import software.amazon.awssdk.transfer.s3.model.Upload;
@@ -696,6 +698,107 @@ default Copy copy(Consumer<CopyRequest.Builder> copyRequestBuilder) {
696698
return copy(CopyRequest.builder().applyMutation(copyRequestBuilder).build());
697699
}
698700

701+
/**
702+
* Downloads an object using a pre-signed URL to a local file. For non-file-based downloads, you may use
703+
* {@link #downloadWithPresignedUrl(PresignedDownloadRequest)} instead.
704+
* <p>
705+
* This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client,
706+
* providing enhanced throughput and reliability for large objects. Progress can be monitored
707+
* through {@link TransferListener}s attached to the request.
708+
* <p>
709+
* The SDK will create a new file if the provided destination doesn't exist. If the file already exists,
710+
* it will be replaced. In the event of an error, the SDK will <b>NOT</b> attempt to delete
711+
* the file, leaving it as-is.
712+
* <p>
713+
* Note: The result of the operation doesn't support pause and resume functionality.
714+
* </p>
715+
* <p>
716+
* <b>Usage Example:</b>
717+
* {@snippet :
718+
* S3TransferManager transferManager = S3TransferManager.create();
719+
*
720+
* // Create presigned URL (typically done by another service)
721+
* PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder()
722+
* .presignedUrl(presignedUrl)
723+
* .build();
724+
*
725+
* PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder()
726+
* .presignedUrlDownloadRequest(presignedRequest)
727+
* .destination(Paths.get("downloaded-file.txt"))
728+
* .addTransferListener(
729+
* LoggingTransferListener.create())
730+
* .build();
731+
*
732+
* FileDownload download = transferManager.downloadFileWithPresignedUrl(request);
733+
* download.completionFuture().join();
734+
* }
735+
*
736+
* @param presignedDownloadFileRequest the presigned download file request
737+
* @return A {@link FileDownload} that can be used to track the ongoing transfer
738+
* @see #downloadFileWithPresignedUrl(Consumer)
739+
* @see #downloadWithPresignedUrl(PresignedDownloadRequest)
740+
*/
741+
default FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) {
742+
throw new UnsupportedOperationException();
743+
}
744+
745+
/**
746+
* This is a convenience method that creates an instance of the {@link PresignedDownloadFileRequest} builder,
747+
* avoiding the need to create one manually via {@link PresignedDownloadFileRequest#builder()}.
748+
* <p>
749+
* Note: The result of the operation doesn't support pause and resume functionality.
750+
* </p>
751+
*
752+
* @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)
753+
*/
754+
default FileDownload downloadFileWithPresignedUrl(
755+
Consumer<PresignedDownloadFileRequest.Builder> presignedDownloadFileRequest) {
756+
return downloadFileWithPresignedUrl(
757+
PresignedDownloadFileRequest.builder().applyMutation(presignedDownloadFileRequest).build());
758+
}
759+
760+
/**
761+
* Downloads an object using a pre-signed URL through the given {@link AsyncResponseTransformer}. For downloading
762+
* to a file, you may use {@link #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)} instead.
763+
* <p>
764+
* This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client,
765+
* providing enhanced throughput and reliability for large objects. Progress can be monitored
766+
* through {@link TransferListener}s attached to the request.
767+
* <p>
768+
* Note: The result of the operation doesn't support pause and resume functionality.
769+
* </p>
770+
* <p>
771+
* <b>Usage Example (downloading to memory - not suitable for large objects):</b>
772+
* {@snippet :
773+
* S3TransferManager transferManager = S3TransferManager.create();
774+
*
775+
* // Create presigned URL (typically done by another service)
776+
* PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder()
777+
* .presignedUrl(presignedUrl)
778+
* .build();
779+
*
780+
* PresignedDownloadRequest<ResponseBytes<GetObjectResponse>> request =
781+
* PresignedDownloadRequest.builder()
782+
* .presignedUrlDownloadRequest(presignedRequest)
783+
* .responseTransformer(AsyncResponseTransformer.toBytes())
784+
* .addTransferListener(LoggingTransferListener.create())
785+
* .build();
786+
*
787+
* Download<ResponseBytes<GetObjectResponse>> download = transferManager.downloadWithPresignedUrl(request);
788+
* ResponseBytes<GetObjectResponse> result = download.completionFuture().join().result();
789+
* }
790+
*
791+
* @param presignedDownloadRequest the presigned download request
792+
* @param <ResultT> The type of data the {@link AsyncResponseTransformer} produces
793+
* @return A {@link Download} that can be used to track the ongoing transfer
794+
* @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)
795+
* @see AsyncResponseTransformer
796+
*/
797+
default <ResultT> Download<ResultT> downloadWithPresignedUrl(
798+
PresignedDownloadRequest<ResultT> presignedDownloadRequest) {
799+
throw new UnsupportedOperationException();
800+
}
801+
699802
/**
700803
* Create an {@code S3TransferManager} using the default values.
701804
* <p>

0 commit comments

Comments
 (0)