Skip to content

Add presigned URL download support to S3 Transfer Manager #6364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: feature/master/pre-signed-url-getobject
Choose a base branch
from
Open

Add presigned URL download support to S3 Transfer Manager #6364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
592f9a4
PresignedUrl multipart, progress tracking support with transfermanager
jencymaryjoseph Aug 21, 2025
9d121c5
Adding multipart Integ test
jencymaryjoseph Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions ...ransfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,13 @@ public class S3IntegrationTestBase extends AwsTestBase {
protected static S3Client s3;

protected static S3AsyncClient s3Async;
protected static S3AsyncClient s3multiAsync;

protected static S3AsyncClient s3CrtAsync;

protected static S3TransferManager tmCrt;
protected static S3TransferManager tmJava;
protected static S3TransferManager tmMultipartJava;

/**
* Loads the AWS account info for the integration tests and creates an S3
Expand All @@ -71,6 +73,8 @@ public static void setUpForAllIntegTests() throws Exception {
System.setProperty("aws.crt.debugnative", "true");
s3 = s3ClientBuilder().build();
s3Async = s3AsyncClientBuilder().build();
s3multiAsync = s3AsyncClientBuilder().multipartEnabled(true).build();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Redundant, s3AsyncClientBuilder() already sets multipartEnabled(true)


s3CrtAsync = S3CrtAsyncClient.builder()
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.region(DEFAULT_REGION)
Expand All @@ -81,15 +85,21 @@ public static void setUpForAllIntegTests() throws Exception {
tmJava = S3TransferManager.builder()
.s3Client(s3Async)
.build();
tmMultipartJava = S3TransferManager.builder()
.s3Client(s3multiAsync)
.build();

}

@AfterAll
public static void cleanUpForAllIntegTests() {
s3.close();
s3Async.close();
s3multiAsync.close();
s3CrtAsync.close();
tmCrt.close();
tmJava.close();
tmMultipartJava.close();
CrtResource.waitForNoResources();
}

Expand Down Expand Up @@ -181,4 +191,9 @@ static Stream<Arguments> transferManagers() {
Arguments.of(tmJava));
}

static Stream<Arguments> javaTransferManagerOnly() {
return Stream.of(
Arguments.of(tmJava),
Arguments.of(tmMultipartJava));
}
}
126 changes: 126 additions & 0 deletions ...tware/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.awssdk.transfer.s3;

import static org.assertj.core.api.Assertions.assertThat;
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest;
import software.amazon.awssdk.testutils.RandomTempFile;
import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;
import software.amazon.awssdk.utils.Md5Utils;

public class S3TransferManagerPresignedUrlDownloadIntegrationTest extends S3IntegrationTestBase {
private static final String BUCKET = temporaryBucketName(S3TransferManagerPresignedUrlDownloadIntegrationTest.class);
private static final String SMALL_KEY = "small-key";
private static final String LARGE_KEY = "large-key";
private static final int SMALL_OBJ_SIZE = 5 * 1024 * 1024;
private static final int LARGE_OBJ_SIZE = 16 * 1024 * 1024;

private static File smallFile;
private static File largeFile;
private static S3Presigner presigner;

@BeforeAll
public static void setup() throws IOException {
createBucket(BUCKET);
smallFile = new RandomTempFile(SMALL_OBJ_SIZE);
largeFile = new RandomTempFile(LARGE_OBJ_SIZE);
s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SMALL_KEY).build(), smallFile.toPath());
s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(LARGE_KEY).build(), largeFile.toPath());
presigner = S3Presigner.builder()
.region(DEFAULT_REGION)
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.build();
}

@AfterAll
public static void cleanup() {
if (presigner != null) {
presigner.close();
}
deleteBucketAndAllContents(BUCKET);
}

@ParameterizedTest
@MethodSource("javaTransferManagerOnly")
void downloadFileWithPresignedUrl_smallFile_downloadedCorrectly(S3TransferManager tm) throws Exception {
PresignedGetObjectRequest presignedRequest = createPresignedRequest(SMALL_KEY);
Path downloadPath = RandomTempFile.randomUncreatedFile().toPath();
PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder()
.presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder()
.presignedUrl(presignedRequest.url())
.build())
.destination(downloadPath)
.addTransferListener(LoggingTransferListener.create())
.build();

FileDownload download = tm.downloadFileWithPresignedUrl(request);
CompletedFileDownload completed = download.completionFuture().join();

assertThat(Files.exists(downloadPath)).isTrue();
assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(smallFile));
assertThat(completed.response().responseMetadata().requestId()).isNotNull();
}

@ParameterizedTest
@MethodSource("javaTransferManagerOnly")
void downloadFileWithPresignedUrl_largeFile_downloadedCorrectly(S3TransferManager tm) throws Exception {
PresignedGetObjectRequest presignedRequest = createPresignedRequest(LARGE_KEY);
Path downloadPath = RandomTempFile.randomUncreatedFile().toPath();
PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder()
.presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder()
.presignedUrl(presignedRequest.url())
.build())
.destination(downloadPath)
.addTransferListener(LoggingTransferListener.create())
.build();

FileDownload download = tm.downloadFileWithPresignedUrl(request);
CompletedFileDownload completed = download.completionFuture().join();

assertThat(Files.exists(downloadPath)).isTrue();
assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(largeFile));
assertThat(completed.response().responseMetadata().requestId()).isNotNull();
}

private static PresignedGetObjectRequest createPresignedRequest(String key) {
return presigner.presignGetObject(GetObjectPresignRequest.builder()
.signatureDuration(Duration.ofMinutes(10))
.getObjectRequest(GetObjectRequest.builder()
.bucket(BUCKET)
.key(key)
.build())
.build());
}
}
103 changes: 103 additions & 0 deletions ...-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
import software.amazon.awssdk.transfer.s3.model.DownloadRequest;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
import software.amazon.awssdk.transfer.s3.model.FileUpload;
import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest;
import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest;
import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload;
import software.amazon.awssdk.transfer.s3.model.Upload;
Expand Down Expand Up @@ -696,6 +698,107 @@ default Copy copy(Consumer<CopyRequest.Builder> copyRequestBuilder) {
return copy(CopyRequest.builder().applyMutation(copyRequestBuilder).build());
}

/**
* Downloads an object using a pre-signed URL to a local file. For non-file-based downloads, you may use
* {@link #downloadWithPresignedUrl(PresignedDownloadRequest)} instead.
* <p>
* This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client,
* providing enhanced throughput and reliability for large objects. Progress can be monitored
* through {@link TransferListener}s attached to the request.
* <p>
* The SDK will create a new file if the provided destination doesn't exist. If the file already exists,
* it will be replaced. In the event of an error, the SDK will <b>NOT</b> attempt to delete
* the file, leaving it as-is.
* <p>
* Note: The result of the operation doesn't support pause and resume functionality.
* </p>
* <p>
* <b>Usage Example:</b>
* {@snippet :
* S3TransferManager transferManager = S3TransferManager.create();
*
* // Create presigned URL (typically done by another service)
* PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder()
* .presignedUrl(presignedUrl)
* .build();
*
* PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder()
* .presignedUrlDownloadRequest(presignedRequest)
* .destination(Paths.get("downloaded-file.txt"))
* .addTransferListener(
* LoggingTransferListener.create())
* .build();
*
* FileDownload download = transferManager.downloadFileWithPresignedUrl(request);
* download.completionFuture().join();
* }
*
* @param presignedDownloadFileRequest the presigned download file request
* @return A {@link FileDownload} that can be used to track the ongoing transfer
* @see #downloadFileWithPresignedUrl(Consumer)
* @see #downloadWithPresignedUrl(PresignedDownloadRequest)
*/
default FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) {
throw new UnsupportedOperationException();
}

/**
* This is a convenience method that creates an instance of the {@link PresignedDownloadFileRequest} builder,
* avoiding the need to create one manually via {@link PresignedDownloadFileRequest#builder()}.
* <p>
* Note: The result of the operation doesn't support pause and resume functionality.
* </p>
*
* @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)
*/
default FileDownload downloadFileWithPresignedUrl(
Consumer<PresignedDownloadFileRequest.Builder> presignedDownloadFileRequest) {
return downloadFileWithPresignedUrl(
PresignedDownloadFileRequest.builder().applyMutation(presignedDownloadFileRequest).build());
}

/**
* Downloads an object using a pre-signed URL through the given {@link AsyncResponseTransformer}. For downloading
* to a file, you may use {@link #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)} instead.
* <p>
* This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client,
* providing enhanced throughput and reliability for large objects. Progress can be monitored
* through {@link TransferListener}s attached to the request.
* <p>
* Note: The result of the operation doesn't support pause and resume functionality.
* </p>
* <p>
* <b>Usage Example (downloading to memory - not suitable for large objects):</b>
* {@snippet :
* S3TransferManager transferManager = S3TransferManager.create();
*
* // Create presigned URL (typically done by another service)
* PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder()
* .presignedUrl(presignedUrl)
* .build();
*
* PresignedDownloadRequest<ResponseBytes<GetObjectResponse>> request =
* PresignedDownloadRequest.builder()
* .presignedUrlDownloadRequest(presignedRequest)
* .responseTransformer(AsyncResponseTransformer.toBytes())
* .addTransferListener(LoggingTransferListener.create())
* .build();
*
* Download<ResponseBytes<GetObjectResponse>> download = transferManager.downloadWithPresignedUrl(request);
* ResponseBytes<GetObjectResponse> result = download.completionFuture().join().result();
* }
*
* @param presignedDownloadRequest the presigned download request
* @param <ResultT> The type of data the {@link AsyncResponseTransformer} produces
* @return A {@link Download} that can be used to track the ongoing transfer
* @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)
* @see AsyncResponseTransformer
*/
default <ResultT> Download<ResultT> downloadWithPresignedUrl(
PresignedDownloadRequest<ResultT> presignedDownloadRequest) {
throw new UnsupportedOperationException();
}

/**
* Create an {@code S3TransferManager} using the default values.
* <p>
Expand Down
Loading