diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java index 1b4f7f105a38..b20445275a30 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java @@ -55,11 +55,13 @@ public class S3IntegrationTestBase extends AwsTestBase { protected static S3Client s3; protected static S3AsyncClient s3Async; + protected static S3AsyncClient s3multiAsync; protected static S3AsyncClient s3CrtAsync; protected static S3TransferManager tmCrt; protected static S3TransferManager tmJava; + protected static S3TransferManager tmMultipartJava; /** * Loads the AWS account info for the integration tests and creates an S3 @@ -71,6 +73,8 @@ public static void setUpForAllIntegTests() throws Exception { System.setProperty("aws.crt.debugnative", "true"); s3 = s3ClientBuilder().build(); s3Async = s3AsyncClientBuilder().build(); + s3multiAsync = s3AsyncClientBuilder().multipartEnabled(true).build(); + s3CrtAsync = S3CrtAsyncClient.builder() .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .region(DEFAULT_REGION) @@ -81,6 +85,9 @@ public static void setUpForAllIntegTests() throws Exception { tmJava = S3TransferManager.builder() .s3Client(s3Async) .build(); + tmMultipartJava = S3TransferManager.builder() + .s3Client(s3multiAsync) + .build(); } @@ -88,8 +95,11 @@ public static void setUpForAllIntegTests() throws Exception { public static void cleanUpForAllIntegTests() { s3.close(); s3Async.close(); + s3multiAsync.close(); s3CrtAsync.close(); tmCrt.close(); + tmJava.close(); + tmMultipartJava.close(); CrtResource.waitForNoResources(); } @@ -181,4 +191,9 @@ static Stream transferManagers() { Arguments.of(tmJava)); } + static Stream javaTransferManagerOnly() { + return Stream.of( + Arguments.of(tmJava), + Arguments.of(tmMultipartJava)); + } } diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java new file mode 100644 index 000000000000..b9e6b688a042 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java @@ -0,0 +1,126 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource;; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.FileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener; +import software.amazon.awssdk.utils.Md5Utils; + +public class S3TransferManagerPresignedUrlDownloadIntegrationTest extends S3IntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3TransferManagerPresignedUrlDownloadIntegrationTest.class); + private static final String SMALL_KEY = "small-key"; + private static final String LARGE_KEY = "large-key"; + private static final int SMALL_OBJ_SIZE = 5 * 1024 * 1024; + private static final int LARGE_OBJ_SIZE = 16 * 1024 * 1024; + + private static File smallFile; + private static File largeFile; + private static S3Presigner presigner; + + @BeforeAll + public static void setup() throws IOException { + createBucket(BUCKET); + smallFile = new RandomTempFile(SMALL_OBJ_SIZE); + largeFile = new RandomTempFile(LARGE_OBJ_SIZE); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SMALL_KEY).build(), smallFile.toPath()); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(LARGE_KEY).build(), largeFile.toPath()); + presigner = S3Presigner.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build(); + } + + @AfterAll + public static void cleanup() { + if (presigner != null) { + presigner.close(); + } + deleteBucketAndAllContents(BUCKET); + } + + @ParameterizedTest + @MethodSource("javaTransferManagerOnly") + void downloadFileWithPresignedUrl_smallFile_downloadedCorrectly(S3TransferManager tm) throws Exception { + PresignedGetObjectRequest presignedRequest = createPresignedRequest(SMALL_KEY); + Path downloadPath = RandomTempFile.randomUncreatedFile().toPath(); + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedRequest.url()) + .build()) + .destination(downloadPath) + .addTransferListener(LoggingTransferListener.create()) + .build(); + + FileDownload download = tm.downloadFileWithPresignedUrl(request); + CompletedFileDownload completed = download.completionFuture().join(); + + assertThat(Files.exists(downloadPath)).isTrue(); + assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(smallFile)); + assertThat(completed.response().responseMetadata().requestId()).isNotNull(); + } + + @ParameterizedTest + @MethodSource("javaTransferManagerOnly") + void downloadFileWithPresignedUrl_largeFile_downloadedCorrectly(S3TransferManager tm) throws Exception { + PresignedGetObjectRequest presignedRequest = createPresignedRequest(LARGE_KEY); + Path downloadPath = RandomTempFile.randomUncreatedFile().toPath(); + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedRequest.url()) + .build()) + .destination(downloadPath) + .addTransferListener(LoggingTransferListener.create()) + .build(); + + FileDownload download = tm.downloadFileWithPresignedUrl(request); + CompletedFileDownload completed = download.completionFuture().join(); + + assertThat(Files.exists(downloadPath)).isTrue(); + assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(largeFile)); + assertThat(completed.response().responseMetadata().requestId()).isNotNull(); + } + + private static PresignedGetObjectRequest createPresignedRequest(String key) { + return presigner.presignGetObject(GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(10)) + .getObjectRequest(GetObjectRequest.builder() + .bucket(BUCKET) + .key(key) + .build()) + .build()); + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 5bf9b55ed657..b1d4c6011996 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -41,6 +41,8 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -696,6 +698,107 @@ default Copy copy(Consumer copyRequestBuilder) { return copy(CopyRequest.builder().applyMutation(copyRequestBuilder).build()); } + /** + * Downloads an object using a pre-signed URL to a local file. For non-file-based downloads, you may use + * {@link #downloadWithPresignedUrl(PresignedDownloadRequest)} instead. + *

+ * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client, + * providing enhanced throughput and reliability for large objects. Progress can be monitored + * through {@link TransferListener}s attached to the request. + *

+ * The SDK will create a new file if the provided destination doesn't exist. If the file already exists, + * it will be replaced. In the event of an error, the SDK will NOT attempt to delete + * the file, leaving it as-is. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ *

+ * Usage Example: + * {@snippet : + * S3TransferManager transferManager = S3TransferManager.create(); + * + * // Create presigned URL (typically done by another service) + * PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder() + * .presignedUrl(presignedUrl) + * .build(); + * + * PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + * .presignedUrlDownloadRequest(presignedRequest) + * .destination(Paths.get("downloaded-file.txt")) + * .addTransferListener( + * LoggingTransferListener.create()) + * .build(); + * + * FileDownload download = transferManager.downloadFileWithPresignedUrl(request); + * download.completionFuture().join(); + * } + * + * @param presignedDownloadFileRequest the presigned download file request + * @return A {@link FileDownload} that can be used to track the ongoing transfer + * @see #downloadFileWithPresignedUrl(Consumer) + * @see #downloadWithPresignedUrl(PresignedDownloadRequest) + */ + default FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + throw new UnsupportedOperationException(); + } + + /** + * This is a convenience method that creates an instance of the {@link PresignedDownloadFileRequest} builder, + * avoiding the need to create one manually via {@link PresignedDownloadFileRequest#builder()}. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ * + * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + */ + default FileDownload downloadFileWithPresignedUrl( + Consumer presignedDownloadFileRequest) { + return downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder().applyMutation(presignedDownloadFileRequest).build()); + } + + /** + * Downloads an object using a pre-signed URL through the given {@link AsyncResponseTransformer}. For downloading + * to a file, you may use {@link #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)} instead. + *

+ * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client, + * providing enhanced throughput and reliability for large objects. Progress can be monitored + * through {@link TransferListener}s attached to the request. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ *

+ * Usage Example (downloading to memory - not suitable for large objects): + * {@snippet : + * S3TransferManager transferManager = S3TransferManager.create(); + * + * // Create presigned URL (typically done by another service) + * PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder() + * .presignedUrl(presignedUrl) + * .build(); + * + * PresignedDownloadRequest> request = + * PresignedDownloadRequest.builder() + * .presignedUrlDownloadRequest(presignedRequest) + * .responseTransformer(AsyncResponseTransformer.toBytes()) + * .addTransferListener(LoggingTransferListener.create()) + * .build(); + * + * Download> download = transferManager.downloadWithPresignedUrl(request); + * ResponseBytes result = download.completionFuture().join().result(); + * } + * + * @param presignedDownloadRequest the presigned download request + * @param The type of data the {@link AsyncResponseTransformer} produces + * @return A {@link Download} that can be used to track the ongoing transfer + * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + * @see AsyncResponseTransformer + */ + default Download downloadWithPresignedUrl( + PresignedDownloadRequest presignedDownloadRequest) { + throw new UnsupportedOperationException(); + } + /** * Create an {@code S3TransferManager} using the default values. *

diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index 29bcdc34d93f..c8dbc5cfe28b 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -78,6 +78,8 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -238,7 +240,6 @@ private boolean isS3ClientMultipartEnabled() { return s3AsyncClient instanceof MultipartS3AsyncClient; } - /** * Can be overridden by subclasses to provide different implementation */ @@ -426,8 +427,8 @@ public final FileDownload resumeDownloadFile(ResumableFileDownload resumableFile Optional optCtx = multipartDownloadResumeContext(resumableFileDownload.downloadFileRequest().getObjectRequest()); if (optCtx.map(MultipartDownloadResumeContext::isComplete).orElse(false)) { - log.debug(() -> "The multipart download associated to the provided ResumableFileDownload is already completed, " - + "nothing to resume"); + log.debug(() -> "The multipart download associated to the provided ResumableFileDownload is already " + + "completed, nothing to resume"); return completedDownload(resumableFileDownload, optCtx.get()); } @@ -562,6 +563,76 @@ public final Copy copy(CopyRequest copyRequest) { return new DefaultCopy(returnFuture, progressUpdater.progress()); } + @Override + public final FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + Validate.paramNotNull(presignedDownloadFileRequest, "presignedDownloadFileRequest"); + + AsyncResponseTransformer responseTransformer = + AsyncResponseTransformer.toFile(presignedDownloadFileRequest.destination(), + FileTransformerConfiguration.defaultCreateOrReplaceExisting()); + + CompletableFuture returnFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(presignedDownloadFileRequest, null); + progressUpdater.transferInitiated(); + + responseTransformer = isS3ClientMultipartEnabled() + ? progressUpdater.wrapResponseTransformerForMultipartDownload( + responseTransformer, GetObjectRequest.builder().build()) + : progressUpdater.wrapResponseTransformer(responseTransformer); + progressUpdater.registerCompletion(returnFuture); + + try { + CompletableFuture future = s3AsyncClient.presignedUrlExtension().getObject( + presignedDownloadFileRequest.presignedUrlDownloadRequest(), responseTransformer); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, future); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, + res -> CompletedFileDownload.builder() + .response(res) + .build()); + } catch (Throwable throwable) { + returnFuture.completeExceptionally(throwable); + } + + return new DefaultFileDownload(returnFuture, progressUpdater.progress(), () -> null, null); + } + + @Override + public final Download downloadWithPresignedUrl( + PresignedDownloadRequest presignedDownloadRequest) { + Validate.paramNotNull(presignedDownloadRequest, "presignedDownloadRequest"); + + AsyncResponseTransformer responseTransformer = + presignedDownloadRequest.responseTransformer(); + + CompletableFuture> returnFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(presignedDownloadRequest, null); + progressUpdater.transferInitiated(); + + responseTransformer = isS3ClientMultipartEnabled() + ? progressUpdater.wrapResponseTransformerForMultipartDownload( + responseTransformer, GetObjectRequest.builder().build()) + : progressUpdater.wrapResponseTransformer(responseTransformer); + progressUpdater.registerCompletion(returnFuture); + + try { + CompletableFuture future = s3AsyncClient.presignedUrlExtension().getObject( + presignedDownloadRequest.presignedUrlDownloadRequest(), responseTransformer); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, future); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, + r -> CompletedDownload.builder() + .result(r) + .build()); + } catch (Throwable throwable) { + returnFuture.completeExceptionally(throwable); + } + + return new DefaultDownload<>(returnFuture, progressUpdater.progress()); + } + @Override public final void close() { if (isDefaultS3AsyncClient) { diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java index b0bf72e91404..3b77c3f7281b 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java @@ -42,12 +42,16 @@ public final class DefaultTransferProgressSnapshot private DefaultTransferProgressSnapshot(Builder builder) { if (builder.totalBytes != null) { Validate.isNotNegative(builder.totalBytes, "totalBytes"); - Validate.isTrue(builder.transferredBytes <= builder.totalBytes, - "transferredBytes (%s) must not be greater than totalBytes (%s)", - builder.transferredBytes, builder.totalBytes); } Validate.paramNotNull(builder.transferredBytes, "byteTransferred"); - this.transferredBytes = Validate.isNotNegative(builder.transferredBytes, "transferredBytes"); + long validatedTransferredBytes = Validate.isNotNegative(builder.transferredBytes, "transferredBytes"); + + // Clamp transferredBytes to not exceed totalBytes to handle race conditions + if (builder.totalBytes != null && validatedTransferredBytes > builder.totalBytes) { + validatedTransferredBytes = builder.totalBytes; + } + + this.transferredBytes = validatedTransferredBytes; this.totalBytes = builder.totalBytes; this.sdkResponse = builder.sdkResponse; } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultTransferProgressSnapshotTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultTransferProgressSnapshotTest.java index 95f78ff621eb..ada2784871a8 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultTransferProgressSnapshotTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultTransferProgressSnapshotTest.java @@ -24,13 +24,14 @@ public class DefaultTransferProgressSnapshotTest { @Test - public void bytesTransferred_greaterThan_transferSize_shouldThrow() { - DefaultTransferProgressSnapshot.Builder builder = DefaultTransferProgressSnapshot.builder() - .transferredBytes(2L) - .totalBytes(1L); - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("transferredBytes (2) must not be greater than totalBytes (1)"); + public void bytesTransferred_greaterThan_transferSize_shouldClamp() { + TransferProgressSnapshot snapshot = DefaultTransferProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(1L) + .build(); + // transferredBytes should be clamped to totalBytes to handle race conditions + assertThat(snapshot.transferredBytes()).isEqualTo(1L); + assertThat(snapshot.totalBytes()).hasValue(1L); } @Test diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java new file mode 100644 index 000000000000..c034f51747ba --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java @@ -0,0 +1,151 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.CompletedDownload; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; + +/** + * Test for S3TransferManager presigned URL download functionality. + */ +class S3TransferManagerPresignedUrlDownloadTest { + private S3AsyncClient mockS3AsyncClient; + private AsyncPresignedUrlExtension mockPresignedUrlExtension; + private GenericS3TransferManager tm; + private TransferManagerConfiguration configuration; + + @BeforeEach + public void methodSetup() { + mockS3AsyncClient = mock(S3AsyncClient.class); + mockPresignedUrlExtension = mock(AsyncPresignedUrlExtension.class); + configuration = mock(TransferManagerConfiguration.class); + + when(mockS3AsyncClient.presignedUrlExtension()).thenReturn(mockPresignedUrlExtension); + + tm = new GenericS3TransferManager(mockS3AsyncClient, + mock(UploadDirectoryHelper.class), + configuration, + mock(DownloadDirectoryHelper.class)); + } + + @AfterEach + public void methodTeardown() { + tm.close(); + } + + @Test + void downloadFileWithPresignedUrl_withValidRequest_returnsResponse() throws Exception { + GetObjectResponse response = GetObjectResponse.builder().build(); + when(mockPresignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + URL presignedUrl = new URL("https://test-bucket.s3.amazonaws.com/test-key?presigned=true"); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(Paths.get("/tmp/test")) + .build(); + + CompletedFileDownload completedFileDownload = tm.downloadFileWithPresignedUrl(request) + .completionFuture() + .join(); + + assertThat(completedFileDownload.response()).isEqualTo(response); + } + + @Test + void downloadWithPresignedUrl_withValidRequest_returnsResponse() throws Exception { + ResponseBytes responseBytes = ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), "test".getBytes()); + when(mockPresignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(CompletableFuture.completedFuture(responseBytes)); + + URL presignedUrl = new URL("https://test-bucket.s3.amazonaws.com/test-key?presigned=true"); + + PresignedDownloadRequest> request = PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .build(); + + CompletedDownload> completedDownload = tm.downloadWithPresignedUrl(request) + .completionFuture() + .join(); + + assertThat(completedDownload.result()).isEqualTo(responseBytes); + } + + @Test + void downloadFileWithPresignedUrl_withConsumerBuilder_returnsResponse() throws Exception { + GetObjectResponse response = GetObjectResponse.builder().build(); + when(mockPresignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + URL presignedUrl = new URL("https://test-bucket.s3.amazonaws.com/test-key?presigned=true"); + + CompletedFileDownload completedFileDownload = tm.downloadFileWithPresignedUrl( + request -> request.presignedUrlDownloadRequest(p -> p.presignedUrl(presignedUrl)) + .destination(Paths.get("/tmp/test"))) + .completionFuture() + .join(); + + assertThat(completedFileDownload.response()).isEqualTo(response); + } + + @Test + void downloadFileWithPresignedUrl_whenCancelled_shouldForwardCancellation() throws Exception { + CompletableFuture s3Future = new CompletableFuture<>(); + when(mockPresignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(s3Future); + + URL presignedUrl = new URL("https://test-bucket.s3.amazonaws.com/test-key?presigned=true"); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(Paths.get("/tmp/test")) + .build(); + + CompletableFuture future = tm.downloadFileWithPresignedUrl(request) + .completionFuture(); + + future.cancel(true); + assertThat(s3Future).isCancelled(); + } +} \ No newline at end of file diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java new file mode 100644 index 000000000000..29e7fd960b96 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.awssdk.services.s3.presignedurl; + +import org.junit.jupiter.api.BeforeAll; +import software.amazon.awssdk.services.s3.S3AsyncClient; + + +public class AsyncPresignedUrlExtensionMultipartIntegrationTest extends AsyncPresignedUrlExtensionTestSuite { + + @BeforeAll + static void setUpIntegrationTest() { + S3AsyncClient s3AsyncClient = s3AsyncClientBuilder() + .multipartEnabled(true) + .build(); + presignedUrlExtension = s3AsyncClient.presignedUrlExtension(); + } + + @Override + protected S3AsyncClient createS3AsyncClient() { + return s3AsyncClientBuilder().build(); + } +} \ No newline at end of file diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java index eee244ca3485..6aaac73c9d88 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java @@ -86,7 +86,7 @@ static void setUpTestSuite() throws Exception { testGetObjectKey = generateRandomObjectKey(); testLargeObjectKey = generateRandomObjectKey() + "-large"; testObjectContent = "Hello AsyncPresignedUrlExtension Integration Test"; - testLargeObjectContent = randomAscii(5 * 1024 * 1024).getBytes(StandardCharsets.UTF_8); + testLargeObjectContent = randomAscii(15 * 1024 * 1024).getBytes(StandardCharsets.UTF_8); try (ByteArrayInputStream originalStream = new ByteArrayInputStream(testLargeObjectContent)) { expectedLargeObjectMd5 = Md5Utils.md5AsBase64(originalStream); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index 975ae0339e78..145567e150c0 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -109,20 +109,20 @@ private void makeRangeRequest(int partIndex, GetObjectResponse> asyncResponseTransformer) { PresignedUrlDownloadRequest partRequest = createRangedGetRequest(partIndex); log.debug(() -> "Sending range request for part " + partIndex + " with range=" + partRequest.range()); - + requestsSent.incrementAndGet(); s3AsyncClient.presignedUrlExtension() - .getObject(partRequest, asyncResponseTransformer) - .whenComplete((response, error) -> { - if (error != null) { - log.debug(() -> "Error encountered during part request for part " + partIndex); - handleError(error); - return; - } - if (validatePart(response, partIndex, asyncResponseTransformer)) { - requestMoreIfNeeded(completedParts.get()); - } - }); + .getObject(partRequest, asyncResponseTransformer) + .whenComplete((response, error) -> { + if (error != null) { + log.debug(() -> "Error encountered during part request for part " + partIndex); + handleError(error); + return; + } + if (validatePart(response, partIndex, asyncResponseTransformer)) { + requestMoreIfNeeded(completedParts.get()); + } + }); } private boolean validatePart(GetObjectResponse response, int partIndex, @@ -144,7 +144,7 @@ private boolean validatePart(GetObjectResponse response, int partIndex, handleError(validationError.get()); return false; } - + if (totalContentLength == null && responseContentRange != null) { Optional parsedContentLength = MultipartDownloadUtils.parseContentRangeForTotalSize(responseContentRange); if (!parsedContentLength.isPresent()) { @@ -186,13 +186,33 @@ private Optional validateResponse(GetObjectResponse response if (contentRange == null) { return Optional.of(PresignedUrlDownloadHelper.missingContentRangeHeader()); } - + Long contentLength = response.contentLength(); if (contentLength == null || contentLength < 0) { return Optional.of(PresignedUrlDownloadHelper.invalidContentLength()); } long expectedStartByte = partIndex * configuredPartSizeInBytes; + + // For the first part, we need to determine the actual object size from the response + if (partIndex == 0 && totalContentLength == null) { + // Parse total content length from the Content-Range header for validation + Optional parsedContentLength = MultipartDownloadUtils.parseContentRangeForTotalSize(contentRange); + if (parsedContentLength.isPresent()) { + long actualTotalLength = parsedContentLength.get(); + // If the object is smaller than our part size, we should expect the full object + if (actualTotalLength <= configuredPartSizeInBytes) { + String expectedRange = "bytes 0-" + (actualTotalLength - 1) + "/"; + if (!contentRange.startsWith(expectedRange)) { + return Optional.of(SdkClientException.create( + "Content-Range mismatch for small object. Expected range starting with: " + expectedRange + + ", but got: " + contentRange)); + } + return Optional.empty(); // Skip further validation for small objects + } + } + } + long expectedEndByte; if (totalContentLength != null) { expectedEndByte = Math.min(expectedStartByte + configuredPartSizeInBytes - 1, totalContentLength - 1); @@ -202,10 +222,24 @@ private Optional validateResponse(GetObjectResponse response String expectedRange = "bytes " + expectedStartByte + "-" + expectedEndByte + "/"; if (!contentRange.startsWith(expectedRange)) { return Optional.of(SdkClientException.create( - "Content-Range mismatch. Expected range starting with: " + expectedRange + + "Content-Range mismatch. Expected range starting with: " + expectedRange + ", but got: " + contentRange)); } + // Skip part size validation if we already handled small object case above + if (partIndex == 0 && totalContentLength == null) { + Optional parsedContentLength = MultipartDownloadUtils.parseContentRangeForTotalSize(contentRange); + if (parsedContentLength.isPresent() && parsedContentLength.get() <= configuredPartSizeInBytes) { + // For small objects, the content length should match the actual object size + if (!contentLength.equals(parsedContentLength.get())) { + return Optional.of(SdkClientException.create( + String.format("Small object content length validation failed. Expected: %d, but got: %d", + parsedContentLength.get(), contentLength))); + } + return Optional.empty(); // Skip remaining validation + } + } + long expectedPartSize; if (totalContentLength != null && partIndex == totalParts - 1) { expectedPartSize = totalContentLength - (partIndex * configuredPartSizeInBytes); @@ -215,19 +249,19 @@ private Optional validateResponse(GetObjectResponse response if (!contentLength.equals(expectedPartSize)) { return Optional.of(SdkClientException.create( String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", - partIndex, expectedPartSize, contentLength))); + partIndex, expectedPartSize, contentLength))); } long actualStartByte = MultipartDownloadUtils.parseStartByteFromContentRange(contentRange); if (actualStartByte != expectedStartByte) { return Optional.of(SdkClientException.create( - "Content range offset mismatch for part " + partIndex + + "Content range offset mismatch for part " + partIndex + ". Expected start: " + expectedStartByte + ", but got: " + actualStartByte)); } - + return Optional.empty(); } - + private int calculateTotalParts(long contentLength, long partSize) { return (int) Math.ceil((double) contentLength / partSize); } @@ -246,7 +280,7 @@ private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { } String rangeHeader = BYTES_RANGE_PREFIX + startByte + "-" + endByte; PresignedUrlDownloadRequest.Builder builder = presignedUrlDownloadRequest.toBuilder() - .range(rangeHeader); + .range(rangeHeader); if (partIndex > 0 && eTag != null) { builder.ifMatch(eTag); }