Skip to content

Commit cd0172d

Browse files
authored
Remove def 16MB read chunk size (#5941)
* Remove def 16MB read chunk size This commit removes the hardcoded 16MiB read chunk size for uploadFile() in the TransferManager. This was originally a workaround to improve performance when using CRT; however this is no longer necessary because file uploads are done entirely in CRT without the SDK/Java layer doing any reads. Without this default size, the default read chunk size reverts back to the default from `FileAsyncRequestBody` which is 16KiB. * Fix test
1 parent d7ed24d commit cd0172d

File tree

2 files changed

+3
-5
lines changed

2 files changed

+3
-5
lines changed

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import static software.amazon.awssdk.services.s3.multipart.S3MultipartExecutionAttribute.MULTIPART_DOWNLOAD_RESUME_CONTEXT;
2121
import static software.amazon.awssdk.services.s3.multipart.S3MultipartExecutionAttribute.PAUSE_OBSERVABLE;
2222
import static software.amazon.awssdk.services.s3.multipart.S3MultipartExecutionAttribute.RESUME_TOKEN;
23-
import static software.amazon.awssdk.transfer.s3.SizeConstant.MB;
2423
import static software.amazon.awssdk.transfer.s3.internal.utils.ResumableRequestConverter.toDownloadFileRequestAndTransformer;
2524

2625
import java.util.Optional;
@@ -94,7 +93,6 @@
9493

9594
@SdkInternalApi
9695
class GenericS3TransferManager implements S3TransferManager {
97-
protected static final int DEFAULT_FILE_UPLOAD_CHUNK_SIZE = (int) (16 * MB);
9896
private static final Logger log = Logger.loggerFor(S3TransferManager.class);
9997
private static final PauseResumeHelper PAUSE_RESUME_HELPER = new PauseResumeHelper();
10098
private final S3AsyncClient s3AsyncClient;
@@ -180,7 +178,6 @@ public FileUpload uploadFile(UploadFileRequest uploadFileRequest) {
180178
AsyncRequestBody requestBody =
181179
FileAsyncRequestBody.builder()
182180
.path(uploadFileRequest.source())
183-
.chunkSizeInBytes(DEFAULT_FILE_UPLOAD_CHUNK_SIZE)
184181
.build();
185182

186183
CompletableFuture<CompletedFileUpload> returnFuture = new CompletableFuture<>();

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3JavaMultipartTransferProgressListenerTest.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ void listeners_reports_ErrorsWhenCancelled(boolean multipartEnabled) {
179179
assertMockOnFailure(transferListenerMock);
180180
}
181181

182-
@ParameterizedTest
182+
@ParameterizedTest(name = "multipartEnabled = {0}")
183183
@ValueSource(booleans = {true, false})
184184
void listeners_reports_ProgressWhenSuccess(boolean multipartEnabled) {
185185
S3AsyncClient s3Async = s3AsyncClient(multipartEnabled);
@@ -209,7 +209,8 @@ void listeners_reports_ProgressWhenSuccess(boolean multipartEnabled) {
209209
Mockito.verify(transferListenerMock, times(1)).transferInitiated(ArgumentMatchers.any());
210210
Mockito.verify(transferListenerMock, times(1)).transferComplete(ArgumentMatchers.any());
211211

212-
int numTimesBytesTransferred = multipartEnabled ? 2 : 1;
212+
// when false, the generic S3 TM will read 16KiB chunks, so OBJ_SIZE / 16KiB = 16MiB / 16KiB = 1024
213+
int numTimesBytesTransferred = multipartEnabled ? 2 : 1024;
213214
Mockito.verify(transferListenerMock, times(numTimesBytesTransferred)).bytesTransferred(ArgumentMatchers.any());
214215
}
215216

0 commit comments

Comments
 (0)