Skip to content

Commit b64302f

Browse files
RanVakninRanVaknin
committed
Adding integ test and serialization unit test coverage
1 parent 35d70a4 commit b64302f

File tree

5 files changed

+101
-15
lines changed

5 files changed

+101
-15
lines changed

‎.changes/next-release/feature-AmazonS3TransferManager-21d3a50.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"type": "feature",
33
"category": "Amazon S3 Transfer Manager",
44
"contributor": "",
5-
"description": "Add support for etag validation in resumableFileDownload"
5+
"description": "Add support for etag validation in resumableFileDownload: restart paused downloads when etag does not match"
66
}

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadPauseResumeIntegrationTest.java

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,16 @@
2121
import static software.amazon.awssdk.transfer.s3.SizeConstant.MB;
2222

2323
import java.io.File;
24+
import java.io.IOException;
2425
import java.nio.charset.StandardCharsets;
2526
import java.nio.file.Files;
2627
import java.nio.file.Path;
2728
import java.time.Duration;
29+
import java.util.List;
2830
import java.util.Optional;
2931
import org.apache.commons.lang3.RandomStringUtils;
32+
import org.apache.logging.log4j.Level;
33+
import org.apache.logging.log4j.core.LogEvent;
3034
import org.junit.jupiter.api.AfterAll;
3135
import org.junit.jupiter.api.BeforeAll;
3236
import org.junit.jupiter.params.ParameterizedTest;
@@ -38,7 +42,10 @@
3842
import software.amazon.awssdk.core.waiters.WaiterAcceptor;
3943
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
4044
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
45+
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
46+
import software.amazon.awssdk.testutils.LogCaptor;
4147
import software.amazon.awssdk.testutils.RandomTempFile;
48+
import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
4249
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;
4350
import software.amazon.awssdk.transfer.s3.model.FileDownload;
4451
import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
@@ -70,6 +77,56 @@ public static void cleanup() {
7077
sourceFile.delete();
7178
}
7279

80+
@ParameterizedTest
81+
@MethodSource("transferManagers")
82+
void pauseAndResume_ObjectEtagChange_shouldRestartDownload(S3TransferManager tm) throws IOException {
83+
Path path = RandomTempFile.randomUncreatedFile().toPath();
84+
85+
TestDownloadListener testDownloadListener = new TestDownloadListener();
86+
DownloadFileRequest request = DownloadFileRequest.builder()
87+
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
88+
.destination(path)
89+
.addTransferListener(testDownloadListener)
90+
.build();
91+
FileDownload download = tm.downloadFile(request);
92+
waitUntilFirstByteBufferDelivered(download);
93+
94+
ResumableFileDownload resumableFileDownload = download.pause();
95+
log.debug(() -> "Paused: " + resumableFileDownload);
96+
97+
String originalEtag = testDownloadListener.getObjectResponse.eTag();
98+
99+
File newSourceFile = new RandomTempFile(OBJ_SIZE);
100+
PutObjectResponse putResponse = s3.putObject(PutObjectRequest.builder()
101+
.bucket(BUCKET)
102+
.key(KEY)
103+
.build(), newSourceFile.toPath());
104+
105+
String newEtag = putResponse.eTag();
106+
assertThat(newEtag).isNotEqualTo(originalEtag);
107+
try (LogCaptor logCaptor = LogCaptor.create(Level.DEBUG)) {
108+
109+
FileDownload resumedFileDownload = tm.resumeDownloadFile(resumableFileDownload);
110+
CompletedFileDownload completedDownload = resumedFileDownload.completionFuture().join();
111+
112+
assertThat(completedDownload.response().eTag()).isEqualTo(newEtag);
113+
assertThat(testDownloadListener.transferInitiatedCount == 2).isTrue();
114+
115+
List<LogEvent> logEvents = logCaptor.loggedEvents();
116+
StringBuilder sb = new StringBuilder();
117+
logEvents.forEach(logEvent -> {
118+
sb.append(logEvent.getMessage().getFormattedMessage());
119+
});
120+
121+
assertThat(sb)
122+
.contains(String.format("The ETag of the requested object in bucket (%s) with key (%s) "
123+
+ "has changed since the last "
124+
+ "pause. The SDK will download the S3 object from "
125+
+ "the beginning",
126+
BUCKET, KEY));
127+
}
128+
}
129+
73130
@ParameterizedTest
74131
@MethodSource("transferManagers")
75132
void pauseAndResume_ObjectNotChanged_shouldResumeDownload(S3TransferManager tm) {
@@ -187,8 +244,14 @@ private static void waitUntilFirstByteBufferDelivered(FileDownload download) {
187244
}
188245

189246
private static final class TestDownloadListener implements TransferListener {
247+
private int transferInitiatedCount = 0;
190248
private GetObjectResponse getObjectResponse;
191249

250+
@Override
251+
public void transferInitiated(Context.TransferInitiated context) {
252+
transferInitiatedCount++;
253+
}
254+
192255
@Override
193256
public void bytesTransferred(Context.BytesTransferred context) {
194257
Optional<SdkResponse> sdkResponse = context.progressSnapshot().sdkResponse();

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultFileDownload.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,18 @@ private ResumableFileDownload doPause() {
6666
completionFuture.cancel(true);
6767

6868
Instant s3objectLastModified = null;
69+
String s3objectEtag = null;
6970
Long totalSizeInBytes = null;
7071
TransferProgressSnapshot snapshot = progress.snapshot();
7172

7273
if (snapshot.sdkResponse().isPresent() && snapshot.sdkResponse().get() instanceof GetObjectResponse) {
7374
GetObjectResponse getObjectResponse = (GetObjectResponse) snapshot.sdkResponse().get();
7475
s3objectLastModified = getObjectResponse.lastModified();
76+
s3objectEtag = getObjectResponse.eTag();
7577
totalSizeInBytes = getObjectResponse.contentLength();
7678
} else if (resumedDownload != null) {
7779
s3objectLastModified = resumedDownload.s3ObjectLastModified().orElse(null);
80+
s3objectEtag = resumedDownload.s3ObjectEtag().orElse(null);
7881
totalSizeInBytes = resumedDownload.totalSizeInBytes().isPresent() ? resumedDownload.totalSizeInBytes().getAsLong()
7982
: null;
8083
}
@@ -87,6 +90,7 @@ private ResumableFileDownload doPause() {
8790
return ResumableFileDownload.builder()
8891
.downloadFileRequest(request)
8992
.s3ObjectLastModified(s3objectLastModified)
93+
.s3ObjectEtag(s3objectEtag)
9094
.fileLastModified(fileLastModified)
9195
.bytesTransferred(length)
9296
.totalSizeInBytes(totalSizeInBytes)

‎services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/serialization/ResumableFileDownloadSerializerTest.java

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ class ResumableFileDownloadSerializerTest {
4949
private static final Instant DATE1 = parseIso8601Date("2022-05-13T21:55:52.529Z");
5050
private static final Instant DATE2 = parseIso8601Date("2022-05-15T21:50:11.308Z");
5151
private static final Map<String, GetObjectRequest> GET_OBJECT_REQUESTS;
52+
private static final String ETAG = "acbd18db4cc2f85cedef654fccc4a4d8";
5253

5354
static {
5455
Map<String, GetObjectRequest> requests = new HashMap<>();
@@ -82,6 +83,7 @@ void serializeDeserialize_fromStoredString_ShouldWork() {
8283
.getObjectRequest(GET_OBJECT_REQUESTS.get("ALL_TYPES")))
8384
.bytesTransferred(1000L)
8485
.fileLastModified(parseIso8601Date("2022-03-08T10:15:30Z"))
86+
.s3ObjectEtag(ETAG)
8587
.totalSizeInBytes(5000L)
8688
.s3ObjectLastModified(parseIso8601Date("2022-03-10T08:21:00Z"))
8789
.build();
@@ -147,6 +149,7 @@ void serializeDeserialize_withCompletedParts_persistCompletedParts() {
147149
.getObjectRequest(GET_OBJECT_REQUESTS.get("ALL_TYPES")))
148150
.bytesTransferred(1000L)
149151
.fileLastModified(parseIso8601Date("2022-03-08T10:15:30Z"))
152+
.s3ObjectEtag(ETAG)
150153
.totalSizeInBytes(5000L)
151154
.s3ObjectLastModified(parseIso8601Date("2022-03-10T08:21:00Z"))
152155
.completedParts(Arrays.asList(1, 2, 3))
@@ -170,26 +173,31 @@ public static Collection<ResumableFileDownload> downloadObjects() {
170173
private static List<ResumableFileDownload> differentGetObjects() {
171174
return GET_OBJECT_REQUESTS.values()
172175
.stream()
173-
.map(request -> resumableFileDownload(1000L, null, DATE1, null, downloadRequest(PATH, request)))
176+
.map(request -> resumableFileDownload(1000L, null, DATE1, null, null, downloadRequest(PATH,
177+
request)))
174178
.collect(Collectors.toList());
175179
}
176180

177181
private static List<ResumableFileDownload> differentDownloadSettings() {
178182
DownloadFileRequest request = downloadRequest(PATH, GET_OBJECT_REQUESTS.get("STANDARD"));
179183
return Arrays.asList(
180-
resumableFileDownload(null, null, null, null, request),
181-
resumableFileDownload(1000L, null, null, null, request),
182-
resumableFileDownload(1000L, null, DATE1, null, request),
183-
resumableFileDownload(1000L, 2000L, DATE1, DATE2, request),
184-
resumableFileDownload(Long.MAX_VALUE, Long.MAX_VALUE, DATE1, DATE2, request),
185-
resumableFileDownload(1000L, 2000L, DATE1, DATE2, request, Arrays.asList(1, 2, 3))
184+
resumableFileDownload(null, null, null, null, null, request),
185+
resumableFileDownload(1000L, null, null, null, null, request),
186+
resumableFileDownload(1000L, null, DATE1, null, null, request),
187+
resumableFileDownload(1000L, 2000L, DATE1, DATE2, null, request),
188+
resumableFileDownload(Long.MAX_VALUE, Long.MAX_VALUE, DATE1, DATE2, null, request),
189+
resumableFileDownload(1000L, 2000L, DATE1, DATE2, null, request, Arrays.asList(1, 2, 3)),
190+
resumableFileDownload(1000L, 2000L, DATE1, DATE2, ETAG, request, Arrays.asList(1, 2, 3)),
191+
resumableFileDownload(1000L, 2000L, DATE1, DATE2, ETAG, request),
192+
resumableFileDownload(1000L, 2000L, DATE1, null, ETAG, request)
186193
);
187194
}
188195

189196
private static ResumableFileDownload resumableFileDownload(Long bytesTransferred,
190197
Long totalSizeInBytes,
191198
Instant fileLastModified,
192199
Instant s3ObjectLastModified,
200+
String s3ObjectEtag,
193201
DownloadFileRequest request) {
194202
ResumableFileDownload.Builder builder = ResumableFileDownload.builder()
195203
.downloadFileRequest(request)
@@ -203,16 +211,20 @@ private static ResumableFileDownload resumableFileDownload(Long bytesTransferred
203211
if (s3ObjectLastModified != null) {
204212
builder.s3ObjectLastModified(s3ObjectLastModified);
205213
}
214+
if (s3ObjectEtag != null) {
215+
builder.s3ObjectEtag(s3ObjectEtag);
216+
}
206217
return builder.build();
207218
}
208219
private static ResumableFileDownload resumableFileDownload(Long bytesTransferred,
209220
Long totalSizeInBytes,
210221
Instant fileLastModified,
211222
Instant s3ObjectLastModified,
223+
String s3ObjectEtag,
212224
DownloadFileRequest request,
213225
List<Integer> completedParts) {
214226
ResumableFileDownload dl = resumableFileDownload(
215-
bytesTransferred, totalSizeInBytes, fileLastModified, s3ObjectLastModified, request);
227+
bytesTransferred, totalSizeInBytes, fileLastModified, s3ObjectLastModified, s3ObjectEtag, request);
216228
return dl.copy(b -> b.completedParts(completedParts));
217229
}
218230

@@ -225,17 +237,21 @@ private static DownloadFileRequest downloadRequest(Path path, GetObjectRequest r
225237

226238
private static final String SERIALIZED_DOWNLOAD_OBJECT = "{\"bytesTransferred\":1000,\"fileLastModified\":1646734530.000,"
227239
+ "\"totalSizeInBytes\":5000,\"s3ObjectLastModified\":1646900460"
228-
+ ".000,\"downloadFileRequest\":{\"destination\":\"test/request\","
240+
+ ".000,\"s3ObjectEtag\":\"acbd18db4cc2f85cedef654fccc4a4d8\","
241+
+ "\"downloadFileRequest\":{\"destination\":\"test/request\","
229242
+ "\"getObjectRequest\":{\"Bucket\":\"BUCKET\","
230243
+ "\"If-Modified-Since\":1577880630.000,\"Key\":\"KEY\","
231244
+ "\"x-amz-request-payer\":\"requester\",\"partNumber\":1,"
232245
+ "\"x-amz-checksum-mode\":\"ENABLED\"}},\"completedParts\":[]}";
233246

247+
248+
234249
private static final String SERIALIZED_DOWNLOAD_OBJECT_WITH_COMPLETED_PARTS =
235250
"{\"bytesTransferred\":1000,"
236251
+ "\"fileLastModified\":1646734530.000,"
237252
+ "\"totalSizeInBytes\":5000,\"s3ObjectLastModified\":1646900460"
238-
+ ".000,\"downloadFileRequest\":{\"destination\":\"test/request\","
253+
+ ".000,\"s3ObjectEtag\":\"acbd18db4cc2f85cedef654fccc4a4d8\","
254+
+ "\"downloadFileRequest\":{\"destination\":\"test/request\","
239255
+ "\"getObjectRequest\":{\"Bucket\":\"BUCKET\","
240256
+ "\"If-Modified-Since\":1577880630.000,\"Key\":\"KEY\","
241257
+ "\"x-amz-request-payer\":\"requester\",\"partNumber\":1,"

‎services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ResumableRequestConverterTest.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@
4747
import software.amazon.awssdk.utils.Pair;
4848

4949
class ResumableRequestConverterTest {
50+
private static final String ETAG_FOO = "acbd18db4cc2f85cedef654fccc4a4d8";
51+
private static final String ETAG_BAR = "37b51d194a7513e45b56f6524f2d51f2";
52+
private static final String ETAG_BAR_UPPERCASE = "ddc35f88fa71b6ef142ae61f35364653";
5053

5154
private File file;
5255

@@ -126,10 +129,10 @@ void toDownloadFileAndTransformer_fileLastModifiedTimeChanged_shouldStartFromBeg
126129

127130
static Stream<Arguments> EtagTestCases() {
128131
return Stream.of(
129-
Arguments.of("acbd18db4cc2f85cedef654fccc4a4d8", null), // etag of "foo"
130-
Arguments.of("37b51d194a7513e45b56f6524f2d51f2", "bytes=1000-2000"), // etag of "bar"
132+
Arguments.of(ETAG_FOO, null),
133+
Arguments.of(ETAG_BAR, "bytes=1000-2000"),
131134
Arguments.of(null, "bytes=1000-2000"),
132-
Arguments.of("ddc35f88fa71b6ef142ae61f35364653", null) // etag of "Bar"
135+
Arguments.of(ETAG_BAR_UPPERCASE, null)
133136
);
134137
}
135138

@@ -216,7 +219,7 @@ private static HeadObjectResponse headObjectResponse(Instant s3ObjectLastModifie
216219
.builder()
217220
.contentLength(2000L)
218221
.lastModified(s3ObjectLastModified)
219-
.eTag("37b51d194a7513e45b56f6524f2d51f2")
222+
.eTag(ETAG_BAR)
220223
.build();
221224
}
222225

0 commit comments

Comments
 (0)