Skip to content

Commit f13539f

Browse files
RanVakninRanVaknin
authored
Fix handling of empty S3 objects in S3ChecksumValidatingInputStream (#6628)
* Fix handling of empty S3 objects in S3ChecksumValidatingInputStream * Add integ tests * Added changelog
1 parent 39e3b6f commit f13539f

File tree

5 files changed

+128
-4
lines changed

5 files changed

+128
-4
lines changed

‎.changes/next-release/bugfix-S3-7ca353f.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "bugfix",
3+
"category": "S3",
4+
"contributor": "",
5+
"description": "Fixed single-byte `read()` on empty S3 objects returning checksum metadata instead of EOF"
6+
}

‎services/s3/src/it/java/software/amazon/awssdk/services/s3/urlconnection/EmptyFileS3IntegrationTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,17 @@
1818
import static org.assertj.core.api.Assertions.assertThat;
1919
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
2020

21+
import java.util.concurrent.CompletableFuture;
2122
import org.junit.jupiter.api.AfterAll;
2223
import org.junit.jupiter.api.BeforeAll;
2324
import org.junit.jupiter.api.Test;
25+
import software.amazon.awssdk.core.ResponseInputStream;
26+
import software.amazon.awssdk.core.async.AsyncRequestBody;
27+
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
2428
import software.amazon.awssdk.core.sync.RequestBody;
29+
import software.amazon.awssdk.services.s3.S3AsyncClient;
2530
import software.amazon.awssdk.services.s3.S3Client;
31+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
2632

2733
public class EmptyFileS3IntegrationTest extends UrlHttpConnectionS3IntegrationTestBase {
2834
private static final String BUCKET = temporaryBucketName(EmptyFileS3IntegrationTest.class);
@@ -54,4 +60,50 @@ public void s3EmptyFileContentLengthIsCorrectWithoutChecksumValidationEnabled()
5460
assertThat(s3.getObject(r -> r.bucket(BUCKET).key("x")).response().contentLength()).isEqualTo(0);
5561
}
5662
}
63+
64+
@Test
65+
public void s3EmptyFileGetAsInputStreamReturnsEOF() throws Exception{
66+
try (S3Client s3 = s3ClientBuilder().build()) {
67+
s3.putObject(r -> r.bucket(BUCKET).key("x"), RequestBody.empty());
68+
69+
ResponseInputStream<GetObjectResponse> stream =
70+
s3.getObject(r -> r.bucket(BUCKET).key("x"));
71+
72+
assertThat(stream.read()).isEqualTo(-1);
73+
74+
stream.close();
75+
}
76+
}
77+
78+
@Test
79+
public void syncEmptyObjectWithChecksumValidation_arrayRead_returnsEOF() throws Exception {
80+
try (S3Client s3 = s3ClientBuilder().build()) {
81+
s3.putObject(r -> r.bucket(BUCKET).key("x"), RequestBody.empty());
82+
83+
ResponseInputStream<GetObjectResponse> stream =
84+
s3.getObject(r -> r.bucket(BUCKET).key("x"));
85+
86+
byte[] buffer = new byte[100];
87+
int bytesRead = stream.read(buffer);
88+
89+
assertThat(bytesRead).isEqualTo(-1);
90+
91+
stream.close();
92+
}
93+
}
94+
95+
@Test
96+
public void asyncEmptyObjectWithChecksumValidation_returnsEmpty() throws Exception {
97+
try (S3AsyncClient s3Async = S3AsyncClient.builder().region(DEFAULT_REGION).build()) {
98+
s3Async.putObject(r -> r.bucket(BUCKET).key("x"),AsyncRequestBody.empty()).join();
99+
100+
101+
CompletableFuture<byte[]> future = s3Async.getObject(r -> r.bucket(BUCKET).key("x"),
102+
AsyncResponseTransformer.toBytes()).thenApply(
103+
res -> res.asByteArray());
104+
105+
byte[] content = future.join();
106+
assertThat(content).isEmpty();
107+
}
108+
}
57109
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/checksums/S3ChecksumValidatingInputStream.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,18 @@ public S3ChecksumValidatingInputStream(InputStream in, SdkChecksum cksum, long s
5858
*/
5959
@Override
6060
public int read() throws IOException {
61+
if (strippedLength == 0 && lengthRead == 0) {
62+
for (int i = 0; i < CHECKSUM_SIZE; i++) {
63+
int b = inputStream.read();
64+
if (b != -1) {
65+
streamChecksum[i] = (byte) b;
66+
}
67+
}
68+
lengthRead = CHECKSUM_SIZE;
69+
validateAndThrow();
70+
return -1;
71+
}
72+
6173
int read = inputStream.read();
6274

6375
if (read != -1 && lengthRead < strippedLength) {

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/checksums/S3ChecksumValidatingInputStreamTest.java‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package software.amazon.awssdk.services.s3.checksums;
1717

1818
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
19+
import static org.junit.jupiter.api.Assertions.assertEquals;
1920
import static org.junit.jupiter.api.Assertions.fail;
2021

2122
import java.io.ByteArrayInputStream;
@@ -56,20 +57,51 @@ public static void populateData() {
5657

5758
@Test
5859
public void validChecksumSucceeds() throws IOException {
59-
InputStream validatingInputStream = newValidatingStream(testData);
60+
InputStream validatingInputStream = newValidatingStream(testData, TEST_DATA_SIZE, CHECKSUM_SIZE);
6061
byte[] dataFromValidatingStream = IoUtils.toByteArray(validatingInputStream);
6162

6263
assertArrayEquals(testDataWithoutChecksum, dataFromValidatingStream);
6364
}
6465

66+
@Test
67+
public void emptyObjectSingleByteReadReturnsEOF() throws IOException {
68+
Md5Checksum checksum = new Md5Checksum();
69+
byte[] checksumBytes = checksum.getChecksumBytes();
70+
byte[] emptyWithChecksum = new byte[CHECKSUM_SIZE];
71+
72+
for (int i = 0; i < CHECKSUM_SIZE; i++) {
73+
emptyWithChecksum[i] = checksumBytes[i];
74+
}
75+
76+
InputStream validatingInputStream = newValidatingStream(emptyWithChecksum, 0, CHECKSUM_SIZE);
77+
78+
assertEquals(-1, validatingInputStream.read());
79+
}
80+
81+
@Test
82+
public void emptyObjectByteArrayReadReturnsEOF() throws IOException {
83+
Md5Checksum checksum = new Md5Checksum();
84+
byte[] checksumBytes = checksum.getChecksumBytes();
85+
byte[] emptyWithChecksum = new byte[CHECKSUM_SIZE];
86+
87+
for (int i = 0; i < CHECKSUM_SIZE; i++) {
88+
emptyWithChecksum[i] = checksumBytes[i];
89+
}
90+
91+
InputStream validatingInputStream = newValidatingStream(emptyWithChecksum, 0, CHECKSUM_SIZE);
92+
byte[] buffer = new byte[1];
93+
94+
assertEquals(-1, validatingInputStream.read(buffer));
95+
}
96+
6597
@Test
6698
public void invalidChecksumFails() throws IOException {
6799
for (int i = 0; i < testData.length; i++) {
68100
// Make sure that corruption of any byte in the test data causes a checksum validation failure.
69101
byte[] corruptedChecksumData = Arrays.copyOf(testData, testData.length);
70102
corruptedChecksumData[i] = (byte) ~corruptedChecksumData[i];
71103

72-
InputStream validatingInputStream = newValidatingStream(corruptedChecksumData);
104+
InputStream validatingInputStream = newValidatingStream(corruptedChecksumData, TEST_DATA_SIZE, CHECKSUM_SIZE);
73105

74106
try {
75107
IoUtils.toByteArray(validatingInputStream);
@@ -80,9 +112,9 @@ public void invalidChecksumFails() throws IOException {
80112
}
81113
}
82114

83-
private InputStream newValidatingStream(byte[] dataFromS3) {
115+
private InputStream newValidatingStream(byte[] dataFromS3, int testDataSize, int checkSumSize) {
84116
return new S3ChecksumValidatingInputStream(new ByteArrayInputStream(dataFromS3),
85117
new Md5Checksum(),
86-
TEST_DATA_SIZE + CHECKSUM_SIZE);
118+
testDataSize + checkSumSize);
87119
}
88120
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/checksums/S3ChecksumValidatingPublisherTest.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,28 @@ public void checksumValidationFailure_throwsSdkClientException_NotNPE() {
169169
assertFalse(s.hasCompleted());
170170
}
171171

172+
@Test
173+
public void emptyObjectReturnsNoData() {
174+
Md5Checksum checksum = new Md5Checksum();
175+
byte[] checksumBytes = checksum.getChecksumBytes();
176+
byte[] emptyWithChecksum = new byte[CHECKSUM_SIZE];
177+
for (int i = 0; i < CHECKSUM_SIZE; i++) {
178+
emptyWithChecksum[i] = checksumBytes[i];
179+
}
180+
181+
final TestPublisher driver = new TestPublisher();
182+
final TestSubscriber s = new TestSubscriber();
183+
final S3ChecksumValidatingPublisher p = new S3ChecksumValidatingPublisher(driver, new Md5Checksum(), CHECKSUM_SIZE);
184+
p.subscribe(s);
185+
186+
driver.doOnNext(ByteBuffer.wrap(emptyWithChecksum));
187+
driver.doOnComplete();
188+
189+
assertArrayEquals(new byte[0], s.receivedData());
190+
assertTrue(s.hasCompleted());
191+
assertFalse(s.isOnErrorCalled());
192+
}
193+
172194
private class TestSubscriber implements Subscriber<ByteBuffer> {
173195
final List<ByteBuffer> received;
174196
boolean completed;

0 commit comments

Comments
 (0)