Skip to content

Commit 37ad067

Browse files
author
Anirav Kareddy
committed
fixes
1 parent 764ac43 commit 37ad067

File tree

2 files changed

+115
-106
lines changed

2 files changed

+115
-106
lines changed

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
env:
4949
AWS_DEFAULTS_MODE: standard
5050
AWS_RETRY_MODE: adaptive
51-
AWS_MAX_ATTEMPTS: '10'
51+
AWS_MAX_ATTEMPTS: '20'
5252
AWS_SDK_CLIENT_LOGGING: debug
5353
run: |
5454
export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }}
@@ -58,7 +58,7 @@ jobs:
5858
export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }}
5959
export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }}
6060
export AWS_REGION=${{ vars.CI_AWS_REGION }}
61-
mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient
61+
mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient
6262
shell: bash
6363

6464
- name: Package JAR

‎src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java‎

Lines changed: 113 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -334,11 +334,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException {
334334
// Tight bound on the custom buffer size limit of 32MiB
335335
final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1;
336336

337-
byte[] data = new byte[(int) fileSizeExceedingDefaultLimit];
338-
for(int j=0; j < data.length; j++) {
339-
data[j] = (byte) (j % 256);
340-
}
341-
final InputStream largeObjectStream = new ByteArrayInputStream(data);
337+
final InputStream largeObjectStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit);
342338
v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder()
343339
.bucket(BUCKET)
344340
.key(objectKey)
@@ -357,7 +353,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException {
357353
.key(objectKey));
358354

359355
// Create a new ByteArrayInputStream for comparison
360-
InputStream expectedStream = new ByteArrayInputStream(data);
356+
InputStream expectedStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit);
361357
assertTrue(IOUtils.contentEquals(expectedStream, response));
362358
response.close();
363359

@@ -370,121 +366,134 @@ public void customSetBufferSizeWithLargeObject() throws IOException {
370366

371367
@Test
372368
public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException {
369+
int success=0, failures = 0;
373370
for(int i=0; i < 100; i++) {
374-
final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async");
375-
376-
Security.addProvider(new BouncyCastleProvider());
377-
Provider provider = Security.getProvider("BC");
378-
379-
// V3 Client with custom max buffer size 32 MiB.
380-
S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder()
381-
.aesKey(AES_KEY)
382-
.cryptoProvider(provider)
383-
.setBufferSize(32 * 1024 * 1024)
384-
.build();
385-
386-
// V3 Client with default buffer size (i.e. 64MiB)
387-
// When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode.
388-
S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder()
389-
.aesKey(AES_KEY)
390-
.cryptoProvider(provider)
391-
.enableDelayedAuthenticationMode(true)
392-
.build();
371+
try {
372+
final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async");
373+
374+
Security.addProvider(new BouncyCastleProvider());
375+
Provider provider = Security.getProvider("BC");
376+
377+
// V3 Client with custom max buffer size 32 MiB.
378+
S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder()
379+
.aesKey(AES_KEY)
380+
.cryptoProvider(provider)
381+
.setBufferSize(32 * 1024 * 1024)
382+
.build();
383+
384+
// V3 Client with default buffer size (i.e. 64MiB)
385+
// When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode.
386+
S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder()
387+
.aesKey(AES_KEY)
388+
.cryptoProvider(provider)
389+
.enableDelayedAuthenticationMode(true)
390+
.build();
391+
392+
// Tight bound on the custom buffer size limit of 32MiB
393+
final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1;
394+
final InputStream largeObjectStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit);
395+
396+
ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor();
397+
CompletableFuture<PutObjectResponse> futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder()
398+
.bucket(BUCKET)
399+
.key(objectKey)
400+
.build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor));
393401

394-
// Tight bound on the custom buffer size limit of 32MiB
395-
final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1;
396-
byte[] data = new byte[(int) fileSizeExceedingDefaultLimit];
397-
for(int j=0; j < data.length; j++) {
398-
data[j] = (byte) (j % 256);
399-
}
400-
final InputStream largeObjectStream = new ByteArrayInputStream(data);
401-
ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor();
402-
CompletableFuture<PutObjectResponse> futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder()
403-
.bucket(BUCKET)
404-
.key(objectKey)
405-
.build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor));
402+
futurePut.join();
403+
largeObjectStream.close();
404+
singleThreadExecutor.shutdown();
406405

407-
futurePut.join();
408-
largeObjectStream.close();
409-
singleThreadExecutor.shutdown();
406+
try {
407+
// Object is larger than Buffer, so getObject fails
408+
CompletableFuture<ResponseInputStream<GetObjectResponse>> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder
409+
.bucket(BUCKET)
410+
.key(objectKey), AsyncResponseTransformer.toBlockingInputStream());
411+
futureResponse.join();
412+
} catch (CompletionException e) {
413+
assertEquals(S3EncryptionClientException.class, e.getCause().getClass());
414+
}
410415

411-
try {
412-
// Object is larger than Buffer, so getObject fails
413-
CompletableFuture<ResponseInputStream<GetObjectResponse>> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder
416+
// You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds)
417+
CompletableFuture<ResponseInputStream<GetObjectResponse>> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder
414418
.bucket(BUCKET)
415419
.key(objectKey), AsyncResponseTransformer.toBlockingInputStream());
416-
futureResponse.join();
417-
} catch (CompletionException e) {
418-
assertEquals(S3EncryptionClientException.class, e.getCause().getClass());
419-
}
420+
ResponseInputStream<GetObjectResponse> output = futureGet.join();
420421

421-
// You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds)
422-
CompletableFuture<ResponseInputStream<GetObjectResponse>> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder
423-
.bucket(BUCKET)
424-
.key(objectKey), AsyncResponseTransformer.toBlockingInputStream());
425-
ResponseInputStream<GetObjectResponse> output = futureGet.join();
422+
InputStream expectedStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit);
423+
assertTrue(IOUtils.contentEquals(expectedStream, output));
426424

427-
ByteArrayInputStream expectedStream = new ByteArrayInputStream(data);
428-
assertTrue(IOUtils.contentEquals(expectedStream, output));
425+
output.close();
429426

430-
output.close();
427+
// Cleanup
428+
deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB);
429+
v3ClientWithBuffer32MiB.close();
430+
v3ClientWithDelayedAuth.close();
431431

432-
// Cleanup
433-
deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB);
434-
v3ClientWithBuffer32MiB.close();
435-
v3ClientWithDelayedAuth.close();
432+
success++;
433+
} catch (Exception e) {
434+
failures++;
435+
}
436436
}
437+
System.out.println("Success: "+success+" Failures: "+failures);
437438
}
438439

439440
@Test
440441
public void delayedAuthModeWithLargeObject() throws IOException {
442+
int success = 0, failures = 0;
441443
for(int i=0; i < 10; i++) {
442-
final String objectKey = appendTestSuffix("large-object-test");
443-
444-
Security.addProvider(new BouncyCastleProvider());
445-
Provider provider = Security.getProvider("BC");
446-
447-
// V3 Client
448-
S3Client v3Client = S3EncryptionClient.builder()
449-
.aesKey(AES_KEY)
450-
.cryptoProvider(provider)
451-
.build();
452-
453-
// Tight bound on the default limit of 64MiB
454-
final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1;
455-
final InputStream largeObjectStream = new BufferedInputStream(
456-
new BoundedInputStream(fileSizeExceedingDefaultLimit)
457-
);
458-
v3Client.putObject(PutObjectRequest.builder()
459-
.bucket(BUCKET)
460-
.key(objectKey)
461-
.build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit));
462-
463-
largeObjectStream.close();
464-
465-
// Delayed Authentication is not enabled, so getObject fails
466-
assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder
467-
.bucket(BUCKET)
468-
.key(objectKey)));
469-
470-
S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder()
471-
.aesKey(AES_KEY)
472-
.enableDelayedAuthenticationMode(true)
473-
.build();
474-
475-
// Once enabled, the getObject request passes
476-
ResponseInputStream<GetObjectResponse> response = v3ClientWithDelayedAuth.getObject(builder -> builder
477-
.bucket(BUCKET)
478-
.key(objectKey));
479-
480-
481-
assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response));
482-
response.close();
483-
484-
// Cleanup
485-
deleteObject(BUCKET, objectKey, v3Client);
486-
v3Client.close();
444+
try {
445+
final String objectKey = appendTestSuffix("large-object-test");
446+
447+
Security.addProvider(new BouncyCastleProvider());
448+
Provider provider = Security.getProvider("BC");
449+
450+
// V3 Client
451+
S3Client v3Client = S3EncryptionClient.builder()
452+
.aesKey(AES_KEY)
453+
.cryptoProvider(provider)
454+
.build();
455+
456+
// Tight bound on the default limit of 64MiB
457+
final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1;
458+
final InputStream largeObjectStream = new BufferedInputStream(
459+
new BoundedInputStream(fileSizeExceedingDefaultLimit)
460+
);
461+
v3Client.putObject(PutObjectRequest.builder()
462+
.bucket(BUCKET)
463+
.key(objectKey)
464+
.build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit));
465+
466+
largeObjectStream.close();
467+
468+
// Delayed Authentication is not enabled, so getObject fails
469+
assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder
470+
.bucket(BUCKET)
471+
.key(objectKey)));
472+
473+
S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder()
474+
.aesKey(AES_KEY)
475+
.enableDelayedAuthenticationMode(true)
476+
.build();
477+
478+
// Once enabled, the getObject request passes
479+
ResponseInputStream<GetObjectResponse> response = v3ClientWithDelayedAuth.getObject(builder -> builder
480+
.bucket(BUCKET)
481+
.key(objectKey));
482+
483+
484+
assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response));
485+
response.close();
486+
487+
// Cleanup
488+
deleteObject(BUCKET, objectKey, v3Client);
489+
v3Client.close();
490+
491+
success++;
492+
} catch (Exception e) {
493+
failures++;
494+
}
487495
}
496+
System.out.println("Success: "+success+" Failures: "+failures);
488497
}
489498

490499
@Test

0 commit comments

Comments
 (0)