Skip to content

Commit ecf6e6c

Browse files
imabhichowimabhichow
authored
feat: add configuration option to set max buffer size (#166)
* feat: Add Configuration option to set max buffer size
1 parent 84baad8 commit ecf6e6c

File tree

8 files changed

+245
-108
lines changed

8 files changed

+245
-108
lines changed

‎src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@
4242
import java.util.function.Consumer;
4343
import java.util.function.Function;
4444

45+
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_BUFFER_SIZE_BYTES;
46+
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MAX_ALLOWED_BUFFER_SIZE_BYTES;
47+
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MIN_ALLOWED_BUFFER_SIZE_BYTES;
4548
import static software.amazon.encryption.s3.internal.ApiNameVersion.API_NAME_INTERCEPTOR;
4649

4750
/**
@@ -56,6 +59,7 @@ public class S3AsyncEncryptionClient extends DelegatingS3AsyncClient {
5659
private final boolean _enableLegacyUnauthenticatedModes;
5760
private final boolean _enableDelayedAuthenticationMode;
5861
private final boolean _enableMultipartPutObject;
62+
private final long _bufferSize;
5963

6064
private S3AsyncEncryptionClient(Builder builder) {
6165
super(builder._wrappedClient);
@@ -65,6 +69,7 @@ private S3AsyncEncryptionClient(Builder builder) {
6569
_enableLegacyUnauthenticatedModes = builder._enableLegacyUnauthenticatedModes;
6670
_enableDelayedAuthenticationMode = builder._enableDelayedAuthenticationMode;
6771
_enableMultipartPutObject = builder._enableMultipartPutObject;
72+
_bufferSize = builder._bufferSize;
6873
}
6974

7075
/**
@@ -181,6 +186,7 @@ public <T> CompletableFuture<T> getObject(GetObjectRequest getObjectRequest,
181186
.cryptoMaterialsManager(_cryptoMaterialsManager)
182187
.enableLegacyUnauthenticatedModes(_enableLegacyUnauthenticatedModes)
183188
.enableDelayedAuthentication(_enableDelayedAuthenticationMode)
189+
.bufferSize(_bufferSize)
184190
.build();
185191

186192
return pipeline.getObject(getObjectRequest, asyncResponseTransformer);
@@ -200,9 +206,9 @@ public CompletableFuture<DeleteObjectResponse> deleteObject(DeleteObjectRequest
200206
final DeleteObjectRequest actualRequest = deleteObjectRequest.toBuilder()
201207
.overrideConfiguration(API_NAME_INTERCEPTOR)
202208
.build();
203-
final CompletableFuture<DeleteObjectResponse> response = _wrappedClient.deleteObject(actualRequest);
209+
final CompletableFuture<DeleteObjectResponse> response = _wrappedClient.deleteObject(actualRequest);
204210
final String instructionObjectKey = deleteObjectRequest.key() + ".instruction";
205-
final CompletableFuture<DeleteObjectResponse> instructionResponse = _wrappedClient.deleteObject(builder -> builder
211+
final CompletableFuture<DeleteObjectResponse> instructionResponse = _wrappedClient.deleteObject(builder -> builder
206212
.overrideConfiguration(API_NAME_INTERCEPTOR)
207213
.bucket(deleteObjectRequest.bucket())
208214
.key(instructionObjectKey));
@@ -257,6 +263,7 @@ public static class Builder {
257263
private boolean _enableMultipartPutObject = false;
258264
private Provider _cryptoProvider = null;
259265
private SecureRandom _secureRandom = new SecureRandom();
266+
private long _bufferSize = -1L;
260267

261268
private Builder() {
262269
}
@@ -434,6 +441,22 @@ public Builder enableMultipartPutObject(boolean _enableMultipartPutObject) {
434441
return this;
435442
}
436443

444+
/**
445+
* Sets the buffer size for safe authentication used when delayed authentication mode is disabled.
446+
* If buffer size is not given during client configuration, default buffer size is set to 64MiB.
447+
* @param bufferSize the desired buffer size in Bytes.
448+
* @return Returns a reference to this object so that method calls can be chained together.
449+
* @throws S3EncryptionClientException if the specified buffer size is outside the allowed bounds
450+
*/
451+
public Builder setBufferSize(long bufferSize) {
452+
if (bufferSize < MIN_ALLOWED_BUFFER_SIZE_BYTES || bufferSize > MAX_ALLOWED_BUFFER_SIZE_BYTES) {
453+
throw new S3EncryptionClientException("Invalid buffer size: " + bufferSize + " Bytes. Buffer size must be between " + MIN_ALLOWED_BUFFER_SIZE_BYTES + " and " + MAX_ALLOWED_BUFFER_SIZE_BYTES + " Bytes.");
454+
}
455+
456+
this._bufferSize = bufferSize;
457+
return this;
458+
}
459+
437460
/**
438461
* Allows the user to pass an instance of {@link Provider} to be used
439462
* for cryptographic operations. By default, the S3 Encryption Client
@@ -476,6 +499,14 @@ public S3AsyncEncryptionClient build() {
476499
throw new S3EncryptionClientException("Exactly one must be set of: crypto materials manager, keyring, AES key, RSA key pair, KMS key id");
477500
}
478501

502+
if (_bufferSize >= 0) {
503+
if (_enableDelayedAuthenticationMode) {
504+
throw new S3EncryptionClientException("Buffer size cannot be set when delayed authentication mode is enabled");
505+
}
506+
} else {
507+
_bufferSize = DEFAULT_BUFFER_SIZE_BYTES;
508+
}
509+
479510
if (_keyring == null) {
480511
if (_aesKey != null) {
481512
_keyring = AesKeyring.builder()

‎src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@
6666
import java.util.concurrent.Future;
6767
import java.util.function.Consumer;
6868

69+
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_BUFFER_SIZE_BYTES;
6970
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX;
71+
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MAX_ALLOWED_BUFFER_SIZE_BYTES;
72+
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MIN_ALLOWED_BUFFER_SIZE_BYTES;
7073
import static software.amazon.encryption.s3.S3EncryptionClientUtilities.instructionFileKeysToDelete;
7174
import static software.amazon.encryption.s3.internal.ApiNameVersion.API_NAME_INTERCEPTOR;
7275

@@ -88,6 +91,7 @@ public class S3EncryptionClient extends DelegatingS3Client {
8891
private final boolean _enableDelayedAuthenticationMode;
8992
private final boolean _enableMultipartPutObject;
9093
private final MultipartUploadObjectPipeline _multipartPipeline;
94+
private final long _bufferSize;
9195

9296
private S3EncryptionClient(Builder builder) {
9397
super(builder._wrappedClient);
@@ -99,6 +103,7 @@ private S3EncryptionClient(Builder builder) {
99103
_enableDelayedAuthenticationMode = builder._enableDelayedAuthenticationMode;
100104
_enableMultipartPutObject = builder._enableMultipartPutObject;
101105
_multipartPipeline = builder._multipartPipeline;
106+
_bufferSize = builder._bufferSize;
102107
}
103108

104109
/**
@@ -224,6 +229,7 @@ public <T> T getObject(GetObjectRequest getObjectRequest,
224229
.cryptoMaterialsManager(_cryptoMaterialsManager)
225230
.enableLegacyUnauthenticatedModes(_enableLegacyUnauthenticatedModes)
226231
.enableDelayedAuthentication(_enableDelayedAuthenticationMode)
232+
.bufferSize(_bufferSize)
227233
.build();
228234

229235
try {
@@ -498,6 +504,7 @@ public static class Builder {
498504
private Provider _cryptoProvider = null;
499505
private SecureRandom _secureRandom = new SecureRandom();
500506
private boolean _enableLegacyUnauthenticatedModes = false;
507+
private long _bufferSize = -1L;
501508

502509
private Builder() {
503510
}
@@ -686,6 +693,22 @@ public Builder enableMultipartPutObject(boolean _enableMultipartPutObject) {
686693
return this;
687694
}
688695

696+
/**
697+
* Sets the buffer size for safe authentication used when delayed authentication mode is disabled.
698+
* If buffer size is not given during client configuration, default buffer size is set to 64MiB.
699+
* @param bufferSize the desired buffer size in Bytes.
700+
* @return Returns a reference to this object so that method calls can be chained together.
701+
* @throws S3EncryptionClientException if the specified buffer size is outside the allowed bounds
702+
*/
703+
public Builder setBufferSize(long bufferSize) {
704+
if (bufferSize < MIN_ALLOWED_BUFFER_SIZE_BYTES || bufferSize > MAX_ALLOWED_BUFFER_SIZE_BYTES) {
705+
throw new S3EncryptionClientException("Invalid buffer size: " + bufferSize + " Bytes. Buffer size must be between " + MIN_ALLOWED_BUFFER_SIZE_BYTES + " and " + MAX_ALLOWED_BUFFER_SIZE_BYTES + " Bytes.");
706+
}
707+
708+
this._bufferSize = bufferSize;
709+
return this;
710+
}
711+
689712
/**
690713
* Allows the user to pass an instance of {@link Provider} to be used
691714
* for cryptographic operations. By default, the S3 Encryption Client
@@ -728,6 +751,14 @@ public S3EncryptionClient build() {
728751
throw new S3EncryptionClientException("Exactly one must be set of: crypto materials manager, keyring, AES key, RSA key pair, KMS key id");
729752
}
730753

754+
if (_bufferSize >= 0) {
755+
if (_enableDelayedAuthenticationMode) {
756+
throw new S3EncryptionClientException("Buffer size cannot be set when delayed authentication mode is enabled");
757+
}
758+
} else {
759+
_bufferSize = DEFAULT_BUFFER_SIZE_BYTES;
760+
}
761+
731762
if (_wrappedClient == null) {
732763
_wrappedClient = S3Client.create();
733764
}

‎src/main/java/software/amazon/encryption/s3/S3EncryptionClientUtilities.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
66
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
7+
import software.amazon.encryption.s3.algorithms.AlgorithmSuite;
78

89
import java.util.List;
910
import java.util.stream.Collectors;
@@ -15,6 +16,13 @@
1516
public class S3EncryptionClientUtilities {
1617

1718
public static final String INSTRUCTION_FILE_SUFFIX = ".instruction";
19+
public static final long MIN_ALLOWED_BUFFER_SIZE_BYTES = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF.cipherBlockSizeBytes();
20+
public static final long MAX_ALLOWED_BUFFER_SIZE_BYTES = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF.cipherMaxContentLengthBytes();
21+
22+
/**
23+
* The Default Buffer Size for Safe authentication is set to 64MiB.
24+
*/
25+
public static final long DEFAULT_BUFFER_SIZE_BYTES = 64 * 1024 * 1024;
1826

1927
/**
2028
* For a given DeleteObjectsRequest, return a list of ObjectIdentifiers

‎src/main/java/software/amazon/encryption/s3/internal/BufferedAesGcmContentStrategy.java‎

Lines changed: 0 additions & 83 deletions
This file was deleted.

‎src/main/java/software/amazon/encryption/s3/internal/BufferedCipherPublisher.java‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
import org.reactivestreams.Subscriber;
66
import software.amazon.awssdk.core.async.SdkPublisher;
7-
import software.amazon.encryption.s3.legacy.internal.RangedGetUtils;
87
import software.amazon.encryption.s3.materials.CryptographicMaterials;
98

109
import java.nio.ByteBuffer;
@@ -13,30 +12,23 @@ public class BufferedCipherPublisher implements SdkPublisher<ByteBuffer> {
1312

1413
private final SdkPublisher<ByteBuffer> wrappedPublisher;
1514
private final Long contentLength;
16-
private final long[] range;
17-
private final String contentRange;
18-
private final int cipherTagLengthBits;
1915
private final CryptographicMaterials materials;
2016
private final byte[] iv;
17+
private final long bufferSize;
2118

2219
public BufferedCipherPublisher(final SdkPublisher<ByteBuffer> wrappedPublisher, final Long contentLength,
23-
long[] range, String contentRange, int cipherTagLengthBits,
24-
final CryptographicMaterials materials, final byte[] iv) {
20+
final CryptographicMaterials materials, final byte[] iv, final long bufferSize) {
2521
this.wrappedPublisher = wrappedPublisher;
2622
this.contentLength = contentLength;
27-
this.range = range;
28-
this.contentRange = contentRange;
29-
this.cipherTagLengthBits = cipherTagLengthBits;
3023
this.materials = materials;
3124
this.iv = iv;
25+
this.bufferSize = bufferSize;
3226
}
3327

3428
@Override
3529
public void subscribe(Subscriber<? super ByteBuffer> subscriber) {
3630
// Wrap the (customer) subscriber in a CipherSubscriber, then subscribe it
3731
// to the wrapped (ciphertext) publisher
38-
Subscriber<? super ByteBuffer> wrappedSubscriber = RangedGetUtils.adjustToDesiredRange(subscriber, range,
39-
contentRange, cipherTagLengthBits);
40-
wrappedPublisher.subscribe(new BufferedCipherSubscriber(wrappedSubscriber, contentLength, materials, iv));
32+
wrappedPublisher.subscribe(new BufferedCipherSubscriber(subscriber, contentLength, materials, iv, bufferSize));
4133
}
4234
}

‎src/main/java/software/amazon/encryption/s3/internal/BufferedCipherSubscriber.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,6 @@
2525
*/
2626
public class BufferedCipherSubscriber implements Subscriber<ByteBuffer> {
2727

28-
// 64MiB ought to be enough for most usecases
29-
private static final long BUFFERED_MAX_CONTENT_LENGTH_MiB = 64;
30-
private static final long BUFFERED_MAX_CONTENT_LENGTH_BYTES = 1024 * 1024 * BUFFERED_MAX_CONTENT_LENGTH_MiB;
31-
3228
private final AtomicInteger contentRead = new AtomicInteger(0);
3329
private final AtomicBoolean doneFinal = new AtomicBoolean(false);
3430
private final Subscriber<? super ByteBuffer> wrappedSubscriber;
@@ -40,16 +36,17 @@ public class BufferedCipherSubscriber implements Subscriber<ByteBuffer> {
4036
private byte[] outputBuffer;
4137
private final Queue<ByteBuffer> buffers = new ConcurrentLinkedQueue<>();
4238

43-
BufferedCipherSubscriber(Subscriber<? super ByteBuffer> wrappedSubscriber, Long contentLength, CryptographicMaterials materials, byte[] iv) {
39+
BufferedCipherSubscriber(Subscriber<? super ByteBuffer> wrappedSubscriber, Long contentLength, CryptographicMaterials materials, byte[] iv, long bufferSizeInBytes) {
4440
this.wrappedSubscriber = wrappedSubscriber;
4541
if (contentLength == null) {
4642
throw new S3EncryptionClientException("contentLength cannot be null in buffered mode. To enable unbounded " +
4743
"streaming, reconfigure the S3 Encryption Client with Delayed Authentication mode enabled.");
4844
}
49-
if (contentLength > BUFFERED_MAX_CONTENT_LENGTH_BYTES) {
50-
throw new S3EncryptionClientException(String.format("The object you are attempting to decrypt exceeds the maximum content " +
51-
"length allowed in default mode. Please enable Delayed Authentication mode to decrypt objects larger" +
52-
"than %d", BUFFERED_MAX_CONTENT_LENGTH_MiB));
45+
if (contentLength > bufferSizeInBytes) {
46+
throw new S3EncryptionClientException(String.format("The object you are attempting to decrypt exceeds the maximum buffer size: " + bufferSizeInBytes +
47+
" for the default (buffered) mode. Either increase your buffer size when configuring your client, " +
48+
"or enable Delayed Authentication mode to disable buffered decryption."));
49+
5350
}
5451
this.contentLength = Math.toIntExact(contentLength);
5552
this.materials = materials;

0 commit comments

Comments
 (0)