Merge branch 'main' of github.com:aws/amazon-s3-encryption-client-java into inst-file-put

Author: kessplas

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## [3.3.5](https://github.com/aws/aws-s3-encryption-client-java/compare/v3.3.4...v3.3.5) (2025-05-21)
+
+### Fixes
+
+* determine effective contentLength, account for tagLength on decrypt ([#463](https://github.com/aws/aws-s3-encryption-client-java/issues/463)) ([969d721](https://github.com/aws/aws-s3-encryption-client-java/commit/969d7213b7bd6250fbce159bd5705a19ee439f23))
+* disable low-level Multipart Upload in Async client ([#461](https://github.com/aws/aws-s3-encryption-client-java/issues/461)) ([599f941](https://github.com/aws/aws-s3-encryption-client-java/commit/599f9417335efac4cf952e555a99bbc6d7d8cc0f))
+* support PutObjectResponse fields ([#462](https://github.com/aws/aws-s3-encryption-client-java/issues/462)) ([dec503b](https://github.com/aws/aws-s3-encryption-client-java/commit/dec503b49f3e57113de16fcc861ee1ecedbd2ff6))
+
+### Maintenance
+
+* Revert "Amazon S3 Encryption Client 3.3.5 Release -- 2025-05-20" ([#465](https://github.com/aws/aws-s3-encryption-client-java/issues/465)) ([3f9ac8e](https://github.com/aws/aws-s3-encryption-client-java/commit/3f9ac8e419f5ed55f59fb0daf742fc2945eea9d0))
+* update dependency needed for semantic-release  ([#464](https://github.com/aws/aws-s3-encryption-client-java/issues/464)) ([0fd3b58](https://github.com/aws/aws-s3-encryption-client-java/commit/0fd3b5826964b41a07a4c004fd0500404e347360))
+
+## [3.3.4](https://github.com/aws/aws-s3-encryption-client-java/compare/v3.3.3...v3.3.4) (2025-05-12)
+
+### Fixes
+
+* Add details to error message ([#459](https://github.com/aws/aws-s3-encryption-client-java/issues/459)) ([0d32b4a](https://github.com/aws/aws-s3-encryption-client-java/commit/0d32b4a81662c5dc8ca85cf6f5dd09252f014b6e)), closes [#458](https://github.com/aws/aws-s3-encryption-client-java/issues/458)
+* Support all PutObjectRequest fields ([#458](https://github.com/aws/aws-s3-encryption-client-java/issues/458)) ([99cce95](https://github.com/aws/aws-s3-encryption-client-java/commit/99cce95ab8e376f4a096401e56a88d6fc34ace44))
+
 ## [3.3.3](https://github.com/aws/aws-s3-encryption-client-java/compare/v3.3.2...v3.3.3) (2025-05-05)
 
 ### Fixes

codebuild/release/version.yml

@@ -19,6 +19,7 @@ phases:
       - npm install @semantic-release/changelog -d
       - npm install @semantic-release/exec -d
       - npm install @semantic-release/git -d
+      - npm install conventional-changelog-conventionalcommits -d
       - npm install --save conventional-changelog
     runtime-versions:
       nodejs: 16
@@ -30,4 +31,4 @@ phases:
   build:
     commands:
       # semantic-release uses config stored in ~/.releaserc
-      - npx semantic-release --branches $BRANCH --no-ci
+      - npx semantic-release --branches $BRANCH --no-ci

pom.xml

@@ -6,7 +6,7 @@
 
   <groupId>software.amazon.encryption.s3</groupId>
   <artifactId>amazon-s3-encryption-client-java</artifactId>
-  <version>3.3.3</version>
+  <version>3.3.5</version>
   <packaging>jar</packaging>
 
   <name>Amazon S3 Encryption Client</name>

src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java

@@ -20,6 +20,10 @@
 import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
 import software.amazon.awssdk.services.s3.S3Configuration;
 import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
 import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
 import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
@@ -30,6 +34,8 @@
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Request;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
 import software.amazon.encryption.s3.internal.GetEncryptedObjectPipeline;
 import software.amazon.encryption.s3.internal.InstructionFileConfig;
@@ -256,6 +262,24 @@ public CompletableFuture<DeleteObjectsResponse> deleteObjects(DeleteObjectsReque
                 .build());
     }
 
+    @Override
+    public CompletableFuture<CreateMultipartUploadResponse> createMultipartUpload(CreateMultipartUploadRequest createMultipartUploadRequest) {
+        throw new UnsupportedOperationException("The S3 Async Encryption Client does not support low-level multipart uploads. " +
+                "Please use Multipart PutObject or the default (synchronous) client to use this API.");
+    }
+
+    @Override
+    public CompletableFuture<UploadPartResponse> uploadPart(UploadPartRequest uploadPartRequest, AsyncRequestBody asyncRequestBody) {
+        throw new UnsupportedOperationException("The S3 Async Encryption Client does not support low-level multipart uploads. " +
+                "Please use Multipart PutObject or the default (synchronous) client to use this API.");
+    }
+
+    @Override
+    public CompletableFuture<CompleteMultipartUploadResponse> completeMultipartUpload(CompleteMultipartUploadRequest completeMultipartUploadRequest) {
+        throw new UnsupportedOperationException("The S3 Async Encryption Client does not support low-level multipart uploads. " +
+                "Please use Multipart PutObject or the default (synchronous) client to use this API.");
+    }
+
     /**
      * Closes the wrapped {@link S3AsyncClient} instance.
      */

src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java

@@ -46,6 +46,7 @@
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.encryption.s3.algorithms.AlgorithmSuite;
+import software.amazon.encryption.s3.internal.ConvertSDKRequests;
 import software.amazon.encryption.s3.internal.GetEncryptedObjectPipeline;
 import software.amazon.encryption.s3.internal.InstructionFileConfig;
 import software.amazon.encryption.s3.internal.MultiFileOutputStream;
@@ -192,11 +193,7 @@ public PutObjectResponse putObject(PutObjectRequest putObjectRequest, RequestBod
 
         if (_enableMultipartPutObject) {
             try {
-                CompleteMultipartUploadResponse completeResponse = multipartPutObject(putObjectRequest, requestBody);
-                PutObjectResponse response = PutObjectResponse.builder()
-                        .eTag(completeResponse.eTag())
-                        .build();
-                return response;
+                return multipartPutObject(putObjectRequest, requestBody);
             } catch (Throwable e) {
                 throw new S3EncryptionClientException("Exception while performing Multipart Upload PutObject", e);
             }
@@ -275,7 +272,7 @@ public <T> T getObject(GetObjectRequest getObjectRequest,
         }
     }
 
-    private CompleteMultipartUploadResponse multipartPutObject(PutObjectRequest request, RequestBody requestBody) throws Throwable {
+    private PutObjectResponse multipartPutObject(PutObjectRequest request, RequestBody requestBody) throws Throwable {
         // Similar logic exists in the MultipartUploadObjectPipeline,
         // but the request types do not match so refactoring is not possible
         final long contentLength;
@@ -355,7 +352,7 @@ private CompleteMultipartUploadResponse multipartPutObject(PutObjectRequest requ
             outputStream.cleanup();
         }
         // Complete upload
-        return observer.onCompletion(partETags);
+        return ConvertSDKRequests.convertResponse(observer.onCompletion(partETags));
     }
 
     private <T extends Throwable> T onAbort(UploadObjectObserver observer, T t) {

src/main/java/software/amazon/encryption/s3/internal/CipherSubscriber.java

@@ -21,6 +21,7 @@ public class CipherSubscriber implements Subscriber<ByteBuffer> {
     private final Long contentLength;
     private final boolean isLastPart;
     private final int tagLength;
+    private final boolean isEncrypt;
     private final AtomicBoolean finalBytesCalled = new AtomicBoolean(false);
 
     private byte[] outputBuffer;
@@ -31,6 +32,7 @@ public class CipherSubscriber implements Subscriber<ByteBuffer> {
         this.cipher = materials.getCipher(iv);
         this.isLastPart = isLastPart;
         this.tagLength = materials.algorithmSuite().cipherTagLengthBytes();
+        this.isEncrypt = (CipherMode.DECRYPT != materials.cipherMode());
     }
 
     CipherSubscriber(Subscriber<? super ByteBuffer> wrappedSubscriber, Long contentLength, CryptographicMaterials materials, byte[] iv) {
@@ -56,7 +58,9 @@ public void onNext(ByteBuffer byteBuffer) {
                 // Note that while the JCE Javadoc specifies that the outputBuffer is null in this case,
                 // in practice SunJCE and ACCP return an empty buffer instead, hence checks for
                 // null OR length == 0.
-                if (contentRead.get() + tagLength >= contentLength) {
+
+                // tagLength should only be added on Encrypt
+                if (contentRead.get() + (isEncrypt ? tagLength : 0) >= contentLength) {
                     // All content has been read, so complete to get the final bytes
                     finalBytes();
                     return;
@@ -84,7 +88,7 @@ public void onNext(ByteBuffer byteBuffer) {
                  Calling `wrappedSubscriber.onNext` more than once for `request(1)`
                  violates the Reactive Streams specification and can cause exceptions downstream.
                 */
-                if (contentRead.get() + tagLength >= contentLength) {
+                if (contentRead.get() + (isEncrypt ? tagLength : 0) >= contentLength) {
                     // All content has been read; complete the stream.
                     finalBytes();
                 } else {
@@ -125,9 +129,10 @@ public void onError(Throwable t) {
     public void onComplete() {
         // In rare cases, e.g. when the last part of a low-level MPU has 0 length,
         // onComplete will be called before onNext is called once.
-        if (contentRead.get() + tagLength <= contentLength) {
-            finalBytes();
-        }
+        // So, call finalBytes here just in case there's any unsent data left.
+        // Most likely, finalBytes has already been called by the last onNext,
+        // but finalBytes guards against multiple invocations so it's safe to call again.
+        finalBytes();
         wrappedSubscriber.onComplete();
     }
 

src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java

@@ -1,14 +1,18 @@
 package software.amazon.encryption.s3.internal;
 
+import java.time.Instant;
+import java.util.Map;
+
+import org.apache.commons.logging.LogFactory;
 import software.amazon.awssdk.services.s3.model.ChecksumType;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
-import java.time.Instant;
-import java.util.Map;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 
 public class ConvertSDKRequests {
 
-  public static CreateMultipartUploadRequest convert(PutObjectRequest request) {
+  public static CreateMultipartUploadRequest convertRequest(PutObjectRequest request) {
 
     final CreateMultipartUploadRequest.Builder output = CreateMultipartUploadRequest.builder();
     request
@@ -121,7 +125,16 @@ public static CreateMultipartUploadRequest convert(PutObjectRequest request) {
             default:
               // Rather than silently dropping the value,
               // we loudly signal that we don't know how to handle this field.
-              throw new IllegalArgumentException("Unknown PutObjectRequest field " + f.locationName() + ".");
+              throw new IllegalArgumentException(
+                f.locationName() + " is an unknown field. " +
+                  "The S3 Encryption Client does not recognize this option and cannot set it on the CreateMultipartUploadRequest." +
+                  "This may be a new S3 feature." +
+                  "Please report this to the Amazon S3 Encryption Client for Java: " +
+                  "https://github.com/aws/amazon-s3-encryption-client-java/issues." +
+                  "To work around this issue you can disable multi part upload," +
+                  "use the Async client, or not set this value on PutObject." +
+                  "You may be able to update this value after the PutObject request completes."
+              );
           }
         }
       });
@@ -131,6 +144,77 @@ public static CreateMultipartUploadRequest convert(PutObjectRequest request) {
       .build();
   }
 
+  public static PutObjectResponse convertResponse(CompleteMultipartUploadResponse response) {
+    final PutObjectResponse.Builder output = PutObjectResponse.builder();
+    response
+            .toBuilder()
+            .sdkFields()
+            .forEach(f -> {
+              final Object value = f.getValueOrDefault(response);
+              if (value != null) {
+                switch (f.memberName()) {
+                  case "ETag":
+                    output.eTag((String) value);
+                    break;
+                  case "Expiration":
+                    output.expiration((String) value);
+                    break;
+                  case "ChecksumCRC32":
+                    output.checksumCRC32((String) value);
+                    break;
+                  case "ChecksumCRC32C":
+                    output.checksumCRC32C((String) value);
+                    break;
+                  case "ChecksumCRC64NVME":
+                    output.checksumCRC64NVME((String) value);
+                    break;
+                  case "ChecksumSHA1":
+                    output.checksumSHA1((String) value);
+                    break;
+                  case "ChecksumSHA256":
+                    output.checksumSHA256((String) value);
+                    break;
+                  case "ChecksumType":
+                    output.checksumType((String) value);
+                    break;
+                  case "ServerSideEncryption":
+                    output.serverSideEncryption((String) value);
+                    break;
+                  case "VersionId":
+                    output.versionId((String) value);
+                    break;
+                  case "SSEKMSKeyId":
+                    output.ssekmsKeyId((String) value);
+                    break;
+                  case "BucketKeyEnabled":
+                    output.bucketKeyEnabled((Boolean) value);
+                    break;
+                  case "RequestCharged":
+                    output.requestCharged((String) value);
+                    break;
+                  // Ignored fields: Location, Bucket, Key
+                  case "Location":
+                  case "Bucket":
+                  case "Key":
+                    // These fields exist only in CompleteMultipartUploadResponse, not in PutObjectResponse
+                    break;
+                  default:
+                    // We should NOT throw an exception for unknown fields because
+                    // once the object is stored, we expect to return a successful response.
+                    // Emit a log at info level for awareness.
+                    LogFactory.getLog(ConvertSDKRequests.class).info(f.memberName() + " returned in CompleteMultipartUploadResponse for "
+                            + response.key() + " is an unknown field." +
+                            "The S3 Encryption Client does not recognize this option and cannot set it on the CompleteMultipartUploadResponse." +
+                            "This may be a new S3 feature." +
+                            "Please report this to the Amazon S3 Encryption Client for Java: " +
+                            "https://github.com/aws/amazon-s3-encryption-client-java/issues."
+                    );
+                }
+              }
+            });
+    return output.build();
+  }
+
   private static boolean isStringStringMap(Object value) {
     if (!(value instanceof Map)) {
       return false;

src/main/java/software/amazon/encryption/s3/internal/UploadObjectObserver.java

@@ -44,7 +44,7 @@ public UploadObjectObserver init(PutObjectRequest req,
 
     public String onUploadCreation(PutObjectRequest req) {
         CreateMultipartUploadResponse res =
-                s3EncryptionClient.createMultipartUpload(ConvertSDKRequests.convert(req));
+                s3EncryptionClient.createMultipartUpload(ConvertSDKRequests.convertRequest(req));
         return this.uploadId = res.uploadId();
     }
 

src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java

@@ -910,16 +910,46 @@ public void wrappedClientMultipartUploadThrowsException() throws IOException {
     }
 
     @Test
-    public void S3AsyncClientBuilderForbidsMultipartEnabled() throws IOException {
+    public void s3AsyncClientBuilderForbidsMultipartEnabled() {
         assertThrows(
             UnsupportedOperationException.class,
             () -> S3AsyncEncryptionClient.builder().multipartEnabled(Boolean.TRUE));
     }
 
     @Test
-    public void S3AsyncClientBuilderForbidsMultipartConfiguration() throws IOException {
+    public void s3AsyncClientBuilderForbidsMultipartConfiguration() {
         assertThrows(
             UnsupportedOperationException.class,
             () -> S3AsyncEncryptionClient.builder().multipartConfiguration(MultipartConfiguration.builder().build()));
     }
+
+    @Test
+    public void s3AsyncClientForbidsCreateMultipartUpload() {
+        S3AsyncClient s3AsyncClient = S3AsyncEncryptionClient.builder()
+                .kmsKeyId("fails")
+                .build();
+
+        assertThrows(UnsupportedOperationException.class, () -> s3AsyncClient.createMultipartUpload(builder -> builder.bucket(BUCKET).key("expected-fail").build()).join());
+        s3AsyncClient.close();
+    }
+
+    @Test
+    public void s3AsyncClientForbidsUploadPart() {
+        S3AsyncClient s3AsyncClient = S3AsyncEncryptionClient.builder()
+                .kmsKeyId("fails")
+                .build();
+
+        assertThrows(UnsupportedOperationException.class, () -> s3AsyncClient.uploadPart(builder -> builder.uploadId("fail").build(), AsyncRequestBody.fromString("fail")).join());
+        s3AsyncClient.close();
+    }
+
+    @Test
+    public void s3AsyncClientForbidsCompleteMultipartUpload() {
+        S3AsyncClient s3AsyncClient = S3AsyncEncryptionClient.builder()
+                .kmsKeyId("fails")
+                .build();
+
+        assertThrows(UnsupportedOperationException.class, () -> s3AsyncClient.completeMultipartUpload(builder -> builder.uploadId("fail").build()).join());
+        s3AsyncClient.close();
+    }
 }