Skip to content

Commit 32a920f

Browse files
sandeepvinayaksandeepvinayak
authored
blobstore: Add SSE in multi-part upload (#112)
1 parent 581005b commit 32a920f

File tree

23 files changed

+457
-14
lines changed

23 files changed

+457
-14
lines changed

‎blob/blob-ali/src/main/java/com/salesforce/multicloudj/blob/ali/AliBlobStore.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ protected ListBlobsPageResponse doListPage(ListBlobsPageRequest request) {
361361
protected MultipartUpload doInitiateMultipartUpload(final MultipartUploadRequest request){
362362
InitiateMultipartUploadRequest initiateMultipartUploadRequest = transformer.toInitiateMultipartUploadRequest(request);
363363
InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
364-
return transformer.toMultipartUpload(initiateMultipartUploadResult, request.getMetadata());
364+
return transformer.toMultipartUpload(initiateMultipartUploadResult, request.getMetadata(), request.getKmsKeyId());
365365
}
366366

367367
/**

‎blob/blob-ali/src/main/java/com/salesforce/multicloudj/blob/ali/AliTransformer.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,15 +202,22 @@ public BlobMetadata toBlobMetadata(String key, ObjectMetadata metadata) {
202202
public InitiateMultipartUploadRequest toInitiateMultipartUploadRequest(MultipartUploadRequest request) {
203203
ObjectMetadata metadata = new ObjectMetadata();
204204
metadata.setUserMetadata(request.getMetadata());
205+
206+
if (request.getKmsKeyId() != null && !request.getKmsKeyId().isEmpty()) {
207+
metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION);
208+
metadata.setHeader(OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION_KEY_ID, request.getKmsKeyId());
209+
}
210+
205211
return new InitiateMultipartUploadRequest(getBucket(), request.getKey(), metadata);
206212
}
207213

208-
public MultipartUpload toMultipartUpload(InitiateMultipartUploadResult initiateMultipartUploadResult, Map<String, String> metadata) {
214+
public MultipartUpload toMultipartUpload(InitiateMultipartUploadResult initiateMultipartUploadResult, Map<String, String> metadata, String kmsKeyId) {
209215
return MultipartUpload.builder()
210216
.bucket(initiateMultipartUploadResult.getBucketName())
211217
.key(initiateMultipartUploadResult.getKey())
212218
.id(initiateMultipartUploadResult.getUploadId())
213219
.metadata(metadata)
220+
.kmsKeyId(kmsKeyId)
214221
.build();
215222
}
216223

‎blob/blob-ali/src/test/java/com/salesforce/multicloudj/blob/ali/AliBlobStoreTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,36 @@ void testDoInitiateMultipartUpload() {
542542
assertEquals(metadata, actualRequest.getObjectMetadata().getUserMetadata());
543543
}
544544

545+
@Test
546+
void testDoInitiateMultipartUploadWithKms() {
547+
InitiateMultipartUploadResult mockResponse = mock(InitiateMultipartUploadResult.class);
548+
doReturn("bucket-1").when(mockResponse).getBucketName();
549+
doReturn("object-1").when(mockResponse).getKey();
550+
doReturn("mpu-id").when(mockResponse).getUploadId();
551+
when(mockOssClient.initiateMultipartUpload((InitiateMultipartUploadRequest) any())).thenReturn(mockResponse);
552+
Map<String, String> metadata = Map.of("key-1", "value-1");
553+
String kmsKeyId = "test-kms-key-id";
554+
MultipartUploadRequest request = new MultipartUploadRequest.Builder()
555+
.withKey("object-1")
556+
.withMetadata(metadata)
557+
.withKmsKeyId(kmsKeyId)
558+
.build();
559+
560+
MultipartUpload response = ali.initiateMultipartUpload(request);
561+
562+
ArgumentCaptor<InitiateMultipartUploadRequest> requestCaptor = ArgumentCaptor.forClass(InitiateMultipartUploadRequest.class);
563+
verify(mockOssClient, times(1)).initiateMultipartUpload(requestCaptor.capture());
564+
InitiateMultipartUploadRequest actualRequest = requestCaptor.getValue();
565+
assertEquals("object-1", actualRequest.getKey());
566+
assertEquals("bucket-1", actualRequest.getBucketName());
567+
assertEquals(metadata, actualRequest.getObjectMetadata().getUserMetadata());
568+
assertEquals(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION, actualRequest.getObjectMetadata().getServerSideEncryption());
569+
assertEquals(kmsKeyId, actualRequest.getObjectMetadata().getRawMetadata().get(OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION_KEY_ID));
570+
571+
// Verify the response has KMS key
572+
assertEquals(kmsKeyId, response.getKmsKeyId());
573+
}
574+
545575
@Test
546576
void testDoUploadMultipartPart() {
547577
UploadPartResult mockResponse = mock(UploadPartResult.class);

‎blob/blob-ali/src/test/java/com/salesforce/multicloudj/blob/ali/AliTransformerTest.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,14 +350,32 @@ void testToMultipartUpload() {
350350
doReturn("uploadId").when(initiateMultipartUploadResult).getUploadId();
351351
Map<String, String> metadata = Map.of("key1", "value1", "key2", "value2");
352352

353-
var actual = transformer.toMultipartUpload(initiateMultipartUploadResult, metadata);
353+
var actual = transformer.toMultipartUpload(initiateMultipartUploadResult, metadata, null);
354354

355355
assertEquals(BUCKET, actual.getBucket());
356356
assertEquals("key", actual.getKey());
357357
assertEquals("uploadId", actual.getId());
358358
assertEquals(metadata, actual.getMetadata());
359359
}
360360

361+
@Test
362+
void testToMultipartUploadWithKms() {
363+
InitiateMultipartUploadResult initiateMultipartUploadResult = mock(InitiateMultipartUploadResult.class);
364+
doReturn(BUCKET).when(initiateMultipartUploadResult).getBucketName();
365+
doReturn("key").when(initiateMultipartUploadResult).getKey();
366+
doReturn("uploadId").when(initiateMultipartUploadResult).getUploadId();
367+
Map<String, String> metadata = Map.of("key1", "value1", "key2", "value2");
368+
String kmsKeyId = "test-kms-key-id";
369+
370+
var actual = transformer.toMultipartUpload(initiateMultipartUploadResult, metadata, kmsKeyId);
371+
372+
assertEquals(BUCKET, actual.getBucket());
373+
assertEquals("key", actual.getKey());
374+
assertEquals("uploadId", actual.getId());
375+
assertEquals(metadata, actual.getMetadata());
376+
assertEquals(kmsKeyId, actual.getKmsKeyId());
377+
}
378+
361379
@Test
362380
void testToUploadPartRequest() {
363381
Map<String, String> metadata = Map.of("key1", "value1", "key2", "value2");

‎blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsBlobStore.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,7 @@ protected MultipartUpload doInitiateMultipartUpload(final MultipartUploadRequest
369369
.key(createMultipartUploadResponse.key())
370370
.id(createMultipartUploadResponse.uploadId())
371371
.metadata(request.getMetadata())
372+
.kmsKeyId(request.getKmsKeyId())
372373
.build();
373374
}
374375

‎blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ public PutObjectRequest toRequest(UploadRequest request) {
150150
.tagging(Tagging.builder().tagSet(tags).build());
151151

152152
if (request.getKmsKeyId() != null && !request.getKmsKeyId().isEmpty()) {
153-
builder.serverSideEncryption("aws:kms")
153+
builder.serverSideEncryption(ServerSideEncryption.AWS_KMS)
154154
.ssekmsKeyId(request.getKmsKeyId());
155155
}
156156

@@ -292,11 +292,17 @@ byte[] eTagToMD5(String eTag) {
292292
}
293293

294294
public CreateMultipartUploadRequest toCreateMultipartUploadRequest(MultipartUploadRequest request) {
295-
return CreateMultipartUploadRequest.builder()
295+
CreateMultipartUploadRequest.Builder builder = CreateMultipartUploadRequest.builder()
296296
.bucket(getBucket())
297297
.key(request.getKey())
298-
.metadata(request.getMetadata())
299-
.build();
298+
.metadata(request.getMetadata());
299+
300+
if (request.getKmsKeyId() != null && !request.getKmsKeyId().isEmpty()) {
301+
builder.serverSideEncryption(ServerSideEncryption.AWS_KMS)
302+
.ssekmsKeyId(request.getKmsKeyId());
303+
}
304+
305+
return builder.build();
300306
}
301307

302308
public UploadPartRequest toUploadPartRequest(MultipartUpload mpu, MultipartPart mpp) {

‎blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsBlobStoreTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
import software.amazon.awssdk.services.s3.model.PutObjectTaggingResponse;
7171
import software.amazon.awssdk.services.s3.model.S3Exception;
7272
import software.amazon.awssdk.services.s3.model.S3Object;
73+
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
7374
import software.amazon.awssdk.services.s3.model.Tag;
7475
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
7576
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
@@ -687,6 +688,40 @@ void testDoInitiateMultipartUpload() {
687688
assertEquals("mpu-id", response.getId());
688689
}
689690

691+
@Test
692+
void testDoInitiateMultipartUploadWithKms() {
693+
CreateMultipartUploadResponse mockResponse = mock(CreateMultipartUploadResponse.class);
694+
doReturn("bucket-1").when(mockResponse).bucket();
695+
doReturn("object-1").when(mockResponse).key();
696+
doReturn("mpu-id").when(mockResponse).uploadId();
697+
when(mockS3Client.createMultipartUpload((CreateMultipartUploadRequest) any())).thenReturn(mockResponse);
698+
Map<String, String> metadata = Map.of("key-1", "value-1");
699+
String kmsKeyId = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012";
700+
MultipartUploadRequest request = new MultipartUploadRequest.Builder()
701+
.withKey("object-1")
702+
.withMetadata(metadata)
703+
.withKmsKeyId(kmsKeyId)
704+
.build();
705+
706+
MultipartUpload response = aws.initiateMultipartUpload(request);
707+
708+
// Verify the request is mapped to the SDK with KMS encryption
709+
ArgumentCaptor<CreateMultipartUploadRequest> requestCaptor = ArgumentCaptor.forClass(CreateMultipartUploadRequest.class);
710+
verify(mockS3Client, times(1)).createMultipartUpload(requestCaptor.capture());
711+
CreateMultipartUploadRequest actualRequest = requestCaptor.getValue();
712+
assertEquals("object-1", actualRequest.key());
713+
assertEquals("bucket-1", actualRequest.bucket());
714+
assertEquals(metadata, actualRequest.metadata());
715+
assertEquals(ServerSideEncryption.AWS_KMS, actualRequest.serverSideEncryption());
716+
assertEquals(kmsKeyId, actualRequest.ssekmsKeyId());
717+
718+
// Verify the response is mapped back properly with KMS key
719+
assertEquals("object-1", response.getKey());
720+
assertEquals("bucket-1", response.getBucket());
721+
assertEquals("mpu-id", response.getId());
722+
assertEquals(kmsKeyId, response.getKmsKeyId());
723+
}
724+
690725
@Test
691726
void testDoUploadMultipartPart() {
692727
UploadPartResponse mockResponse = mock(UploadPartResponse.class);

‎blob/blob-aws/src/test/resources/mappings/delete-4lfap3itgx.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"id" : "db22fc9f-3cea-41b0-a18f-6dc0b9985c0c",
3+
"name" : "chameleon-jcloud_conformance-tests_multipart-withkms",
4+
"request" : {
5+
"url" : "/chameleon-jcloud/conformance-tests/multipart-withKms",
6+
"method" : "DELETE"
7+
},
8+
"response" : {
9+
"status" : 204,
10+
"headers" : {
11+
"Server" : "AmazonS3",
12+
"x-amz-request-id" : "QEXVC59YXBQPMRTM",
13+
"x-amz-id-2" : "+N1BJ/b+RhzccdF12ZsqyJoepI5nCc2kEh7xEBuVFq3VzqwJ2NjDrAWrwP0LXcOqz6FKCgtYyuA=",
14+
"Date" : "Thu, 30 Oct 2025 03:16:06 GMT"
15+
}
16+
},
17+
"uuid" : "db22fc9f-3cea-41b0-a18f-6dc0b9985c0c",
18+
"persistent" : true,
19+
"insertionIndex" : 541
20+
}

‎blob/blob-aws/src/test/resources/mappings/delete-rxwqfxe3lf.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"id" : "1771bde0-65c2-481e-9256-2f4318863074",
3+
"name" : "chameleon-jcloud_conformance-tests_multipart-withkms",
4+
"request" : {
5+
"url" : "/chameleon-jcloud/conformance-tests/multipart-withKms?uploadId=HP3yS0r4PKOhIfexxxwxciB6pikz.Mk0xzdlaZdnwSCFX9NNc2EUsYmNL_RS77aRIcK3.taL6VBdjpgwVFpkb6CKVP1vWsn2wTNKAtm_P3c9G_7smpYdQerWRyT4seAKvmaaeVOi4HbR2Pidr_sEPkI3WwVrVc59Yqu7IuVfzO9FW_VUXOBvYnk5qsKuQSTUh6p5UmvZXrYTOGtJHq0C9w--",
6+
"method" : "DELETE"
7+
},
8+
"response" : {
9+
"status" : 204,
10+
"headers" : {
11+
"Server" : "AmazonS3",
12+
"x-amz-request-id" : "QEXKDM9VY5PD64D6",
13+
"x-amz-id-2" : "5rVY8XuJLUAYgfYr1eRG8hujkZlDHlnLtKiMUW6uslOvv955eC4sb3QkF7EDLBB3BJV2Q1OuF+M=",
14+
"Date" : "Thu, 30 Oct 2025 03:16:06 GMT"
15+
}
16+
},
17+
"uuid" : "1771bde0-65c2-481e-9256-2f4318863074",
18+
"persistent" : true,
19+
"insertionIndex" : 540
20+
}

‎blob/blob-aws/src/test/resources/mappings/get-ukvs6x01b5.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"id" : "19ea6fb7-e8e3-4194-a40a-83a5373fd128",
3+
"name" : "chameleon-jcloud_conformance-tests_multipart-withkms",
4+
"request" : {
5+
"url" : "/chameleon-jcloud/conformance-tests/multipart-withKms?uploadId=HP3yS0r4PKOhIfexxxwxciB6pikz.Mk0xzdlaZdnwSCFX9NNc2EUsYmNL_RS77aRIcK3.taL6VBdjpgwVFpkb6CKVP1vWsn2wTNKAtm_P3c9G_7smpYdQerWRyT4seAKvmaaeVOi4HbR2Pidr_sEPkI3WwVrVc59Yqu7IuVfzO9FW_VUXOBvYnk5qsKuQSTUh6p5UmvZXrYTOGtJHq0C9w--",
6+
"method" : "GET"
7+
},
8+
"response" : {
9+
"status" : 200,
10+
"body" : "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ListPartsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Bucket>chameleon-jcloud</Bucket><Key>conformance-tests/multipart-withKms</Key><UploadId>HP3yS0r4PKOhIfexxxwxciB6pikz.Mk0xzdlaZdnwSCFX9NNc2EUsYmNL_RS77aRIcK3.taL6VBdjpgwVFpkb6CKVP1vWsn2wTNKAtm_P3c9G_7smpYdQerWRyT4seAKvmaaeVOi4HbR2Pidr_sEPkI3WwVrVc59Yqu7IuVfzO9FW_VUXOBvYnk5qsKuQSTUh6p5UmvZXrYTOGtJHq0C9w--</UploadId><Initiator><ID>arn:aws:sts::654654370895:assumed-role/PCSKAdministratorAccessRole/PCSK-sandeeppal@d2999721-cb95-450c-8455-96b96fdcdc84</ID><DisplayName>PCSKAdministratorAccessRole/PCSK-sandeeppal@d2999721-cb95-450c-8455-96b96fdcdc84</DisplayName></Initiator><Owner><ID>b6eaab2af32ba61bce00b8c9aceaa1c649844388ec2c3827bbd3e2b06f798cf1</ID></Owner><StorageClass>STANDARD</StorageClass><PartNumberMarker>0</PartNumberMarker><NextPartNumberMarker>2</NextPartNumberMarker><MaxParts>1000</MaxParts><IsTruncated>false</IsTruncated><Part><PartNumber>1</PartNumber><LastModified>2025-10-30T03:16:01.000Z</LastModified><ETag>&quot;ddbbf3e1d34182a6a0233f9e3b607419&quot;</ETag><Size>5242880</Size></Part><Part><PartNumber>2</PartNumber><LastModified>2025-10-30T03:16:03.000Z</LastModified><ETag>&quot;8582535128d55f015eb60cc3d05b3355&quot;</ETag><Size>5242880</Size></Part></ListPartsResult>",
11+
"headers" : {
12+
"Server" : "AmazonS3",
13+
"x-amz-request-id" : "D13SSBSNEPX0SRBQ",
14+
"x-amz-id-2" : "mERBuJQtkoMlMy1yy6m4LemDsZlQzIpXwywQrfWgwDWaatzKG/g6HfyWsFlqOjZVD9X7bqgQi7o=",
15+
"Date" : "Thu, 30 Oct 2025 03:16:04 GMT",
16+
"Content-Type" : "application/xml"
17+
}
18+
},
19+
"uuid" : "19ea6fb7-e8e3-4194-a40a-83a5373fd128",
20+
"persistent" : true,
21+
"insertionIndex" : 544
22+
}

0 commit comments

Comments
 (0)