Skip to content

Commit 9e87bc1

Browse files
LihaoLiuXsLihaoLiuXs
authored
Add directory operations and storageClass for GCP blob (salesforce#93)
1 parent efd5eb4 commit 9e87bc1

File tree

20 files changed

+2901
-63
lines changed

20 files changed

+2901
-63
lines changed

‎blob/blob-ali/src/main/java/com/salesforce/multicloudj/blob/ali/AliTransformer.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ protected ObjectMetadata generateObjectMetadata(UploadRequest uploadRequest) {
7575
metadata.setUserMetadata(uploadRequest.getMetadata());
7676
metadata.setObjectTagging(uploadRequest.getTags());
7777

78+
// Set storage class if provided
79+
if (uploadRequest.getStorageClass() != null && !uploadRequest.getStorageClass().isEmpty()) {
80+
metadata.setHeader("x-oss-storage-class", uploadRequest.getStorageClass());
81+
}
82+
7883
if (uploadRequest.getKmsKeyId() != null && !uploadRequest.getKmsKeyId().isEmpty()) {
7984
metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION);
8085
metadata.setHeader(OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION_KEY_ID, uploadRequest.getKmsKeyId());

‎blob/blob-ali/src/test/java/com/salesforce/multicloudj/blob/ali/AliTransformerTest.java‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.salesforce.multicloudj.blob.driver.UploadRequest;
2525
import org.apache.commons.lang3.tuple.Pair;
2626
import org.junit.jupiter.api.Test;
27+
import static org.junit.jupiter.api.Assertions.assertNull;
2728

2829
import java.io.ByteArrayInputStream;
2930
import java.io.File;
@@ -587,4 +588,116 @@ void testToListObjectsRequest() {
587588
assertEquals(request.getPaginationToken(), actual.getMarker());
588589
assertEquals(request.getMaxResults(), actual.getMaxKeys());
589590
}
591+
592+
@Test
593+
void testGenerateObjectMetadataWithStorageClass() {
594+
UploadRequest uploadRequest = UploadRequest.builder()
595+
.withKey("test-key")
596+
.withMetadata(Map.of("key1", "value1"))
597+
.withTags(Map.of("tag1", "value1"))
598+
.withStorageClass("IA")
599+
.build();
600+
601+
ObjectMetadata result = transformer.generateObjectMetadata(uploadRequest);
602+
603+
assertEquals(Map.of("key1", "value1"), result.getUserMetadata());
604+
assertNotNull(result);
605+
}
606+
607+
@Test
608+
void testGenerateObjectMetadataWithStandardStorageClass() {
609+
UploadRequest uploadRequest = UploadRequest.builder()
610+
.withKey("test-key")
611+
.withStorageClass("Standard")
612+
.build();
613+
614+
ObjectMetadata result = transformer.generateObjectMetadata(uploadRequest);
615+
616+
// Verify the metadata object was created successfully
617+
assertNotNull(result);
618+
}
619+
620+
@Test
621+
void testGenerateObjectMetadataWithArchiveStorageClass() {
622+
UploadRequest uploadRequest = UploadRequest.builder()
623+
.withKey("test-key")
624+
.withStorageClass("Archive")
625+
.build();
626+
627+
ObjectMetadata result = transformer.generateObjectMetadata(uploadRequest);
628+
629+
// Verify the metadata object was created successfully
630+
assertNotNull(result);
631+
}
632+
633+
@Test
634+
void testGenerateObjectMetadataWithNullStorageClass() {
635+
UploadRequest uploadRequest = UploadRequest.builder()
636+
.withKey("test-key")
637+
.withStorageClass(null)
638+
.build();
639+
640+
ObjectMetadata result = transformer.generateObjectMetadata(uploadRequest);
641+
642+
// Verify the metadata object was created successfully
643+
assertNotNull(result);
644+
}
645+
646+
@Test
647+
void testGenerateObjectMetadataWithEmptyStorageClass() {
648+
UploadRequest uploadRequest = UploadRequest.builder()
649+
.withKey("test-key")
650+
.withStorageClass("")
651+
.build();
652+
653+
ObjectMetadata result = transformer.generateObjectMetadata(uploadRequest);
654+
655+
// Verify the metadata object was created successfully
656+
assertNotNull(result);
657+
}
658+
659+
@Test
660+
void testGenerateObjectMetadataWithoutStorageClass() {
661+
UploadRequest uploadRequest = UploadRequest.builder()
662+
.withKey("test-key")
663+
.withMetadata(Map.of("key1", "value1"))
664+
.withTags(Map.of("tag1", "value1"))
665+
.build();
666+
667+
ObjectMetadata result = transformer.generateObjectMetadata(uploadRequest);
668+
669+
assertEquals(Map.of("key1", "value1"), result.getUserMetadata());
670+
assertNotNull(result);
671+
}
672+
673+
@Test
674+
void testToPutObjectRequestWithStorageClass() {
675+
UploadRequest uploadRequest = UploadRequest.builder()
676+
.withKey("test-key")
677+
.withStorageClass("IA")
678+
.build();
679+
680+
InputStream inputStream = new ByteArrayInputStream("test data".getBytes());
681+
com.aliyun.oss.model.PutObjectRequest result = transformer.toPutObjectRequest(uploadRequest, inputStream);
682+
683+
assertEquals(BUCKET, result.getBucketName());
684+
assertEquals("test-key", result.getKey());
685+
assertNotNull(result.getMetadata());
686+
}
687+
688+
@Test
689+
void testToPutObjectRequestWithFileAndStorageClass() {
690+
UploadRequest uploadRequest = UploadRequest.builder()
691+
.withKey("test-key")
692+
.withStorageClass("Archive")
693+
.build();
694+
695+
File file = new File("test-file.txt");
696+
com.aliyun.oss.model.PutObjectRequest result = transformer.toPutObjectRequest(uploadRequest, file);
697+
698+
assertEquals(BUCKET, result.getBucketName());
699+
assertEquals("test-key", result.getKey());
700+
// Verify the request was created successfully with metadata
701+
assertNotNull(result.getMetadata());
702+
}
590703
}

‎blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
import com.salesforce.multicloudj.blob.driver.MultipartUploadRequest;
2121
import com.salesforce.multicloudj.blob.driver.PresignedUrlRequest;
2222
import com.salesforce.multicloudj.blob.driver.UploadPartResponse;
23+
import com.salesforce.multicloudj.common.exceptions.InvalidArgumentException;
24+
import com.salesforce.multicloudj.common.exceptions.UnSupportedOperationException;
25+
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
26+
import software.amazon.awssdk.services.s3.model.StorageClass;
2327
import com.salesforce.multicloudj.blob.driver.UploadRequest;
2428
import com.salesforce.multicloudj.common.util.HexUtil;
2529
import software.amazon.awssdk.core.async.AsyncRequestBody;
@@ -149,10 +153,21 @@ public PutObjectRequest toRequest(UploadRequest request) {
149153
builder.serverSideEncryption("aws:kms")
150154
.ssekmsKeyId(request.getKmsKeyId());
151155
}
152-
156+
157+
// Set storage class if provided
158+
if (request.getStorageClass() != null && !request.getStorageClass().isEmpty()) {
159+
try {
160+
StorageClass awsStorageClass = StorageClass.fromValue(request.getStorageClass());
161+
builder.storageClass(awsStorageClass);
162+
} catch (IllegalArgumentException e) {
163+
throw new InvalidArgumentException("Invalid storage class: " + request.getStorageClass(), e);
164+
}
165+
}
166+
153167
return builder.build();
154168
}
155169

170+
156171
public GetObjectRequest toRequest(DownloadRequest request) {
157172
var builder = GetObjectRequest
158173
.builder()

‎blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java‎

Lines changed: 161 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import software.amazon.awssdk.services.s3.model.PutObjectTaggingRequest;
3737
import software.amazon.awssdk.services.s3.model.S3Object;
3838
import software.amazon.awssdk.services.s3.model.Tag;
39+
import software.amazon.awssdk.services.s3.model.Tagging;
3940
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
4041
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
4142
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
@@ -48,6 +49,7 @@
4849
import software.amazon.awssdk.transfer.s3.model.FailedFileUpload;
4950
import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest;
5051
import software.amazon.awssdk.transfer.s3.model.UploadFileRequest;
52+
import static org.junit.jupiter.api.Assertions.assertThrows;
5153

5254
import java.io.ByteArrayInputStream;
5355
import java.io.InputStream;
@@ -97,7 +99,7 @@ void testUpload() {
9799
.bucket(BUCKET)
98100
.key(key)
99101
.metadata(metadata)
100-
.tagging("tag-key=tag-value")
102+
.tagging(Tagging.builder().tagSet(List.of(Tag.builder().key("tag-key").value("tag-value").build())).build())
101103
.build();
102104

103105
assertEquals(expected, transformer.toRequest(request));
@@ -517,6 +519,7 @@ void testToPutObjectPresignRequestWithKmsKey() {
517519
assertEquals(kmsKeyId, actualRequest.putObjectRequest().ssekmsKeyId());
518520
}
519521

522+
520523
@Test
521524
void testToPutObjectPresignRequestWithoutKmsKey() {
522525
Map<String, String> metadata = Map.of("some-key", "some-value");
@@ -702,4 +705,161 @@ public void testToBlobIdentifiers() {
702705
assertNull(blobIdentifiers.get(i).getVersionId());
703706
}
704707
}
708+
709+
@Test
710+
void testUploadRequestWithStorageClass() {
711+
var key = "some-key";
712+
var metadata = Map.of("some-key", "some-value");
713+
var tags = Map.of("tag-key", "tag-value");
714+
var storageClass = "STANDARD_IA";
715+
716+
var request = UploadRequest
717+
.builder()
718+
.withKey(key)
719+
.withMetadata(metadata)
720+
.withTags(tags)
721+
.withStorageClass(storageClass)
722+
.build();
723+
724+
var result = transformer.toRequest(request);
725+
726+
assertEquals(BUCKET, result.bucket());
727+
assertEquals(key, result.key());
728+
assertEquals(metadata, result.metadata());
729+
assertEquals("tag-key=tag-value", result.tagging());
730+
assertEquals(software.amazon.awssdk.services.s3.model.StorageClass.STANDARD_IA, result.storageClass());
731+
}
732+
733+
@Test
734+
void testUploadRequestWithStandardStorageClass() {
735+
var key = "some-key";
736+
var storageClass = "STANDARD";
737+
738+
var request = UploadRequest
739+
.builder()
740+
.withKey(key)
741+
.withStorageClass(storageClass)
742+
.build();
743+
744+
var result = transformer.toRequest(request);
745+
746+
assertEquals(software.amazon.awssdk.services.s3.model.StorageClass.STANDARD, result.storageClass());
747+
}
748+
749+
@Test
750+
void testUploadRequestWithGlacierStorageClass() {
751+
var key = "some-key";
752+
var storageClass = "GLACIER";
753+
754+
var request = UploadRequest
755+
.builder()
756+
.withKey(key)
757+
.withStorageClass(storageClass)
758+
.build();
759+
760+
var result = transformer.toRequest(request);
761+
762+
assertEquals(software.amazon.awssdk.services.s3.model.StorageClass.GLACIER, result.storageClass());
763+
}
764+
765+
@Test
766+
void testUploadRequestWithIntelligentTieringStorageClass() {
767+
var key = "some-key";
768+
var storageClass = "INTELLIGENT_TIERING";
769+
770+
var request = UploadRequest
771+
.builder()
772+
.withKey(key)
773+
.withStorageClass(storageClass)
774+
.build();
775+
776+
var result = transformer.toRequest(request);
777+
778+
assertEquals(software.amazon.awssdk.services.s3.model.StorageClass.INTELLIGENT_TIERING, result.storageClass());
779+
}
780+
781+
@Test
782+
void testUploadRequestWithDeepArchiveStorageClass() {
783+
var key = "some-key";
784+
var storageClass = "DEEP_ARCHIVE";
785+
786+
var request = UploadRequest
787+
.builder()
788+
.withKey(key)
789+
.withStorageClass(storageClass)
790+
.build();
791+
792+
var result = transformer.toRequest(request);
793+
794+
assertEquals(software.amazon.awssdk.services.s3.model.StorageClass.DEEP_ARCHIVE, result.storageClass());
795+
}
796+
797+
@Test
798+
void testUploadRequestWithGlacierIrStorageClass() {
799+
var key = "some-key";
800+
var storageClass = "GLACIER_IR";
801+
802+
var request = UploadRequest
803+
.builder()
804+
.withKey(key)
805+
.withStorageClass(storageClass)
806+
.build();
807+
808+
var result = transformer.toRequest(request);
809+
810+
assertEquals(software.amazon.awssdk.services.s3.model.StorageClass.GLACIER_IR, result.storageClass());
811+
}
812+
813+
@Test
814+
void testUploadRequestWithNullStorageClass() {
815+
var key = "some-key";
816+
817+
var request = UploadRequest
818+
.builder()
819+
.withKey(key)
820+
.withStorageClass(null)
821+
.build();
822+
823+
var result = transformer.toRequest(request);
824+
825+
assertNull(result.storageClass());
826+
}
827+
828+
@Test
829+
void testUploadRequestWithEmptyStorageClass() {
830+
var key = "some-key";
831+
832+
var request = UploadRequest
833+
.builder()
834+
.withKey(key)
835+
.withStorageClass("")
836+
.build();
837+
838+
var result = transformer.toRequest(request);
839+
840+
assertNull(result.storageClass());
841+
}
842+
843+
844+
@Test
845+
void testUploadRequestWithoutStorageClass() {
846+
var key = "some-key";
847+
var metadata = Map.of("some-key", "some-value");
848+
var tags = Map.of("tag-key", "tag-value");
849+
850+
var request = UploadRequest
851+
.builder()
852+
.withKey(key)
853+
.withMetadata(metadata)
854+
.withTags(tags)
855+
.build();
856+
857+
var result = transformer.toRequest(request);
858+
859+
assertEquals(BUCKET, result.bucket());
860+
assertEquals(key, result.key());
861+
assertEquals(metadata, result.metadata());
862+
assertEquals("tag-key=tag-value", result.tagging());
863+
assertNull(result.storageClass());
864+
}
705865
}

‎blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/async/driver/BlobStoreAsyncBridge.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -227,16 +227,16 @@ public Class<? extends SubstrateSdkException> getException(Throwable t) {
227227

228228
@Override
229229
public CompletableFuture<DirectoryDownloadResponse> downloadDirectory(DirectoryDownloadRequest directoryDownloadRequest){
230-
throw new UnsupportedOperationException("Feature not implemented");
230+
return CompletableFuture.supplyAsync(() -> blobStore.downloadDirectory(directoryDownloadRequest), executorService);
231231
}
232232

233233
@Override
234234
public CompletableFuture<DirectoryUploadResponse> uploadDirectory(DirectoryUploadRequest directoryUploadRequest) {
235-
throw new UnsupportedOperationException("Feature not implemented");
235+
return CompletableFuture.supplyAsync(() -> blobStore.uploadDirectory(directoryUploadRequest), executorService);
236236
}
237237

238238
@Override
239239
public CompletableFuture<Void> deleteDirectory(String prefix) {
240-
throw new UnsupportedOperationException("Feature not implemented");
240+
return CompletableFuture.runAsync(() -> blobStore.deleteDirectory(prefix), executorService);
241241
}
242242
}

0 commit comments

Comments
 (0)