Skip to content

Commit f331125

Browse files
LihaoLiuXsLihaoLiuXs
authored
blobStore: Directory tagging (#217)
1 parent 4146d97 commit f331125

File tree

8 files changed

+218
-47
lines changed

8 files changed

+218
-47
lines changed

‎blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -578,12 +578,26 @@ public DirectoryDownloadResponse toDirectoryDownloadResponse(CompletedDirectoryD
578578
}
579579

580580
public UploadDirectoryRequest toUploadDirectoryRequest(DirectoryUploadRequest request) {
581-
return UploadDirectoryRequest.builder()
581+
UploadDirectoryRequest.Builder builder = UploadDirectoryRequest.builder()
582582
.bucket(getBucket())
583583
.source(Paths.get(request.getLocalSourceDirectory()))
584584
.maxDepth(request.isIncludeSubFolders() ? Integer.MAX_VALUE : 1)
585-
.s3Prefix(request.getPrefix())
586-
.build();
585+
.s3Prefix(request.getPrefix());
586+
587+
// Merge tags into the existing PutObjectRequest per file; putObjectRequest(Consumer) would replace it and drop bucket/key.
588+
if (request.getTags() != null && !request.getTags().isEmpty()) {
589+
List<Tag> tagSet = request.getTags().entrySet().stream()
590+
.map(e -> Tag.builder().key(e.getKey()).value(e.getValue()).build())
591+
.collect(Collectors.toList());
592+
builder.uploadFileRequestTransformer(fileRequestBuilder -> {
593+
PutObjectRequest existing = fileRequestBuilder.build().putObjectRequest();
594+
fileRequestBuilder.putObjectRequest(existing.toBuilder()
595+
.tagging(Tagging.builder().tagSet(tagSet).build())
596+
.build());
597+
});
598+
}
599+
600+
return builder.build();
587601
}
588602

589603
public DirectoryUploadResponse toDirectoryUploadResponse(CompletedDirectoryUpload completedDirectoryUpload) {

‎blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStore.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
4444
import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration;
4545
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
46-
import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
4746
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
4847
import software.amazon.awssdk.http.nio.netty.ProxyConfiguration;
4948
import software.amazon.awssdk.regions.Region;
@@ -52,7 +51,6 @@
5251
import software.amazon.awssdk.services.s3.S3Configuration;
5352
import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
5453
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
55-
import software.amazon.awssdk.services.s3.crt.S3CrtProxyConfiguration;
5654
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
5755
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
5856
import software.amazon.awssdk.services.s3.model.Part;
@@ -79,7 +77,6 @@
7977
import java.util.List;
8078
import java.util.Map;
8179
import java.util.concurrent.CompletableFuture;
82-
import java.util.concurrent.ExecutionException;
8380
import java.util.concurrent.Executors;
8481
import java.util.function.Consumer;
8582
import java.util.stream.Collectors;

‎blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -698,6 +698,33 @@ void testToUploadDirectoryRequest() {
698698
assertTrue(request.maxDepth().isPresent());
699699
}
700700

701+
@Test
702+
void testToUploadDirectoryRequest_WithTags() {
703+
// Given
704+
Map<String, String> tags = Map.of("tag1", "value1", "tag2", "value2");
705+
DirectoryUploadRequest directoryUploadRequest = DirectoryUploadRequest.builder()
706+
.localSourceDirectory("/home/documents")
707+
.prefix("/files")
708+
.includeSubFolders(true)
709+
.tags(tags)
710+
.build();
711+
712+
// When
713+
UploadDirectoryRequest request = transformer.toUploadDirectoryRequest(directoryUploadRequest);
714+
715+
// Then
716+
assertEquals(BUCKET, request.bucket());
717+
assertTrue(request.maxDepth().isPresent());
718+
assertEquals(Integer.MAX_VALUE, request.maxDepth().getAsInt());
719+
assertTrue(request.s3Prefix().isPresent());
720+
assertEquals("/files", request.s3Prefix().get());
721+
assertEquals("/home/documents", request.source().toString());
722+
723+
// Note: AWS SDK 2.35.0 doesn't support tagging in directory uploads via UploadDirectoryRequest
724+
// Tags would need to be applied post-upload or when AWS SDK is upgraded
725+
assertNotNull(request);
726+
}
727+
701728
@Test
702729
void testToDirectoryUploadResponse() {
703730
Exception exception1 = new RuntimeException("Exception1!");

‎blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java‎

Lines changed: 108 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1205,45 +1205,115 @@ void doDownloadDirectory() throws ExecutionException, InterruptedException {
12051205
}
12061206

12071207
@Test
1208-
void doUploadDirectory() throws ExecutionException, InterruptedException {
1209-
DirectoryUpload awsResponseFuture = mock(DirectoryUpload.class);
1210-
CompletedDirectoryUpload awsResponse = mock(CompletedDirectoryUpload.class);
1211-
doReturn(awsResponseFuture).when(mockS3TransferManager).uploadDirectory(any(UploadDirectoryRequest.class));
1212-
doReturn(future(awsResponse)).when(awsResponseFuture).completionFuture();
1213-
1214-
Exception failedUploadException = new RuntimeException("Fake exception!");
1215-
Path failedUploadPath = Paths.get("/home/documents/files/business/taxes.csv");
1216-
UploadFileRequest uploadFileRequest = UploadFileRequest.builder()
1217-
.source(failedUploadPath)
1218-
.putObjectRequest(mock(PutObjectRequest.class))
1219-
.build();
1220-
List<FailedFileUpload> failedTransfers = List.of(FailedFileUpload.builder()
1221-
.request(uploadFileRequest)
1222-
.exception(failedUploadException)
1223-
.build());
1224-
doReturn(failedTransfers).when(awsResponse).failedTransfers();
1225-
1226-
String source = "/home/documents";
1227-
DirectoryUploadRequest uploadRequest = DirectoryUploadRequest.builder()
1228-
.localSourceDirectory(source)
1229-
.prefix("files/")
1230-
.includeSubFolders(true)
1231-
.build();
1232-
1233-
// Perform the request
1234-
DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
1235-
1236-
// Verify the wiring
1237-
ArgumentCaptor<UploadDirectoryRequest> requestCaptor = ArgumentCaptor.forClass(UploadDirectoryRequest.class);
1238-
verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
1239-
var actualCapturedValue = requestCaptor.getValue();
1240-
assertEquals(BUCKET, actualCapturedValue.bucket());
1241-
assertEquals(source, actualCapturedValue.source().toString());
1208+
void doUploadDirectory() throws ExecutionException, InterruptedException, IOException {
1209+
// Create a temporary directory with test files
1210+
Path tempDir = Files.createTempDirectory("test-upload-dir");
1211+
try {
1212+
// Create test files
1213+
Path file1 = tempDir.resolve("file1.txt");
1214+
Path file2 = tempDir.resolve("subdir").resolve("file2.txt");
1215+
Files.createDirectories(file2.getParent());
1216+
Files.write(file1, "content1".getBytes());
1217+
Files.write(file2, "content2".getBytes());
1218+
1219+
// Mock transfer manager directory upload
1220+
DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
1221+
CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
1222+
doReturn(mockDirectoryUpload).when(mockS3TransferManager).uploadDirectory(any(UploadDirectoryRequest.class));
1223+
doReturn(CompletableFuture.completedFuture(mockCompletedUpload)).when(mockDirectoryUpload).completionFuture();
1224+
doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
1225+
1226+
DirectoryUploadRequest uploadRequest = DirectoryUploadRequest.builder()
1227+
.localSourceDirectory(tempDir.toString())
1228+
.prefix("files/")
1229+
.includeSubFolders(true)
1230+
.build();
1231+
1232+
// Perform the request
1233+
DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
1234+
1235+
// Verify the results
1236+
assertNotNull(response);
1237+
assertTrue(response.getFailedTransfers().isEmpty());
1238+
1239+
// Verify transfer manager uploadDirectory was called with correct request
1240+
ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
1241+
ArgumentCaptor.forClass(UploadDirectoryRequest.class);
1242+
verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
1243+
UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
1244+
assertEquals(BUCKET, capturedRequest.bucket());
1245+
assertEquals(tempDir, capturedRequest.source());
1246+
assertEquals("files/", capturedRequest.s3Prefix().orElse(null));
1247+
} finally {
1248+
// Clean up
1249+
Files.walk(tempDir)
1250+
.sorted((a, b) -> b.compareTo(a))
1251+
.forEach(path -> {
1252+
try {
1253+
Files.deleteIfExists(path);
1254+
} catch (IOException e) {
1255+
// Ignore cleanup errors
1256+
}
1257+
});
1258+
}
1259+
}
12421260

1243-
// Verify the results
1244-
assertEquals(1, response.getFailedTransfers().size());
1245-
assertEquals(failedUploadException, response.getFailedTransfers().get(0).getException());
1246-
assertEquals(failedUploadPath, response.getFailedTransfers().get(0).getSource());
1261+
@Test
1262+
void doUploadDirectory_WithTags() throws ExecutionException, InterruptedException, IOException {
1263+
// Create a temporary directory with test files
1264+
Path tempDir = Files.createTempDirectory("test-upload-dir-tags");
1265+
try {
1266+
// Create test files
1267+
Path file1 = tempDir.resolve("file1.txt");
1268+
Path file2 = tempDir.resolve("subdir").resolve("file2.txt");
1269+
Files.createDirectories(file2.getParent());
1270+
Files.write(file1, "content1".getBytes());
1271+
Files.write(file2, "content2".getBytes());
1272+
1273+
Map<String, String> tags = Map.of("tag1", "value1", "tag2", "value2");
1274+
1275+
// Mock transfer manager directory upload
1276+
DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
1277+
CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
1278+
doReturn(mockDirectoryUpload).when(mockS3TransferManager).uploadDirectory(any(UploadDirectoryRequest.class));
1279+
doReturn(CompletableFuture.completedFuture(mockCompletedUpload)).when(mockDirectoryUpload).completionFuture();
1280+
doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
1281+
1282+
DirectoryUploadRequest uploadRequest = DirectoryUploadRequest.builder()
1283+
.localSourceDirectory(tempDir.toString())
1284+
.prefix("files/")
1285+
.includeSubFolders(true)
1286+
.tags(tags)
1287+
.build();
1288+
1289+
// Perform the request
1290+
DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
1291+
1292+
// Verify the results
1293+
assertNotNull(response);
1294+
assertTrue(response.getFailedTransfers().isEmpty());
1295+
1296+
// Verify transfer manager uploadDirectory was called with correct request (including tags via transformer)
1297+
ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
1298+
ArgumentCaptor.forClass(UploadDirectoryRequest.class);
1299+
verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
1300+
UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
1301+
assertEquals(BUCKET, capturedRequest.bucket());
1302+
assertEquals(tempDir, capturedRequest.source());
1303+
assertEquals("files/", capturedRequest.s3Prefix().orElse(null));
1304+
assertNotNull(capturedRequest.uploadFileRequestTransformer());
1305+
} finally {
1306+
// Clean up
1307+
Files.walk(tempDir)
1308+
.sorted((a, b) -> b.compareTo(a))
1309+
.forEach(path -> {
1310+
try {
1311+
Files.deleteIfExists(path);
1312+
} catch (IOException e) {
1313+
// Ignore cleanup errors
1314+
}
1315+
});
1316+
}
12471317
}
12481318

12491319
@Test

‎blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DirectoryUploadRequest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import lombok.Builder;
44
import lombok.Getter;
55

6+
import java.util.Map;
7+
68
/**
79
* Wrapper object for directory upload data
810
*/
@@ -12,4 +14,8 @@ public class DirectoryUploadRequest {
1214
private final String localSourceDirectory;
1315
private final String prefix;
1416
private final boolean includeSubFolders;
17+
/**
18+
* (Optional parameter) The map of tagName to tagValue to be associated with all blobs in the directory
19+
*/
20+
private final Map<String, String> tags;
1521
}

‎blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/UploadRequest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public class UploadRequest {
5656
* (Optional parameter) Object lock configuration for WORM protection.
5757
*/
5858
private final ObjectLockConfiguration objectLock;
59-
59+
6060
private UploadRequest(Builder builder) {
6161
this.key = builder.key;
6262
this.contentLength = builder.contentLength;

‎blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStore.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -592,8 +592,17 @@ protected DirectoryUploadResponse doUploadDirectory(DirectoryUploadRequest direc
592592
// Generate blob key
593593
String blobKey = transformer.toBlobKey(sourceDir, filePath, directoryUploadRequest.getPrefix());
594594

595-
// Upload file to GCS - use same approach as single file upload
596-
com.google.cloud.storage.BlobInfo blobInfo = com.google.cloud.storage.BlobInfo.newBuilder(getBucket(), blobKey).build();
595+
// Build metadata map with tags if provided
596+
Map<String, String> metadata = new HashMap<>();
597+
if (directoryUploadRequest.getTags() != null && !directoryUploadRequest.getTags().isEmpty()) {
598+
directoryUploadRequest.getTags().forEach((tagName, tagValue) ->
599+
metadata.put(TAG_PREFIX + tagName, tagValue));
600+
}
601+
602+
// Upload file to GCS with tags applied
603+
com.google.cloud.storage.BlobInfo blobInfo = com.google.cloud.storage.BlobInfo.newBuilder(getBucket(), blobKey)
604+
.setMetadata(metadata.isEmpty() ? null : metadata)
605+
.build();
597606
storage.createFrom(blobInfo, filePath);
598607
} catch (Exception e) {
599608
failedUploads.add(FailedBlobUpload.builder()

‎blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStoreTest.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@
113113
import static org.mockito.Mockito.lenient;
114114
import static org.mockito.Mockito.mock;
115115
import static org.mockito.Mockito.never;
116+
import static org.mockito.Mockito.times;
116117
import static org.mockito.Mockito.verify;
117118
import static org.mockito.Mockito.when;
118119

@@ -1794,6 +1795,53 @@ void testUploadDirectory_Success() throws Exception {
17941795
verify(mockStorage).createFrom(any(BlobInfo.class), eq(file2));
17951796
}
17961797

1798+
@Test
1799+
void testUploadDirectory_WithTags() throws Exception {
1800+
// Given
1801+
Map<String, String> tags = Map.of("tag1", "value1", "tag2", "value2");
1802+
DirectoryUploadRequest request = DirectoryUploadRequest.builder()
1803+
.localSourceDirectory(tempDir.toString())
1804+
.prefix("uploads/")
1805+
.includeSubFolders(true)
1806+
.tags(tags)
1807+
.build();
1808+
1809+
// Create test files in temp directory
1810+
Path file1 = tempDir.resolve("file1.txt");
1811+
Path file2 = tempDir.resolve("subdir").resolve("file2.txt");
1812+
Files.createDirectories(file2.getParent());
1813+
Files.write(file1, "content1".getBytes());
1814+
Files.write(file2, "content2".getBytes());
1815+
1816+
List<Path> filePaths = List.of(file1, file2);
1817+
when(mockTransformer.toFilePaths(request)).thenReturn(filePaths);
1818+
when(mockTransformer.toBlobKey(eq(tempDir), eq(file1), eq("uploads/")))
1819+
.thenReturn("uploads/file1.txt");
1820+
when(mockTransformer.toBlobKey(eq(tempDir), eq(file2), eq("uploads/")))
1821+
.thenReturn("uploads/subdir/file2.txt");
1822+
1823+
// When
1824+
DirectoryUploadResponse response = gcpBlobStore.uploadDirectory(request);
1825+
1826+
// Then
1827+
assertNotNull(response);
1828+
assertTrue(response.getFailedTransfers().isEmpty());
1829+
1830+
// Verify that tags are applied to both files
1831+
ArgumentCaptor<BlobInfo> blobInfoCaptor = ArgumentCaptor.forClass(BlobInfo.class);
1832+
verify(mockStorage, times(2)).createFrom(blobInfoCaptor.capture(), any(Path.class));
1833+
1834+
List<BlobInfo> capturedBlobInfos = blobInfoCaptor.getAllValues();
1835+
assertEquals(2, capturedBlobInfos.size());
1836+
1837+
// Verify tags are present in metadata with TAG_PREFIX
1838+
for (BlobInfo blobInfo : capturedBlobInfos) {
1839+
assertNotNull(blobInfo.getMetadata());
1840+
assertEquals("value1", blobInfo.getMetadata().get("gcp-tag-tag1"));
1841+
assertEquals("value2", blobInfo.getMetadata().get("gcp-tag-tag2"));
1842+
}
1843+
}
1844+
17971845
@Test
17981846
void testUploadDirectory_WithFailures() throws Exception {
17991847
// Given

0 commit comments

Comments
 (0)