feat: Folder provisioning implementation (#94)

* feat: Folder provisioning implementation

* feat: Folder provisioning implementation adjustments (dataAddress validation, formatting)

* feat: Folder provisioning implementation -dependencies update

Author: dorkau80

DEPENDENCIES

@@ -33,10 +33,10 @@ maven/mavencentral/com.azure/azure-core-http-netty/1.13.9, MIT AND Apache-2.0, a
 maven/mavencentral/com.azure/azure-core-management/1.11.5, MIT, approved, #10033
 maven/mavencentral/com.azure/azure-core-management/1.11.7, MIT, approved, #10033
 maven/mavencentral/com.azure/azure-core/1.43.0, MIT AND Apache-2.0, approved, #10548
-maven/mavencentral/com.azure/azure-core/1.44.1, , restricted, clearlydefined
+maven/mavencentral/com.azure/azure-core/1.44.1, MIT, approved, clearlydefined
 maven/mavencentral/com.azure/azure-identity/1.10.1, MIT AND Apache-2.0, approved, #10086
 maven/mavencentral/com.azure/azure-json/1.1.0, MIT AND Apache-2.0, approved, #10547
-maven/mavencentral/com.azure/azure-messaging-eventgrid/4.19.0, , restricted, clearlydefined
+maven/mavencentral/com.azure/azure-messaging-eventgrid/4.19.0, MIT, approved, clearlydefined
 maven/mavencentral/com.azure/azure-security-keyvault-keys/4.7.1, MIT, approved, #10872
 maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.7.0, MIT, approved, #10868
 maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.7.1, MIT, approved, #10868
@@ -182,15 +182,16 @@ maven/mavencentral/io.setl/rdf-urdna/1.1, Apache-2.0, approved, clearlydefined
 maven/mavencentral/io.swagger.core.v3/swagger-annotations-jakarta/2.1.13, Apache-2.0, approved, clearlydefined
 maven/mavencentral/io.swagger.core.v3/swagger-annotations-jakarta/2.2.15, Apache-2.0, approved, #5947
 maven/mavencentral/io.swagger.core.v3/swagger-annotations-jakarta/2.2.18, Apache-2.0, approved, #5947
+maven/mavencentral/io.swagger.core.v3/swagger-annotations-jakarta/2.2.19, Apache-2.0, approved, #5947
 maven/mavencentral/io.swagger.core.v3/swagger-core-jakarta/2.1.13, Apache-2.0, approved, clearlydefined
 maven/mavencentral/io.swagger.core.v3/swagger-core-jakarta/2.2.15, Apache-2.0, approved, #5929
 maven/mavencentral/io.swagger.core.v3/swagger-core-jakarta/2.2.18, Apache-2.0, approved, #5929
 maven/mavencentral/io.swagger.core.v3/swagger-integration-jakarta/2.1.13, Apache-2.0, approved, clearlydefined
-maven/mavencentral/io.swagger.core.v3/swagger-integration-jakarta/2.2.15, Apache-2.0, approved, clearlydefined
-maven/mavencentral/io.swagger.core.v3/swagger-integration-jakarta/2.2.18, , restricted, clearlydefined
+maven/mavencentral/io.swagger.core.v3/swagger-integration-jakarta/2.2.15, Apache-2.0, approved, #11475
+maven/mavencentral/io.swagger.core.v3/swagger-integration-jakarta/2.2.18, Apache-2.0, approved, #11475
 maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2-jakarta/2.1.13, Apache-2.0, approved, clearlydefined
-maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2-jakarta/2.2.15, Apache-2.0, approved, clearlydefined
-maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2-jakarta/2.2.18, , restricted, clearlydefined
+maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2-jakarta/2.2.15, Apache-2.0, approved, #11477
+maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2-jakarta/2.2.18, Apache-2.0, approved, #11477
 maven/mavencentral/io.swagger.core.v3/swagger-models-jakarta/2.1.13, Apache-2.0, approved, clearlydefined
 maven/mavencentral/io.swagger.core.v3/swagger-models-jakarta/2.2.15, Apache-2.0, approved, #5919
 maven/mavencentral/io.swagger.core.v3/swagger-models-jakarta/2.2.18, Apache-2.0, approved, #5919
@@ -409,21 +410,15 @@ maven/mavencentral/org.junit-pioneer/junit-pioneer/2.1.0, EPL-2.0, approved, #10
 maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.0, EPL-2.0, approved, #9714
 maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.1, EPL-2.0, approved, #9714
 maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.9.2, EPL-2.0, approved, #3133
-maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.10.0, EPL-2.0, approved, #9711
 maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.10.1, EPL-2.0, approved, #9711
 maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.9.2, EPL-2.0, approved, #3125
-maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.0, EPL-2.0, approved, #9708
 maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.1, EPL-2.0, approved, #9708
 maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.9.2, EPL-2.0, approved, #3134
-maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.0, EPL-2.0, approved, #9715
 maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.1, EPL-2.0, approved, #9715
 maven/mavencentral/org.junit.platform/junit-platform-commons/1.9.2, EPL-2.0, approved, #3130
-maven/mavencentral/org.junit.platform/junit-platform-engine/1.10.0, EPL-2.0, approved, #9709
 maven/mavencentral/org.junit.platform/junit-platform-engine/1.10.1, EPL-2.0, approved, #9709
 maven/mavencentral/org.junit.platform/junit-platform-engine/1.9.2, EPL-2.0, approved, #3128
-maven/mavencentral/org.junit.platform/junit-platform-launcher/1.10.0, EPL-2.0, approved, #9704
 maven/mavencentral/org.junit.platform/junit-platform-launcher/1.10.1, EPL-2.0, approved, #9704
-maven/mavencentral/org.junit/junit-bom/5.10.0, EPL-2.0, approved, #9844
 maven/mavencentral/org.junit/junit-bom/5.10.1, EPL-2.0, approved, #9844
 maven/mavencentral/org.junit/junit-bom/5.9.2, EPL-2.0, approved, #4711
 maven/mavencentral/org.jvnet.mimepull/mimepull/1.9.15, CDDL-1.1 OR GPL-2.0-only WITH Classpath-exception-2.0, approved, CQ21484

extensions/common/azure/azure-blob-core/src/main/java/org/eclipse/edc/azure/blob/AzureBlobStoreSchema.java

@@ -27,6 +27,7 @@ private AzureBlobStoreSchema() {
     public static final String CONTAINER_NAME = "container";
     public static final String ACCOUNT_NAME = "account";
     public static final String BLOB_NAME = "blobName";
+    public static final String BLOB_PREFIX = "blobPrefix";
     public static final String FOLDER_NAME = "folderName";
     public static final String CORRELATION_ID = "correlationId";
 }

extensions/common/azure/azure-blob-core/src/main/java/org/eclipse/edc/azure/blob/api/BlobStoreApi.java

@@ -35,6 +35,16 @@ public interface BlobStoreApi {
 
     List<BlobItem> listContainer(String accountName, String containerName);
 
+    /**
+     * List all blobs from given folder in the container on a storage account.
+     *
+     * @param accountName The name of the storage account
+     * @param containerName The name of the container within the storage account
+     * @param directory The name of the folder within the container of the storage account
+     * @return Lazy loaded list of blobs from folder specified by the input parameters
+     */
+    List<BlobItem> listContainerFolder(String accountName, String containerName, String directory);
+
     void putBlob(String accountName, String containerName, String blobName, byte[] data);
 
     String createAccountSas(String accountName, String containerName, String racwxdl, OffsetDateTime expiry);

extensions/common/azure/azure-blob-core/src/main/java/org/eclipse/edc/azure/blob/api/BlobStoreApiImpl.java

@@ -20,6 +20,7 @@
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
 import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.ListBlobsOptions;
 import com.azure.storage.blob.sas.BlobContainerSasPermission;
 import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
 import com.azure.storage.common.StorageSharedKeyCredential;
@@ -75,6 +76,12 @@ public List<BlobItem> listContainer(String accountName, String containerName) {
         return getBlobServiceClient(accountName).getBlobContainerClient(containerName).listBlobs().stream().toList();
     }
 
+    @Override
+    public List<BlobItem> listContainerFolder(String accountName, String containerName, String directory) {
+        var options = new ListBlobsOptions().setPrefix(directory);
+        return getBlobServiceClient(accountName).getBlobContainerClient(containerName).listBlobs(options, null).stream().toList();
+    }
+
     @Override
     public void putBlob(String accountName, String containerName, String blobName, byte[] data) {
         var blobServiceClient = getBlobServiceClient(accountName);
@@ -105,16 +112,15 @@ private BlobServiceClient getBlobServiceClient(String accountName) {
         }
 
         var accountKey = vault.resolveSecret(accountName + "-key1");
-
-        if (accountKey == null) {
-            throw new IllegalArgumentException("No Object Storage credential found in vault!");
-        }
-
         var endpoint = createEndpoint(accountName);
-        var blobServiceClient = new BlobServiceClientBuilder()
-                .credential(createCredential(accountKey, accountName))
-                .endpoint(endpoint)
-                .buildClient();
+
+        var blobServiceClient = accountKey == null ?
+                new BlobServiceClientBuilder().credential(new DefaultAzureCredentialBuilder().build())
+                        .endpoint(endpoint)
+                        .buildClient() :
+                new BlobServiceClientBuilder().credential(createCredential(accountKey, accountName))
+                        .endpoint(endpoint)
+                        .buildClient();
 
         cache.put(accountName, blobServiceClient);
         return blobServiceClient;

extensions/common/azure/azure-blob-core/src/main/java/org/eclipse/edc/azure/blob/validator/AzureStorageValidator.java

@@ -35,6 +35,7 @@ public class AzureStorageValidator {
     private static final int BLOB_MAX_LENGTH = 1024;
     private static final int METADATA_MIN_LENGTH = 1;
     private static final int METADATA_MAX_LENGTH = 4096;
+    private static final int PATH_SEGMENTS_MAX = 254;
     private static final Pattern ACCOUNT_REGEX = Pattern.compile("^[a-z0-9]+$");
     private static final Pattern CONTAINER_REGEX = Pattern.compile("^[a-z0-9]+(-[a-z0-9]+)*$");
     private static final Pattern METADATA_REGEX = Pattern.compile("^[ -~]*$"); // US-ASCII
@@ -44,10 +45,13 @@ public class AzureStorageValidator {
     private static final String CONTAINER = "container";
     private static final String KEY_NAME = "keyName";
     private static final String METADATA = "metadata";
+    private static final String PREFIX = "prefix";
+
+    private static final String INVALID_PREFIX = "Invalid %s prefix, prefix must end with a '/' character";
     private static final String INVALID_RESOURCE_NAME = "Invalid %s name";
     private static final String INVALID_RESOURCE_NAME_LENGTH = "Invalid %s name length, the name must be between %s and %s characters long";
     private static final String RESOURCE_NAME_EMPTY = "Invalid %s name, the name may not be null, empty or blank";
-    private static final String TOO_MANY_PATH_SEGMENTS = "The number of URL path segments (strings between '/' characters) as part of the blob name cannot exceed 254.";
+    private static final String TOO_MANY_PATH_SEGMENTS = "The number of URL path segments (strings between '/' characters) as part of the blob name cannot exceed %s.";
 
     /**
      * Checks if an account name is valid.
@@ -87,12 +91,25 @@ public static void validateContainerName(String containerName) {
      */
     public static void validateBlobName(String blobName) {
         checkLength(blobName, BLOB, BLOB_MIN_LENGTH, BLOB_MAX_LENGTH);
+        checkSegments(blobName);
+    }
 
-        var slashCount = blobName.chars().filter(ch -> ch == '/').count();
+    /**
+     * Checks if a blob prefix is valid.
+     * The restriction is based on Azure Blob Storage folder 'virtualization' which is base on the forward slash (/)
+     * used in the blob path as delimiter. Prefix has to ends with '/'.
+     *
+     * @param blobPrefix A String representing the blob prefix to validate.
+     * @throws IllegalArgumentException if the string does not represent a valid prefix name.
+     */
+    public static void validateBlobPrefix(String blobPrefix) {
+        checkLength(blobPrefix, PREFIX, BLOB_MIN_LENGTH, BLOB_MAX_LENGTH);
+        checkSegments(blobPrefix);
 
-        if (slashCount >= 254) {
-            throw new IllegalArgumentException(TOO_MANY_PATH_SEGMENTS);
+        if (!blobPrefix.endsWith("/")) {
+            throw new IllegalArgumentException(String.format(INVALID_PREFIX, blobPrefix));
         }
+
     }
 
     /**
@@ -133,4 +150,12 @@ private static void checkLength(String name, String resourceType, int minLength,
             throw new IllegalArgumentException(String.format(INVALID_RESOURCE_NAME_LENGTH, resourceType, minLength, maxLength));
         }
     }
+
+    private static void checkSegments(String name) {
+        var slashCount = name.chars().filter(ch -> ch == '/').count();
+
+        if (slashCount >= PATH_SEGMENTS_MAX) {
+            throw new IllegalArgumentException(String.format(TOO_MANY_PATH_SEGMENTS, PATH_SEGMENTS_MAX));
+        }
+    }
 }

extensions/common/azure/azure-blob-core/src/test/java/org/eclipse/edc/azure/blob/validator/AzureStorageValidatorTest.java

@@ -90,6 +90,20 @@ void validateBlobName_fail(String input) {
                 .isThrownBy(() -> AzureStorageValidator.validateBlobName(input));
     }
 
+    @ParameterizedTest
+    @ArgumentsSource(ValidBlobPrefixProvider.class)
+    void validateBlobPrefix_success(String input) {
+        AzureStorageValidator.validateBlobPrefix(input);
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(InvalidBlobPrefixProvider.class)
+    @NullSource
+    void validateBlobPrefix_fail(String input) {
+        assertThatExceptionOfType(IllegalArgumentException.class)
+                .isThrownBy(() -> AzureStorageValidator.validateBlobPrefix(input));
+    }
+
     @ParameterizedTest
     @ValueSource(strings = {  "abcdefghijklmnop", "-", "a/%!_- $K1~"})
     void validateMetadata_success(String input) {
@@ -129,6 +143,33 @@ public Stream<? extends Arguments> provideArguments(ExtensionContext context) th
         }
     }
 
+    private static class InvalidBlobPrefixProvider implements ArgumentsProvider {
+        @Override
+        public Stream<? extends Arguments> provideArguments(ExtensionContext context) throws Exception {
+            return Stream.of(
+                    Arguments.of(""),
+                    Arguments.of("dfhjdhfjhsd"),
+                    Arguments.of("abcdefghijklmnop".repeat(64) + "/"),
+                    Arguments.of("a/b".repeat(253) + "/")
+                    );
+        }
+    }
+
+    private static class ValidBlobPrefixProvider implements ArgumentsProvider {
+        @Override
+        public Stream<? extends Arguments> provideArguments(ExtensionContext context) throws Exception {
+            return Stream.of(
+                    Arguments.of("geq/"),
+                    Arguments.of("Qja143/"),
+                    Arguments.of("ABE/"),
+                    Arguments.of("a name/"),
+                    Arguments.of("end space /"),
+                    Arguments.of("je`~3j4k%$':\\/"),
+                    Arguments.of("abcdefghijklmnop".repeat(63) + "/"),
+                    Arguments.of("a/b".repeat(252) + "/"));
+        }
+    }
+
     private static class InvalidMetadataProvider implements ArgumentsProvider {
         @Override
         public Stream<? extends Arguments> provideArguments(ExtensionContext context) throws Exception {

extensions/common/azure/azure-blob-core/src/testFixtures/java/org/eclipse/edc/azure/blob/testfixtures/AzureStorageTestFixtures.java

@@ -50,6 +50,10 @@ public static String createBlobName() {
         return "blob-" + UUID.randomUUID();
     }
 
+    public static String createBlobPrefix() {
+        return "blobFolder-" + UUID.randomUUID() + "/";
+    }
+
     public static String createSharedKey() {
         return "SK-" + UUID.randomUUID();
     }

extensions/data-plane/data-plane-azure-storage/README.md

@@ -4,42 +4,85 @@
 
 This module contains a Data Plane extension to copy data to and from Azure Blob storage.
 
-When used as a source, it currently only supports copying a single blob.
+When used as a source, it supports copying a single or multiple blobs.
 
 The source `keyName` should reference a vault entry containing a storage [Shared Key](https://docs.microsoft.com/rest/api/storageservices/authorize-with-shared-key).
 
 The destination `keyName` should reference a vault entry containing a JSON-serialized `AzureSasToken` object wrapping a [storage access signature](https://docs.microsoft.com/azure/storage/common/storage-sas-overview).
 
+### AzureStorage DataAddress Configuration
+
+The behavior of blobs transfers can be customized using DataAddress properties.
+
+- When blobPrefix is present, transfer all blobs with names that start with the specified prefix.
+- When blobPrefix is not present, transfer only the blob with a name matching the blobName property.
+- Precedence: blobPrefix takes precedence over blobName when determining which objects to transfer. It allows for both multiple blobs transfers and fetching a single blob when necessary.
+
+>Note: Using blobPrefix introduces an additional step to list all blobs whose name match the specified prefix.
+
+
+An example source address:
+
+- Single blob:
+```json
+{
+  "dataAddress": {
+    "properties": {
+      "type": "AzureStorage",
+      "container": "containerName",
+      "account": "accountName",
+      "blobName": "test/blob.bin",
+      "keyName": "(see above)"
+    }
+  }
+}
+```
+- Multiple blobs:
+```json
+{
+  "dataAddress": {
+    "properties": {
+      "type": "AzureStorage",
+      "container": "containerName",
+      "account": "accountName",
+      "blobPrefix": "test/",
+      "keyName": "(see above)"
+    }
+  }
+}
+```
 An example destination address:
+
+- Single blob:
 ```json
 {
     "dataDestination": {
         "properties": {
             "type": "AzureStorage",
             "container": "containerName",
             "account": "accountName",
-            "folderName": "test/",
+            "folderName": "destinationFolder/",
             "blobName": "new-name",
             "keyName": "(see above)"
         }
     }
 }
 ```
 
-An example source address:
+- Multiple blobs:
 ```json
 {
-  "dataAddress": {
-    "properties": {
-      "type": "AzureStorage",
-      "container": "containerName",
-      "account": "accountName",
-      "blobName": "test/blob.bin",
-      "keyName": "(see above)"
+    "dataDestination": {
+        "properties": {
+            "type": "AzureStorage",
+            "container": "containerName",
+            "account": "accountName",
+            "folderName": "destinationFolder/",
+            "keyName": "(see above)"
+        }
     }
-  }
 }
 ```
-The `folderName` and the `blobName` are optional properties.
+The `folderName` and the `blobName` are optional properties in destination address.
 
 

extensions/data-plane/data-plane-azure-storage/src/main/java/org/eclipse/edc/connector/dataplane/azure/storage/pipeline/AzureStorageDataSink.java

@@ -42,6 +42,7 @@ public class AzureStorageDataSink extends ParallelSink {
     private String containerName;
     private String folderName;
     private String blobName;
+    private String blobPrefix;
     private String sharedAccessSignature;
     private BlobStoreApi blobStoreApi;
     private DataFlowRequest request;
@@ -111,7 +112,7 @@ private StreamResult<Object> getTransferResult(Exception e, String logMessage, O
     }
 
     String getDestinationBlobName(String partName) {
-        var name = !StringUtils.isNullOrEmpty(blobName) ? blobName : partName;
+        var name = !StringUtils.isNullOrEmpty(blobName) && StringUtils.isNullOrBlank(blobPrefix) ? blobName : partName;
         if (!StringUtils.isNullOrEmpty(folderName)) {
             return folderName.endsWith("/") ? folderName + name : folderName + "/" + name;
         } else {
@@ -149,6 +150,11 @@ public Builder blobName(String blobName) {
             return this;
         }
 
+        public Builder blobPrefix(String blobPrefix) {
+            sink.blobPrefix = blobPrefix;
+            return this;
+        }
+
         public Builder sharedAccessSignature(String sharedAccessSignature) {
             sink.sharedAccessSignature = sharedAccessSignature;
             return this;