[Storage][Datamovement] Fix copying of empty directories under prefixes (Azure#51233)

Author: jalauzon-msft

sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_1ebb672122"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_f1a4120258"
 }

sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs

@@ -156,21 +156,21 @@ protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAs
         {
             // Suffix the slash when searching if there's a prefix specified,
             // to only list blobs in the specified virtual directory.
-            string fullPrefix = string.IsNullOrEmpty(DirectoryPrefix) ?
+            string sourcePrefix = string.IsNullOrEmpty(DirectoryPrefix) ?
                 "" :
                 string.Concat(DirectoryPrefix, Constants.PathBackSlashDelimiter);
 
-            Queue<string> prefixes = new();
-            prefixes.Enqueue(fullPrefix); // Start with the initial prefix
+            Queue<string> paths = new();
+            paths.Enqueue(sourcePrefix); // Start with the initial prefix
 
-            while (prefixes.Count > 0)
+            while (paths.Count > 0)
             {
-                string currentPrefix = prefixes.Dequeue();
+                string currentPath = paths.Dequeue();
 
                 int childCount = 0;
                 await foreach (BlobHierarchyItem blobHierarchyItem in BlobContainerClient.GetBlobsByHierarchyAsync(
                     traits: BlobTraits.Metadata,
-                    prefix: currentPrefix,
+                    prefix: currentPath,
                     delimiter: Constants.PathBackSlashDelimiter,
                     cancellationToken: cancellationToken).ConfigureAwait(false))
                 {
@@ -186,9 +186,9 @@ protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAs
                     else if (blobHierarchyItem.IsPrefix)
                     {
                         // Return the blob virtual directory as a StorageResourceContainer
-                        yield return GetChildStorageResourceContainer(blobHierarchyItem.Prefix.Substring(fullPrefix.Length));
+                        yield return GetChildStorageResourceContainer(blobHierarchyItem.Prefix.Substring(sourcePrefix.Length));
                         // Enqueue the prefix for further traversal
-                        prefixes.Enqueue(blobHierarchyItem.Prefix);
+                        paths.Enqueue(blobHierarchyItem.Prefix);
                     }
                 }
 
@@ -200,10 +200,13 @@ protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAs
                 // with the folder metadata set which represents a directory stub on HNS accounts. No other
                 // properties will be copied from the source. We only do this for empty directories because non-empty
                 // directories are created automatically.
-                if (childCount == 0 && destinationContainer is BlobStorageResourceContainer destBlobContainer)
+                if (childCount == 0 &&
+                    currentPath != sourcePrefix && // If doing an empty copy
+                    destinationContainer is BlobStorageResourceContainer destBlobContainer)
                 {
+                    // Remove source prefix and add destination prefix
                     BlockBlobStorageResource destinationDirectoryResource = destBlobContainer.GetBlobAsStorageResource(
-                        currentPrefix,
+                        destBlobContainer.ApplyOptionalPrefix(currentPath.Substring(sourcePrefix.Length)),
                         BlobType.Block) as BlockBlobStorageResource;
                     await destinationDirectoryResource.CreateEmptyDirectoryStubAsync(cancellationToken).ConfigureAwait(false);
                 }
@@ -219,7 +222,7 @@ protected override StorageResourceCheckpointDetails GetSourceCheckpointDetails()
         protected override StorageResourceCheckpointDetails GetDestinationCheckpointDetails()
             => new BlobDestinationCheckpointDetails(_options);
 
-        private string ApplyOptionalPrefix(string path)
+        internal string ApplyOptionalPrefix(string path)
             => IsDirectory
                 ? string.Join("/", DirectoryPrefix, path)
                 : path;

sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobDirectoryToDirectoryTests.cs

@@ -87,7 +87,8 @@ protected internal override BlockBlobClient GetSourceBlob(BlobContainerClient co
 
         private async Task PopulateVirtualDirectoryContainer(
             BlobContainerClient containerClient,
-            long objectLength)
+            long objectLength,
+            string prefix = default)
         {
             byte[] data = GetRandomBuffer(objectLength);
             Metadata folderMetadata = new Dictionary<string, string>()
@@ -110,16 +111,18 @@ private async Task PopulateVirtualDirectoryContainer(
             ];
             foreach (string file in files)
             {
+                string fileName = prefix != default ? $"{prefix}/{file}" : file;
                 using MemoryStream stream = new(data);
-                BlobClient blobClient = containerClient.GetBlobClient(file);
+                BlobClient blobClient = containerClient.GetBlobClient(fileName);
                 await blobClient.UploadAsync(stream);
             }
 
             // List of empty directories to be created
             string[] emptyDirs = ["emptyDir", "recursiveDir/emptySubDir"];
             foreach (string dir in emptyDirs)
             {
-                BlobClient blobClient = containerClient.GetBlobClient(dir);
+                string dirName = prefix != default ? $"{prefix}/{dir}" : dir;
+                BlobClient blobClient = containerClient.GetBlobClient(dirName);
                 await blobClient.UploadAsync(Stream.Null, new BlobUploadOptions()
                 {
                     Metadata = folderMetadata
@@ -128,7 +131,9 @@ private async Task PopulateVirtualDirectoryContainer(
         }
 
         [RecordedTest]
-        public async Task DirectoryCopyWithVirtualDirectories([Values(true, false)] bool hns)
+        public async Task DirectoryCopyWithVirtualDirectories(
+            [Values(true, false)] bool hns,
+            [Values(true, false)] bool usePrefix)
         {
             BlobServiceClient serviceClient = SourceClientBuilder.GetServiceClientFromOauthConfig(
                 hns ? Tenants.TestConfigHierarchicalNamespace : Tenants.TestConfigDefault,
@@ -137,7 +142,9 @@ public async Task DirectoryCopyWithVirtualDirectories([Values(true, false)] bool
             await using DisposingBlobContainer sourceContainer = await SourceClientBuilder.GetTestContainerAsync(serviceClient);
             await using DisposingBlobContainer destinationContainer = await DestinationClientBuilder.GetTestContainerAsync(serviceClient);
 
-            await PopulateVirtualDirectoryContainer(sourceContainer.Container, 1024);
+            string sourcePrefix = usePrefix ? "source" : default;
+            string destinationPrefix = usePrefix ? "destination" : default;
+            await PopulateVirtualDirectoryContainer(sourceContainer.Container, 1024, prefix: sourcePrefix);
 
             TransferManager transferManager = new();
             BlobsStorageResourceProvider blobProvider = new(TestEnvironment.Credential);
@@ -146,8 +153,8 @@ public async Task DirectoryCopyWithVirtualDirectories([Values(true, false)] bool
             TestEventsRaised testEventsRaised = new(transferOptions);
 
             TransferOperation transfer = await transferManager.StartTransferAsync(
-                GetSourceStorageResourceContainer(sourceContainer.Container, default),
-                GetSourceStorageResourceContainer(destinationContainer.Container, default),
+                GetSourceStorageResourceContainer(sourceContainer.Container, sourcePrefix),
+                GetSourceStorageResourceContainer(destinationContainer.Container, destinationPrefix),
                 transferOptions);
 
             CancellationTokenSource tokenSource = new(TimeSpan.FromSeconds(30));
@@ -158,8 +165,8 @@ await TestTransferWithTimeout.WaitForCompletionAsync(
 
             testEventsRaised.AssertUnexpectedFailureCheck();
             await VerifyResultsAsync(
-                sourceContainer.Container, string.Empty,
-                destinationContainer.Container, string.Empty);
+                sourceContainer.Container, sourcePrefix,
+                destinationContainer.Container, destinationPrefix);
         }
     }
 }

sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferBlobDirectoryCopyTestBase.cs

@@ -285,17 +285,19 @@ protected override async Task VerifyResultsAsync(
 
             // List all files in source blob folder path
             List<string> sourceFileNames = new List<string>();
-
-            // Get source directory client and list the paths
-            await foreach (Page<BlobItem> page in sourceContainer.GetBlobsAsync(prefix: sourcePrefix, cancellationToken: cancellationToken).AsPages())
+            await foreach (Page<BlobItem> page in sourceContainer.GetBlobsAsync(
+                prefix: !string.IsNullOrEmpty(sourcePrefix) ? sourcePrefix + '/' : sourcePrefix,
+                cancellationToken: cancellationToken).AsPages())
             {
                 sourceFileNames.AddRange(page.Values.Select(
                     (BlobItem item) => !string.IsNullOrEmpty(sourcePrefix) ? item.Name.Substring(sourcePrefix.Length + 1) : item.Name));
             }
 
             // List all files in the destination blob folder path
             List<string> destinationFileNames = new List<string>();
-            await foreach (Page<BlobItem> page in destinationContainer.GetBlobsAsync(prefix: destinationPrefix, cancellationToken: cancellationToken).AsPages())
+            await foreach (Page<BlobItem> page in destinationContainer.GetBlobsAsync(
+                prefix: !string.IsNullOrEmpty(destinationPrefix) ? destinationPrefix + '/' : destinationPrefix,
+                cancellationToken: cancellationToken).AsPages())
             {
                 destinationFileNames.AddRange(page.Values.Select(
                     (BlobItem item) => !string.IsNullOrEmpty(destinationPrefix) ? item.Name.Substring(destinationPrefix.Length + 1) : item.Name));