[Storage][DataMovement] Fix transfers using root BlobStorageResourceContainer (Azure#36023)

jalauzon-msft · web-flow · commit ed179888f12c · 2023-05-05T13:21:45.000-07:00
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs
@@ -43,8 +43,9 @@ public BlobStorageResourceContainer(BlobContainerClient blobContainerClient, Blo
 
         /// <summary>
         /// Gets the path of the storage resource.
+        /// Return empty string since we are using the root of the container.
         /// </summary>
-        public override string Path => _blobContainerClient.Name;
+        public override string Path => string.Empty;
 
         /// <summary>
         /// Gets the URL of the storage resource.
diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs
@@ -173,7 +173,10 @@ private async IAsyncEnumerable<JobPartInternal> GetStorageResourcesAsync()
                 StorageResourceBase current = enumerator.Current;
                 if (lastResource != default)
                 {
-                    string sourceName = lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+                    string sourceName = string.IsNullOrEmpty(_sourceResourceContainer.Path)
+                        ? lastResource.Path
+                        : lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+
                     if (!existingSources.Contains(sourceName))
                     {
                         // Because AsyncEnumerable doesn't let us know which storage resource is the last resource
@@ -210,7 +213,10 @@ private async IAsyncEnumerable<JobPartInternal> GetStorageResourcesAsync()
                 {
                     // Return last part but enable the part to be the last job part of the entire job
                     // so we know that we've finished listing in the container
-                    string lastSourceName = lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+                    string lastSourceName = string.IsNullOrEmpty(_sourceResourceContainer.Path)
+                        ? lastResource.Path
+                        : lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+
                     lastPart = await ServiceToServiceJobPart.CreateJobPartAsync(
                             job: this,
                             partNumber: partNumber,
diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs
@@ -171,7 +171,10 @@ private async IAsyncEnumerable<JobPartInternal> GetStorageResourcesAsync()
                 StorageResourceBase current = enumerator.Current;
                 if (lastResource != default)
                 {
-                    string sourceName = lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+                    string sourceName = string.IsNullOrEmpty(_sourceResourceContainer.Path)
+                        ? lastResource.Path
+                        : lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+
                     if (!existingSources.Contains(sourceName))
                     {
                         // Because AsyncEnumerable doesn't let us know which storage resource is the last resource
@@ -208,7 +211,10 @@ private async IAsyncEnumerable<JobPartInternal> GetStorageResourcesAsync()
                 {
                     // Return last part but enable the part to be the last job part of the entire job
                     // so we know that we've finished listing in the container
-                    string lastSourceName = lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+                    string lastSourceName = string.IsNullOrEmpty(_sourceResourceContainer.Path)
+                        ? lastResource.Path
+                        : lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+
                     lastPart = await StreamToUriJobPart.CreateJobPartAsync(
                             job: this,
                             partNumber: partNumber,
diff --git a/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamTransferJob.cs
@@ -171,7 +171,10 @@ private async IAsyncEnumerable<JobPartInternal> GetStorageResourcesAsync()
                 StorageResourceBase current = enumerator.Current;
                 if (lastResource != default)
                 {
-                    string sourceName = lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+                    string sourceName = string.IsNullOrEmpty(_sourceResourceContainer.Path)
+                        ? lastResource.Path
+                        : lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+
                     if (!existingSources.Contains(sourceName))
                     {
                         // Because AsyncEnumerable doesn't let us know which storage resource is the last resource
@@ -208,7 +211,10 @@ private async IAsyncEnumerable<JobPartInternal> GetStorageResourcesAsync()
                 {
                     // Return last part but enable the part to be the last job part of the entire job
                     // so we know that we've finished listing in the container
-                    string lastSourceName = lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+                    string lastSourceName = string.IsNullOrEmpty(_sourceResourceContainer.Path)
+                        ? lastResource.Path
+                        : lastResource.Path.Substring(_sourceResourceContainer.Path.Length + 1);
+
                     lastPart = await UriToStreamJobPart.CreateJobPartAsync(
                             job: this,
                             partNumber: partNumber,
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferAsyncCopyDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferAsyncCopyDirectoryTests.cs
@@ -26,7 +26,7 @@ public StartTransferAsyncCopyDirectoryTests(bool async, BlobClientOptions.Servic
         /// <summary>
         /// Upload and verify the contents of the blob
         ///
-        /// By default in this function an event arguement will be added to the options event handler
+        /// By default in this function an event argument will be added to the options event handler
         /// to detect when the upload has finished.
         /// </summary>
         /// <param name="container">The source container which will contains the source blobs</param>
@@ -441,6 +441,48 @@ await CopyBlobDirectoryAndVerify(
                 options: options).ConfigureAwait(false);
         }
 
+        [Test]
+        [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082
+        public async Task BlockBlobDirectoryToDirectory_Root()
+        {
+            // Arrange
+            string[] files = { "file1", "dir1/file1", "dir1/file2", "dir1/file3", "dir2/file1" };
+            BinaryData data = BinaryData.FromString("Hello World");
+
+            await using DisposingBlobContainer source = await GetTestContainerAsync();
+            await using DisposingBlobContainer destination = await GetTestContainerAsync();
+
+            foreach (string file in files)
+            {
+                await source.Container.UploadBlobAsync(file, data);
+            }
+
+            TransferManager transferManager = new TransferManager();
+
+            StorageResourceContainer sourceResource =
+                new BlobStorageResourceContainer(source.Container);
+            StorageResourceContainer destinationResource = new BlobStorageResourceContainer(
+                destination.Container,
+                new BlobStorageResourceContainerOptions()
+                {
+                    CopyMethod = TransferCopyMethod.AsyncCopy,
+                });
+
+            // Act
+            DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource);
+
+            CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+            await transfer.AwaitCompletion(tokenSource.Token);
+
+            // Assert
+            Assert.AreEqual(StorageTransferStatus.Completed, transfer.TransferStatus);
+
+            IEnumerable<string> destinationFiles =
+                (await destination.Container.GetBlobsAsync().ToEnumerableAsync()).Select(b => b.Name);
+
+            Assert.IsTrue(destinationFiles.OrderBy(f => f).SequenceEqual(files.OrderBy(f => f)));
+        }
+
         #region Single Concurrency
         private async Task CreateBlobDirectoryTree(
             BlobContainerClient client,
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadDirectoryTests.cs
@@ -3,18 +3,16 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Core.TestFramework;
 using Azure.Storage.Blobs;
-using Azure.Storage.DataMovement.Models;
+using Azure.Storage.Blobs.Models;
 using Azure.Storage.DataMovement.Blobs;
+using Azure.Storage.DataMovement.Models;
 using NUnit.Framework;
-using System.IO;
-using Azure.Storage.Blobs.Models;
-using Microsoft.Extensions.Options;
-using Microsoft.CodeAnalysis;
 
 namespace Azure.Storage.DataMovement.Tests
 {
@@ -28,7 +26,7 @@ public StartTransferDownloadDirectoryTests(bool async, BlobClientOptions.Service
         /// <summary>
         /// Upload and verify the contents of the blob
         ///
-        /// By default in this function an event arguement will be added to the options event handler
+        /// By default in this function an event argument will be added to the options event handler
         /// to detect when the upload has finished.
         /// </summary>
         /// <param name="sourceContainer">The source container which will contains the source blobs</param>
@@ -330,6 +328,40 @@ await DownloadBlobDirectoryAndVerify(
                 blobNames,
                 destinationFolder).ConfigureAwait(false);
         }
+
+        [Test]
+        [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082
+        public async Task DownloadDirectoryAsync_Root()
+        {
+            // Arrange
+            string[] files = { "file1", "dir1/file1", "dir1/file2", "dir1/file3", "dir2/file1" };
+            BinaryData data = BinaryData.FromString("Hello World");
+
+            await using DisposingBlobContainer source = await GetTestContainerAsync();
+            using DisposingLocalDirectory destination = DisposingLocalDirectory.GetTestDirectory();
+
+            foreach (string file in files)
+            {
+                await source.Container.UploadBlobAsync(file, data);
+            }
+
+            TransferManager transferManager = new TransferManager();
+
+            StorageResourceContainer sourceResource =
+                new BlobStorageResourceContainer(source.Container);
+            StorageResourceContainer destinationResource =
+                new LocalDirectoryStorageResourceContainer(destination.DirectoryPath);
+
+            DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource);
+
+            CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+            await transfer.AwaitCompletion(tokenSource.Token);
+
+            IEnumerable<string> destinationFiles = ListFilesInDirectory(destination.DirectoryPath)
+                .Select(f => f.Substring(destination.DirectoryPath.Length + 1).Replace("\\", "/"));
+
+            Assert.IsTrue(destinationFiles.OrderBy(f => f).SequenceEqual(files.OrderBy(f => f)));
+        }
         #endregion DirectoryDownloadTests
 
         #region Single Concurrency
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs
@@ -3,10 +3,8 @@
 
 using System;
 using System.Collections.Generic;
-using System.Drawing;
 using System.IO;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Core.TestFramework;
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs
@@ -1,5 +1,6 @@
 ﻿// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
+
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -13,9 +14,7 @@
 using Azure.Storage.DataMovement.Blobs;
 using Azure.Storage.DataMovement.Models;
 using Microsoft.CodeAnalysis;
-using Microsoft.Extensions.Options;
 using NUnit.Framework;
-using NUnit.Framework.Internal;
 
 namespace Azure.Storage.DataMovement.Tests
 {
@@ -363,6 +362,47 @@ await UploadBlobDirectoryAndVerify(
                 destinationPrefix: dirName,
                 waitTimeInSec: 10);
         }
+
+        [Test]
+        [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082
+        public async Task DirectoryUpload_Root()
+        {
+            // Arrange
+            using DisposingLocalDirectory source = DisposingLocalDirectory.GetTestDirectory();
+            await using DisposingBlobContainer destination = await GetTestContainerAsync();
+
+            string file1 = await CreateRandomFileAsync(source.DirectoryPath, size:10);
+
+            string dir1 = CreateRandomDirectory(source.DirectoryPath);
+            string file2 = await CreateRandomFileAsync(dir1, size: 10);
+            string file3 = await CreateRandomFileAsync(dir1, size: 10);
+            string file4 = await CreateRandomFileAsync(dir1, size: 10);
+
+            string dir2 = CreateRandomDirectory(source.DirectoryPath);
+            string file5 = await CreateRandomFileAsync(dir2, size: 10);
+
+            string[] files = {file1, file2, file3, file4, file5};
+
+            TransferManager transferManager = new TransferManager();
+
+            StorageResourceContainer sourceResource =
+                new LocalDirectoryStorageResourceContainer(source.DirectoryPath);
+            StorageResourceContainer destinationResource =
+                new BlobStorageResourceContainer(destination.Container);
+
+            DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource);
+
+            CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+            await transfer.AwaitCompletion(tokenSource.Token);
+
+            IEnumerable<string> destinationFiles =
+                (await destination.Container.GetBlobsAsync().ToEnumerableAsync()).Select(b => b.Name);
+
+            Assert.IsTrue(files
+                .Select(f => f.Substring(source.DirectoryPath.Length + 1).Replace("\\", "/"))
+                .OrderBy(f => f)
+                .SequenceEqual(destinationFiles.OrderBy(f => f)));
+        }
         #endregion
 
         #region DirectoryUploadTests
