[Storage][DataMovement] Fix large file transfers for certain destination types (Azure#48321)

Author: jalauzon-msft

sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs

@@ -48,7 +48,7 @@ public static ArgumentException InvalidService(char s)
             => new ArgumentException($"Invalid service: '{s}'");
 
         public static ArgumentException InsufficientStorageTransferOptions(long streamLength, long statedMaxBlockSize, long necessaryMinBlockSize)
-            => new ArgumentException($"Cannot upload {streamLength} bytes with a maximum transfer size of {statedMaxBlockSize} bytes per block. Please increase the StorageTransferOptions.MaximumTransferSize to at least {necessaryMinBlockSize}.");
+            => new ArgumentException($"Cannot transfer {streamLength} bytes with a maximum transfer size of {statedMaxBlockSize} bytes per block. Please increase the TransferOptions.MaximumTransferChunkSize to at least {necessaryMinBlockSize}.");
 
         public static InvalidDataException HashMismatch(string hashHeaderName)
             => new InvalidDataException($"{hashHeaderName} did not match hash of recieved data.");

sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md

@@ -1,14 +1,9 @@
 # Release History
 
-## 12.1.0-beta.1 (Unreleased)
-
-### Features Added
-
-### Breaking Changes
+## 12.0.1 (Unreleased)
 
 ### Bugs Fixed
-
-### Other Changes
+- Fixed an issue that would prevent transfers of large files (>200 GiB) for certain destination resource types.
 
 ## 12.0.0 (2025-02-11)
 

sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs

@@ -31,6 +31,8 @@ internal class AppendBlobStorageResource : StorageResourceItemInternal
 
         protected override long MaxSupportedChunkSize => Constants.Blob.Append.MaxAppendBlockBytes;
 
+        protected override int MaxSupportedChunkCount => Constants.Blob.Append.MaxBlocks;
+
         protected override long? Length => ResourceProperties?.ResourceLength;
 
         internal AppendBlobStorageResource()

sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <AssemblyTitle>Microsoft Azure.Storage.DataMovement.Blobs client library</AssemblyTitle>
-    <Version>12.1.0-beta.1</Version>
+    <Version>12.0.1</Version>
     <!--The ApiCompatVersion is managed automatically and should not generally be modified manually.-->
     <ApiCompatVersion>12.0.0</ApiCompatVersion>
     <DefineConstants>BlobDataMovementSDK;$(DefineConstants)</DefineConstants>

sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs

@@ -40,6 +40,8 @@ internal class BlockBlobStorageResource : StorageResourceItemInternal
 
         protected override long MaxSupportedChunkSize => Constants.Blob.Block.MaxStageBytes;
 
+        protected override int MaxSupportedChunkCount => Constants.Blob.Block.MaxBlocks;
+
         protected override long? Length => ResourceProperties?.ResourceLength;
 
         /// <summary>

sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs

@@ -32,6 +32,8 @@ internal class PageBlobStorageResource : StorageResourceItemInternal
 
         protected override long MaxSupportedChunkSize => Constants.Blob.Page.MaxPageBlockBytes;
 
+        protected override int MaxSupportedChunkCount => int.MaxValue;
+
         protected override long? Length => ResourceProperties?.ResourceLength;
 
         public PageBlobStorageResource()

sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/AppendBlobDirectoryToShareDirectoryTests.cs

@@ -45,7 +45,7 @@ protected override async Task CreateObjectInSourceAsync(
             }
             else
             {
-                var data = new byte[0];
+                byte[] data = GetRandomBuffer(objectLength ?? 0);
                 using (var stream = new MemoryStream(data))
                 {
                     await UploadAppendBlocksAsync(

sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/AppendBlobToShareFileTests.cs

@@ -42,7 +42,7 @@ public class AppendBlobToShareFileTests : StartTransferCopyTestBase
         private const string _fileResourcePrefix = "test-file-";
         private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file.";
         protected readonly object _serviceVersion;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private readonly string[] _defaultContentLanguageFile = { "en-US" };
         private const string _defaultContentLanguageBlob = "en-US";
         private const string _defaultContentDisposition = "inline";
@@ -122,9 +122,11 @@ protected override async Task<AppendBlobClient> GetSourceObjectClientAsync(
                 }
                 else
                 {
-                    var data = GetRandomBuffer(objectLength.Value);
-                    using Stream originalStream = await CreateLimitedMemoryStream(objectLength.Value);
-                    await UploadAppendBlocksAsync(blobClient, originalStream, cancellationToken);
+                    byte[] data = GetRandomBuffer(objectLength.Value);
+                    using (var stream = new MemoryStream(data))
+                    {
+                        await UploadAppendBlocksAsync(blobClient, stream, cancellationToken);
+                    }
                 }
             }
             Uri sourceUri = blobClient.GenerateSasUri(Sas.BlobSasPermissions.All, Recording.UtcNow.AddDays(1));

sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/BlockBlobDirectoryToShareDirectoryTests.cs

@@ -58,7 +58,7 @@ await blobClient.UploadAsync(
             }
             else
             {
-                var data = new byte[0];
+                byte[] data = GetRandomBuffer(objectLength ?? 0);
                 using (var stream = new MemoryStream(data))
                 {
                     await blobClient.UploadAsync(

sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/BlockBlobToShareFileTests.cs

@@ -38,7 +38,7 @@ public class BlockBlobToShareFileTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private readonly string[] _defaultContentLanguageFile = { "en-US" };
         private const string _defaultContentLanguageBlob = "en-US";
         private const string _defaultContentDisposition = "inline";
@@ -136,10 +136,11 @@ await blobClient.UploadAsync(
                 }
                 else
                 {
-                    var data = GetRandomBuffer(objectLength.Value);
-                    using Stream originalStream = await CreateLimitedMemoryStream(objectLength.Value);
-                    await blobClient.UploadAsync(
-                        content: originalStream,
+                    byte[] data = GetRandomBuffer(objectLength.Value);
+                    using (var stream = new MemoryStream(data))
+                    {
+                        await blobClient.UploadAsync(
+                        content: stream,
                         new BlobUploadOptions()
                         {
                             AccessTier = _defaultAccessTier,
@@ -153,6 +154,7 @@ await blobClient.UploadAsync(
                             }
                         },
                         cancellationToken: cancellationToken);
+                    }
                 }
             }
             Uri sourceUri = blobClient.GenerateSasUri(Sas.BlobSasPermissions.All, Recording.UtcNow.AddDays(1));