[Storage] Support high throughput append blob (#23484)

yifanz7 · blueww · web-flow · commit 29b7a3b78907 · 2023-11-22T13:48:20.000+08:00
* [Storage] Support append 100MB block

* Update chunk size

* Remove redundant line

* Add in missing changelog items from previous releases

---------

Co-authored-by: Wei Wei (AZURE) &lt;weiwei@microsoft.com&gt;
diff --git a/src/Storage/Storage.Management/ChangeLog.md b/src/Storage/Storage.Management/ChangeLog.md
@@ -18,6 +18,9 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Supported high throughput append blob
+    - `Set-AzStorageBlobContent`
+    - `Copy-AzStorageBlob`    
 
 ## Version 6.0.0
 * Supported customer initiated migration
@@ -68,6 +71,12 @@
     - `New-AzStorageAccount`
 * Updated help file of `New-AzStorageQueueSASToken`
 
+## Version 5.9.1
+* Added support for customer initiated account migration 
+    - `Start-AzStorageAccountMigration`
+    - `Get-AzStorageAccountMigration`
+* Supported Storage account planned failover new properties 
+    - `Get-AzStorageAccount`
 
 ## Version 5.9.0
 * Supported OAuth authentication on File service cmdlets
@@ -145,6 +154,26 @@
 * Support get a file share object without get share properties. For pipeline to file/directory cmdlets with OAuth authentication.
     - `Get-AzStorageShare`
 
+## Version 5.6.2
+* Removed the API version limitation on File cmdlets
+
+## Version 5.6.1
+* Support OAuth authentication on File service cmdlets
+    - `New-AzStorageContext`
+    - `Get-AzStorageFile`
+    - `Get-AzStorageFileContent`
+    - `Get-AzStorageFileCopyState`
+    - `New-AzStorageDirectory`
+    - `Remove-AzStorageDirectory`
+    - `Remove-AzStorageFile`
+    - `Set-AzStorageFileContent`
+    - `Start-AzStorageFileCopy`
+    - `Stop-AzStorageFileCopy`
+    - `Get-AzStorageFileHandle`
+    - `Close-AzStorageFileHandle`
+* Support get a file share object without get share properties. For pipeline to file/directory cmdlets with OAuth authentication.
+    - `Get-AzStorageShare`
+
 ## Version 5.6.0
 * Supported rename file and directory
     - `Rename-AzStorageFile`
@@ -204,6 +233,12 @@
 * Returned AllowedCopyScope in get account result
     - `Get-AzStorageAccount`
 
+## Version 5.2.2
+* Supported Planned and Unplanned types in Storage account failover type 
+    - `Invoke-AzStorageAccountFailover`
+* Supported TierToCold and TierToHot in Storage account management policy 
+    - `Add-AzStorageAccountManagementPolicyAction`
+
 ## Version 5.2.0
 * Supported MaxPageSize, Include, and Filter parameters for listing encryption scopes 
     - `Get-AzStorageEncryptionScope`
diff --git a/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs b/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs
@@ -466,9 +466,10 @@ private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel,
                         long appendCopyOffset = 0;
                         progressHandler.Report(appendCopyOffset);
                         long appendContentLenLeft = srcProperties.ContentLength;
+                        long appendChunkSize = GetAppendBlockLength(srcProperties.ContentLength);
                         while (appendContentLenLeft > 0)
                         {
-                            long appendContentSize = appendContentLenLeft < size4MB ? appendContentLenLeft : size4MB;
+                            long appendContentSize = appendContentLenLeft < appendChunkSize ? appendContentLenLeft : appendChunkSize;
 
                             Track2Models.AppendBlobAppendBlockFromUriOptions appendBlobAppendBlockFromUriOptions = new Track2Models.AppendBlobAppendBlockFromUriOptions
                             {
diff --git a/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs b/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs
@@ -394,7 +394,9 @@ protected override void DoEndProcessing()
                 Tuple<string, StorageBlob.CloudBlob> uploadRequest = UploadRequests.DequeueRequest();
                 IStorageBlobManagement localChannel = Channel;
                 Func<long, Task> taskGenerator;
-                if (!UseTrack2Sdk())
+                long fileSize = new FileInfo(ResolvedFileName).Length;
+
+                if (!UseTrack2Sdk() && (this.BlobType.ToLower() != AppendBlobType.ToLower() || fileSize <= (long)size4MB * maxBlockCount))
                 {
                     //Upload with DMlib
                     taskGenerator = (taskId) => Upload2Blob(taskId, localChannel, uploadRequest.Item1, uploadRequest.Item2);
@@ -542,32 +544,39 @@ internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagemen
                         }
 
                         // Upload blob content
-                        byte[] uploadcache4MB = null;
+                        //For page blob will always use 4MB
+                        long chunksize = size4MB;
+
+                        // for append blob, the chunksize can be at most 100MB, will make it multiple of 8MB.
+                        if (string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase))
+                        {
+                            chunksize = GetAppendBlockLength(fileSize);
+                        }
+                        byte[] uploadcacheLong = null;
                         byte[] uploadcache = null;
                         progressHandler.Report(0);
                         long offset = 0;
                         while (offset < fileSize)
                         {
                             // Get chunk size and prepare cache
-                            int chunksize = size4MB;
                             if (chunksize <= (fileSize - offset)) // Chunk size will be 4MB
                             {
-                                if (uploadcache4MB == null)
+                                if (uploadcacheLong == null)
                                 {
-                                    uploadcache4MB = new byte[size4MB];
+                                    uploadcacheLong = new byte[chunksize];
                                 }
-                                uploadcache = uploadcache4MB;
+                                uploadcache = uploadcacheLong;
                             }
                             else // last chunk can < 4MB
                             {
                                 chunksize = (int)(fileSize - offset);
-                                if (uploadcache4MB == null)
+                                if (uploadcacheLong == null)
                                 {
                                     uploadcache = new byte[chunksize];
                                 }
                                 else
                                 {
-                                    uploadcache = uploadcache4MB;
+                                    uploadcache = uploadcacheLong;
                                 }
                             }
 
diff --git a/src/Storage/Storage/Blob/StorageDataMovementCmdletBase.cs b/src/Storage/Storage/Blob/StorageDataMovementCmdletBase.cs
@@ -32,6 +32,10 @@ public class StorageDataMovementCmdletBase : StorageCloudBlobCmdletBase, IDispos
 
         protected const int size256MB = 256 * 1024 * 1024;
 
+        protected const int maxBlockCount = 50000;
+
+        protected const int maxBlockSize = 100 * 1024 * 1024;
+
         /// <summary>
         /// block blob type
         /// </summary>
@@ -189,14 +193,35 @@ public static long GetBlockLength(long contentLength)
             {
                 return contentLength;
             }
-            long blockLength = contentLength / 50000;
+            long blockLength = contentLength / maxBlockCount;
             if (blockLength % (size8MB) != 0)
             {
                 blockLength = (blockLength / (size8MB) + 1) * (size8MB);
             }
             return blockLength > 0 ? blockLength : contentLength;
         }
 
+        /// <summary>
+        /// Get the block size from append blob length
+        /// </summary>
+        public static long GetAppendBlockLength(long contentLength)
+        {
+            if (contentLength <= size8MB)
+            {
+                return contentLength;
+            }
+            long blockLength = contentLength / maxBlockCount;
+            if (blockLength <= size8MB)
+            {
+                return size8MB;
+            }
+            if (blockLength % (size4MB) != 0)
+            {
+                blockLength = (blockLength / (size4MB) + 1) * (size4MB);
+            }
+            return blockLength;
+        }
+
         /// <summary>
         /// Get the block id arrary from block blob length, block size and blob name
         /// </summary>

Original file line number	Diff line number	Diff line change
`@@ -466,9 +466,10 @@ private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel,`
`466`	`466`	`long appendCopyOffset = 0;`
`467`	`467`	`progressHandler.Report(appendCopyOffset);`
`468`	`468`	`long appendContentLenLeft = srcProperties.ContentLength;`
	`469`	`+ long appendChunkSize = GetAppendBlockLength(srcProperties.ContentLength);`
`469`	`470`	`while (appendContentLenLeft > 0)`
`470`	`471`	`{`
`471`		`- long appendContentSize = appendContentLenLeft < size4MB ? appendContentLenLeft : size4MB;`
	`472`	`+ long appendContentSize = appendContentLenLeft < appendChunkSize ? appendContentLenLeft : appendChunkSize;`
`472`	`473`
`473`	`474`	`Track2Models.AppendBlobAppendBlockFromUriOptions appendBlobAppendBlockFromUriOptions = new Track2Models.AppendBlobAppendBlockFromUriOptions`
`474`	`475`	`{`
Original file line number	Diff line number	Diff line change
`@@ -394,7 +394,9 @@ protected override void DoEndProcessing()`
`394`	`394`	`Tuple<string, StorageBlob.CloudBlob> uploadRequest = UploadRequests.DequeueRequest();`
`395`	`395`	`IStorageBlobManagement localChannel = Channel;`
`396`	`396`	`Func<long, Task> taskGenerator;`
`397`		`- if (!UseTrack2Sdk())`
	`397`	`+ long fileSize = new FileInfo(ResolvedFileName).Length;`
	`398`	`+`
	`399`	`+ if (!UseTrack2Sdk() && (this.BlobType.ToLower() != AppendBlobType.ToLower() \|\| fileSize <= (long)size4MB * maxBlockCount))`
`398`	`400`	`{`
`399`	`401`	`//Upload with DMlib`
`400`	`402`	`taskGenerator = (taskId) => Upload2Blob(taskId, localChannel, uploadRequest.Item1, uploadRequest.Item2);`
`@@ -542,32 +544,39 @@ internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagemen`
`542`	`544`	`}`
`543`	`545`
`544`	`546`	`// Upload blob content`
`545`		`- byte[] uploadcache4MB = null;`
	`547`	`+ //For page blob will always use 4MB`
	`548`	`+ long chunksize = size4MB;`
	`549`	`+`
	`550`	`+ // for append blob, the chunksize can be at most 100MB, will make it multiple of 8MB.`
	`551`	`+ if (string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase))`
	`552`	`+ {`
	`553`	`+ chunksize = GetAppendBlockLength(fileSize);`
	`554`	`+ }`
	`555`	`+ byte[] uploadcacheLong = null;`
`546`	`556`	`byte[] uploadcache = null;`
`547`	`557`	`progressHandler.Report(0);`
`548`	`558`	`long offset = 0;`
`549`	`559`	`while (offset < fileSize)`
`550`	`560`	`{`
`551`	`561`	`// Get chunk size and prepare cache`
`552`		`- int chunksize = size4MB;`
`553`	`562`	`if (chunksize <= (fileSize - offset)) // Chunk size will be 4MB`
`554`	`563`	`{`
`555`		`- if (uploadcache4MB == null)`
	`564`	`+ if (uploadcacheLong == null)`
`556`	`565`	`{`
`557`		`- uploadcache4MB = new byte[size4MB];`
	`566`	`+ uploadcacheLong = new byte[chunksize];`
`558`	`567`	`}`
`559`		`- uploadcache = uploadcache4MB;`
	`568`	`+ uploadcache = uploadcacheLong;`
`560`	`569`	`}`
`561`	`570`	`else // last chunk can < 4MB`
`562`	`571`	`{`
`563`	`572`	`chunksize = (int)(fileSize - offset);`
`564`		`- if (uploadcache4MB == null)`
	`573`	`+ if (uploadcacheLong == null)`
`565`	`574`	`{`
`566`	`575`	`uploadcache = new byte[chunksize];`
`567`	`576`	`}`
`568`	`577`	`else`
`569`	`578`	`{`
`570`		`- uploadcache = uploadcache4MB;`
	`579`	`+ uploadcache = uploadcacheLong;`
`571`	`580`	`}`
`572`	`581`	`}`
`573`	`582`