[Storage] Remove MD5 related code from B/F upload cmdlet. (Azure#27666)

Author: blueww

src/Storage/Storage.Management/ChangeLog.md

@@ -18,6 +18,9 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Removed MD5 from blob and file upload in some scenarios
+    - `Set-AzStorageFileContent`
+    - `Set-AzStorageBlobContent`
 
 ## Version 8.4.0
 * When create Storage context, allowed user input StorageAccountName

src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs

@@ -36,6 +36,7 @@
 using Azure.Storage.Blobs.Specialized;
 using Azure;
 using System.Linq;
+using System.Runtime.InteropServices;
 
 namespace Microsoft.WindowsAzure.Commands.Storage.Blob
 {
@@ -699,8 +700,7 @@ public override void ExecuteCmdlet()
                 ValidateBlobProperties(BlobProperties);
             }
 
-            // if FIPS policy is enabled, must use native MD5
-            if (fipsEnabled)
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
             {
                 CloudStorageAccount.UseV1MD5 = false;
             }

src/Storage/Storage/Blob/StorageDataMovementCmdletBase.cs

@@ -23,6 +23,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob
     using OpContext = Microsoft.Azure.Storage.OperationContext;
     using System.Collections.Generic;
     using System.Security.Cryptography;
+    using Microsoft.Azure.Documents;
 
     public class StorageDataMovementCmdletBase : StorageCloudBlobCmdletBase, IDisposable
     {
@@ -199,18 +200,18 @@ public static string[] GetBlockIDs(long contentLength, long blockLength, string
                 blockCount++;
             }
             List<string> blockIDs = new List<string>();
-            string blockIdPrefix = Convert.ToBase64String(MD5.Create().ComputeHash(System.Text.Encoding.UTF8.GetBytes(blobname)));
             for (int i = 0; i < (int)blockCount; i++)
             {
-                string idNo = i.ToString();
-                while (idNo.Length < 5)
-                {
-                    idNo = "0" + idNo;
-                }
-                string blockID = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(blockIdPrefix + idNo));
+                string blockID = GenerateBlockId(i * blockLength);
                 blockIDs.Add(blockID);
             }
             return blockIDs.ToArray();
         }
+        public static string GenerateBlockId(long offset)
+        {
+            byte[] id = new byte[48]; // 48 raw bytes => 64 byte string once Base64 encoded
+            BitConverter.GetBytes(offset).CopyTo(id, 0);
+            return Convert.ToBase64String(id);
+        }
     }
 }

src/Storage/Storage/Common/StorageCloudCmdletBase.cs

@@ -670,23 +670,5 @@ protected override void StopProcessing()
             _cancellationTokenSource.Cancel();
             base.StopProcessing();
         }
-
-        /// <summary>
-        /// true if FIPS policy is enabled on the current machine
-        /// </summary>
-        public static bool fipsEnabled { get; } = IsFIPSEnabled();
-
-        internal static bool IsFIPSEnabled()
-        {
-            try
-            {
-                System.Security.Cryptography.MD5.Create();
-                return false;
-            }
-            catch (System.Reflection.TargetInvocationException)
-            {
-                return true;
-            }
-        }
     }
 }

src/Storage/Storage/File/Cmdlet/SetAzureStorageFileContent.cs

@@ -22,7 +22,6 @@ namespace Microsoft.WindowsAzure.Commands.Storage.File.Cmdlet
     using Microsoft.Azure.Storage.DataMovement;
     using Microsoft.Azure.Storage.File;
     using Microsoft.WindowsAzure.Commands.Common;
-    using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
     using Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel;
     using Microsoft.WindowsAzure.Commands.Storage.Common;
     using Microsoft.WindowsAzure.Commands.Utilities.Common;
@@ -37,7 +36,6 @@ namespace Microsoft.WindowsAzure.Commands.Storage.File.Cmdlet
     using System.Threading.Tasks;
     using LocalConstants = Microsoft.WindowsAzure.Commands.Storage.File.Constants;
 
-    [CmdletOutputBreakingChangeWithVersion(typeof(AzureStorageFile), "14.0.0", "9.0.0", ChangeDescription = "The ContentHash properties will be removed from the uploaded Azure file when file size > 1TB, or upload with Oauth credencial, or with -DisAllowTrailingDot.")]
     [Cmdlet("Set", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageFileContent", SupportsShouldProcess = true, DefaultParameterSetName = LocalConstants.ShareNameParameterSetName), OutputType(typeof(AzureStorageFile))]
     public class SetAzureStorageFileContent : StorageFileDataManagementCmdletBase, IDynamicParameters
     {
@@ -130,17 +128,9 @@ public override void ExecuteCmdlet()
             }
             long fileSize = localFile.Length;
 
-            // if FIPS policy is enabled, must use native MD5 for DMlib. 
-            if (fipsEnabled)
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
             {
-                if (fileSize < sizeTB)
-                {
-                    CloudStorageAccount.UseV1MD5 = false;
-                }
-                else // use Track2 SDK
-                {
-                    WriteWarning("The uploaded file won't have Content MD5 hash, since caculate MD5 hash fail, most possiblly caused by FIPS is enabled on this machine.");
-                }
+                CloudStorageAccount.UseV1MD5 = false;
             }
 
             bool isDirectory;
@@ -251,12 +241,6 @@ await fileClient.CreateAsync(fileSize,
 
                         List<Task> runningTasks = new List<Task>();
 
-                        IncrementalHash hash = null;
-                        if (!fipsEnabled)
-                        {
-                            hash = IncrementalHash.CreateHash(HashAlgorithmName.MD5);
-                        }
-
                         using (FileStream stream = File.OpenRead(localFile.FullName))
                         {
                             byte[] buffer = null;
@@ -268,10 +252,6 @@ await fileClient.CreateAsync(fileSize,
                                 buffer = new byte[targetBlockSize];
 
                                 int actualBlockSize = await stream.ReadAsync(buffer: buffer, offset: 0, count: (int)targetBlockSize);
-                                if (!fipsEnabled && hash != null)
-                                {
-                                    hash.AppendData(buffer, 0, actualBlockSize);
-                                }
 
                                 Task task = UploadFileRangAsync(fileClient,
                                     new HttpRange(offset, actualBlockSize),
@@ -305,24 +285,6 @@ await fileClient.CreateAsync(fileSize,
                             await Task.WhenAll(runningTasks).ConfigureAwait(false);
                         }
 
-                        // Need set file ContentHash
-                        if ((!fipsEnabled && hash != null))
-                        {
-                            ShareFileHttpHeaders header = null;
-                            if (!fipsEnabled && hash != null)
-                            {
-                                header = new ShareFileHttpHeaders();
-                                header.ContentHash = hash.GetHashAndReset();
-                            }
-
-                            // set file header and attributes to the file
-                            ShareFileSetHttpHeadersOptions httpHeadersOptions = new ShareFileSetHttpHeadersOptions
-                            {
-                                HttpHeaders = header,
-                            };
-                            fileClient.SetHttpHeaders(httpHeadersOptions);
-                        }
-
                         if (this.PassThru)
                         {
                             // TODO: should make sure track1 file object attributes get?