Display download progress in human readable format for Invoke-WebRequest (PowerShell#14611)

bergmeister · web-flow · commit d3d81c3e4453 · 2022-09-19T11:16:21.000-07:00
diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs
@@ -458,7 +458,7 @@ internal override void ProcessResponse(HttpResponseMessage response)
             }
             else if (ShouldSaveToOutFile)
             {
-                StreamHelper.SaveStreamToFile(baseResponseStream, QualifiedOutFile, this, _cancelToken.Token);
+                StreamHelper.SaveStreamToFile(baseResponseStream, QualifiedOutFile, this, response.Content.Headers.ContentLength.GetValueOrDefault(), _cancelToken.Token);
             }
 
             if (!string.IsNullOrEmpty(StatusCodeVariable))
diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebResponseObject.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebResponseObject.Common.cs
@@ -204,7 +204,7 @@ private void SetResponse(HttpResponseMessage response, Stream contentStream)
                 }
 
                 int initialCapacity = (int)Math.Min(contentLength, StreamHelper.DefaultReadBuffer);
-                _rawContentStream = new WebResponseContentMemoryStream(st, initialCapacity, null);
+                _rawContentStream = new WebResponseContentMemoryStream(st, initialCapacity, cmdlet: null, response.Content.Headers.ContentLength.GetValueOrDefault());
             }
             // set the position of the content stream to the beginning
             _rawContentStream.Position = 0;
diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs
@@ -38,7 +38,11 @@ internal override void ProcessResponse(HttpResponseMessage response)
             if (ShouldWriteToPipeline)
             {
                 // creating a MemoryStream wrapper to response stream here to support IsStopping.
-                responseStream = new WebResponseContentMemoryStream(responseStream, StreamHelper.ChunkSize, this);
+                responseStream = new WebResponseContentMemoryStream(
+                    responseStream,
+                    StreamHelper.ChunkSize,
+                    this,
+                    response.Content.Headers.ContentLength.GetValueOrDefault());
                 WebResponseObject ro = WebResponseObjectFactory.GetResponseObject(response, responseStream, this.Context);
                 ro.RelationLink = _relationLink;
                 WriteObject(ro);
@@ -52,7 +56,7 @@ internal override void ProcessResponse(HttpResponseMessage response)
 
             if (ShouldSaveToOutFile)
             {
-                StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, _cancelToken.Token);
+                StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, response.Content.Headers.ContentLength.GetValueOrDefault(), _cancelToken.Token);
             }
         }
 
diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs
@@ -24,6 +24,7 @@ internal class WebResponseContentMemoryStream : MemoryStream
     {
         #region Data
 
+        private readonly long? _contentLength;
         private readonly Stream _originalStreamToProxy;
         private bool _isInitialized = false;
         private readonly Cmdlet _ownerCmdlet;
@@ -37,9 +38,11 @@ internal class WebResponseContentMemoryStream : MemoryStream
         /// <param name="stream"></param>
         /// <param name="initialCapacity"></param>
         /// <param name="cmdlet">Owner cmdlet if any.</param>
-        internal WebResponseContentMemoryStream(Stream stream, int initialCapacity, Cmdlet cmdlet)
+        /// <param name="contentLength">Expected download size in Bytes.</param>
+        internal WebResponseContentMemoryStream(Stream stream, int initialCapacity, Cmdlet cmdlet, long? contentLength)
             : base(initialCapacity)
         {
+            this._contentLength = contentLength;
             _originalStreamToProxy = stream;
             _ownerCmdlet = cmdlet;
         }
@@ -218,14 +221,24 @@ private void Initialize()
             _isInitialized = true;
             try
             {
-                long totalLength = 0;
+                long totalRead = 0;
                 byte[] buffer = new byte[StreamHelper.ChunkSize];
                 ProgressRecord record = new(StreamHelper.ActivityId, WebCmdletStrings.ReadResponseProgressActivity, "statusDescriptionPlaceholder");
-                for (int read = 1; read > 0; totalLength += read)
+                string totalDownloadSize = _contentLength is null ? "???" : Utils.DisplayHumanReadableFileSize((long)_contentLength);
+                for (int read = 1; read > 0; totalRead += read)
                 {
                     if (_ownerCmdlet != null)
                     {
-                        record.StatusDescription = StringUtil.Format(WebCmdletStrings.ReadResponseProgressStatus, totalLength);
+                        record.StatusDescription = StringUtil.Format(
+                            WebCmdletStrings.ReadResponseProgressStatus,
+                            Utils.DisplayHumanReadableFileSize(totalRead),
+                            totalDownloadSize);
+
+                        if (_contentLength > 0)
+                        {
+                            record.PercentComplete = Math.Min((int)(totalRead * 100 / (long)_contentLength), 100);
+                        }
+
                         _ownerCmdlet.WriteProgress(record);
 
                         if (_ownerCmdlet.IsStopping)
@@ -244,13 +257,13 @@ private void Initialize()
 
                 if (_ownerCmdlet != null)
                 {
-                    record.StatusDescription = StringUtil.Format(WebCmdletStrings.ReadResponseComplete, totalLength);
+                    record.StatusDescription = StringUtil.Format(WebCmdletStrings.ReadResponseComplete, totalRead);
                     record.RecordType = ProgressRecordType.Completed;
                     _ownerCmdlet.WriteProgress(record);
                 }
 
                 // make sure the length is set appropriately
-                base.SetLength(totalLength);
+                base.SetLength(totalRead);
                 base.Seek(0, SeekOrigin.Begin);
             }
             catch (Exception)
@@ -276,7 +289,7 @@ internal static class StreamHelper
 
         #region Static Methods
 
-        internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, CancellationToken cancellationToken)
+        internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, long? contentLength, CancellationToken cancellationToken)
         {
             if (cmdlet == null)
             {
@@ -289,12 +302,22 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet,
                 ActivityId,
                 WebCmdletStrings.WriteRequestProgressActivity,
                 WebCmdletStrings.WriteRequestProgressStatus);
+            string totalDownloadSize = contentLength is null ? "???" : Utils.DisplayHumanReadableFileSize((long)contentLength);
 
             try
             {
                 while (!copyTask.Wait(1000, cancellationToken))
                 {
-                    record.StatusDescription = StringUtil.Format(WebCmdletStrings.WriteRequestProgressStatus, output.Position);
+                    record.StatusDescription = StringUtil.Format(
+                        WebCmdletStrings.WriteRequestProgressStatus,
+                        Utils.DisplayHumanReadableFileSize(output.Position),
+                        totalDownloadSize);
+
+                    if (contentLength != null && contentLength > 0)
+                    {
+                        record.PercentComplete = Math.Min((int)(output.Position * 100 / (long)contentLength), 100);
+                    }
+
                     cmdlet.WriteProgress(record);
                 }
 
@@ -316,13 +339,14 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet,
         /// <param name="stream">Input stream.</param>
         /// <param name="filePath">Output file name.</param>
         /// <param name="cmdlet">Current cmdlet (Invoke-WebRequest or Invoke-RestMethod).</param>
+        /// <param name="contentLength">Expected download size in Bytes.</param>
         /// <param name="cancellationToken">CancellationToken to track the cmdlet cancellation.</param>
-        internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet, CancellationToken cancellationToken)
+        internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet, long? contentLength, CancellationToken cancellationToken)
         {
             // If the web cmdlet should resume, append the file instead of overwriting.
             FileMode fileMode = cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume ? FileMode.Append : FileMode.Create;
             using FileStream output = new(filePath, fileMode, FileAccess.Write, FileShare.Read);
-            WriteToStream(stream, output, cmdlet, cancellationToken);
+            WriteToStream(stream, output, cmdlet, contentLength, cancellationToken);
         }
 
         private static string StreamToString(Stream stream, Encoding encoding)
diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx
@@ -199,13 +199,13 @@
     <value>The cmdlet cannot run because the following parameter is missing: Proxy. Provide a valid proxy URI for the Proxy parameter when using the ProxyCredential or ProxyUseDefaultCredentials parameters, then retry.</value>
   </data>
   <data name="ReadResponseComplete" xml:space="preserve">
-    <value>Reading web response completed. (Number of bytes read: {0})</value>
+    <value>Reading web response stream completed. Downloaded: {0}</value>
   </data>
   <data name="ReadResponseProgressActivity" xml:space="preserve">
-    <value>Reading web response</value>
+    <value>Reading web response stream</value>
   </data>
   <data name="ReadResponseProgressStatus" xml:space="preserve">
-    <value>Reading response stream... (Number of bytes read: {0})</value>
+    <value>Downloaded: {0} of {1}</value>
   </data>
   <data name="RequestTimeout" xml:space="preserve">
     <value>The operation has timed out.</value>
@@ -223,7 +223,7 @@
     <value>Web request status</value>
   </data>
   <data name="WriteRequestProgressStatus" xml:space="preserve">
-    <value>Number of bytes processed: {0}</value>
+    <value>Downloaded: {0} of {1}</value>
   </data>
   <data name="JsonNetModuleRequired" xml:space="preserve">
     <value>The ConvertTo-Json and ConvertFrom-Json cmdlets require the 'Json.Net' module. {0}</value>
diff --git a/src/System.Management.Automation/engine/Utils.cs b/src/System.Management.Automation/engine/Utils.cs
@@ -5,8 +5,6 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
-using System.ComponentModel;
-using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.IO;
@@ -17,7 +15,6 @@
 using System.Management.Automation.Security;
 using System.Numerics;
 using System.Reflection;
-using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Security;
 #if !UNIX
@@ -1547,6 +1544,21 @@ internal static bool IsComObject(object obj)
 
             return oldMode;
         }
+                
+        internal static string DisplayHumanReadableFileSize(long bytes)
+        {
+            return bytes switch
+            {
+                < 1024 and >= 0 => $"{bytes} Bytes",
+                < 1048576 and >= 1024 => $"{(bytes / 1024.0).ToString("0.0")} KB",
+                < 1073741824 and >= 1048576 => $"{(bytes / 1048576.0).ToString("0.0")} MB",
+                < 1099511627776 and >= 1073741824 => $"{(bytes / 1073741824.0).ToString("0.000")} GB",
+                < 1125899906842624 and >= 1099511627776 => $"{(bytes / 1099511627776.0).ToString("0.00000")} TB",
+                < 1152921504606847000 and >= 1125899906842624 => $"{(bytes / 1125899906842624.0).ToString("0.0000000")} PB",
+                >= 1152921504606847000 => $"{(bytes / 1152921504606847000.0).ToString("0.000000000")} EB",
+                _ => $"0 Bytes",
+            };
+        }
     }
 }
 
diff --git a/test/xUnit/csharp/test_Utils.cs b/test/xUnit/csharp/test_Utils.cs
@@ -22,6 +22,23 @@ public static void TestIsWinPEHost()
             Assert.False(Utils.IsWinPEHost());
         }
 
+        [Theory]
+        [InlineData(long.MinValue, "0 Bytes")]
+        [InlineData(-1, "0 Bytes")]
+        [InlineData(0, "0 Bytes")]
+        [InlineData(1, "1 Bytes")]
+        [InlineData(1024, "1.0 KB")]
+        [InlineData(3000, "2.9 KB")]
+        [InlineData(1024 * 1024, "1.0 MB")]
+        [InlineData(1024 * 1024 * 1024, "1.000 GB")]
+        [InlineData((long)(1024 * 1024 * 1024) * 1024, "1.00000 TB")]
+        [InlineData((long)(1024 * 1024 * 1024) * 1024 * 1024, "1.0000000 PB")]
+        [InlineData(long.MaxValue, "8.000000000 EB")]
+        public static void DisplayHumanReadableFileSize(long bytes, string expected)
+        {
+            Assert.Equal(expected, Utils.DisplayHumanReadableFileSize(bytes));
+        }
+
         [Fact]
         public static void TestHistoryStack()
         {

Original file line number	Diff line number	Diff line change
`@@ -458,7 +458,7 @@ internal override void ProcessResponse(HttpResponseMessage response)`
`458`	`458`	`}`
`459`	`459`	`else if (ShouldSaveToOutFile)`
`460`	`460`	`{`
`461`		`- StreamHelper.SaveStreamToFile(baseResponseStream, QualifiedOutFile, this, _cancelToken.Token);`
	`461`	`+ StreamHelper.SaveStreamToFile(baseResponseStream, QualifiedOutFile, this, response.Content.Headers.ContentLength.GetValueOrDefault(), _cancelToken.Token);`
`462`	`462`	`}`
`463`	`463`
`464`	`464`	`if (!string.IsNullOrEmpty(StatusCodeVariable))`
Original file line number	Diff line number	Diff line change
`@@ -204,7 +204,7 @@ private void SetResponse(HttpResponseMessage response, Stream contentStream)`
`204`	`204`	`}`
`205`	`205`
`206`	`206`	`int initialCapacity = (int)Math.Min(contentLength, StreamHelper.DefaultReadBuffer);`
`207`		`- _rawContentStream = new WebResponseContentMemoryStream(st, initialCapacity, null);`
	`207`	`+ _rawContentStream = new WebResponseContentMemoryStream(st, initialCapacity, cmdlet: null, response.Content.Headers.ContentLength.GetValueOrDefault());`
`208`	`208`	`}`
`209`	`209`	`// set the position of the content stream to the beginning`
`210`	`210`	`_rawContentStream.Position = 0;`
Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,11 @@ internal override void ProcessResponse(HttpResponseMessage response)`
`38`	`38`	`if (ShouldWriteToPipeline)`
`39`	`39`	`{`
`40`	`40`	`// creating a MemoryStream wrapper to response stream here to support IsStopping.`
`41`		`- responseStream = new WebResponseContentMemoryStream(responseStream, StreamHelper.ChunkSize, this);`
	`41`	`+ responseStream = new WebResponseContentMemoryStream(`
	`42`	`+ responseStream,`
	`43`	`+ StreamHelper.ChunkSize,`
	`44`	`+ this,`
	`45`	`+ response.Content.Headers.ContentLength.GetValueOrDefault());`
`42`	`46`	`WebResponseObject ro = WebResponseObjectFactory.GetResponseObject(response, responseStream, this.Context);`
`43`	`47`	`ro.RelationLink = _relationLink;`
`44`	`48`	`WriteObject(ro);`
`@@ -52,7 +56,7 @@ internal override void ProcessResponse(HttpResponseMessage response)`
`52`	`56`
`53`	`57`	`if (ShouldSaveToOutFile)`
`54`	`58`	`{`
`55`		`- StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, _cancelToken.Token);`
	`59`	`+ StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, response.Content.Headers.ContentLength.GetValueOrDefault(), _cancelToken.Token);`
`56`	`60`	`}`
`57`	`61`	`}`
`58`	`62`