Minor improvements to compression (#8555)

Author: Martin-Molinero

Common/Extensions.cs

@@ -926,13 +926,10 @@ public static string GetString(this byte[] bytes, Encoding encoding = null)
         public static string ToMD5(this string str)
         {
             var builder = new StringBuilder(32);
-            using (var md5Hash = MD5.Create())
+            var data = MD5.HashData(Encoding.UTF8.GetBytes(str));
+            for (var i = 0; i < 16; i++)
             {
-                var data = md5Hash.ComputeHash(Encoding.UTF8.GetBytes(str));
-                for (var i = 0; i < 16; i++)
-                {
-                    builder.Append(data[i].ToStringInvariant("x2"));
-                }
+                builder.Append(data[i].ToStringInvariant("x2"));
             }
             return builder.ToString();
         }
@@ -945,13 +942,10 @@ public static string ToMD5(this string str)
         public static string ToSHA256(this string data)
         {
             var hash = new StringBuilder(64);
-            using (var crypt = SHA256.Create())
+            var crypto = SHA256.HashData(Encoding.UTF8.GetBytes(data));
+            for (var i = 0; i < 32; i++)
             {
-                var crypto = crypt.ComputeHash(Encoding.UTF8.GetBytes(data));
-                for (var i = 0; i < 32; i++)
-                {
-                    hash.Append(crypto[i].ToStringInvariant("x2"));
-                }
+                hash.Append(crypto[i].ToStringInvariant("x2"));
             }
             return hash.ToString();
         }

Compression/Compression.cs

@@ -297,18 +297,88 @@ public static async Task<Dictionary<string, string>> UnzipDataAsync(Stream strea
         /// <returns>The zipped file as a byte array</returns>
         public static byte[] ZipBytes(byte[] bytes, string zipEntryName)
         {
-            using (var memoryStream = new MemoryStream())
+            using var memoryStream = new MemoryStream();
+            ZipBytesAsync(memoryStream, bytes, zipEntryName, null).ConfigureAwait(false).GetAwaiter().GetResult();
+            return memoryStream.ToArray();
+        }
+
+        /// <summary>
+        /// Performs an in memory zip of the specified bytes in the target stream
+        /// </summary>
+        /// <param name="target">The target stream</param>
+        /// <param name="data">The file contents in bytes to be zipped</param>
+        /// <param name="zipEntryName">The zip entry name</param>
+        /// <param name="mode">The archive mode</param>
+        /// <param name="compressionLevel">The desired compression level</param>
+        /// <returns>The zipped file as a byte array</returns>
+        public static async Task ZipBytesAsync(Stream target, byte[] data, string zipEntryName, ZipArchiveMode? mode = null,
+            CompressionLevel? compressionLevel = null)
+        {
+            await ZipBytesAsync(target, [new KeyValuePair<byte[], string>(data, zipEntryName)], mode, compressionLevel).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Performs an in memory zip of the specified bytes in the target stream
+        /// </summary>
+        /// <param name="target">The target stream</param>
+        /// <param name="data">The file contents in bytes to be zipped</param>
+        /// <param name="mode">The archive mode</param>
+        /// <param name="compressionLevel">The desired compression level</param>
+        /// <returns>The zipped file as a byte array</returns>
+        public static async Task ZipBytesAsync(Stream target, IEnumerable<KeyValuePair<byte[], string>> data, ZipArchiveMode? mode = null,
+            CompressionLevel? compressionLevel = null)
+        {
+            compressionLevel ??= CompressionLevel.SmallestSize;
+            using var archive = new ZipArchive(target, mode ?? ZipArchiveMode.Create, true);
+            foreach (var kvp in data)
             {
-                using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
+                var entry = archive.CreateEntry(kvp.Value, compressionLevel.Value);
+                using var entryStream = entry.Open();
+                await entryStream.WriteAsync(kvp.Key).ConfigureAwait(false);
+            }
+        }
+
+        /// <summary>
+        /// Performs an in memory zip of the specified stream in the target stream
+        /// </summary>
+        /// <param name="target">The target stream</param>
+        /// <param name="data">The file contents in bytes to be zipped</param>
+        /// <param name="mode">The archive mode</param>
+        /// <param name="compressionLevel">The desired compression level</param>
+        /// <returns>The zipped file as a byte array</returns>
+        public static async Task ZipStreamsAsync(string target, IEnumerable<KeyValuePair<string, Stream>> data, ZipArchiveMode? mode = null,
+            CompressionLevel? compressionLevel = null)
+        {
+            using var fileStream = mode == ZipArchiveMode.Update
+                ? new FileStream(target, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)
+                : new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None);
+            await ZipStreamsAsync(fileStream, data, mode, compressionLevel).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Performs an in memory zip of the specified stream in the target stream
+        /// </summary>
+        /// <param name="target">The target stream</param>
+        /// <param name="data">The file contents in bytes to be zipped</param>
+        /// <param name="mode">The archive mode</param>
+        /// <param name="compressionLevel">The desired compression level</param>
+        /// <param name="leaveStreamOpen">True to leave the taget stream open</param>
+        /// <returns>The zipped file as a byte array</returns>
+        public static async Task ZipStreamsAsync(Stream target, IEnumerable<KeyValuePair<string, Stream>> data, ZipArchiveMode? mode = null,
+            CompressionLevel? compressionLevel = null, bool leaveStreamOpen = false)
+        {
+            compressionLevel ??= CompressionLevel.SmallestSize;
+            using var archive = new ZipArchive(target, mode ?? ZipArchiveMode.Create, leaveStreamOpen);
+            foreach (var kvp in data)
+            {
+                if (archive.Mode == ZipArchiveMode.Update)
                 {
-                    var entry = archive.CreateEntry(zipEntryName);
-                    using (var entryStream = entry.Open())
-                    {
-                        entryStream.Write(bytes, 0, bytes.Length);
-                    }
+                    var existingEntry = archive.GetEntry(kvp.Key);
+                    existingEntry?.Delete();
                 }
-                // 'ToArray' after disposing of 'ZipArchive' since it finishes writing all the data
-                return memoryStream.ToArray();
+                var entry = archive.CreateEntry(kvp.Key, compressionLevel.Value);
+                using var entryStream = entry.Open();
+                await kvp.Value.CopyToAsync(entryStream).ConfigureAwait(false);
             }
         }
 

Tests/Compression/CompressionTests.cs

@@ -38,6 +38,25 @@ public void ReadLinesCountMatchesLineCount()
             Assert.AreEqual(expected, actual);
         }
 
+        [Test]
+        public void ZipsStream()
+        {
+            const string zipName = "stream_entry.zip";
+            File.Delete(zipName);
+            const string fileContents = "this is the contents of a file!";
+            using var memoryStream = new MemoryStream();
+            var array = Encoding.UTF8.GetBytes(fileContents);
+            memoryStream.Write(array);
+            memoryStream.Position = 0;
+
+            QuantConnect.Compression.ZipStreamsAsync(zipName, [new ("entry", memoryStream)]).Wait();
+
+            using var file = File.OpenRead(zipName);
+            using var streamReader = QuantConnect.Compression.UnzipStreamToStreamReader(file);
+            var contents = streamReader.ReadToEnd();
+            Assert.AreEqual(fileContents, contents);
+        }
+
         [Test]
         public void ZipBytes()
         {
@@ -61,7 +80,7 @@ public void ZipBytesReturnsByteArrayWithCorrectLength()
             var fileBytes = File.ReadAllBytes(file);
             var zippedBytes = QuantConnect.Compression.ZipBytes(fileBytes, "entry");
 
-            Assert.AreEqual(OS.IsWindows ? 905921 : 906121, zippedBytes.Length);
+            Assert.AreEqual(OS.IsWindows ? 905693 : 906121, zippedBytes.Length);
         }
 
         [Test]