Implement gz compression

Add test for UncompressedDigest

Fix #29

Author: eerhardt

Microsoft.NET.Build.Containers/Descriptor.cs

@@ -29,6 +29,15 @@ public readonly record struct Descriptor
     [JsonPropertyName("digest")]
     public string Digest { get; init; }
 
+    /// <summary>
+    /// Digest of the uncompressed content, specifying algorithm and value.
+    /// </summary>
+    /// <remarks>
+    /// <see href="https://github.com/opencontainers/image-spec/blob/7b36cea86235157d78528944cb94c3323ee0905c/descriptor.md#digests"/>
+    /// </remarks>
+    [JsonIgnore]
+    public string UncompressedDigest { get; init; }
+
     /// <summary>
     /// Size, in bytes, of the raw content.
     /// </summary>

Microsoft.NET.Build.Containers/Image.cs

@@ -56,7 +56,7 @@ public void AddLayer(Layer l)
     {
         newLayers.Add(l);
         manifest["layers"]!.AsArray().Add(l.Descriptor);
-        config["rootfs"]!["diff_ids"]!.AsArray().Add(l.Descriptor.Digest); // TODO: this should be the descriptor of the UNCOMPRESSED tarball (once we turn on compression)
+        config["rootfs"]!["diff_ids"]!.AsArray().Add(l.Descriptor.UncompressedDigest);
         RecalculateDigest();
     }
 

Microsoft.NET.Build.Containers/Layer.cs

@@ -1,4 +1,5 @@
 using System.Formats.Tar;
+using System.IO.Compression;
 using System.Security.Cryptography;
 
 namespace Microsoft.NET.Build.Containers;
@@ -26,37 +27,44 @@ public static Layer FromFiles(IEnumerable<(string path, string containerPath)> f
     {
         long fileSize;
         Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
+        byte[] uncompressedHash;
 
         string tempTarballPath = ContentStore.GetTempFile();
         using (FileStream fs = File.Create(tempTarballPath))
         {
-            // using (GZipStream gz = new(fs, CompressionMode.Compress)) // TODO: https://github.com/dotnet/sdk-container-builds/issues/29
-            using (TarWriter writer = new(fs, TarEntryFormat.Gnu, leaveOpen: true))
+            using (HashDigestGZipStream gz = new(fs, leaveOpen: true))
             {
-                foreach (var item in fileList)
+                using (TarWriter writer = new(gz, TarEntryFormat.Gnu, leaveOpen: true))
                 {
-                    // Docker treats a COPY instruction that copies to a path like `/app` by
-                    // including `app/` as a directory, with no leading slash. Emulate that here.
-                    string containerPath = item.containerPath.TrimStart(PathSeparators);
+                    foreach (var item in fileList)
+                    {
+                        // Docker treats a COPY instruction that copies to a path like `/app` by
+                        // including `app/` as a directory, with no leading slash. Emulate that here.
+                        string containerPath = item.containerPath.TrimStart(PathSeparators);
+
+                        writer.WriteEntry(item.path, containerPath);
+                    }
+                } // Dispose of the TarWriter before getting the hash so the final data get written to the tar stream
 
-                    writer.WriteEntry(item.path, containerPath);
-                }
+                uncompressedHash = gz.GetHash();
             }
 
             fileSize = fs.Length;
-
+            
             fs.Position = 0;
 
             SHA256.HashData(fs, hash);
         }
 
         string contentHash = Convert.ToHexString(hash).ToLowerInvariant();
+        string uncompressedContentHash = Convert.ToHexString(uncompressedHash).ToLowerInvariant();
 
         Descriptor descriptor = new()
         {
-            MediaType = "application/vnd.docker.image.rootfs.diff.tar", // TODO: configurable? gzip always?
+            MediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip", // TODO: configurable? gzip always?
             Size = fileSize,
-            Digest = $"sha256:{contentHash}"
+            Digest = $"sha256:{contentHash}",
+            UncompressedDigest = $"sha256:{uncompressedContentHash}",
         };
 
         string storedContent = ContentStore.PathForDescriptor(descriptor);
@@ -76,4 +84,74 @@ public static Layer FromFiles(IEnumerable<(string path, string containerPath)> f
 
     private readonly static char[] PathSeparators = new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar };
 
+    /// <summary>
+    /// A stream capable of computing the hash digest of raw uncompressed data while also compressing it.
+    /// </summary>
+    private sealed class HashDigestGZipStream : Stream
+    {
+        private readonly SHA256 hashAlgorithm;
+        private readonly CryptoStream sha256Stream;
+        private readonly Stream compressionStream;
+
+        public HashDigestGZipStream(Stream writeStream, bool leaveOpen)
+        {
+            hashAlgorithm = SHA256.Create();
+            sha256Stream = new CryptoStream(Stream.Null, hashAlgorithm, CryptoStreamMode.Write);
+            compressionStream = new GZipStream(writeStream, CompressionMode.Compress, leaveOpen);
+        }
+
+        public override bool CanWrite => true;
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            sha256Stream.Write(buffer, offset, count);
+            compressionStream.Write(buffer, offset, count);
+        }
+
+        public override void Write(ReadOnlySpan<byte> buffer)
+        {
+            sha256Stream.Write(buffer);
+            compressionStream.Write(buffer);
+        }
+
+        public override void Flush()
+        {
+            sha256Stream.Flush();
+            compressionStream.Flush();
+        }
+
+        internal byte[] GetHash()
+        {
+            sha256Stream.FlushFinalBlock();
+            return hashAlgorithm.Hash!;
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            try
+            {
+                sha256Stream.Dispose();
+                compressionStream.Dispose();
+            }
+            finally
+            {
+                base.Dispose(disposing);
+            }
+        }
+
+        // This class is never used with async writes, but if it ever is, implement these overrides
+        public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+            => throw new NotImplementedException();
+        public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken)
+            => throw new NotImplementedException();
+
+        public override bool CanRead => false;
+        public override bool CanSeek => false;
+        public override long Length => throw new NotImplementedException();
+        public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
+
+        public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException();
+        public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException();
+        public override void SetLength(long value) => throw new NotImplementedException();
+    }
 }

Test.Microsoft.NET.Build.Containers.Filesystem/LayerEndToEnd.cs

@@ -1,4 +1,5 @@
 using Microsoft.NET.Build.Containers;
+using System.IO.Compression;
 using System.Security.Cryptography;
 
 namespace Test.Microsoft.NET.Build.Containers.Filesystem;
@@ -21,19 +22,10 @@ public void SingleFileInFolder()
         Console.WriteLine(l.Descriptor);
 
         //Assert.AreEqual("application/vnd.oci.image.layer.v1.tar", l.Descriptor.MediaType); // TODO: configurability
-        Assert.AreEqual(2048, l.Descriptor.Size);
+        Assert.IsTrue(l.Descriptor.Size is >= 136 and <= 138, $"'l.Descriptor.Size' should be between 136 and 138, but is {l.Descriptor.Size}"); // TODO: determinism!
         //Assert.AreEqual("sha256:26140bc75f2fcb3bf5da7d3b531d995c93d192837e37df0eb5ca46e2db953124", l.Descriptor.Digest); // TODO: determinism!
 
-        Assert.AreEqual(l.Descriptor.Size, new FileInfo(l.BackingFile).Length);
-
-        byte[] hashBytes;
-
-        using (FileStream fs = File.OpenRead(l.BackingFile))
-        {
-            hashBytes = SHA256.HashData(fs);
-        }
-
-        Assert.AreEqual(Convert.ToHexString(hashBytes), l.Descriptor.Digest.Substring("sha256:".Length), ignoreCase: true);
+        VerifyDescriptorInfo(l);
     }
 
     [TestMethod]
@@ -59,19 +51,33 @@ public void TwoFilesInTwoFolders()
         Console.WriteLine(l.Descriptor);
 
         //Assert.AreEqual("application/vnd.oci.image.layer.v1.tar", l.Descriptor.MediaType); // TODO: configurability
-        Assert.AreEqual(3072, l.Descriptor.Size);
+        Assert.IsTrue(l.Descriptor.Size is >= 169 and <= 173, $"'l.Descriptor.Size' should be between 169 and 173, but is {l.Descriptor.Size}"); // TODO: determinism!
         //Assert.AreEqual("sha256:26140bc75f2fcb3bf5da7d3b531d995c93d192837e37df0eb5ca46e2db953124", l.Descriptor.Digest); // TODO: determinism!
 
+        VerifyDescriptorInfo(l);
+    }
+
+    private static void VerifyDescriptorInfo(Layer l)
+    {
         Assert.AreEqual(l.Descriptor.Size, new FileInfo(l.BackingFile).Length);
 
         byte[] hashBytes;
+        byte[] uncompressedHashBytes;
 
         using (FileStream fs = File.OpenRead(l.BackingFile))
         {
             hashBytes = SHA256.HashData(fs);
+
+            fs.Position = 0;
+
+            using (GZipStream decompressionStream = new GZipStream(fs, CompressionMode.Decompress))
+            {
+                uncompressedHashBytes = SHA256.HashData(decompressionStream);
+            }
         }
 
         Assert.AreEqual(Convert.ToHexString(hashBytes), l.Descriptor.Digest.Substring("sha256:".Length), ignoreCase: true);
+        Assert.AreEqual(Convert.ToHexString(uncompressedHashBytes), l.Descriptor.UncompressedDigest.Substring("sha256:".Length), ignoreCase: true);
     }
 
     TransientTestFolder testSpecificArtifactRoot;