Use zlib in System.IO.Compression on Unix

Updates the Unix implementation of System.IO.Compression to use zlib.  For the most part, the Windows implementation "just works"; the primary differences are the name of the library on Unix and the size of a data type being used in the z_stream data structure.

Additionally, on Windows we're planning to distribute the native library with the managed library, but on Unix we're currently planning on relying on the platform's package management to provide the native zlib library.  As such, I've added a fallback mechanism, such that if zlib isn't available, we use the managed implementation that'll work everywhere.

Author: stephentoub

src/Common/src/Interop/Unix/Interop.Libraries.cs

@@ -8,5 +8,6 @@ private static partial class Libraries
         internal const string Libc = "libc";                   // C library
         internal const string LibCoreClr= "libcoreclr";        // CoreCLR runtime
         internal const string LibCrypto = "libcrypto";         // OpenSSL crypto library
+        internal const string Zlib = "libz";                   // zlib compression library
     }
 }

src/System.IO.Compression/src/System.IO.Compression.csproj

@@ -52,16 +52,17 @@
     <Compile Include="System\IO\Compression\ZipVersionNeededValues.cs" />
     <Compile Include="System\IO\Compression\ZLibException.cs" />
     <Compile Include="System\IO\Compression\ZLibNative.cs" />
-  </ItemGroup>
-  <ItemGroup>
     <Compile Include="Interop\Interop.zlib.cs" />
-    <Compile Include="$(CommonPath)\Interop\Windows\Interop.Libraries.cs" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
     <Compile Include="System\IO\Compression\DeflateStream.Windows.cs" />
+    <Compile Include="System\IO\Compression\ZLibNative.Windows.cs" />
+    <Compile Include="$(CommonPath)\Interop\Windows\Interop.Libraries.cs" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetsUnix)' == 'true' ">
     <Compile Include="System\IO\Compression\DeflateStream.Unix.cs" />
+    <Compile Include="System\IO\Compression\ZLibNative.Unix.cs" />
+    <Compile Include="$(CommonPath)\Interop\Unix\Interop.Libraries.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="project.json" />

src/System.IO.Compression/src/System/IO/Compression/DeflateStream.Unix.cs

@@ -1,13 +1,27 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System.Diagnostics;
+
 namespace System.IO.Compression
 {
     public partial class DeflateStream : Stream
     {
         private static WorkerType GetDeflaterType()
         {
-            return WorkerType.Managed; // TODO: Switch to ZLib once interop is worked out
+            try
+            {
+                // Make any P/Invoke into zlib to ensure we're able to find and use it.
+                // If we are, then use zlib.
+                Interop.zlib.zlibCompileFlags(); 
+                return WorkerType.ZLib;
+            }
+            catch
+            {
+                // Otherwise, fallback to managed implementation if zlib isn't available
+                Debug.Fail("zlib unavailable");
+                return WorkerType.Managed;
+            }
         }
     }
 }

src/System.IO.Compression/src/System/IO/Compression/ZLibNative.Unix.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Runtime.InteropServices;
+
+namespace System.IO.Compression
+{
+    internal static partial class ZLibNative
+    {
+        /// <summary>
+        /// ZLib stream descriptor data structure
+        /// Do not construct instances of <code>ZStream</code> explicitly.
+        /// Always use <code>ZLibNative.DeflateInit2_</code> or <code>ZLibNative.InflateInit2_</code> instead.
+        /// Those methods will wrap this structure into a <code>SafeHandle</code> and thus make sure that it is always disposed correctly.
+        /// </summary>
+        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
+        internal struct ZStream
+        {
+            internal IntPtr nextIn;     //Bytef    *next_in;  /* next input byte */
+            internal UIntPtr availIn;   //uInt     avail_in;  /* number of bytes available at next_in */
+            internal UIntPtr totalIn;   //uLong    total_in;  /* total nb of input bytes read so far */
+
+            internal IntPtr nextOut;    //Bytef    *next_out; /* next output byte should be put there */
+            internal UIntPtr availOut;  //uInt     avail_out; /* remaining free space at next_out */
+            internal UIntPtr totalOut;  //uLong    total_out; /* total nb of bytes output so far */
+
+            internal IntPtr msg;        //char     *msg;      /* last error message, NULL if no error */
+
+            internal IntPtr state;      //struct internal_state FAR *state; /* not visible by applications */
+
+            internal IntPtr zalloc;     //alloc_func zalloc;  /* used to allocate the internal state */
+            internal IntPtr zfree;      //free_func  zfree;   /* used to free the internal state */
+            internal IntPtr opaque;     //voidpf   opaque;    /* private data object passed to zalloc and zfree */
+
+            internal int dataType;      //int      data_type; /* best guess about the data type: binary or text */
+            internal UIntPtr adler;     //uLong    adler;     /* adler32 value of the uncompressed data */
+            internal UIntPtr reserved;  //uLong    reserved;  /* reserved for future use */
+        }
+
+        /// <summary>Casts a uint to a native zlib uLong.</summary>
+        private static UIntPtr CastUInt32ToNativeuLong(uint value)
+        {
+            // Several members of ZStream are defined by zlib as uLong, which is 32-bits on
+            // Windows, but 32-bits on 32-bit Unix and 64-bits on 64-bit Unix.  The consuming code
+            // works in terms of a 32-bit value, so we need to cast back and forth in
+            // a platform-agnostic way, hence we have a PAL helper on Unix and one on Windows.
+            return (UIntPtr)value;
+        }
+    }
+}

src/System.IO.Compression/src/System/IO/Compression/ZLibNative.Windows.cs

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Runtime.InteropServices;
+
+namespace System.IO.Compression
+{
+    internal static partial class ZLibNative
+    {
+        /// <summary>
+        /// ZLib stream descriptor data structure
+        /// Do not construct instances of <code>ZStream</code> explicitly.
+        /// Always use <code>ZLibNative.DeflateInit2_</code> or <code>ZLibNative.InflateInit2_</code> instead.
+        /// Those methods will wrap this structure into a <code>SafeHandle</code> and thus make sure that it is always disposed correctly.
+        /// </summary>
+        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
+        internal struct ZStream
+        {
+            internal IntPtr nextIn;     //Bytef    *next_in;  /* next input byte */
+            internal uint availIn;      //uInt     avail_in;  /* number of bytes available at next_in */
+            internal uint totalIn;      //uLong    total_in;  /* total nb of input bytes read so far */
+
+            internal IntPtr nextOut;    //Bytef    *next_out; /* next output byte should be put there */
+            internal uint availOut;     //uInt     avail_out; /* remaining free space at next_out */
+            internal uint totalOut;     //uLong    total_out; /* total nb of bytes output so far */
+
+            internal IntPtr msg;        //char     *msg;      /* last error message, NULL if no error */
+
+            internal IntPtr state;      //struct internal_state FAR *state; /* not visible by applications */
+
+            internal IntPtr zalloc;     //alloc_func zalloc;  /* used to allocate the internal state */
+            internal IntPtr zfree;      //free_func  zfree;   /* used to free the internal state */
+            internal IntPtr opaque;     //voidpf   opaque;    /* private data object passed to zalloc and zfree */
+
+            internal int dataType;      //int      data_type; /* best guess about the data type: binary or text */
+            internal uint adler;        //uLong    adler;     /* adler32 value of the uncompressed data */
+            internal uint reserved;     //uLong    reserved;  /* reserved for future use */
+        }
+
+        /// <summary>Casts a uint to a native zlib uLong.</summary>
+        private static uint CastUInt32ToNativeuLong(uint value)
+        {
+            return value;
+        }
+
+    }
+}

src/System.IO.Compression/src/System/IO/Compression/ZLibNative.cs

@@ -1,13 +1,10 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System.Text;
-using System.Runtime.InteropServices;
-using System.Runtime.Versioning;
-using System.Runtime.CompilerServices;
-using Microsoft.Win32.SafeHandles;
 using System.Diagnostics.Contracts;
+using System.Runtime.InteropServices;
 using System.Security;
+using System.Text;
 
 namespace System.IO.Compression
 {
@@ -17,7 +14,7 @@ namespace System.IO.Compression
     /// 
     /// See also: How to choose a compression level (in comments to <code>CompressionLevel</code>.
     /// </summary>
-    internal static class ZLibNative
+    internal static partial class ZLibNative
     {
         #region Constants defined in zlib.h
 
@@ -178,41 +175,6 @@ public enum CompressionMethod : int
                                                           // More is faster and better compression with more memory usage.
         #endregion  // Defaults for ZLib parameters
 
-
-        #region ZLib stream descriptor data structure
-
-        /// <summary>
-        /// Do not construct instances of <code>ZStream</code> explicitly.
-        /// Always use <code>ZLibNative.DeflateInit2_</code> or <code>ZLibNative.InflateInit2_</code> instead.
-        /// Those methods will wrap this structure into a <code>SafeHandle</code> and thus make sure that it is always disposed correctly.
-        /// </summary>
-        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
-        internal struct ZStream
-        {
-            internal IntPtr nextIn;     //Bytef    *next_in;  /* next input byte */
-            internal UInt32 availIn;    //uInt     avail_in;  /* number of bytes available at next_in */
-            internal UInt32 totalIn;    //uLong    total_in;  /* total nb of input bytes read so far */
-
-            internal IntPtr nextOut;    //Bytef    *next_out; /* next output byte should be put there */
-            internal UInt32 availOut;   //uInt     avail_out; /* remaining free space at next_out */
-            internal UInt32 totalOut;   //uLong    total_out; /* total nb of bytes output so far */
-
-            internal IntPtr msg;        //char     *msg;      /* last error message, NULL if no error */
-
-            internal IntPtr state;      //struct internal_state FAR *state; /* not visible by applications */
-
-            internal IntPtr zalloc;     //alloc_func zalloc;  /* used to allocate the internal state */
-            internal IntPtr zfree;      //free_func  zfree;   /* used to free the internal state */
-            internal IntPtr opaque;     //voidpf   opaque;    /* private data object passed to zalloc and zfree */
-
-            internal Int32 dataType;    //int      data_type; /* best guess about the data type: binary or text */
-            internal UInt32 adler;      //uLong    adler;     /* adler32 value of the uncompressed data */
-            internal UInt32 reserved;   //uLong    reserved;  /* reserved for future use */
-        }
-
-        #endregion  // ZLib stream descriptor data structure
-
-
         /**
          * Do not remove the nested typing of types inside of <code>System.IO.Compression.ZLibNative</code>.
          * This was done on purpose to:
@@ -314,12 +276,10 @@ public IntPtr NextIn
 
             public UInt32 AvailIn
             {
-                [SecurityCritical] get { return _zStream.availIn; }
-                [SecurityCritical] set { _zStream.availIn = value; }
+                [SecurityCritical] get { return (uint)_zStream.availIn; }
+                [SecurityCritical] set { _zStream.availIn = CastUInt32ToNativeuLong(value); }
             }
 
-            public UInt32 TotalIn {[SecurityCritical] get { return _zStream.totalIn; } }
-
             public IntPtr NextOut
             {
                 [SecurityCritical] get { return _zStream.nextOut; }
@@ -328,16 +288,10 @@ public IntPtr NextOut
 
             public UInt32 AvailOut
             {
-                [SecurityCritical] get { return _zStream.availOut; }
-                [SecurityCritical] set { _zStream.availOut = value; }
+                [SecurityCritical] get { return (uint)_zStream.availOut; }
+                [SecurityCritical] set { _zStream.availOut = CastUInt32ToNativeuLong(value); }
             }
 
-            public UInt32 TotalOut {[SecurityCritical] get { return _zStream.totalOut; } }
-
-            public Int32 DataType {[SecurityCritical] get { return _zStream.dataType; } }
-
-            public UInt32 Adler {[SecurityCritical] get { return _zStream.adler; } }
-
             #endregion  // Expose fields on ZStream for use by user / Fx code (add more as required)