Enable automatic compression format detection (#208)

Co-authored-by: Cory Benfield <[email protected]>

Author: dnadoba

Sources/NIOHTTPCompression/HTTPDecompression.swift

@@ -94,18 +94,41 @@ public enum NIOHTTPDecompression {
                 return nil
             }
         }
-
-        var window: CInt {
-            switch self {
-            case .deflate:
-                return 15
-            case .gzip:
-                return 15 + 16
-            }
-        }
     }
 
     struct Decompressor {
+        /// `15` is the base two logarithm of the maximum window size (the size of the history buffer). It should be in the range 8..15 for this version of the library.
+        /// `32` enables automatic detection of gzip or zlib compression format with automatic header detection.
+        ///
+        /// Documentation from https://www.zlib.net/manual.html:
+        /// The windowBits parameter is the base two logarithm of the maximum window size (the size of the history buffer).
+        /// It should be in the range 8..15 for this version of the library.
+        /// The default value is 15 if inflateInit is used instead.
+        /// windowBits must be greater than or equal to the windowBits value provided to deflateInit2() while compressing,
+        /// or it must be equal to 15 if deflateInit2() was not used.
+        /// If a compressed stream with a larger window size is given as input,
+        /// inflate() will return with the error code Z_DATA_ERROR instead of trying to allocate a larger window.
+        /// windowBits can also be zero to request that inflate use the window size in the zlib header of the compressed stream.
+        /// windowBits can also be –8..–15 for raw inflate.
+        /// In this case, -windowBits determines the window size.
+        /// inflate() will then process raw deflate data, not looking for a zlib or gzip header, not generating a check value,
+        /// and not looking for any check values for comparison at the end of the stream.
+        /// This is for use with other formats that use the deflate compressed data format such as zip.
+        /// Those formats provide their own check values.
+        /// If a custom format is developed using the raw deflate format for compressed data,
+        /// it is recommended that a check value such as an Adler-32 or a CRC-32 be applied to the uncompressed data as is done in the zlib,
+        /// gzip, and zip formats. For most applications, the zlib format should be used as is.
+        /// Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits.
+        /// windowBits can also be greater than 15 for optional gzip decoding.
+        /// Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection,
+        /// or add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR).
+        /// If a gzip stream is being decoded, strm->adler is a CRC-32 instead of an Adler-32.
+        /// Unlike the gunzip utility and gzread() (see below), inflate() will not automatically decode concatenated gzip members.
+        /// inflate() will return Z_STREAM_END at the end of the gzip member.
+        /// The state would need to be reset to continue decoding a subsequent gzip member.
+        /// This must be done if there is more data after a gzip member, in order for the decompression to be compliant with the gzip standard (RFC 1952).
+        static let windowBitsWithAutomaticCompressionFormatDetection: Int32 = 15 + 32
+        
         private let limit: NIOHTTPDecompression.DecompressionLimit
         private var stream = z_stream()
         private var inflated = 0
@@ -125,13 +148,13 @@ public enum NIOHTTPDecompression {
             return result
         }
 
-        mutating func initializeDecoder(encoding: NIOHTTPDecompression.CompressionAlgorithm) throws {
+        mutating func initializeDecoder() throws {
             self.stream.zalloc = nil
             self.stream.zfree = nil
             self.stream.opaque = nil
             self.inflated = 0
 
-            let rc = CNIOExtrasZlib_inflateInit2(&self.stream, encoding.window)
+            let rc = CNIOExtrasZlib_inflateInit2(&self.stream, Self.windowBitsWithAutomaticCompressionFormatDetection)
             guard rc == Z_OK else {
                 throw NIOHTTPDecompression.DecompressionError.initializationError(Int(rc))
             }

Sources/NIOHTTPCompression/HTTPRequestDecompressor.swift

@@ -55,7 +55,7 @@ public final class NIOHTTPRequestDecompressor: ChannelDuplexHandler, RemovableCh
                 let length = head.headers[canonicalForm: "Content-Length"].first.flatMap({ Int($0) })
             {
                 do {
-                    try self.decompressor.initializeDecoder(encoding: algorithm)
+                    try self.decompressor.initializeDecoder()
                     self.compression = Compression(algorithm: algorithm, contentLength: length)
                 } catch let error {
                     context.fireErrorCaught(error)

Sources/NIOHTTPCompression/HTTPResponseDecompressor.swift

@@ -71,7 +71,7 @@ public final class NIOHTTPResponseDecompressor: ChannelDuplexHandler, RemovableC
             do {
                 if let algorithm = algorithm {
                     self.compression = Compression(algorithm: algorithm, compressedLength: 0)
-                    try self.decompressor.initializeDecoder(encoding: algorithm)
+                    try self.decompressor.initializeDecoder()
                 }
 
                 context.fireChannelRead(data)

Tests/NIOHTTPCompressionTests/HTTPRequestDecompressorTest.swift

@@ -103,13 +103,20 @@ class HTTPRequestDecompressorTest: XCTestCase {
         try channel.pipeline.addHandler(NIOHTTPRequestDecompressor(limit: .none)).wait()
 
         let body = Array(repeating: testString, count: 1000).joined()
-
-        for algorithm in [nil, "gzip", "deflate"] {
+        let algorithms: [(actual: String, announced: String)?] = [
+            nil,
+            (actual: "gzip", announced: "gzip"),
+            (actual: "deflate", announced: "deflate"),
+            (actual: "gzip", announced: "deflate"),
+            (actual: "deflate", announced: "gzip"),
+        ]
+        
+        for algorithm in algorithms {
             let compressed: ByteBuffer
             var headers = HTTPHeaders()
             if let algorithm = algorithm {
-                headers.add(name: "Content-Encoding", value: algorithm)
-                compressed = compress(ByteBuffer.of(string: body), algorithm)
+                headers.add(name: "Content-Encoding", value: algorithm.announced)
+                compressed = compress(ByteBuffer.of(string: body), algorithm.actual)
             } else {
                 compressed = ByteBuffer.of(string: body)
             }

Tests/NIOHTTPCompressionTests/HTTPResponseDecompressorTest.swift

@@ -166,13 +166,20 @@ class HTTPResponseDecompressorTest: XCTestCase {
         for _ in 1...1000 {
             body += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."
         }
-
-        for algorithm in [nil, "gzip", "deflate"] {
+        let algorithms: [(actual: String, announced: String)?] = [
+            nil,
+            (actual: "gzip", announced: "gzip"),
+            (actual: "deflate", announced: "deflate"),
+            (actual: "gzip", announced: "deflate"),
+            (actual: "deflate", announced: "gzip"),
+        ]
+
+        for algorithm in algorithms {
             let compressed: ByteBuffer
             var headers = HTTPHeaders()
             if let algorithm = algorithm {
-                headers.add(name: "Content-Encoding", value: algorithm)
-                compressed = compress(ByteBuffer.of(string: body), algorithm)
+                headers.add(name: "Content-Encoding", value: algorithm.announced)
+                compressed = compress(ByteBuffer.of(string: body), algorithm.actual)
             } else {
                 compressed = ByteBuffer.of(string: body)
             }
@@ -213,13 +220,21 @@ class HTTPResponseDecompressorTest: XCTestCase {
         for _ in 1...1000 {
             body += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."
         }
-
-        for algorithm in [nil, "gzip", "deflate"] {
+        
+        let algorithms: [(actual: String, announced: String)?] = [
+            nil,
+            (actual: "gzip", announced: "gzip"),
+            (actual: "deflate", announced: "deflate"),
+            (actual: "gzip", announced: "deflate"),
+            (actual: "deflate", announced: "gzip"),
+        ]
+
+        for algorithm in algorithms {
             let compressed: ByteBuffer
             var headers = HTTPHeaders()
             if let algorithm = algorithm {
-                headers.add(name: "Content-Encoding", value: algorithm)
-                compressed = compress(ByteBuffer.of(string: body), algorithm)
+                headers.add(name: "Content-Encoding", value: algorithm.announced)
+                compressed = compress(ByteBuffer.of(string: body), algorithm.actual)
             } else {
                 compressed = ByteBuffer.of(string: body)
             }