Merge pull request #103 from pycompression/py3.11_2

Make code compatible with python 3.11 onwards

Author: rhpvorderman

src/isal/igzip.py

@@ -28,7 +28,7 @@
 import struct
 import sys
 import time
-from typing import List, Optional, SupportsInt
+from typing import Optional, SupportsInt
 import _compression  # noqa: I201  # Not third-party
 
 from . import igzip_lib, isal_zlib
@@ -49,7 +49,7 @@
 try:
     BadGzipFile = gzip.BadGzipFile  # type: ignore
 except AttributeError:  # Versions lower than 3.8 do not have BadGzipFile
-    BadGzipFile = OSError
+    BadGzipFile = OSError  # type: ignore
 
 
 # The open method was copied from the CPython source with minor adjustments.
@@ -349,11 +349,13 @@ def _gzip_header_end(data: bytes) -> int:
         raise BadGzipFile(f"Not a gzipped file ({repr(data[:2])})")
     if method != 8:
         raise BadGzipFile("Unknown compression method")
+    if not flags:  # Likely when data compressed in memory
+        return 10
     pos = 10
     if flags & FEXTRA:
         if len(data) < pos + 2:
             raise eof_error
-        xlen = int.from_bytes(data[pos: pos + 2], "little", signed=False)
+        xlen, = struct.unpack("<H", data[pos: pos+2])
         pos += 2 + xlen
     if flags & FNAME:
         pos = data.find(b"\x00", pos) + 1
@@ -367,12 +369,12 @@ def _gzip_header_end(data: bytes) -> int:
     if flags & FHCRC:
         if len(data) < pos + 2:
             raise eof_error
-        header_crc = int.from_bytes(data[pos: pos + 2], "little", signed=False)
+        header_crc, = struct.unpack("<H", data[pos: pos+2])
         # CRC is stored as a 16-bit integer by taking last bits of crc32.
         crc = isal_zlib.crc32(data[:pos]) & 0xFFFF
         if header_crc != crc:
-            raise BadGzipFile(f"Corrupted header. Checksums do not "
-                              f"match: {crc} != {header_crc}")
+            raise BadGzipFile(f"Corrupted gzip header. Checksums do not "
+                              f"match: {crc:04x} != {header_crc:04x}")
         pos += 2
     return pos
 
@@ -381,26 +383,25 @@ def decompress(data):
     """Decompress a gzip compressed string in one shot.
     Return the decompressed string.
     """
-    all_blocks: List[bytes] = []
+    decompressed_members = []
     while True:
-        if data == b"":
-            break
+        if not data:  # Empty data returns empty bytestring
+            return b"".join(decompressed_members)
         header_end = _gzip_header_end(data)
-        do = isal_zlib.decompressobj(-15)
-        block = do.decompress(data[header_end:]) + do.flush()
+        # Use a zlib raw deflate compressor
+        do = isal_zlib.decompressobj(wbits=-isal_zlib.MAX_WBITS)
+        # Read all the data except the header
+        decompressed = do.decompress(data[header_end:])
         if not do.eof or len(do.unused_data) < 8:
             raise EOFError("Compressed file ended before the end-of-stream "
                            "marker was reached")
-        checksum, length = struct.unpack("<II", do.unused_data[:8])
-        crc = isal_zlib.crc32(block)
-        if crc != checksum:
+        crc, length = struct.unpack("<II", do.unused_data[:8])
+        if crc != isal_zlib.crc32(decompressed):
             raise BadGzipFile("CRC check failed")
-        if length != len(block):
+        if length != (len(decompressed) & 0xffffffff):
             raise BadGzipFile("Incorrect length of data produced")
-        all_blocks.append(block)
-        # Remove all padding null bytes and start next block.
+        decompressed_members.append(decompressed)
         data = do.unused_data[8:].lstrip(b"\x00")
-    return b"".join(all_blocks)
 
 
 def _argument_parser():

tests/test_igzip.py

@@ -27,6 +27,7 @@
 import os
 import re
 import shutil
+import struct
 import sys
 import tempfile
 import zlib
@@ -343,19 +344,23 @@ def test_header_too_short():
 
 
 def test_header_corrupt():
-    header = b"\x1f\x8b\x08\x02\x00\x00\x00\x00\x00\xff"
+    header = (b"\x1f\x8b\x08\x1f\x00\x00\x00\x00\x00\xff"  # All flags set
+              b"\x05\x00"  # Xlen = 5
+              b"extra"
+              b"name\x00"
+              b"comment\x00")
     # Create corrupt checksum by using wrong seed.
     crc = zlib.crc32(header, 50) & 0xFFFF
     true_crc = zlib.crc32(header) & 0xFFFF
-    header += crc.to_bytes(2, "little")
+    header += struct.pack("<H", crc)
 
     data = isal_zlib.compress(b"", wbits=-15)
     trailer = b"\x00" * 8
     compressed = header + data + trailer
     with pytest.raises(igzip.BadGzipFile) as error:
         igzip.decompress(compressed)
-    error.match(f"Corrupted header. "
-                f"Checksums do not match: {true_crc} != {crc}")
+    error.match(f"Corrupted gzip header. "
+                f"Checksums do not match: {true_crc:04x} != {crc:04x}")
 
 
 TRUNCATED_HEADERS = [