Merge pull request #63 from pycompression/release_0.8.1

Release 0.8.1

Author: rhpvorderman

.github/workflows/ci.yml

@@ -81,7 +81,9 @@ jobs:
         run: sudo apt install yasm
         if: runner.os == 'Linux'
       - name: Install build dependencies (Macos)
-        run: brew install nasm automake autoconf
+        # Install yasm because nasm does not work when building wheels.
+        # Probably because of nasm-filter.sh not filtering all flags that can not be used.
+        run: brew install yasm automake autoconf
         if: runner.os == 'macOS'
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]
@@ -141,7 +143,7 @@ jobs:
       - name: Install cibuildwheel twine wheel
         run: python -m pip install cibuildwheel twine wheel
       - name: Install build dependencies (Macos)
-        run: brew install nasm automake autoconf
+        run: brew install yasm automake autoconf
         if: runner.os == 'macOS'
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]

CHANGELOG.rst

@@ -7,6 +7,12 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.8.1
+-----------------
++ Fix a bug where multi-member gzip files where read incorrectly due to an
+  offset error. This was caused by ISA-L's decompressobj having a small
+  bitbuffer which was not taken properly into account in some circumstances.
+
 version 0.8.0
 -----------------
 + Speed up ``igzip.compress`` and ``igzip.decompress`` by improving the

setup.py

@@ -169,7 +169,7 @@ def build_isa_l(compiler_command: str, compiler_options: str):
 
 setup(
     name="isal",
-    version="0.8.0",
+    version="0.8.1",
     description="Faster zlib and gzip compatible compression and "
                 "decompression by providing python bindings for the ISA-L "
                 "library.",
@@ -197,7 +197,7 @@ def build_isa_l(compiler_command: str, compiler_options: str):
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Cython",
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 4 - Beta",
         "Topic :: System :: Archiving :: Compression",
         "License :: OSI Approved :: MIT License",
         "Operating System :: POSIX :: Linux",

src/isal/__init__.py

@@ -39,4 +39,4 @@
     "__version__"
 ]
 
-__version__ = "0.8.0"
+__version__ = "0.8.1"

src/isal/igzip.py

@@ -29,6 +29,7 @@
 import sys
 import time
 from typing import List, Optional, SupportsInt
+import _compression  # noqa: I201  # Not third-party
 
 from . import isal_zlib
 
@@ -204,11 +205,35 @@ def write(self, data):
         return length
 
 
+class _PaddedFile(gzip._PaddedFile):
+    # Overwrite _PaddedFile from gzip as its prepend method assumes that
+    # the prepended data is always read from its _buffer. Unfortunately in
+    # isal_zlib.decompressobj there is a bitbuffer as well which may be added.
+    # So an extra check is added to prepend to ensure no extra data in front
+    # of the buffer was present. (Negative self._read).
+    def prepend(self, prepend=b''):
+        if self._read is not None:
+            # Assume data was read since the last prepend() call
+            self._read -= len(prepend)
+            if self._read >= 0:
+                return
+            # If self._read is negative the data was read further back and
+            # the buffer needs to be reset.
+        self._buffer = prepend
+        self._length = len(self._buffer)
+        self._read = 0
+
+
 class _IGzipReader(gzip._GzipReader):
     def __init__(self, fp):
-        super().__init__(fp)
-        self._decomp_factory = isal_zlib.decompressobj
-        self._decompressor = self._decomp_factory(**self._decomp_args)
+        # Call the init method of gzip._GzipReader's parent here.
+        # It is not very invasive and allows us to override _PaddedFile
+        _compression.DecompressReader.__init__(
+            self, _PaddedFile(fp), isal_zlib.decompressobj,
+            wbits=-isal_zlib.MAX_WBITS)
+        # Set flag indicating start of a new member
+        self._new_member = True
+        self._last_mtime = None
 
     def _add_read_data(self, data):
         # Use faster isal crc32 calculation and update the stream size in place

tests/data/README.md

@@ -12,3 +12,9 @@ dwgsim -N 10000 -e 0.0010 -E 0.0010 -1 152 -2 152 -c 0 -S1 -z 1 -Q 5 \
 reference.fasta test
 gzip -c -9 test.bwa.read1.fastq > test.fastq.gz
 ```
+
+concatenated.fastq.gz was created with:
+```
+gzip -cd tests/data/test.fastq.gz | head -n 1000 | gzip -c -1 >> test2.fastq.gz
+gzip -cd tests/data/test.fastq.gz | head -n 1000 | gzip -c -1 >> test2.fastq.gz
+```

tests/data/concatenated.fastq.gz

@@ -288,3 +288,11 @@ def test_header_corrupt():
 def test_truncated_header(trunc):
     with pytest.raises(EOFError):
         igzip.decompress(trunc)
+
+
+def test_concatenated_gzip():
+    concat = Path(__file__).parent / "data" / "concatenated.fastq.gz"
+    data = gzip.decompress(concat.read_bytes())
+    with igzip.open(concat, "rb") as igzip_h:
+        result = igzip_h.read()
+    assert data == result