Skip to content

Commit 56aa7e3

Browse files
committed
Fix multi-member gzip bug caused by bitbuffer and prepend method
Due to the bitbuffer the prepended data can be bigger than the original buffer. This was not expected by the _PaddedFile object
1 parent 57a5f15 commit 56aa7e3

File tree

1 file changed

+28
-3
lines changed

1 file changed

+28
-3
lines changed

src/isal/igzip.py

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import sys
3030
import time
3131
from typing import List, Optional, SupportsInt
32+
import _compression # noqa: I201 # Not third-party
3233

3334
from . import isal_zlib
3435

@@ -204,11 +205,35 @@ def write(self, data):
204205
return length
205206

206207

208+
class _PaddedFile(gzip._PaddedFile):
209+
# Overwrite _PaddedFile from gzip as its prepend method assumes that
210+
# the prepended data is always read from its _buffer. Unfortunately in
211+
# isal_zlib.decompressobj there is a bitbuffer as well which may be added.
212+
# So an extra check is added to prepend to ensure no extra data in front
213+
# of the buffer was present. (Negative self._read).
214+
def prepend(self, prepend=b''):
215+
if self._read is not None:
216+
# Assume data was read since the last prepend() call
217+
self._read -= len(prepend)
218+
if self._read >= 0:
219+
return
220+
# If self._read is negative the data was read further back and
221+
# the buffer needs to be reset.
222+
self._buffer = prepend
223+
self._length = len(self._buffer)
224+
self._read = 0
225+
226+
207227
class _IGzipReader(gzip._GzipReader):
208228
def __init__(self, fp):
209-
super().__init__(fp)
210-
self._decomp_factory = isal_zlib.decompressobj
211-
self._decompressor = self._decomp_factory(**self._decomp_args)
229+
# Call the init method of gzip._GzipReader's parent here.
230+
# It is not very invasive and allows us to override _PaddedFile
231+
_compression.DecompressReader.__init__(
232+
self, _PaddedFile(fp), isal_zlib.decompressobj,
233+
wbits=-isal_zlib.MAX_WBITS)
234+
# Set flag indicating start of a new member
235+
self._new_member = True
236+
self._last_mtime = None
212237

213238
def _add_read_data(self, data):
214239
# Use faster isal crc32 calculation and update the stream size in place

0 commit comments

Comments
 (0)