Remove igzipdecompressor

Author: rhpvorderman

src/isal/igzip_lib.pyx

@@ -22,7 +22,6 @@
 # cython: binding=True
 
 from libc.stdint cimport UINT64_MAX, UINT32_MAX
-from libc.string cimport memmove, memcpy
 from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
 from cpython.buffer cimport PyBUF_C_CONTIGUOUS, PyObject_GetBuffer, PyBuffer_Release
 from cpython.bytes cimport PyBytes_FromStringAndSize
@@ -258,170 +257,6 @@ cdef bytes view_bitbuffer(inflate_state * stream):
         # the correct order.
         return (read_in >> remainder).to_bytes(8, "little")[:read_in_length]
 
-cdef class IgzipDecompressor:
-    """Decompress object for handling streaming decompression."""
-    cdef public bytes unused_data
-    cdef public bint eof
-    cdef public bint needs_input
-    cdef inflate_state stream
-    cdef unsigned char * input_buffer
-    cdef size_t input_buffer_size
-    cdef Py_ssize_t avail_in_real
-
-    def __dealloc__(self):
-        if self.input_buffer != NULL:
-            PyMem_Free(self.input_buffer)
-
-    def __cinit__(self,
-                  flag=ISAL_DEFLATE,
-                  hist_bits=ISAL_DEF_MAX_HIST_BITS,
-                  zdict = None):
-        isal_inflate_init(&self.stream)
-
-        self.stream.hist_bits = hist_bits
-        self.stream.crc_flag = flag
-        cdef Py_ssize_t zdict_length
-        if zdict:
-            zdict_length = len(zdict)
-            if zdict_length > UINT32_MAX:
-                raise OverflowError("zdict length does not fit in an unsigned int")
-            err = isal_inflate_set_dict(&self.stream, zdict, zdict_length)
-            if err != COMP_OK:
-                check_isal_deflate_rc(err)
-        self.unused_data = b""
-        self.eof = False
-        self.input_buffer = NULL
-        self.input_buffer_size = 0
-        self.avail_in_real = 0
-        self.needs_input = True
-        
-    def _view_bitbuffer(self):
-        """Shows the 64-bitbuffer of the internal inflate_state. It contains
-        a maximum of 8 bytes. This data is already read-in so is not part
-        of the unconsumed tail."""
-        return view_bitbuffer(&self.stream)
-
-    cdef decompress_buf(self, Py_ssize_t max_length, unsigned char ** obuf):
-        obuf[0] = NULL
-        cdef Py_ssize_t obuflen = DEF_BUF_SIZE_I
-        cdef int err
-        if obuflen > max_length:
-            obuflen = max_length
-        while True:
-            obuflen = arrange_output_buffer_with_maximum(&self.stream, obuf, obuflen, max_length)
-            if obuflen == -1:
-                raise MemoryError("Unsufficient memory for buffer allocation")
-            elif obuflen == -2:
-                break
-            arrange_input_buffer(&self.stream, &self.avail_in_real)
-            err = isal_inflate(&self.stream)
-            self.avail_in_real += self.stream.avail_in
-            if err != ISAL_DECOMP_OK:
-                check_isal_inflate_rc(err)
-            if self.stream.block_state == ISAL_BLOCK_FINISH:
-                self.eof = 1
-                break
-            elif self.avail_in_real == 0:
-                break
-        return
-
-    def decompress(self, data, Py_ssize_t max_length = -1):
-        """
-        Decompress data, returning a bytes object containing the uncompressed
-        data corresponding to at least part of the data in string.
-
-        :param max_length: if non-zero then the return value will be no longer
-                           than max_length. Unprocessed data will be in the
-                           unconsumed_tail attribute.
-        """
-        if self.eof:
-            raise EOFError("End of stream already reached")
-        cdef bint input_buffer_in_use
-        
-        cdef Py_ssize_t hard_limit
-        if max_length < 0:
-            hard_limit = PY_SSIZE_T_MAX
-        else:
-            hard_limit = max_length
-
-        cdef unsigned int avail_now
-        cdef unsigned int avail_total
-        # Cython makes sure error is handled when acquiring buffer fails.
-        cdef Py_buffer buffer_data
-        cdef Py_buffer* buffer = &buffer_data
-        PyObject_GetBuffer(data, buffer, PyBUF_C_CONTIGUOUS)
-        cdef Py_ssize_t ibuflen = buffer.len
-        cdef unsigned char * data_ptr = <unsigned char*>buffer.buf
-
-
-        cdef bint max_length_reached = False
-        cdef unsigned char * tmp
-        cdef size_t offset
-        # Initialise output buffer
-        cdef unsigned char *obuf = NULL
-
-        try:
-            if self.stream.next_in != NULL:
-                avail_now = (self.input_buffer + self.input_buffer_size) - \
-                            (self.stream.next_in + self.avail_in_real)
-                avail_total = self.input_buffer_size - self.avail_in_real
-                if avail_total < ibuflen:
-                    offset = self.stream.next_in - self.input_buffer
-                    new_size = self.input_buffer_size + ibuflen - avail_now
-                    tmp = <unsigned char*>PyMem_Realloc(self.input_buffer, new_size)
-                    if tmp == NULL:
-                        raise MemoryError()
-                    self.input_buffer = tmp
-                    self.input_buffer_size = new_size
-                    self.stream.next_in = self.input_buffer + offset
-                elif avail_now < ibuflen:
-                    memmove(self.input_buffer, self.stream.next_in,
-                            self.avail_in_real)
-                    self.stream.next_in = self.input_buffer
-                memcpy(<void *>(self.stream.next_in + self.avail_in_real), data_ptr, buffer.len)
-                self.avail_in_real += ibuflen
-                input_buffer_in_use = 1
-            else:
-                self.stream.next_in = data_ptr
-                self.avail_in_real = ibuflen
-                input_buffer_in_use = 0
-
-            self.decompress_buf(hard_limit, &obuf)
-            if obuf == NULL:
-                self.stream.next_in = NULL
-                return b""
-            if self.eof:
-                self.needs_input = False
-                if self.avail_in_real > 0:
-                    new_data = PyBytes_FromStringAndSize(<char *>self.stream.next_in, self.avail_in_real)
-                    self.unused_data = self._view_bitbuffer() + new_data
-            elif self.avail_in_real == 0:
-                self.stream.next_in = NULL
-                self.needs_input = True
-            else:
-                self.needs_input = False
-                if not input_buffer_in_use:
-                    # Discard buffer if to small.
-                    # Resizing may needlessly copy the current contents.
-                    if self.input_buffer != NULL and self.input_buffer_size < self.avail_in_real:
-                        PyMem_Free(self.input_buffer)
-                        self.input_buffer = NULL
-
-                    # Allocate of necessary
-                    if self.input_buffer == NULL:
-                        self.input_buffer = <unsigned char *>PyMem_Malloc(self.avail_in_real)
-                        if self.input_buffer == NULL:
-                            raise MemoryError()
-                        self.input_buffer_size = self.avail_in_real
-
-                    # Copy tail
-                    memcpy(self.input_buffer, self.stream.next_in, self.avail_in_real)
-                    self.stream.next_in = self.input_buffer
-            return PyBytes_FromStringAndSize(<char*>obuf, self.stream.next_out - obuf)
-        finally:
-            PyBuffer_Release(buffer)
-            PyMem_Free(obuf)
-
 
 cdef int mem_level_to_bufsize(int compression_level, int mem_level, unsigned int *bufsize):
     """

tests/test_igzip_lib.py

@@ -19,12 +19,9 @@
 # SOFTWARE.
 
 import itertools
-import os
-import pickle
 from typing import NamedTuple
 
 from isal import igzip_lib
-from isal.igzip_lib import IgzipDecompressor
 
 import pytest
 
@@ -57,159 +54,3 @@ def test_compress_decompress(level, flag: Flag, hist_bits):
     comp = igzip_lib.compress(DATA, level, flag.comp, hist_bits)
     decomp = igzip_lib.decompress(comp, flag.decomp, hist_bits)
     assert decomp == DATA
-
-
-class TestIgzipDecompressor():
-    # Tests adopted from CPython's test_bz2.py
-    TEXT = DATA
-    DATA = igzip_lib.compress(DATA)
-    BAD_DATA = b"Not a valid deflate block"
-
-    def test_decompress(self):
-        decomp = IgzipDecompressor()
-        decompressed = decomp.decompress(self.DATA)
-        assert decompressed == self.TEXT
-
-    def testDecompressChunks10(self):
-        igzd = IgzipDecompressor()
-        text = b''
-        n = 0
-        while True:
-            str = self.DATA[n*10:(n+1)*10]
-            if not str:
-                break
-            text += igzd.decompress(str)
-            n += 1
-        assert text == self.TEXT
-
-    def testDecompressUnusedData(self):
-        igzd = IgzipDecompressor()
-        unused_data = b"this is unused data"
-        text = igzd.decompress(self.DATA+unused_data)
-        assert text == self.TEXT
-        assert igzd.unused_data == unused_data
-
-    def testEOFError(self):
-        igzd = IgzipDecompressor()
-        igzd.decompress(self.DATA)
-        with pytest.raises(EOFError):
-            igzd.decompress(b"anything")
-        with pytest.raises(EOFError):
-            igzd.decompress(b"")
-
-    def testDecompress4G(self):
-        # "Test igzdecompressor.decompress() with >4GiB input"
-        size = 4 * 1024 ** 3 + 100  # 4 GiB + 100
-        blocksize = 10 * 1024 * 1024
-        block = os.urandom(blocksize)
-        try:
-            data = block * (size // blocksize + 1)
-            compressed = igzip_lib.compress(data)
-            igzd = IgzipDecompressor()
-            decompressed = igzd.decompress(compressed)
-            assert decompressed == data
-        finally:
-            data = None
-            compressed = None
-            decompressed = None
-
-    def testPickle(self):
-        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
-            with pytest.raises(TypeError):
-                pickle.dumps(IgzipDecompressor(), proto)
-
-    def testDecompressorChunksMaxsize(self):
-        igzd = IgzipDecompressor()
-        max_length = 100
-        out = []
-
-        # Feed some input
-        len_ = len(self.DATA) - 64
-        out.append(igzd.decompress(self.DATA[:len_],
-                                   max_length=max_length))
-        assert not igzd.needs_input
-        assert len(out[-1]) == max_length
-
-        # Retrieve more data without providing more input
-        out.append(igzd.decompress(b'', max_length=max_length))
-        assert not igzd.needs_input
-        assert len(out[-1]) == max_length
-
-        # Retrieve more data while providing more input
-        out.append(igzd.decompress(self.DATA[len_:],
-                                   max_length=max_length))
-        assert len(out[-1]) == max_length
-
-        # Retrieve remaining uncompressed data
-        while not igzd.eof:
-            out.append(igzd.decompress(b'', max_length=max_length))
-            assert len(out[-1]) <= max_length
-
-        out = b"".join(out)
-        assert out == self.TEXT
-        assert igzd.unused_data == b""
-
-    def test_decompressor_inputbuf_1(self):
-        # Test reusing input buffer after moving existing
-        # contents to beginning
-        igzd = IgzipDecompressor()
-        out = []
-
-        # Create input buffer and fill it
-        assert igzd.decompress(self.DATA[:100], max_length=0) == b''
-
-        # Retrieve some results, freeing capacity at beginning
-        # of input buffer
-        out.append(igzd.decompress(b'', 2))
-
-        # Add more data that fits into input buffer after
-        # moving existing data to beginning
-        out.append(igzd.decompress(self.DATA[100:105], 15))
-
-        # Decompress rest of data
-        out.append(igzd.decompress(self.DATA[105:]))
-        assert b''.join(out) == self.TEXT
-
-    def test_decompressor_inputbuf_2(self):
-        # Test reusing input buffer by appending data at the
-        # end right away
-        igzd = IgzipDecompressor()
-        out = []
-
-        # Create input buffer and empty it
-        assert igzd.decompress(self.DATA[:200], max_length=0) == b''
-        out.append(igzd.decompress(b''))
-
-        # Fill buffer with new data
-        out.append(igzd.decompress(self.DATA[200:280], 2))
-
-        # Append some more data, not enough to require resize
-        out.append(igzd.decompress(self.DATA[280:300], 2))
-
-        # Decompress rest of data
-        out.append(igzd.decompress(self.DATA[300:]))
-        assert b''.join(out) == self.TEXT
-
-    def test_decompressor_inputbuf_3(self):
-        # Test reusing input buffer after extending it
-
-        igzd = IgzipDecompressor()
-        out = []
-
-        # Create almost full input buffer
-        out.append(igzd.decompress(self.DATA[:200], 5))
-
-        # Add even more data to it, requiring resize
-        out.append(igzd.decompress(self.DATA[200:300], 5))
-
-        # Decompress rest of data
-        out.append(igzd.decompress(self.DATA[300:]))
-        assert b''.join(out) == self.TEXT
-
-    def test_failure(self):
-        igzd = IgzipDecompressor()
-        with pytest.raises(Exception):
-            igzd.decompress(self.BAD_DATA * 30)
-        # Make sure there are no internal consistencies
-        with pytest.raises(Exception):
-            igzd.decompress(self.BAD_DATA * 30)