Throw an overflow error on big header sizes

Author: rhpvorderman

src/isal/igzip.py

@@ -264,7 +264,7 @@ def decompress(data):
     gzip member is guaranteed to be present.
     """
     fp = io.BytesIO(data)
-    reader = _GzipReader(fp, len(data))
+    reader = _GzipReader(fp, max(len(data), 16))
     return reader.readall()
 
 

src/isal/isal_zlibmodule.c

@@ -1282,6 +1282,16 @@ GzipReader__new__(PyTypeObject *type, PyObject *args, PyObject *kwargs)
             args, kwargs, format, keywords, &fp, &buffer_size)) {
         return NULL;
     }
+    if (buffer_size < 16) {
+        // Necessary to distinguish between truncated headers and headers
+        // which are too big. A header is at least 10 bytes, but may contain
+        // more depending on flags.
+        PyErr_Format(
+            PyExc_ValueError,
+            "buffersize must be at least 16, got %zd", buffer_size
+        );
+        return NULL;
+    }
     GzipReader *self = PyObject_New(GzipReader, type);
     self->buffer_size = buffer_size;
     self->input_buffer = PyMem_Malloc(self->buffer_size);
@@ -1323,6 +1333,15 @@ static inline ssize_t GzipReader_read_from_file(GzipReader *self)
     current_pos = input_buffer;
     buffer_end = input_buffer + remaining;
     size_t read_in_size = self->buffer_size - remaining;
+    if (read_in_size == 0) {
+        // The buffer is already full of data but the current position could not
+        // progress. This happens when the header is too large.
+        PyErr_Format(
+            PyExc_OverflowError, 
+            "header does not fit into buffer of size %zu",
+            self->buffer_size);
+        return -1;
+    }
     PyObject *bufview = PyMemoryView_FromMemory((char *)buffer_end, read_in_size, PyBUF_WRITE);
     if (bufview == NULL) {
         return -1;

tests/test_igzip.py

@@ -431,6 +431,26 @@ def test_truncated_header(trunc):
     with pytest.raises(EOFError):
         igzip.decompress(trunc)
 
+def test_very_long_header_in_data():
+    # header with a very long filename.
+    header = (b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff" + 256 * 1024 * b"A" +
+              b"\x00")
+    compressed = header + isal_zlib.compress(b"", 3, -15) + 8 * b"\00"
+    assert igzip.decompress(compressed) == b""
+
+
+def test_very_long_header_in_file():
+    # header with a very long filename.
+    header = (b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff" +
+              igzip.READ_BUFFER_SIZE * 2 * b"A" +
+              b"\x00")
+    compressed = header + isal_zlib.compress(b"", 3, -15) + 8 * b"\00"
+    f = io.BytesIO(compressed)
+    with pytest.raises(OverflowError) as error:
+        with igzip.open(f, "rb") as gzip_file:
+            gzip_file.read()
+    error.match(f"header does not fit into buffer of size {igzip.READ_BUFFER_SIZE}")
+
 
 def test_concatenated_gzip():
     concat = Path(__file__).parent / "data" / "concatenated.fastq.gz"