Skip to content

Commit 30d85fd

Browse files
committed
Remove repeated comments for a general overview
1 parent f366401 commit 30d85fd

File tree

1 file changed

+43
-38
lines changed

1 file changed

+43
-38
lines changed

src/isal/isal_zlib.pyx

Lines changed: 43 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,45 @@
2424
"""
2525
Implementation of the zlib module using the ISA-L libraries.
2626
"""
27+
###############################################################################
28+
### README FOR DEVELOPERS: IMPLEMENTATION OF THIS MODULE ###
29+
###############################################################################
30+
#
31+
# This module is implemented with zlibmodule.c as example. Since there is a lot
32+
# of duplication in zlibmodule.c there is a lot of duplication in this module.
33+
# It is not always possible to route repetitive calls to a subroutine.
34+
# Therefore the main methods are explained here.
35+
#
36+
# All compress, decompress and flush implementations are basically the same.
37+
# 1. Get a buffer from the input data
38+
# 2. Initialise an output buffer
39+
# 3. Initialise a isal_zstream(for compression) or inflate_state (for
40+
# decompression). Hereafter referred as stream.
41+
# 4. The amount of available input bytes is set on the stream. This is either
42+
# the maximum amount possible (in the case the input data is equal or larger
43+
# than the maximum amount). Or the length of the (remaining) input data.
44+
# 5. The amount of available output bytes is set on the stream. Buffer is
45+
# enlarged as needed.
46+
# 6. inflate/deflate/flush action
47+
# 7. Check for errors in the action.
48+
# 8. Was the output buffer completely filled? (stream.avail_out == 0). If so go
49+
# to 5. Special case: decompressobj. If the output buffer is at max_length
50+
# continue.
51+
# 9. Was all the input read? if not go to 4. Alternatively in the case of
52+
# decompression: was the end of the stream reached? if not go to 4.
53+
# 10. In case of decompression with leftover input data. For a decompressobj
54+
# this is stored in unconsumed_tail / unused_data.
55+
# 11. Convert output buffer to bytes object and return.
56+
#
57+
# Errors are raised in the main functions as much as possible to prevent cdef
58+
# functions returning PyObjects that need to be transformed in C variables.
59+
# In cases where this is not possible, C variables were set using pointers.
60+
# Allowing repeated use of functions while limiting the number of python
61+
# interactions.
62+
#
63+
###############################################################################
64+
65+
2766
import warnings
2867
import zlib
2968

@@ -139,6 +178,9 @@ cdef Py_ssize_t arrange_output_buffer_with_maximum(stream_or_state *stream,
139178
unsigned char **buffer,
140179
Py_ssize_t length,
141180
Py_ssize_t max_length):
181+
# The zlibmodule.c function builds a PyBytes object. A unsigned char *
182+
# is build here because building raw PyObject * stuff in cython is somewhat
183+
# harder due to cython's interference. FIXME
142184
cdef Py_ssize_t occupied
143185
cdef Py_ssize_t new_length
144186
cdef unsigned char * new_buffer
@@ -225,31 +267,21 @@ def compress(data,
225267
# initialise helper variables
226268
cdef int err
227269

228-
# Implementation imitated from CPython's zlibmodule.c
229270
try:
230271
while True:
231-
# This loop runs n times (at least twice). n-1 times to fill the input
232-
# buffer with data. The nth time the input is empty. In that case
233-
# stream.flush is set to FULL_FLUSH and the end_of_stream is activated.
234272
arrange_input_buffer(&stream, &ibuflen)
235273
if ibuflen == 0:
236274
stream.flush = FULL_FLUSH
237275
stream.end_of_stream = 1
238276
else:
239277
stream.flush = NO_FLUSH
240278

241-
# This loop reads all the input bytes. The check is at the end,
242-
# because when flush = FULL_FLUSH the input buffer is empty. But
243-
# this loop still needs to run one time.
244279
while True:
245280
bufsize = arrange_output_buffer(&stream, &obuf, bufsize)
246281
if bufsize == -1:
247282
raise MemoryError("Unsufficient memory for buffer allocation")
248283
err = isal_deflate(&stream)
249284
if err != COMP_OK:
250-
# There is some python interacting when possible exceptions
251-
# Are raised. So we remain in pure C code if we check for
252-
# COMP_OK first.
253285
check_isal_deflate_rc(err)
254286
if stream.avail_out != 0:
255287
break
@@ -302,23 +334,16 @@ def decompress(data,
302334
cdef unsigned char * obuf = NULL
303335
cdef int err
304336

305-
# Implementation imitated from CPython's zlibmodule.c
306337
try:
307338
while True:
308339
arrange_input_buffer(&stream, &ibuflen)
309340

310-
# This loop reads all the input bytes. The check is at the end,
311-
# because when the block state is not at FINISH, the function needs
312-
# to be called again.
313341
while True:
314342
bufsize = arrange_output_buffer(&stream, &obuf, bufsize)
315343
if bufsize == -1:
316344
raise MemoryError("Unsufficient memory for buffer allocation")
317345
err = isal_inflate(&stream)
318346
if err != ISAL_DECOMP_OK:
319-
# There is some python interacting when possible exceptions
320-
# Are raised. So we remain in pure C code if we check for
321-
# COMP_OK first.
322347
check_isal_inflate_rc(err)
323348
if stream.avail_out != 0:
324349
break
@@ -445,19 +470,13 @@ cdef class Compress:
445470
cdef int err
446471
try:
447472
while True:
448-
# This loop runs n times (at least twice). n-1 times to fill the input
449-
# buffer with data. The nth time the input is empty. In that case
450-
# stream.flush is set to FULL_FLUSH and the end_of_stream is activated.
451473
arrange_input_buffer(&self.stream, &ibuflen)
452474
while True:
453475
obuflen = arrange_output_buffer(&self.stream, &obuf, obuflen)
454476
if obuflen== -1:
455477
raise MemoryError("Unsufficient memory for buffer allocation")
456478
err = isal_deflate(&self.stream)
457479
if err != COMP_OK:
458-
# There is some python interacting when possible exceptions
459-
# Are raised. So we remain in pure C code if we check for
460-
# COMP_OK first.
461480
check_isal_deflate_rc(err)
462481
if self.stream.avail_out != 0:
463482
break
@@ -504,11 +523,8 @@ cdef class Compress:
504523
raise MemoryError("Unsufficient memory for buffer allocation")
505524
err = isal_deflate(&self.stream)
506525
if err != COMP_OK:
507-
# There is some python interacting when possible exceptions
508-
# Are raised. So we remain in pure C code if we check for
509-
# COMP_OK first.
510526
check_isal_deflate_rc(err)
511-
if self.stream.avail_out != 0: # All input is processed and therefore all output flushed.
527+
if self.stream.avail_out != 0:
512528
break
513529
if self.stream.avail_in != 0:
514530
raise AssertionError("There should be no available input after flushing.")
@@ -629,8 +645,6 @@ cdef class Decompress:
629645
obuflen = hard_limit
630646

631647
try:
632-
# This loop reads all the input bytes. If there are no input bytes
633-
# anymore the output is written.
634648
while True:
635649
arrange_input_buffer(&self.stream, &ibuflen)
636650
while True:
@@ -643,9 +657,6 @@ cdef class Decompress:
643657
break
644658
err = isal_inflate(&self.stream)
645659
if err != ISAL_DECOMP_OK:
646-
# There is some python interacting when possible exceptions
647-
# Are raised. So we remain in pure C code if we check for
648-
# COMP_OK first.
649660
check_isal_inflate_rc(err)
650661
if self.stream.block_state == ISAL_BLOCK_FINISH or self.stream.avail_out != 0:
651662
break
@@ -688,13 +699,7 @@ cdef class Decompress:
688699
raise MemoryError("Unsufficient memory for buffer allocation")
689700
err = isal_inflate(&self.stream)
690701
if err != ISAL_DECOMP_OK:
691-
# There is some python interacting when possible exceptions
692-
# Are raised. So we remain in pure C code if we check for
693-
# COMP_OK first.
694702
check_isal_inflate_rc(err)
695-
# Instead of output buffer resizing as the zlibmodule.c example
696-
# the data is appended to a list.
697-
# TODO: Improve this with the buffer protocol.
698703
if self.stream.avail_out != 0 or self.stream.block_state == ISAL_BLOCK_FINISH:
699704
break
700705
if self.stream.block_state == ISAL_BLOCK_FINISH or ibuflen == 0:

0 commit comments

Comments
 (0)