Switch Buffers to memoryviews & remove extra copies/allocations (#656)

When this was written in the code, Python's Buffer Protocol support was inconsistent across Python versions (specifically on Python 2.7). Since Python 2.7 reached EOL and it was dropped from Numcodecs, the Python Buffer Protocol support has become more consistent.

At this stage the `memoryview` object, which Cython also supports, does all the same things that `Buffer` would do for us. Plus it is builtin to the Python standard library. It behaves similarly in a lot of ways.

Given this, switch the code over to `memoryview`s internally and drop `Buffer`.

<hr>

Additionally have pushed changes to this PR to improve overall memory usage. This eliminates some unneeded copies that occurred at the ended of some codecs. Also have eliminated some temporary allocations used in some codec pipelines by allocating output buffers earlier and changing operations to act in-place. This should eliminate some spiky behavior seen recently with codecs.

Author: jakirkham

docs/release.rst

@@ -37,6 +37,13 @@ Fixes
 ~~~~~
 * Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs.
   By :user:`Norman Rzepka <normanrz>`, :issue:`685`
+* Preallocate output buffers and resize directly as needed.
+  By :user:`John Kirkham <jakirkham>`, :issue:`656`
+
+Maintenance
+~~~~~~~~~~~
+* Replace internal ``Buffer`` usage with ``memoryview``\ s.
+  By :user:`John Kirkham <jakirkham>`, :issue:`656`
 
 .. _release_0.15.0:
 

numcodecs/blosc.pyx

@@ -9,12 +9,11 @@ import os
 from deprecated import deprecated
 
 
-from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS, PyBUF_WRITEABLE
-from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING
+from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize
+from cpython.memoryview cimport PyMemoryView_GET_BUFFER
 
+from .compat_ext cimport PyBytes_RESIZE, ensure_continguous_memoryview
 
-from .compat_ext cimport Buffer
-from .compat_ext import Buffer
 from .compat import ensure_contiguous_ndarray
 from .abc import Codec
 
@@ -154,17 +153,16 @@ def _cbuffer_sizes(source):
 
     """
     cdef:
-        Buffer buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
         size_t nbytes, cbytes, blocksize
 
-    # obtain buffer
-    buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
 
     # determine buffer size
-    blosc_cbuffer_sizes(buffer.ptr, &nbytes, &cbytes, &blocksize)
-
-    # release buffers
-    buffer.release()
+    blosc_cbuffer_sizes(source_pb.buf, &nbytes, &cbytes, &blocksize)
 
     return nbytes, cbytes, blocksize
 
@@ -173,16 +171,15 @@ cbuffer_sizes = deprecated(_cbuffer_sizes)
 def cbuffer_complib(source):
     """Return the name of the compression library used to compress `source`."""
     cdef:
-        Buffer buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
 
-    # obtain buffer
-    buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
 
     # determine buffer size
-    complib = blosc_cbuffer_complib(buffer.ptr)
-
-    # release buffers
-    buffer.release()
+    complib = blosc_cbuffer_complib(source_pb.buf)
 
     complib = complib.decode('ascii')
 
@@ -202,18 +199,17 @@ def _cbuffer_metainfo(source):
 
     """
     cdef:
-        Buffer buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
         size_t typesize
         int flags
 
-    # obtain buffer
-    buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
 
     # determine buffer size
-    blosc_cbuffer_metainfo(buffer.ptr, &typesize, &flags)
-
-    # release buffers
-    buffer.release()
+    blosc_cbuffer_metainfo(source_pb.buf, &typesize, &flags)
 
     # decompose flags
     if flags & BLOSC_DOSHUFFLE:
@@ -263,28 +259,34 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE,
     """
 
     cdef:
-        char *source_ptr
-        char *dest_ptr
-        Buffer source_buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
+        const char* source_ptr
         size_t nbytes, itemsize
         int cbytes
         bytes dest
+        char* dest_ptr
 
     # check valid cname early
     cname_str = cname.decode('ascii')
     if cname_str not in list_compressors():
         _err_bad_cname(cname_str)
 
-    # setup source buffer
-    source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
-    source_ptr = source_buffer.ptr
-    nbytes = source_buffer.nbytes
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
+
+    # extract metadata
+    source_ptr = <const char*>source_pb.buf
+    nbytes = source_pb.len
+
+    # validate typesize
     if isinstance(typesize, int):
         if typesize < 1:
             raise ValueError(f"Cannot use typesize {typesize} less than 1.")
         itemsize = typesize
     else:
-        itemsize = source_buffer.itemsize
+        itemsize = source_pb.itemsize
 
     # determine shuffle
     if shuffle == AUTOSHUFFLE:
@@ -333,16 +335,14 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE,
                                             cname, blocksize, 1)
 
     finally:
-
-        # release buffers
-        source_buffer.release()
+        pass
 
     # check compression was successful
     if cbytes <= 0:
         raise RuntimeError('error during blosc compression: %d' % cbytes)
 
     # resize after compression
-    dest = dest[:cbytes]
+    PyBytes_RESIZE(dest, cbytes)
 
     return dest
 
@@ -366,30 +366,36 @@ def decompress(source, dest=None):
     """
     cdef:
         int ret
-        char *source_ptr
-        char *dest_ptr
-        Buffer source_buffer
-        Buffer dest_buffer = None
+        memoryview source_mv
+        const Py_buffer* source_pb
+        const char* source_ptr
+        memoryview dest_mv
+        Py_buffer* dest_pb
+        char* dest_ptr
         size_t nbytes, cbytes, blocksize
 
-    # setup source buffer
-    source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
-    source_ptr = source_buffer.ptr
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
+
+    # get source pointer
+    source_ptr = <const char*>source_pb.buf
 
     # determine buffer size
     blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize)
 
     # setup destination buffer
     if dest is None:
         # allocate memory
-        dest = PyBytes_FromStringAndSize(NULL, nbytes)
-        dest_ptr = PyBytes_AS_STRING(dest)
-        dest_nbytes = nbytes
+        dest_1d = dest = PyBytes_FromStringAndSize(NULL, nbytes)
     else:
-        arr = ensure_contiguous_ndarray(dest)
-        dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
-        dest_ptr = dest_buffer.ptr
-        dest_nbytes = dest_buffer.nbytes
+        dest_1d = ensure_contiguous_ndarray(dest)
+
+    # obtain dest memoryview
+    dest_mv = memoryview(dest_1d)
+    dest_pb = PyMemoryView_GET_BUFFER(dest_mv)
+    dest_ptr = <char*>dest_pb.buf
+    dest_nbytes = dest_pb.len
 
     try:
 
@@ -408,11 +414,7 @@ def decompress(source, dest=None):
                 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1)
 
     finally:
-
-        # release buffers
-        source_buffer.release()
-        if dest_buffer is not None:
-            dest_buffer.release()
+        pass
 
     # handle errors
     if ret <= 0:
@@ -449,14 +451,20 @@ def _decompress_partial(source, start, nitems, dest=None):
         int encoding_size
         int nitems_bytes
         int start_bytes
-        char *source_ptr
-        char *dest_ptr
-        Buffer source_buffer
-        Buffer dest_buffer = None
+        memoryview source_mv
+        const Py_buffer* source_pb
+        const char* source_ptr
+        memoryview dest_mv
+        Py_buffer* dest_pb
+        char* dest_ptr
+        size_t dest_nbytes
 
-    # setup source buffer
-    source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
-    source_ptr = source_buffer.ptr
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
+
+    # setup source pointer
+    source_ptr = <const char*>source_pb.buf
 
     # get encoding size from source buffer header
     encoding_size = source[3]
@@ -467,26 +475,25 @@ def _decompress_partial(source, start, nitems, dest=None):
 
     # setup destination buffer
     if dest is None:
-        dest = PyBytes_FromStringAndSize(NULL, nitems_bytes)
-        dest_ptr = PyBytes_AS_STRING(dest)
-        dest_nbytes = nitems_bytes
+        # allocate memory
+        dest_1d = dest = PyBytes_FromStringAndSize(NULL, nitems_bytes)
     else:
-        arr = ensure_contiguous_ndarray(dest)
-        dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
-        dest_ptr = dest_buffer.ptr
-        dest_nbytes = dest_buffer.nbytes
+        dest_1d = ensure_contiguous_ndarray(dest)
+
+    # obtain dest memoryview
+    dest_mv = memoryview(dest_1d)
+    dest_pb = PyMemoryView_GET_BUFFER(dest_mv)
+    dest_ptr = <char*>dest_pb.buf
+    dest_nbytes = dest_pb.len
 
     # try decompression
     try:
         if dest_nbytes < nitems_bytes:
             raise ValueError('destination buffer too small; expected at least %s, '
                              'got %s' % (nitems_bytes, dest_nbytes))
         ret = blosc_getitem(source_ptr, start, nitems, dest_ptr)
-
     finally:
-        source_buffer.release()
-        if dest_buffer is not None:
-            dest_buffer.release()
+        pass
 
     # ret refers to the number of bytes returned from blosc_getitem.
     if ret <= 0:

numcodecs/compat_ext.pxd

@@ -1,12 +1,11 @@
 # cython: language_level=3
 
 
-cdef class Buffer:
-    cdef:
-        char *ptr
-        Py_buffer buffer
-        size_t nbytes
-        size_t itemsize
-        bint acquired
-
-    cpdef release(self)
+cdef extern from *:
+    """
+    #define PyBytes_RESIZE(b, n) _PyBytes_Resize(&b, n)
+    """
+    int PyBytes_RESIZE(object b, Py_ssize_t n) except -1
+
+
+cpdef memoryview ensure_continguous_memoryview(obj)

numcodecs/compat_ext.pyx

@@ -3,26 +3,17 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
-from cpython.buffer cimport PyObject_GetBuffer, PyBuffer_Release
 
+from cpython.buffer cimport PyBuffer_IsContiguous
+from cpython.memoryview cimport PyMemoryView_GET_BUFFER
 
-from .compat import ensure_contiguous_ndarray
 
-
-cdef class Buffer:
-    """Convenience class for buffer interface."""
-
-    def __cinit__(self, obj, int flags):
-        PyObject_GetBuffer(obj, &(self.buffer), flags)
-        self.acquired = True
-        self.ptr = <char *> self.buffer.buf
-        self.itemsize = self.buffer.itemsize
-        self.nbytes = self.buffer.len
-
-    cpdef release(self):
-        if self.acquired:
-            PyBuffer_Release(&(self.buffer))
-            self.acquired = False
-
-    def __dealloc__(self):
-        self.release()
+cpdef memoryview ensure_continguous_memoryview(obj):
+    cdef memoryview mv
+    if type(obj) is memoryview:
+        mv = <memoryview>obj
+    else:
+        mv = memoryview(obj)
+    if not PyBuffer_IsContiguous(PyMemoryView_GET_BUFFER(mv), b'A'):
+        raise BufferError("Expected contiguous memory")
+    return mv