Skip to content

Commit 07c7001

Browse files
FrancescAltedFrancescAlted
committed
Use buffers in the stack for the buffer protocol
1 parent acb5f88 commit 07c7001

File tree

1 file changed

+58
-72
lines changed

1 file changed

+58
-72
lines changed

‎src/blosc2/blosc2_ext.pyx‎

Lines changed: 58 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -642,8 +642,8 @@ def cbuffer_sizes(src):
642642
cpdef compress(src, int32_t typesize=8, int clevel=9, filter=blosc2.Filter.SHUFFLE, codec=blosc2.Codec.BLOSCLZ):
643643
set_compressor(codec)
644644
cdef int32_t len_src = <int32_t> len(src)
645-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
646-
PyObject_GetBuffer(src, buf, PyBUF_SIMPLE)
645+
cdef Py_buffer buf
646+
PyObject_GetBuffer(src, &buf, PyBUF_SIMPLE)
647647
dest = bytes(buf.len + BLOSC2_MAX_OVERHEAD)
648648
cdef int32_t len_dest = <int32_t> len(dest)
649649
cdef int size
@@ -654,8 +654,7 @@ cpdef compress(src, int32_t typesize=8, int clevel=9, filter=blosc2.Filter.SHUFF
654654
size = blosc2_compress(clevel, filter_, <int32_t> typesize, buf.buf, <int32_t> buf.len, _dest, len_dest)
655655
else:
656656
size = blosc2_compress(clevel, filter_, <int32_t> typesize, buf.buf, <int32_t> buf.len, <void*> <char *> dest, len_dest)
657-
PyBuffer_Release(buf)
658-
free(buf)
657+
PyBuffer_Release(&buf)
659658
if size > 0:
660659
return dest[:size]
661660
else:
@@ -672,14 +671,13 @@ def decompress(src, dst=None, as_bytearray=False):
672671
typed_view_src = mem_view_src.cast('B')
673672
_check_comp_length('src', len(typed_view_src))
674673
blosc2_cbuffer_sizes(<void*>&typed_view_src[0], &nbytes, &cbytes, &blocksize)
675-
cdef Py_buffer *buf
674+
cdef Py_buffer buf
676675
if dst is not None:
677-
buf = <Py_buffer *> malloc(sizeof(Py_buffer))
678-
PyObject_GetBuffer(dst, buf, PyBUF_SIMPLE)
676+
PyObject_GetBuffer(dst, &buf, PyBUF_SIMPLE)
679677
if buf.len == 0:
680678
raise ValueError("The dst length must be greater than 0")
681679
size = blosc1_decompress(<void*>&typed_view_src[0], buf.buf, buf.len)
682-
PyBuffer_Release(buf)
680+
PyBuffer_Release(&buf)
683681
else:
684682
dst = PyBytes_FromStringAndSize(NULL, nbytes)
685683
if dst is None:
@@ -850,8 +848,8 @@ def compress2(src, **kwargs):
850848
create_cparams_from_kwargs(&cparams, kwargs)
851849

852850
cdef blosc2_context *cctx
853-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
854-
PyObject_GetBuffer(src, buf, PyBUF_SIMPLE)
851+
cdef Py_buffer buf
852+
PyObject_GetBuffer(src, &buf, PyBUF_SIMPLE)
855853
cdef int size
856854
cdef int32_t len_dest = <int32_t> (buf.len + BLOSC2_MAX_OVERHEAD)
857855
dest = bytes(len_dest)
@@ -865,8 +863,7 @@ def compress2(src, **kwargs):
865863
else:
866864
size = blosc2_compress_ctx(cctx, buf.buf, <int32_t> buf.len, _dest, len_dest)
867865
blosc2_free_ctx(cctx)
868-
PyBuffer_Release(buf)
869-
free(buf)
866+
PyBuffer_Release(&buf)
870867
if size < 0:
871868
raise RuntimeError("Could not compress the data")
872869
elif size == 0:
@@ -900,10 +897,9 @@ def decompress2(src, dst=None, **kwargs):
900897
cdef int32_t cbytes
901898
cdef int32_t blocksize
902899
blosc2_cbuffer_sizes(<void*>&typed_view_src[0], &nbytes, &cbytes, &blocksize)
903-
cdef Py_buffer *buf
900+
cdef Py_buffer buf
904901
if dst is not None:
905-
buf = <Py_buffer *> malloc(sizeof(Py_buffer))
906-
PyObject_GetBuffer(dst, buf, PyBUF_SIMPLE)
902+
PyObject_GetBuffer(dst, &buf, PyBUF_SIMPLE)
907903
if buf.len == 0:
908904
blosc2_free_ctx(dctx)
909905
raise ValueError("The dst length must be greater than 0")
@@ -914,7 +910,7 @@ def decompress2(src, dst=None, **kwargs):
914910
else:
915911
size = blosc2_decompress_ctx(dctx, view, cbytes, buf.buf, nbytes)
916912
blosc2_free_ctx(dctx)
917-
PyBuffer_Release(buf)
913+
PyBuffer_Release(&buf)
918914
else:
919915
dst = PyBytes_FromStringAndSize(NULL, nbytes)
920916
if dst is None:
@@ -980,7 +976,6 @@ cdef get_chunk_repeatval(blosc2_cparams cparams, const int32_t nbytes,
980976
if blosc2_chunk_repeatval(cparams, nbytes, dest, destsize, repeatval.buf) < 0:
981977
free(dest)
982978
PyBuffer_Release(repeatval)
983-
free(repeatval)
984979
raise RuntimeError("Problems when creating the repeated values chunk")
985980

986981

@@ -1076,11 +1071,10 @@ cdef class SChunk:
10761071
self.schunk.chunksize = chunksize
10771072
cdef const uint8_t[:] typed_view
10781073
cdef int64_t index
1079-
cdef Py_buffer *buf
1074+
cdef Py_buffer buf
10801075
cdef uint8_t *buf_ptr
10811076
if data is not None and len(data) > 0:
1082-
buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1083-
PyObject_GetBuffer(data, buf, PyBUF_SIMPLE)
1077+
PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE)
10841078
buf_ptr = <uint8_t *> buf.buf
10851079
len_data = buf.len
10861080
nchunks = len_data // chunksize + 1 if len_data % chunksize != 0 else len_data // chunksize
@@ -1091,9 +1085,9 @@ cdef class SChunk:
10911085
index = i * chunksize
10921086
nchunks_ = blosc2_schunk_append_buffer(self.schunk, buf_ptr + index, len_chunk)
10931087
if nchunks_ != (i + 1):
1094-
PyBuffer_Release(buf)
1088+
PyBuffer_Release(&buf)
10951089
raise RuntimeError("An error occurred while appending the chunks")
1096-
PyBuffer_Release(buf)
1090+
PyBuffer_Release(&buf)
10971091

10981092
@property
10991093
def c_schunk(self):
@@ -1223,11 +1217,10 @@ cdef class SChunk:
12231217
raise RuntimeError("Could not create decompression context")
12241218

12251219
def append_data(self, data):
1226-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1227-
PyObject_GetBuffer(data, buf, PyBUF_SIMPLE)
1220+
cdef Py_buffer buf
1221+
PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE)
12281222
rc = blosc2_schunk_append_buffer(self.schunk, buf.buf, <int32_t> buf.len)
1229-
PyBuffer_Release(buf)
1230-
free(buf)
1223+
PyBuffer_Release(&buf)
12311224
if rc < 0:
12321225
raise RuntimeError("Could not append the buffer")
12331226
return rc
@@ -1252,31 +1245,29 @@ cdef class SChunk:
12521245
else:
12531246
raise ValueError("value size in bytes must match with typesize")
12541247
array = np.array([value], dtype=dtype)
1255-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1256-
PyObject_GetBuffer(array, buf, PyBUF_SIMPLE)
1248+
cdef Py_buffer buf
1249+
PyObject_GetBuffer(array, &buf, PyBUF_SIMPLE)
12571250
# Create chunk with repeated values
12581251
nchunks = nitems // self.chunkshape
12591252
cdef blosc2_schunk *c_schunk = <blosc2_schunk *> self.c_schunk
12601253
cdef blosc2_cparams *cparams = self.schunk.storage.cparams
12611254
chunksize = BLOSC_EXTENDED_HEADER_LENGTH + self.typesize
12621255
cdef void *chunk = malloc(chunksize)
1263-
get_chunk_repeatval(dereference(cparams), self.chunksize, chunk, chunksize, buf)
1256+
get_chunk_repeatval(dereference(cparams), self.chunksize, chunk, chunksize, &buf)
12641257

12651258
for i in range(nchunks):
12661259
if blosc2_schunk_append_chunk(self.schunk, <uint8_t *>chunk, True) < 0:
12671260
free(chunk)
1268-
PyBuffer_Release(buf)
1269-
free(buf)
1261+
PyBuffer_Release(&buf)
12701262
raise RuntimeError("Error while appending the chunk")
12711263
# Create and append last chunk if it is smaller than chunkshape
12721264
remainder = nitems % self.chunkshape
12731265
rc = 0
12741266
if remainder != 0:
1275-
get_chunk_repeatval(dereference(cparams), remainder * self.typesize, chunk, chunksize, buf)
1267+
get_chunk_repeatval(dereference(cparams), remainder * self.typesize, chunk, chunksize, &buf)
12761268
rc = blosc2_schunk_append_chunk(self.schunk, <uint8_t *>chunk, True)
12771269
free(chunk)
1278-
PyBuffer_Release(buf)
1279-
free(buf)
1270+
PyBuffer_Release(&buf)
12801271
if rc < 0:
12811272
raise RuntimeError("Error while appending the chunk")
12821273

@@ -1297,14 +1288,13 @@ cdef class SChunk:
12971288
if needs_free:
12981289
free(chunk)
12991290

1300-
cdef Py_buffer *buf
1291+
cdef Py_buffer buf
13011292
if dst is not None:
1302-
buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1303-
PyObject_GetBuffer(dst, buf, PyBUF_SIMPLE)
1293+
PyObject_GetBuffer(dst, &buf, PyBUF_SIMPLE)
13041294
if buf.len == 0:
13051295
raise ValueError("The dst length must be greater than 0")
13061296
size = blosc2_schunk_decompress_chunk(self.schunk, nchunk, buf.buf, <int32_t>buf.len)
1307-
PyBuffer_Release(buf)
1297+
PyBuffer_Release(&buf)
13081298
else:
13091299
dst = PyBytes_FromStringAndSize(NULL, nbytes)
13101300
if dst is None:
@@ -1363,8 +1353,8 @@ cdef class SChunk:
13631353

13641354
def insert_data(self, nchunk, data, copy):
13651355
cdef blosc2_context *cctx
1366-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1367-
PyObject_GetBuffer(data, buf, PyBUF_SIMPLE)
1356+
cdef Py_buffer buf
1357+
PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE)
13681358
cdef int size
13691359
cdef int32_t len_chunk = <int32_t> (buf.len + BLOSC2_MAX_OVERHEAD)
13701360
cdef uint8_t* chunk = <uint8_t*> malloc(len_chunk)
@@ -1374,8 +1364,7 @@ cdef class SChunk:
13741364
size = blosc2_compress_ctx(self.schunk.cctx, buf.buf, <int32_t> buf.len, chunk, len_chunk)
13751365
else:
13761366
size = blosc2_compress_ctx(self.schunk.cctx, buf.buf, <int32_t> buf.len, chunk, len_chunk)
1377-
PyBuffer_Release(buf)
1378-
free(buf)
1367+
PyBuffer_Release(&buf)
13791368
if size < 0:
13801369
raise RuntimeError("Could not compress the data")
13811370
elif size == 0:
@@ -1402,8 +1391,8 @@ cdef class SChunk:
14021391
return rc
14031392

14041393
def update_data(self, nchunk, data, copy):
1405-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1406-
PyObject_GetBuffer(data, buf, PyBUF_SIMPLE)
1394+
cdef Py_buffer buf
1395+
PyObject_GetBuffer(data, &buf, PyBUF_SIMPLE)
14071396
cdef int size
14081397
cdef int32_t len_chunk = <int32_t> (buf.len + BLOSC2_MAX_OVERHEAD)
14091398
cdef uint8_t* chunk = <uint8_t*> malloc(len_chunk)
@@ -1413,8 +1402,7 @@ cdef class SChunk:
14131402
else:
14141403
size = blosc2_compress_ctx(self.schunk.cctx, buf.buf, <int32_t> buf.len, chunk, len_chunk)
14151404

1416-
PyBuffer_Release(buf)
1417-
free(buf)
1405+
PyBuffer_Release(&buf)
14181406
if size < 0:
14191407
raise RuntimeError("Could not compress the data")
14201408
elif size == 0:
@@ -1437,14 +1425,13 @@ cdef class SChunk:
14371425
return b''
14381426

14391427
cdef Py_ssize_t nbytes = (stop - start) * self.schunk.typesize
1440-
cdef Py_buffer *buf
1428+
cdef Py_buffer buf
14411429
if out is not None:
1442-
buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1443-
PyObject_GetBuffer(out, buf, PyBUF_SIMPLE)
1430+
PyObject_GetBuffer(out, &buf, PyBUF_SIMPLE)
14441431
if buf.len < nbytes:
14451432
raise ValueError("Not enough space for writing the slice in out")
14461433
rc = blosc2_schunk_get_slice_buffer(self.schunk, start, stop, buf.buf)
1447-
PyBuffer_Release(buf)
1434+
PyBuffer_Release(&buf)
14481435
else:
14491436
out = PyBytes_FromStringAndSize(NULL, nbytes)
14501437
if out is None:
@@ -1463,8 +1450,8 @@ cdef class SChunk:
14631450

14641451
cdef int64_t nbytes = (stop - start) * self.schunk.typesize
14651452

1466-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1467-
PyObject_GetBuffer(value, buf, PyBUF_SIMPLE)
1453+
cdef Py_buffer buf
1454+
PyObject_GetBuffer(value, &buf, PyBUF_SIMPLE)
14681455
cdef uint8_t *buf_ptr = <uint8_t *> buf.buf
14691456
cdef int64_t buf_pos = 0
14701457
cdef uint8_t *data
@@ -1516,7 +1503,7 @@ cdef class SChunk:
15161503
buf_pos += chunksize
15171504
else:
15181505
rc = blosc2_schunk_set_slice_buffer(self.schunk, start, stop, buf.buf)
1519-
PyBuffer_Release(buf)
1506+
PyBuffer_Release(&buf)
15201507
if rc < 0:
15211508
raise RuntimeError("Error while setting the slice")
15221509

@@ -1797,18 +1784,17 @@ cdef int aux_udf(udf_udata *udata, int64_t nchunk, int32_t nblock,
17971784
cdef int64_t start[B2ND_MAX_DIM]
17981785
cdef int64_t slice_shape[B2ND_MAX_DIM]
17991786
cdef int64_t blockshape_int64[B2ND_MAX_DIM]
1800-
cdef Py_buffer *buf
1787+
cdef Py_buffer buf
18011788
if padding:
18021789
for i in range(udata.array.ndim):
18031790
start[i] = 0
18041791
slice_shape[i] = blockshape[i]
18051792
blockshape_int64[i] = udata.array.blockshape[i]
1806-
buf = <Py_buffer *> malloc(sizeof(Py_buffer))
1807-
PyObject_GetBuffer(output, buf, PyBUF_SIMPLE)
1793+
PyObject_GetBuffer(output, &buf, PyBUF_SIMPLE)
18081794
rc = b2nd_copy_buffer2(udata.array.ndim, typesize,
18091795
buf.buf, slice_shape, start, slice_shape,
18101796
params_output, blockshape_int64, start)
1811-
PyBuffer_Release(buf)
1797+
PyBuffer_Release(&buf)
18121798
_check_rc(rc, "Could not copy the result into the buffer")
18131799

18141800
return 0
@@ -2088,13 +2074,13 @@ cdef schunk_is_ndarray(blosc2_schunk* schunk):
20882074

20892075

20902076
def schunk_from_cframe(cframe, copy=False):
2091-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
2092-
PyObject_GetBuffer(cframe, buf, PyBUF_SIMPLE)
2077+
cdef Py_buffer buf
2078+
PyObject_GetBuffer(cframe, &buf, PyBUF_SIMPLE)
20932079
cdef blosc2_schunk *schunk_ = blosc2_schunk_from_buffer(<uint8_t *>buf.buf, buf.len, copy)
20942080
if schunk_ == NULL:
20952081
raise RuntimeError("Could not get the schunk from the cframe")
20962082
schunk = blosc2.SChunk(_schunk=PyCapsule_New(schunk_, <char *> "blosc2_schunk*", NULL))
2097-
PyBuffer_Release(buf)
2083+
PyBuffer_Release(&buf)
20982084
if not copy:
20992085
schunk._avoid_cframe_free(True)
21002086
return schunk
@@ -2441,8 +2427,8 @@ cdef class NDArray:
24412427
def set_slice(self, key, ndarray):
24422428
ndim = self.ndim
24432429
start, stop = key
2444-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
2445-
PyObject_GetBuffer(ndarray, buf, PyBUF_SIMPLE)
2430+
cdef Py_buffer buf
2431+
PyObject_GetBuffer(ndarray, &buf, PyBUF_SIMPLE)
24462432

24472433
cdef int64_t[B2ND_MAX_DIM] buffershape_, start_, stop_
24482434
for i in range(ndim):
@@ -2452,7 +2438,7 @@ cdef class NDArray:
24522438

24532439
_check_rc(b2nd_set_slice_cbuffer(buf.buf, buffershape_, buf.len, start_, stop_, self.array),
24542440
"Error while setting the slice")
2455-
PyBuffer_Release(buf)
2441+
PyBuffer_Release(&buf)
24562442

24572443
return self
24582444

@@ -2715,12 +2701,12 @@ def full(shape, chunks, blocks, fill_value, dtype, **kwargs):
27152701

27162702
dtype = np.dtype(dtype)
27172703
nparr = np.array([fill_value], dtype=dtype)
2718-
cdef Py_buffer *val = <Py_buffer *> malloc(sizeof(Py_buffer))
2719-
PyObject_GetBuffer(nparr, val, PyBUF_SIMPLE)
2704+
cdef Py_buffer val
2705+
PyObject_GetBuffer(nparr, &val, PyBUF_SIMPLE)
27202706

27212707
cdef b2nd_array_t *array
27222708
_check_rc(b2nd_full(ctx, &array, val.buf), "Could not create full array")
2723-
PyBuffer_Release(val)
2709+
PyBuffer_Release(&val)
27242710

27252711
ndarray = blosc2.NDArray(_schunk=PyCapsule_New(array.sc, <char *> "blosc2_schunk*", NULL),
27262712
_array=PyCapsule_New(array, <char *> "b2nd_array_t*", NULL))
@@ -2748,8 +2734,8 @@ def from_buffer(buf, shape, chunks, blocks, dtype, **kwargs):
27482734

27492735
def asarray(ndarray, chunks, blocks, **kwargs):
27502736
interface = ndarray.__array_interface__
2751-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
2752-
PyObject_GetBuffer(ndarray, buf, PyBUF_SIMPLE)
2737+
cdef Py_buffer buf
2738+
PyObject_GetBuffer(ndarray, &buf, PyBUF_SIMPLE)
27532739

27542740
shape = interface["shape"]
27552741
dtype = interface["typestr"]
@@ -2763,7 +2749,7 @@ def asarray(ndarray, chunks, blocks, **kwargs):
27632749
cdef b2nd_array_t *array
27642750
_check_rc(b2nd_from_cbuffer(ctx, &array, <void *> <char *> buf.buf, buf.len),
27652751
"Error while creating the NDArray")
2766-
PyBuffer_Release(buf)
2752+
PyBuffer_Release(&buf)
27672753
ndarray = blosc2.NDArray(_schunk=PyCapsule_New(array.sc, <char *> "blosc2_schunk*", NULL),
27682754
_array=PyCapsule_New(array, <char *> "b2nd_array_t*", NULL))
27692755
_check_rc(b2nd_free_ctx(ctx), "Error while freeing the context")
@@ -2773,8 +2759,8 @@ def asarray(ndarray, chunks, blocks, **kwargs):
27732759

27742760

27752761
def ndarray_from_cframe(cframe, copy=False):
2776-
cdef Py_buffer *buf = <Py_buffer *> malloc(sizeof(Py_buffer))
2777-
PyObject_GetBuffer(cframe, buf, PyBUF_SIMPLE)
2762+
cdef Py_buffer buf
2763+
PyObject_GetBuffer(cframe, &buf, PyBUF_SIMPLE)
27782764
cdef b2nd_array_t *array
27792765
cdef int rc
27802766
rc = b2nd_from_cframe(<uint8_t *>buf.buf, buf.len, copy, &array)
@@ -2783,7 +2769,7 @@ def ndarray_from_cframe(cframe, copy=False):
27832769
ndarray = blosc2.NDArray(_schunk=PyCapsule_New(array.sc, <char *> "blosc2_schunk*", NULL),
27842770
_array=PyCapsule_New(array, <char *> "b2nd_array_t*", NULL))
27852771

2786-
PyBuffer_Release(buf)
2772+
PyBuffer_Release(&buf)
27872773
if not copy:
27882774
ndarray._schunk._avoid_cframe_free(True)
27892775
return ndarray

0 commit comments

Comments
 (0)