Skip to content

Commit 0883a1c

Browse files
committed
Minor memory optimisations
1 parent 5a14db4 commit 0883a1c

File tree

1 file changed

+28
-21
lines changed

1 file changed

+28
-21
lines changed

src/blosc2/ndarray.py

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1500,44 +1500,51 @@ def get_fselection_numpy(self, key: list | np.ndarray) -> np.ndarray: # noqa: C
15001500
end = None
15011501
flat_shape = ()
15021502

1503-
for num, i in enumerate(_slice):
1504-
if isinstance(i, np.ndarray): # collecting arrays
1505-
if end is not None:
1506-
raise ValueError("Cannot use slices between arrays of integers in index")
1507-
arrs.append(i.flatten())
1508-
if begin is None:
1509-
begin = num
1510-
else: # slice
1511-
if arrs: # flush arrays
1512-
arr = np.stack(arrs)
1513-
arrs = []
1514-
end = num
1515-
flat_shape += (arr.shape[-1],)
1516-
flat_shape += ((i.stop - i.start + (i.step - 1)) // i.step,)
1517-
1518-
# flush at the end if arrays remain
1519-
if arrs:
1520-
arr = np.stack(arrs)
1503+
if len(_slice) == 1: # 1D fast path possible if no slices
1504+
arr = _slice[0].reshape(1, -1)
15211505
flat_shape += (arr.shape[-1],)
1506+
begin = 0
1507+
else:
1508+
for num, i in enumerate(_slice):
1509+
if isinstance(i, np.ndarray): # collecting arrays
1510+
if end is not None:
1511+
raise ValueError("Cannot use slices between arrays of integers in index")
1512+
arrs.append(i.reshape(-1)) # flatten does copy
1513+
if begin is None:
1514+
begin = num
1515+
else: # slice
1516+
if arrs: # flush arrays
1517+
arr = np.stack(arrs)
1518+
arrs = []
1519+
end = num
1520+
flat_shape += (arr.shape[-1],)
1521+
flat_shape += ((i.stop - i.start + (i.step - 1)) // i.step,)
1522+
1523+
# flush at the end if arrays remain
1524+
if arrs:
1525+
arr = np.stack(arrs)
1526+
flat_shape += (arr.shape[-1],)
1527+
15221528
# out_shape could have new dims if indexing arrays are not all 1D
15231529
# (we have just flattened them so need to handle accordingly)
1530+
prior_tuple = _slice[:begin]
1531+
post_tuple = _slice[end:] if end is not None else ()
15241532
divider = chunks[begin:end]
15251533
chunked_arr = arr.T // divider
15261534
unique_chunks, chunk_nitems = np.unique(chunked_arr, axis=0, return_counts=True)
1527-
if len(arr) == 1: # 1D chunks:
1535+
if len(arr) == 1: # 1D chunks, can avoid loading whole chunks
15281536
idx_order = np.argsort(arr.squeeze(axis=0), axis=-1) # sort by real index
15291537
else:
15301538
idx_order = np.lexsort(
15311539
tuple(a for a in reversed(chunked_arr.T))
15321540
) # sort by chunks (can't sort by index since larger index could belong to lower chunk)
15331541
# e.g. chunks of (100, 10) means (50, 15) has chunk idx (0,1) but (60,5) has (0, 0)
1542+
del chunked_arr # no longer need this
15341543
sorted_idxs = arr[:, idx_order]
15351544
out = np.empty(flat_shape, dtype=self.dtype)
15361545
shape = np.array(shape)
15371546

15381547
chunk_nitems_cumsum = np.cumsum(chunk_nitems)
1539-
prior_tuple = _slice[:begin]
1540-
post_tuple = _slice[end:] if end is not None else ()
15411548
cprior_slices = [
15421549
slice_to_chunktuple(s, c) for s, c in zip(prior_tuple, chunks[:begin], strict=True)
15431550
]

0 commit comments

Comments
 (0)