Merge remote-tracking branch 'origin/release-v4.7' into develop

Author: nfrasser

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+exclude: |
+  (?x)^(tests/data)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
@@ -10,7 +12,7 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.0
+    rev: v0.9.9
     hooks:
       - id: ruff
         name: ruff check
@@ -24,7 +26,7 @@ repos:
         name: ruff format
         alias: format
   - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.391
+    rev: v1.1.396
     hooks:
       - id: pyright
         additional_dependencies:

cryosparc/dataset/__init__.py

@@ -51,6 +51,7 @@
 
 import numpy as n
 
+from ..constants import ONE_MIB
 from ..errors import DatasetLoadError
 from ..stream import AsyncReadable, Streamable
 from ..util import bopen, default_rng, random_integers, u32bytesle, u32intle
@@ -635,8 +636,12 @@ def _load_numpy(
     ):
         import os
 
-        # disable mmap by setting CRYOSPARC_DATASET_MMAP=false
-        if os.getenv("CRYOSPARC_DATASET_MMAP", "true").lower() == "true" and isinstance(file, (str, PurePath)):
+        # disable mmap by setting CRYOSPARC_DATASET_MMAP=false or dataset is small
+        if (
+            os.getenv("CRYOSPARC_DATASET_MMAP", "true").lower() == "true"
+            and isinstance(file, (str, PurePath))
+            and os.stat(file).st_size > ONE_MIB
+        ):
             # Use mmap to avoid loading full record array into memory
             # cast path to a string for older numpy/python
             mmap_mode, f = "r", str(file)

cryosparc/dataset/core.pyi

@@ -51,7 +51,7 @@ class Data:
     def ncol(self) -> int: ...
     def nrow(self) -> int: ...
     def key(self, index: int) -> str: ...
-    def type(self, field: str) -> int: ...
+    def type(self, field: str) -> DsetType: ...
     def has(self, field: str) -> bool: ...
     def addrows(self, num: int) -> None: ...
     def addcol_scalar(self, field: str, dtype: int) -> None: ...

cryosparc/tools.py

@@ -60,7 +60,6 @@
 assert model_registry
 registry.finalize()  # no more models may be registered after this
 
-ONE_MIB = 2**20  # bytes in one mebibyte
 
 LICENSE_REGEX = re.compile(r"[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}")
 """Regular expression for matching CryoSPARC license IDs."""

tests/data/4k_dataset.cs

@@ -394,3 +394,10 @@ def test_allocate_many_together():
             allocated.append(Dataset(1))
         assert len(allocated) == 66_000
         del allocated
+
+
+def test_load_4k():
+    # Check that a 4kiB dataset (same as linux page size) loads correctly
+    # (numpy bug https://github.com/numpy/numpy/pull/27723 fixed in 2.2)
+    d = Dataset.load("tests/data/4k_dataset.cs")
+    assert len(d) == 0