Merge branch 'main' into ig/itemsize_blosc

ilan-gold · ilan-gold · commit 28015ae44205 · 2025-03-03T14:26:09.000+01:00
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -13,8 +13,8 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.12", "3.13"]
-        # macos-13 is an intel runner, macos-14 is a arm64 runner
-        platform: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        # macos-13 is an intel runner, macos-14 is an arm64 runner
+        platform: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-13, macos-14]
 
     defaults:
       run:
@@ -28,7 +28,7 @@ jobs:
           fetch-depth: 0 # required for version resolution
 
       - name: Set up Conda
-        uses: conda-incubator/setup-miniconda@v3.1.0
+        uses: conda-incubator/setup-miniconda@v3.1.1
         with:
           channels: conda-forge
           miniforge-version: latest
@@ -50,20 +50,13 @@ jobs:
       - name: Install numcodecs
         run: |
           export DISABLE_NUMCODECS_AVX2=""
-          # TODO: put back zfpy import when it supports numpy 2.0
-          python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec]
+          python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
 
       - name: Install zarr-python
         # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out
         # so we can have some tests of our minimum version of numpy (1.24)
         if: matrix.python-version != '3.11'
-        # TODO: remove --pre option when zarr v3 is out
-        run: python -m pip install --pre zarr>=3.0.0b2
-
-      # This is used to test with zfpy, which does not yet support numpy 2.0
-      - name: Install older numpy and zfpy
-        if: matrix.python-version == '3.11'
-        run: python -m pip install -v ".[zfpy]"
+        run: python -m pip install zarr>=3
 
       - name: List installed packages
         run: python -m pip list
diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml
@@ -13,8 +13,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # macos-13 is an intel runner, macos-14 is a arm64 runner
-        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        # macos-13 is an intel runner, macos-14 is an arm64 runner
+        os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-13, macos-14]
     env:
       CIBW_TEST_COMMAND: python -c "import numcodecs"
       CIBW_BUILD: "cp311-* cp312-* cp313-*"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,19 +14,19 @@ repos:
     - id: debug-statements
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.4
+    rev: v0.9.4
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
     - id: ruff-format
 
   - repo: https://github.com/scientific-python/cookie
-    rev: 2024.08.19
+    rev: 2025.01.22
     hooks:
     - id: sp-repo-review
 
   -   repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v1.14.0
+      rev: v1.14.1
       hooks:
       -   id: mypy
           args: [--config-file, pyproject.toml]
diff --git a/docs/release.rst b/docs/release.rst
@@ -11,11 +11,38 @@ Release notes
     Unreleased
     ----------
 
+
 .. _unreleased:
 
 Unreleased
 ----------
 
+Enhancements
+~~~~~~~~~~~~
+
+* Add support for the Linux AArch64 architecture, and bump the minimum
+macOS deployment target for x86_64 to 10.13.
+  By :user:`Agriya Khetarpal <agriyakhetarpal>`, :issue:`288`.
+
+Improvements
+~~~~~~~~~~~~
+* Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec.
+  By :user:`Cas Wognum <cwognum>`.
+* Add ``typesize`` argument to ``Blosc`` to allow for buffers that are passed to ``encode``
+  use that information.  zarr v3 currently has its Blosc codec as bytes-to-bytes but does retain
+  the size information so using it here allows for massive compression ratio gains.
+  By :user:`Ilan Gold <ilan-gold>`
+
+Fixes
+~~~~~
+* Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs.
+  By :user:`Norman Rzepka <normanrz>`, :issue:`685`
+
+.. _release_0.15.0:
+
+0.15.0
+------
+
 Breaking changes
 ~~~~~~~~~~~~~~~~
 * All arguments to the ``PCodec`` constructor except for ``level``
@@ -24,7 +51,7 @@ Breaking changes
 
 Deprecations
 ~~~~~~~~~~~~
-The following ``blosc`` funcitons are deprecated, with no replacement.
+The following ``blosc`` functions are deprecated, with no replacement.
 This is because they are not intended to be public API.
 
 - ``numcodecs.blosc.init``
@@ -36,11 +63,16 @@ This is because they are not intended to be public API.
 In addition, ``numcodecs.blosc.decompress_partial`` is deprecated as
 has always been experimental and there is no equivalent in the official
 blsoc Python package.
+By :user:`David Stansby <dstansby>`, :issue`619`
 
 Fixes
 ~~~~~
 * Fixes issue with ``Delta`` Zarr 3 codec not working with ``astype``.
   By :user:`Norman Rzepka <normanrz>`, :issue:`664`
+* Cleanup ``PCodec`` soft dependency.
+  Previously importing ``numcodecs.pcodec`` would work if ``pcodec`` is not installed,
+  but now it will fail to import. This mirrors the behaviour of other optional dependencies.
+  By :user:`John Kirkham <jakirkham>`, :issue:`647`
 * Fixes issues with the upcoming ``zarr`` 3.0.0 release.
   By :user:`Norman Rzepka <normanrz>`, :issue:`675`
 
@@ -63,10 +95,8 @@ Improvements
   Import errors caused by optional dependencies (ZFPY, MsgPack, CRC32C, and PCodec)
   are still silently caught.
   By :user:`David Stansby <dstansby>`, :issue:`550`.
-* Add ``typesize`` argument to ``Blosc`` to allow for buffers that are passed to ``encode``
-  use that information.  zarr v3 currently has its Blosc codec as bytes-to-bytes but does retain
-  the size information so using it here allows for massive compression ratio gains.
-  By :user:`Ilan Gold <ilan-gold>`
+
+.. _release_0.14.1:
 
 0.14.1
 ------
diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py
@@ -13,7 +13,7 @@
 
 _crc32c: Optional[ModuleType] = None
 with suppress(ImportError):
-    import crc32c as _crc32c  # type: ignore[no-redef]
+    import crc32c as _crc32c  # type: ignore[no-redef, unused-ignore]
 
 if TYPE_CHECKING:  # pragma: no cover
     from typing_extensions import Buffer
diff --git a/numcodecs/delta.py b/numcodecs/delta.py
@@ -63,12 +63,7 @@ def encode(self, buf):
         enc[0] = arr[0]
 
         # compute differences
-        # using np.subtract for in-place operations
-        if arr.dtype == bool:
-            np.not_equal(arr[1:], arr[:-1], out=enc[1:])
-        else:
-            np.subtract(arr[1:], arr[:-1], out=enc[1:])
-
+        enc[1:] = np.diff(arr)
         return enc
 
     def decode(self, buf, out=None):
diff --git a/numcodecs/errors.py b/numcodecs/errors.py
@@ -0,0 +1,26 @@
+"""
+This module defines custom exceptions that are raised in the `numcodecs` codebase.
+"""
+
+
+class UnknownCodecError(ValueError):
+    """
+    An exception that is raised when trying to receive a codec that has not been registered.
+
+    Parameters
+    ----------
+    codec_id : str
+        Codec identifier.
+
+    Examples
+    ----------
+    >>> import numcodecs
+    >>> numcodecs.get_codec({"codec_id": "unknown"})
+    Traceback (most recent call last):
+        ...
+    UnknownCodecError: codec not available: 'unknown'
+    """
+
+    def __init__(self, codec_id: str):
+        self.codec_id = codec_id
+        super().__init__(f"codec not available: '{codec_id}'")
diff --git a/numcodecs/json.py b/numcodecs/json.py
@@ -68,7 +68,7 @@ def __init__(
     def encode(self, buf):
         try:
             buf = np.asarray(buf)
-        except ValueError:
+        except ValueError:  # pragma: no cover
             buf = np.asarray(buf, dtype=object)
         items = np.atleast_1d(buf).tolist()
         items.append(buf.dtype.str)
diff --git a/numcodecs/pcodec.py b/numcodecs/pcodec.py
@@ -2,12 +2,7 @@
 
 from numcodecs.abc import Codec
 from numcodecs.compat import ensure_contiguous_ndarray
-
-try:
-    from pcodec import ChunkConfig, DeltaSpec, ModeSpec, PagingSpec, standalone
-except ImportError:  # pragma: no cover
-    standalone = None
-
+from pcodec import ChunkConfig, DeltaSpec, ModeSpec, PagingSpec, standalone
 
 DEFAULT_MAX_PAGE_N = 262144
 
@@ -58,9 +53,6 @@ def __init__(
         delta_encoding_order: int | None = None,
         equal_pages_up_to: int = DEFAULT_MAX_PAGE_N,
     ):
-        if standalone is None:  # pragma: no cover
-            raise ImportError("pcodec must be installed to use the PCodec codec.")
-
         # note that we use `level` instead of `compression_level` to
         # match other codecs
         self.level = level
diff --git a/numcodecs/quantize.py b/numcodecs/quantize.py
@@ -65,9 +65,9 @@ def encode(self, buf):
         precision = 10.0**-self.digits
         exp = math.log10(precision)
         if exp < 0:
-            exp = int(math.floor(exp))
+            exp = math.floor(exp)
         else:
-            exp = int(math.ceil(exp))
+            exp = math.ceil(exp)
         bits = math.ceil(math.log2(10.0**-exp))
         scale = 2.0**bits
         enc = np.around(scale * arr) / scale
diff --git a/numcodecs/registry.py b/numcodecs/registry.py
@@ -5,6 +5,7 @@
 from importlib.metadata import EntryPoints, entry_points
 
 from numcodecs.abc import Codec
+from numcodecs.errors import UnknownCodecError
 
 logger = logging.getLogger("numcodecs")
 codec_registry: dict[str, Codec] = {}
@@ -50,7 +51,7 @@ def get_codec(config):
         register_codec(cls, codec_id=codec_id)
     if cls:
         return cls.from_config(config)
-    raise ValueError(f'codec not available: {codec_id!r}')
+    raise UnknownCodecError(f"{codec_id!r}")
 
 
 def register_codec(cls, codec_id=None):
diff --git a/numcodecs/tests/test_pcodec.py b/numcodecs/tests/test_pcodec.py
@@ -1,11 +1,8 @@
 import numpy as np
 import pytest
 
-from numcodecs.pcodec import PCodec
-
 try:
-    # initializing codec triggers ImportError
-    PCodec()
+    from numcodecs.pcodec import PCodec
 except ImportError:  # pragma: no cover
     pytest.skip("pcodec not available", allow_module_level=True)
 
diff --git a/numcodecs/tests/test_registry.py b/numcodecs/tests/test_registry.py
@@ -3,11 +3,12 @@
 import pytest
 
 import numcodecs
+from numcodecs.errors import UnknownCodecError
 from numcodecs.registry import get_codec
 
 
 def test_registry_errors():
-    with pytest.raises(ValueError):
+    with pytest.raises(UnknownCodecError, match='foo'):
         get_codec({'id': 'foo'})
 
 
diff --git a/numcodecs/tests/test_vlen_bytes.py b/numcodecs/tests/test_vlen_bytes.py
@@ -1,3 +1,4 @@
+import sys
 import unittest
 
 import numpy as np
@@ -84,6 +85,9 @@ def test_decode_errors():
         codec.decode(enc, out=np.zeros(10, dtype='i4'))
 
 
+# TODO: fix this test on GitHub actions somehow...
+# See https://github.com/zarr-developers/numcodecs/issues/683
+@pytest.mark.skipif(sys.platform == "darwin", reason="Test is failing on macOS on GitHub actions.")
 def test_encode_none():
     a = np.array([b'foo', None, b'bar'], dtype=object)
     codec = VLenBytes()
diff --git a/numcodecs/tests/test_zarr3.py b/numcodecs/tests/test_zarr3.py
@@ -91,7 +91,7 @@ def test_generic_compressor(
         (numcodecs.zarr3.Delta, {"dtype": "float32"}),
         (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 25.5}),
         (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}),
-        (numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float64"}),
+        (numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float32"}),
     ],
     ids=[
         "delta",
@@ -225,11 +225,11 @@ def test_generic_bytes_codec(
 ):
     try:
         codec_class()._codec  # noqa: B018
-    except ValueError as e:
+    except ValueError as e:  # pragma: no cover
         if "codec not available" in str(e):
             pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
         else:
-            raise  # pragma: no cover
+            raise
     except ImportError as e:  # pragma: no cover
         pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
 
@@ -272,3 +272,8 @@ def test_delta_astype(store: StorePath):
 def test_repr():
     codec = numcodecs.zarr3.LZ4(level=5)
     assert repr(codec) == "LZ4(codec_name='numcodecs.lz4', codec_config={'level': 5})"
+
+
+def test_to_dict():
+    codec = numcodecs.zarr3.LZ4(level=5)
+    assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}}
diff --git a/numcodecs/zarr3.py b/numcodecs/zarr3.py
@@ -112,6 +112,7 @@ def from_dict(cls, data: dict[str, JSON]) -> Self:
 
     def to_dict(self) -> dict[str, JSON]:
         codec_config = self.codec_config.copy()
+        codec_config.pop("id", None)
         return {
             "name": self.codec_name,
             "configuration": codec_config,
@@ -270,7 +271,7 @@ def __init__(self, **codec_config: JSON) -> None:
         super().__init__(**codec_config)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle:
-        if array_spec.dtype.itemsize != self.codec_config.get("elementsize"):
+        if self.codec_config.get("elementsize", None) is None:
             return Shuffle(**{**self.codec_config, "elementsize": array_spec.dtype.itemsize})
         return self  # pragma: no cover
 
@@ -307,7 +308,7 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
         return chunk_spec
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset:
-        if str(array_spec.dtype) != self.codec_config.get("dtype"):
+        if self.codec_config.get("dtype", None) is None:
             return FixedScaleOffset(**{**self.codec_config, "dtype": str(array_spec.dtype)})
         return self
 
@@ -320,7 +321,7 @@ def __init__(self, **codec_config: JSON) -> None:
         super().__init__(**codec_config)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize:
-        if str(array_spec.dtype) != self.codec_config.get("dtype"):
+        if self.codec_config.get("dtype", None) is None:
             return Quantize(**{**self.codec_config, "dtype": str(array_spec.dtype)})
         return self
 
@@ -355,8 +356,7 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
         return replace(chunk_spec, dtype=np.dtype(self.codec_config["encode_dtype"]))  # type: ignore[arg-type]
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> AsType:
-        decode_dtype = self.codec_config.get("decode_dtype")
-        if str(array_spec.dtype) != decode_dtype:
+        if self.codec_config.get("decode_dtype", None) is None:
             return AsType(**{**self.codec_config, "decode_dtype": str(array_spec.dtype)})
         return self
 
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/setup.py b/setup.py
