Merge zarr-developers/main into jakirkham/use_mv

Author: jakirkham

.github/workflows/ci.yaml

@@ -13,94 +13,60 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.12", "3.13"]
-        # macos-13 is an intel runner, macos-14 is a arm64 runner
-        platform: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        # macos-13 is an intel runner, macos-14 is an arm64 runner
+        platform: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-13, macos-14]
+
+    defaults:
+      run:
+        shell: bash -el {0}
 
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
         with:
           submodules: recursive
+          fetch-depth: 0 # required for version resolution
 
       - name: Set up Conda
-        uses: conda-incubator/[email protected].0
+        uses: conda-incubator/[email protected].1
         with:
-            channels: conda-forge
-            miniforge-version: latest
-            python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          miniforge-version: latest
+          python-version: ${{ matrix.python-version }}
 
-      - name: Show info about `base` environment
-        shell: "bash -l {0}"
-        run: |
-          conda info
-          conda config --show-sources
-          conda list --show-channel-urls
-
-      - name: Set up `env`
-        shell: "bash -l {0}"
-        run: >
-          conda create -n env
-          c-compiler cxx-compiler
-          python=${{matrix.python-version}} wheel pip
+      - name: Install compilers
+        run: conda install -y c-compiler cxx-compiler
 
       - name: Install clang
-        shell: "bash -l {0}"
         if: matrix.platform == 'macos-13'
-        run: |
-          conda activate env
-          conda install -y 'clang>=12.0.1,<17'
+        run: conda install -y 'clang>=12.0.1,<17'
 
-      - name: Show info about `env` environment
-        shell: "bash -l {0}"
+      - name: Show conda environment info
         run: |
-          conda list --show-channel-urls -n env
+          conda info
+          conda config --show-sources
+          conda list --show-channel-urls
 
       - name: Install numcodecs
-        shell: "bash -l {0}"
         run: |
-          conda activate env
           export DISABLE_NUMCODECS_AVX2=""
-          # TODO: put back zfpy import when it supports numpy 2.0
-          python -m pip install -v -e .[test,test_extras,msgpack,crc32c]
-
-      - name: Install pcodec
-        if: matrix.python-version != '3.13'
-        shell: "bash -l {0}"
-        run: |
-            conda activate env
-            python -m pip install -v ".[pcodec]"
+          python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
 
       - name: Install zarr-python
-        shell: "bash -l {0}"
         # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out
         # so we can have some tests of our minimum version of numpy (1.24)
         if: matrix.python-version != '3.11'
-        run: |
-          conda activate env
-          # TODO: remove --pre option when zarr v3 is out
-          python -m pip install --pre zarr>=3.0.0b2
-
-      # This is used to test with zfpy, which does not yet support numpy 2.0
-      - name: Install older numpy and zfpy
-        if: matrix.python-version == '3.11'
-        shell: "bash -l {0}"
-        run: |
-          conda activate env
-          python -m pip install -v ".[zfpy]"
+        run: python -m pip install zarr>=3
 
       - name: List installed packages
-        shell: "bash -l {0}"
-        run: |
-          conda activate env
-          python -m pip list
+        run: python -m pip list
 
       - name: Run tests
         shell: "bash -l {0}"
-        run: |
-          conda activate env
-          pytest -v
+        run: pytest -v
 
       - uses: codecov/codecov-action@v5
         with:
+          fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true

.github/workflows/wheel.yaml

@@ -13,8 +13,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # macos-13 is an intel runner, macos-14 is a arm64 runner
-        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        # macos-13 is an intel runner, macos-14 is an arm64 runner
+        os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-13, macos-14]
     env:
       CIBW_TEST_COMMAND: python -c "import numcodecs"
       CIBW_BUILD: "cp311-* cp312-* cp313-*"
@@ -26,7 +26,7 @@ jobs:
         with:
           submodules: true
 
-      - uses: pypa/cibuildwheel@v2.21.3
+      - uses: pypa/cibuildwheel@v2.23.0
 
       - uses: actions/upload-artifact@v4
         with:

.pre-commit-config.yaml

@@ -1,11 +1,10 @@
 exclude: ^fixture/  # files with trailing whitespaces on purpose
 ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autoupdate_schedule: "monthly"
   autofix_commit_msg: "style: pre-commit fixes"
   autofix_prs: false
 default_stages: [pre-commit, pre-push]
-default_language_version:
-  python: python3
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
@@ -15,20 +14,20 @@ repos:
     - id: debug-statements
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.4
+    rev: v0.9.9
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
     - id: ruff-format
 
   - repo: https://github.com/scientific-python/cookie
-    rev: 2024.08.19
+    rev: 2025.01.22
     hooks:
     - id: sp-repo-review
 
   -   repo: https://github.com/pre-commit/mirrors-mypy
-      rev: 'v1.13.0'
+      rev: v1.15.0
       hooks:
       -   id: mypy
           args: [--config-file, pyproject.toml]
-          additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr>=3.0.0b2']
+          additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr>=3.0.0rc1']

adhoc/blosc_memleak_check.py

@@ -1,10 +1,9 @@
 import sys
 
+import numcodecs
 import numpy as np
 from numpy.testing import assert_array_equal
 
-import numcodecs
-
 codec = numcodecs.Blosc()
 data = np.arange(int(sys.argv[1]))
 for _ in range(int(sys.argv[2])):

docs/compression/blosc.rst

@@ -18,15 +18,9 @@ Blosc
 Helper functions
 ----------------
 
-.. autofunction:: init
-.. autofunction:: destroy
-.. autofunction:: compname_to_compcode
 .. autofunction:: list_compressors
 .. autofunction:: get_nthreads
 .. autofunction:: set_nthreads
-.. autofunction:: cbuffer_sizes
 .. autofunction:: cbuffer_complib
-.. autofunction:: cbuffer_metainfo
 .. autofunction:: compress
 .. autofunction:: decompress
-.. autofunction:: decompress_partial

docs/release.rst

@@ -11,8 +11,93 @@ Release notes
     Unreleased
     ----------
 
+
 .. _unreleased:
 
+Unreleased
+----------
+
+Enhancements
+~~~~~~~~~~~~
+
+* Add support for the Linux AArch64 architecture, and bump the minimum
+macOS deployment target for x86_64 to 10.13.
+  By :user:`Agriya Khetarpal <agriyakhetarpal>`, :issue:`288`.
+
+Improvements
+~~~~~~~~~~~~
+* Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec.
+  By :user:`Cas Wognum <cwognum>`.
+* Add ``typesize`` argument to ``Blosc`` to allow for buffers that are passed to ``encode``
+  use that information.  zarr v3 currently has its Blosc codec as bytes-to-bytes but does retain
+  the size information so using it here allows for massive compression ratio gains.
+  By :user:`Ilan Gold <ilan-gold>`
+
+Fixes
+~~~~~
+* Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs.
+  By :user:`Norman Rzepka <normanrz>`, :issue:`685`
+
+.. _release_0.15.0:
+
+0.15.0
+------
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+* All arguments to the ``PCodec`` constructor except for ``level``
+  are now keyword only, to support the updated API.
+  By :user:`Sam Levang <slevang>`, :issue:`623`
+
+Deprecations
+~~~~~~~~~~~~
+The following ``blosc`` functions are deprecated, with no replacement.
+This is because they are not intended to be public API.
+
+- ``numcodecs.blosc.init``
+- ``numcodecs.blosc.destroy``
+- ``numcodecs.blosc.compname_to_compcode``
+- ``numcodecs.blosc.cbuffer_sizes``
+- ``numcodecs.blosc.cbuffer_metainfo``
+
+In addition, ``numcodecs.blosc.decompress_partial`` is deprecated as
+has always been experimental and there is no equivalent in the official
+blsoc Python package.
+By :user:`David Stansby <dstansby>`, :issue`619`
+
+Fixes
+~~~~~
+* Fixes issue with ``Delta`` Zarr 3 codec not working with ``astype``.
+  By :user:`Norman Rzepka <normanrz>`, :issue:`664`
+* Cleanup ``PCodec`` soft dependency.
+  Previously importing ``numcodecs.pcodec`` would work if ``pcodec`` is not installed,
+  but now it will fail to import. This mirrors the behaviour of other optional dependencies.
+  By :user:`John Kirkham <jakirkham>`, :issue:`647`
+* Fixes issues with the upcoming ``zarr`` 3.0.0 release.
+  By :user:`Norman Rzepka <normanrz>`, :issue:`675`
+
+* Removed Version Check: The previous code included a check for the `NumPy` version
+  and a warning if the version was incompatible with `zfpy`.
+  This check has been removed because `zfpy` now supports the newer versions of `NumPy`.
+  By :user:`Meher Gajula <me-her>`, :issue:`672`
+
+Improvements
+~~~~~~~~~~~~
+* Add support for ``pcodec`` 0.3. This exposes the new ``delta_spec``
+  and ``paging_spec`` arguments, but maintains full backwards
+  compatibility for data written with older package versions.
+  By :user:`Sam Levang <slevang>`, :issue:`623`
+* If an import error is raised when trying to define a codec that is *not*
+  an optional dependency, it is no longer silently caught. Instead it will
+  be propagated to the user, as this indicates an issue with the installed
+  package.
+
+  Import errors caused by optional dependencies (ZFPY, MsgPack, CRC32C, and PCodec)
+  are still silently caught.
+  By :user:`David Stansby <dstansby>`, :issue:`550`.
+
+.. _release_0.14.1:
+
 0.14.1
 ------
 

docs/zarr3.rst

@@ -1,10 +1,12 @@
+.. _Zarr 3 codecs:
+
 Zarr 3 codecs
 =============
 .. automodule:: numcodecs.zarr3
 
 
-Bytes-to-bytes codecs
----------------------
+Compressors (bytes-to-bytes codecs)
+-----------------------------------
 .. autoclass:: Blosc()
 
     .. autoattribute:: codec_name
@@ -38,58 +40,61 @@ Bytes-to-bytes codecs
     .. autoattribute:: codec_name
 
 
-Array-to-array codecs
----------------------
-.. autoclass:: Delta()
+Checksum codecs (bytes-to-bytes codecs)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Need to be used as ``compressors`` in zarr-python.
+
+.. autoclass:: CRC32()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: BitRound()
+.. autoclass:: CRC32C()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: FixedScaleOffset()
+.. autoclass:: Adler32()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: Quantize()
+.. autoclass:: Fletcher32()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: PackBits()
+.. autoclass:: JenkinsLookup3()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: AsType()
 
-    .. autoattribute:: codec_name
+Filters (array-to-array codecs)
+-------------------------------
+.. autoclass:: Delta()
 
+    .. autoattribute:: codec_name
 
-Bytes-to-bytes checksum codecs
-------------------------------
-.. autoclass:: CRC32()
+.. autoclass:: BitRound()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: CRC32C()
+.. autoclass:: FixedScaleOffset()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: Adler32()
+.. autoclass:: Quantize()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: Fletcher32()
+.. autoclass:: PackBits()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: JenkinsLookup3()
+.. autoclass:: AsType()
 
     .. autoattribute:: codec_name
 
 
-Array-to-bytes codecs
----------------------
+
+Serializers (array-to-bytes codecs)
+-----------------------------------
 .. autoclass:: PCodec()
 
     .. autoattribute:: codec_name