Merge branch 'main' into pre-commit-ci-update-config

Author: d-v-b

.github/workflows/ci-i386.yml

@@ -27,34 +27,45 @@ jobs:
           arch: x86
           packages: >
             build-base
-            python3
-            python3-dev
+            cargo
+            cmake
+            cython
             git
-            py3-pip
-            py3-pytest
-            uv
             meson
             pkgconf
+            rust
+            sed
+            sudo
+            uv
+            zstd
+            python3
+            python3-dev
             py3-numpy
             py3-numpy-dev
-            zstd
+            py3-pip
+            py3-pytest
 
-      - name: Install numcodecs
+      - name: Install zfp
         run: |
-          export DISABLE_NUMCODECS_AVX2=""
           uv venv
-          uv pip install -v -e .[test,test_extras,msgpack,crc32c]
+          PYTHON_INCLUDE=$(uv run python -c 'from sysconfig import get_paths; print(get_paths()["include"])');
+          PYTHON_LIB=$(uv run python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))');
+          git clone https://github.com/LLNL/zfp
+          sed -i "s/distutils.sysconfig/sysconfig/g" zfp/python/scikit-build-cmake/FindPythonExtensions.cmake
+          mkdir zfp/build;
+          uv run cmake zfp -B zfp/build -DBUILD_ZFPY=ON -DBUILD_TESTING=OFF -DPYTHON_LIBRARY=$PYTHON_LIB -DPYTHON_INCLUDE_DIR=$PYTHON_INCLUDE
+          uv run make -j -C zfp/build
+          uv run sudo make -C zfp/build install
+          uv add "zfpy @ ./zfp"
         shell: alpine.sh {0}
 
-
-      - name: Install zarr-python
-        # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out
-        # so we can have some tests of our minimum version of numpy (1.24)
-        if: matrix.python-version != '3.11'
-        run: uv add zarr>=3
+      - name: Install numcodecs
+        run: |
+          export DISABLE_NUMCODECS_AVX2=""
+          uv venv
+          uv pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
         shell: alpine.sh {0}
 
-
       - name: List installed packages
         run: uv pip list
         shell: alpine.sh {0}

.github/workflows/ci.yaml

@@ -52,12 +52,6 @@ jobs:
           export DISABLE_NUMCODECS_AVX2=""
           python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
 
-      - name: Install zarr-python
-        # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out
-        # so we can have some tests of our minimum version of numpy (1.24)
-        if: matrix.python-version != '3.11'
-        run: python -m pip install zarr>=3
-
       - name: List installed packages
         run: python -m pip list
 
@@ -75,6 +69,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        zarr-version: ["312", "313"]
 
     defaults:
       run:
@@ -93,10 +89,10 @@ jobs:
           pixi-version: v0.49.0
           cache: false
 
-      - name: List deps
+      - name: List installed packages
         shell: "bash -l {0}"
-        run: pixi run -e default hatch run test:list-deps
+        run: pixi run ls-deps-${{matrix.zarr-version}}
 
-      - name: Run tests
+      - name: Run tests with Zarr ${{ matrix.zarr-version }}
         shell: "bash -l {0}"
-        run: pixi run -e default hatch run test:test-zarr
+        run: pixi run test-zarr-${{ matrix.zarr-version }}

.github/workflows/wheel.yaml

@@ -23,8 +23,8 @@ jobs:
         os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-13, macos-14]
     env:
       CIBW_TEST_COMMAND: python -c "import numcodecs"
-      CIBW_BUILD: "cp311-* cp312-* cp313-*"
-      CIBW_SKIP: "pp* *-musllinux_* *win32 *_i686 *_s390x"
+      CIBW_BUILD: "cp311-* cp312-* cp313-* cp314-*"
+      CIBW_SKIP: "*-musllinux_* *win32 *_i686 *_s390x"
       # note: CIBW_ENVIRONMENT is now set in pyproject.toml
 
     steps:
@@ -33,7 +33,7 @@ jobs:
           submodules: true
           fetch-depth: 0  # required for version resolution for nightly wheels
 
-      - uses: pypa/cibuildwheel@v2.23.2
+      - uses: pypa/cibuildwheel@v3.1.0
 
       - uses: actions/upload-artifact@v4
         with:
@@ -69,11 +69,11 @@ jobs:
     needs: [build_wheels, build_sdist]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v6
         with:
           name: sdist
           path: dist
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v6
         with:
           pattern: wheels-*
           path: dist
@@ -89,11 +89,11 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v6
         with:
           name: sdist
           path: dist
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v6
         with:
           pattern: wheels-*
           path: dist
@@ -110,7 +110,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v6
         with:
           pattern: wheels-*
           path: dist

.pre-commit-config.yaml

@@ -30,4 +30,4 @@ repos:
       hooks:
       -   id: mypy
           args: [--config-file, pyproject.toml]
-          additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr>=3']
+          additional_dependencies: [numpy, pytest, google-crc32c, crc32c, zfpy, 'zarr>=3.1.3']

numcodecs/checksum32.py

@@ -14,7 +14,7 @@
 
 _crc32c: ModuleType | None = None
 with suppress(ImportError):
-    import crc32c as _crc32c  # type: ignore[no-redef, unused-ignore]
+    import google_crc32c as _crc32c  # type: ignore[no-redef, unused-ignore]
 
 CHECKSUM_LOCATION = Literal['start', 'end']
 
@@ -179,5 +179,11 @@ class CRC32C(Checksum32):
         """
 
         codec_id = 'crc32c'
-        checksum = _crc32c.crc32c  # type: ignore[union-attr]
         location = 'end'
+
+        @staticmethod
+        def checksum(data: Buffer, value: int = 0) -> int:
+            if value == 0:
+                return _crc32c.value(data)  # type: ignore[union-attr]
+            else:
+                return _crc32c.extend(value, data)  # type: ignore[union-attr]

numcodecs/tests/test_checksum32.py

@@ -145,6 +145,25 @@ def test_crc32c_checksum():
     assert np.frombuffer(buf, dtype="<u4", offset=(len(buf) - 4))[0] == np.uint32(4218238699)
 
 
+@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed")
+def test_crc32c_incremental():
+    """Test that CRC32C.checksum supports incremental calculation via value parameter."""
+    # Test incremental checksum calculation (for API compatibility)
+    data1 = np.frombuffer(b"hello", dtype='uint8')
+    data2 = np.frombuffer(b" world", dtype='uint8')
+    full_data = np.frombuffer(b"hello world", dtype='uint8')
+
+    # Calculate checksum in one go
+    checksum_full = CRC32C.checksum(full_data)
+
+    # Calculate incrementally using the value parameter
+    checksum_part1 = CRC32C.checksum(data1, 0)
+    checksum_part2 = CRC32C.checksum(data2, checksum_part1)
+
+    # Both methods should produce the same result
+    assert checksum_full == checksum_part2
+
+
 @pytest.mark.parametrize("codec", codecs)
 def test_err_checksum(codec):
     arr = np.arange(0, 64, dtype="uint8")