Merge pull request #8 from eugo-inc/zarr-developers-main

Sync upstream

Author: BwL1289

.github/workflows/ci.yaml

@@ -13,94 +13,60 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.12", "3.13"]
-        # macos-13 is an intel runner, macos-14 is a arm64 runner
-        platform: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        # macos-13 is an intel runner, macos-14 is an arm64 runner
+        platform: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-13, macos-14]
+
+    defaults:
+      run:
+        shell: bash -el {0}
 
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
         with:
           submodules: recursive
+          fetch-depth: 0 # required for version resolution
 
       - name: Set up Conda
-        uses: conda-incubator/[email protected].0
+        uses: conda-incubator/[email protected].1
         with:
-            channels: conda-forge
-            miniforge-version: latest
-            python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          miniforge-version: latest
+          python-version: ${{ matrix.python-version }}
 
-      - name: Show info about `base` environment
-        shell: "bash -l {0}"
-        run: |
-          conda info
-          conda config --show-sources
-          conda list --show-channel-urls
-
-      - name: Set up `env`
-        shell: "bash -l {0}"
-        run: >
-          conda create -n env
-          c-compiler cxx-compiler
-          python=${{matrix.python-version}} wheel pip
+      - name: Install compilers
+        run: conda install -y c-compiler cxx-compiler
 
       - name: Install clang
-        shell: "bash -l {0}"
         if: matrix.platform == 'macos-13'
-        run: |
-          conda activate env
-          conda install -y 'clang>=12.0.1,<17'
+        run: conda install -y 'clang>=12.0.1,<17'
 
-      - name: Show info about `env` environment
-        shell: "bash -l {0}"
+      - name: Show conda environment info
         run: |
-          conda list --show-channel-urls -n env
+          conda info
+          conda config --show-sources
+          conda list --show-channel-urls
 
       - name: Install numcodecs
-        shell: "bash -l {0}"
         run: |
-          conda activate env
           export DISABLE_NUMCODECS_AVX2=""
-          # TODO: put back zfpy import when it supports numpy 2.0
-          python -m pip install -v -e .[test,test_extras,msgpack,crc32c]
-
-      - name: Install pcodec
-        if: matrix.python-version != '3.13'
-        shell: "bash -l {0}"
-        run: |
-            conda activate env
-            python -m pip install -v ".[pcodec]"
+          python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
 
       - name: Install zarr-python
-        shell: "bash -l {0}"
         # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out
         # so we can have some tests of our minimum version of numpy (1.24)
         if: matrix.python-version != '3.11'
-        run: |
-          conda activate env
-          # TODO: remove --pre option when zarr v3 is out
-          python -m pip install --pre zarr>=3.0.0b2
-
-      # This is used to test with zfpy, which does not yet support numpy 2.0
-      - name: Install older numpy and zfpy
-        if: matrix.python-version == '3.11'
-        shell: "bash -l {0}"
-        run: |
-          conda activate env
-          python -m pip install -v ".[zfpy]"
+        run: python -m pip install zarr>=3
 
       - name: List installed packages
-        shell: "bash -l {0}"
-        run: |
-          conda activate env
-          python -m pip list
+        run: python -m pip list
 
       - name: Run tests
         shell: "bash -l {0}"
-        run: |
-          conda activate env
-          pytest -v
+        run: pytest -v
 
       - uses: codecov/codecov-action@v5
         with:
+          fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true

.github/workflows/wheel.yaml

@@ -13,8 +13,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # macos-13 is an intel runner, macos-14 is a arm64 runner
-        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        # macos-13 is an intel runner, macos-14 is an arm64 runner
+        os: [ubuntu-latest, ubuntu-22.04-arm, windows-latest, macos-13, macos-14]
     env:
       CIBW_TEST_COMMAND: python -c "import numcodecs"
       CIBW_BUILD: "cp311-* cp312-* cp313-*"
@@ -26,7 +26,7 @@ jobs:
         with:
           submodules: true
 
-      - uses: pypa/cibuildwheel@v2.21.3
+      - uses: pypa/cibuildwheel@v2.23.2
 
       - uses: actions/upload-artifact@v4
         with:

.pre-commit-config.yaml

@@ -1,11 +1,10 @@
 exclude: ^fixture/  # files with trailing whitespaces on purpose
 ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autoupdate_schedule: "monthly"
   autofix_commit_msg: "style: pre-commit fixes"
   autofix_prs: false
 default_stages: [pre-commit, pre-push]
-default_language_version:
-  python: python3
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
@@ -15,20 +14,20 @@ repos:
     - id: debug-statements
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.4
+    rev: v0.11.12
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
     - id: ruff-format
 
   - repo: https://github.com/scientific-python/cookie
-    rev: 2024.08.19
+    rev: 2025.05.02
     hooks:
     - id: sp-repo-review
 
   -   repo: https://github.com/pre-commit/mirrors-mypy
-      rev: 'v1.13.0'
+      rev: v1.16.0
       hooks:
       -   id: mypy
           args: [--config-file, pyproject.toml]
-          additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr>=3.0.0b2']
+          additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr>=3']

adhoc/blosc_memleak_check.py

@@ -1,10 +1,9 @@
 import sys
 
+import numcodecs
 import numpy as np
 from numpy.testing import assert_array_equal
 
-import numcodecs
-
 codec = numcodecs.Blosc()
 data = np.arange(int(sys.argv[1]))
 for _ in range(int(sys.argv[2])):

docs/compression/blosc.rst

@@ -18,15 +18,9 @@ Blosc
 Helper functions
 ----------------
 
-.. autofunction:: init
-.. autofunction:: destroy
-.. autofunction:: compname_to_compcode
 .. autofunction:: list_compressors
 .. autofunction:: get_nthreads
 .. autofunction:: set_nthreads
-.. autofunction:: cbuffer_sizes
 .. autofunction:: cbuffer_complib
-.. autofunction:: cbuffer_metainfo
 .. autofunction:: compress
 .. autofunction:: decompress
-.. autofunction:: decompress_partial

docs/compression/zstd.rst

@@ -7,6 +7,9 @@ Zstd
     .. autoattribute:: codec_id
     .. automethod:: encode
     .. automethod:: decode
+        .. note::
+            If the compressed data does not contain the decompressed size, streaming
+            decompression will be used.
     .. automethod:: get_config
     .. automethod:: from_config
 

docs/contributing.rst

@@ -203,12 +203,11 @@ contributors.
 Merging pull requests
 ~~~~~~~~~~~~~~~~~~~~~
 
-Pull requests submitted by an external contributor should be reviewed and approved by at least
-one core developers before being merged. Ideally, pull requests submitted by a core developer
-should be reviewed and approved by at least one other core developers before being merged.
+Pull requests should be reviewed and approved by at least one core developer
+(other than the pull request author) before being merged.
 
-Pull requests should not be merged until all CI checks have passed (Travis, AppVeyor,
-Codecov) against code that has had the latest main merged in.
+Pull requests should not be merged until all CI checks have passed against code
+that has had the latest main merged in.
 
 Compatibility and versioning policies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -278,6 +277,10 @@ hard-and-fast rules, e.g., it is fine to make a minor release to make a single n
 feature available; equally, it is fine to make a minor release that includes a number of
 changes.
 
+When making a minor release, open an issue stating your intention so other developers
+know that a release is planned. At least a week's notice should be given for other
+developers to be aware of and possibly add to the contents of the release.
+
 Major releases obviously need to be given careful consideration, and should be done as
 infrequently as possible, as they will break existing code and/or affect data
 compatibility in some way.