Merge branch 'main' into fix_pcodec_soft

Author: dstansby

.github/workflows/ci.yaml

@@ -16,89 +16,55 @@ jobs:
         # macos-13 is an intel runner, macos-14 is a arm64 runner
         platform: [ubuntu-latest, windows-latest, macos-13, macos-14]
 
+    defaults:
+      run:
+        shell: bash -el {0}
+
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
         with:
           submodules: recursive
+          fetch-depth: 0 # required for version resolution
 
       - name: Set up Conda
         uses: conda-incubator/[email protected]
         with:
-            channels: conda-forge
-            miniforge-version: latest
-            python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          miniforge-version: latest
+          python-version: ${{ matrix.python-version }}
 
-      - name: Show info about `base` environment
-        shell: "bash -l {0}"
-        run: |
-          conda info
-          conda config --show-sources
-          conda list --show-channel-urls
-
-      - name: Set up `env`
-        shell: "bash -l {0}"
-        run: >
-          conda create -n env
-          c-compiler cxx-compiler
-          python=${{matrix.python-version}} wheel pip
+      - name: Install compilers
+        run: conda install -y c-compiler cxx-compiler
 
       - name: Install clang
-        shell: "bash -l {0}"
         if: matrix.platform == 'macos-13'
-        run: |
-          conda activate env
-          conda install -y 'clang>=12.0.1,<17'
+        run: conda install -y 'clang>=12.0.1,<17'
 
-      - name: Show info about `env` environment
-        shell: "bash -l {0}"
+      - name: Show conda environment info
         run: |
-          conda list --show-channel-urls -n env
+          conda info
+          conda config --show-sources
+          conda list --show-channel-urls
 
       - name: Install numcodecs
-        shell: "bash -l {0}"
         run: |
-          conda activate env
           export DISABLE_NUMCODECS_AVX2=""
-          # TODO: put back zfpy import when it supports numpy 2.0
-          python -m pip install -v -e .[test,test_extras,msgpack,crc32c]
-
-      - name: Install pcodec
-        if: matrix.python-version != '3.13'
-        shell: "bash -l {0}"
-        run: |
-            conda activate env
-            python -m pip install -v ".[pcodec]"
+          python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
 
       - name: Install zarr-python
-        shell: "bash -l {0}"
         # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out
         # so we can have some tests of our minimum version of numpy (1.24)
         if: matrix.python-version != '3.11'
-        run: |
-          conda activate env
-          # TODO: remove --pre option when zarr v3 is out
-          python -m pip install --pre zarr>=3.0.0b2
-
-      # This is used to test with zfpy, which does not yet support numpy 2.0
-      - name: Install older numpy and zfpy
-        if: matrix.python-version == '3.11'
-        shell: "bash -l {0}"
-        run: |
-          conda activate env
-          python -m pip install -v ".[zfpy]"
+        # TODO: remove --pre option when zarr v3 is out
+        run: python -m pip install --pre zarr>=3.0.0b2
 
       - name: List installed packages
-        shell: "bash -l {0}"
-        run: |
-          conda activate env
-          python -m pip list
+        run: python -m pip list
 
       - name: Run tests
         shell: "bash -l {0}"
-        run: |
-          conda activate env
-          pytest -v
+        run: pytest -v
 
       - uses: codecov/codecov-action@v5
         with:

.pre-commit-config.yaml

@@ -1,11 +1,10 @@
 exclude: ^fixture/  # files with trailing whitespaces on purpose
 ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autoupdate_schedule: "monthly"
   autofix_commit_msg: "style: pre-commit fixes"
   autofix_prs: false
 default_stages: [pre-commit, pre-push]
-default_language_version:
-  python: python3
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
@@ -15,7 +14,7 @@ repos:
     - id: debug-statements
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.1
+    rev: v0.8.4
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -27,8 +26,8 @@ repos:
     - id: sp-repo-review
 
   -   repo: https://github.com/pre-commit/mirrors-mypy
-      rev: 'v1.13.0'
+      rev: v1.14.0
       hooks:
       -   id: mypy
           args: [--config-file, pyproject.toml]
-          additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr>=3.0.0b2']
+          additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr>=3.0.0rc1']

adhoc/blosc_memleak_check.py

@@ -1,10 +1,9 @@
 import sys
 
+import numcodecs
 import numpy as np
 from numpy.testing import assert_array_equal
 
-import numcodecs
-
 codec = numcodecs.Blosc()
 data = np.arange(int(sys.argv[1]))
 for _ in range(int(sys.argv[2])):

docs/compression/blosc.rst

@@ -18,15 +18,9 @@ Blosc
 Helper functions
 ----------------
 
-.. autofunction:: init
-.. autofunction:: destroy
-.. autofunction:: compname_to_compcode
 .. autofunction:: list_compressors
 .. autofunction:: get_nthreads
 .. autofunction:: set_nthreads
-.. autofunction:: cbuffer_sizes
 .. autofunction:: cbuffer_complib
-.. autofunction:: cbuffer_metainfo
 .. autofunction:: compress
 .. autofunction:: decompress
-.. autofunction:: decompress_partial

docs/release.rst

@@ -22,6 +22,20 @@ Breaking changes
   are now keyword only, to support the updated API.
   By :user:`Sam Levang <slevang>`, :issue:`623`
 
+Deprecations
+~~~~~~~~~~~~
+The following ``blosc`` funcitons are deprecated, with no replacement.
+This is because they are not intended to be public API.
+
+- ``numcodecs.blosc.init``
+- ``numcodecs.blosc.destroy``
+- ``numcodecs.blosc.compname_to_compcode``
+- ``numcodecs.blosc.cbuffer_sizes``
+- ``numcodecs.blosc.cbuffer_metainfo``
+
+In addition, ``numcodecs.blosc.decompress_partial`` is deprecated as
+has always been experimental and there is no equivalent in the official
+blsoc Python package.
 
 Fixes
 ~~~~~
@@ -31,7 +45,13 @@ Fixes
   Previously importing ``numcodecs.pcodec`` would work if ``pcodec`` is not installed,
   but now it will fail to import. This mirrors the behaviour of other optional dependencies.
   By :user:`John Kirkham <jakirkham>`, :issue:`647`
+* Fixes issues with the upcoming ``zarr`` 3.0.0 release.
+  By :user:`Norman Rzepka <normanrz>`, :issue:`675`
 
+* Removed Version Check: The previous code included a check for the `NumPy` version
+  and a warning if the version was incompatible with `zfpy`.
+  This check has been removed because `zfpy` now supports the newer versions of `NumPy`.
+  By :user:`Meher Gajula <me-her>`, :issue:`672`
 
 Improvements
 ~~~~~~~~~~~~

docs/zarr3.rst

@@ -5,8 +5,8 @@ Zarr 3 codecs
 .. automodule:: numcodecs.zarr3
 
 
-Bytes-to-bytes codecs
----------------------
+Compressors (bytes-to-bytes codecs)
+-----------------------------------
 .. autoclass:: Blosc()
 
     .. autoattribute:: codec_name
@@ -40,58 +40,61 @@ Bytes-to-bytes codecs
     .. autoattribute:: codec_name
 
 
-Array-to-array codecs
----------------------
-.. autoclass:: Delta()
+Checksum codecs (bytes-to-bytes codecs)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Need to be used as ``compressors`` in zarr-python.
+
+.. autoclass:: CRC32()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: BitRound()
+.. autoclass:: CRC32C()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: FixedScaleOffset()
+.. autoclass:: Adler32()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: Quantize()
+.. autoclass:: Fletcher32()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: PackBits()
+.. autoclass:: JenkinsLookup3()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: AsType()
 
-    .. autoattribute:: codec_name
+Filters (array-to-array codecs)
+-------------------------------
+.. autoclass:: Delta()
 
+    .. autoattribute:: codec_name
 
-Bytes-to-bytes checksum codecs
-------------------------------
-.. autoclass:: CRC32()
+.. autoclass:: BitRound()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: CRC32C()
+.. autoclass:: FixedScaleOffset()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: Adler32()
+.. autoclass:: Quantize()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: Fletcher32()
+.. autoclass:: PackBits()
 
     .. autoattribute:: codec_name
 
-.. autoclass:: JenkinsLookup3()
+.. autoclass:: AsType()
 
     .. autoattribute:: codec_name
 
 
-Array-to-bytes codecs
----------------------
+
+Serializers (array-to-bytes codecs)
+-----------------------------------
 .. autoclass:: PCodec()
 
     .. autoattribute:: codec_name

notebooks/benchmark_vlen.ipynb

@@ -24,9 +24,8 @@
     }
    ],
    "source": [
-    "import numpy as np\n",
-    "\n",
     "import numcodecs\n",
+    "import numpy as np\n",
     "\n",
     "numcodecs.__version__"
    ]

numcodecs/__init__.py

@@ -49,7 +49,7 @@
     ncores = multiprocessing.cpu_count()
 except OSError:  # pragma: no cover
     ncores = 1
-blosc.init()
+blosc._init()
 blosc.set_nthreads(min(8, ncores))
 atexit.register(blosc.destroy)
 

numcodecs/astype.py

@@ -49,9 +49,7 @@ def encode(self, buf):
         arr = ensure_ndarray(buf).view(self.decode_dtype)
 
         # convert and copy
-        enc = arr.astype(self.encode_dtype)
-
-        return enc
+        return arr.astype(self.encode_dtype)
 
     def decode(self, buf, out=None):
         # normalise input
@@ -61,9 +59,7 @@ def decode(self, buf, out=None):
         dec = enc.astype(self.decode_dtype)
 
         # handle output
-        out = ndarray_copy(dec, out)
-
-        return out
+        return ndarray_copy(dec, out)
 
     def get_config(self):
         return {

numcodecs/base64.py

@@ -13,8 +13,7 @@ def encode(self, buf):
         # normalise inputs
         buf = ensure_contiguous_ndarray(buf)
         # do compression
-        compressed = _base64.standard_b64encode(buf)
-        return compressed
+        return _base64.standard_b64encode(buf)
 
     def decode(self, buf, out=None):
         # normalise inputs