fix/revert 792 (#802)

* Update pyproject.toml

* update pyproject.toml

* abstract over crc32c modules

* Fix dependency groups/optional-dependencies

* Add deprecation warning

* Put back test and docs optional dependencies

* Keep test-extras

* Update deps in tests

* Add env for local testing

* Only emit warning once

* Add test

* Lint

* Don't install pcodec with python 3.14

* Ignore slow google_crc32c warning

---------

Co-authored-by: Max Jones <[email protected]>

Author: d-v-b

.github/workflows/ci-i386.yml

@@ -63,7 +63,12 @@ jobs:
         run: |
           export DISABLE_NUMCODECS_AVX2=""
           uv venv
-          uv pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
+          # TODO: Remove this conditional when pcodec supports Python 3.14
+          if [[ "${{ matrix.python-version }}" == "3.14" ]]; then
+            uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy]
+          else
+            uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]
+          fi
         shell: alpine.sh {0}
 
       - name: List installed packages

.github/workflows/ci.yaml

@@ -50,7 +50,12 @@ jobs:
       - name: Install numcodecs
         run: |
           export DISABLE_NUMCODECS_AVX2=""
-          python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy]
+          # TODO: Remove this conditional when pcodec supports Python 3.14
+          if [[ "${{ matrix.python-version }}" == "3.14" ]]; then
+            python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy]
+          else
+            python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]
+          fi
 
       - name: List installed packages
         run: python -m pip list

docs/contributing.rst

@@ -101,6 +101,10 @@ like the following::
     $ source ~/pyenv/numcodecs-dev/bin/activate
     $ pip install -e .[docs,test,msgpack,zfpy]
 
+You may need to initialize the submodule for c-blosc:
+
+    $ git submodule update --init --recursive
+
 To verify that your development environment is working, you can run the unit tests::
 
     $ pytest -v

numcodecs/checksum32.py

@@ -1,8 +1,9 @@
 import abc
+import importlib.util
 import struct
+import warnings
 import zlib
-from contextlib import suppress
-from types import ModuleType
+from collections.abc import Callable
 from typing import Literal
 
 import numpy as np
@@ -12,9 +13,32 @@
 from .compat import ensure_contiguous_ndarray, ndarray_copy
 from .jenkins import jenkins_lookup3
 
-_crc32c: ModuleType | None = None
-with suppress(ImportError):
-    import google_crc32c as _crc32c  # type: ignore[no-redef, unused-ignore]
+crc32c_checksum: Callable[[Buffer, int], int] | None
+
+warnings.filterwarnings('once', message='crc32c usage is deprecated.*', category=DeprecationWarning)
+
+if importlib.util.find_spec("google_crc32c") is not None:
+    import google_crc32c
+
+    def crc32c_checksum(data: Buffer, value: int = 0) -> int:
+        if value == 0:
+            return google_crc32c.value(data)
+        else:
+            return google_crc32c.extend(value, data)
+
+elif importlib.util.find_spec("crc32c") is not None:
+    warnings.warn(
+        "crc32c usage is deprecated since numcodecs v0.16.4. "
+        "It is recommended to install google_crc32c instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    import crc32c
+
+    def crc32c_checksum(data: Buffer, value: int = 0) -> int:
+        return crc32c.crc32c(data, value)
+else:
+    crc32c_checksum = None
 
 CHECKSUM_LOCATION = Literal['start', 'end']
 
@@ -167,7 +191,7 @@ def decode(self, buf, out=None):
         return memoryview(b[:-4])
 
 
-if _crc32c:
+if crc32c_checksum is not None:
 
     class CRC32C(Checksum32):
         """Codec add a crc32c checksum to the buffer.
@@ -183,7 +207,4 @@ class CRC32C(Checksum32):
 
         @staticmethod
         def checksum(data: Buffer, value: int = 0) -> int:
-            if value == 0:
-                return _crc32c.value(data)  # type: ignore[union-attr]
-            else:
-                return _crc32c.extend(value, data)  # type: ignore[union-attr]
+            return crc32c_checksum(data, value)  # type: ignore[misc]

numcodecs/tests/test_checksum32.py

@@ -1,3 +1,4 @@
+import importlib
 import itertools
 from contextlib import suppress
 
@@ -15,6 +16,9 @@
 )
 
 has_crc32c = False
+has_google_crc32c = importlib.util.find_spec("google_crc32c") is not None
+has_legacy_crc32c = importlib.util.find_spec("crc32c") is not None
+
 with suppress(ImportError):
     from numcodecs.checksum32 import CRC32C
 
@@ -104,7 +108,7 @@ def test_err_location():
     with pytest.raises(ValueError):
         Adler32(location="foo")
     if not has_crc32c:
-        pytest.skip("Needs `crc32c` installed")
+        pytest.skip("Needs `crc32c` or `google_crc32c` installed")
     with pytest.raises(ValueError):
         CRC32C(location="foo")
 
@@ -118,11 +122,15 @@ def test_err_location():
         "Adler32(location='end')",
         pytest.param(
             "CRC32C(location='start')",
-            marks=pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed"),
+            marks=pytest.mark.skipif(
+                not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed"
+            ),
         ),
         pytest.param(
             "CRC32C(location='end')",
-            marks=pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed"),
+            marks=pytest.mark.skipif(
+                not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed"
+            ),
         ),
     ],
 )
@@ -141,7 +149,7 @@ def test_backwards_compatibility(codec_id, codec_instance):
     check_backwards_compatibility(codec_id, arrays, [codec_instance])
 
 
-@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed")
+@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed")
 def test_backwards_compatibility_crc32c():
     check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()])
 
@@ -164,14 +172,14 @@ def test_err_out_too_small(codec):
         codec.decode(codec.encode(arr), out)
 
 
-@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed")
+@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed")
 def test_crc32c_checksum():
     arr = np.arange(0, 64, dtype="uint8")
     buf = CRC32C(location="end").encode(arr)
     assert np.frombuffer(buf, dtype="<u4", offset=(len(buf) - 4))[0] == np.uint32(4218238699)
 
 
-@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed")
+@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed")
 def test_crc32c_incremental():
     """Test that CRC32C.checksum supports incremental calculation via value parameter."""
     # Test incremental checksum calculation (for API compatibility)
@@ -197,3 +205,23 @@ def test_err_checksum(codec):
     buf[-1] = 0  # corrupt the checksum
     with pytest.raises(RuntimeError):
         codec.decode(buf)
+
+
+@pytest.mark.skipif(
+    has_google_crc32c or not has_legacy_crc32c,
+    reason="Only runs when legacy `crc32c` is used (not `google_crc32c`)",
+)
+def test_crc32c_deprecation_warning():
+    """Test that using legacy crc32c (not google_crc32c) issues a deprecation warning."""
+    import sys
+
+    # Remove the module from cache to force re-import and trigger the warning
+    if 'numcodecs.checksum32' in sys.modules:
+        del sys.modules['numcodecs.checksum32']
+
+    with pytest.warns(
+        DeprecationWarning, match="crc32c usage is deprecated since numcodecs v0.16.4"
+    ):
+        import numcodecs.checksum32
+
+        importlib.reload(numcodecs.checksum32)

pyproject.toml

@@ -47,6 +47,21 @@ Documentation = "https://numcodecs.readthedocs.io/"
 Homepage = "https://github.com/zarr-developers/numcodecs"
 
 [project.optional-dependencies]
+msgpack = [
+    "msgpack",
+]
+zfpy = [
+    "zfpy>=1.0.0"
+]
+pcodec = [
+    "pcodec>=0.3,<0.4",
+]
+crc32c = [
+    "crc32c>=2.7",
+]
+google_crc32c = [
+    "google-crc32c>=1.5"
+]
 docs = [
     "sphinx",
     "sphinx-issues",
@@ -84,19 +99,6 @@ test-zarr-main = [
     "crc32c",
 ]
 
-msgpack = [
-    "msgpack",
-]
-zfpy = [
-    "zfpy>=1.0.0"
-]
-pcodec = [
-    "pcodec>=0.3,<0.4",
-]
-crc32c = [
-    "google-crc32c>=1.5",
-]
-
 [tool.setuptools]
 package-dir = {"" = "."}
 packages = ["numcodecs", "numcodecs.tests"]
@@ -152,6 +154,7 @@ log_cli_level = "INFO"
 xfail_strict = true
 filterwarnings = [
     "error",
+    "ignore:As the c extension couldn't be imported:RuntimeWarning", # Ignore warning about pure python google_crc32c (on Python 3.14)
 ]
 
 [tool.cibuildwheel]
@@ -251,20 +254,39 @@ conflicts = [
     ]
 ]
 
-[tool.pixi.project]
+[tool.pixi.workspace]
 channels = ["conda-forge"]
 platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"]
 
 [tool.pixi.dependencies]
+python = "=3.14"
 clang = ">=19.1.7,<20"
 c-compiler = ">=1.9.0,<2"
 cxx-compiler = ">=1.9.0,<2"
 uv = "*"
 
+[tool.pixi.feature.test.pypi-dependencies]
+numcodecs = { path = ".", editable = true, extras = ["test","test_extras","msgpack","zfpy"] }
+
+[tool.pixi.feature.test-google-crc32c.pypi-dependencies]
+numcodecs = { path = ".", editable = true, extras = ["google_crc32c"] }
+
+[tool.pixi.feature.test-crc32c.pypi-dependencies]
+numcodecs = { path = ".", editable = true, extras = ["crc32c"] }
+
+[tool.pixi.environments]
+default = { solve-group = "default" }
+test = ["test"]
+test-crc32c = ["test", "test-crc32c"]
+test-google-crc32c = ["test", "test-google-crc32c"]
+
 [tool.pixi.tasks]
 ls-deps-312 = "uv run --group test-zarr-312 uv pip freeze"
 ls-deps-313 = "uv run --group test-zarr-313 uv pip freeze"
 ls-deps-main = "uv run --group test-zarr-main uv pip freeze"
 test-zarr-312 = "uv run --group test-zarr-312 pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py"
 test-zarr-313 = "uv run --group test-zarr-313 pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py"
 test-zarr-main = "uv run --group test-zarr-main pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py"
+
+[tool.pixi.feature.test.tasks]
+run-tests = "pytest -v"