Merge branch 'main' into zarr3-codecs

normanrz · web-flow · commit c9c8f5ed45d3 · 2024-07-09T15:22:26.000+02:00
diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml
@@ -26,7 +26,7 @@ jobs:
         with:
           submodules: true
 
-      - uses: pypa/cibuildwheel@v2.16.5
+      - uses: pypa/cibuildwheel@v2.19.2
 
       - uses: actions/upload-artifact@v4
         with:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.4.10'
+    rev: 'v0.5.1'
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
diff --git a/docs/contributing.rst b/docs/contributing.rst
@@ -92,12 +92,12 @@ Creating a development environment
 To work with the NumCodecs source code, it is recommended to set up a Python virtual
 environment and install all NumCodecs dependencies using the same versions as are used by
 the core developers and continuous integration services. Assuming you have a Python
-3 interpreter already installed, and have also installed the virtualenv package, and
-you have cloned the NumCodecs source code and your current working directory is the root of
-the repository, you can do something like the following::
+3 interpreter already installed matching the ``requires-python`` constraint from
+``pyproject.toml``, and you have cloned the NumCodecs source code and your
+current working directory is the root of the repository, you can do something
+like the following::
 
-    $ mkdir -p ~/pyenv/numcodecs-dev
-    $ virtualenv --no-site-packages --python=/usr/bin/python3.9 ~/pyenv/numcodecs-dev
+    $ python3 -m venv ~/pyenv/numcodecs-dev
     $ source ~/pyenv/numcodecs-dev/bin/activate
     $ pip install -e .[docs,test,msgpack,zfpy]
 
diff --git a/docs/release.rst b/docs/release.rst
@@ -31,6 +31,8 @@ Fix
   By :user:`Elliott Sales de Andrade <QuLogic>`, :issue:`487`.
 * Fix Upgrade to Zstd 1.5.5 due to potential corruption.
   By :user:`Mark Kittisopikul <mkitti>`, :issue:`429`
+* Add version constraint(<2.0) for numpy in zfpy.
+  By :user:`Tom Liang <px39n>`, :issue:`540`.
 
 Maintenance
 ~~~~~~~~~~~
diff --git a/fixture/pcodec/codec.00/config.json b/fixture/pcodec/codec.00/config.json
@@ -1,8 +1,7 @@
 {
     "delta_encoding_order": null,
     "equal_pages_up_to": 262144,
-    "float_mult_spec": "enabled",
     "id": "pcodec",
-    "int_mult_spec": "enabled",
-    "level": 8
+    "level": 8,
+    "mode_spec": "auto"
 }
diff --git a/fixture/pcodec/codec.01/config.json b/fixture/pcodec/codec.01/config.json
@@ -1,8 +1,7 @@
 {
     "delta_encoding_order": null,
     "equal_pages_up_to": 262144,
-    "float_mult_spec": "enabled",
     "id": "pcodec",
-    "int_mult_spec": "enabled",
-    "level": 1
+    "level": 1,
+    "mode_spec": "auto"
 }
diff --git a/fixture/pcodec/codec.02/config.json b/fixture/pcodec/codec.02/config.json
@@ -1,8 +1,7 @@
 {
     "delta_encoding_order": null,
     "equal_pages_up_to": 262144,
-    "float_mult_spec": "enabled",
     "id": "pcodec",
-    "int_mult_spec": "enabled",
-    "level": 5
+    "level": 5,
+    "mode_spec": "auto"
 }
diff --git a/fixture/pcodec/codec.03/config.json b/fixture/pcodec/codec.03/config.json
@@ -1,8 +1,7 @@
 {
     "delta_encoding_order": null,
     "equal_pages_up_to": 262144,
-    "float_mult_spec": "enabled",
     "id": "pcodec",
-    "int_mult_spec": "enabled",
-    "level": 9
+    "level": 9,
+    "mode_spec": "auto"
 }
diff --git a/fixture/pcodec/codec.04/config.json b/fixture/pcodec/codec.04/config.json
@@ -1,8 +1,7 @@
 {
     "delta_encoding_order": null,
     "equal_pages_up_to": 262144,
-    "float_mult_spec": "disabled",
     "id": "pcodec",
-    "int_mult_spec": "disabled",
-    "level": 8
+    "level": 8,
+    "mode_spec": "classic"
 }
diff --git a/fixture/pcodec/codec.05/config.json b/fixture/pcodec/codec.05/config.json
@@ -1,8 +1,7 @@
 {
     "delta_encoding_order": null,
     "equal_pages_up_to": 300,
-    "float_mult_spec": "enabled",
     "id": "pcodec",
-    "int_mult_spec": "enabled",
-    "level": 8
+    "level": 8,
+    "mode_spec": "auto"
 }
diff --git a/numcodecs/categorize.py b/numcodecs/categorize.py
@@ -44,12 +44,12 @@ def __init__(self, labels, dtype, astype='u1'):
             raise TypeError("only unicode ('U') and object ('O') dtypes are " "supported")
         self.labels = [ensure_text(label) for label in labels]
         self.astype = np.dtype(astype)
-        if self.astype == object:
+        if self.astype == np.dtype(object):
             raise TypeError('encoding as object array not supported')
 
     def encode(self, buf):
         # normalise input
-        if self.dtype == object:
+        if self.dtype == np.dtype(object):
             arr = np.asarray(buf, dtype=object)
         else:
             arr = ensure_ndarray(buf).view(self.dtype)
diff --git a/numcodecs/delta.py b/numcodecs/delta.py
@@ -47,7 +47,7 @@ def __init__(self, dtype, astype=None):
             self.astype = self.dtype
         else:
             self.astype = np.dtype(astype)
-        if self.dtype == object or self.astype == object:
+        if self.dtype == np.dtype(object) or self.astype == np.dtype(object):
             raise ValueError('object arrays are not supported')
 
     def encode(self, buf):
diff --git a/numcodecs/fixedscaleoffset.py b/numcodecs/fixedscaleoffset.py
@@ -78,7 +78,7 @@ def __init__(self, offset, scale, dtype, astype=None):
             self.astype = self.dtype
         else:
             self.astype = np.dtype(astype)
-        if self.dtype == object or self.astype == object:
+        if self.dtype == np.dtype(object) or self.astype == np.dtype(object):
             raise ValueError('object arrays are not supported')
 
     def encode(self, buf):
diff --git a/numcodecs/pcodec.py b/numcodecs/pcodec.py
@@ -5,7 +5,7 @@
 from numcodecs.compat import ensure_contiguous_ndarray
 
 try:
-    from pcodec import standalone, ChunkConfig, PagingSpec
+    from pcodec import standalone, ChunkConfig, PagingSpec, ModeSpec
 except ImportError:  # pragma: no cover
     standalone = None
 
@@ -20,8 +20,8 @@ class PCodec(numcodecs.abc.Codec):
 
     See `PCodec Repo <https://github.com/mwlon/pcodec>`_ for more information.
 
-    PCodec supports only the following numerical dtypes: uint32, unit64, int32,
-    int64, float32, and float64.
+    PCodec supports only the following numerical dtypes: uint16, uint32, uint64,
+    int16, int32, int64, float16, float32, and float64.
 
     Parameters
     ----------
@@ -31,14 +31,11 @@ class PCodec(numcodecs.abc.Codec):
     delta_encoding_order : init or None
         Either a delta encoding level from 0-7 or None. If set to None, pcodec
         will try to infer the optimal delta encoding order.
-    int_mult_spec : {'enabled', 'disabled'}
-        If enabled, pcodec will consider using int mult mode, which can
-        substantially improve compression ratio but decrease speed in some cases
-        for integer types.
-    float_mult_spec : {'enabled', 'disabled'}
-        If enabled, pcodec will consider using float mult mode, which can
-        substantially improve compression ratio but decrease speed in some cases
-        for float types.
+    mode_spec : {'auto', 'classic'}
+        Configures whether Pcodec should try to infer the best "mode" or
+        structure of the data (e.g. approximate multiples of 0.1) to improve
+        compression ratio, or skip this step and just use the numbers as-is
+        (Classic mode).
     equal_pages_up_to : int
         Divide the chunk into equal pages of up to this many numbers.
     """
@@ -49,9 +46,9 @@ def __init__(
         self,
         level: int = 8,
         delta_encoding_order: Optional[int] = None,
-        int_mult_spec: Literal["enabled", "disabled"] = "enabled",
-        float_mult_spec: Literal["enabled", "disabled"] = "enabled",
         equal_pages_up_to: int = 262144,
+        # TODO one day, add support for the Try* mode specs
+        mode_spec: Literal['auto', 'classic'] = 'auto',
     ):
         if standalone is None:  # pragma: no cover
             raise ImportError("pcodec must be installed to use the PCodec codec.")
@@ -60,20 +57,25 @@ def __init__(
         # match other codecs
         self.level = level
         self.delta_encoding_order = delta_encoding_order
-        self.int_mult_spec = int_mult_spec
-        self.float_mult_spec = float_mult_spec
         self.equal_pages_up_to = equal_pages_up_to
+        self.mode_spec = mode_spec
 
     def encode(self, buf):
         buf = ensure_contiguous_ndarray(buf)
 
+        match self.mode_spec:
+            case 'auto':
+                mode_spec = ModeSpec.auto()
+            case 'classic':
+                mode_spec = ModeSpec.classic()
+            case _:
+                raise ValueError(f"unknown value for mode_spec: {self.mode_spec}")
         paging_spec = PagingSpec.equal_pages_up_to(self.equal_pages_up_to)
 
         config = ChunkConfig(
             compression_level=self.level,
             delta_encoding_order=self.delta_encoding_order,
-            int_mult_spec=self.int_mult_spec,
-            float_mult_spec=self.float_mult_spec,
+            mode_spec=mode_spec,
             paging_spec=paging_spec,
         )
         return standalone.simple_compress(buf, config)
diff --git a/numcodecs/tests/test_pcodec.py b/numcodecs/tests/test_pcodec.py
@@ -24,7 +24,7 @@
     PCodec(level=1),
     PCodec(level=5),
     PCodec(level=9),
-    PCodec(float_mult_spec="disabled", int_mult_spec="disabled"),
+    PCodec(mode_spec='classic'),
     PCodec(equal_pages_up_to=300),
 ]
 
@@ -57,10 +57,15 @@ def test_config():
     check_config(codec)
 
 
+def test_invalid_config_error():
+    with pytest.raises(ValueError):
+        codec = PCodec(mode_spec='bogus')
+        check_encode_decode_array_to_bytes(arrays[0], codec)
+
+
 def test_repr():
     check_repr(
-        "PCodec(delta_encoding_order=None, equal_pages_up_to=262144, float_mult_spec='enabled', "
-        "int_mult_spec='enabled', level=3)"
+        "PCodec(delta_encoding_order=None, equal_pages_up_to=262144, level=3, mode_spec='auto')"
     )
 
 
diff --git a/numcodecs/zfpy.py b/numcodecs/zfpy.py
@@ -1,9 +1,26 @@
 from contextlib import suppress
+from importlib.metadata import PackageNotFoundError, version
+import warnings
 
 _zfpy = None
-with suppress(ImportError):
-    import zfpy as _zfpy
 
+_zfpy_version: tuple = ()
+with suppress(PackageNotFoundError):
+    _zfpy_version = tuple(map(int, version("zfpy").split(".")))
+
+if _zfpy_version:
+    # Check NumPy version
+    _numpy_version: tuple = tuple(map(int, version("numpy").split('.')))
+    if _numpy_version >= (2, 0, 0) and _zfpy_version <= (1, 0, 1):  # pragma: no cover
+        _zfpy_version = ()
+        warnings.warn(
+            "NumPy version >= 2.0.0 detected. The zfpy library is incompatible with this version of NumPy. "
+            "Please downgrade to NumPy < 2.0.0 or wait for an update from zfpy.",
+            UserWarning,
+        )
+    else:
+        with suppress(ImportError):
+            import zfpy as _zfpy
 
 if _zfpy:
     from .abc import Codec
diff --git a/pyproject.toml b/pyproject.toml
@@ -64,9 +64,10 @@ msgpack = [
 ]
 zfpy = [
     "zfpy>=1.0.0",
+    "numpy<2.0.0",    
 ]
 pcodec = [
-    "pcodec>=0.1.0",
+    "pcodec>=0.2.0",
 ]
 
 [project.entry-points."zarr.codecs"]

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"delta_encoding_order": null,`
`3`	`3`	`"equal_pages_up_to": 262144,`
`4`		`- "float_mult_spec": "enabled",`
`5`	`4`	`"id": "pcodec",`
`6`		`- "int_mult_spec": "enabled",`
`7`		`- "level": 8`
	`5`	`+ "level": 8,`
	`6`	`+ "mode_spec": "auto"`
`8`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"delta_encoding_order": null,`
`3`	`3`	`"equal_pages_up_to": 300,`
`4`		`- "float_mult_spec": "enabled",`
`5`	`4`	`"id": "pcodec",`
`6`		`- "int_mult_spec": "enabled",`
`7`		`- "level": 8`
	`5`	`+ "level": 8,`
	`6`	`+ "mode_spec": "auto"`
`8`	`7`	`}`