Merge branch 'support/v2' into array-hexdigest-zlibng-v2

Author: dstansby

.readthedocs.yaml

@@ -4,6 +4,9 @@ build:
   os: ubuntu-20.04
   tools:
     python: "3.11"
+  jobs:
+    post_checkout:
+      - git fetch --unshallow || true
 
 sphinx:
   configuration: docs/conf.py

docs/conf.py

@@ -44,6 +44,7 @@
     "sphinx_issues",
     "sphinx_copybutton",
     "sphinx_design",
+    "pytest_doctestplus.sphinx.doctestplus",
 ]
 
 numpydoc_show_class_members = False
@@ -72,8 +73,8 @@
 copyright = "2024, Zarr Developers"
 author = "Zarr Developers"
 
-version = get_version("zarr")
 release = get_version("zarr")
+version = get_version("zarr")
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/release.rst

@@ -20,8 +20,8 @@ Release notes
 
 .. _release_2.18.4:
 
-2.18.4 (unreleased)
--------------------
+2.18.4
+------
 
 Enhancements
 ~~~~~~~~~~~~
@@ -36,6 +36,9 @@ Maintenance
   ``zarr-python`` if you can install it, but to reduce our maintenance
   burden we will no longer run our compatibility tests for it.
   By :user:`David Stansby <dstansby>` (:issue:`2344`).
+* Excluded versions 0.14.0 and 0.14.1 of numcodecs, due to a bug in the implementation of
+  the Delta filter (see https://github.com/zarr-developers/numcodecs/issues/653 for more information).
+  By :user:`David Stansby <dstansby>` (:issue:`2544`).
 
 Deprecations
 ~~~~~~~~~~~~

docs/tutorial.rst

@@ -809,7 +809,7 @@ Another storage alternative is the :class:`zarr.storage.DBMStore` class, added
 in Zarr version 2.2. This class allows any DBM-style database to be used for
 storing an array or group. Here is an example using a Berkeley DB B-tree
 database for storage (requires `bsddb3
-<https://www.jcea.es/programacion/pybsddb.htm>`_ to be installed)::
+<https://www.jcea.es/programacion/pybsddb.htm>`_ to be installed):
 
 .. doctest-requires:: bsddb3
 

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=64.0.0", "setuptools-scm>1.5.4"]
+requires = ["setuptools>=64", "setuptools-scm>8"]
 build-backend = "setuptools.build_meta"
 
 
@@ -13,7 +13,7 @@ dependencies = [
     'asciitree',
     'numpy>=1.24',
     'fasteners; sys_platform != "emscripten"',
-    'numcodecs>=0.10.0',
+    'numcodecs>=0.10.0,!=0.14.0,!=0.14.1',
 ]
 dynamic = ["version"]
 classifiers = [
@@ -42,7 +42,8 @@ docs = [
     'sphinx-copybutton',
     'pydata-sphinx-theme',
     'numpydoc',
-    'numcodecs[msgpack]',
+    'numcodecs[msgpack]!=0.14.0,!=0.14.1',
+    'pytest-doctestplus',
 ]
 
 [project.urls]
@@ -67,9 +68,8 @@ packages = ["zarr", "zarr._storage", "zarr.tests"]
 license-files = ["LICENSE.txt"]
 
 [tool.setuptools_scm]
-version_scheme = "guess-next-dev"
 local_scheme = "dirty-tag"
-write_to = "zarr/version.py"
+version_file = "zarr/version.py"
 
 [tool.ruff]
 line-length = 100

requirements_dev_optional.txt

@@ -15,7 +15,7 @@ pymongo==4.10.1
 # optional test requirements
 coverage
 pytest-cov==5.0.0
-pytest-doctestplus==1.2.1
+pytest-doctestplus==1.3.0
 pytest-timeout==2.3.1
 h5py==3.12.1
 fsspec==2023.12.2

zarr/convenience.py

@@ -31,6 +31,8 @@
 
 _builtin_open = open  # builtin open is later shadowed by a local open function
 
+__doctest_requires__ = {("*"): ["numpy>=2.2"]}
+
 
 def _check_and_update_path(store: BaseStore, path):
     if getattr(store, "_store_version", 2) > 2 and not path:
@@ -174,13 +176,13 @@ def save_array(store: StoreLike, arr, *, zarr_version=None, path=None, **kwargs)
         >>> arr = np.arange(10000)
         >>> zarr.save_array('data/example.zarr', arr)
         >>> zarr.load('data/example.zarr')
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
 
     Save an array to a single file (uses a :class:`ZipStore`)::
 
         >>> zarr.save_array('data/example.zip', arr)
         >>> zarr.load('data/example.zip')
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
 
     """
     may_need_closing = _might_close(store)
@@ -234,9 +236,9 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs):
         >>> loader
         <LazyLoader: arr_0, arr_1>
         >>> loader['arr_0']
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
         >>> loader['arr_1']
-        array([10000,  9999,  9998, ...,     3,     2,     1])
+        array([10000,  9999,  9998, ...,     3,     2,     1], shape=(10000,))
 
     Save several arrays using named keyword arguments::
 
@@ -245,9 +247,9 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs):
         >>> loader
         <LazyLoader: bar, foo>
         >>> loader['foo']
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
         >>> loader['bar']
-        array([10000,  9999,  9998, ...,     3,     2,     1])
+        array([10000,  9999,  9998, ...,     3,     2,     1], shape=(10000,))
 
     Store several arrays in a single zip file (uses a :class:`ZipStore`)::
 
@@ -256,9 +258,9 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs):
         >>> loader
         <LazyLoader: bar, foo>
         >>> loader['foo']
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
         >>> loader['bar']
-        array([10000,  9999,  9998, ...,     3,     2,     1])
+        array([10000,  9999,  9998, ...,     3,     2,     1], shape=(10000,))
 
     Notes
     -----
@@ -316,13 +318,13 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs):
         >>> arr = np.arange(10000)
         >>> zarr.save('data/example.zarr', arr)
         >>> zarr.load('data/example.zarr')
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
 
     Save an array to a Zip file (uses a :class:`ZipStore`)::
 
         >>> zarr.save('data/example.zip', arr)
         >>> zarr.load('data/example.zip')
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
 
     Save several arrays to a directory on the file system (uses a
     :class:`DirectoryStore` and stores arrays in a group)::
@@ -336,9 +338,9 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs):
         >>> loader
         <LazyLoader: arr_0, arr_1>
         >>> loader['arr_0']
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
         >>> loader['arr_1']
-        array([10000,  9999,  9998, ...,     3,     2,     1])
+        array([10000,  9999,  9998, ...,     3,     2,     1], shape=(10000,))
 
     Save several arrays using named keyword arguments::
 
@@ -347,9 +349,9 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs):
         >>> loader
         <LazyLoader: bar, foo>
         >>> loader['foo']
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
         >>> loader['bar']
-        array([10000,  9999,  9998, ...,     3,     2,     1])
+        array([10000,  9999,  9998, ...,     3,     2,     1], shape=(10000,))
 
     Store several arrays in a single zip file (uses a :class:`ZipStore`)::
 
@@ -358,9 +360,9 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs):
         >>> loader
         <LazyLoader: bar, foo>
         >>> loader['foo']
-        array([   0,    1,    2, ..., 9997, 9998, 9999])
+        array([   0,    1,    2, ..., 9997, 9998, 9999], shape=(10000,))
         >>> loader['bar']
-        array([10000,  9999,  9998, ...,     3,     2,     1])
+        array([10000,  9999,  9998, ...,     3,     2,     1], shape=(10000,))
 
     See Also
     --------

zarr/core.py

@@ -61,6 +61,7 @@
 )
 
 __all__ = ["Array"]
+__doctest_requires__ = {("*"): ["numpy>=2.2"]}
 
 
 # noinspection PyUnresolvedReferences
@@ -2793,33 +2794,33 @@ def view(
             >>> a = zarr.array(data, chunks=1000, filters=filters)
             >>> a[:]
             array(['female', 'male', 'female', ..., 'male', 'male', 'female'],
-                  dtype='<U6')
+                  shape=(10000,), dtype='<U6')
             >>> v = a.view(dtype='u1', filters=[])
             >>> v.is_view
             True
             >>> v[:]
-            array([1, 2, 1, ..., 2, 2, 1], dtype=uint8)
+            array([1, 2, 1, ..., 2, 2, 1], shape=(10000,), dtype=uint8)
 
         Views can be used to modify data:
 
             >>> x = v[:]
             >>> x.sort()
             >>> v[:] = x
             >>> v[:]
-            array([1, 1, 1, ..., 2, 2, 2], dtype=uint8)
+            array([1, 1, 1, ..., 2, 2, 2], shape=(10000,), dtype=uint8)
             >>> a[:]
             array(['female', 'female', 'female', ..., 'male', 'male', 'male'],
-                  dtype='<U6')
+                  shape=(10000,), dtype='<U6')
 
         View as a different dtype with the same item size:
 
             >>> data = np.random.randint(0, 2, size=10000, dtype='u1')
             >>> a = zarr.array(data, chunks=1000)
             >>> a[:]
-            array([0, 0, 1, ..., 1, 0, 0], dtype=uint8)
+            array([0, 0, 1, ..., 1, 0, 0], shape=(10000,), dtype=uint8)
             >>> v = a.view(dtype=bool)
             >>> v[:]
-            array([False, False,  True, ...,  True, False, False])
+            array([False, False,  True, ...,  True, False, False], shape=(10000,))
             >>> np.all(a[:].view(dtype=bool) == v[:])
             np.True_
 
@@ -2841,10 +2842,10 @@ def view(
 
             >>> a = zarr.full(10000, chunks=1000, fill_value=-1, dtype='i1')
             >>> a[:]
-            array([-1, -1, -1, ..., -1, -1, -1], dtype=int8)
+            array([-1, -1, -1, ..., -1, -1, -1], shape=(10000,), dtype=int8)
             >>> v = a.view(fill_value=42)
             >>> v[:]
-            array([42, 42, 42, ..., 42, 42, 42], dtype=int8)
+            array([42, 42, 42, ..., 42, 42, 42], shape=(10000,), dtype=int8)
 
         Note that resizing or appending to views is not permitted:
 

zarr/storage.py

@@ -2069,13 +2069,11 @@ class DBMStore(Store):
     <https://www.jcea.es/programacion/pybsddb.htm>`_ package is installed, a
     Berkeley DB database can be used::
 
-    .. doctest-requires:: bsddb3
-
-        >>> import bsddb3
-        >>> store = zarr.DBMStore('data/array.bdb', open=bsddb3.btopen)
-        >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True)
-        >>> z[...] = 42
-        >>> store.close()
+        >>> import bsddb3  # doctest: +SKIP
+        >>> store = zarr.DBMStore('data/array.bdb', open=bsddb3.btopen) # doctest: +SKIP
+        >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) # doctest: +SKIP
+        >>> z[...] = 42 # doctest: +SKIP
+        >>> store.close() # doctest: +SKIP
 
     Notes
     -----

zarr/tests/test_core.py

@@ -1,5 +1,4 @@
 import atexit
-import os
 import sys
 import pickle
 import shutil
@@ -75,7 +74,6 @@
 from zarr.util import buffer_size
 from zarr.tests.util import (
     abs_container,
-    have_bsddb3,
     have_fsspec,
     have_lmdb,
     have_sqlite3,
@@ -2083,20 +2081,6 @@ def test_nbytes_stored(self):
         pass  # not implemented
 
 
-@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3")
-class TestArrayWithDBMStoreBerkeleyDB(TestArray):
-    def create_store(self):
-        import bsddb3
-
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStore(path, flag="n", open=bsddb3.btopen)
-        return store
-
-    def test_nbytes_stored(self):
-        pass  # not implemented
-
-
 @pytest.mark.skipif(have_lmdb is False, reason="needs lmdb")
 class TestArrayWithLMDBStore(TestArray):
     def create_store(self):
@@ -2840,21 +2824,6 @@ def test_nbytes_stored(self):
         pass  # not implemented
 
 
-@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
-@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3")
-class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayV3):
-    def create_store(self) -> DBMStoreV3:
-        import bsddb3
-
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStoreV3(path, flag="n", open=bsddb3.btopen)
-        return store
-
-    def test_nbytes_stored(self):
-        pass  # not implemented
-
-
 @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
 @pytest.mark.skipif(have_lmdb is False, reason="needs lmdb")
 class TestArrayWithLMDBStoreV3(TestArrayV3):