Merge branch 'main' into feat/latency-store

Author: d-v-b

.github/workflows/test.yml

@@ -93,3 +93,22 @@ jobs:
     - name: Run Tests
       run: |
         hatch env run --env ${{ matrix.dependency-set }} run
+
+  test-complete:
+    name: Test complete
+
+    needs:
+      [
+        test,
+        test-upstream-and-min-deps,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check failure
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled')
+        run: exit 1
+      - name: Success
+        run: echo Success!

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.1
+    rev: v0.8.2
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]

docs/contributing.rst

@@ -92,12 +92,11 @@ the following::
     $ mkdir -p ~/pyenv/zarr-dev
     $ python -m venv ~/pyenv/zarr-dev
     $ source ~/pyenv/zarr-dev/bin/activate
-    $ pip install -r requirements_dev_minimal.txt -r requirements_dev_numpy.txt
-    $ pip install -e .[docs]
+    $ pip install -e .[test,docs]
 
 To verify that your development environment is working, you can run the unit tests::
 
-    $ python -m pytest -v zarr
+    $ python -m pytest -v tests
 
 Creating a branch
 ~~~~~~~~~~~~~~~~~
@@ -149,7 +148,7 @@ and invoke::
 Some tests require optional dependencies to be installed, otherwise
 the tests will be skipped. To install all optional dependencies, run::
 
-    $ pip install -r requirements_dev_optional.txt
+    $ pip install pytest-doctestplus
 
 To also run the doctests within docstrings (requires optional
 dependencies to be installed), run::
@@ -234,7 +233,7 @@ should run and pass as doctests under Python 3.8. To run doctests,
 activate your development environment, install optional requirements,
 and run::
 
-    $ python -m pytest -v --doctest-plus zarr
+    $ python -m pytest -v --doctest-plus tests
 
 Zarr uses Sphinx for documentation, hosted on readthedocs.org. Documentation is
 written in the RestructuredText markup language (.rst files) in the ``docs`` folder.

docs/installation.rst

@@ -1,27 +1,37 @@
 Installation
 ============
 
-Zarr depends on NumPy. It is generally best to `install NumPy
-<https://numpy.org/doc/stable/user/install.html>`_ first using whatever method is most
-appropriate for your operating system and Python distribution. Other dependencies should be
-installed automatically if using one of the installation methods below.
+pip
+---
 
-Note: Zarr has endorsed `Scientific-Python SPEC 0 <https://scientific-python.org/specs/spec-0000/>`_ and now follows the version support window as outlined below: 
+.. code-block:: console
 
-- Python: 36 months after initial release
-- Core package dependencies (e.g. NumPy): 24 months after initial release
+    $ pip install zarr
 
-Install Zarr from PyPI::
+There are a number of optional dependency groups you can install for extra functionality.
+These can be installed using ``pip install "zarr[<extra>]"``, e.g. ``pip install "zarr[gpu]"``
 
-    $ pip install zarr
+- ``gpu``: support for GPUs
+- ``fsspec``: support for reading/writing to remote data stores
+- ``tree``: support for pretty printing of directory trees
 
-Alternatively, install Zarr via conda::
+conda
+-----
+
+.. code-block:: console
 
     $ conda install -c conda-forge zarr
 
-To install the latest development version of Zarr, you can use pip with the
-latest GitHub main::
+Conda does not support optional dependencies, so you will have to manually install any packages
+needed to enable extra functionality.
+
+Dependency support
+------------------
+Zarr has endorsed `Scientific-Python SPEC 0 <https://scientific-python.org/specs/spec-0000/>`_ and now follows the version support window as outlined below:
 
-    $ pip install git+https://github.com/zarr-developers/zarr-python.git
+- Python: 36 months after initial release
+- Core package dependencies (e.g. NumPy): 24 months after initial release
 
-To work with Zarr source code in development, see `Contributing <contributing.html>`_.
+Development
+-----------
+To install the latest development version of Zarr, see `the contributing guide <contributing.html>`_.

docs/release.rst

@@ -18,10 +18,32 @@ Release notes
    See `GH1777 <https://github.com/zarr-developers/zarr-python/issues/1777>`_ for more details on the upcoming
    3.0 release.
 
+.. release_3.0.0-beta:
+
+3.0.0-beta series
+-----------------
+
+.. warning::
+   Zarr-Python 3.0.0-beta is a pre-release of the upcoming 3.0 release. This release is not feature complete or
+   expected to be ready for production applications.
+
+.. note::
+   The complete release notes for 3.0 have not been added to this document yet. See the
+   `3.0.0-beta <https://github.com/zarr-developers/zarr-python/releases/tag/v3.0.0-beta>`_ release on GitHub
+   for a record of changes included in this release.
+
+Dependency Changes
+~~~~~~~~~~~~~~~~~~
+
+* fsspec was moved from a required dependency to an optional one. Users should install
+  fsspec and any relevant implementations (e.g. s3fs) before using the ``RemoteStore``.
+  By :user:`Joe Hamman <jhamman>` :issue:`2391`.
+
+
 .. release_3.0.0-alpha:
 
-3.0.0-alpha
------------
+3.0.0-alpha series
+------------------
 
 .. warning::
    Zarr-Python 3.0.0-alpha is a pre-release of the upcoming 3.0 release. This release is not feature complete or

pyproject.toml

@@ -28,8 +28,7 @@ requires-python = ">=3.11"
 dependencies = [
     'numpy>=1.25',
     'numcodecs[crc32c]>=0.14',
-    'fsspec>=2022.10.0',
-    'typing_extensions>=4.6',
+    'typing_extensions>=4.9',
     'donfig>=0.8',
 ]
 
@@ -54,16 +53,16 @@ license = {text = "MIT License"}
 keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"]
 
 [project.optional-dependencies]
+fsspec = [
+    "fsspec>=2023.10.0",
+]
 test = [
     "coverage",
     "pytest",
     "pytest-cov",
-    "msgpack",
     "s3fs",
     "pytest-asyncio",
     "moto[s3]",
-    "flask-cors",
-    "flask",
     "requests",
     "mypy",
     "hypothesis",
@@ -224,7 +223,7 @@ dependencies = [
     'fsspec==2022.10.0',
     's3fs==2022.10.0',
     'universal_pathlib==0.0.22',
-    'typing_extensions==4.6.*',  # 4.5 needed for @deprecated, 4.6 for Buffer
+    'typing_extensions==4.9.*',
     'donfig==0.8.*',
     # test deps
     'hypothesis',
@@ -342,6 +341,7 @@ ignore_errors = true
 
 [[tool.mypy.overrides]]
 module = [
+    "zarr.testing.stateful",  # lots of hypothesis decorator errors
     "tests.package_with_entrypoint.*",
     "tests.test_codecs.test_codecs",
     "tests.test_codecs.test_transpose",

src/zarr/api/asynchronous.py

@@ -7,6 +7,7 @@
 
 import numpy as np
 import numpy.typing as npt
+from typing_extensions import deprecated
 
 from zarr.core.array import Array, AsyncArray, get_array_metadata
 from zarr.core.buffer import NDArrayLike
@@ -493,8 +494,29 @@ async def save_group(
     await asyncio.gather(*aws)
 
 
-async def tree(*args: Any, **kwargs: Any) -> None:
-    raise NotImplementedError
+@deprecated("Use AsyncGroup.tree instead.")
+async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = None) -> Any:
+    """Provide a rich display of the hierarchy.
+
+    Parameters
+    ----------
+    grp : Group
+        Zarr or h5py group.
+    expand : bool, optional
+        Only relevant for HTML representation. If True, tree will be fully expanded.
+    level : int, optional
+        Maximum depth to descend into hierarchy.
+
+    Returns
+    -------
+    TreeRepr
+        A pretty-printable object displaying the hierarchy.
+
+    .. deprecated:: 3.0.0
+        `zarr.tree()` is deprecated and will be removed in a future release.
+        Use `group.tree()` instead.
+    """
+    return await grp.tree(expand=expand, level=level)
 
 
 async def array(

src/zarr/api/synchronous.py

@@ -2,6 +2,8 @@
 
 from typing import TYPE_CHECKING, Any, Literal
 
+from typing_extensions import deprecated
+
 import zarr.api.asynchronous as async_api
 from zarr._compat import _deprecate_positional_args
 from zarr.core.array import Array, AsyncArray
@@ -155,8 +157,9 @@ def save_group(
     )
 
 
-def tree(*args: Any, **kwargs: Any) -> None:
-    return sync(async_api.tree(*args, **kwargs))
+@deprecated("Use Group.tree instead.")
+def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> Any:
+    return sync(async_api.tree(grp._async_group, expand=expand, level=level))
 
 
 # TODO: add type annotations for kwargs

src/zarr/core/chunk_grids.py

@@ -138,6 +138,9 @@ def normalize_chunks(chunks: Any, shape: tuple[int, ...], typesize: int) -> tupl
             s if c == -1 or c is None else int(c) for s, c in zip(shape, chunks, strict=False)
         )
 
+    if not all(isinstance(c, numbers.Integral) for c in chunks):
+        raise TypeError("non integer value in chunks")
+
     return tuple(int(c) for c in chunks)
 
 

src/zarr/core/group.py

@@ -1449,7 +1449,7 @@ async def tree(self, expand: bool | None = None, level: int | None = None) -> An
         from zarr.core._tree import group_tree_async
 
         if expand is not None:
-            raise NotImplementedError("'expanded' is not yet implemented.")
+            raise NotImplementedError("'expand' is not yet implemented.")
         return await group_tree_async(self, max_depth=level)
 
     async def empty(
@@ -1847,8 +1847,8 @@ def __repr__(self) -> str:
     async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group:
         """Update the attributes of this group.
 
-        Example
-        -------
+        Examples
+        --------
         >>> import zarr
         >>> group = zarr.group()
         >>> await group.update_attributes_async({"foo": "bar"})
@@ -1947,8 +1947,9 @@ def synchronizer(self) -> None:
 
     def update_attributes(self, new_attributes: dict[str, Any]) -> Group:
         """Update the attributes of this group.
-        Example
-        -------
+
+        Examples
+        --------
         >>> import zarr
         >>> group = zarr.group()
         >>> group.update_attributes({"foo": "bar"})
@@ -2027,8 +2028,8 @@ def __contains__(self, member: str) -> bool:
     def groups(self) -> Generator[tuple[str, Group], None]:
         """Return the sub-groups of this group as a generator of (name, group) pairs.
 
-        Example
-        -------
+        Examples
+        --------
         >>> import zarr
         >>> group = zarr.group()
         >>> group.create_group("subgroup")