Merge branch 'main' of github.com:zarr-developers/zarr-python into feat/read-funcs

Author: d-v-b

.github/workflows/test.yml

@@ -93,3 +93,22 @@ jobs:
     - name: Run Tests
       run: |
         hatch env run --env ${{ matrix.dependency-set }} run
+
+  test-complete:
+    name: Test complete
+
+    needs:
+      [
+        test,
+        test-upstream-and-min-deps,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check failure
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled')
+        run: exit 1
+      - name: Success
+        run: echo Success!

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.0
+    rev: v0.8.2
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]

docs/contributing.rst

@@ -92,12 +92,11 @@ the following::
     $ mkdir -p ~/pyenv/zarr-dev
     $ python -m venv ~/pyenv/zarr-dev
     $ source ~/pyenv/zarr-dev/bin/activate
-    $ pip install -r requirements_dev_minimal.txt -r requirements_dev_numpy.txt
-    $ pip install -e .[docs]
+    $ pip install -e .[test,docs]
 
 To verify that your development environment is working, you can run the unit tests::
 
-    $ python -m pytest -v zarr
+    $ python -m pytest -v tests
 
 Creating a branch
 ~~~~~~~~~~~~~~~~~
@@ -149,7 +148,7 @@ and invoke::
 Some tests require optional dependencies to be installed, otherwise
 the tests will be skipped. To install all optional dependencies, run::
 
-    $ pip install -r requirements_dev_optional.txt
+    $ pip install pytest-doctestplus
 
 To also run the doctests within docstrings (requires optional
 dependencies to be installed), run::
@@ -234,7 +233,7 @@ should run and pass as doctests under Python 3.8. To run doctests,
 activate your development environment, install optional requirements,
 and run::
 
-    $ python -m pytest -v --doctest-plus zarr
+    $ python -m pytest -v --doctest-plus tests
 
 Zarr uses Sphinx for documentation, hosted on readthedocs.org. Documentation is
 written in the RestructuredText markup language (.rst files) in the ``docs`` folder.

docs/installation.rst

@@ -1,27 +1,37 @@
 Installation
 ============
 
-Zarr depends on NumPy. It is generally best to `install NumPy
-<https://numpy.org/doc/stable/user/install.html>`_ first using whatever method is most
-appropriate for your operating system and Python distribution. Other dependencies should be
-installed automatically if using one of the installation methods below.
+pip
+---
 
-Note: Zarr has endorsed `Scientific-Python SPEC 0 <https://scientific-python.org/specs/spec-0000/>`_ and now follows the version support window as outlined below: 
+.. code-block:: console
 
-- Python: 36 months after initial release
-- Core package dependencies (e.g. NumPy): 24 months after initial release
+    $ pip install zarr
 
-Install Zarr from PyPI::
+There are a number of optional dependency groups you can install for extra functionality.
+These can be installed using ``pip install "zarr[<extra>]"``, e.g. ``pip install "zarr[gpu]"``
 
-    $ pip install zarr
+- ``gpu``: support for GPUs
+- ``fsspec``: support for reading/writing to remote data stores
+- ``tree``: support for pretty printing of directory trees
 
-Alternatively, install Zarr via conda::
+conda
+-----
+
+.. code-block:: console
 
     $ conda install -c conda-forge zarr
 
-To install the latest development version of Zarr, you can use pip with the
-latest GitHub main::
+Conda does not support optional dependencies, so you will have to manually install any packages
+needed to enable extra functionality.
+
+Dependency support
+------------------
+Zarr has endorsed `Scientific-Python SPEC 0 <https://scientific-python.org/specs/spec-0000/>`_ and now follows the version support window as outlined below:
 
-    $ pip install git+https://github.com/zarr-developers/zarr-python.git
+- Python: 36 months after initial release
+- Core package dependencies (e.g. NumPy): 24 months after initial release
 
-To work with Zarr source code in development, see `Contributing <contributing.html>`_.
+Development
+-----------
+To install the latest development version of Zarr, see `the contributing guide <contributing.html>`_.

docs/release.rst

@@ -18,10 +18,32 @@ Release notes
    See `GH1777 <https://github.com/zarr-developers/zarr-python/issues/1777>`_ for more details on the upcoming
    3.0 release.
 
+.. release_3.0.0-beta:
+
+3.0.0-beta series
+-----------------
+
+.. warning::
+   Zarr-Python 3.0.0-beta is a pre-release of the upcoming 3.0 release. This release is not feature complete or
+   expected to be ready for production applications.
+
+.. note::
+   The complete release notes for 3.0 have not been added to this document yet. See the
+   `3.0.0-beta <https://github.com/zarr-developers/zarr-python/releases/tag/v3.0.0-beta>`_ release on GitHub
+   for a record of changes included in this release.
+
+Dependency Changes
+~~~~~~~~~~~~~~~~~~
+
+* fsspec was moved from a required dependency to an optional one. Users should install
+  fsspec and any relevant implementations (e.g. s3fs) before using the ``RemoteStore``.
+  By :user:`Joe Hamman <jhamman>` :issue:`2391`.
+
+
 .. release_3.0.0-alpha:
 
-3.0.0-alpha
------------
+3.0.0-alpha series
+------------------
 
 .. warning::
    Zarr-Python 3.0.0-alpha is a pre-release of the upcoming 3.0 release. This release is not feature complete or

pyproject.toml

@@ -28,8 +28,7 @@ requires-python = ">=3.11"
 dependencies = [
     'numpy>=1.25',
     'numcodecs[crc32c]>=0.14',
-    'fsspec>=2022.10.0',
-    'typing_extensions>=4.6',
+    'typing_extensions>=4.9',
     'donfig>=0.8',
 ]
 
@@ -54,16 +53,16 @@ license = {text = "MIT License"}
 keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"]
 
 [project.optional-dependencies]
+fsspec = [
+    "fsspec>=2023.10.0",
+]
 test = [
     "coverage",
     "pytest",
     "pytest-cov",
-    "msgpack",
     "s3fs",
     "pytest-asyncio",
     "moto[s3]",
-    "flask-cors",
-    "flask",
     "requests",
     "mypy",
     "hypothesis",
@@ -81,7 +80,7 @@ gpu = [
 docs = [
     'sphinx==8.1.3',
     'sphinx-autobuild>=2021.3.14',
-    'sphinx-autoapi==3.3.3',
+    'sphinx-autoapi==3.4.0',
     'sphinx_design',
     'sphinx-issues',
     'sphinx-copybutton',
@@ -224,7 +223,7 @@ dependencies = [
     'fsspec==2022.10.0',
     's3fs==2022.10.0',
     'universal_pathlib==0.0.22',
-    'typing_extensions==4.6.*',  # 4.5 needed for @deprecated, 4.6 for Buffer
+    'typing_extensions==4.9.*',
     'donfig==0.8.*',
     # test deps
     'hypothesis',
@@ -294,19 +293,14 @@ extend-select = [
     "W",    # pycodestyle warnings
 ]
 ignore = [
-    "ANN101", # deprecated
-    "ANN102", # deprecated
     "ANN401",
-    "PT004",  # deprecated
-    "PT005",  # deprecated
     "PT011",  # TODO: apply this rule
     "PT012",  # TODO: apply this rule
     "RET505",
     "RET506",
     "RUF005",
     "SIM108",
     "TRY003",
-    "UP027",  # deprecated
     "UP038",  # https://github.com/astral-sh/ruff/issues/7871
     # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
     "W191",
@@ -347,6 +341,7 @@ ignore_errors = true
 
 [[tool.mypy.overrides]]
 module = [
+    "zarr.testing.stateful",  # lots of hypothesis decorator errors
     "tests.package_with_entrypoint.*",
     "tests.test_codecs.test_codecs",
     "tests.test_codecs.test_transpose",

src/zarr/api/asynchronous.py

@@ -7,6 +7,7 @@
 
 import numpy as np
 import numpy.typing as npt
+from typing_extensions import deprecated
 
 from zarr.core.array import Array, AsyncArray, get_array_metadata
 from zarr.core.buffer import NDArrayLike
@@ -535,8 +536,29 @@ async def save_group(
     await asyncio.gather(*aws)
 
 
-async def tree(*args: Any, **kwargs: Any) -> None:
-    raise NotImplementedError
+@deprecated("Use AsyncGroup.tree instead.")
+async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = None) -> Any:
+    """Provide a rich display of the hierarchy.
+
+    Parameters
+    ----------
+    grp : Group
+        Zarr or h5py group.
+    expand : bool, optional
+        Only relevant for HTML representation. If True, tree will be fully expanded.
+    level : int, optional
+        Maximum depth to descend into hierarchy.
+
+    Returns
+    -------
+    TreeRepr
+        A pretty-printable object displaying the hierarchy.
+
+    .. deprecated:: 3.0.0
+        `zarr.tree()` is deprecated and will be removed in a future release.
+        Use `group.tree()` instead.
+    """
+    return await grp.tree(expand=expand, level=level)
 
 
 async def array(
@@ -743,12 +765,12 @@ async def open_group(
         Store or path to directory in file system or name of zip file.
 
         Strings are interpreted as paths on the local file system
-        and used as the ``root`` argument to :class:`zarr.store.LocalStore`.
+        and used as the ``root`` argument to :class:`zarr.storage.LocalStore`.
 
         Dictionaries are used as the ``store_dict`` argument in
-        :class:`zarr.store.MemoryStore``.
+        :class:`zarr.storage.MemoryStore``.
 
-        By default (``store=None``) a new :class:`zarr.store.MemoryStore`
+        By default (``store=None``) a new :class:`zarr.storage.MemoryStore`
         is created.
 
     mode : {'r', 'r+', 'a', 'w', 'w-'}, optional

src/zarr/api/synchronous.py

@@ -2,6 +2,8 @@
 
 from typing import TYPE_CHECKING, Any, Literal
 
+from typing_extensions import deprecated
+
 import zarr.api.asynchronous as async_api
 from zarr._compat import _deprecate_positional_args
 from zarr.core.array import Array, AsyncArray
@@ -165,8 +167,9 @@ def save_group(
     )
 
 
-def tree(*args: Any, **kwargs: Any) -> None:
-    return sync(async_api.tree(*args, **kwargs))
+@deprecated("Use Group.tree instead.")
+def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> Any:
+    return sync(async_api.tree(grp._async_group, expand=expand, level=level))
 
 
 # TODO: add type annotations for kwargs

src/zarr/codecs/_v2.py

@@ -5,7 +5,7 @@
 from typing import TYPE_CHECKING
 
 import numcodecs
-from numcodecs.compat import ensure_ndarray_like
+from numcodecs.compat import ensure_bytes, ensure_ndarray_like
 
 from zarr.abc.codec import ArrayBytesCodec
 from zarr.registry import get_ndbuffer_class
@@ -68,6 +68,9 @@ async def _encode_single(
     ) -> Buffer | None:
         chunk = chunk_array.as_ndarray_like()
 
+        # ensure contiguous and correct order
+        chunk = chunk.astype(chunk_spec.dtype, order=chunk_spec.order, copy=False)
+
         # apply filters
         if self.filters:
             for f in self.filters:
@@ -83,6 +86,7 @@ async def _encode_single(
         else:
             cdata = chunk
 
+        cdata = ensure_bytes(cdata)
         return chunk_spec.prototype.buffer.from_bytes(cdata)
 
     def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: