Merge branch 'main' into literate-byte-ranges

Author: maxrjones

.github/workflows/test.yml

@@ -94,13 +94,38 @@ jobs:
       run: |
         hatch env run --env ${{ matrix.dependency-set }} run
 
+  doctests:
+    name: doctests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 # required for hatch version discovery, which is needed for numcodecs.zarr3
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+        cache: 'pip'
+    - name: Install Hatch
+      run: |
+        python -m pip install --upgrade pip
+        pip install hatch
+    - name: Set Up Hatch Env
+      run: |
+        hatch env create doctest
+        hatch env run -e doctest list-env
+    - name: Run Tests
+      run: |
+        hatch env run --env doctest run
+
   test-complete:
     name: Test complete
 
     needs:
       [
         test,
         test-upstream-and-min-deps,
+        doctests
       ]
     if: always()
     runs-on: ubuntu-latest
@@ -111,4 +136,4 @@ jobs:
           contains(needs.*.result, 'cancelled')
         run: exit 1
       - name: Success
-        run: echo Success!
+        run: echo Success!

.gitignore

@@ -51,7 +51,10 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
-docs/_autoapi
+docs/api
+docs/data
+data
+data.zip
 
 # PyBuilder
 target/

data/donotdelete

@@ -52,7 +52,7 @@ help:
 .PHONY: clean
 clean:
 	rm -rf $(BUILDDIR)/*
-	rm -rf $(BUILDDIR)/../_autoapi
+	rm -rf $(BUILDDIR)/../api
 
 .PHONY: html
 html:

docs/Makefile

@@ -0,0 +1,24 @@
+About
+=====
+
+Zarr is a format for the storage of chunked, compressed, N-dimensional arrays
+inspired by `HDF5 <https://www.hdfgroup.org/HDF5/>`_, `h5py
+<https://www.h5py.org/>`_ and `bcolz <https://bcolz.readthedocs.io/>`_.
+
+These documents describe the Zarr-Python implementation. More information
+about the Zarr format can be found on the `main website <https://zarr.dev>`_.
+
+Projects using Zarr
+-------------------
+
+If you are using Zarr-Python, we would `love to hear about it
+<https://github.com/zarr-developers/community/issues/19>`_.
+
+Funding
+-------
+The project is fiscally sponsored by `NumFOCUS <https://numfocus.org/>`_, a US
+501(c)(3) public charity, and development is supported by the
+`MRC Centre for Genomics and Global Health <https://www.cggh.org>`_
+and the `Chan Zuckerberg Initiative <https://chanzuckerberg.com/>`_.
+
+.. _NumCodecs: https://numcodecs.readthedocs.io/

docs/about.rst

@@ -17,6 +17,7 @@
 import sys
 from typing import Any
 
+import sphinx
 import sphinx.application
 
 from importlib.metadata import version as get_version
@@ -56,10 +57,24 @@
 autoapi_add_toctree_entry = False
 autoapi_generate_api_docs = True
 autoapi_member_order = "groupwise"
-autoapi_root = "_autoapi"
+autoapi_root = "api"
 autoapi_keep_files = True
 autoapi_options = [ 'members', 'undoc-members', 'show-inheritance', 'show-module-summary', 'imported-members', ]
 
+def skip_submodules(
+        app: sphinx.application.Sphinx,
+        what: str,
+        name: str,
+        obj: object,
+        skip: bool,
+        options: dict[str, Any]
+      ) -> bool:
+    # Skip documenting zarr.codecs submodules
+    # codecs are documented in the main zarr.codecs namespace
+    if what == "module" and name.startswith("zarr.codecs."):
+        skip = True
+    return skip
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -87,7 +102,13 @@
      "spec/v1": 'https://zarr-specs.readthedocs.io/en/latest/v1/v1.0.html',
      "spec/v2": "https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html",
      "spec/v3": "https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html",
-     "license": "https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt"
+     "license": "https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt",
+     "tutorial": "user-guide",
+     "getting-started": "quickstart",
+     "release": "developers/release.html",
+     "roadmap": "developers/roadmap.html",
+     "installation": "user-guide/installation.html",
+     "api": "api/zarr/index"
 }
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -179,6 +200,7 @@
 
 def setup(app: sphinx.application.Sphinx) -> None:
     app.add_css_file("custom.css")
+    app.connect("autoapi-skip-member", skip_submodules)
 
 
 # The name of an image file (relative to this directory) to use as a favicon of

docs/api/index.rst

@@ -1,5 +1,7 @@
-Contributing
-============
+.. _dev-guide-contributing:
+
+Contributing to Zarr
+====================
 
 Zarr is a community maintained project. We welcome contributions in the form of bug
 reports, bug fixes, documentation, enhancement proposals and more. This page provides
@@ -46,8 +48,7 @@ a bug report:
    interpreter can be obtained by running a Python interactive session, e.g.::
 
     $ python
-    Python 3.6.1 (default, Mar 22 2017, 06:17:05)
-    [GCC 6.3.0 20170321] on linux
+      Python 3.12.7 | packaged by conda-forge | (main, Oct  4 2024, 15:57:01) [Clang 17.0.6 ] on darwin
 
 Enhancement proposals
 ---------------------
@@ -73,7 +74,8 @@ The Zarr source code is hosted on GitHub at the following location:
 * `https://github.com/zarr-developers/zarr-python <https://github.com/zarr-developers/zarr-python>`_
 
 You will need your own fork to work on the code. Go to the link above and hit
-the "Fork" button. Then clone your fork to your local machine::
+the `"Fork" <https://github.com/zarr-developers/zarr-python/fork>`_ button.
+Then clone your fork to your local machine::
 
     $ git clone [email protected]:your-user-name/zarr-python.git
     $ cd zarr-python
@@ -82,21 +84,21 @@ the "Fork" button. Then clone your fork to your local machine::
 Creating a development environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To work with the Zarr source code, it is recommended to set up a Python virtual
-environment and install all Zarr dependencies using the same versions as are used by
-the core developers and continuous integration services. Assuming you have a Python
-3 interpreter already installed, and you have cloned the Zarr source code and your
-current working directory is the root of the repository, you can do something like
-the following::
+To work with the Zarr source code, it is recommended to use
+`hatch <https://hatch.pypa.io/latest/index.html>`_ to create and manage development
+environments. Hatch will automatically install all Zarr dependencies using the same
+versions as are used by the core developers and continuous integration services.
+Assuming you have a Python 3 interpreter already installed, and you have cloned the
+Zarr source code and your current working directory is the root of the repository,
+you can do something like the following::
 
-    $ mkdir -p ~/pyenv/zarr-dev
-    $ python -m venv ~/pyenv/zarr-dev
-    $ source ~/pyenv/zarr-dev/bin/activate
-    $ pip install -e .[test,docs]
+    $ pip install hatch
+    $ hatch env show  # list all available environments
 
-To verify that your development environment is working, you can run the unit tests::
+To verify that your development environment is working, you can run the unit tests
+for one of the test environments, e.g.::
 
-    $ python -m pytest -v tests
+    $ hatch env run --env test.py3.12-2.1-optional run
 
 Creating a branch
 ~~~~~~~~~~~~~~~~~
@@ -109,9 +111,7 @@ new, separate branch for each piece of work you want to do. E.g.::
 
     git checkout main
     git fetch upstream
-    git rebase upstream/main
-    git push
-    git checkout -b shiny-new-feature
+    git checkout -b shiny-new-feature upstream/main
     git push -u origin shiny-new-feature
 
 This changes your working directory to the 'shiny-new-feature' branch. Keep any changes in
@@ -129,54 +129,27 @@ merge conflicts, these need to be resolved before submitting a pull request.
 Alternatively, you can merge the changes in from upstream/main instead of rebasing,
 which can be simpler::
 
-    git fetch upstream
-    git merge upstream/main
+    git pull upstream main
 
 Again, any conflicts need to be resolved before submitting a pull request.
 
 Running the test suite
 ~~~~~~~~~~~~~~~~~~~~~~
 
-Zarr includes a suite of unit tests, as well as doctests included in
-function and class docstrings and in the tutorial and storage
-spec. The simplest way to run the unit tests is to activate your
-development environment (see `creating a development environment`_ above)
-and invoke::
-
-    $ python -m pytest -v zarr
-
-Some tests require optional dependencies to be installed, otherwise
-the tests will be skipped. To install all optional dependencies, run::
-
-    $ pip install pytest-doctestplus
-
-To also run the doctests within docstrings (requires optional
-dependencies to be installed), run::
-
-    $ python -m pytest -v --doctest-plus zarr
-
-To run the doctests within the tutorial and storage spec (requires
-optional dependencies to be installed), run::
-
-    $ python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst
-
-Note that some tests also require storage services to be running
-locally. To run the Azure Blob Service storage tests, run an Azure
-storage emulator (e.g., azurite) and set the environment variable
-``ZARR_TEST_ABS=1``. If you're using Docker to run azurite, start the service with::
+Zarr includes a suite of unit tests. The simplest way to run the unit tests
+is to activate your development environment
+(see `creating a development environment`_ above) and invoke::
 
-    docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --loose --blobHost 0.0.0.0
-
-To run the Mongo DB storage tests, run a Mongo
-server locally and set the environment variable ``ZARR_TEST_MONGO=1``.
-To run the Redis storage tests, run a Redis server locally on port
-6379 and set the environment variable ``ZARR_TEST_REDIS=1``.
+    $ hatch env run --env test.py3.12-2.1-optional run
 
 All tests are automatically run via GitHub Actions for every pull
 request and must pass before code can be accepted. Test coverage is
-also collected automatically via the Codecov service, and total
-coverage over all builds must be 100% (although individual builds
-may be lower due to Python 2/3 or other differences).
+also collected automatically via the Codecov service.
+
+.. note::
+    Previous versions of Zarr-Python made extensive use of doctests. These tests were
+    not maintained during the 3.0 refactor but may be brought back in the future.
+    See :issue:`2614` for more details.
 
 Code standards - using pre-commit
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -205,15 +178,17 @@ If you would like to skip the failing checks and push the code for further discu
 the ``--no-verify`` option with ``git commit``.
 
 
-
 Test coverage
 ~~~~~~~~~~~~~
 
-Zarr maintains 100% test coverage under the latest Python stable release (currently
-Python 3.8). Both unit tests and docstring doctests are included when computing
-coverage. Running::
+.. note::
+    Test coverage for Zarr-Python 3 is currently not at 100%. This is a known issue and help
+    is welcome to bring test coverage back to 100%. See :issue:`2613` for more details.
+
+Zarr strives to maintain 100% test coverage under the latest Python stable release
+Both unit tests and docstring doctests are included when computing coverage. Running::
 
-    $ python -m pytest -v --cov=zarr --cov-config=pyproject.toml zarr
+    $ hatch env run --env test.py3.12-2.1-optional run-coverage
 
 will automatically run the test suite with coverage and produce a coverage report.
 This should be 100% before code can be accepted into the main code base.
@@ -229,28 +204,28 @@ Docstrings for user-facing classes and functions should follow the
 `numpydoc
 <https://numpydoc.readthedocs.io/en/stable/format.html#docstring-standard>`_
 standard, including sections for Parameters and Examples. All examples
-should run and pass as doctests under Python 3.8. To run doctests,
-activate your development environment, install optional requirements,
-and run::
-
-    $ python -m pytest -v --doctest-plus tests
+should run and pass as doctests under Python 3.11.
 
 Zarr uses Sphinx for documentation, hosted on readthedocs.org. Documentation is
 written in the RestructuredText markup language (.rst files) in the ``docs`` folder.
 The documentation consists both of prose and API documentation. All user-facing classes
-and functions should be included in the API documentation, under the ``docs/api``
-folder. Any new features or important usage information should be included in the
-tutorial (``docs/tutorial.rst``). Any changes should also be included in the release
-notes (``docs/release.rst``).
+and functions are included in the API documentation, under the ``docs/api`` folder
+using the `autodoc <https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html>`_
+extension to sphinx. Any new features or important usage information should be included in the
+user-guide (``docs/user-guide``). Any changes should also be included in the release
+notes (``docs/developers/release.rst``).
 
 The documentation can be built locally by running::
 
-    $ cd docs
-    $ make clean; make html
-    $ open _build/html/index.html
+    $ hatch --env docs run build
 
 The resulting built documentation will be available in the ``docs/_build/html`` folder.
 
+Hatch can also be used to serve continuously updating version of the documentation
+during development at `http://0.0.0.0:8000/ <http://0.0.0.0:8000/>`_. This can be done by running::
+
+    $ hatch --env docs run serve
+
 Development best practices, policies and procedures
 ---------------------------------------------------
 
@@ -329,14 +304,7 @@ implements storage spec version 3, then the next library release should have ver
 number 3.0.0. Note however that the major version number of the Zarr library may not
 always correspond to the spec version number. For example, Zarr versions 2.x, 3.x, and
 4.x might all implement the same version of the storage spec and thus maintain data
-format compatibility, although they will not maintain API compatibility. The version number
-of the storage specification that is currently implemented is stored under the
-``zarr.meta.ZARR_FORMAT`` variable.
-
-Note that the Zarr test suite includes a data fixture and tests to try and ensure that
-data format compatibility is not accidentally broken. See the
-:func:`test_format_compatibility` function in the :mod:`tests.test_storage` module
-for details.
+format compatibility, although they will not maintain API compatibility.
 
 When to make a release
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -358,7 +326,7 @@ Release procedure
 
 .. note::
 
-   Most of the release process is now handled by github workflow which should
+   Most of the release process is now handled by GitHub workflow which should
    automatically push a release to PyPI if a tag is pushed.
 
 Before releasing, make sure that all pull requests which will be

docs/conf.py

@@ -0,0 +1,10 @@
+
+Developer's Guide
+-----------------
+
+.. toctree::
+    :maxdepth: 1
+
+    contributing
+    release
+    roadmap