Merge branch 'main' into deterministic-chunk-padding

brokkoli71 · brokkoli71 · commit 6ccfe95d9b9c · 2025-01-23T15:13:52.000+01:00
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -3,7 +3,7 @@
 TODO:
 * [ ] Add unit tests and/or doctests in docstrings
 * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions
-* [ ] New/modified features documented in docs/tutorial.rst
-* [ ] Changes documented in docs/release.rst
+* [ ] New/modified features documented in `docs/user-guide/*.rst`
+* [ ] Changes documented as a new file in `changes/`
 * [ ] GitHub Actions have all passed
 * [ ] Test coverage is 100% (Codecov passes)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,15 +1,7 @@
 ---
 version: 2
 updates:
-  # Updates for v3 branch (the default branch)
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    groups:
-      actions:
-        patterns:
-          - "*"
+  # Updates for main
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
@@ -19,19 +11,19 @@ updates:
         patterns:
           - "*"
 
-  # Same updates, but for main branch
+  # Updates for support/v2 branch
   - package-ecosystem: "pip"
     directory: "/"
-    target-branch: "main"
+    target-branch: "support/v2"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     groups:
       requirements:
         patterns:
           - "*"
   - package-ecosystem: "github-actions"
     directory: "/"
-    target-branch: "main"
+    target-branch: "support/v2"
     schedule:
       interval: "weekly"
     groups:
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -1,2 +1,4 @@
-needs release notes:
-- all: ['!docs/release.rst']
+- needs release notes:
+  - all:
+    - changed-files:
+      - any-glob-to-any-file: 'changes/*.rst'
diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml
@@ -4,11 +4,14 @@ on:
   - pull_request_target
 
 jobs:
-  triage:
+  labeler:
     if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }} && ${{ github.event.pull_request.user.login != 'pre-commit-ci[bot]' }}
+    permissions:
+      contents: read
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@main
+    - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9  # v5.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: true
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ ci:
 default_stages: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.1
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -49,3 +49,7 @@ repos:
     rev: v1.8.0
     hooks:
       - id: numpydoc-validation
+  - repo: https://github.com/twisted/towncrier
+    rev: 23.11.0
+    hooks:
+      - id: towncrier-check
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -4,6 +4,13 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.12"
+  jobs:
+    pre_build:
+      - |
+        if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ];
+        then
+          towncrier build --version Unreleased --yes;
+        fi
 
 sphinx:
   configuration: docs/conf.py
diff --git a/changes/.gitignore b/changes/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/changes/2533.bigfix.rst b/changes/2533.bigfix.rst
@@ -0,0 +1 @@
+Wrap sync fsspec filesystems with AsyncFileSystemWrapper in xarray.to_zarr
diff --git a/changes/2681.bugfix.rst b/changes/2681.bugfix.rst
@@ -0,0 +1 @@
+Added backwards compatibility for Zarr format 2 structured arrays.
diff --git a/changes/2736.doc.rst b/changes/2736.doc.rst
@@ -0,0 +1,2 @@
+Changed the machinery for creating changelog entries.
+Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file.
diff --git a/changes/README.md b/changes/README.md
@@ -0,0 +1,14 @@
+Writing a changelog entry
+-------------------------
+
+Please put a new file in this directory named `xxxx.<type>.rst`, where
+
+- `xxxx` is the pull request number associated with this entry
+- `<type>` is one of:
+  - feature
+  - bugfix
+  - doc
+  - removal
+  - misc
+
+Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-python`.
diff --git a/docs/conf.py b/docs/conf.py
@@ -15,15 +15,12 @@
 
 import os
 import sys
+from importlib.metadata import version as get_version
 from typing import Any
 
 import sphinx
 import sphinx.application
 
-from importlib.metadata import version as get_version
-
-import sphinx
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst
@@ -98,7 +98,7 @@ you can do something like the following::
 To verify that your development environment is working, you can run the unit tests
 for one of the test environments, e.g.::
 
-    $ hatch env run --env test.py3.12-2.1-optional run
+    $ hatch env run --env test.py3.12-2.1-optional run-pytest
 
 Creating a branch
 ~~~~~~~~~~~~~~~~~
@@ -140,7 +140,7 @@ Zarr includes a suite of unit tests. The simplest way to run the unit tests
 is to activate your development environment
 (see `creating a development environment`_ above) and invoke::
 
-    $ hatch env run --env test.py3.12-2.1-optional run
+    $ hatch env run --env test.py3.12-2.1-optional run-pytest
 
 All tests are automatically run via GitHub Actions for every pull
 request and must pass before code can be accepted. Test coverage is
@@ -190,9 +190,13 @@ Both unit tests and docstring doctests are included when computing coverage. Run
 
     $ hatch env run --env test.py3.12-2.1-optional run-coverage
 
-will automatically run the test suite with coverage and produce a coverage report.
+will automatically run the test suite with coverage and produce a XML coverage report.
 This should be 100% before code can be accepted into the main code base.
 
+You can also generate an HTML coverage report by running::
+
+     $ hatch env run --env test.py3.12-2.1-optional run-coverage-html
+
 When submitting a pull request, coverage will also be collected across all supported
 Python versions via the Codecov service, and will be reported back within the pull
 request. Codecov coverage must also be 100% before code can be accepted.
@@ -212,8 +216,8 @@ The documentation consists both of prose and API documentation. All user-facing
 and functions are included in the API documentation, under the ``docs/api`` folder
 using the `autodoc <https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html>`_
 extension to sphinx. Any new features or important usage information should be included in the
-user-guide (``docs/user-guide``). Any changes should also be included in the release
-notes (``docs/release-notes.rst``).
+user-guide (``docs/user-guide``). Any changes should also be included as a new file in the
+:file:`changes` directory.
 
 The documentation can be built locally by running::
 
@@ -331,11 +335,9 @@ Release procedure
 
 Pre-release
 """""""""""
-1. Make sure that all pull requests which will be
-   included in the release have been properly documented in
-   :file:`docs/release-notes.rst`.
-2. Rename the "Unreleased" section heading in :file:`docs/release-notes.rst`
-   to the version you are about to release.
+1. Make sure that all pull requests which will be included in the release
+   have been properly documented as changelog files in :file:`changes`.
+2. Run ``towncrier build --version x.y.z`` to create the changelog.
 
 Releasing
 """""""""
@@ -348,7 +350,7 @@ appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`).
 
 Set the description of the release to::
 
-    See release notes https://zarr.readthedocs.io/en/stable/release.html#release-0-0-0
+    See release notes https://zarr.readthedocs.io/en/stable/release-notes.html#release-0-0-0
 
 replacing the correct version numbers. For pre-release versions,
 the URL should omit the pre-release suffix, e.g. "a1" or "rc1".
diff --git a/docs/release-notes.rst b/docs/release-notes.rst
@@ -1,33 +1,49 @@
 Release notes
 =============
 
-Unreleased
-----------
+.. towncrier release notes start
 
-New features
-~~~~~~~~~~~~
+
+3.0.1 (Jan. 17, 2025)
+---------------------
 
 Bug fixes
 ~~~~~~~~~
 * Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`).
 
-* Fixes a bug that prevented reading Zarr format 2 data with consolidated metadata written using ``zarr-python`` version 2 (:issue:`2694`).
+* Fixes a bug that prevented reading Zarr format 2 data with consolidated
+  metadata written using ``zarr-python`` version 2 (:issue:`2694`).
+
+* Ensure that compressor=None results in no compression when writing Zarr
+  format 2 data (:issue:`2708`).
+
+* Fix for empty consolidated metadata dataset: backwards compatibility with
+  Zarr-Python 2 (:issue:`2695`).
+
+Documentation
+~~~~~~~~~~~~~
+* Add v3.0.0 release announcement banner (:issue:`2677`).
 
-* Ensure that compressor=None results in no compression when writing Zarr format 2 data (:issue:`2708`)
+* Quickstart guide alignment with V3 API (:issue:`2697`).
 
-Behaviour changes
-~~~~~~~~~~~~~~~~~
+* Fix doctest failures related to numcodecs 0.15 (:issue:`2727`).
 
 Other
 ~~~~~
 * Removed some unnecessary files from the source distribution
-  to reduce its size. (:issue:`2686`)
+  to reduce its size. (:issue:`2686`).
+
+* Enable codecov in GitHub actions (:issue:`2682`).
+
+* Speed up hypothesis tests (:issue:`2650`).
+
+* Remove multiple imports for an import name (:issue:`2723`).
 
 
 .. _release_3.0.0:
 
-3.0.0
------
+3.0.0 (Jan. 9, 2025)
+--------------------
 
 3.0.0 is a new major release of Zarr-Python, with many breaking changes.
 See the :ref:`v3 migration guide` for a listing of what's changed.
diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst
@@ -241,7 +241,7 @@ built-in delta filter::
    >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000)
    >>> z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors)
    >>> z.compressors
-   (_make_bytes_bytes_codec.<locals>._Codec(codec_name='numcodecs.lzma', codec_config={'id': 'lzma', 'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),)
+   (LZMA(codec_name='numcodecs.lzma', codec_config={'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),)
 
 The default compressor can be changed by setting the value of the using Zarr's
 :ref:`user-guide-config`, e.g.::
@@ -292,7 +292,7 @@ Here is an example using a delta filter with the Blosc compressor::
    Order              : C
    Read-only          : False
    Store type         : LocalStore
-   Filters            : (_make_array_array_codec.<locals>._Codec(codec_name='numcodecs.delta', codec_config={'id': 'delta', 'dtype': 'int32'}),)
+   Filters            : (Delta(codec_name='numcodecs.delta', codec_config={'dtype': 'int32'}),)
    Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
    Compressors        : (BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=1, shuffle=<BloscShuffle.shuffle: 'shuffle'>, blocksize=0),)
    No. bytes          : 400000000 (381.5M)
diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst
@@ -53,6 +53,7 @@ This is the current default configuration::
                                                    'level': 0}},
               'v2_default_filters': {'bytes': [{'id': 'vlen-bytes'}],
                                      'numeric': None,
+                                     'raw': None,
                                      'string': [{'id': 'vlen-utf8'}]},
               'v3_default_compressors': {'bytes': [{'configuration': {'checksum': False,
                                                                       'level': 0},
diff --git a/pyproject.toml b/pyproject.toml
@@ -85,6 +85,7 @@ test = [
 ]
 optional = ["rich", "universal-pathlib"]
 docs = [
+    # Doc building
     'sphinx==8.1.3',
     'sphinx-autobuild>=2021.3.14',
     'sphinx-autoapi==3.4.0',
@@ -94,6 +95,9 @@ docs = [
     'sphinx-reredirects',
     'pydata-sphinx-theme',
     'numpydoc',
+    # Changelog generation
+    'towncrier',
+    # Optional dependencies to run examples
     'numcodecs[msgpack]',
     'rich',
     's3fs',
@@ -149,7 +153,9 @@ features = ["gpu"]
 [tool.hatch.envs.test.scripts]
 run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
 run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
+run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
 run = "run-coverage --no-cov"
+run-pytest = "run"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
 run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
@@ -228,11 +234,12 @@ See Spec 0000 for details and drop schedule: https://scientific-python.org/specs
 """
 python = "3.11"
 dependencies = [
+    'zarr[remote]',
     'packaging==22.*',
     'numpy==1.25.*',
     'numcodecs==0.14.*',  # 0.14 needed for zarr3 codecs
-    'fsspec==2022.10.0',
-    's3fs==2022.10.0',
+    'fsspec==2023.10.0',
+    's3fs==2023.10.0',
     'universal_pathlib==0.0.22',
     'typing_extensions==4.9.*',
     'donfig==0.8.*',
@@ -326,8 +333,6 @@ ignore = [
     "Q003",
     "COM812",
     "COM819",
-    "ISC001",
-    "ISC002",
 ]
 
 [tool.ruff.lint.extend-per-file-ignores]
@@ -414,3 +419,9 @@ checks = [
     "PR05",
     "PR06",
 ]
+
+[tool.towncrier]
+directory = 'changes'
+filename = "docs/release-notes.rst"
+underlines = ["-", "~", "^"]
+issue_format = ":issue:`{issue}`"
diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py
@@ -86,9 +86,9 @@ async def get(
         self, prototype: BufferPrototype, byte_range: ByteRequest | None = None
     ) -> Buffer | None:
         assert byte_range is None, "byte_range is not supported within shards"
-        assert (
-            prototype == default_buffer_prototype()
-        ), f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}"
+        assert prototype == default_buffer_prototype(), (
+            f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}"
+        )
         return self.shard_dict.get(self.chunk_coords)
 
 
diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py
@@ -112,7 +112,6 @@
     _parse_bytes_bytes_codec,
     get_pipeline_class,
 )
-from zarr.storage import StoreLike
 from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path
 
 if TYPE_CHECKING:
diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py
@@ -470,7 +470,9 @@ def all_equal(self, other: Any, equal_nan: bool = True) -> bool:
         # every single time we have to write data?
         _data, other = np.broadcast_arrays(self._data, other)
         return np.array_equal(
-            self._data, other, equal_nan=equal_nan if self._data.dtype.kind not in "USTO" else False
+            self._data,
+            other,
+            equal_nan=equal_nan if self._data.dtype.kind not in "USTOV" else False,
         )
 
     def fill(self, value: Any) -> None:
diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py
diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py
diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py
diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py
diff --git a/tests/test_config.py b/tests/test_config.py
diff --git a/tests/test_properties.py b/tests/test_properties.py
diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py
diff --git a/tests/test_v2.py b/tests/test_v2.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Wrap sync fsspec filesystems with AsyncFileSystemWrapper in xarray.to_zarr`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Added backwards compatibility for Zarr format 2 structured arrays.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Changed the machinery for creating changelog entries.`
	`2`	+Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file.