Merge branch 'main' into kyle/object-store

kylebarron · kylebarron · commit 0f8820d00d5c · 2025-01-17T12:28:13.000-05:00
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -3,7 +3,7 @@
 TODO:
 * [ ] Add unit tests and/or doctests in docstrings
 * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions
-* [ ] New/modified features documented in docs/tutorial.rst
-* [ ] Changes documented in docs/release.rst
+* [ ] New/modified features documented in `docs/user-guide/*.rst`
+* [ ] Changes documented in `docs/release-notes.rst`
 * [ ] GitHub Actions have all passed
 * [ ] Test coverage is 100% (Codecov passes)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,15 +1,7 @@
 ---
 version: 2
 updates:
-  # Updates for v3 branch (the default branch)
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    groups:
-      actions:
-        patterns:
-          - "*"
+  # Updates for main
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
@@ -19,19 +11,19 @@ updates:
         patterns:
           - "*"
 
-  # Same updates, but for main branch
+  # Updates for support/v2 branch
   - package-ecosystem: "pip"
     directory: "/"
-    target-branch: "main"
+    target-branch: "support/v2"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     groups:
       requirements:
         patterns:
           - "*"
   - package-ecosystem: "github-actions"
     directory: "/"
-    target-branch: "main"
+    target-branch: "support/v2"
     schedule:
       interval: "weekly"
     groups:
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -1,2 +1,2 @@
 needs release notes:
-- all: ['!docs/release.rst']
+- all: ['!docs/release-notes.rst']
diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml
@@ -8,7 +8,7 @@ jobs:
     if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }} && ${{ github.event.pull_request.user.login != 'pre-commit-ci[bot]' }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@main
+    - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9  # v5.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: true
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ ci:
 default_stages: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.1
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]
diff --git a/docs/conf.py b/docs/conf.py
@@ -15,15 +15,12 @@
 
 import os
 import sys
+from importlib.metadata import version as get_version
 from typing import Any
 
 import sphinx
 import sphinx.application
 
-from importlib.metadata import version as get_version
-
-import sphinx
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst
@@ -329,10 +329,16 @@ Release procedure
    Most of the release process is now handled by GitHub workflow which should
    automatically push a release to PyPI if a tag is pushed.
 
-Before releasing, make sure that all pull requests which will be
-included in the release have been properly documented in
-`docs/release.rst`.
-
+Pre-release
+"""""""""""
+1. Make sure that all pull requests which will be
+   included in the release have been properly documented in
+   :file:`docs/release-notes.rst`.
+2. Rename the "Unreleased" section heading in :file:`docs/release-notes.rst`
+   to the version you are about to release.
+
+Releasing
+"""""""""
 To make a new release, go to
 https://github.com/zarr-developers/zarr-python/releases and
 click "Draft a new release". Choose a version number prefixed
@@ -355,5 +361,8 @@ https://readthedocs.io. Full releases will be available under
 pre-releases will be available under
 `/latest <https://zarr.readthedocs.io/en/latest>`_.
 
-Also review and merge the https://github.com/conda-forge/zarr-feedstock
-pull request that will be automatically generated.
+Post-release
+""""""""""""
+
+- Review and merge the pull request on the `conda-forge feedstock <https://github.com/conda-forge/zarr-feedstock>`_ that will be automatically generated.
+- Create a new "Unreleased" section in the release notes
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
@@ -74,7 +74,7 @@ Zarr supports data compression and filters. For example, to use Blosc compressio
     ...    "data/example-3.zarr",
     ...    mode="w", shape=(100, 100),
     ...    chunks=(10, 10), dtype="f4",
-    ...    compressor=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.SHUFFLE)
+    ...    compressors=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
     ... )
     >>> z[:, :] = np.random.random((100, 100))
     >>>
@@ -101,7 +101,7 @@ Zarr allows you to create hierarchical groups, similar to directories::
     >>> root = zarr.group("data/example-2.zarr")
     >>> foo = root.create_group(name="foo")
     >>> bar = root.create_array(
-    ...     name="bar", shape=(100, 10), chunks=(10, 10)
+    ...     name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4"
     ... )
     >>> spam = foo.create_array(name="spam", shape=(10,), dtype="i4")
     >>>
@@ -112,6 +112,7 @@ Zarr allows you to create hierarchical groups, similar to directories::
     >>> # print the hierarchy
     >>> root.tree()
     /
+    ├── bar (100, 10) float32
     └── foo
         └── spam (10,) int32
     <BLANKLINE>
@@ -130,7 +131,7 @@ using external libraries like `s3fs <https://s3fs.readthedocs.io>`_ or
 
     >>> import s3fs # doctest: +SKIP
     >>>
-    >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10)) # doctest: +SKIP
+    >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10), dtype="f4") # doctest: +SKIP
     >>> z[:, :] = np.random.random((100, 100)) # doctest: +SKIP
 
 A single-file store can also be created using the the :class:`zarr.storage.ZipStore`::
diff --git a/docs/release-notes.rst b/docs/release-notes.rst
@@ -4,21 +4,58 @@ Release notes
 Unreleased
 ----------
 
-New features
-~~~~~~~~~~~~
+Bug fixes
+~~~~~~~~~
+
+Features
+~~~~~~~~
+
+Documentation
+~~~~~~~~~~~~~
+
+Other
+~~~~~
+
+3.0.1 (Jan. 17, 2025)
+---------------------
 
 Bug fixes
 ~~~~~~~~~
-* Fixes ``order`` argument for Zarr format 2 arrays.
-  By :user:`Norman Rzepka <normanrz>` (:issue:`2679`).
+* Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`).
+
+* Fixes a bug that prevented reading Zarr format 2 data with consolidated
+  metadata written using ``zarr-python`` version 2 (:issue:`2694`).
+
+* Ensure that compressor=None results in no compression when writing Zarr
+  format 2 data (:issue:`2708`).
+
+* Fix for empty consolidated metadata dataset: backwards compatibility with
+  Zarr-Python 2 (:issue:`2695`).
+
+Documentation
+~~~~~~~~~~~~~
+* Add v3.0.0 release announcement banner (:issue:`2677`).
+
+* Quickstart guide alignment with V3 API (:issue:`2697`).
+
+* Fix doctest failures related to numcodecs 0.15 (:issue:`2727`).
+
+Other
+~~~~~
+* Removed some unnecessary files from the source distribution
+  to reduce its size. (:issue:`2686`).
+
+* Enable codecov in GitHub actions (:issue:`2682`).
+
+* Speed up hypothesis tests (:issue:`2650`).
+
+* Remove multiple imports for an import name (:issue:`2723`).
 
-Behaviour changes
-~~~~~~~~~~~~~~~~~
 
 .. _release_3.0.0:
 
-3.0.0
------
+3.0.0 (Jan. 9, 2025)
+--------------------
 
 3.0.0 is a new major release of Zarr-Python, with many breaking changes.
 See the :ref:`v3 migration guide` for a listing of what's changed.
diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst
@@ -241,7 +241,7 @@ built-in delta filter::
    >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000)
    >>> z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors)
    >>> z.compressors
-   (_make_bytes_bytes_codec.<locals>._Codec(codec_name='numcodecs.lzma', codec_config={'id': 'lzma', 'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),)
+   (LZMA(codec_name='numcodecs.lzma', codec_config={'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),)
 
 The default compressor can be changed by setting the value of the using Zarr's
 :ref:`user-guide-config`, e.g.::
@@ -292,7 +292,7 @@ Here is an example using a delta filter with the Blosc compressor::
    Order              : C
    Read-only          : False
    Store type         : LocalStore
-   Filters            : (_make_array_array_codec.<locals>._Codec(codec_name='numcodecs.delta', codec_config={'id': 'delta', 'dtype': 'int32'}),)
+   Filters            : (Delta(codec_name='numcodecs.delta', codec_config={'dtype': 'int32'}),)
    Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
    Compressors        : (BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=1, shuffle=<BloscShuffle.shuffle: 'shuffle'>, blocksize=0),)
    No. bytes          : 400000000 (381.5M)
diff --git a/pyproject.toml b/pyproject.toml
@@ -2,6 +2,13 @@
 requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "/.github",
+  "/bench",
+  "/docs",
+  "/notebooks"
+]
 
 [project]
 name = "zarr"
@@ -224,11 +231,12 @@ See Spec 0000 for details and drop schedule: https://scientific-python.org/specs
 """
 python = "3.11"
 dependencies = [
+    'zarr[remote]',
     'packaging==22.*',
     'numpy==1.25.*',
     'numcodecs==0.14.*',  # 0.14 needed for zarr3 codecs
-    'fsspec==2022.10.0',
-    's3fs==2022.10.0',
+    'fsspec==2023.10.0',
+    's3fs==2023.10.0',
     'universal_pathlib==0.0.22',
     'typing_extensions==4.9.*',
     'donfig==0.8.*',
@@ -322,8 +330,6 @@ ignore = [
     "Q003",
     "COM812",
     "COM819",
-    "ISC001",
-    "ISC002",
 ]
 
 [tool.ruff.lint.extend-per-file-ignores]
diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py
@@ -86,9 +86,9 @@ async def get(
         self, prototype: BufferPrototype, byte_range: ByteRequest | None = None
     ) -> Buffer | None:
         assert byte_range is None, "byte_range is not supported within shards"
-        assert (
-            prototype == default_buffer_prototype()
-        ), f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}"
+        assert prototype == default_buffer_prototype(), (
+            f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}"
+        )
         return self.shard_dict.get(self.chunk_coords)
 
 
diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py
@@ -112,7 +112,6 @@
     _parse_bytes_bytes_codec,
     get_pipeline_class,
 )
-from zarr.storage import StoreLike
 from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path
 
 if TYPE_CHECKING:
@@ -4131,15 +4130,22 @@ def _parse_chunk_encoding_v3(
 
 
 def _parse_deprecated_compressor(
-    compressor: CompressorLike | None, compressors: CompressorsLike
+    compressor: CompressorLike | None, compressors: CompressorsLike, zarr_format: int = 3
 ) -> CompressorsLike | None:
-    if compressor:
+    if compressor != "auto":
         if compressors != "auto":
             raise ValueError("Cannot specify both `compressor` and `compressors`.")
-        warn(
-            "The `compressor` argument is deprecated. Use `compressors` instead.",
-            category=UserWarning,
-            stacklevel=2,
-        )
-        compressors = (compressor,)
+        if zarr_format == 3:
+            warn(
+                "The `compressor` argument is deprecated. Use `compressors` instead.",
+                category=UserWarning,
+                stacklevel=2,
+            )
+        if compressor is None:
+            # "no compression"
+            compressors = ()
+        else:
+            compressors = (compressor,)
+    elif zarr_format == 2 and compressor == compressors == "auto":
+        compressors = ({"id": "blosc"},)
     return compressors
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
@@ -31,7 +31,7 @@
     create_array,
 )
 from zarr.core.attributes import Attributes
-from zarr.core.buffer import Buffer, default_buffer_prototype
+from zarr.core.buffer import default_buffer_prototype
 from zarr.core.common import (
     JSON,
     ZARR_JSON,
@@ -573,8 +573,8 @@ def _from_bytes_v2(
             v2_consolidated_metadata = json.loads(consolidated_metadata_bytes.to_bytes())
             v2_consolidated_metadata = v2_consolidated_metadata["metadata"]
             # We already read zattrs and zgroup. Should we ignore these?
-            v2_consolidated_metadata.pop(".zattrs")
-            v2_consolidated_metadata.pop(".zgroup")
+            v2_consolidated_metadata.pop(".zattrs", None)
+            v2_consolidated_metadata.pop(".zgroup", None)
 
             consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict)
 
@@ -1011,7 +1011,7 @@ async def create_array(
         shards: ShardsLike | None = None,
         filters: FiltersLike = "auto",
         compressors: CompressorsLike = "auto",
-        compressor: CompressorLike = None,
+        compressor: CompressorLike = "auto",
         serializer: SerializerLike = "auto",
         fill_value: Any | None = 0,
         order: MemoryOrder | None = None,
@@ -1114,8 +1114,9 @@ async def create_array(
         AsyncArray
 
         """
-
-        compressors = _parse_deprecated_compressor(compressor, compressors)
+        compressors = _parse_deprecated_compressor(
+            compressor, compressors, zarr_format=self.metadata.zarr_format
+        )
         return await create_array(
             store=self.store_path,
             name=name,
@@ -2244,7 +2245,7 @@ def create_array(
         shards: ShardsLike | None = None,
         filters: FiltersLike = "auto",
         compressors: CompressorsLike = "auto",
-        compressor: CompressorLike = None,
+        compressor: CompressorLike = "auto",
         serializer: SerializerLike = "auto",
         fill_value: Any | None = 0,
         order: MemoryOrder | None = "C",
@@ -2346,7 +2347,9 @@ def create_array(
         -------
         AsyncArray
         """
-        compressors = _parse_deprecated_compressor(compressor, compressors)
+        compressors = _parse_deprecated_compressor(
+            compressor, compressors, zarr_format=self.metadata.zarr_format
+        )
         return Array(
             self._sync(
                 self._async_group.create_array(
diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py
diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py
diff --git a/tests/test_group.py b/tests/test_group.py
diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py
diff --git a/tests/test_properties.py b/tests/test_properties.py
diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py
diff --git a/tests/test_v2.py b/tests/test_v2.py

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`needs release notes:`
`2`		`-- all: ['!docs/release.rst']`
	`2`	`+- all: ['!docs/release-notes.rst']`