Merge branch 'main' of https://github.com/zarr-developers/zarr-python into doc/3.0-updates

Author: jhamman

.pre-commit-config.yaml

@@ -28,9 +28,10 @@ repos:
         files: src|tests
         additional_dependencies:
           # Package dependencies
+          - packaging
           - donfig
           - numcodecs[crc32c]
-          - numpy
+          - numpy==2.1  # until https://github.com/numpy/numpy/issues/28034 is resolved
           - typing_extensions
           - universal-pathlib
           # Tests

README-v3.md

@@ -26,6 +26,7 @@ maintainers = [
 requires-python = ">=3.11"
 # If you add a new dependency here, please also add it to .pre-commit-config.yml
 dependencies = [
+    'packaging>=22.0',
     'numpy>=1.25',
     'numcodecs[crc32c]>=0.14',
     'typing_extensions>=4.9',
@@ -177,6 +178,7 @@ serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0"
 [tool.hatch.envs.upstream]
 python = "3.13"
 dependencies = [
+    'packaging @ git+https://github.com/pypa/packaging',
     'numpy',  # from scientific-python-nightly-wheels
     'numcodecs @ git+https://github.com/zarr-developers/numcodecs',
     'fsspec @ git+https://github.com/fsspec/filesystem_spec',
@@ -210,6 +212,7 @@ See Spec 0000 for details and drop schedule: https://scientific-python.org/specs
 """
 python = "3.11"
 dependencies = [
+    'packaging==22.*',
     'numpy==1.25.*',
     'numcodecs==0.14.*',  # 0.14 needed for zarr3 codecs
     'fsspec==2022.10.0',

pyproject.toml

@@ -17,10 +17,12 @@
     ChunkCoords,
     MemoryOrder,
     ZarrFormat,
+    parse_dtype,
 )
 from zarr.core.config import config
 from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata
 from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata
+from zarr.core.metadata.v2 import _default_filters_and_compressor
 from zarr.errors import NodeTypeValidationError
 from zarr.storage import (
     StoreLike,
@@ -401,7 +403,7 @@ async def save_array(
     arr : ndarray
         NumPy array with data to save.
     zarr_format : {2, 3, None}, optional
-        The zarr format to use when saving.
+        The zarr format to use when saving (default is 3 if not specified).
     path : str or None, optional
         The path within the store where the array will be saved.
     storage_options : dict
@@ -817,19 +819,45 @@ async def create(
     shape : int or tuple of ints
         Array shape.
     chunks : int or tuple of ints, optional
-        Chunk shape. If True, will be guessed from `shape` and `dtype`. If
-        False, will be set to `shape`, i.e., single chunk for the whole array.
-        If an int, the chunk size in each dimension will be given by the value
-        of `chunks`. Default is True.
+        The shape of the array's chunks.
+        V2 only. V3 arrays should use `chunk_shape` instead.
+        If not specified, default values are guessed based on the shape and dtype.
     dtype : str or dtype, optional
         NumPy dtype.
+    chunk_shape : int or tuple of ints, optional
+        The shape of the Array's chunks (default is None).
+        V3 only. V2 arrays should use `chunks` instead.
+    chunk_key_encoding : ChunkKeyEncoding, optional
+        A specification of how the chunk keys are represented in storage.
+        V3 only. V2 arrays should use `dimension_separator` instead.
+        Default is ``("default", "/")``.
+    codecs : Sequence of Codecs or dicts, optional
+        An iterable of Codec or dict serializations of Codecs. The elements of
+        this collection specify the transformation from array values to stored bytes.
+        V3 only. V2 arrays should use ``filters`` and ``compressor`` instead.
+
+        If no codecs are provided, default codecs will be used:
+
+        - For numeric arrays, the default is ``BytesCodec`` and ``ZstdCodec``.
+        - For Unicode strings, the default is ``VLenUTF8Codec``.
+        - For bytes or objects, the default is ``VLenBytesCodec``.
+
+        These defaults can be changed by modifying the value of ``array.v3_default_codecs`` in :mod:`zarr.core.config`.
     compressor : Codec, optional
-        Primary compressor.
-    fill_value : object
+        Primary compressor to compress chunk data.
+        V2 only. V3 arrays should use ``codecs`` instead.
+
+        If neither ``compressor`` nor ``filters`` are provided, a default compressor will be used:
+
+        - For numeric arrays, the default is ``ZstdCodec``.
+        - For Unicode strings, the default is ``VLenUTF8Codec``.
+        - For bytes or objects, the default is ``VLenBytesCodec``.
+
+        These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`.    fill_value : object
         Default value to use for uninitialized portions of the array.
     order : {'C', 'F'}, optional
         Memory layout to be used within each chunk.
-        Default is set in Zarr's config (`array.order`).
+        If not specified, default is taken from the Zarr config ```array.order```.
     store : Store or str
         Store or path to directory in file system or name of zip file.
     synchronizer : object, optional
@@ -844,6 +872,8 @@ async def create(
         for storage of both chunks and metadata.
     filters : sequence of Codecs, optional
         Sequence of filters to use to encode chunk data prior to compression.
+        V2 only. If neither ``compressor`` nor ``filters`` are provided, a default
+        compressor will be used. (see ``compressor`` for details).
     cache_metadata : bool, optional
         If True, array configuration metadata will be cached for the
         lifetime of the object. If False, array metadata will be reloaded
@@ -859,7 +889,8 @@ async def create(
         A codec to encode object arrays, only needed if dtype=object.
     dimension_separator : {'.', '/'}, optional
         Separator placed between the dimensions of a chunk.
-
+        V2 only. V3 arrays should use ``chunk_key_encoding`` instead.
+        Default is ".".
         .. versionadded:: 2.8
 
     write_empty_chunks : bool, optional
@@ -875,6 +906,7 @@ async def create(
 
     zarr_format : {2, 3, None}, optional
         The zarr format to use when saving.
+        Default is 3.
     meta_array : array-like, optional
         An array instance to use for determining arrays to create and return
         to users. Use `numpy.empty(())` by default.
@@ -894,9 +926,13 @@ async def create(
         or _default_zarr_version()
     )
 
-    if zarr_format == 2 and chunks is None:
-        chunks = shape
-    elif zarr_format == 3 and chunk_shape is None:
+    if zarr_format == 2:
+        if chunks is None:
+            chunks = shape
+        dtype = parse_dtype(dtype, zarr_format)
+        if not filters and not compressor:
+            filters, compressor = _default_filters_and_compressor(dtype)
+    elif zarr_format == 3 and chunk_shape is None:  # type: ignore[redundant-expr]
         if chunks is not None:
             chunk_shape = chunks
             chunks = None

src/zarr/api/asynchronous.py

@@ -1,10 +1,5 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    import numpy as np
-
 from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle
 from zarr.codecs.bytes import BytesCodec, Endian
 from zarr.codecs.crc32c_ import Crc32cCodec
@@ -13,7 +8,6 @@
 from zarr.codecs.transpose import TransposeCodec
 from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec
 from zarr.codecs.zstd import ZstdCodec
-from zarr.core.metadata.v3 import DataType
 
 __all__ = [
     "BloscCname",
@@ -30,15 +24,3 @@
     "VLenUTF8Codec",
     "ZstdCodec",
 ]
-
-
-def _get_default_array_bytes_codec(
-    np_dtype: np.dtype[Any],
-) -> BytesCodec | VLenUTF8Codec | VLenBytesCodec:
-    dtype = DataType.from_numpy(np_dtype)
-    if dtype == DataType.string:
-        return VLenUTF8Codec()
-    elif dtype == DataType.bytes:
-        return VLenBytesCodec()
-    else:
-        return BytesCodec()

src/zarr/codecs/__init__.py

@@ -5,6 +5,7 @@
 from typing import TYPE_CHECKING
 
 import numcodecs
+import numpy as np
 from numcodecs.compat import ensure_bytes, ensure_ndarray_like
 
 from zarr.abc.codec import ArrayBytesCodec
@@ -46,7 +47,17 @@ async def _decode_single(
         # special case object dtype, because incorrect handling can lead to
         # segfaults and other bad things happening
         if chunk_spec.dtype != object:
-            chunk = chunk.view(chunk_spec.dtype)
+            try:
+                chunk = chunk.view(chunk_spec.dtype)
+            except TypeError:
+                # this will happen if the dtype of the chunk
+                # does not match the dtype of the array spec i.g. if
+                # the dtype of the chunk_spec is a string dtype, but the chunk
+                # is an object array. In this case, we need to convert the object
+                # array to the correct dtype.
+
+                chunk = np.array(chunk).astype(chunk_spec.dtype)
+
         elif chunk.dtype != object:
             # If we end up here, someone must have hacked around with the filters.
             # We cannot deal with object arrays unless there is an object