top-level functions for reading, creating data (#2463)

* add functions for easy read-only data access

* sync funcs

* make read-only funcs top-level exports

* add create_array, create_group, and tests

* add top-level imports

* add test for top-level exports

* add test for read

* add asserts

* Apply suggestions from code review

* handle sharding in create_array

* tweak

* make logic of _auto_partition better for shard shape

* add dtype parsing, and tweak auto_partitioning func

* sketch of docstring; remove auto chunks / shard shape

* tweak docstring

* docstrings

* ensure tests pass

* tuple -> list

* allow data in create_array

* docstring

* remove auto_partition

* make shape shapelike

* use create_array everywhere in group class

* remove readers

* fix dodgy imports

* compressors -> compression, auto chunking, auto sharding, auto compression, auto filters

* use sane shard shape when there are too few chunks

* fix: allow user-specified filters and compression

* np.dtype[np.generic] -> np.dtype[Any]

* handle singleton compressor / filters input

* default codec config now uses the full config dict

* test for auto sharding

* test

* adds a shards property

* add (typed) functions for resolving codecs

* better codec parsing

* add warning if auto sharding is used

* remove read_array

* rename compression to compressors, and make the docstring for create_array more clear on what filters and compressors mean

* compression -> compressors, shard_shape -> shards, chunk_shape -> chunks

* use typerror instead of valuerror; docstring

* default order is None

* fix circular dep

* format

* fix some tests

* use filters=auto and compressors=auto in Group.create_array

* compression -> compressors

* Update src/zarr/core/group.py

Co-authored-by: Norman Rzepka <[email protected]>

* fix mypy

* narrow type of filters param and compression param

* remove data kwarg to create_array

* mypy fixes

* ensure that we accept dict form of compressor in _parse_chunk_encoding_v2

* fix properties test

* add tests for compressors and filters kwargs to create_array

* add tests for codec inference

* add test for illegal shards kwarg for v2 arrays

* remove redundant test function

* tests and types

* rm print

* types

* resolve cyclic import

* add create_array to async and sync API

* docs for create_array

* rename (Async)Array.create to _create

* adds array_bytes_codec kwarg

* tests

* tests for no filters+compressors

* widen type of FiltersParam to include single numcodecs codec instances

* don't alias None to default codecs in _create_v2

* allow single codec instances for filters, and None for filters / compressor, and condense some tests

* add docstring for None

* single-item tuple for compressors in v2

* Update src/zarr/core/array.py

* tweaks

* pr feedback 1

* tests

* mypy

* rename array_bytes_codec to serializer

* Update src/zarr/api/asynchronous.py

Co-authored-by: Joe Hamman <[email protected]>

* docstrings

* *params -> *like

* *params -> *like, in tests

* adds deprecated compressor arg to Group.create_array

* docs

---------

Co-authored-by: Joe Hamman <[email protected]>
Co-authored-by: Norman Rzepka <[email protected]>
Co-authored-by: Joe Hamman <[email protected]>

Author: 3 people

src/zarr/__init__.py

@@ -6,6 +6,8 @@
     copy_all,
     copy_store,
     create,
+    create_array,
+    create_group,
     empty,
     empty_like,
     full,
@@ -46,6 +48,8 @@
     "copy_all",
     "copy_store",
     "create",
+    "create_array",
+    "create_group",
     "empty",
     "empty_like",
     "full",

src/zarr/api/asynchronous.py

@@ -9,23 +9,23 @@
 import numpy.typing as npt
 from typing_extensions import deprecated
 
-from zarr.core.array import Array, AsyncArray, get_array_metadata
-from zarr.core.array_spec import ArrayConfig, ArrayConfigParams
+from zarr.core.array import Array, AsyncArray, create_array, get_array_metadata
+from zarr.core.array_spec import ArrayConfig, ArrayConfigLike
 from zarr.core.buffer import NDArrayLike
 from zarr.core.common import (
     JSON,
     AccessModeLiteral,
     ChunkCoords,
     MemoryOrder,
     ZarrFormat,
+    _default_zarr_format,
     _warn_order_kwarg,
     _warn_write_empty_chunks_kwarg,
     parse_dtype,
 )
-from zarr.core.config import config
 from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata
 from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata
-from zarr.core.metadata.v2 import _default_filters_and_compressor
+from zarr.core.metadata.v2 import _default_compressor, _default_filters
 from zarr.errors import NodeTypeValidationError
 from zarr.storage import (
     StoreLike,
@@ -49,6 +49,7 @@
     "copy_all",
     "copy_store",
     "create",
+    "create_array",
     "empty",
     "empty_like",
     "full",
@@ -150,11 +151,6 @@ def _handle_zarr_version_or_format(
     return zarr_format
 
 
-def _default_zarr_version() -> ZarrFormat:
-    """Return the default zarr_version"""
-    return cast(ZarrFormat, int(config.get("default_zarr_version", 3)))
-
-
 async def consolidate_metadata(
     store: StoreLike,
     path: str | None = None,
@@ -300,8 +296,8 @@ async def open(
     path : str or None, optional
         The path within the store to open.
     storage_options : dict
-        If using an fsspec URL to create the store, these will be passed to
-        the backend implementation. Ignored otherwise.
+        If the store is backed by an fsspec-based implementation, then this dict will be passed to
+        the Store constructor for that implementation. Ignored otherwise.
     **kwargs
         Additional parameters are passed through to :func:`zarr.creation.open_array` or
         :func:`zarr.hierarchy.open_group`.
@@ -417,7 +413,7 @@ async def save_array(
     """
     zarr_format = (
         _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
-        or _default_zarr_version()
+        or _default_zarr_format()
     )
     if not isinstance(arr, NDArrayLike):
         raise TypeError("arr argument must be numpy or other NDArrayLike array")
@@ -429,7 +425,7 @@ async def save_array(
     shape = arr.shape
     chunks = getattr(arr, "chunks", None)  # for array-likes with chunks attribute
     overwrite = kwargs.pop("overwrite", None) or _infer_overwrite(mode)
-    new = await AsyncArray.create(
+    new = await AsyncArray._create(
         store_path,
         zarr_format=zarr_format,
         shape=shape,
@@ -477,7 +473,7 @@ async def save_group(
             zarr_version=zarr_version,
             zarr_format=zarr_format,
         )
-        or _default_zarr_version()
+        or _default_zarr_format()
     )
 
     for arg in args:
@@ -657,7 +653,7 @@ async def group(
     try:
         return await AsyncGroup.open(store=store_path, zarr_format=zarr_format)
     except (KeyError, FileNotFoundError):
-        _zarr_format = zarr_format or _default_zarr_version()
+        _zarr_format = zarr_format or _default_zarr_format()
         return await AsyncGroup.from_store(
             store=store_path,
             zarr_format=_zarr_format,
@@ -666,6 +662,56 @@ async def group(
         )
 
 
+async def create_group(
+    *,
+    store: StoreLike,
+    path: str | None = None,
+    overwrite: bool = False,
+    zarr_format: ZarrFormat | None = None,
+    attributes: dict[str, Any] | None = None,
+    storage_options: dict[str, Any] | None = None,
+) -> AsyncGroup:
+    """Create a group.
+
+    Parameters
+    ----------
+    store : Store or str
+        Store or path to directory in file system.
+    path : str, optional
+        Group path within store.
+    overwrite : bool, optional
+        If True, pre-existing data at ``path`` will be deleted before
+        creating the group.
+    zarr_format : {2, 3, None}, optional
+        The zarr format to use when saving.
+        If no ``zarr_format`` is provided, the default format will be used.
+        This default can be changed by modifying the value of ``default_zarr_format``
+        in :mod:`zarr.core.config`.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
+
+    Returns
+    -------
+    AsyncGroup
+        The new group.
+    """
+
+    if zarr_format is None:
+        zarr_format = _default_zarr_format()
+
+    mode: Literal["a"] = "a"
+
+    store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
+
+    return await AsyncGroup.from_store(
+        store=store_path,
+        zarr_format=zarr_format,
+        overwrite=overwrite,
+        attributes=attributes,
+    )
+
+
 async def open_group(
     store: StoreLike | None = None,
     *,  # Note: this is a change from v2
@@ -768,7 +814,7 @@ async def open_group(
         pass
     if mode in _CREATE_MODES:
         overwrite = _infer_overwrite(mode)
-        _zarr_format = zarr_format or _default_zarr_version()
+        _zarr_format = zarr_format or _default_zarr_format()
         return await AsyncGroup.from_store(
             store_path,
             zarr_format=_zarr_format,
@@ -813,7 +859,7 @@ async def create(
     codecs: Iterable[Codec | dict[str, JSON]] | None = None,
     dimension_names: Iterable[str] | None = None,
     storage_options: dict[str, Any] | None = None,
-    config: ArrayConfig | ArrayConfigParams | None = None,
+    config: ArrayConfig | ArrayConfigLike | None = None,
     **kwargs: Any,
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
     """Create an array.
@@ -843,8 +889,8 @@ async def create(
         If no codecs are provided, default codecs will be used:
 
         - For numeric arrays, the default is ``BytesCodec`` and ``ZstdCodec``.
-        - For Unicode strings, the default is ``VLenUTF8Codec``.
-        - For bytes or objects, the default is ``VLenBytesCodec``.
+        - For Unicode strings, the default is ``VLenUTF8Codec`` and ``ZstdCodec``.
+        - For bytes or objects, the default is ``VLenBytesCodec`` and ``ZstdCodec``.
 
         These defaults can be changed by modifying the value of ``array.v3_default_codecs`` in :mod:`zarr.core.config`.
     compressor : Codec, optional
@@ -857,7 +903,8 @@ async def create(
         - For Unicode strings, the default is ``VLenUTF8Codec``.
         - For bytes or objects, the default is ``VLenBytesCodec``.
 
-        These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`.    fill_value : object
+        These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`.
+    fill_value : object
         Default value to use for uninitialized portions of the array.
     order : {'C', 'F'}, optional
         Deprecated in favor of the ``config`` keyword argument.
@@ -878,8 +925,8 @@ async def create(
         for storage of both chunks and metadata.
     filters : sequence of Codecs, optional
         Sequence of filters to use to encode chunk data prior to compression.
-        V2 only. If neither ``compressor`` nor ``filters`` are provided, a default
-        compressor will be used. (see ``compressor`` for details).
+        V2 only. If no ``filters`` are provided, a default set of filters will be used.
+        These defaults can be changed by modifying the value of ``array.v2_default_filters`` in :mod:`zarr.core.config`.
     cache_metadata : bool, optional
         If True, array configuration metadata will be cached for the
         lifetime of the object. If False, array metadata will be reloaded
@@ -914,7 +961,7 @@ async def create(
     storage_options : dict
         If using an fsspec URL to create the store, these will be passed to
         the backend implementation. Ignored otherwise.
-    config : ArrayConfig or ArrayConfigParams, optional
+    config : ArrayConfig or ArrayConfigLike, optional
         Runtime configuration of the array. If provided, will override the
         default values from `zarr.config.array`.
 
@@ -925,15 +972,17 @@ async def create(
     """
     zarr_format = (
         _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
-        or _default_zarr_version()
+        or _default_zarr_format()
     )
 
     if zarr_format == 2:
         if chunks is None:
             chunks = shape
         dtype = parse_dtype(dtype, zarr_format)
-        if not filters and not compressor:
-            filters, compressor = _default_filters_and_compressor(dtype)
+        if not filters:
+            filters = _default_filters(dtype)
+        if not compressor:
+            compressor = _default_compressor(dtype)
     elif zarr_format == 3 and chunk_shape is None:  # type: ignore[redundant-expr]
         if chunks is not None:
             chunk_shape = chunks
@@ -971,7 +1020,7 @@ async def create(
         mode = "a"
     store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
 
-    config_dict: ArrayConfigParams = {}
+    config_dict: ArrayConfigLike = {}
 
     if write_empty_chunks is not None:
         if config is not None:
@@ -994,7 +1043,7 @@ async def create(
 
     config_parsed = ArrayConfig.from_dict(config_dict)
 
-    return await AsyncArray.create(
+    return await AsyncArray._create(
         store_path,
         shape=shape,
         chunks=chunks,
@@ -1173,7 +1222,7 @@ async def open_array(
         If using an fsspec URL to create the store, these will be passed to
         the backend implementation. Ignored otherwise.
     **kwargs
-        Any keyword arguments to pass to ``create``.
+        Any keyword arguments to pass to :func:`create`.
 
     Returns
     -------
@@ -1196,7 +1245,7 @@ async def open_array(
     except FileNotFoundError:
         if not store_path.read_only and mode in _CREATE_MODES:
             overwrite = _infer_overwrite(mode)
-            _zarr_format = zarr_format or _default_zarr_version()
+            _zarr_format = zarr_format or _default_zarr_format()
             return await create(
                 store=store_path,
                 zarr_format=_zarr_format,