Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
fd6ecd1
add functions for easy read-only data access
d-v-b Nov 4, 2024
fa343f5
sync funcs
d-v-b Nov 4, 2024
d95eba8
make read-only funcs top-level exports
d-v-b Nov 4, 2024
f6765bc
Merge branch 'main' into feat/read-funcs
d-v-b Nov 5, 2024
5d8445b
add create_array, create_group, and tests
d-v-b Nov 5, 2024
9526571
add top-level imports
d-v-b Nov 5, 2024
90bf421
Merge branch 'feat/read-funcs' of github.com:d-v-b/zarr-python into f…
d-v-b Nov 5, 2024
de280a7
add test for top-level exports
d-v-b Nov 5, 2024
d9878cf
add test for read
d-v-b Nov 5, 2024
e5217ce
add asserts
d-v-b Nov 5, 2024
40cc7af
Apply suggestions from code review
d-v-b Nov 5, 2024
d7ce58b
Merge branch 'main' into feat/read-funcs
d-v-b Nov 5, 2024
a0dfe18
Merge branch 'main' into feat/read-funcs
d-v-b Nov 5, 2024
98bc328
Merge branch 'main' into feat/read-funcs
d-v-b Nov 12, 2024
16f5cc2
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Nov 29, 2024
4b45ebf
handle sharding in create_array
d-v-b Dec 10, 2024
750a439
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 10, 2024
7a5cbe7
tweak
d-v-b Dec 10, 2024
215ff96
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 18, 2024
489e2a2
make logic of _auto_partition better for shard shape
d-v-b Dec 18, 2024
05dd0d8
add dtype parsing, and tweak auto_partitioning func
d-v-b Dec 18, 2024
3fbfc21
sketch of docstring; remove auto chunks / shard shape
d-v-b Dec 19, 2024
b348737
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 19, 2024
5025ad6
tweak docstring
d-v-b Dec 19, 2024
e204a32
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 20, 2024
68465db
docstrings
d-v-b Dec 20, 2024
d7bb121
ensure tests pass
d-v-b Dec 20, 2024
99cc8f5
tuple -> list
d-v-b Dec 20, 2024
a39457f
allow data in create_array
d-v-b Dec 20, 2024
3f0a3e0
docstring
d-v-b Dec 20, 2024
26ced00
remove auto_partition
d-v-b Dec 20, 2024
af55ac4
make shape shapelike
d-v-b Dec 20, 2024
07f07ea
use create_array everywhere in group class
d-v-b Dec 20, 2024
bc552ce
remove readers
d-v-b Dec 20, 2024
74f731a
fix dodgy imports
d-v-b Dec 20, 2024
43877c0
compressors -> compression, auto chunking, auto sharding, auto compre…
d-v-b Dec 21, 2024
c693fb4
use sane shard shape when there are too few chunks
d-v-b Dec 21, 2024
4c18aaa
Merge branch 'main' into feat/read-funcs
jhamman Dec 22, 2024
dba2594
fix: allow user-specified filters and compression
d-v-b Dec 22, 2024
669ad72
np.dtype[np.generic] -> np.dtype[Any]
d-v-b Dec 22, 2024
ae1832d
handle singleton compressor / filters input
d-v-b Dec 22, 2024
5cb6dd8
default codec config now uses the full config dict
normanrz Dec 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/zarr/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
copy_all,
copy_store,
create,
create_array,
create_group,
empty,
empty_like,
full,
Expand Down Expand Up @@ -46,6 +48,8 @@
"copy_all",
"copy_store",
"create",
"create_array",
"create_group",
"empty",
"empty_like",
"full",
Expand Down
113 changes: 97 additions & 16 deletions src/zarr/api/asynchronous.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@
ChunkCoords,
MemoryOrder,
ZarrFormat,
_default_zarr_version,
_warn_order_kwarg,
_warn_write_empty_chunks_kwarg,
parse_dtype,
)
from zarr.core.config import config
from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata
from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata
from zarr.core.metadata.v2 import _default_filters_and_compressor
from zarr.core.metadata.v2 import _default_compressor, _default_filters
from zarr.errors import NodeTypeValidationError
from zarr.storage import (
StoreLike,
Expand Down Expand Up @@ -150,11 +150,6 @@ def _handle_zarr_version_or_format(
return zarr_format


def _default_zarr_version() -> ZarrFormat:
"""Return the default zarr_version"""
return cast(ZarrFormat, int(config.get("default_zarr_version", 3)))


async def consolidate_metadata(
store: StoreLike,
path: str | None = None,
Expand Down Expand Up @@ -300,8 +295,8 @@ async def open(
path : str or None, optional
The path within the store to open.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.
If the store is backed by an fsspec-based implementation, then this dict will be passed to
the Store constructor for that implementation. Ignored otherwise.
**kwargs
Additional parameters are passed through to :func:`zarr.creation.open_array` or
:func:`zarr.hierarchy.open_group`.
Expand Down Expand Up @@ -666,6 +661,54 @@ async def group(
)


async def create_group(
*,
store: StoreLike,
path: str | None = None,
overwrite: bool = False,
zarr_format: ZarrFormat | None = None,
attributes: dict[str, Any] | None = None,
storage_options: dict[str, Any] | None = None,
) -> AsyncGroup:
"""Create a group.

Parameters
----------
store : Store or str
Store or path to directory in file system.
path : str, optional
Group path within store.
overwrite : bool, optional
If True, pre-existing data at ``path`` will be deleted before
creating the group.
zarr_format : {2, 3, None}, optional
The zarr format to use when saving.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.

Returns
-------
g : group
The new group.
"""

if zarr_format is None:
zarr_format = _default_zarr_version()

# TODO: fix this when modes make sense. It should be `w` for overwriting, `w-` otherwise
mode: Literal["a"] = "a"

store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)

return await AsyncGroup.from_store(
store=store_path,
zarr_format=zarr_format,
overwrite=overwrite,
attributes=attributes,
)


async def open_group(
store: StoreLike | None = None,
*, # Note: this is a change from v2
Expand Down Expand Up @@ -843,8 +886,8 @@ async def create(
If no codecs are provided, default codecs will be used:

- For numeric arrays, the default is ``BytesCodec`` and ``ZstdCodec``.
- For Unicode strings, the default is ``VLenUTF8Codec``.
- For bytes or objects, the default is ``VLenBytesCodec``.
- For Unicode strings, the default is ``VLenUTF8Codec`` and ``ZstdCodec``.
- For bytes or objects, the default is ``VLenBytesCodec`` and ``ZstdCodec``.

These defaults can be changed by modifying the value of ``array.v3_default_codecs`` in :mod:`zarr.core.config`.
compressor : Codec, optional
Expand All @@ -857,7 +900,8 @@ async def create(
- For Unicode strings, the default is ``VLenUTF8Codec``.
- For bytes or objects, the default is ``VLenBytesCodec``.

These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`. fill_value : object
These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`.
fill_value : object
Default value to use for uninitialized portions of the array.
order : {'C', 'F'}, optional
Deprecated in favor of the ``config`` keyword argument.
Expand All @@ -878,8 +922,8 @@ async def create(
for storage of both chunks and metadata.
filters : sequence of Codecs, optional
Sequence of filters to use to encode chunk data prior to compression.
V2 only. If neither ``compressor`` nor ``filters`` are provided, a default
compressor will be used. (see ``compressor`` for details).
V2 only. If no ``filters`` are provided, a default set of filters will be used.
These defaults can be changed by modifying the value of ``array.v2_default_filters`` in :mod:`zarr.core.config`.
cache_metadata : bool, optional
If True, array configuration metadata will be cached for the
lifetime of the object. If False, array metadata will be reloaded
Expand Down Expand Up @@ -932,8 +976,10 @@ async def create(
if chunks is None:
chunks = shape
dtype = parse_dtype(dtype, zarr_format)
if not filters and not compressor:
filters, compressor = _default_filters_and_compressor(dtype)
if not filters:
filters = _default_filters(dtype)
if not compressor:
compressor = _default_compressor(dtype)
elif zarr_format == 3 and chunk_shape is None: # type: ignore[redundant-expr]
if chunks is not None:
chunk_shape = chunks
Expand Down Expand Up @@ -1015,6 +1061,41 @@ async def create(
)


async def read_array(
store: StoreLike,
*,
path: str | None = None,
zarr_format: ZarrFormat | None = None,
storage_options: dict[str, Any] | None = None,
) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]:
"""Create an array for reading. Wraps `:func:zarr.api.asynchronous.create`.
See the documentation of that function for details.

Parameters
----------
store : Store or str
Store or path to directory in file system or name of zip file.
path : str, optional
Path under which the array is stored.
zarr_format : {2, 3, None}, optional
The zarr format to require. The default value of ``None`` will first look for Zarr v3 data,
then Zarr v2 data, then fail if neither format is found.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.

Returns
-------
z : array
The array.
"""
store_path = await make_store_path(store, path=path, mode="r", storage_options=storage_options)
return await AsyncArray.open(
store=store_path,
zarr_format=zarr_format,
)


async def empty(
shape: ChunkCoords, **kwargs: Any
) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
Expand Down
28 changes: 28 additions & 0 deletions src/zarr/api/synchronous.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing_extensions import deprecated

import zarr.api.asynchronous as async_api
import zarr.core.array
from zarr._compat import _deprecate_positional_args
from zarr.core.array import Array, AsyncArray
from zarr.core.group import Group
Expand Down Expand Up @@ -523,6 +524,29 @@ def open_group(
)


def create_group(
store: StoreLike,
*,
path: str | None = None,
zarr_format: ZarrFormat | None = None,
overwrite: bool = False,
attributes: dict[str, Any] | None = None,
storage_options: dict[str, Any] | None = None,
) -> Group:
return Group(
sync(
async_api.create_group(
store=store,
path=path,
overwrite=overwrite,
storage_options=storage_options,
zarr_format=zarr_format,
attributes=attributes,
)
)
)


# TODO: add type annotations for kwargs
def create(
shape: ChunkCoords | int,
Expand Down Expand Up @@ -675,6 +699,10 @@ def create(
)


def create_array(*args: Any, **kwargs: Any) -> Array:
return Array(sync(zarr.core.array.create_array(*args, **kwargs)))


# TODO: add type annotations for kwargs
def empty(shape: ChunkCoords, **kwargs: Any) -> Array:
"""Create an empty array.
Expand Down
Loading
Loading