Skip to content

[do not merge] sketch of a refactor for core modules #2780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[do not merge] sketch of a refactor for core modules #2780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions src/zarr/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,31 @@
from zarr._version import version as __version__
from zarr.api.synchronous import (
array,
consolidate_metadata,
copy,
copy_all,
copy_store,
create,
create_array,
create_group,
empty,
empty_like,
full,
full_like,
group,
load,
ones,
ones_like,
open,
open_array,
open_consolidated,
open_group,
open_like,
save,
save_array,
save_group,
tree,
zeros,
zeros_like,
)
from zarr.core.array import Array, AsyncArray
from zarr.core.config import config
from zarr.core.group import AsyncGroup, Group
from zarr.core.group.async import AsyncGroup
from zarr.core.group.sync import Group, consolidate_metadata, create_group, group, open_group, save_group, tree

# in case setuptools scm screw up and find version to be 0.0.0
assert not __version__.startswith("0.0.0")
Expand Down
250 changes: 4 additions & 246 deletions src/zarr/api/asynchronous.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
_warn_write_empty_chunks_kwarg,
parse_dtype,
)
from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata
from zarr.core.group.core import open_group
from zarr.core.group.metadata import ConsolidatedMetadata
from zarr.core.group import AsyncGroup, GroupMetadata
from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata
from zarr.core.metadata.v2 import _default_compressor, _default_filters
from zarr.errors import NodeTypeValidationError
Expand Down Expand Up @@ -52,7 +54,7 @@
"empty_like",
"full",
"full_like",
"group",
,
"load",
"ones",
"ones_like",
Expand Down Expand Up @@ -577,250 +579,6 @@ async def array(
return z


async def group(
*, # Note: this is a change from v2
store: StoreLike | None = None,
overwrite: bool = False,
chunk_store: StoreLike | None = None, # not used
cache_attrs: bool | None = None, # not used, default changed
synchronizer: Any | None = None, # not used
path: str | None = None,
zarr_version: ZarrFormat | None = None, # deprecated
zarr_format: ZarrFormat | None = None,
meta_array: Any | None = None, # not used
attributes: dict[str, JSON] | None = None,
storage_options: dict[str, Any] | None = None,
) -> AsyncGroup:
"""Create a group.

Parameters
----------
store : Store or str, optional
Store or path to directory in file system.
overwrite : bool, optional
If True, delete any pre-existing data in `store` at `path` before
creating the group.
chunk_store : Store, optional
Separate storage for chunks. If not provided, `store` will be used
for storage of both chunks and metadata.
cache_attrs : bool, optional
If True (default), user attributes will be cached for attribute read
operations. If False, user attributes are reloaded from the store prior
to all attribute read operations.
synchronizer : object, optional
Array synchronizer.
path : str, optional
Group path within store.
meta_array : array-like, optional
An array instance to use for determining arrays to create and return
to users. Use `numpy.empty(())` by default.
zarr_format : {2, 3, None}, optional
The zarr format to use when saving.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.

Returns
-------
g : group
The new group.
"""

zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)

mode: AccessModeLiteral
if overwrite:
mode = "w"
else:
mode = "r+"
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)

if chunk_store is not None:
warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)
if cache_attrs is not None:
warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2)
if synchronizer is not None:
warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2)
if meta_array is not None:
warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)

if attributes is None:
attributes = {}

try:
return await AsyncGroup.open(store=store_path, zarr_format=zarr_format)
except (KeyError, FileNotFoundError):
_zarr_format = zarr_format or _default_zarr_format()
return await AsyncGroup.from_store(
store=store_path,
zarr_format=_zarr_format,
overwrite=overwrite,
attributes=attributes,
)


async def create_group(
*,
store: StoreLike,
path: str | None = None,
overwrite: bool = False,
zarr_format: ZarrFormat | None = None,
attributes: dict[str, Any] | None = None,
storage_options: dict[str, Any] | None = None,
) -> AsyncGroup:
"""Create a group.

Parameters
----------
store : Store or str
Store or path to directory in file system.
path : str, optional
Group path within store.
overwrite : bool, optional
If True, pre-existing data at ``path`` will be deleted before
creating the group.
zarr_format : {2, 3, None}, optional
The zarr format to use when saving.
If no ``zarr_format`` is provided, the default format will be used.
This default can be changed by modifying the value of ``default_zarr_format``
in :mod:`zarr.core.config`.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.

Returns
-------
AsyncGroup
The new group.
"""

if zarr_format is None:
zarr_format = _default_zarr_format()

mode: Literal["a"] = "a"

store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)

return await AsyncGroup.from_store(
store=store_path,
zarr_format=zarr_format,
overwrite=overwrite,
attributes=attributes,
)


async def open_group(
store: StoreLike | None = None,
*, # Note: this is a change from v2
mode: AccessModeLiteral = "a",
cache_attrs: bool | None = None, # not used, default changed
synchronizer: Any = None, # not used
path: str | None = None,
chunk_store: StoreLike | None = None, # not used
storage_options: dict[str, Any] | None = None,
zarr_version: ZarrFormat | None = None, # deprecated
zarr_format: ZarrFormat | None = None,
meta_array: Any | None = None, # not used
attributes: dict[str, JSON] | None = None,
use_consolidated: bool | str | None = None,
) -> AsyncGroup:
"""Open a group using file-mode-like semantics.

Parameters
----------
store : Store, str, or mapping, optional
Store or path to directory in file system or name of zip file.

Strings are interpreted as paths on the local file system
and used as the ``root`` argument to :class:`zarr.storage.LocalStore`.

Dictionaries are used as the ``store_dict`` argument in
:class:`zarr.storage.MemoryStore``.

By default (``store=None``) a new :class:`zarr.storage.MemoryStore`
is created.

mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
cache_attrs : bool, optional
If True (default), user attributes will be cached for attribute read
operations. If False, user attributes are reloaded from the store prior
to all attribute read operations.
synchronizer : object, optional
Array synchronizer.
path : str, optional
Group path within store.
chunk_store : Store or str, optional
Store or path to directory in file system or name of zip file.
storage_options : dict
If using an fsspec URL to create the store, these will be passed to
the backend implementation. Ignored otherwise.
meta_array : array-like, optional
An array instance to use for determining arrays to create and return
to users. Use `numpy.empty(())` by default.
attributes : dict
A dictionary of JSON-serializable values with user-defined attributes.
use_consolidated : bool or str, default None
Whether to use consolidated metadata.

By default, consolidated metadata is used if it's present in the
store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file
for Zarr format 2).

To explicitly require consolidated metadata, set ``use_consolidated=True``,
which will raise an exception if consolidated metadata is not found.

To explicitly *not* use consolidated metadata, set ``use_consolidated=False``,
which will fall back to using the regular, non consolidated metadata.

Zarr format 2 allowed configuring the key storing the consolidated metadata
(``.zmetadata`` by default). Specify the custom key as ``use_consolidated``
to load consolidated metadata from a non-default key.

Returns
-------
g : group
The new group.
"""

zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)

if cache_attrs is not None:
warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2)
if synchronizer is not None:
warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2)
if meta_array is not None:
warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)
if chunk_store is not None:
warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)

store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)

if attributes is None:
attributes = {}

try:
if mode in _READ_MODES:
return await AsyncGroup.open(
store_path, zarr_format=zarr_format, use_consolidated=use_consolidated
)
except (KeyError, FileNotFoundError):
pass
if mode in _CREATE_MODES:
overwrite = _infer_overwrite(mode)
_zarr_format = zarr_format or _default_zarr_format()
return await AsyncGroup.from_store(
store_path,
zarr_format=_zarr_format,
overwrite=overwrite,
attributes=attributes,
)
raise FileNotFoundError(f"Unable to find group: {store_path}")


async def create(
shape: ChunkCoords | int,
*, # Note: this is a change from v2
Expand Down
Loading
Loading