docs

Author: d-v-b

docs/quickstart.rst

@@ -119,6 +119,28 @@ Zarr allows you to create hierarchical groups, similar to directories::
 
 This creates a group with two datasets: ``foo`` and ``bar``.
 
+Batch Hierarchy Creation
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Zarr provides tools for creating a collection of arrays and groups with a single function call.
+Suppose we want to copy existing groups and arrays into a new storage backend:
+
+    >>> # Create nested groups and add arrays
+    >>> root = zarr.group("data/example-3.zarr", attributes={'name': 'root'})
+    >>> foo = root.create_group(name="foo")
+    >>> bar = root.create_array(
+    ...     name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4"
+    ... )
+    >>> nodes = {'': root.metadata} | {k: v.metadata for k,v in root.members()}
+    >>> print(nodes)
+    >>> from zarr.storage import MemoryStore
+    >>> new_nodes = dict(zarr.create_hierarchy(store=MemoryStore(), nodes=nodes))
+    >>> new_root = new_nodes['']
+    >>> assert new_root.attrs == root.attrs
+
+Note that :func:`zarr.create_hierarchy` will only initialize arrays and groups -- copying array data must
+be done in a separate step.
+
 Persistent Storage
 ------------------
 

docs/user-guide/groups.rst

@@ -75,6 +75,31 @@ For more information on groups see the :class:`zarr.Group` API docs.
 
 .. _user-guide-diagnostics:
 
+Batch Group Creation
+--------------------
+
+You can also create multiple groups concurrently with a single function call. :func:`zarr.create_hierarchy` takes
+a :class:`zarr.storage.Store` instance and a dict of ``key : metadata`` pairs, parses that dict, and
+writes metadata documents to storage:
+
+   >>> from zarr import create_hierarchy
+   >>> from zarr.core.group import GroupMetadata
+   >>> from zarr.storage import LocalStore
+   >>> node_spec = {'a/b/c': GroupMetadata()}
+   >>> nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec))
+   >>> print(sorted(nodes_created.items(), key=lambda kv: len(kv[0])))
+   [('', <Group file://data>), ('a', <Group file://data/a>), ('a/b', <Group file://data/a/b>), ('a/b/c', <Group file://data/a/b/c>)]
+
+Note that we only specified a single group named ``a/b/c``, but 4 groups were created. These additional groups
+were created to ensure that the desired node ``a/b/c`` is connected to the root group ``''`` by a sequence
+of intermediate groups. :func:`zarr.create_hierarchy` normalizes the ``nodes`` keyword argument to
+ensure that the resulting hierarchy is complete, i.e. all groups or arrays are connected to the root
+of the hierarchy via intermediate groups.
+
+Because :func:`zarr.create_hierarchy` concurrently creates metadata documents, it's more efficient
+than repeated calls to :func:`create_group` or :func:`create_array`, provided you can statically define
+the metadata for the groups and arrays you want to create.
+
 Array and group diagnostics
 ---------------------------
 

src/zarr/__init__.py

@@ -52,8 +52,6 @@
     "create_array",
     "create_group",
     "create_hierarchy",
-    "create_nodes",
-    "create_rooted_hierarchy",
     "empty",
     "empty_like",
     "full",

src/zarr/core/group.py

@@ -2866,15 +2866,10 @@ async def create_hierarchy(
     """
     Create a complete zarr hierarchy from a collection of metadata objects.
 
-    This function will parse its input to ensure that the hierarchy is valid. In this context,
-    "valid" means that the following requirements are met:
-    * All nodes have the same zarr format.
-    * There are no nodes descending from arrays.
-    * There are no implicit groups. Any implicit groups will be inserted as needed. For example,
-    an input like ```{'a': GroupMetadata, 'a/b/c': GroupMetadata}``` defines an implicit group at
-    the path ```a/b```, and also at the root of the hierarchy, which we denote with the empty string.
-    After parsing, that group will be added and the input will be:
-    ```{'': GroupMetadata, 'a': GroupMetadata, 'a/b': GroupMetadata, 'a/b/c': GroupMetadata}```
+    This function will parse its input to ensure that the hierarchy is complete. Any implicit groups
+    will be inserted as needed. For example, an input like
+    ```{'a/b': GroupMetadata}``` will be parsed to
+    ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```
 
     After input parsing, this function then creates all the nodes in the hierarchy concurrently.
 
@@ -2886,22 +2881,38 @@ async def create_hierarchy(
     store : Store
         The storage backend to use.
     nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata]
-        A dictionary defining the hierarchy. The keys are the paths of the nodes
-        in the hierarchy, and the values are the metadata of the nodes. The
-        metadata must be either an instance of GroupMetadata, ArrayV3Metadata
-        or ArrayV2Metadata.
+        A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy,
+        relative to the root of the ``Store``. The root of the store can be specified with the empty
+        string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that
+        all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the
+        same hierarchy.
     overwrite : bool
         Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is
         raised instead of overwriting an existing array or group.
 
+        This function will not erase an existing group unless that group is explicitly named in
+        ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a
+        group already exists at path ``a``, then this function will leave the group at ``a`` as-is.
+
     Yields
     ------
-    AsyncGroup | AsyncArray
-        The created nodes in the order they are created.
+    tuple[str, AsyncGroup | AsyncArray]
+        This function yields (path, node) pairs, in the order the nodes were created.
 
     Examples
     --------
-
+    from zarr.api.asynchronous import create_hierarchy
+    from zarr.storage import MemoryStore
+    from zarr.core.group import GroupMetadata
+    import asyncio
+    store = MemoryStore()
+    nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})}
+
+    async def run():
+        print(dict([x async for x in create_hierarchy(store=store, nodes=nodes)]))
+
+    asyncio.run(run())
+    # {'a': <AsyncGroup memory://140345143770112/a>, '': <AsyncGroup memory://140345143770112>}
     """
     # normalize the keys to be valid paths
     nodes_normed_keys = _normalize_path_keys(nodes)

src/zarr/core/sync_group.py

@@ -57,25 +57,50 @@ def create_hierarchy(
     """
     Create a complete zarr hierarchy from a collection of metadata objects.
 
-    Groups that are implicitly defined by the input will be created as needed.
+    This function will parse its input to ensure that the hierarchy is complete. Any implicit groups
+    will be inserted as needed. For example, an input like
+    ```{'a/b': GroupMetadata}``` will be parsed to
+    ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```
 
-    This function takes a parsed hierarchy dictionary and creates all the nodes in the hierarchy
-    concurrently. Arrays and Groups are yielded in the order they are created.
+    After input parsing, this function then creates all the nodes in the hierarchy concurrently.
+
+    Arrays and Groups are yielded in the order they are created. This order is not stable and
+    should not be relied on.
 
     Parameters
     ----------
     store : Store
         The storage backend to use.
     nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata]
-        A dictionary defining the hierarchy. The keys are the paths of the nodes
-        in the hierarchy, and the values are the metadata of the nodes. The
-        metadata must be either an instance of GroupMetadata, ArrayV3Metadata
-        or ArrayV2Metadata.
+        A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy,
+        relative to the root of the ``Store``. The root of the store can be specified with the empty
+        string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that
+        all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the
+        same hierarchy.
+    overwrite : bool
+        Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is
+        raised instead of overwriting an existing array or group.
+
+        This function will not erase an existing group unless that group is explicitly named in
+        ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a
+        group already exists at path ``a``, then this function will leave the group at ``a`` as-is.
 
     Yields
     ------
-    Group | Array
-        The created nodes in the order they are created.
+    tuple[str, Group | Array]
+        This function yields (path, node) pairs, in the order the nodes were created.
+
+    Examples
+    --------
+    from zarr import create_hierarchy
+    from zarr.storage import MemoryStore
+    from zarr.core.group import GroupMetadata
+
+    store = MemoryStore()
+    nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})}
+    nodes_created = dict(create_hierarchy(store=store, nodes=nodes))
+    print(nodes)
+    # {'a': GroupMetadata(attributes={'name': 'leaf'}, zarr_format=3, consolidated_metadata=None, node_type='group')}
     """
     coro = create_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite)