Merge branch 'main' into km/child-of-array

Author: d-v-b

.github/workflows/gpu_test.yml

@@ -30,6 +30,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v5
+      with:
+        fetch-depth: 0  # grab all branches and tags
     # - name: cuda-toolkit
     #   uses: Jimver/[email protected]
     #   id: cuda-toolkit

changes/1798.feature.rst

@@ -0,0 +1,2 @@
+Add a command-line interface to migrate v2 Zarr metadata to v3. Corresponding functions are also
+provided under zarr.metadata.

changes/3436.feature.rst

@@ -0,0 +1,2 @@
+Adds a registry for chunk key encodings for extensibility.
+This allows users to implement a custom `ChunkKeyEncoding`, which can be registered via `register_chunk_key_encoding` or as an entry point under `zarr.chunk_key_encoding`.

docs/user-guide/cli.rst

@@ -0,0 +1,127 @@
+.. _user-guide-cli:
+
+Command-line interface
+========================
+
+Zarr-Python provides a command-line interface that enables:
+
+- migration of Zarr v2 metadata to v3
+- removal of v2 or v3 metadata
+
+To see available commands run the following in a terminal:
+
+.. code-block:: bash
+
+    $ zarr --help
+
+or to get help on individual commands:
+
+.. code-block:: bash
+
+    $ zarr migrate --help
+
+    $ zarr remove-metadata --help
+
+
+Migrate metadata from v2 to v3
+------------------------------
+
+Migrate to a separate location
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To migrate a Zarr array/group's metadata from v2 to v3 run:
+
+.. code-block:: bash
+
+    $ zarr migrate v3 path/to/input.zarr path/to/output.zarr
+
+This will write new ``zarr.json`` files to ``output.zarr``, leaving ``input.zarr`` un-touched.
+Note - this will migrate the entire Zarr hierarchy, so if ``input.zarr`` contains multiple groups/arrays,
+new ``zarr.json`` will be made for all of them.
+
+Migrate in-place
+~~~~~~~~~~~~~~~~
+
+If you'd prefer to migrate the metadata in-place run:
+
+.. code-block:: bash
+
+    $ zarr migrate v3 path/to/input.zarr
+
+This will write new ``zarr.json`` files to ``input.zarr``, leaving the existing v2 metadata un-touched.
+
+To open the array/group using the new metadata use:
+
+.. code-block:: python
+
+    >>> import zarr
+    >>> zarr_with_v3_metadata = zarr.open('path/to/input.zarr', zarr_format=3)
+
+Once you are happy with the conversion, you can run the following to remove the old v2 metadata:
+
+.. code-block:: bash
+
+    $ zarr remove-metadata v2 path/to/input.zarr
+
+Note there is also a shortcut to migrate and remove v2 metadata in one step:
+
+.. code-block:: bash
+
+    $ zarr migrate v3 path/to/input.zarr --remove-v2-metadata
+
+
+Remove metadata
+----------------
+
+Remove v2 metadata using:
+
+.. code-block:: bash
+
+    $ zarr remove-metadata v2 path/to/input.zarr
+
+or v3 with:
+
+.. code-block:: bash
+
+    $ zarr remove-metadata v3 path/to/input.zarr
+
+By default, this will only allow removal of metadata if a valid alternative exists. For example, you can't
+remove v2 metadata unless v3 metadata exists at that location.
+
+To override this behaviour use ``--force``:
+
+.. code-block:: bash
+
+    $ zarr remove-metadata v3 path/to/input.zarr --force
+
+
+Dry run
+--------
+All commands provide a ``--dry-run`` option that will log changes that would be made on a real run, without creating
+or modifying any files.
+
+.. code-block:: bash
+
+    $ zarr migrate v3 path/to/input.zarr --dry-run
+
+    Dry run enabled - no new files will be created or changed. Log of files that would be created on a real run:
+    Saving metadata to path/to/input.zarr/zarr.json
+
+
+Verbose
+--------
+You can also add ``--verbose`` **before** any command, to see a full log of its actions:
+
+.. code-block:: bash
+
+    $ zarr --verbose migrate v3 path/to/input.zarr
+
+    $ zarr --verbose remove-metadata v2 path/to/input.zarr
+
+
+Equivalent functions
+--------------------
+All features of the command-line interface are also available via functions under
+:mod:`zarr.metadata`.
+
+

docs/user-guide/consolidated_metadata.rst

@@ -49,44 +49,41 @@ that can be used.:
    >>> from pprint import pprint
    >>> pprint(dict(consolidated_metadata.items()))
    {'a': ArrayV3Metadata(shape=(1,),
-                          data_type=Float64(endianness='little'),
-                          chunk_grid=RegularChunkGrid(chunk_shape=(1,)),
-                          chunk_key_encoding=DefaultChunkKeyEncoding(name='default',
-                                                                     separator='/'),
-                          fill_value=np.float64(0.0),
-                          codecs=(BytesCodec(endian=<Endian.little: 'little'>),
-                                  ZstdCodec(level=0, checksum=False)),
-                          attributes={},
-                          dimension_names=None,
-                          zarr_format=3,
-                          node_type='array',
-                          storage_transformers=()),
-     'b': ArrayV3Metadata(shape=(2, 2),
-                          data_type=Float64(endianness='little'),
-                          chunk_grid=RegularChunkGrid(chunk_shape=(2, 2)),
-                          chunk_key_encoding=DefaultChunkKeyEncoding(name='default',
-                                                                     separator='/'),
-                          fill_value=np.float64(0.0),
-                          codecs=(BytesCodec(endian=<Endian.little: 'little'>),
-                                  ZstdCodec(level=0, checksum=False)),
-                          attributes={},
-                          dimension_names=None,
-                          zarr_format=3,
-                          node_type='array',
-                          storage_transformers=()),
-     'c': ArrayV3Metadata(shape=(3, 3, 3),
-                          data_type=Float64(endianness='little'),
-                          chunk_grid=RegularChunkGrid(chunk_shape=(3, 3, 3)),
-                          chunk_key_encoding=DefaultChunkKeyEncoding(name='default',
-                                                                     separator='/'),
-                          fill_value=np.float64(0.0),
-                          codecs=(BytesCodec(endian=<Endian.little: 'little'>),
-                                  ZstdCodec(level=0, checksum=False)),
-                          attributes={},
-                          dimension_names=None,
-                          zarr_format=3,
-                          node_type='array',
-                          storage_transformers=())}
+                         data_type=Float64(endianness='little'),
+                         chunk_grid=RegularChunkGrid(chunk_shape=(1,)),
+                         chunk_key_encoding=DefaultChunkKeyEncoding(separator='/'),
+                         fill_value=np.float64(0.0),
+                         codecs=(BytesCodec(endian=<Endian.little: 'little'>),
+                                 ZstdCodec(level=0, checksum=False)),
+                         attributes={},
+                         dimension_names=None,
+                         zarr_format=3,
+                         node_type='array',
+                         storage_transformers=()),
+    'b': ArrayV3Metadata(shape=(2, 2),
+                         data_type=Float64(endianness='little'),
+                         chunk_grid=RegularChunkGrid(chunk_shape=(2, 2)),
+                         chunk_key_encoding=DefaultChunkKeyEncoding(separator='/'),
+                         fill_value=np.float64(0.0),
+                         codecs=(BytesCodec(endian=<Endian.little: 'little'>),
+                                 ZstdCodec(level=0, checksum=False)),
+                         attributes={},
+                         dimension_names=None,
+                         zarr_format=3,
+                         node_type='array',
+                         storage_transformers=()),
+    'c': ArrayV3Metadata(shape=(3, 3, 3),
+                         data_type=Float64(endianness='little'),
+                         chunk_grid=RegularChunkGrid(chunk_shape=(3, 3, 3)),
+                         chunk_key_encoding=DefaultChunkKeyEncoding(separator='/'),
+                         fill_value=np.float64(0.0),
+                         codecs=(BytesCodec(endian=<Endian.little: 'little'>),
+                                 ZstdCodec(level=0, checksum=False)),
+                         attributes={},
+                         dimension_names=None,
+                         zarr_format=3,
+                         node_type='array',
+                         storage_transformers=())}
 
 Operations on the group to get children automatically use the consolidated metadata.:
 

docs/user-guide/index.rst

@@ -13,6 +13,7 @@ User guide
     storage
     config
     v3_migration
+    cli
 
 Advanced Topics
 ---------------

pyproject.toml

@@ -68,6 +68,7 @@ remote = [
 gpu = [
     "cupy-cuda12x",
 ]
+cli = ["typer"]
 # Development extras
 test = [
     "coverage>=7.10",
@@ -114,6 +115,9 @@ docs = [
     'pytest'
 ]
 
+[project.scripts]
+zarr = "zarr._cli.cli:app"
+
 
 [project.urls]
 issues = "https://github.com/zarr-developers/zarr-python/issues"
@@ -164,7 +168,7 @@ deps = ["minimal", "optional"]
 
 [tool.hatch.envs.test.overrides]
 matrix.deps.dependencies = [
-  {value = "zarr[remote, remote_tests, test, optional]", if = ["optional"]}
+  {value = "zarr[remote, remote_tests, test, optional, cli]", if = ["optional"]}
 ]
 
 [tool.hatch.envs.test.scripts]

src/zarr/__init__.py

@@ -1,3 +1,7 @@
+import functools
+import logging
+from typing import Literal
+
 from zarr._version import version as __version__
 from zarr.api.synchronous import (
     array,
@@ -37,6 +41,8 @@
 # in case setuptools scm screw up and find version to be 0.0.0
 assert not __version__.startswith("0.0.0")
 
+_logger = logging.getLogger(__name__)
+
 
 def print_debug_info() -> None:
     """
@@ -85,6 +91,58 @@ def print_packages(packages: list[str]) -> None:
     print_packages(optional)
 
 
+# The decorator ensures this always returns the same handler (and it is only
+# attached once).
+@functools.cache
+def _ensure_handler() -> logging.Handler:
+    """
+    The first time this function is called, attach a `StreamHandler` using the
+    same format as `logging.basicConfig` to the Zarr-Python root logger.
+
+    Return this handler every time this function is called.
+    """
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+    _logger.addHandler(handler)
+    return handler
+
+
+def set_log_level(
+    level: Literal["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+) -> None:
+    """Set the logging level for Zarr-Python.
+
+    Zarr-Python uses the standard library `logging` framework under the root
+    logger 'zarr'.  This is a helper function to:
+
+    - set Zarr-Python's root logger level
+    - set the root logger handler's level, creating the handler
+      if it does not exist yet
+
+    Parameters
+    ----------
+    level : str
+        The logging level to set.
+    """
+    _logger.setLevel(level)
+    _ensure_handler().setLevel(level)
+
+
+def set_format(log_format: str) -> None:
+    """Set the format of logging messages from Zarr-Python.
+
+    Zarr-Python uses the standard library `logging` framework under the root
+    logger 'zarr'. This sets the format of log messages from the root logger's StreamHandler.
+
+    Parameters
+    ----------
+    log_format : str
+        A string determining the log format (as defined in the standard library's `logging` module
+        for logging.Formatter)
+    """
+    _ensure_handler().setFormatter(logging.Formatter(fmt=log_format))
+
+
 __all__ = [
     "Array",
     "AsyncArray",