Merge branch 'main' into speed-up-tests

Author: dcherian

.gitignore

@@ -51,7 +51,7 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
-docs/_autoapi
+docs/api
 docs/data
 data
 data.zip

docs/Makefile

@@ -52,7 +52,7 @@ help:
 .PHONY: clean
 clean:
 	rm -rf $(BUILDDIR)/*
-	rm -rf $(BUILDDIR)/../_autoapi
+	rm -rf $(BUILDDIR)/../api
 
 .PHONY: html
 html:

docs/api/index.rst

@@ -57,7 +57,7 @@
 autoapi_add_toctree_entry = False
 autoapi_generate_api_docs = True
 autoapi_member_order = "groupwise"
-autoapi_root = "_autoapi"
+autoapi_root = "api"
 autoapi_keep_files = True
 autoapi_options = [ 'members', 'undoc-members', 'show-inheritance', 'show-module-summary', 'imported-members', ]
 
@@ -108,6 +108,7 @@ def skip_submodules(
      "release": "developers/release.html",
      "roadmap": "developers/roadmap.html",
      "installation": "user-guide/installation.html",
+     "api": "api/zarr/index"
 }
 
 # The language for content autogenerated by Sphinx. Refer to documentation

docs/conf.py

@@ -10,7 +10,7 @@ Zarr-Python
 
     quickstart
     user-guide/index
-    api/index
+    API reference <api/zarr/index>
     developers/index
     developers/release
     about
@@ -25,7 +25,7 @@ Zarr-Python
 
 Zarr-Python is a Python library for reading and writing Zarr groups and arrays. Highlights include:
 
-* Specification support for both Zarr v2 and v3.
+* Specification support for both Zarr format 2 and 3.
 * Create and read from N-dimensional arrays using NumPy-like semantics.
 * Flexible storage enables reading and writing from local, cloud and in-memory stores.
 * High performance: Enables fast I/O with support for asynchronous I/O and multi-threading.
@@ -81,12 +81,12 @@ Zarr-Python is a Python library for reading and writing Zarr groups and arrays.
 
         +++
 
-        .. button-ref:: api/index
+        .. button-ref:: api/zarr/index
             :expand:
             :color: dark
             :click-parent:
 
-            To the API reference guide
+            To the API reference
 
     .. grid-item-card::
         :img-top: _static/index_contribute.svg

docs/index.rst

@@ -168,8 +168,8 @@ argument accepted by all array creation functions. For example::
    >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000)
    >>> z = zarr.create_array(store='data/example-5.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors)
    >>> z[:] = data
-   >>> z.metadata.codecs
-   [BytesCodec(endian=<Endian.little: 'little'>), BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=3, shuffle=<BloscShuffle.bitshuffle: 'bitshuffle'>, blocksize=0)]
+   >>> z.compressors
+   (BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=3, shuffle=<BloscShuffle.bitshuffle: 'bitshuffle'>, blocksize=0),)
 
 This array above will use Blosc as the primary compressor, using the Zstandard
 algorithm (compression level 3) internally within Blosc, and with the
@@ -188,7 +188,9 @@ which can be used to print useful diagnostics, e.g.::
    Order              : C
    Read-only          : False
    Store type         : LocalStore
-   Codecs             : [{'endian': <Endian.little: 'little'>}, {'typesize': 4, 'cname': <BloscCname.zstd: 'zstd'>, 'clevel': 3, 'shuffle': <BloscShuffle.bitshuffle: 'bitshuffle'>, 'blocksize': 0}]
+   Filters            : ()
+   Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
+   Compressors        : (BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=3, shuffle=<BloscShuffle.bitshuffle: 'bitshuffle'>, blocksize=0),)
    No. bytes          : 400000000 (381.5M)
 
 The :func:`zarr.Array.info_complete` method inspects the underlying store and
@@ -203,7 +205,9 @@ prints additional diagnostics, e.g.::
    Order              : C
    Read-only          : False
    Store type         : LocalStore
-   Codecs             : [{'endian': <Endian.little: 'little'>}, {'typesize': 4, 'cname': <BloscCname.zstd: 'zstd'>, 'clevel': 3, 'shuffle': <BloscShuffle.bitshuffle: 'bitshuffle'>, 'blocksize': 0}]
+   Filters            : ()
+   Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
+   Compressors        : (BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=3, shuffle=<BloscShuffle.bitshuffle: 'bitshuffle'>, blocksize=0),)
    No. bytes          : 400000000 (381.5M)
    No. bytes stored   : 9696302
    Storage ratio      : 41.3
@@ -223,8 +227,8 @@ here is an array using Gzip compression, level 1::
    >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000)
    >>> z = zarr.create_array(store='data/example-6.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=zarr.codecs.GzipCodec(level=1))
    >>> z[:] = data
-   >>> z.metadata.codecs
-   [BytesCodec(endian=<Endian.little: 'little'>), GzipCodec(level=1)]
+   >>> z.compressors
+   (GzipCodec(level=1),)
 
 Here is an example using LZMA from NumCodecs_ with a custom filter pipeline including LZMA's
 built-in delta filter::
@@ -236,23 +240,24 @@ built-in delta filter::
    >>> compressors = LZMA(filters=lzma_filters)
    >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000)
    >>> z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors)
-   >>> z.metadata.codecs
-   [BytesCodec(endian=<Endian.little: 'little'>), _make_bytes_bytes_codec.<locals>._Codec(codec_name='numcodecs.lzma', codec_config={'id': 'lzma', 'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]})]
+   >>> z.compressors
+   (_make_bytes_bytes_codec.<locals>._Codec(codec_name='numcodecs.lzma', codec_config={'id': 'lzma', 'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),)
 
 The default compressor can be changed by setting the value of the using Zarr's
 :ref:`user-guide-config`, e.g.::
 
    >>> with zarr.config.set({'array.v2_default_compressor.numeric': {'id': 'blosc'}}):
    ...     z = zarr.create_array(store={}, shape=(100000000,), chunks=(1000000,), dtype='int32', zarr_format=2)
-   >>> z.metadata.filters
-   >>> z.metadata.compressor
-   Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
+   >>> z.filters
+   ()
+   >>> z.compressors
+   (Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0),)
 
 To disable compression, set ``compressors=None`` when creating an array, e.g.::
 
    >>> z = zarr.create_array(store='data/example-8.zarr', shape=(100000000,), chunks=(1000000,), dtype='int32', compressors=None)
-   >>> z.metadata.codecs
-   [BytesCodec(endian=<Endian.little: 'little'>)]
+   >>> z.compressors
+   ()
 
 .. _user-guide-filters:
 
@@ -287,7 +292,9 @@ Here is an example using a delta filter with the Blosc compressor::
    Order              : C
    Read-only          : False
    Store type         : LocalStore
-   Codecs             : [{'codec_name': 'numcodecs.delta', 'codec_config': {'id': 'delta', 'dtype': 'int32'}}, {'endian': <Endian.little: 'little'>}, {'typesize': 4, 'cname': <BloscCname.zstd: 'zstd'>, 'clevel': 1, 'shuffle': <BloscShuffle.shuffle: 'shuffle'>, 'blocksize': 0}]
+   Filters            : (_make_array_array_codec.<locals>._Codec(codec_name='numcodecs.delta', codec_config={'id': 'delta', 'dtype': 'int32'}),)
+   Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
+   Compressors        : (BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=1, shuffle=<BloscShuffle.shuffle: 'shuffle'>, blocksize=0),)
    No. bytes          : 400000000 (381.5M)
 
 For more information about available filter codecs, see the `Numcodecs
@@ -600,11 +607,13 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za
   Order              : C
   Read-only          : False
   Store type         : LocalStore
-  Codecs             : [{'chunk_shape': (100, 100), 'codecs': ({'endian': <Endian.little: 'little'>}, {'level': 0, 'checksum': False}), 'index_codecs': ({'endian': <Endian.little: 'little'>}, {}), 'index_location': <ShardingCodecIndexLocation.end: 'end'>}]
+  Filters            : ()
+  Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
+  Compressors        : (ZstdCodec(level=0, checksum=False),)
   No. bytes          : 100000000 (95.4M)
   No. bytes stored   : 3981060
   Storage ratio      : 25.1
-  Chunks Initialized : 100
+  Shards Initialized : 100
 
 In this example a shard shape of (1000, 1000) and a chunk shape of (100, 100) is used.
 This means that 10*10 chunks are stored in each shard, and there are 10*10 shards in total.

docs/user-guide/arrays.rst

@@ -28,7 +28,7 @@ Configuration options include the following:
 
 - Default Zarr format ``default_zarr_version``
 - Default array order in memory ``array.order``
-- Default codecs ``array.v3_default_codecs`` and ``array.v2_default_compressor``
+- Default filters, serializers and compressors, e.g. ``array.v3_default_filters``, ``array.v3_default_serializer``, ``array.v3_default_compressors``, ``array.v2_default_filters`` and ``array.v2_default_compressor``
 - Whether empty chunks are written to storage ``array.write_empty_chunks``
 - Async and threading options, e.g. ``async.concurrency`` and ``threading.max_workers``
 - Selections of implementations of codecs, codec pipelines and buffers
@@ -54,19 +54,20 @@ This is the current default configuration::
               'v2_default_filters': {'bytes': [{'id': 'vlen-bytes'}],
                                      'numeric': None,
                                      'string': [{'id': 'vlen-utf8'}]},
-              'v3_default_codecs': {'bytes': [{'name': 'vlen-bytes'},
-                                              {'configuration': {'checksum': False,
-                                                                 'level': 0},
-                                               'name': 'zstd'}],
-                                    'numeric': [{'configuration': {'endian': 'little'},
-                                                 'name': 'bytes'},
-                                                {'configuration': {'checksum': False,
-                                                                   'level': 0},
-                                                 'name': 'zstd'}],
-                                    'string': [{'name': 'vlen-utf8'},
-                                               {'configuration': {'checksum': False,
-                                                                  'level': 0},
-                                                'name': 'zstd'}]},
+              'v3_default_compressors': {'bytes': [{'configuration': {'checksum': False,
+                                                                      'level': 0},
+                                                    'name': 'zstd'}],
+                                         'numeric': [{'configuration': {'checksum': False,
+                                                                        'level': 0},
+                                                      'name': 'zstd'}],
+                                         'string': [{'configuration': {'checksum': False,
+                                                                       'level': 0},
+                                                     'name': 'zstd'}]},
+              'v3_default_filters': {'bytes': [], 'numeric': [], 'string': []},
+              'v3_default_serializer': {'bytes': {'name': 'vlen-bytes'},
+                                        'numeric': {'configuration': {'endian': 'little'},
+                                                    'name': 'bytes'},
+                                        'string': {'name': 'vlen-utf8'}},
               'write_empty_chunks': False},
     'async': {'concurrency': 10, 'timeout': None},
     'buffer': 'zarr.core.buffer.cpu.Buffer',

docs/user-guide/config.rst

@@ -52,8 +52,8 @@ that can be used.:
                           chunk_key_encoding=DefaultChunkKeyEncoding(name='default',
                                                                      separator='/'),
                           fill_value=np.float64(0.0),
-                          codecs=[BytesCodec(endian=<Endian.little: 'little'>),
-                                  ZstdCodec(level=0, checksum=False)],
+                          codecs=(BytesCodec(endian=<Endian.little: 'little'>),
+                                  ZstdCodec(level=0, checksum=False)),
                           attributes={},
                           dimension_names=None,
                           zarr_format=3,
@@ -65,8 +65,8 @@ that can be used.:
                           chunk_key_encoding=DefaultChunkKeyEncoding(name='default',
                                                                      separator='/'),
                           fill_value=np.float64(0.0),
-                          codecs=[BytesCodec(endian=<Endian.little: 'little'>),
-                                  ZstdCodec(level=0, checksum=False)],
+                          codecs=(BytesCodec(endian=<Endian.little: 'little'>),
+                                  ZstdCodec(level=0, checksum=False)),
                           attributes={},
                           dimension_names=None,
                           zarr_format=3,
@@ -78,8 +78,8 @@ that can be used.:
                           chunk_key_encoding=DefaultChunkKeyEncoding(name='default',
                                                                      separator='/'),
                           fill_value=np.float64(0.0),
-                          codecs=[BytesCodec(endian=<Endian.little: 'little'>),
-                                  ZstdCodec(level=0, checksum=False)],
+                          codecs=(BytesCodec(endian=<Endian.little: 'little'>),
+                                  ZstdCodec(level=0, checksum=False)),
                           attributes={},
                           dimension_names=None,
                           zarr_format=3,

docs/user-guide/consolidated_metadata.rst

@@ -10,8 +10,8 @@ Custom codecs
 -------------
 
 .. note::
-    This section explains how custom codecs can be created for Zarr version 3 data. For Zarr
-    version 2, codecs should subclass the
+    This section explains how custom codecs can be created for Zarr format 3 arrays. For Zarr
+    format 2, codecs should subclass the
     `numcodecs.abc.Codec <https://numcodecs.readthedocs.io/en/stable/abc.html#numcodecs.abc.Codec>`_
     base class and register through
     `numcodecs.registry.register_codec <https://numcodecs.readthedocs.io/en/stable/registry.html#numcodecs.registry.register_codec>`_.
@@ -66,7 +66,7 @@ strongly recommended to prefix the codec identifier with a unique name. For exam
 the codecs from ``numcodecs`` are prefixed with ``numcodecs.``, e.g. ``numcodecs.delta``.
 
 .. note::
-    Note that the extension mechanism for the Zarr version 3 is still under development.
+    Note that the extension mechanism for the Zarr format 3 is still under development.
     Requirements for custom codecs including the choice of codec identifiers might
     change in the future.
 

docs/user-guide/extending.rst

@@ -109,7 +109,9 @@ property. E.g.::
    Order              : C
    Read-only          : False
    Store type         : MemoryStore
-   Codecs             : [{'endian': <Endian.little: 'little'>}, {'level': 0, 'checksum': False}]
+   Filters            : ()
+   Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
+   Compressors        : (ZstdCodec(level=0, checksum=False),)
    No. bytes          : 8000000 (7.6M)
    No. bytes stored   : 1432
    Storage ratio      : 5586.6
@@ -123,7 +125,9 @@ property. E.g.::
    Order              : C
    Read-only          : False
    Store type         : MemoryStore
-   Codecs             : [{'endian': <Endian.little: 'little'>}, {'level': 0, 'checksum': False}]
+   Filters            : ()
+   Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
+   Compressors        : (ZstdCodec(level=0, checksum=False),)
    No. bytes          : 4000000 (3.8M)
 
 Groups also have the :func:`zarr.Group.tree` method, e.g.::