Move metadata handling to a class (updated) (#839)

* fix conflicts

* cleanup naming

* zip move

* fix erasability test

* test for warning

* please flake

* remove uncovered lines

* remove uncovered lines in tests

* pragma no cover for exceptional case

* minor docstring fixes

add assert statements to test_capabilities

* pep8 fix

* avoid NumPy 1.21.0 due to numpy/numpy#19325

* move Store class and some helper functions to zarr._storage.store

update version in Store docstring

* BUG: ABSStore should inherit from Store

* pep8 fix

* TST: make CustomMapping a subclass of Store

TST: initialize stores with KVStore(dict()) instead of bare dict()

* update version mentioned in Store docstring

* update version mentioned in warning message

* use Store._ensure_store in Attributes class

ensures Attributes.store is a Store

* TST: add Attributes test case ensuring store gets coerced to a Store

* use Store._ensure_store in normalize_store_arg

ensures open_array, etc can work when the user supplies a dict

* TST: make sure high level creation functions also work when passed a dict for store

* TST: add test case with group initialized from dict

* TST: add test case with Array initialized from dict

* change CustomMapping back to type object, not Store

want to test the non-Store code path in _ensure_store

* pep8 fixes

* update/fix new hierarchy test case to complete code coverage

* create a BaseStore parent for Store

BaseStore does not have the listdir or rmdir methods

cleaned up some type declerations, making sure mypy passes

* Convert functions in zarr.meta to class methods

This is done to ease transition to Zarr v3 support.
When adding v3 support, we can override encode and decode methods to account for changes
in the metadata format.
The classmethods are also exported under the old function names for backwards compatibility.

Co-authored-by: Matthias Bussonier <[email protected]>

* Add a _metadata_class attribute to the Store class

Because existing functions allow coerce of dict to store, there are a lot of
hasattr calls here. We can remove these checks if we start enforcing that the
input MUST be a Store.

Use of this _metadata_class will ease the transition to v3

* Use _metadata_class attribute in test_storage.py

This will make it easier to reuse existing testing code when adding v3 support

* remove unused imports

use _metadata_class methods

* remove unnecessary hasattr checks

In these cases self._store was created using _ensure_store, so it will always have the attribute

* test migrate1to2 with store=dict() and store=KVStore(dict())

* pep8: remove unused imports

* misc

* s

* remove unused FLOAT_FILLS

* mypy fixes

* sync metadata methods with current master branch

* flake8

* restore is_erasable check to rmdir function

Otherwise the save_array doc example fails to write to a ZipStore

Co-authored-by: Matthias Bussonnier <[email protected]>
Co-authored-by: Josh Moore <[email protected]>
Co-authored-by: jmoore <[email protected]>

Author: 3 people

zarr/_storage/store.py

@@ -1,6 +1,7 @@
 from collections.abc import MutableMapping
 from typing import Any, List, Optional, Union
 
+from zarr.meta import Metadata2
 from zarr.util import normalize_storage_path
 
 # v2 store keys
@@ -32,6 +33,8 @@ class BaseStore(MutableMapping):
     _writeable = True
     _erasable = True
     _listable = True
+    _store_version = 2
+    _metadata_class = Metadata2
 
     def is_readable(self):
         return self._readable
@@ -114,6 +117,7 @@ class Store(BaseStore):
     .. added: 2.11.0
 
     """
+
     def listdir(self, path: str = "") -> List[str]:
         path = normalize_storage_path(path)
         return _listdir_from_keys(self, path)

zarr/attrs.py

@@ -1,6 +1,5 @@
 from collections.abc import MutableMapping
 
-from zarr.meta import parse_metadata
 from zarr._storage.store import Store
 from zarr.util import json_dumps
 
@@ -40,7 +39,7 @@ def _get_nosync(self):
         except KeyError:
             d = dict()
         else:
-            d = parse_metadata(data)
+            d = self.store._metadata_class.parse_metadata(data)
         return d
 
     def asdict(self):

zarr/core.py

@@ -31,7 +31,6 @@
     is_scalar,
     pop_fields,
 )
-from zarr.meta import decode_array_metadata, encode_array_metadata
 from zarr.storage import array_meta_key, attrs_key, getsize, listdir, BaseStore
 from zarr.util import (
     all_equal,
@@ -210,7 +209,7 @@ def _load_metadata_nosync(self):
         else:
 
             # decode and store metadata as instance members
-            meta = decode_array_metadata(meta_bytes)
+            meta = self._store._metadata_class.decode_array_metadata(meta_bytes)
             self._meta = meta
             self._shape = meta['shape']
             self._chunks = meta['chunks']
@@ -267,7 +266,7 @@ def _flush_metadata_nosync(self):
                     compressor=compressor_config, fill_value=self._fill_value,
                     order=self._order, filters=filters_config)
         mkey = self._key_prefix + array_meta_key
-        self._store[mkey] = encode_array_metadata(meta)
+        self._store[mkey] = self._store._metadata_class.encode_array_metadata(meta)
 
     @property
     def store(self):

zarr/hierarchy.py

@@ -14,7 +14,6 @@
     GroupNotFoundError,
     ReadOnlyError,
 )
-from zarr.meta import decode_group_metadata
 from zarr.storage import (
     BaseStore,
     MemoryStore,
@@ -134,8 +133,7 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
         except KeyError:
             raise GroupNotFoundError(path)
         else:
-            meta = decode_group_metadata(meta_bytes)
-            self._meta = meta
+            self._meta = self._store._metadata_class.decode_group_metadata(meta_bytes)
 
         # setup attributes
         akey = self._key_prefix + attrs_key