Remove consolidated functionality, add tests

Author: Martin Durant

zarr/creation.py

@@ -357,7 +357,8 @@ def array(data, **kwargs):
 def open_array(store=None, mode='a', shape=None, chunks=True, dtype=None,
                compressor='default', fill_value=0, order='C', synchronizer=None,
                filters=None, cache_metadata=True, cache_attrs=True, path=None,
-               object_codec=None, chunk_store=None, **kwargs):
+               object_codec=None, chunk_store=None, storage_options=None,
+               **kwargs):
     """Open an array using file-mode-like semantics.
 
     Parameters
@@ -403,6 +404,9 @@ def open_array(store=None, mode='a', shape=None, chunks=True, dtype=None,
         A codec to encode object arrays, only needed if dtype=object.
     chunk_store : MutableMapping or string, optional
         Store or path to directory in file system or name of zip file.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
 
     Returns
     -------
@@ -439,7 +443,6 @@ def open_array(store=None, mode='a', shape=None, chunks=True, dtype=None,
 
     # handle polymorphic store arg
     clobber = mode == 'w'
-    storage_options = kwargs.pop("storage_options", None)
     store = normalize_store_arg(store, clobber=clobber, storage_options=storage_options)
     if chunk_store is not None:
         chunk_store = normalize_store_arg(chunk_store, clobber=clobber,

zarr/hierarchy.py

@@ -1118,6 +1118,9 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
         Group path within store.
     chunk_store : MutableMapping or string, optional
         Store or path to directory in file system or name of zip file.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
 
     Returns
     -------
@@ -1140,9 +1143,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
     """
 
     # handle polymorphic store arg
-    store = _normalize_store_arg(store, storage_options=storage_options)
+    clobber = mode != 'r'
+    store = _normalize_store_arg(store, clobber=clobber, storage_options=storage_options)
     if chunk_store is not None:
-        chunk_store = _normalize_store_arg(chunk_store, storage_options=storage_options)
+        chunk_store = _normalize_store_arg(chunk_store, clobber=clobber, storage_options=storage_options)
     path = normalize_storage_path(path)
 
     # ensure store is initialized

zarr/storage.py

@@ -948,9 +948,30 @@ def atexit_rmglob(path,
 
 
 class FSStore(MutableMapping):
+    """Wraps an fsspec.FSMap to give access to arbitrary filesystems
+
+    Requires that ``fsspec`` is installed, as well as any additional
+    requirements for the protocol chosen.
+
+    Parameters
+    ----------
+    url : str
+        The destination to map. Should include protocol and path,
+        like "s3://bucket/root"
+    normalize_keys : bool
+    key_separator : str
+        Character to use when constructing the target path strings
+        for data keys
+    mode : str
+        "w" for writable, "r" for read-only
+    exceptions : list of Exception subclasses
+        When accessing data, any of these exceptions will be treated
+        as a missing key
+    storage_options : passed to the fsspec implementation
+    """
 
     def __init__(self, url, normalize_keys=True, key_separator='.',
-                 mode='w', consolidated=False, metadata_key='.zmetadata',
+                 mode='w',
                  exceptions=(KeyError, PermissionError, IOError),
                  **storage_options):
         import fsspec
@@ -961,24 +982,8 @@ def __init__(self, url, normalize_keys=True, key_separator='.',
         self.fs = self.map.fs  # for direct operations
         self.mode = mode
         self.exceptions = exceptions
-        # TODO: should warn if consolidated and write mode?
         if self.fs.exists(url) and not self.fs.isdir(url):
             err_fspath_exists_notdir(url)
-        self.consolidated = consolidated
-        self.metadata_key = metadata_key
-        if consolidated:
-            self.meta = json.loads(self.map.get(metadata_key, b"{}").decode())
-            if mode == 'r' or 'zarr_consolidated_format' in self.meta:
-                consolidated_format = self.meta.get('zarr_consolidated_format', None)
-                if consolidated_format != 1:  # pragma: no cover
-                    raise MetadataError('unsupported zarr consolidated metadata format: %s' %
-                                        consolidated_format)
-            else:
-                self.meta['zarr_consolidated_format'] = 1
-
-    @staticmethod
-    def _is_meta(key):
-        return key.split('/')[-1] in [attrs_key, group_meta_key, array_meta_key]
 
     def _normalize_key(self, key):
         key = normalize_storage_path(key).lstrip('/')
@@ -988,8 +993,6 @@ def _normalize_key(self, key):
         return key.lower() if self.normalize_keys else key
 
     def __getitem__(self, key):
-        if self.consolidated and self._is_meta(key):
-            return self.meta[key].encode()  # expect bytes out
         key = self._normalize_key(key)
         try:
             return self.map[key]
@@ -999,9 +1002,6 @@ def __getitem__(self, key):
     def __setitem__(self, key, value):
         if self.mode == 'r':
             err_read_only()
-        if self.consolidated and self._is_meta(key):
-            self.meta[key] = value.decode()
-            self.map[self.metadata_key] = json.dumps(self.meta).encode()
         key = self._normalize_key(key)
         path = self.dir_path(key)
         value = ensure_contiguous_ndarray(value)
@@ -1015,9 +1015,6 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         if self.mode == 'r':
             err_read_only()
-        if self.consolidated and self._is_meta(key):
-            del self.meta[key]
-            self.map[self.metadata_key] = json.dumps(self.meta).encode()
         key = self._normalize_key(key)
         path = self.dir_path(key)
         if self.fs.isdir(path):
@@ -1026,8 +1023,6 @@ def __delitem__(self, key):
             del self.map[key]
 
     def __contains__(self, key):
-        if self.consolidated and self._is_meta(key):
-            return key in self.meta
         key = self._normalize_key(key)
         return key in self.map
 
@@ -1071,8 +1066,6 @@ def getsize(self, path=None):
     def clear(self):
         if self.mode == 'r':
             err_read_only()
-        if self.consolidated:
-            self.meta = {}
         self.map.clear()
 
 

zarr/tests/test_storage.py

@@ -852,35 +852,6 @@ def test_complex(self):
         assert store["foo"] == b"hello"
         assert 'foo' in os.listdir(path2)
 
-    def test_consolidated(self):
-        path = tempfile.mkdtemp()
-        store = FSStore(path, consolidated=True)
-        store[".zarray"] = b"{}"
-        assert ".zmetadata" in os.listdir(path)
-        del store[".zarray"]
-        with pytest.raises(KeyError):
-            store[".zarray"]
-        store[".zarray"] = b"{}"
-
-        os.remove(os.path.join(path, ".zarray"))
-        store2 = FSStore(path, mode='r', consolidated=True)
-        assert store != store2
-        assert ".zarray" in store2
-        assert store2[".zarray"] == b"{}"
-        with pytest.raises(PermissionError):
-            store2[".zarray"] = b"{{}}"
-        with pytest.raises(PermissionError):
-            del store2[".zarray"]
-        with pytest.raises(PermissionError):
-            store2.clear()
-        with pytest.raises(PermissionError):
-            store2.rmdir("any")
-        with pytest.raises(ValueError):
-            FSStore(os.path.join(path, ".zmetadata"))
-
-        store.clear()
-        assert ".zmetadata" not in store
-
     def test_not_fsspec(self):
         import zarr
         path = tempfile.mkdtemp()
@@ -890,6 +861,24 @@ def test_not_fsspec(self):
             zarr.open_group(path, mode='w', storage_options={"some": "kwargs"})
         zarr.open_array("file://" + path, mode='w', shape=(1,), dtype="f8")
 
+    def test_create(self):
+        import zarr
+        path1 = tempfile.mkdtemp()
+        path2 = tempfile.mkdtemp()
+        g = zarr.open_group("file://" + path1, mode='w',
+                            storage_options={"auto_mkdir": True})
+        a = g.create_dataset("data", shape=(8,))
+        a[:4] = [0, 1, 2, 3]
+        assert "data" in os.listdir(path1)
+        assert ".zgroup" in os.listdir(path1)
+
+        g = zarr.open_group("simplecache::file://" + path1, mode='r',
+                            storage_options={"cache_storage": path2,
+                                             "same_names": True})
+        assert g.data[:].tolist() == [0, 1, 2, 3, 0, 0, 0, 0]
+        with pytest.raises(PermissionError):
+            g.data[:] = 1
+
 
 class TestNestedDirectoryStore(TestDirectoryStore, unittest.TestCase):