Merge pull request #645 from andrewfulton9/fsspec_indexer_error

Author: Carreau

docs/release.rst

@@ -10,6 +10,9 @@ This release will be the first release of Zarr not supporting Python 3.5.
 * End Python 3.5 support.
   By :user:`Chris Barnes <clbarnes>`; :issue:`602`.
 
+* Add `Array` tests for FSStore.
+  By :user:`Andrew Fulton <andrewfulton9>`; :issue: `644`.
+
 2.5.0
 -----
 

requirements_dev_optional.txt

@@ -19,6 +19,6 @@ pytest-doctestplus==0.4.0
 pytest-remotedata==0.3.2
 h5py==2.10.0
 s3fs==0.5.1; python_version > '3.6'
-fsspec==0.8.3; python_version > '3.6'
+fsspec==0.8.4; python_version > '3.6'
 moto>=1.3.14; python_version > '3.6'
 flask

zarr/core.py

@@ -1020,7 +1020,8 @@ def _get_selection(self, indexer, out=None, fields=None):
             check_array_shape('out', out, out_shape)
 
         # iterate over chunks
-        if not hasattr(self.chunk_store, "getitems"):
+        if not hasattr(self.chunk_store, "getitems") or \
+           any(map(lambda x: x == 0, self.shape)):
             # sequentially get one key at a time from storage
             for chunk_coords, chunk_selection, out_selection in indexer:
 
@@ -1535,7 +1536,8 @@ def _set_selection(self, indexer, value, fields=None):
             check_array_shape('value', value, sel_shape)
 
         # iterate over chunks in range
-        if not hasattr(self.store, "setitems") or self._synchronizer is not None:
+        if not hasattr(self.store, "setitems") or self._synchronizer is not None \
+           or any(map(lambda x: x == 0, self.shape)):
             # iterative approach
             for chunk_coords, chunk_selection, out_selection in indexer:
 

zarr/storage.py

@@ -1113,7 +1113,9 @@ def rmdir(self, path=None):
             raise ReadOnlyError()
         store_path = self.dir_path(path)
         if self.fs.isdir(store_path):
-            self.fs.rm(store_path, recursive=True)
+            # sometimes doesn't delete on first try with S3
+            while self.fs.exists(store_path):
+                self.fs.rm(store_path, recursive=True)
 
     def getsize(self, path=None):
         store_path = self.dir_path(path)

zarr/tests/test_core.py

@@ -20,9 +20,11 @@
 from zarr.n5 import N5Store, n5_keywords
 from zarr.storage import (ABSStore, DBMStore, DirectoryStore, LMDBStore,
                           LRUStoreCache, NestedDirectoryStore, SQLiteStore,
-                          atexit_rmglob, atexit_rmtree, init_array, init_group)
+                          FSStore, atexit_rmglob, atexit_rmtree, init_array,
+                          init_group
+                          )
 from zarr.util import buffer_size
-from zarr.tests.util import skip_test_env_var
+from zarr.tests.util import skip_test_env_var, have_fsspec
 
 
 # noinspection PyMethodMayBeStatic
@@ -2366,3 +2368,42 @@ def create_array(read_only=False, **kwargs):
     def test_store_has_bytes_values(self):
         # skip as the cache has no control over how the store provides values
         pass
+
+
+@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
+class TestArrayWithFSStore(TestArray):
+
+    @staticmethod
+    def create_array(read_only=False, **kwargs):
+        path = mkdtemp()
+        atexit.register(shutil.rmtree, path)
+        store = FSStore(path)
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
+        kwargs.setdefault('compressor', Blosc())
+        init_array(store, **kwargs)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
+
+    def test_hexdigest(self):
+        # Check basic 1-D array
+        z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
+        assert 'f710da18d45d38d4aaf2afd7fb822fdd73d02957' == z.hexdigest()
+
+        # Check basic 1-D array with different type
+        z = self.create_array(shape=(1050,), chunks=100, dtype='<f4')
+        assert '1437428e69754b1e1a38bd7fc9e43669577620db' == z.hexdigest()
+
+        # Check basic 2-D array
+        z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
+        assert '6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe' == z.hexdigest()
+
+        # Check basic 1-D array with some data
+        z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
+        z[200:400] = np.arange(200, 400, dtype='i4')
+        assert '4c0a76fb1222498e09dcd92f7f9221d6cea8b40e' == z.hexdigest()
+
+        # Check basic 1-D array with attributes
+        z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
+        z.attrs['foo'] = 'bar'
+        assert '05b0663ffe1785f38d3a459dec17e57a18f254af' == z.hexdigest()