Merge pull request #650 from martindurant/small_storage_fixes

Fixes #649

Author: martindurant

zarr/convenience.py

@@ -73,7 +73,10 @@ def open(store=None, mode='a', **kwargs):
     path = kwargs.get('path', None)
     # handle polymorphic store arg
     clobber = mode == 'w'
-    store = normalize_store_arg(store, clobber=clobber)
+    # we pass storage options explicitly, since normalize_store_arg might construct
+    # a store if the input is a fsspec-compatible URL
+    store = normalize_store_arg(store, clobber=clobber,
+                                storage_options=kwargs.pop("storage_options", {}))
     path = normalize_storage_path(path)
 
     if mode in {'w', 'w-', 'x'}:

zarr/core.py

@@ -1688,7 +1688,7 @@ def _chunk_getitems(self, lchunk_coords, lchunk_selection, out, lout_selection,
             out_is_ndarray = False
 
         ckeys = [self._chunk_key(ch) for ch in lchunk_coords]
-        cdatas = self.chunk_store.getitems(ckeys)
+        cdatas = self.chunk_store.getitems(ckeys, on_error="omit")
         for ckey, chunk_select, out_select in zip(ckeys, lchunk_selection, lout_selection):
             if ckey in cdatas:
                 self._process_chunk(out, cdatas[ckey], chunk_select, drop_axes,

zarr/storage.py

@@ -1038,7 +1038,7 @@ def _normalize_key(self, key):
             key = '/'.join(bits + [end.replace('.', self.key_separator)])
         return key.lower() if self.normalize_keys else key
 
-    def getitems(self, keys):
+    def getitems(self, keys, **kwargs):
         keys = [self._normalize_key(key) for key in keys]
         return self.map.getitems(keys, on_error="omit")
 

zarr/tests/test_storage.py

@@ -955,6 +955,11 @@ def test_s3(self):
 
         assert g.data[:].tolist() == [0, 1, 2, 3, 0, 0, 0, 0]
 
+        # test via convenience
+        g = zarr.open("s3://test/out.zarr", mode='r',
+                      storage_options=self.s3so)
+        assert g.data[:].tolist() == [0, 1, 2, 3, 0, 0, 0, 0]
+
     @pytest.mark.usefixtures("s3")
     def test_s3_complex(self):
         import zarr