From 79e86e7cf0930d7ef7f3e6405112df2a2a3c13c1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Jul 2023 20:36:09 -0400 Subject: [PATCH 1/7] chore: refactor tests to inherit create_array from a base class --- zarr/storage.py | 2 +- zarr/tests/test_core.py | 846 +++++++++++----------------------------- zarr/util.py | 6 +- 3 files changed, 238 insertions(+), 616 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index ef1bd64955..c91f2f1cf0 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -300,7 +300,7 @@ def _require_parent_group( def init_array( store: StoreLike, - shape: Tuple[int, ...], + shape: Union[int, Tuple[int, ...]], chunks: Union[bool, int, Tuple[int, ...]] = True, dtype=None, compressor="default", diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index ab1a6e8aa7..a25eed0d24 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3,10 +3,10 @@ import sys import pickle import shutil +from typing import Any, Literal, Optional, Tuple, Union import unittest from itertools import zip_longest from tempfile import mkdtemp - import numpy as np import packaging.version import pytest @@ -19,6 +19,7 @@ import zarr from zarr._storage.store import ( + BaseStore, v3_api_available, ) from .._storage.v3_storage_transformers import ShardingStorageTransformer, v3_sharding_available @@ -42,6 +43,7 @@ init_array, init_group, meta_root, + normalize_store_arg ) from zarr._storage.v3 import ( ABSStoreV3, @@ -62,16 +64,59 @@ # noinspection PyMethodMayBeStatic -class TestArray(unittest.TestCase): - +class TestArray(): version = 2 root = '' - KVStoreClass = KVStore + path = '' + compressor = Zlib(level=1) + filters = None + dimension_separator: Literal["/", ".", None] = None + cache_metadata = True + cache_attrs = True + partial_decompress: bool = False + write_empty_chunks = True + read_only = False + storage_transformers: Tuple[Any, ...] = () + + def create_store(self) -> BaseStore: + return KVStore(dict()) + + # used by child classes + def create_chunk_store(self) -> Optional[BaseStore]: + return None + + def create_storage_transformers(self, shape) -> Tuple[Any, ...]: + return () + + def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): + store = self.create_store() + chunk_store = self.create_chunk_store() + # keyword arguments for array initialization + init_array_kwargs = { + "path": kwargs.pop("path", self.path), + "compressor": self.compressor, + "chunk_store": chunk_store, + "storage_transformers": self.create_storage_transformers(shape), + } + + # keyword arguments for array instantiation + access_array_kwargs = { + "path": init_array_kwargs["path"], + "read_only": kwargs.pop("read_only", self.read_only), + "chunk_store": chunk_store, + "cache_metadata": kwargs.pop("cache_metadata", self.cache_metadata), + "cache_attrs": kwargs.pop("cache_attrs", self.cache_attrs), + "partial_decompress": kwargs.pop("partial_decompress", self.partial_decompress), + "write_empty_chunks": kwargs.pop("write_empty_chunks", self.write_empty_chunks), + } + init_array(store, shape, **{**init_array_kwargs, **kwargs}) + + return Array(store, **access_array_kwargs) def test_array_init(self): # normal initialization - store = self.KVStoreClass(dict()) + store = self.create_store() init_array(store, shape=100, chunks=10, dtype=" BaseStore: path = mkdtemp() atexit.register(shutil.rmtree, path) - store = DirectoryStoreV3(path) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, - cache_metadata=cache_metadata, cache_attrs=cache_attrs, - write_empty_chunks=write_empty_chunks) + return DirectoryStoreV3(path) def test_nbytes_stored(self): # dict as store @@ -2960,87 +2764,52 @@ def test_nbytes_stored(self): @skip_test_env_var("ZARR_TEST_ABS") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithABSStoreV3(TestArrayWithABSStore, TestArrayWithPathV3): - - @staticmethod - def absstore(): +class TestArrayWithABSStoreV3(TestArrayV3): + def create_store(self) -> ABSStoreV3: client = abs_container() store = ABSStoreV3(client=client) store.rmdir() return store - def create_array(self, array_path='arr1', read_only=False, **kwargs): - store = self.absstore() - kwargs.setdefault('compressor', Zlib(1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) - - # TODO: TestArrayWithN5StoreV3 # class TestArrayWithN5StoreV3(TestArrayWithDirectoryStoreV3): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithDBMStoreV3(TestArrayWithDBMStore, TestArrayWithPathV3): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mktemp(suffix='.anydbm') - atexit.register(atexit_rmglob, path + '*') - store = DBMStoreV3(path, flag='n') - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_attrs=cache_attrs, - cache_metadata=cache_metadata, write_empty_chunks=write_empty_chunks) +class TestArrayWithDBMStoreV3(TestArrayV3): + def create_store(self) -> DBMStoreV3: + path = mktemp(suffix=".anydbm") + atexit.register(atexit_rmglob, path + "*") + store = DBMStoreV3(path, flag="n") + return store def test_nbytes_stored(self): pass # not implemented @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayWithDBMStoreBerkeleyDB, TestArrayWithPathV3): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayV3): + def create_store(self) -> DBMStoreV3: bsddb3 = pytest.importorskip("bsddb3") - path = mktemp(suffix='.dbm') + path = mktemp(suffix=".dbm") atexit.register(os.remove, path) - store = DBMStoreV3(path, flag='n', open=bsddb3.btopen) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + store = DBMStoreV3(path, flag="n", open=bsddb3.btopen) + return store def test_nbytes_stored(self): pass # not implemented @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithLMDBStoreV3(TestArrayWithLMDBStore, TestArrayWithPathV3): +class TestArrayWithLMDBStoreV3(TestArrayV3): + lmdb_buffers = True - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): + def create_store(self) -> LMDBStoreV3: pytest.importorskip("lmdb") - path = mktemp(suffix='.lmdb') + path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) - store = LMDBStoreV3(path, buffers=True) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + store = LMDBStoreV3(path, buffers=self.lmdb_buffers) + return store def test_store_has_bytes_values(self): pass # returns values as memoryviews/buffers instead of bytes @@ -3050,42 +2819,21 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithLMDBStoreV3NoBuffers(TestArrayWithLMDBStoreNoBuffers, TestArrayWithPathV3): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - pytest.importorskip("lmdb") - path = mktemp(suffix='.lmdb') - atexit.register(atexit_rmtree, path) - store = LMDBStoreV3(path, buffers=False) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) +class TestArrayWithLMDBStoreV3NoBuffers(TestArrayWithLMDBStoreV3): + lmdb_buffers = False def test_nbytes_stored(self): pass # not implemented @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithSQLiteStoreV3(TestArrayWithPathV3, TestArrayWithSQLiteStore): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithSQLiteStoreV3(TestArrayV3): + def create_store(self): pytest.importorskip("sqlite3") - path = mktemp(suffix='.db') + path = mktemp(suffix=".db") atexit.register(atexit_rmtree, path) store = SQLiteStoreV3(path) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_nbytes_stored(self): pass # not implemented @@ -3142,18 +2890,10 @@ def __contains__(self, item): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithCustomMappingV3(TestArrayWithPathV3, TestArrayWithCustomMapping): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithCustomMappingV3(TestArrayV3): + def create_store(self): store = CustomMappingV3() - kwargs.setdefault('compressor', Zlib(1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) @@ -3171,18 +2911,10 @@ def test_len(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayNoCacheV3(TestArrayWithPathV3, TestArrayNoCache): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayNoCacheV3(TestArrayWithPathV3): + def create_store(self): store = KVStoreV3(dict()) - kwargs.setdefault('compressor', Zlib(level=1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_object_arrays_danger(self): # skip this one as it only works if metadata are cached @@ -3190,18 +2922,10 @@ def test_object_arrays_danger(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithStoreCacheV3(TestArrayWithPathV3, TestArrayWithStoreCache): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithStoreCacheV3(TestArrayV3): + def create_store(self): store = LRUStoreCacheV3(dict(), max_size=None) - kwargs.setdefault('compressor', Zlib(level=1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_store_has_bytes_values(self): # skip as the cache has no control over how the store provides values @@ -3210,25 +2934,17 @@ def test_store_has_bytes_values(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3(TestArrayWithPathV3, TestArrayWithFSStore): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithFSStoreV3(TestArrayV3): + compressor = Blosc() + + def create_store(self): path = mkdtemp() atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', ".") + key_separator = self.dimension_separator store = FSStoreV3( - path, - key_separator=key_separator, - auto_mkdir=True, - **fsspec_mapper_kwargs + path, key_separator=key_separator, auto_mkdir=True, **fsspec_mapper_kwargs ) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def expected(self): return [ @@ -3242,22 +2958,16 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3FromFilesystem(TestArrayWithPathV3, TestArrayWithFSStore): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithFSStoreV3FromFilesystem(TestArrayWithFSStoreV3): + def create_store(self): from fsspec.implementations.local import LocalFileSystem + fs = LocalFileSystem(auto_mkdir=True) path = mkdtemp() atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', ".") + key_separator = self.dimension_separator store = FSStoreV3(path, fs=fs, key_separator=key_separator, **fsspec_mapper_kwargs) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def expected(self): return [ @@ -3271,27 +2981,8 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3PartialRead(TestArrayWithPathV3, TestArrayWithFSStorePartialRead): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - store = FSStoreV3(path) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault("compressor", Blosc()) - init_array(store, path=array_path, **kwargs) - return Array( - store, - path=array_path, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks, - ) +class TestArrayWithFSStoreV3PartialRead(TestArrayWithFSStoreV3): + partial_decompress = True def expected(self): return [ @@ -3306,33 +2997,16 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @pytest.mark.skipif(not v3_sharding_available, reason="sharding is disabled") -class TestArrayWithFSStoreV3PartialReadUncompressedSharded( - TestArrayWithPathV3, TestArrayWithFSStorePartialRead -): +class TestArrayWithFSStoreV3PartialReadUncompressedSharded(TestArrayWithFSStoreV3): + partial_decompress = True + compressor = None - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - store = FSStoreV3(path) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', None) - num_dims = 1 if isinstance(kwargs["shape"], int) else len(kwargs["shape"]) + def create_storage_transformers(self, shape) -> Tuple[Any]: + num_dims = 1 if isinstance(shape, int) else len(shape) sharding_transformer = ShardingStorageTransformer( "indexed", chunks_per_shard=(2, ) * num_dims ) - init_array(store, path=array_path, storage_transformers=[sharding_transformer], **kwargs) - return Array( - store, - path=array_path, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks, - ) + return (sharding_transformer,) def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) @@ -3359,21 +3033,8 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3Nested(TestArrayWithPathV3, TestArrayWithFSStoreNested): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', "/") - store = FSStoreV3(path, key_separator=key_separator, auto_mkdir=True) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) +class TestArrayWithFSStoreV3Nested(TestArrayWithFSStoreV3): + dimension_separator = "/" def expected(self): return [ @@ -3387,28 +3048,8 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3NestedPartialRead(TestArrayWithPathV3, - TestArrayWithFSStoreNestedPartialRead): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', "/") - store = FSStoreV3(path, key_separator=key_separator, auto_mkdir=True) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault("compressor", Blosc()) - init_array(store, path=array_path, **kwargs) - return Array( - store, - path=array_path, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks, - ) +class TestArrayWithFSStoreV3NestedPartialRead(TestArrayWithFSStoreV3): + dimension_separator = "/" def expected(self): return [ @@ -3423,22 +3064,10 @@ def expected(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") class TestArrayWithStorageTransformersV3(TestArrayWithChunkStoreV3): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - store = KVStoreV3(dict()) - # separate chunk store - chunk_store = KVStoreV3(dict()) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - dummy_storage_transformer = DummyStorageTransfomer( - "dummy_type", test_value=DummyStorageTransfomer.TEST_CONSTANT + def create_storage_transformers(self, shape) -> Tuple[Any]: + return ( + DummyStorageTransfomer("dummy_type", test_value=DummyStorageTransfomer.TEST_CONSTANT), ) - init_array(store, path=array_path, chunk_store=chunk_store, - storage_transformers=[dummy_storage_transformer], **kwargs) - return Array(store, path=array_path, read_only=read_only, - chunk_store=chunk_store, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) def expected(self): return [ @@ -3452,23 +3081,14 @@ def expected(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @pytest.mark.skipif(not v3_sharding_available, reason="sharding is disabled") -class TestArrayWithShardingStorageTransformerV3(TestArrayWithPathV3): +class TestArrayWithShardingStorageTransformerV3(TestArrayV3): + compressor = None - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - store = KVStoreV3(dict()) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', None) - num_dims = 1 if isinstance(kwargs["shape"], int) else len(kwargs["shape"]) - sharding_transformer = ShardingStorageTransformer( - "indexed", chunks_per_shard=(2, ) * num_dims + def create_storage_transformers(self, shape) -> Tuple[Any]: + num_dims = (1 if isinstance(shape, int) else len(shape)) + return ( + ShardingStorageTransformer("indexed", chunks_per_shard=(2, ) * num_dims), ) - init_array(store, path=array_path, storage_transformers=[sharding_transformer], **kwargs) - return Array(store, path=array_path, read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) diff --git a/zarr/util.py b/zarr/util.py index 6ba20b96c2..d23d87e402 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -15,7 +15,8 @@ Tuple, TypeVar, Union, - Iterable + Iterable, + cast ) import numpy as np @@ -74,7 +75,7 @@ def json_loads(s: Union[bytes, str]) -> Dict[str, Any]: return json.loads(ensure_text(s, 'utf-8')) -def normalize_shape(shape) -> Tuple[int]: +def normalize_shape(shape: Union[int, Tuple[int, ...], None]) -> Tuple[int, ...]: """Convenience function to normalize the `shape` argument.""" if shape is None: @@ -85,6 +86,7 @@ def normalize_shape(shape) -> Tuple[int]: shape = (int(shape),) # normalize + shape = cast(Tuple[int], shape) shape = tuple(int(s) for s in shape) return shape From f7e14614839b81d0dff724f2f94b694027222ef1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Jul 2023 20:48:44 -0400 Subject: [PATCH 2/7] chore: widen type of shape to variable length tuple --- zarr/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/util.py b/zarr/util.py index d23d87e402..efbb86e4c0 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -86,7 +86,7 @@ def normalize_shape(shape: Union[int, Tuple[int, ...], None]) -> Tuple[int, ...] shape = (int(shape),) # normalize - shape = cast(Tuple[int], shape) + shape = cast(Tuple[int, ...], shape) shape = tuple(int(s) for s in shape) return shape From ff2e5ea0dd7b5a347442da30a715c42d47f86696 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Jul 2023 20:53:44 -0400 Subject: [PATCH 3/7] chore: add importskip barrier for lmdb --- zarr/tests/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index a25eed0d24..85a26a382f 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2060,6 +2060,7 @@ def test_nbytes_stored(self): class TestArrayWithLMDBStore(TestArray): def create_store(self): + pytest.importorskip("lmdb") path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) store = LMDBStore(path, buffers=True) From e449f3c842bc1c9034ac85012926a46b9bfecc72 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Jul 2023 21:41:50 -0400 Subject: [PATCH 4/7] docs: release notes --- docs/release.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 46bd1f025d..269305c2a4 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -30,6 +30,12 @@ Enhancements * **Block Indexing**: Implemented blockwise (chunk blocks) indexing to ``zarr.Array``. By :user:`Altay Sansal ` :issue:`1428` +Maintenance +~~~~~~~~~~~ + +* Refactor the core array tests to reduce code duplication. + By :user:`Davis Bennett ` :issue:`1462`. + .. _release_2.15.0: 2.15.0 From 80e6079e15931a89e78ff02427738fd2cd9f771c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Jul 2023 22:15:41 -0400 Subject: [PATCH 5/7] chore: remove fsstore kwargs variable --- zarr/tests/test_core.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 85a26a382f..a5b35a78d0 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2401,13 +2401,6 @@ def test_store_has_bytes_values(self): pass -fsspec_mapper_kwargs = { - "check": True, - "create": True, - "missing_exceptions": None -} - - @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestArrayWithFSStore(TestArray): compressor = Blosc() @@ -2417,7 +2410,12 @@ def create_store(self): path = mkdtemp() atexit.register(shutil.rmtree, path) key_separator = self.dimension_separator - store = FSStore(path, key_separator=key_separator, auto_mkdir=True, **fsspec_mapper_kwargs) + store = FSStore(path, + key_separator=key_separator, + auto_mkdir=True, + check=True, + create=True, + missing_exceptions=True) return store def expected(self): @@ -2442,7 +2440,12 @@ def create_store(self): path = mkdtemp() atexit.register(shutil.rmtree, path) key_separator = self.dimension_separator - store = FSStore(path, fs=fs, key_separator=key_separator, **fsspec_mapper_kwargs) + store = FSStore(path, + fs=fs, + key_separator=key_separator, + check=True, + create=True, + missing_exceptions=None) return store def expected(self): @@ -2943,7 +2946,12 @@ def create_store(self): atexit.register(shutil.rmtree, path) key_separator = self.dimension_separator store = FSStoreV3( - path, key_separator=key_separator, auto_mkdir=True, **fsspec_mapper_kwargs + path, + key_separator=key_separator, + auto_mkdir=True, + create=True, + check=True, + missing_exceptions=None ) return store @@ -2967,7 +2975,12 @@ def create_store(self): path = mkdtemp() atexit.register(shutil.rmtree, path) key_separator = self.dimension_separator - store = FSStoreV3(path, fs=fs, key_separator=key_separator, **fsspec_mapper_kwargs) + store = FSStoreV3(path, + fs=fs, + key_separator=key_separator, + create=True, + check=True, + missing_exceptions=None) return store def expected(self): From a1cb970882fd61090857e175eed9725f70d6dedd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 13 Jul 2023 22:37:29 -0400 Subject: [PATCH 6/7] chore: fix type error in creation of fsstore --- zarr/tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index a5b35a78d0..3deabcdbae 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2415,7 +2415,7 @@ def create_store(self): auto_mkdir=True, check=True, create=True, - missing_exceptions=True) + missing_exceptions=None) return store def expected(self): From 9d5b41ffc6cfe0739becdaa373634779837d2938 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 14 Jul 2023 10:23:10 -0400 Subject: [PATCH 7/7] chore: add `create_filters` method to TestArray. Pop out `compressor` kwarg in create_array. --- zarr/tests/test_core.py | 37 ++++++++++--------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 3deabcdbae..8bf8789f56 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -85,7 +85,10 @@ def create_store(self) -> BaseStore: def create_chunk_store(self) -> Optional[BaseStore]: return None - def create_storage_transformers(self, shape) -> Tuple[Any, ...]: + def create_storage_transformers(self, shape: Union[int, Tuple[int, ...]]) -> Tuple[Any, ...]: + return () + + def create_filters(self, dtype: Optional[str]) -> Tuple[Any, ...]: return () def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): @@ -94,9 +97,10 @@ def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): # keyword arguments for array initialization init_array_kwargs = { "path": kwargs.pop("path", self.path), - "compressor": self.compressor, + "compressor": kwargs.pop("compressor", self.compressor), "chunk_store": chunk_store, "storage_transformers": self.create_storage_transformers(shape), + "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype", None))) } # keyword arguments for array instantiation @@ -109,6 +113,7 @@ def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): "partial_decompress": kwargs.pop("partial_decompress", self.partial_decompress), "write_empty_chunks": kwargs.pop("write_empty_chunks", self.write_empty_chunks), } + init_array(store, shape, **{**init_array_kwargs, **kwargs}) return Array(store, **access_array_kwargs) @@ -2164,33 +2169,11 @@ class TestArrayWithFilters(TestArray): compressor = Zlib(1) - def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): - - store = self.create_store() - chunk_store = self.create_chunk_store() - dtype = kwargs.get("dtype", None) - filters = [ + def create_filters(self, dtype: Optional[str]) -> Tuple[Any, ...]: + return ( Delta(dtype=dtype), FixedScaleOffset(dtype=dtype, scale=1, offset=0), - ] - init_array_kwargs = { - "path": kwargs.pop("path", self.path), - "compressor": self.compressor, - "chunk_store": chunk_store, - "filters": filters, - } - - access_array_kwargs = { - "path": init_array_kwargs["path"], - "read_only": kwargs.pop("read_only", self.read_only), - "chunk_store": chunk_store, - "cache_metadata": kwargs.pop("cache_metadata", self.cache_metadata), - "cache_attrs": kwargs.pop("cache_attrs", self.cache_attrs), - "write_empty_chunks": kwargs.pop("write_empty_chunks", self.write_empty_chunks), - } - init_array(store, shape, **{**init_array_kwargs, **kwargs}) - - return Array(store, **access_array_kwargs) + ) def expected(self): return [