diff --git a/docs/release.rst b/docs/release.rst index 46bd1f025d..269305c2a4 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -30,6 +30,12 @@ Enhancements * **Block Indexing**: Implemented blockwise (chunk blocks) indexing to ``zarr.Array``. By :user:`Altay Sansal ` :issue:`1428` +Maintenance +~~~~~~~~~~~ + +* Refactor the core array tests to reduce code duplication. + By :user:`Davis Bennett ` :issue:`1462`. + .. _release_2.15.0: 2.15.0 diff --git a/zarr/storage.py b/zarr/storage.py index ef1bd64955..c91f2f1cf0 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -300,7 +300,7 @@ def _require_parent_group( def init_array( store: StoreLike, - shape: Tuple[int, ...], + shape: Union[int, Tuple[int, ...]], chunks: Union[bool, int, Tuple[int, ...]] = True, dtype=None, compressor="default", diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index ab1a6e8aa7..8bf8789f56 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3,10 +3,10 @@ import sys import pickle import shutil +from typing import Any, Literal, Optional, Tuple, Union import unittest from itertools import zip_longest from tempfile import mkdtemp - import numpy as np import packaging.version import pytest @@ -19,6 +19,7 @@ import zarr from zarr._storage.store import ( + BaseStore, v3_api_available, ) from .._storage.v3_storage_transformers import ShardingStorageTransformer, v3_sharding_available @@ -42,6 +43,7 @@ init_array, init_group, meta_root, + normalize_store_arg ) from zarr._storage.v3 import ( ABSStoreV3, @@ -62,16 +64,64 @@ # noinspection PyMethodMayBeStatic -class TestArray(unittest.TestCase): - +class TestArray(): version = 2 root = '' - KVStoreClass = KVStore + path = '' + compressor = Zlib(level=1) + filters = None + dimension_separator: Literal["/", ".", None] = None + cache_metadata = True + cache_attrs = True + partial_decompress: bool = False + write_empty_chunks = True + read_only = False + storage_transformers: Tuple[Any, ...] = () + + def create_store(self) -> BaseStore: + return KVStore(dict()) + + # used by child classes + def create_chunk_store(self) -> Optional[BaseStore]: + return None + + def create_storage_transformers(self, shape: Union[int, Tuple[int, ...]]) -> Tuple[Any, ...]: + return () + + def create_filters(self, dtype: Optional[str]) -> Tuple[Any, ...]: + return () + + def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): + store = self.create_store() + chunk_store = self.create_chunk_store() + # keyword arguments for array initialization + init_array_kwargs = { + "path": kwargs.pop("path", self.path), + "compressor": kwargs.pop("compressor", self.compressor), + "chunk_store": chunk_store, + "storage_transformers": self.create_storage_transformers(shape), + "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype", None))) + } + + # keyword arguments for array instantiation + access_array_kwargs = { + "path": init_array_kwargs["path"], + "read_only": kwargs.pop("read_only", self.read_only), + "chunk_store": chunk_store, + "cache_metadata": kwargs.pop("cache_metadata", self.cache_metadata), + "cache_attrs": kwargs.pop("cache_attrs", self.cache_attrs), + "partial_decompress": kwargs.pop("partial_decompress", self.partial_decompress), + "write_empty_chunks": kwargs.pop("write_empty_chunks", self.write_empty_chunks), + } + + init_array(store, shape, **{**init_array_kwargs, **kwargs}) + + return Array(store, **access_array_kwargs) def test_array_init(self): # normal initialization - store = self.KVStoreClass(dict()) + store = self.create_store() init_array(store, shape=100, chunks=10, dtype=" Tuple[Any, ...]: + return ( Delta(dtype=dtype), FixedScaleOffset(dtype=dtype, scale=1, offset=0), - ] - kwargs.setdefault('filters', filters) - compressor = Zlib(1) - kwargs.setdefault('compressor', compressor) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_attrs=cache_attrs, - cache_metadata=cache_metadata, write_empty_chunks=write_empty_chunks) + ) def expected(self): return [ @@ -2407,19 +2296,14 @@ def __delitem__(self, key): def __contains__(self, item): return item in self.inner + def close(self): + return self.inner.close() + class TestArrayWithCustomMapping(TestArray): - @staticmethod - def create_array(read_only=False, **kwargs): - store = CustomMapping() - kwargs.setdefault('compressor', Zlib(1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + def create_store(self): + return CustomMapping() def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) @@ -2429,18 +2313,6 @@ def test_nbytes_stored(self): class TestArrayNoCache(TestArray): - - @staticmethod - def create_array(read_only=False, **kwargs): - store = KVStore(dict()) - kwargs.setdefault('compressor', Zlib(level=1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) - def test_cache_metadata(self): a1 = self.create_array(shape=100, chunks=10, dtype='i1', cache_metadata=False) path = None if self.version == 2 else a1.path @@ -2504,45 +2376,30 @@ def test_object_arrays_danger(self): class TestArrayWithStoreCache(TestArray): - - @staticmethod - def create_array(read_only=False, **kwargs): - store = LRUStoreCache(dict(), max_size=None) - kwargs.setdefault('compressor', Zlib(level=1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + def create_store(self): + return LRUStoreCache(dict(), max_size=None) def test_store_has_bytes_values(self): # skip as the cache has no control over how the store provides values pass -fsspec_mapper_kwargs = { - "check": True, - "create": True, - "missing_exceptions": None -} - - @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestArrayWithFSStore(TestArray): - @staticmethod - def create_array(read_only=False, **kwargs): + compressor = Blosc() + dimension_separator: Literal[".", "/"] = "." + + def create_store(self): path = mkdtemp() atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', ".") - store = FSStore(path, key_separator=key_separator, auto_mkdir=True, **fsspec_mapper_kwargs) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + key_separator = self.dimension_separator + store = FSStore(path, + key_separator=key_separator, + auto_mkdir=True, + check=True, + create=True, + missing_exceptions=None) + return store def expected(self): return [ @@ -2556,21 +2413,23 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestArrayWithFSStoreFromFilesystem(TestArray): - @staticmethod - def create_array(read_only=False, **kwargs): + compressor = Blosc() + dimension_separator = "." + + def create_store(self): from fsspec.implementations.local import LocalFileSystem + fs = LocalFileSystem(auto_mkdir=True) path = mkdtemp() atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', ".") - store = FSStore(path, fs=fs, key_separator=key_separator, **fsspec_mapper_kwargs) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + key_separator = self.dimension_separator + store = FSStore(path, + fs=fs, + key_separator=key_separator, + check=True, + create=True, + missing_exceptions=None) + return store def expected(self): return [ @@ -2584,24 +2443,14 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestArrayWithFSStorePartialRead(TestArray): - @staticmethod - def create_array(read_only=False, **kwargs): + compressor = Blosc(blocksize=256) + partial_decompress = True + + def create_store(self): path = mkdtemp() atexit.register(shutil.rmtree, path) store = FSStore(path) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault("compressor", Blosc(blocksize=256)) - init_array(store, **kwargs) - return Array( - store, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks - ) + return store def expected(self): return [ @@ -2640,21 +2489,9 @@ def test_read_from_all_blocks(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") -class TestArrayWithFSStoreNested(TestArray): - - @staticmethod - def create_array(read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', "/") - store = FSStore(path, key_separator=key_separator, auto_mkdir=True) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) +class TestArrayWithFSStoreNested(TestArrayWithFSStore): + compressor = Blosc() + dimension_separator = "/" def expected(self): return [ @@ -2667,26 +2504,10 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") -class TestArrayWithFSStoreNestedPartialRead(TestArray): - @staticmethod - def create_array(read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', "/") - store = FSStore(path, key_separator=key_separator, auto_mkdir=True) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault("compressor", Blosc()) - init_array(store, **kwargs) - return Array( - store, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks - ) +class TestArrayWithFSStoreNestedPartialRead(TestArrayWithFSStore): + compressor = Blosc() + dimension_separator = "/" + partial_decompress = True def expected(self): return [ @@ -2730,10 +2551,12 @@ def test_read_from_all_blocks(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") class TestArrayV3(TestArray): - version = 3 root = meta_root - KVStoreClass = KVStoreV3 + path = "arr1" + + def create_store(self): + return KVStoreV3(dict()) def expected(self): # tests for array without path will not be run for v3 stores @@ -2750,25 +2573,11 @@ def expected(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithPathV3(TestArrayWithPath): - - version = 3 - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - store = KVStoreV3(dict()) - kwargs.setdefault('compressor', Zlib(level=1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, - cache_metadata=cache_metadata, cache_attrs=cache_attrs, - write_empty_chunks=write_empty_chunks) +class TestArrayWithPathV3(TestArrayV3): def test_array_init(self): - store = KVStoreV3(dict()) + store = self.create_store() # can initialize an array without a path init_array(store, shape=100, chunks=10, dtype=" BaseStore: path = mkdtemp() atexit.register(shutil.rmtree, path) - store = DirectoryStoreV3(path) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, - cache_metadata=cache_metadata, cache_attrs=cache_attrs, - write_empty_chunks=write_empty_chunks) + return DirectoryStoreV3(path) def test_nbytes_stored(self): # dict as store @@ -2960,87 +2751,52 @@ def test_nbytes_stored(self): @skip_test_env_var("ZARR_TEST_ABS") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithABSStoreV3(TestArrayWithABSStore, TestArrayWithPathV3): - - @staticmethod - def absstore(): +class TestArrayWithABSStoreV3(TestArrayV3): + def create_store(self) -> ABSStoreV3: client = abs_container() store = ABSStoreV3(client=client) store.rmdir() return store - def create_array(self, array_path='arr1', read_only=False, **kwargs): - store = self.absstore() - kwargs.setdefault('compressor', Zlib(1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) - - # TODO: TestArrayWithN5StoreV3 # class TestArrayWithN5StoreV3(TestArrayWithDirectoryStoreV3): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithDBMStoreV3(TestArrayWithDBMStore, TestArrayWithPathV3): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mktemp(suffix='.anydbm') - atexit.register(atexit_rmglob, path + '*') - store = DBMStoreV3(path, flag='n') - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_attrs=cache_attrs, - cache_metadata=cache_metadata, write_empty_chunks=write_empty_chunks) +class TestArrayWithDBMStoreV3(TestArrayV3): + def create_store(self) -> DBMStoreV3: + path = mktemp(suffix=".anydbm") + atexit.register(atexit_rmglob, path + "*") + store = DBMStoreV3(path, flag="n") + return store def test_nbytes_stored(self): pass # not implemented @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayWithDBMStoreBerkeleyDB, TestArrayWithPathV3): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayV3): + def create_store(self) -> DBMStoreV3: bsddb3 = pytest.importorskip("bsddb3") - path = mktemp(suffix='.dbm') + path = mktemp(suffix=".dbm") atexit.register(os.remove, path) - store = DBMStoreV3(path, flag='n', open=bsddb3.btopen) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + store = DBMStoreV3(path, flag="n", open=bsddb3.btopen) + return store def test_nbytes_stored(self): pass # not implemented @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithLMDBStoreV3(TestArrayWithLMDBStore, TestArrayWithPathV3): +class TestArrayWithLMDBStoreV3(TestArrayV3): + lmdb_buffers = True - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): + def create_store(self) -> LMDBStoreV3: pytest.importorskip("lmdb") - path = mktemp(suffix='.lmdb') + path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) - store = LMDBStoreV3(path, buffers=True) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + store = LMDBStoreV3(path, buffers=self.lmdb_buffers) + return store def test_store_has_bytes_values(self): pass # returns values as memoryviews/buffers instead of bytes @@ -3050,42 +2806,21 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithLMDBStoreV3NoBuffers(TestArrayWithLMDBStoreNoBuffers, TestArrayWithPathV3): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - pytest.importorskip("lmdb") - path = mktemp(suffix='.lmdb') - atexit.register(atexit_rmtree, path) - store = LMDBStoreV3(path, buffers=False) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) +class TestArrayWithLMDBStoreV3NoBuffers(TestArrayWithLMDBStoreV3): + lmdb_buffers = False def test_nbytes_stored(self): pass # not implemented @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithSQLiteStoreV3(TestArrayWithPathV3, TestArrayWithSQLiteStore): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithSQLiteStoreV3(TestArrayV3): + def create_store(self): pytest.importorskip("sqlite3") - path = mktemp(suffix='.db') + path = mktemp(suffix=".db") atexit.register(atexit_rmtree, path) store = SQLiteStoreV3(path) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Zlib(1)) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_nbytes_stored(self): pass # not implemented @@ -3142,18 +2877,10 @@ def __contains__(self, item): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithCustomMappingV3(TestArrayWithPathV3, TestArrayWithCustomMapping): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithCustomMappingV3(TestArrayV3): + def create_store(self): store = CustomMappingV3() - kwargs.setdefault('compressor', Zlib(1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) @@ -3171,18 +2898,10 @@ def test_len(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayNoCacheV3(TestArrayWithPathV3, TestArrayNoCache): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayNoCacheV3(TestArrayWithPathV3): + def create_store(self): store = KVStoreV3(dict()) - kwargs.setdefault('compressor', Zlib(level=1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_object_arrays_danger(self): # skip this one as it only works if metadata are cached @@ -3190,18 +2909,10 @@ def test_object_arrays_danger(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithStoreCacheV3(TestArrayWithPathV3, TestArrayWithStoreCache): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithStoreCacheV3(TestArrayV3): + def create_store(self): store = LRUStoreCacheV3(dict(), max_size=None) - kwargs.setdefault('compressor', Zlib(level=1)) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def test_store_has_bytes_values(self): # skip as the cache has no control over how the store provides values @@ -3210,25 +2921,22 @@ def test_store_has_bytes_values(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3(TestArrayWithPathV3, TestArrayWithFSStore): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithFSStoreV3(TestArrayV3): + compressor = Blosc() + + def create_store(self): path = mkdtemp() atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', ".") + key_separator = self.dimension_separator store = FSStoreV3( path, key_separator=key_separator, auto_mkdir=True, - **fsspec_mapper_kwargs + create=True, + check=True, + missing_exceptions=None ) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + return store def expected(self): return [ @@ -3242,22 +2950,21 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3FromFilesystem(TestArrayWithPathV3, TestArrayWithFSStore): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): +class TestArrayWithFSStoreV3FromFilesystem(TestArrayWithFSStoreV3): + def create_store(self): from fsspec.implementations.local import LocalFileSystem + fs = LocalFileSystem(auto_mkdir=True) path = mkdtemp() atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', ".") - store = FSStoreV3(path, fs=fs, key_separator=key_separator, **fsspec_mapper_kwargs) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) + key_separator = self.dimension_separator + store = FSStoreV3(path, + fs=fs, + key_separator=key_separator, + create=True, + check=True, + missing_exceptions=None) + return store def expected(self): return [ @@ -3271,27 +2978,8 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3PartialRead(TestArrayWithPathV3, TestArrayWithFSStorePartialRead): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - store = FSStoreV3(path) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault("compressor", Blosc()) - init_array(store, path=array_path, **kwargs) - return Array( - store, - path=array_path, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks, - ) +class TestArrayWithFSStoreV3PartialRead(TestArrayWithFSStoreV3): + partial_decompress = True def expected(self): return [ @@ -3306,33 +2994,16 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @pytest.mark.skipif(not v3_sharding_available, reason="sharding is disabled") -class TestArrayWithFSStoreV3PartialReadUncompressedSharded( - TestArrayWithPathV3, TestArrayWithFSStorePartialRead -): +class TestArrayWithFSStoreV3PartialReadUncompressedSharded(TestArrayWithFSStoreV3): + partial_decompress = True + compressor = None - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - store = FSStoreV3(path) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', None) - num_dims = 1 if isinstance(kwargs["shape"], int) else len(kwargs["shape"]) + def create_storage_transformers(self, shape) -> Tuple[Any]: + num_dims = 1 if isinstance(shape, int) else len(shape) sharding_transformer = ShardingStorageTransformer( "indexed", chunks_per_shard=(2, ) * num_dims ) - init_array(store, path=array_path, storage_transformers=[sharding_transformer], **kwargs) - return Array( - store, - path=array_path, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks, - ) + return (sharding_transformer,) def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) @@ -3359,21 +3030,8 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3Nested(TestArrayWithPathV3, TestArrayWithFSStoreNested): - - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', "/") - store = FSStoreV3(path, key_separator=key_separator, auto_mkdir=True) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', Blosc()) - init_array(store, path=array_path, **kwargs) - return Array(store, path=array_path, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) +class TestArrayWithFSStoreV3Nested(TestArrayWithFSStoreV3): + dimension_separator = "/" def expected(self): return [ @@ -3387,28 +3045,8 @@ def expected(self): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestArrayWithFSStoreV3NestedPartialRead(TestArrayWithPathV3, - TestArrayWithFSStoreNestedPartialRead): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - path = mkdtemp() - atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', "/") - store = FSStoreV3(path, key_separator=key_separator, auto_mkdir=True) - cache_metadata = kwargs.pop("cache_metadata", True) - cache_attrs = kwargs.pop("cache_attrs", True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault("compressor", Blosc()) - init_array(store, path=array_path, **kwargs) - return Array( - store, - path=array_path, - read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - partial_decompress=True, - write_empty_chunks=write_empty_chunks, - ) +class TestArrayWithFSStoreV3NestedPartialRead(TestArrayWithFSStoreV3): + dimension_separator = "/" def expected(self): return [ @@ -3423,22 +3061,10 @@ def expected(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") class TestArrayWithStorageTransformersV3(TestArrayWithChunkStoreV3): - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - store = KVStoreV3(dict()) - # separate chunk store - chunk_store = KVStoreV3(dict()) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - dummy_storage_transformer = DummyStorageTransfomer( - "dummy_type", test_value=DummyStorageTransfomer.TEST_CONSTANT + def create_storage_transformers(self, shape) -> Tuple[Any]: + return ( + DummyStorageTransfomer("dummy_type", test_value=DummyStorageTransfomer.TEST_CONSTANT), ) - init_array(store, path=array_path, chunk_store=chunk_store, - storage_transformers=[dummy_storage_transformer], **kwargs) - return Array(store, path=array_path, read_only=read_only, - chunk_store=chunk_store, cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) def expected(self): return [ @@ -3452,23 +3078,14 @@ def expected(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @pytest.mark.skipif(not v3_sharding_available, reason="sharding is disabled") -class TestArrayWithShardingStorageTransformerV3(TestArrayWithPathV3): +class TestArrayWithShardingStorageTransformerV3(TestArrayV3): + compressor = None - @staticmethod - def create_array(array_path='arr1', read_only=False, **kwargs): - store = KVStoreV3(dict()) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - write_empty_chunks = kwargs.pop('write_empty_chunks', True) - kwargs.setdefault('compressor', None) - num_dims = 1 if isinstance(kwargs["shape"], int) else len(kwargs["shape"]) - sharding_transformer = ShardingStorageTransformer( - "indexed", chunks_per_shard=(2, ) * num_dims + def create_storage_transformers(self, shape) -> Tuple[Any]: + num_dims = (1 if isinstance(shape, int) else len(shape)) + return ( + ShardingStorageTransformer("indexed", chunks_per_shard=(2, ) * num_dims), ) - init_array(store, path=array_path, storage_transformers=[sharding_transformer], **kwargs) - return Array(store, path=array_path, read_only=read_only, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks) def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) diff --git a/zarr/util.py b/zarr/util.py index 6ba20b96c2..efbb86e4c0 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -15,7 +15,8 @@ Tuple, TypeVar, Union, - Iterable + Iterable, + cast ) import numpy as np @@ -74,7 +75,7 @@ def json_loads(s: Union[bytes, str]) -> Dict[str, Any]: return json.loads(ensure_text(s, 'utf-8')) -def normalize_shape(shape) -> Tuple[int]: +def normalize_shape(shape: Union[int, Tuple[int, ...], None]) -> Tuple[int, ...]: """Convenience function to normalize the `shape` argument.""" if shape is None: @@ -85,6 +86,7 @@ def normalize_shape(shape) -> Tuple[int]: shape = (int(shape),) # normalize + shape = cast(Tuple[int, ...], shape) shape = tuple(int(s) for s in shape) return shape