diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 3e8e9bac66..9422453375 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -176,7 +176,7 @@ print some diagnostics, e.g.:: Read-only : False Compressor : Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE, : blocksize=0) - Store type : builtins.dict + Store type : zarr.storage.DictStore No. bytes : 400000000 (381.5M) No. bytes stored : 3379344 (3.2M) Storage ratio : 118.4 @@ -268,7 +268,7 @@ Here is an example using a delta filter with the Blosc compressor:: Read-only : False Filter [0] : Delta(dtype='>> from numcodecs import Quantize, Adler32 - >>> store, chunk_store = dict(), dict() + >>> store, chunk_store = DictStore(), DictStore() >>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype='f8', ... filters=[Quantize(digits=2, dtype='f8'), Adler32()], ... store=store, chunk_store=chunk_store) @@ -125,7 +125,7 @@ def create(shape, chunks=True, dtype=None, compressor='default', return z -def normalize_store_arg(store, clobber=False, default=dict): +def normalize_store_arg(store, clobber=False, default=DictStore): if store is None: return default() elif isinstance(store, str): diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 17821130eb..6be0e9dc99 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -91,6 +91,9 @@ class Group(MutableMapping): def __init__(self, store, path=None, read_only=False, chunk_store=None, cache_attrs=True, synchronizer=None): + if isinstance(store, dict): + raise TypeError("Please use Zarr's DictStore instead") + self._store = store self._chunk_store = chunk_store self._path = normalize_storage_path(path) diff --git a/zarr/storage.py b/zarr/storage.py index 656b05acf8..4849e4a42b 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -476,12 +476,11 @@ class DictStore(MutableMapping): >>> type(g.store) - Note that the default class when creating an array is the built-in - :class:`dict` class, i.e.:: + Also this is the default class when creating an array. E.g.:: >>> z = zarr.zeros(100) >>> type(z.store) - + Notes ----- diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 97842a6f6c..fd8e1d2146 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -15,7 +15,7 @@ import pytest -from zarr.storage import (DirectoryStore, init_array, init_group, NestedDirectoryStore, +from zarr.storage import (DictStore, DirectoryStore, init_array, init_group, NestedDirectoryStore, DBMStore, LMDBStore, SQLiteStore, atexit_rmtree, atexit_rmglob, LRUStoreCache) from zarr.core import Array @@ -41,7 +41,7 @@ class TestArray(unittest.TestCase): def test_array_init(self): # normal initialization - store = dict() + store = DictStore() init_array(store, shape=100, chunks=10) a = Array(store) assert isinstance(a, Array) @@ -54,7 +54,7 @@ def test_array_init(self): assert "8fecb7a17ea1493d9c1430d04437b4f5b0b34985" == a.hexdigest() # initialize at path - store = dict() + store = DictStore() init_array(store, shape=100, chunks=10, path='foo/bar') a = Array(store, path='foo/bar') assert isinstance(a, Array) @@ -67,18 +67,18 @@ def test_array_init(self): assert "8fecb7a17ea1493d9c1430d04437b4f5b0b34985" == a.hexdigest() # store not initialized - store = dict() + store = DictStore() with pytest.raises(ValueError): Array(store) # group is in the way - store = dict() + store = DictStore() init_group(store, path='baz') with pytest.raises(ValueError): Array(store, path='baz') def create_array(self, read_only=False, **kwargs): - store = dict() + store = DictStore() kwargs.setdefault('compressor', Zlib(level=1)) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) @@ -1255,7 +1255,7 @@ class TestArrayWithPath(TestArray): @staticmethod def create_array(read_only=False, **kwargs): - store = dict() + store = DictStore() cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) init_array(store, path='foo/bar', **kwargs) @@ -1299,18 +1299,14 @@ def test_nbytes_stored(self): if k.startswith('foo/bar/')) assert expect_nbytes_stored == z.nbytes_stored - # mess with store - z.store[z._key_prefix + 'foo'] = list(range(10)) - assert -1 == z.nbytes_stored - class TestArrayWithChunkStore(TestArray): @staticmethod def create_array(read_only=False, **kwargs): - store = dict() + store = DictStore() # separate chunk store - chunk_store = dict() + chunk_store = DictStore() cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) init_array(store, chunk_store=chunk_store, **kwargs) @@ -1353,10 +1349,6 @@ def test_nbytes_stored(self): for v in z.chunk_store.values()) assert expect_nbytes_stored == z.nbytes_stored - # mess with store - z.chunk_store[z._key_prefix + 'foo'] = list(range(10)) - assert -1 == z.nbytes_stored - class TestArrayWithDirectoryStore(TestArray): @@ -1516,7 +1508,7 @@ def test_nbytes_stored(self): class TestArrayWithNoCompressor(TestArray): def create_array(self, read_only=False, **kwargs): - store = dict() + store = DictStore() kwargs.setdefault('compressor', None) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) @@ -1551,7 +1543,7 @@ def test_hexdigest(self): class TestArrayWithBZ2Compressor(TestArray): def create_array(self, read_only=False, **kwargs): - store = dict() + store = DictStore() compressor = BZ2(level=1) kwargs.setdefault('compressor', compressor) cache_metadata = kwargs.pop('cache_metadata', True) @@ -1587,7 +1579,7 @@ def test_hexdigest(self): class TestArrayWithBloscCompressor(TestArray): def create_array(self, read_only=False, **kwargs): - store = dict() + store = DictStore() compressor = Blosc(cname='zstd', clevel=1, shuffle=1) kwargs.setdefault('compressor', compressor) cache_metadata = kwargs.pop('cache_metadata', True) @@ -1630,7 +1622,7 @@ def test_hexdigest(self): class TestArrayWithLZMACompressor(TestArray): def create_array(self, read_only=False, **kwargs): - store = dict() + store = DictStore() compressor = LZMA(preset=1) kwargs.setdefault('compressor', compressor) cache_metadata = kwargs.pop('cache_metadata', True) @@ -1667,7 +1659,7 @@ class TestArrayWithFilters(TestArray): @staticmethod def create_array(read_only=False, **kwargs): - store = dict() + store = DictStore() dtype = kwargs.get('dtype', None) filters = [ Delta(dtype=dtype), @@ -1710,7 +1702,7 @@ def test_astype_no_filters(self): dtype = np.dtype(np.int8) astype = np.dtype(np.float32) - store = dict() + store = DictStore() init_array(store, shape=shape, chunks=10, dtype=dtype) data = np.arange(np.prod(shape), dtype=dtype).reshape(shape) @@ -1830,11 +1822,31 @@ def test_nbytes_stored(self): assert -1 == z.nbytes_stored +class TestArrayWithCustomChunkStore(TestArray): + + @staticmethod + def create_array(read_only=False, **kwargs): + store = CustomMapping() + kwargs["chunk_store"] = CustomMapping() + kwargs.setdefault('compressor', Zlib(1)) + cache_metadata = kwargs.pop('cache_metadata', True) + cache_attrs = kwargs.pop('cache_attrs', True) + init_array(store, **kwargs) + return Array(store, read_only=read_only, cache_metadata=cache_metadata, + cache_attrs=cache_attrs) + + def test_nbytes_stored(self): + z = self.create_array(shape=1000, chunks=100) + assert -1 == z.nbytes_stored + z[:] = 42 + assert -1 == z.nbytes_stored + + class TestArrayNoCache(TestArray): @staticmethod def create_array(read_only=False, **kwargs): - store = dict() + store = DictStore() kwargs.setdefault('compressor', Zlib(level=1)) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) @@ -1906,7 +1918,7 @@ class TestArrayWithStoreCache(TestArray): @staticmethod def create_array(read_only=False, **kwargs): - store = LRUStoreCache(dict(), max_size=None) + store = LRUStoreCache(DictStore(), max_size=None) kwargs.setdefault('compressor', Zlib(level=1)) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 369cf4b55a..d1cbb842ac 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -41,7 +41,7 @@ class TestGroup(unittest.TestCase): @staticmethod def create_store(): # can be overridden in sub-classes - return dict(), None + return DictStore(), None def create_group(self, store=None, path=None, read_only=False, chunk_store=None, synchronizer=None): @@ -948,7 +948,7 @@ class TestGroupWithChunkStore(TestGroup): @staticmethod def create_store(): - return dict(), dict() + return DictStore(), DictStore() def test_chunk_store(self): # setup @@ -979,7 +979,7 @@ class TestGroupWithStoreCache(TestGroup): @staticmethod def create_store(): - store = LRUStoreCache(dict(), max_size=None) + store = LRUStoreCache(DictStore(), max_size=None) return store, None @@ -993,13 +993,13 @@ def test_group(): assert '/' == g.name # usage with custom store - store = dict() + store = DictStore() g = group(store=store) assert isinstance(g, Group) assert store is g.store # overwrite behaviour - store = dict() + store = DictStore() init_array(store, shape=100, chunks=10) with pytest.raises(ValueError): group(store) diff --git a/zarr/tests/test_info.py b/zarr/tests/test_info.py index e0d2330d6b..7ea7859dca 100644 --- a/zarr/tests/test_info.py +++ b/zarr/tests/test_info.py @@ -9,7 +9,7 @@ def test_info(): # setup - g = zarr.group(store=dict(), chunk_store=dict(), + g = zarr.group(store=zarr.DictStore(), chunk_store=zarr.DictStore(), synchronizer=zarr.ThreadSynchronizer()) g.create_group('foo') z = g.zeros('bar', shape=10, filters=[numcodecs.Adler32()]) diff --git a/zarr/tests/test_sync.py b/zarr/tests/test_sync.py index 1593845f1c..c85300716f 100644 --- a/zarr/tests/test_sync.py +++ b/zarr/tests/test_sync.py @@ -19,7 +19,7 @@ from zarr.sync import ThreadSynchronizer, ProcessSynchronizer from zarr.core import Array from zarr.attrs import Attributes -from zarr.storage import init_array, DirectoryStore, init_group, atexit_rmtree +from zarr.storage import init_array, DictStore, DirectoryStore, init_group, atexit_rmtree from zarr.hierarchy import Group @@ -100,7 +100,7 @@ def test_parallel_append(self): class TestArrayWithThreadSynchronizer(TestArray, MixinArraySyncTests): def create_array(self, read_only=False, **kwargs): - store = dict() + store = DictStore() cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) init_array(store, **kwargs)