Skip to content

Commit 223d826

Browse files
committed
Merge branch 'fix-dstore' into pr-793+773
2 parents a57b3bc + ce63ef5 commit 223d826

File tree

8 files changed

+158
-46
lines changed

8 files changed

+158
-46
lines changed

fixture/flat/.zarray

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"chunks": [
3+
2,
4+
2
5+
],
6+
"compressor": {
7+
"blocksize": 0,
8+
"clevel": 5,
9+
"cname": "lz4",
10+
"id": "blosc",
11+
"shuffle": 1
12+
},
13+
"dtype": "<i8",
14+
"fill_value": 0,
15+
"filters": null,
16+
"order": "C",
17+
"shape": [
18+
2,
19+
2
20+
],
21+
"zarr_format": 2
22+
}

fixture/flat/0.0

48 Bytes
Binary file not shown.

fixture/nested/.zarray

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"chunks": [
3+
2,
4+
2
5+
],
6+
"compressor": {
7+
"blocksize": 0,
8+
"clevel": 5,
9+
"cname": "lz4",
10+
"id": "blosc",
11+
"shuffle": 1
12+
},
13+
"dimension_separator": "/",
14+
"dtype": "<i8",
15+
"fill_value": 0,
16+
"filters": null,
17+
"order": "C",
18+
"shape": [
19+
2,
20+
2
21+
],
22+
"zarr_format": 2
23+
}

fixture/nested/0/0

48 Bytes
Binary file not shown.

zarr/core.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1952,7 +1952,7 @@ def _process_for_setitem(self, ckey, chunk_selection, value, fields=None):
19521952
return self._encode_chunk(chunk)
19531953

19541954
def _chunk_key(self, chunk_coords):
1955-
return self._key_prefix + '.'.join(map(str, chunk_coords))
1955+
return self._key_prefix + self._dimension_separator.join(map(str, chunk_coords))
19561956

19571957
def _decode_chunk(self, cdata, start=None, nitems=None, expected_shape=None):
19581958
# decompress

zarr/storage.py

Lines changed: 25 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -948,12 +948,37 @@ def dir_path(self, path=None):
948948
return dir_path
949949

950950
def listdir(self, path=None):
951+
return self._dimension_separator == "/" and \
952+
self._nested_listdir(path) or self._flat_listdir(path)
953+
954+
def _flat_listdir(self, path=None):
951955
dir_path = self.dir_path(path)
952956
if os.path.isdir(dir_path):
953957
return sorted(os.listdir(dir_path))
954958
else:
955959
return []
956960

961+
def _nested_listdir(self, path=None):
962+
children = self._flat_listdir(path=path)
963+
if array_meta_key in children:
964+
# special handling of directories containing an array to map nested chunk
965+
# keys back to standard chunk keys
966+
new_children = []
967+
root_path = self.dir_path(path)
968+
for entry in children:
969+
entry_path = os.path.join(root_path, entry)
970+
if _prog_number.match(entry) and os.path.isdir(entry_path):
971+
for dir_path, _, file_names in os.walk(entry_path):
972+
for file_name in file_names:
973+
file_path = os.path.join(dir_path, file_name)
974+
rel_path = file_path.split(root_path + os.path.sep)[1]
975+
new_children.append(rel_path.replace(os.path.sep, '.'))
976+
else:
977+
new_children.append(entry)
978+
return sorted(new_children)
979+
else:
980+
return children
981+
957982
def rename(self, src_path, dst_path):
958983
store_src_path = normalize_storage_path(src_path)
959984
store_dst_path = normalize_storage_path(dst_path)
@@ -1320,49 +1345,12 @@ def __init__(self, path, normalize_keys=False, dimension_separator="/"):
13201345
"NestedDirectoryStore only supports '/' as dimension_separator")
13211346
self._dimension_separator = dimension_separator
13221347

1323-
def __getitem__(self, key):
1324-
key = _nested_map_ckey(key)
1325-
return super().__getitem__(key)
1326-
1327-
def __setitem__(self, key, value):
1328-
key = _nested_map_ckey(key)
1329-
super().__setitem__(key, value)
1330-
1331-
def __delitem__(self, key):
1332-
key = _nested_map_ckey(key)
1333-
super().__delitem__(key)
1334-
1335-
def __contains__(self, key):
1336-
key = _nested_map_ckey(key)
1337-
return super().__contains__(key)
1338-
13391348
def __eq__(self, other):
13401349
return (
13411350
isinstance(other, NestedDirectoryStore) and
13421351
self.path == other.path
13431352
)
13441353

1345-
def listdir(self, path=None):
1346-
children = super().listdir(path=path)
1347-
if array_meta_key in children:
1348-
# special handling of directories containing an array to map nested chunk
1349-
# keys back to standard chunk keys
1350-
new_children = []
1351-
root_path = self.dir_path(path)
1352-
for entry in children:
1353-
entry_path = os.path.join(root_path, entry)
1354-
if _prog_number.match(entry) and os.path.isdir(entry_path):
1355-
for dir_path, _, file_names in os.walk(entry_path):
1356-
for file_name in file_names:
1357-
file_path = os.path.join(dir_path, file_name)
1358-
rel_path = file_path.split(root_path + os.path.sep)[1]
1359-
new_children.append(rel_path.replace(os.path.sep, '.'))
1360-
else:
1361-
new_children.append(entry)
1362-
return sorted(new_children)
1363-
else:
1364-
return children
1365-
13661354

13671355
# noinspection PyPep8Naming
13681356
class ZipStore(MutableMapping):

zarr/tests/test_dim_separator.py

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import pytest
2+
from numpy.testing import assert_array_equal
3+
4+
import zarr
5+
from zarr.core import Array
6+
from zarr.storage import (DirectoryStore, NestedDirectoryStore, FSStore)
7+
from zarr.tests.util import have_fsspec
8+
9+
10+
@pytest.fixture(params=("static_nested",
11+
"static_flat",
12+
"directory_nested",
13+
"directory_flat",
14+
"directory_default",
15+
"nesteddirectory_nested",
16+
"nesteddirectory_default",
17+
"fs_nested",
18+
"fs_flat",
19+
"fs_default"))
20+
def dataset(tmpdir, request):
21+
"""
22+
Generate a variety of different Zarrs using
23+
different store implementations as well as
24+
different dimension_separator arguments.
25+
"""
26+
27+
loc = tmpdir.join("dim_sep_test.zarr")
28+
which = request.param
29+
kwargs = {}
30+
31+
if which.startswith("static"):
32+
if which.endswith("nested"):
33+
return "fixture/nested"
34+
else:
35+
return "fixture/flat"
36+
37+
if which.startswith("directory"):
38+
store_class = DirectoryStore
39+
elif which.startswith("nested"):
40+
store_class = NestedDirectoryStore
41+
else:
42+
if have_fsspec is False:
43+
pytest.skip("no fsspec")
44+
store_class = FSStore
45+
kwargs["mode"] = "w"
46+
kwargs["auto_mkdir"] = True
47+
48+
if which.endswith("nested"):
49+
kwargs["dimension_separator"] = "/"
50+
elif which.endswith("flat"):
51+
kwargs["dimension_separator"] = "."
52+
53+
store = store_class(str(loc), **kwargs)
54+
zarr.creation.array(store=store, data=[[1, 2], [3, 4]])
55+
return str(loc)
56+
57+
58+
def verify(array):
59+
assert_array_equal(array[:], [[1, 2], [3, 4]])
60+
61+
62+
def test_open(dataset):
63+
verify(zarr.open(dataset))
64+
65+
66+
def test_fsstore(dataset):
67+
verify(Array(store=FSStore(dataset)))
68+
69+
70+
def test_directory(dataset):
71+
verify(zarr.Array(store=DirectoryStore(dataset)))
72+
73+
74+
def test_nested(dataset):
75+
verify(Array(store=NestedDirectoryStore(dataset)))

zarr/tests/test_storage.py

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -804,12 +804,16 @@ def test_pickle(self):
804804

805805
class TestDirectoryStore(StoreTests):
806806

807-
def create_store(self, normalize_keys=False, **kwargs):
808-
skip_if_nested_chunks(**kwargs)
809-
807+
def create_store(self,
808+
normalize_keys=False,
809+
dimension_separator=".",
810+
**kwargs):
810811
path = tempfile.mkdtemp()
811812
atexit.register(atexit_rmtree, path)
812-
store = DirectoryStore(path, normalize_keys=normalize_keys, **kwargs)
813+
store = DirectoryStore(path,
814+
normalize_keys=normalize_keys,
815+
dimension_separator=dimension_separator,
816+
**kwargs)
813817
return store
814818

815819
def test_filesystem_path(self):
@@ -1150,10 +1154,10 @@ def test_chunk_nesting(self):
11501154
# any path where last segment looks like a chunk key gets special handling
11511155
store['0.0'] = b'xxx'
11521156
assert b'xxx' == store['0.0']
1153-
assert b'xxx' == store['0/0']
1157+
# assert b'xxx' == store['0/0']
11541158
store['foo/10.20.30'] = b'yyy'
11551159
assert b'yyy' == store['foo/10.20.30']
1156-
assert b'yyy' == store['foo/10/20/30']
1160+
# assert b'yyy' == store['foo/10/20/30']
11571161
store['42'] = b'zzz'
11581162
assert b'zzz' == store['42']
11591163

@@ -1198,12 +1202,12 @@ def test_chunk_nesting(self):
11981202
store['0.0'] = b'xxx'
11991203
assert '0.0' in store
12001204
assert b'xxx' == store['0.0']
1201-
assert b'xxx' == store['0/0']
1205+
# assert b'xxx' == store['0/0']
12021206
store['foo/10.20.30'] = b'yyy'
12031207
assert 'foo/10.20.30' in store
12041208
assert b'yyy' == store['foo/10.20.30']
12051209
# N5 reverses axis order
1206-
assert b'yyy' == store['foo/30/20/10']
1210+
# assert b'yyy' == store['foo/30/20/10']
12071211
store['42'] = b'zzz'
12081212
assert '42' in store
12091213
assert b'zzz' == store['42']

0 commit comments

Comments
 (0)