Skip to content

Commit 1dc6f47

Browse files
joshmoorejoshmoore
committed
Locally working test_read_all.py
1 parent 225e71a commit 1dc6f47

File tree

8 files changed

+61
-62
lines changed

8 files changed

+61
-62
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,4 @@ Test for compatibility. See [doc/development_overview.md](doc/development_overvi
3131
## Other features
3232

3333
* `touch implementations/{impl}/.skip` to disable an implementation
34+
* Use `make NODEBUG=1 ...` to quiet output

‎implementations/.bash_driver.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,20 @@ set -o pipefail
99
argparse(){
1010
case "${1}" in
1111
write)
12-
>&2 echo "Generating data..."
12+
[[ -z "${NODEBUG}" ]] && >&2 echo "Generating data..."
1313
zi_write;;
1414
list)
1515
shift;
1616
zi_list "$@";;
1717
read)
1818
shift;
19-
>&2 echo "Verifying data..."
19+
[[ -z "${NODEBUG}" ]] && >&2 echo "Verifying data..."
2020
zi_read "$@";;
2121
destroy)
22-
>&2 echo "Tearing down..."
22+
[[ -z "${NODEBUG}" ]] && >&2 echo "Tearing down..."
2323
zi_destroy;;
2424
*)
25-
>&2 echo "Unknown command: ${1}"
25+
[[ -z "${NODEBUG}" ]] && >&2 echo "Unknown command: ${1}"
2626
exit 2;;
2727
esac
2828
}

‎implementations/.conda_driver.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ fi
1818
create_or_activate(){
1919

2020
if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then
21-
>&2 echo "Using $ENVNAME"
21+
[[ -z "${NODEBUG}" ]] && >&2 echo "Using $ENVNAME"
2222
else
23-
>&2 echo "Creating $ENVNAME"
23+
[[ -z "${NODEBUG}" ]] && >&2 echo "Creating $ENVNAME"
2424
$COMMAND env create -n $ENVNAME -f $IMPL/environment.yml
2525
fi
2626
export MAMBA_ROOT_PREFIX=$(mamba info --base -q)
@@ -29,14 +29,14 @@ create_or_activate(){
2929
. $MAMBA_ROOT_PREFIX/etc/profile.d/conda.sh
3030
. $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh
3131

32-
>&2 echo "Activating $ENVNAME"
32+
[[ -z "${NODEBUG}" ]] && >&2 echo "Activating $ENVNAME"
3333
$COMMAND activate $ENVNAME
3434
}
3535

3636
zi_destroy(){
3737

3838
if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then
39-
>&2 echo "Destroying $ENVNAME"
39+
[[ -z "${NODEBUG}" ]] && >&2 echo "Destroying $ENVNAME"
4040
$COMMAND env remove -y -n $ENVNAME
4141
else
4242
>&2 echo "No known env: $ENVNAME"

‎implementations/pyn5/generate_pyn5.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,20 @@ def generate_n5_format(list_only: bool, compressors=pyn5.CompressionType):
2222
f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor)
2323

2424

25+
def verify_format(directory: str, dataset: str):
26+
f = pyn5.File(f"{directory}/{dataset}")# TODO, mode="r")
27+
return f[:]
28+
29+
2530
if __name__ == '__main__':
2631
import sys
2732
import argparse
2833
parser = argparse.ArgumentParser()
2934
parser.add_argument("-list", action="store_true")
3035
parser.add_argument("-verify", action="store_true")
36+
parser.add_argument("args", nargs="*")
3137
ns = parser.parse_args()
3238
if ns.verify:
33-
verify_n5_format()
39+
verify_format(*ns.args)
3440
else:
3541
generate_n5_format(ns.list)

‎implementations/tensorstore/generate_tensorstore.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,10 @@ def generate_n5_format(list_only:bool, compressors=['gzip', None]):
111111
parser = argparse.ArgumentParser()
112112
parser.add_argument("-list", action="store_true")
113113
parser.add_argument("-verify", action="store_true")
114+
parser.add_argument("args", nargs="*")
114115
ns = parser.parse_args()
115116
if ns.verify:
116-
verify_format(ns.known_args)
117+
verify_format(*ns.args)
117118
else:
118119
generate_zarr_format(ns.list)
119120
generate_n5_format(ns.list)

‎implementations/z5py/generate_z5py.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,20 @@ def generate_n5_format(list_only:bool, compressors=['gzip', 'raw']):
4444
f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor)
4545

4646

47+
def verify_format(directory: str, dataset: str):
48+
f = z5py.File(f"{directory}/{dataset}"), mode="r")
49+
return f[:]
50+
51+
4752
if __name__ == '__main__':
4853
import argparse
4954
parser = argparse.ArgumentParser()
5055
parser.add_argument("-list", action="store_true")
5156
parser.add_argument("-verify", action="store_true")
57+
parser.add_argument("args", nargs="*")
5258
ns = parser.parse_args()
5359
if ns.verify:
54-
verify_format(ns.known_args)
60+
verify_format(*ns.args)
5561
else:
5662
generate_zarr_format(ns.list)
5763
generate_n5_format(ns.list)

‎implementations/zarrita/generate_zarrita.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,10 @@ def generate_zr3_format(list_only:bool, codecs=["gzip", "blosc", None], nested=T
6565
parser = argparse.ArgumentParser()
6666
parser.add_argument("-list", action="store_true")
6767
parser.add_argument("-verify", action="store_true")
68+
parser.add_argument("args", nargs="*")
6869
ns = parser.parse_args()
6970
if ns.verify:
70-
verify_format(ns.known_args)
71+
verify_format(*ns.args)
7172
else:
7273
for nested in [False, True]:
7374
for sharded in [False, True]:

‎test/test_read_all.py

Lines changed: 34 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
3535
"""
3636
import os
37+
import sys
3738
import subprocess
3839
from typing import Dict, List
3940
from pathlib import Path
@@ -81,83 +82,64 @@
8182
},
8283
}
8384

84-
def read_with_jzarr(fpath, ds_name, nested=None):
85-
if ds_name == "blosc":
86-
ds_name = "blosc/lz4"
8785

86+
def make_read(name, fpath, ds_name, nested=None):
8887
cmd = (
89-
f"implementations/jzarr/generate_data.sh "
90-
f"-verify {str(fpath)} {ds_name}"
88+
f"make implementations/{name}-read-fast "
89+
f"NODEBUG=1 DIR={str(fpath)} DATASET={ds_name}"
9190
)
9291

9392
# will raise subprocess.CalledProcessError if return code is not 0
94-
subprocess.check_output(cmd, shell=True)
95-
return None
93+
result = subprocess.run(cmd, shell=True,
94+
stdout=subprocess.PIPE,
95+
stderr=subprocess.PIPE)
96+
if result.returncode != 0:
97+
raise Exception(
98+
{
99+
"command": cmd,
100+
"stdout": result.stdout,
101+
"stderr": result.stderr,
102+
}
103+
)
104+
return result
105+
106+
def read_with_jzarr(fpath, ds_name, nested=None):
107+
if ds_name == "blosc":
108+
ds_name = "blosc/lz4"
109+
return make_read("jzarr", fpath, ds_name, nested)
96110

97111

98112
def read_with_zarr(fpath, ds_name, nested):
99-
import zarr
100113
if ds_name == "blosc":
101114
ds_name = "blosc/lz4"
102-
if str(fpath).endswith('.zr'):
103-
if nested:
104-
if 'FSStore' in str(fpath):
105-
store = zarr.storage.FSStore(
106-
os.fspath(fpath), dimension_separator='/', mode='r'
107-
)
108-
else:
109-
store = zarr.storage.NestedDirectoryStore(os.fspath(fpath))
110-
else:
111-
if 'FSStore' in str(fpath):
112-
store = zarr.storage.FSStore(os.fspath(fpath))
113-
else:
114-
store = zarr.storage.DirectoryStore(fpath)
115-
else:
116-
store = os.fspath(fpath)
117-
return zarr.open(store)[ds_name][:]
115+
return make_read("zarr-python", fpath, ds_name, nested)
118116

119117

120118
def read_with_pyn5(fpath, ds_name, nested):
121-
import pyn5
122-
return pyn5.File(fpath)[ds_name][:]
119+
return make_read("pyn5", fpath, ds_name, nested)
123120

124121

125122
def read_with_z5py(fpath, ds_name, nested):
126-
import z5py
127123
if ds_name == "blosc":
128124
ds_name = "blosc/lz4"
129-
return z5py.File(fpath)[ds_name][:]
125+
return make_read("z5py", fpath, ds_name, nested)
130126

131127

132128
def read_with_zarrita(fpath, ds_name, nested):
133-
import zarrita
134129
if ds_name == "blosc":
135130
ds_name = "blosc/lz4"
136-
h = zarrita.get_hierarchy(str(fpath.absolute()))
137-
return h["/" + ds_name][:]
131+
return make_read("zarrita", fpath, ds_name, nested)
138132

139133
def read_with_xtensor_zarr(fpath, ds_name, nested):
140134
if ds_name == "blosc":
141135
ds_name = "blosc/lz4"
142-
fname = "a.npz"
143-
if os.path.exists(fname):
144-
os.remove(fname)
145-
subprocess.check_call(["implementations/xtensor_zarr/build/run_xtensor_zarr", fpath, ds_name])
146-
return np.load(fname)["a"]
136+
return make_read("xtensor_zarr", fpath, ds_name, nested)
147137

148138
def read_with_Rarr(fpath, ds_name, nested):
149139

150140
if ds_name == "blosc":
151141
ds_name = "blosc/lz4"
152-
153-
cmd = (
154-
f"Rscript implementations/Rarr/verify_data_internal.R "
155-
f"{str(fpath)} {ds_name}"
156-
)
157-
158-
# will raise subprocess.CalledProcessError if return code is not 0
159-
subprocess.check_output(cmd, shell=True)
160-
return None
142+
return make_read("Rarr", fpath, ds_name, nested)
161143

162144

163145
EXTENSIONS = {"zarr": ".zr", "N5": ".n5", "zarr-v3": ".zr3"}
@@ -274,11 +256,13 @@ def test_correct_read(fmt, writing_library, reading_library, codec, nested,
274256
f"file not found: {fpath}. Make sure you have generated the data "
275257
"using 'make data'"
276258
)
277-
test = read_fn(fpath, codec, nested)
278-
# Assume if None is returned, the read function has verified.
279-
if test is not None:
280-
assert test.shape == reference.shape
281-
assert np.allclose(test, reference)
259+
result = read_fn(fpath, codec, nested)
260+
261+
if b"Skipping" in result.stderr:
262+
pytest.skip(result.stderr.decode())
263+
else:
264+
print(result.stdout)
265+
print(result.stderr, file=sys.stderr)
282266

283267

284268
def tabulate_test_results(params, per_codec_tables=False):

0 commit comments

Comments
 (0)