diff --git a/docs/release.rst b/docs/release.rst index 45eb9c8a49..c87ebff589 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -41,6 +41,11 @@ Maintenance * CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and upgrade all pinned package requirements. :issue:`308`. +* Corrects handling of ``NaT`` in ``datetime64`` and ``timedelta64`` in various + compressors (by :user:`John Kirkham `; :issue:`344`). + +Acknowledgments +~~~~~~~~~~~~~~~ .. _release_2.2.0: diff --git a/zarr/meta.py b/zarr/meta.py index 7984efb701..c90c12ff38 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -209,6 +209,6 @@ def encode_fill_value(v, dtype): elif dtype.kind == 'U': return v elif dtype.kind in 'mM': - return int(v.view('u8')) + return int(v.view('i8')) else: return v diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 544ec95c41..cbad222edb 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -955,8 +955,9 @@ def test_dtypes(self): dtype = '{}8[{}]'.format(base_type, resolution) z = self.create_array(shape=100, dtype=dtype, fill_value=0) assert z.dtype == np.dtype(dtype) - a = np.random.randint(0, np.iinfo('u8').max, size=z.shape[0], - dtype='u8').view(dtype) + a = np.random.randint(np.iinfo('i8').min, np.iinfo('i8').max, + size=z.shape[0], + dtype='i8').view(dtype) z[:] = a assert_array_equal(a, z[:]) diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index ef2232c234..4c2af854fb 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -135,6 +135,12 @@ def test_full(): z = full(100, chunks=10, fill_value=np.nan, dtype='f8') assert np.all(np.isnan(z[:])) + # NaT + z = full(100, chunks=10, fill_value='NaT', dtype='M8[s]') + assert np.all(np.isnat(z[:])) + z = full(100, chunks=10, fill_value='NaT', dtype='m8[s]') + assert np.all(np.isnat(z[:])) + # byte string dtype v = b'xxx' z = full(100, chunks=10, fill_value=v, dtype='S3') diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index 904c2146a7..12dda299c8 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -116,6 +116,60 @@ def test_encode_decode_array_2(): assert [df.get_config()] == meta_dec['filters'] +def test_encode_decode_array_datetime_timedelta(): + + # some variations + for k in ['m8[s]', 'M8[s]']: + compressor = Blosc(cname='lz4', clevel=3, shuffle=2) + dtype = np.dtype(k) + fill_value = dtype.type("NaT") + meta = dict( + shape=(100, 100), + chunks=(10, 10), + dtype=dtype, + compressor=compressor.get_config(), + fill_value=fill_value, + order=dtype.char, + filters=[] + ) + + meta_json = '''{ + "chunks": [10, 10], + "compressor": { + "id": "blosc", + "clevel": 3, + "cname": "lz4", + "shuffle": 2, + "blocksize": 0 + }, + "dtype": "%s", + "fill_value": -9223372036854775808, + "filters": [], + "order": "%s", + "shape": [100, 100], + "zarr_format": %s + }''' % (dtype.str, dtype.char, ZARR_FORMAT) + + # test encoding + meta_enc = encode_array_metadata(meta) + assert_json_equal(meta_json, meta_enc) + + # test decoding + meta_dec = decode_array_metadata(meta_enc) + assert ZARR_FORMAT == meta_dec['zarr_format'] + assert meta['shape'] == meta_dec['shape'] + assert meta['chunks'] == meta_dec['chunks'] + assert meta['dtype'] == meta_dec['dtype'] + assert meta['compressor'] == meta_dec['compressor'] + assert meta['order'] == meta_dec['order'] + # Based off of this SO answer: https://stackoverflow.com/a/49972198 + assert np.all( + fill_value.view((np.uint8, fill_value.itemsize)) == + meta_dec['fill_value'].view((np.uint8, meta_dec['fill_value'].itemsize)) + ) + assert [] == meta_dec['filters'] + + def test_encode_decode_array_dtype_shape(): meta = dict(