Dask to_zarr now complains about read-only after #1304 merged

[djhoese]

Zarr version

main branch

Numcodecs version

0.11.0

Python Version

3.9

Operating System

Linux - ubuntu-latest

Installation

Pip into conda environment with other unstable dependencies

Description

I'm not sure if there are other changes planned to make this work, but to_zarr in dask which calls create in zarr now seems to open things as read-only even though it is creating a new file.

Steps to reproduce

In [1]: import dask.array as da

In [2]: a = da.random.random((10, 10), chunks=2)

In [3]: a.to_zarr("/tmp/test.zarr", compute=False)
---------------------------------------------------------------------------
ReadOnlyError                             Traceback (most recent call last)
Input In [3], in <cell line: 1>()
----> 1 a.to_zarr("/tmp/test.zarr", compute=False)

File ~/miniconda3/envs/satpy_py39_unstable/lib/python3.9/site-packages/dask/array/core.py:2926, in Array.to_zarr(self, *args, **kwargs)
   2915 def to_zarr(self, *args, **kwargs):
   2916     """Save array to the zarr storage format
   2917 
   2918     See https://zarr.readthedocs.io for details about the format.
   (...)
   2924     dask.array.to_zarr : equivalent function
   2925     """
-> 2926     return to_zarr(self, *args, **kwargs)

File ~/miniconda3/envs/satpy_py39_unstable/lib/python3.9/site-packages/dask/array/core.py:3685, in to_zarr(arr, url, component, storage_options, overwrite, region, compute, return_stored, **kwargs)
   3681     mapper = url
   3683 chunks = [c[0] for c in arr.chunks]
-> 3685 z = zarr.create(
   3686     shape=arr.shape,
   3687     chunks=chunks,
   3688     dtype=arr.dtype,
   3689     store=mapper,
   3690     path=component,
   3691     overwrite=overwrite,
   3692     **kwargs,
   3693 )
   3694 return arr.store(z, lock=False, compute=compute, return_stored=return_stored)

File ~/miniconda3/envs/satpy_py39_unstable/lib/python3.9/site-packages/zarr/creation.py:170, in create(shape, chunks, dtype, compressor, fill_value, order, store, synchronizer, overwrite, path, chunk_store, filters, cache_metadata, cache_attrs, read_only, object_codec, dimension_separator, write_empty_chunks, zarr_version, meta_array, **kwargs)
    167     path = '/'
    169 # initialize array metadata
--> 170 init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor,
    171            fill_value=fill_value, order=order, overwrite=overwrite, path=path,
    172            chunk_store=chunk_store, filters=filters, object_codec=object_codec,
    173            dimension_separator=dimension_separator)
    175 # instantiate array
    176 z = Array(store, path=path, chunk_store=chunk_store, synchronizer=synchronizer,
    177           cache_metadata=cache_metadata, cache_attrs=cache_attrs, read_only=read_only,
    178           write_empty_chunks=write_empty_chunks, meta_array=meta_array)

File ~/miniconda3/envs/satpy_py39_unstable/lib/python3.9/site-packages/zarr/storage.py:436, in init_array(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec, dimension_separator)
    433 if not compressor:
    434     # compatibility with legacy tests using compressor=[]
    435     compressor = None
--> 436 _init_array_metadata(store, shape=shape, chunks=chunks, dtype=dtype,
    437                      compressor=compressor, fill_value=fill_value,
    438                      order=order, overwrite=overwrite, path=path,
    439                      chunk_store=chunk_store, filters=filters,
    440                      object_codec=object_codec,
    441                      dimension_separator=dimension_separator)

File ~/miniconda3/envs/satpy_py39_unstable/lib/python3.9/site-packages/zarr/storage.py:597, in _init_array_metadata(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec, dimension_separator)
    595 key = _prefix_to_array_key(store, _path_to_prefix(path))
    596 if hasattr(store, '_metadata_class'):
--> 597     store[key] = store._metadata_class.encode_array_metadata(meta)  # type: ignore
    598 else:
    599     store[key] = encode_array_metadata(meta)

File ~/miniconda3/envs/satpy_py39_unstable/lib/python3.9/site-packages/zarr/storage.py:1398, in FSStore.__setitem__(self, key, value)
   1396 def __setitem__(self, key, value):
   1397     if self.mode == 'r':
-> 1398         raise ReadOnlyError()
   1399     key = self._normalize_key(key)
   1400     path = self.dir_path(key)

ReadOnlyError: object is read-only

Additional output

No response

[djhoese]

This kind of seems like a bug that has existed for a while and was only just revealed (I'm sure I might be doing something wrong) because this line never passes mode='w' when normalizing the store argument during creation:

zarr-python/zarr/creation.py

Line 148 in 4e633ad

store = normalize_store_arg(store, zarr_version=zarr_version)

[jrbourbeau]

Thanks @djhoese, this was also triggered in Dask's upstream CI build (xref dask/dask#9736). @ravwojdyla @martindurant thoughts on passing mode="w" in create?

[ravwojdyla]

thoughts on passing mode="w" in create?

+1