ValueError: cannot reshape array of size ... #3541
Description
Zarr version
v3.1.3
Numcodecs version
v0.16.2
Python Version
3.11
Operating System
Linux/Mac
Installation
pypi
Description
We build a zarr store (and skip writing empty chunks) in another process. We do NOT use sharding. We then open the dataset after it has been written and see this issue.
Steps to reproduce
import zarr
arr = zarr.open("s3://our/bucket/data.zarr")['variables']
# shape is (1, 10, 285185, 524551)
d = arr[0:1, :, 4096:8192, 327680:331776] # causes issue
arr.metadata
#ArrayV3Metadata(shape=(1, 10, 285185, 524551), data_type=Float32(endianness='little'), chunk_grid=RegularChunkGrid(chunk_shape=(1, 1, 4096, 4096)), chunk_key_encoding=DefaultChunkKeyEncoding(separator='/'), fill_value=np.float32(0.0), codecs=(BytesCodec(endian<Endian.little: 'little'>), ZstdCodec(level=0, checksum=False)), attributes={'scale_factor': 1.0, 'add_offset': 0.0, 'coordinates': 'spatial_ref', '_FillValue': 'AAAAAAAA+H8='}, dimension_names=('time', 'band', 'y', 'x'), zarr_format=3, node_type='array', storage_transformers=())
# zarr debug info
platform: macOS-15.6.1-arm64-arm-64bit
python: 3.11.13
zarr: 3.1.3
**Required dependencies:**
packaging: 25.0
numpy: 2.3.2
numcodecs: 0.16.2
typing_extensions: 4.14.1
donfig: 0.8.1.post1
**Optional dependencies:**
botocore: 1.39.11
fsspec: 2025.7.0
numcodecs: 0.16.2
s3fs: 2025.7.0
gcsfs: 2025.7.0
rich: 14.1.0
**Not Installed:**
cupy-cuda12x
universal-pathlib
obstoreGives us
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[5], [line 4](vscode-notebook-cell:?execution_count=5&line=4)
1 import zarr
3 arr = zarr.open("s3://wbc-test-data-sandbox/mosaics/8be69306cbe9.zarr")['variables']
----> [4](vscode-notebook-cell:?execution_count=5&line=4) d = arr[0:1, :, 4096:8192, 327680:331776]
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/array.py:2806, in Array.__getitem__(self, selection)
2804 return self.vindex[cast("CoordinateSelection | MaskSelection", selection)]
2805 elif is_pure_orthogonal_indexing(pure_selection, self.ndim):
-> [2806](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/array.py:2806) return self.get_orthogonal_selection(pure_selection, fields=fields)
2807 else:
2808 return self.get_basic_selection(cast("BasicSelection", pure_selection), fields=fields)
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/array.py:3245, in Array.get_orthogonal_selection(self, selection, out, fields, prototype)
3243 prototype = default_buffer_prototype()
3244 indexer = OrthogonalIndexer(selection, self.shape, self.metadata.chunk_grid)
-> [3245](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/array.py:3245) return sync(
3246 self._async_array._get_selection(
3247 indexer=indexer, out=out, fields=fields, prototype=prototype
3248 )
3249 )
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/sync.py:163, in sync(coro, loop, timeout)
160 return_result = next(iter(finished)).result()
162 if isinstance(return_result, BaseException):
--> [163](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/sync.py:163) raise return_result
164 else:
165 return return_result
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/sync.py:119, in _runner(coro)
114 """
115 Await a coroutine and return the result of running it. If awaiting the coroutine raises an
116 exception, the exception will be returned.
117 """
118 try:
--> [119](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/sync.py:119) return await coro
120 except Exception as ex:
121 return ex
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/array.py:1525, in AsyncArray._get_selection(self, indexer, prototype, out, fields)
1522 _config = replace(_config, order=self.order)
1524 # reading chunks and decoding them
-> [1525](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/array.py:1525) await self.codec_pipeline.read(
1526 [
1527 (
1528 self.store_path / self.metadata.encode_chunk_key(chunk_coords),
1529 self.metadata.get_chunk_spec(chunk_coords, _config, prototype=prototype),
1530 chunk_selection,
1531 out_selection,
1532 is_complete_chunk,
1533 )
1534 for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer
1535 ],
1536 out_buffer,
1537 drop_axes=indexer.drop_axes,
1538 )
1539 if isinstance(indexer, BasicIndexer) and indexer.shape == ():
1540 return out_buffer.as_scalar()
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/codec_pipeline.py:471, in BatchedCodecPipeline.read(self, batch_info, out, drop_axes)
465 async def read(
466 self,
467 batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]],
468 out: NDBuffer,
469 drop_axes: tuple[int, ...] = (),
470 ) -> None:
--> [471](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/codec_pipeline.py:471) await concurrent_map(
472 [
473 (single_batch_info, out, drop_axes)
474 for single_batch_info in batched(batch_info, self.batch_size)
475 ],
476 self.read_batch,
477 config.get("async.concurrency"),
478 )
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:[99](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:99), in concurrent_map(items, func, limit)
96 async with sem:
97 return await func(*item)
---> 99 return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:[97](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:97), in concurrent_map.<locals>.run(item)
95 async def run(item: tuple[Any]) -> V:
96 async with sem:
---> 97 return await func(*item)
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/codec_pipeline.py:275, in BatchedCodecPipeline.read_batch(self, batch_info, out, drop_axes)
269 else:
270 chunk_bytes_batch = await concurrent_map(
271 [(byte_getter, array_spec.prototype) for byte_getter, array_spec, *_ in batch_info],
272 lambda byte_getter, prototype: byte_getter.get(prototype),
273 config.get("async.concurrency"),
274 )
--> [275](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/codec_pipeline.py:275) chunk_array_batch = await self.decode_batch(
276 [
277 (chunk_bytes, chunk_spec)
278 for chunk_bytes, (_, chunk_spec, *_) in zip(
279 chunk_bytes_batch, batch_info, strict=False
280 )
281 ],
282 )
283 for chunk_array, (_, chunk_spec, chunk_selection, out_selection, _) in zip(
284 chunk_array_batch, batch_info, strict=False
285 ):
286 if chunk_array is not None:
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/codec_pipeline.py:195, in BatchedCodecPipeline.decode_batch(self, chunk_bytes_and_specs)
190 chunk_bytes_batch = await bb_codec.decode(
191 zip(chunk_bytes_batch, chunk_spec_batch, strict=False)
192 )
194 ab_codec, chunk_spec_batch = ab_codec_with_spec
--> [195](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/codec_pipeline.py:195) chunk_array_batch = await ab_codec.decode(
196 zip(chunk_bytes_batch, chunk_spec_batch, strict=False)
197 )
199 for aa_codec, chunk_spec_batch in aa_codecs_with_spec[::-1]:
200 chunk_array_batch = await aa_codec.decode(
201 zip(chunk_array_batch, chunk_spec_batch, strict=False)
202 )
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/abc/codec.py:159, in BaseCodec.decode(self, chunks_and_specs)
143 async def decode(
144 self,
145 chunks_and_specs: Iterable[tuple[CodecOutput | None, ArraySpec]],
146 ) -> Iterable[CodecInput | None]:
147 """Decodes a batch of chunks.
148 Chunks can be None in which case they are ignored by the codec.
149
(...) 157 Iterable[CodecInput | None]
158 """
--> [159](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/abc/codec.py:159) return await _batching_helper(self._decode_single, chunks_and_specs)
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/abc/codec.py:467, in _batching_helper(func, batch_info)
463 async def _batching_helper(
464 func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]],
465 batch_info: Iterable[tuple[CodecInput | None, ArraySpec]],
466 ) -> list[CodecOutput | None]:
--> [467](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/abc/codec.py:467) return await concurrent_map(
468 list(batch_info),
469 _noop_for_none(func),
470 config.get("async.concurrency"),
471 )
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:99, in concurrent_map(items, func, limit)
96 async with sem:
97 return await func(*item)
---> [99](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:99) return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:97, in concurrent_map.<locals>.run(item)
95 async def run(item: tuple[Any]) -> V:
96 async with sem:
---> [97](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/common.py:97) return await func(*item)
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/abc/codec.py:480, in _noop_for_none.<locals>.wrap(chunk, chunk_spec)
478 if chunk is None:
479 return None
--> [480](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/abc/codec.py:480) return await func(chunk, chunk_spec)
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/codecs/bytes.py:93, in BytesCodec._decode_single(self, chunk_bytes, chunk_spec)
91 # ensure correct chunk shape
92 if chunk_array.shape != chunk_spec.shape:
---> [93](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/codecs/bytes.py:93) chunk_array = chunk_array.reshape(
94 chunk_spec.shape,
95 )
96 return chunk_array
File ~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/buffer/core.py:500, in NDBuffer.reshape(self, newshape)
499 def reshape(self, newshape: tuple[int, ...] | Literal[-1]) -> Self:
--> [500](https://file+.vscode-resource.vscode-cdn.net/Users/len/code/wherobots-mosaics/~/code/wherobots-mosaics/.pixi/envs/dev/lib/python3.11/site-packages/zarr/core/buffer/core.py:500) return self.__class__(self._data.reshape(newshape))
ValueError: cannot reshape array of size 1048576 into shape (1,1,4096,4096)
Additional output
No response