Skip to content

Commit 4bec665

Browse files
committed
Only use concurrent_map over chunks within a shard if > 1 groups after coalescing
1 parent c393e41 commit 4bec665

File tree

2 files changed

+446
-11
lines changed

2 files changed

+446
-11
lines changed

src/zarr/codecs/sharding.py

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -737,17 +737,23 @@ async def _load_partial_shard_maybe(
737737

738738
groups = self._coalesce_chunks(chunks, max_gap_bytes, coalesce_max_bytes)
739739

740-
shard_dicts = await concurrent_map(
741-
[(group, byte_getter, prototype) for group in groups],
742-
self._get_group_bytes,
743-
async_concurrency,
744-
)
745-
746740
shard_dict: ShardMutableMapping = {}
747-
for d in shard_dicts:
741+
if len(groups) == 1:
742+
# Avoid thread start overhead when there's only one group
743+
shard_dict_result = await self._get_group_bytes(groups[0], byte_getter, prototype)
748744
# can be None if the ByteGetter returned None when reading chunk data
749-
if d is not None:
750-
shard_dict.update(d)
745+
if shard_dict_result is not None:
746+
shard_dict.update(shard_dict_result)
747+
else:
748+
shard_dicts = await concurrent_map(
749+
[(group, byte_getter, prototype) for group in groups],
750+
self._get_group_bytes,
751+
async_concurrency,
752+
)
753+
754+
for shard_dict_result in shard_dicts:
755+
if shard_dict_result is not None:
756+
shard_dict.update(shard_dict_result)
751757

752758
return shard_dict
753759

0 commit comments

Comments
 (0)