[Diff PR] Sharding storage transformer for v3

.github/workflows/minimal.yml

@@ -24,6 +24,7 @@ jobs:
       shell: "bash -l {0}"
       env:
         ZARR_V3_EXPERIMENTAL_API: 1
+        ZARR_V3_SHARDING: 1
       run: |
         conda activate minimal
         python -m pip install .
@@ -32,6 +33,7 @@ jobs:
       shell: "bash -l {0}"
       env:
         ZARR_V3_EXPERIMENTAL_API: 1
+        ZARR_V3_SHARDING: 1
       run: |
         conda activate minimal
         rm -rf fixture/

.github/workflows/python-package.yml

@@ -70,6 +70,7 @@ jobs:
         ZARR_TEST_MONGO: 1
         ZARR_TEST_REDIS: 1
         ZARR_V3_EXPERIMENTAL_API: 1
+        ZARR_V3_SHARDING: 1
       run: |
         conda activate zarr-env
         mkdir ~/blob_emulator

.github/workflows/windows-testing.yml

@@ -52,6 +52,7 @@ jobs:
         env:
           ZARR_TEST_ABS: 1
           ZARR_V3_EXPERIMENTAL_API: 1
+          ZARR_V3_SHARDING: 1
       - name: Conda info
         shell: bash -l {0}
         run: conda info

docs/release.rst

@@ -18,7 +18,9 @@ Unreleased
   Add two features of the [v3 spec](https://zarr-specs.readthedocs.io/en/latest/core/v3.0.html):
   * storage transformers
   * `get_partial_values` and `set_partial_values`
-  By :user:`Jonathan Striebel <jstriebel>`; :issue:`1096`.
+  * efficient `get_partial_values` implementation for `FSStoreV3`
+  * sharding storage transformer
+  By :user:`Jonathan Striebel <jstriebel>`; :issue:`1096`, :issue:`1111`.
 
 .. _release_2.13.6:
 

zarr/_storage/v3.py

@@ -182,6 +182,35 @@ def rmdir(self, path=None):
             if self.fs.isdir(store_path):
                 self.fs.rm(store_path, recursive=True)
 
+    @property
+    def supports_efficient_get_partial_values(self):
+        return True
+
+    def get_partial_values(self, key_ranges):
+        """Get multiple partial values.
+        key_ranges can be an iterable of key, range pairs,
+        where a range specifies two integers range_start and range_length
+        as a tuple, (range_start, range_length).
+        range_length may be None to indicate to read until the end.
+        range_start may be negative to start reading range_start bytes
+        from the end of the file.
+        A key may occur multiple times with different ranges.
+        Inserts None for missing keys into the returned list."""
+        results = []
+        for key, (range_start, range_length) in key_ranges:
+            key = self._normalize_key(key)
+            path = self.dir_path(key)
+            try:
+                if range_start is None or range_length is None:
+                    end = None
+                else:
+                    end = range_start + range_length
+                result = self.fs.cat_file(path, start=range_start, end=end)
+            except self.map.missing_exceptions:
+                result = None
+            results.append(result)
+        return results
+
 
 class MemoryStoreV3(MemoryStore, StoreV3):