Allow negative slices when indexing chunked data

Changelog.rst

@@ -1,3 +1,10 @@
+Version ?.?.?
+-------------
+
+**2026-??-??**
+
+* Allow negative slices when indexing chunked data by `David Hassell <https://github.com/davidhassell>`_ in https://github.com/NCAS-CMS/pyfive/pull/170
+
 Version 1.0.1
 -------------
 

pyfive/h5d.py

@@ -208,15 +208,18 @@ def get_data(self, args, fillvalue):
         if no_storage:
             return np.full(self.shape, fillvalue, dtype=self.dtype)[args]
 
-
     def iter_chunks(self, args):
-        """ 
-        Iterate over chunks in a chunked dataset. 
-        The optional sel argument is a slice or tuple of slices that defines the region to be used. 
-        If not set, the entire dataspace will be used for the iterator.
-        For each chunk within the given region, the iterator yields a tuple of slices that gives the
-        intersection of the given chunk with the selection area. 
-        This can be used to read data in that chunk.
+        """Iterate over chunks in a chunked dataset.
+
+        The args argument is a (possibly empty) sequence of indices
+        that defines the region to be used. If an empty sequence then
+        the entire dataspace will be used for the iterator.
+
+        For each chunk within the given region, the iterator yields a
+        tuple of indices that gives the intersection of the given
+        chunk with the selection area. This can be used to read data
+        in that chunk.
+
         """
         if not self.__chunk_init_check():
             return None
@@ -381,7 +384,6 @@ def _build_index(self):
         self.__index_built=True
 
     def _get_contiguous_data(self, args, fillvalue):
-
         if isinstance(self._ptype, P5ReferenceType):
             size = self._ptype.size
             if size != 8:

pyfive/indexing.py

@@ -47,10 +47,6 @@ class VindexInvalidSelectionError(_BaseZarrIndexError):
 class BoundsCheckError(_BaseZarrIndexError):
     _msg = "index out of bounds for dimension with length {0}"
 
-class NegativeStepError(IndexError):
-    def __init__(self):
-        super().__init__("only slices with step >= 1 are supported")
-
 # And the rest of the code is the original file.
 
 def is_integer(x):
@@ -207,8 +203,6 @@ class SliceDimIndexer:
     def __init__(self, dim_sel, dim_len, dim_chunk_len):
         # normalize
         self.start, self.stop, self.step = dim_sel.indices(dim_len)
-        if self.step < 1:
-            raise NegativeStepError()
 
         # store attributes
         self.dim_len = dim_len
@@ -631,6 +625,11 @@ def __init__(self, selection, array):
         # normalize list to array
         selection = replace_lists(selection)
 
+        # Reverse negative slices
+        selection, reverse_dims = replace_negative_slices(
+            selection, array._shape
+        )
+
         # setup per-dimension indexers
         dim_indexers = []
         for dim_sel, dim_len, dim_chunk_len in zip(selection, array._shape, array._chunks):
@@ -659,6 +658,8 @@ def __init__(self, selection, array):
         self.dim_indexers = dim_indexers
         self.shape = tuple(s.nitems for s in self.dim_indexers if not isinstance(s, IntDimIndexer))
         self.is_advanced = not is_basic_selection(selection)
+        self.reverse_dims = reverse_dims
+
         if self.is_advanced:
             self.drop_axes = tuple(
                 i
@@ -688,6 +689,34 @@ def __iter__(self):
                 if not is_basic_selection(out_selection):
                     out_selection = ix_(out_selection, self.shape)
 
+            if self.reverse_dims:
+                # A requested negative slice was changed to a positive
+                # slice (e.g. slice(7, 3, -1) -> slice(4, 8, 1)), so
+                # modify the index to the output array to ensure that
+                # the chunk selection goes in the correct place and in
+                # the correct order.
+                #
+                # E.g. For an output array axis of size 7:
+                #        slice(0, 1) -> slice(6, 5, -1)
+                #        slice(1, 4) -> slice(5, 2, -1)
+                #        slice(4, 7) -> slice(2, None, -1)
+                #
+                # Note that the step for the modified output array
+                # slice is always -1
+                out_selection = list(out_selection)
+                for i in self.reverse_dims:
+                    size = self.shape[i]
+                    start, stop, step = out_selection[i].indices(size)
+
+                    start = size - start - 1
+                    stop = size - stop - 1
+                    if stop < 0:
+                        stop = None
+
+                    out_selection[i] = slice(start, stop, -1)
+
+                out_selection = tuple(out_selection)
+
             yield ChunkProjection(chunk_coords, chunk_selection, out_selection)
 
 
@@ -1018,6 +1047,104 @@ def make_slice_selection(selection):
             ls.append(dim_selection)
     return ls
 
+def replace_negative_slices(selection, shape):
+    """Replace negative slices with positve slices.
+
+    A slice object that is decreasing (i.e. with a negative step), is
+    recast as an increasing slice (i.e. with a positive step. For
+    example ``slice(7,3,-1)`` would be cast as ``slice(4,8,1)``. This
+    is to facilitate finding which chunks are touched by the
+    index. The dimensions in the indexed array for which this has
+    occurred are returned so that they can be reversed later.
+
+    All other indices are unchanged.
+
+    Parameters
+    ----------
+    selection : numpy-style indices
+        Indices to an array of the given shape.
+
+    shape : tuple of `int`
+        The shape of the array.
+
+    Returns
+    -------
+    modified_selection : `tuple`
+        The reformatted indices that give the will correct subspace
+        after output dimensions have been reversed, as appropriate.
+
+    reverse_dims : `tuple`
+        The positions in the output array of dimensions that will need
+        reversing. The tuple wll be empty if there were no negative
+        slices.
+
+    """
+    indices = replace_ellipsis(selection, shape)
+
+    # Initialise outputs
+    modified_selection = []
+    reverse_dims = []
+
+    # Dimension positions of the array *after* indexing (some
+    # dimensions might get dropped by integer indices)
+    dim = 0
+
+    # Replace a negative slice index with its reversed version
+    for index, size in zip(selection, shape):
+        if isinstance(index, slice):
+            start, stop, step = index.indices(size)
+
+            if step < 0:
+                # When the slice step is negative, transform the
+                # original slice to a new slice with a positive step
+                # such that the result of the new slice is the reverse
+                # of the result of the original slice.
+                #
+                # For example, if the original slice is slice(6,0,-2)
+                # then the new slice will be slice(2,7,2):
+                #
+                # >>> a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+                # >>> a[slice(6, 0, -2)]
+                # [6, 4, 2]
+                # >>> a[slice(2, 7, 2)]
+                # [2, 4, 6]
+                # >>> a[slice(6, 0, -2)] == list(reversed(a[slice(2, 7, 2)]))
+                # True
+                if stop == -1:
+                    stop = None
+
+                index = slice(start, stop, step)
+                start, stop, step = index.indices(size)
+
+                step *= -1
+                div, mod = divmod(start - stop - 1, step)
+                div_step = div * step
+                start -= div_step
+                stop = start + div_step + 1
+
+                index = slice(start, stop, step)
+                reverse_dims.append(dim)
+
+        elif isinstance(index, int):
+            dim -= 1
+
+        elif isinstance(index, np.ndarray):
+           if not index.ndim:
+               dim -= 1
+
+        elif index == Ellipsis:
+            raise ValueError(
+                "replace_negative_slices(selection, shape) doesn't work "
+                "when selection contains Ellipsis. Consider doing "
+                "selection=pyfive.indexing.replace_ellipsis(selection, shape) "
+                "first"
+            )
+
+        dim += 1
+
+        modified_selection.append(index)
+
+    return tuple(modified_selection), tuple(reverse_dims)
 
 class PartialChunkIterator:
     """Iterator to retrieve the specific coordinates of requested data

tests/test_dataset_indexing.py

@@ -0,0 +1,62 @@
+""" Test pyfive's abililty to index Datasets."""
+import os
+
+import numpy as np
+from numpy.testing import assert_array_equal
+import pytest
+
+import pyfive
+
+DIRNAME = os.path.dirname(__file__)
+DATASET_CHUNKED_FILE = os.path.join(DIRNAME, 'data/chunked.hdf5')
+
+# Define a fixture that opens the chunked file once
+@pytest.fixture(scope="module")
+def chunked_file():
+    with pyfive.File(DATASET_CHUNKED_FILE) as hfile:
+        yield hfile
+
+@pytest.mark.parametrize(
+    "index", [
+        (slice(None, None, -1), slice(None, None, -1)),
+        (slice(None), slice(1000, 1, -2)),
+        (..., slice(None, None, -2)),
+        (slice(None, None, -2), ...),
+        (0, slice(-1, None, -3)),
+        (slice(-1, None, -3), 0),
+     ]
+)
+def test_dataset_indexing_chunked_negative_step_slices(chunked_file, index):
+    """Test orthogonal chunked indexing with negative step slices"""
+    d = chunked_file['dataset1']
+    assert d.shape == (21, 16)
+
+    array = d[...]
+    assert array.shape == d.shape
+
+    assert_array_equal(d[index], array[index])
+
+def test_dataset_indexing_replace_negative_slices():
+    """Test pyfive.indexing.replace_negative_slices"""
+    func = pyfive.indexing.replace_negative_slices
+
+    r = list(range(8))
+    assert func((0, slice(6, 0, -2)), (7, 8, 9)) == (
+        (0, slice(2, 7, 2)), (0,)
+    )
+    assert r[slice(6, 0, -2)] == list(reversed(r[slice(2, 7, 2)]))
+
+    assert func(([1, 2], slice(600, 1, -3), 0), (7, 8, 9)) == (
+        ([1, 2], slice(4, 8, 3), 0), (1,)
+    )
+    assert r[slice(600, None, -3)] == list(reversed(r[slice(1, 8, 3)]))
+
+    r = list(range(9))
+    assert func((0, slice(None), slice(6, 0, -2)), (7, 8, 9)) == (
+        (0, slice(None), slice(2, 7, 2)), (1,)
+    )
+    assert r[slice(6, 0, -2)] == list(reversed(r[slice(2, 7, 2)]))
+
+    # Can't pass in in Ellipsis
+    with pytest.raises(ValueError) as error:
+       func((0, slice(6, 0, -2), ...), (7, 8, 9))