From d16a773f7ab918a6a2775818b302e564b06183ae Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 5 Jan 2026 11:47:14 +0000 Subject: [PATCH 1/4] Add __orthogonal_indexing__ flag --- pyfive/high_level.py | 7 +++++++ tests/test_dataset_indexing.py | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/pyfive/high_level.py b/pyfive/high_level.py index 5eec4034..76b93f6d 100644 --- a/pyfive/high_level.py +++ b/pyfive/high_level.py @@ -490,6 +490,13 @@ def attrs(self): """ attrs attribute. """ return self.id._meta.attributes + @property + def __orthogonal_indexing__(self): + """Flag to indicate whether indexing is orthogonal. + + """ + return self.id.chunks is not None + class DimensionManager(Sequence): """ Represents a collection of dimensions associated with a dataset. """ diff --git a/tests/test_dataset_indexing.py b/tests/test_dataset_indexing.py index 0c79d271..e2fee1b5 100644 --- a/tests/test_dataset_indexing.py +++ b/tests/test_dataset_indexing.py @@ -9,6 +9,7 @@ DIRNAME = os.path.dirname(__file__) DATASET_CHUNKED_FILE = os.path.join(DIRNAME, 'data/chunked.hdf5') +DATASET_CONTIGUOUS_FILE = os.path.join(DIRNAME, 'data/dataset_multidim.hdf5') # Define a fixture that opens the chunked file once @pytest.fixture(scope="module") @@ -16,6 +17,12 @@ def chunked_file(): with pyfive.File(DATASET_CHUNKED_FILE) as hfile: yield hfile +# Define a fixture that opens the contiguous file once +@pytest.fixture(scope="module") +def contiguous_file(): + with pyfive.File(DATASET_CONTIGUOUS_FILE) as hfile: + yield hfile + @pytest.mark.parametrize( "index", [ (slice(None, None, -1), slice(None, None, -1)), @@ -60,3 +67,7 @@ def test_dataset_indexing_replace_negative_slices(): # Can't pass in in Ellipsis with pytest.raises(ValueError) as error: func((0, slice(6, 0, -2), ...), (7, 8, 9)) + +def test_dataset_orthogonal_indexing(chunked_file, contiguous_file): + assert chunked_file['dataset1'].__orthogonal_indexing__ is True + assert contiguous_file['d'].__orthogonal_indexing__ is False From b04ee1245b38f89510662c055f0dd3a0a758e725 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 16 Jan 2026 12:21:27 +0000 Subject: [PATCH 2/4] expand if test --- pyfive/high_level.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pyfive/high_level.py b/pyfive/high_level.py index 9fb64597..2befdbca 100644 --- a/pyfive/high_level.py +++ b/pyfive/high_level.py @@ -16,6 +16,7 @@ from pyfive.dataobjects import DataObjects, DatasetID from pyfive.misc_low_level import SuperBlock from pyfive.h5py import Datatype +from pyfive.p5t import P5VlenStringType, P5ReferenceType, P5SequenceType class Group(Mapping): @@ -506,9 +507,26 @@ def attrs(self): def __orthogonal_indexing__(self): """Flag to indicate whether indexing is orthogonal. - """ - return self.id.chunks is not None + In general, the flag will be `True` if either the data is + contiguous and memory mapped access is not available, or else + the data is chunked. Otherwise the flag will be `False`. + """ + if self.id.chunks is not None: + # Chunked data + return True + + if not (isinstance(self.id._ptype, (P5ReferenceType, P5VlenStringType, P5SequenceType))) and not self.id.posix: + # Contiguous data accessed with pseudo chunking via + # `DatasetID._get_direct_from_contiguous`. + # + # This test is based on the logic of + # `DatasetID._get_contiguous_data`, which decides whether + # or not to call `DatasetID._get_direct_from_contiguous`. + return True + + # All other cases + return False class DimensionManager(Sequence): """Represents a collection of dimensions associated with a dataset.""" From 569c54a259eeb56b2799360f23bd7ea3e30de088 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 22 Jan 2026 13:30:32 +0000 Subject: [PATCH 3/4] linting --- pyfive/high_level.py | 21 ++++++++++++-------- pyfive/indexing.py | 10 +++++----- tests/test_dataset_indexing.py | 35 ++++++++++++++++++++-------------- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/pyfive/high_level.py b/pyfive/high_level.py index 2befdbca..847c35f3 100644 --- a/pyfive/high_level.py +++ b/pyfive/high_level.py @@ -513,21 +513,26 @@ def __orthogonal_indexing__(self): """ if self.id.chunks is not None: - # Chunked data + # Chunked data indexed with + # `DatasetID._get_selection_via_chunks` return True - if not (isinstance(self.id._ptype, (P5ReferenceType, P5VlenStringType, P5SequenceType))) and not self.id.posix: - # Contiguous data accessed with pseudo chunking via - # `DatasetID._get_direct_from_contiguous`. - # - # This test is based on the logic of - # `DatasetID._get_contiguous_data`, which decides whether - # or not to call `DatasetID._get_direct_from_contiguous`. + if ( + not ( + isinstance( + self.id._ptype, (P5ReferenceType, P5VlenStringType, P5SequenceType) + ) + ) + and not self.id.posix + ): + # Contiguous data indexed with + # `DatasetID._get_direct_from_contiguous` return True # All other cases return False + class DimensionManager(Sequence): """Represents a collection of dimensions associated with a dataset.""" diff --git a/pyfive/indexing.py b/pyfive/indexing.py index ae703c5f..b0d6c4bc 100644 --- a/pyfive/indexing.py +++ b/pyfive/indexing.py @@ -655,9 +655,7 @@ def __init__(self, selection, array): selection = replace_lists(selection) # Reverse negative slices - selection, reverse_dims = replace_negative_slices( - selection, array._shape - ) + selection, reverse_dims = replace_negative_slices(selection, array._shape) # setup per-dimension indexers dim_indexers = [] @@ -1092,6 +1090,7 @@ def make_slice_selection(selection): ls.append(dim_selection) return ls + def replace_negative_slices(selection, shape): """Replace negative slices with positve slices. @@ -1174,8 +1173,8 @@ def replace_negative_slices(selection, shape): dim -= 1 elif isinstance(index, np.ndarray): - if not index.ndim: - dim -= 1 + if not index.ndim: + dim -= 1 elif index == Ellipsis: raise ValueError( @@ -1191,6 +1190,7 @@ def replace_negative_slices(selection, shape): return tuple(modified_selection), tuple(reverse_dims) + class PartialChunkIterator: """Iterator to retrieve the specific coordinates of requested data from within a compressed chunk. diff --git a/tests/test_dataset_indexing.py b/tests/test_dataset_indexing.py index e2fee1b5..63e4c73a 100644 --- a/tests/test_dataset_indexing.py +++ b/tests/test_dataset_indexing.py @@ -1,4 +1,5 @@ -""" Test pyfive's abililty to index Datasets.""" +"""Test pyfive's abililty to index Datasets.""" + import os import numpy as np @@ -8,8 +9,9 @@ import pyfive DIRNAME = os.path.dirname(__file__) -DATASET_CHUNKED_FILE = os.path.join(DIRNAME, 'data/chunked.hdf5') -DATASET_CONTIGUOUS_FILE = os.path.join(DIRNAME, 'data/dataset_multidim.hdf5') +DATASET_CHUNKED_FILE = os.path.join(DIRNAME, "data/chunked.hdf5") +DATASET_CONTIGUOUS_FILE = os.path.join(DIRNAME, "data/dataset_multidim.hdf5") + # Define a fixture that opens the chunked file once @pytest.fixture(scope="module") @@ -17,25 +19,28 @@ def chunked_file(): with pyfive.File(DATASET_CHUNKED_FILE) as hfile: yield hfile + # Define a fixture that opens the contiguous file once @pytest.fixture(scope="module") def contiguous_file(): with pyfive.File(DATASET_CONTIGUOUS_FILE) as hfile: yield hfile + @pytest.mark.parametrize( - "index", [ + "index", + [ (slice(None, None, -1), slice(None, None, -1)), (slice(None), slice(1000, 1, -2)), (..., slice(None, None, -2)), (slice(None, None, -2), ...), (0, slice(-1, None, -3)), (slice(-1, None, -3), 0), - ] + ], ) def test_dataset_indexing_chunked_negative_step_slices(chunked_file, index): """Test orthogonal chunked indexing with negative step slices""" - d = chunked_file['dataset1'] + d = chunked_file["dataset1"] assert d.shape == (21, 16) array = d[...] @@ -43,31 +48,33 @@ def test_dataset_indexing_chunked_negative_step_slices(chunked_file, index): assert_array_equal(d[index], array[index]) + def test_dataset_indexing_replace_negative_slices(): """Test pyfive.indexing.replace_negative_slices""" func = pyfive.indexing.replace_negative_slices r = list(range(8)) - assert func((0, slice(6, 0, -2)), (7, 8, 9)) == ( - (0, slice(2, 7, 2)), (0,) - ) + assert func((0, slice(6, 0, -2)), (7, 8, 9)) == ((0, slice(2, 7, 2)), (0,)) assert r[slice(6, 0, -2)] == list(reversed(r[slice(2, 7, 2)])) assert func(([1, 2], slice(600, 1, -3), 0), (7, 8, 9)) == ( - ([1, 2], slice(4, 8, 3), 0), (1,) + ([1, 2], slice(4, 8, 3), 0), + (1,), ) assert r[slice(600, None, -3)] == list(reversed(r[slice(1, 8, 3)])) r = list(range(9)) assert func((0, slice(None), slice(6, 0, -2)), (7, 8, 9)) == ( - (0, slice(None), slice(2, 7, 2)), (1,) + (0, slice(None), slice(2, 7, 2)), + (1,), ) assert r[slice(6, 0, -2)] == list(reversed(r[slice(2, 7, 2)])) # Can't pass in in Ellipsis with pytest.raises(ValueError) as error: - func((0, slice(6, 0, -2), ...), (7, 8, 9)) + func((0, slice(6, 0, -2), ...), (7, 8, 9)) + def test_dataset_orthogonal_indexing(chunked_file, contiguous_file): - assert chunked_file['dataset1'].__orthogonal_indexing__ is True - assert contiguous_file['d'].__orthogonal_indexing__ is False + assert chunked_file["dataset1"].__orthogonal_indexing__ is True + assert contiguous_file["d"].__orthogonal_indexing__ is False From 08827b60d3b4dc54839331ca7473cf31735bf6ba Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 23 Jan 2026 12:46:55 +0000 Subject: [PATCH 4/4] dev --- Changelog.rst | 1 + pyfive/high_level.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index cfb32e6c..b195f0fc 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -3,6 +3,7 @@ Version ?.?.? **2026-??-??** +* Add ``__orthogonal_indexing__`` flag by `David Hassell `_ in https://github.com/NCAS-CMS/pyfive/issue/171 * Allow negative slices when indexing chunked data by `David Hassell `_ in https://github.com/NCAS-CMS/pyfive/pull/170 Version 1.0.1 diff --git a/pyfive/high_level.py b/pyfive/high_level.py index 847c35f3..1ab5e906 100644 --- a/pyfive/high_level.py +++ b/pyfive/high_level.py @@ -507,9 +507,11 @@ def attrs(self): def __orthogonal_indexing__(self): """Flag to indicate whether indexing is orthogonal. - In general, the flag will be `True` if either the data is - contiguous and memory mapped access is not available, or else - the data is chunked. Otherwise the flag will be `False`. + In general, the flag will be `True` if: + + * The data is chunked. + * The data is contiguous and memory mapped access is not + available. """ if self.id.chunks is not None: