From 0c931a4f302ed1750c5505f6c3f3124e33a0b84c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Wed, 1 Oct 2025 13:44:35 +0200 Subject: [PATCH 1/3] pipe VLEN_SEQUENCE through Datatype, add tests for vlen types and datasets,fix Datatype issues by enabling generic TypeID and TypeCompoundID, add many tests --- pyfive/dataobjects.py | 2 + pyfive/datatype_msg.py | 2 +- pyfive/h5d.py | 2 + pyfive/h5py.py | 14 ++- pyfive/h5t.py | 67 +++++++++++- tests/make_dataset_datatypes_file.py | 146 ++++++++++++++++++--------- tests/make_enum_file.py | 44 +++++--- tests/test_dataset_datatypes.py | 75 ++++++++++++++ tests/test_enum_var.py | 37 +++++++ 9 files changed, 321 insertions(+), 68 deletions(-) diff --git a/pyfive/dataobjects.py b/pyfive/dataobjects.py index ca601189..1388904f 100644 --- a/pyfive/dataobjects.py +++ b/pyfive/dataobjects.py @@ -262,6 +262,8 @@ def _attr_value(self, dtype, buf, count, offset): if isinstance(dtype, tuple): if dtype[0] == "ENUMERATION": dtype = np.dtype(dtype[1], metadata={'enum': dtype[2]}) + elif dtype[0] == "COMPOUND": + dtype = np.dtype(dtype[1]) if isinstance(dtype, tuple): dtype_class = dtype[0] diff --git a/pyfive/datatype_msg.py b/pyfive/datatype_msg.py index 9571ae83..a2d7aa5b 100644 --- a/pyfive/datatype_msg.py +++ b/pyfive/datatype_msg.py @@ -157,7 +157,7 @@ def _determine_dtype_compound(self, datatype_msg): prop2['dim_size_4'] == 0 ) if names_valid and dtypes_valid and offsets_valid and props_valid: - return complex_dtype_map[dtype1] + return "COMPOUND", complex_dtype_map[dtype1] raise NotImplementedError("Compound dtype not supported.") diff --git a/pyfive/h5d.py b/pyfive/h5d.py index 08db0645..e19a36cc 100644 --- a/pyfive/h5d.py +++ b/pyfive/h5d.py @@ -87,6 +87,8 @@ def __init__(self, dataobject, pseudo_chunking_size_MB=4): if isinstance(dataobject.dtype,tuple): if dataobject.dtype[0] == 'ENUMERATION': self._dtype = np.dtype(dataobject.dtype[1], metadata={'enum':dataobject.dtype[2]}) + elif dataobject.dtype[0] == 'COMPOUND': + self._dtype = np.dtype(dataobject.dtype[1]) else: self._dtype = dataobject.dtype else: diff --git a/pyfive/h5py.py b/pyfive/h5py.py index 3ff627fb..bb7a32dd 100644 --- a/pyfive/h5py.py +++ b/pyfive/h5py.py @@ -4,7 +4,7 @@ from pyfive.datatype_msg import DatatypeMessage -from pyfive.h5t import TypeEnumID +from pyfive.h5t import TypeID, TypeEnumID, TypeCompoundID import numpy as np from pathlib import PurePosixPath @@ -15,7 +15,17 @@ class Datatype: suitable for use with enumerations. """ def __init__(self, name, hfile, raw_dtype): - self.id = TypeEnumID(raw_dtype) + id = raw_dtype + if isinstance(raw_dtype, tuple): + if raw_dtype[0] == "ENUMERATION": + id = TypeEnumID(raw_dtype[1:]) + elif raw_dtype[0] == "COMPOUND": + id = TypeCompoundID(raw_dtype[1]) + elif raw_dtype[0] == "VLEN_SEQUENCE": + id = TypeID(raw_dtype[1]) + else: + id = TypeID(id) + self.id = id path = PurePosixPath(name) self.name = path.name self.parent = str(path.parent) if str(path.parent) != '' else '/' diff --git a/pyfive/h5t.py b/pyfive/h5t.py index 68a66da1..e03bdb30 100644 --- a/pyfive/h5t.py +++ b/pyfive/h5t.py @@ -74,6 +74,36 @@ def check_dtype(**kwds): else: return None + +class TypeID: + """ + Used by DataType to expose internal structure of a generic + datatype. This is instantiated by pyfive using arcane + hdf5 structure information, and should not normally be + needed by any user code. + """ + def __init__(self, raw_dtype): + """ + Initialised with the raw_dtype read from the message. + This is not the same init signature as h5py! + """ + super().__init__() + dtype = raw_dtype + self.kind = dtype.replace('<', '|') + + def __eq__(self, other): + if type(self) != type(other): + return False + return self.dtype == other.dtype + + @property + def dtype(self): + """ + The numpy dtype. + """ + return np.dtype(self.kind) + + class TypeEnumID: """ Used by DataType to expose internal structure of an enum @@ -87,10 +117,11 @@ def __init__(self, raw_dtype): This is not the same init signature as h5py! """ super().__init__() - enum, dtype, enumdict = raw_dtype + dtype, enumdict = raw_dtype self.metadata = {'enum':enumdict} self.__reversed = None self.kind = dtype.replace('<','|') + def enum_valueof(self, name): """ Get the value associated with an enum name. @@ -105,10 +136,12 @@ def enum_nameof(self, index): Determine the name associated with the given value. """ return self.__reversed[index] + def __eq__(self, other): if type(self) != type(other): return False return self.metadata == other.metadata + @property def dtype(self): """ @@ -121,5 +154,35 @@ def dtype(self): x = my_datatype.id.dtype enum_dict = x.metadata """ - return np.dtype(self.kind,metadata=self.metadata) + return np.dtype(self.kind, metadata=self.metadata) + + +class TypeCompoundID: + """ + Used by DataType to expose internal structure of a compound + datatype. This is instantiated by pyfive using arcane + hdf5 structure information, and should not normally be + needed by any user code. + """ + + def __init__(self, raw_dtype): + """ + Initialised with the raw_dtype read from the message. + This is not the same init signature as h5py! + """ + super().__init__() + dtype = raw_dtype + self.kind = dtype.replace('<', '|') + + def __eq__(self, other): + if type(self) != type(other): + return False + return self.dtype == other.dtype + + @property + def dtype(self): + """ + The numpy dtype. + """ + return np.dtype(self.kind) diff --git a/tests/make_dataset_datatypes_file.py b/tests/make_dataset_datatypes_file.py index 7ee57377..a105af78 100644 --- a/tests/make_dataset_datatypes_file.py +++ b/tests/make_dataset_datatypes_file.py @@ -1,55 +1,101 @@ #! /usr/bin/env python """ Create a HDF5 file with datasets of many datatypes . """ +import sys import h5py import numpy as np +from pathlib import Path -f = h5py.File('dataset_datatypes.hdf5', 'w') - -# signed intergers -common_signed_args = { - 'shape': (4, ), - 'data': -np.arange(4), - 'track_times': False, -} - -f.create_dataset('int08_little', dtype='c8') + f["complex128_little_type"] = np.dtype('c16') + + f.create_dataset('complex64_little', dtype='"]: + for base in ["i", "u", "f"]: + for width in ["1", "2", "4", "8"]: + if base == "f" and width == "1": + continue + tstr = "".join([endian, base, width]) + dtype = h5py.vlen_dtype(np.dtype(tstr)) + f[f"vlen_{tstr}_type"] = dtype + ds = f.create_dataset(f"vlen_{tstr}", (4,), dtype=dtype) + ds[0] = [0] + ds[1] = [0, 1] + ds[2] = [0, 1, 2] + ds[3] = [0, 1, 2, 3] + + +if __name__ == "__main__": + default_path = Path(__file__).parent / "dataset_datatypes.hdf5" + filepath = Path(sys.argv[1]) if len(sys.argv) > 1 else default_path + create_file(filepath) \ No newline at end of file diff --git a/tests/make_enum_file.py b/tests/make_enum_file.py index d20e5826..9b7e94c5 100644 --- a/tests/make_enum_file.py +++ b/tests/make_enum_file.py @@ -3,9 +3,11 @@ (1) the netcdf interface, and (2) the h5py interface """ -from netCDF4 import Dataset +import sys +import netCDF4 import h5py import numpy as np +from pathlib import Path clouds = ['stratus','stratus','missing','nimbus','cumulus','longcloudname'] selection = ['stratus','nimbus','missing','nimbus','longcloudname'] @@ -13,18 +15,34 @@ enum_dict['missing'] = 255 data = [enum_dict[k] for k in selection] -ncd = Dataset('enum_variable.nc','w') -enum_type = ncd.createEnumType(np.uint8,'enum_t', enum_dict) +def create_nc_file(path): + with netCDF4.Dataset(path, mode='w') as ncd: + enum_type = ncd.createEnumType(np.uint8,'enum_t', enum_dict) -dim = ncd.createDimension('axis',5) -enum_var = ncd.createVariable('enum_var',enum_type,'axis', - fill_value=enum_dict['missing']) -enum_var[:] = data -ncd.close() + # add types for checking comparison + enum_type2 = ncd.createEnumType(np.uint8,'enum2_t', enum_dict) + vlen_t = ncd.createVLType(np.int32, "phony_vlen") -hcd = h5py.File('enum_variable.hdf5','w') -dt = h5py.enum_dtype(enum_dict,basetype='i') -assert h5py.check_enum_dtype(dt) == enum_dict -ds = hcd.create_dataset('enum_var', data=data, dtype=dt) -hcd.close() + dim = ncd.createDimension('axis',5) + enum_var = ncd.createVariable('enum_var',enum_type,'axis', + fill_value=enum_dict['missing']) + enum_var[:] = data + +def create_hdf_file(path): + with h5py.File(path,'w') as hcd: + dt = h5py.enum_dtype(enum_dict, basetype='i') + assert h5py.check_enum_dtype(dt) == enum_dict + ds = hcd.create_dataset('enum_var', data=data, dtype=dt) + + +if __name__ == "__main__": + default_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).parent + if len(sys.argv) == 1: + create_nc_file(default_path / 'enum_variable.nc' ) + create_hdf_file(default_path / 'enum_variable.hdf5') + else: + if default_path.suffix == ".hdf5": + create_hdf_file(default_path) + else: + create_nc_file(default_path) diff --git a/tests/test_dataset_datatypes.py b/tests/test_dataset_datatypes.py index 5d7d6792..ce2bd728 100644 --- a/tests/test_dataset_datatypes.py +++ b/tests/test_dataset_datatypes.py @@ -1,13 +1,25 @@ """ Unit tests for pyfive. """ import os +import sys +import subprocess import numpy as np from numpy.testing import assert_array_equal +import pytest import pyfive DIRNAME = os.path.dirname(__file__) DATASET_DATATYPES_HDF5_FILE = os.path.join(DIRNAME, 'dataset_datatypes.hdf5') +MAKE_DATASET_DATATYPES_SCRIPT = os.path.join(DIRNAME, 'make_dataset_datatypes.py') + + +@pytest.fixture(scope="module") +def dataset_datatypes_hdf5(tmp_path_factory): + tmp_dir = tmp_path_factory.mktemp("dataset_datatypes") + path = tmp_dir / "dataset_datatypes.hdf5" + subprocess.run([sys.executable, MAKE_DATASET_DATATYPES_SCRIPT, str(path)], check=True) + return str(path) def test_signed_int_dataset_datatypes(): @@ -38,6 +50,15 @@ def test_signed_int_dataset_datatypes(): assert hfile['int64_big'].dtype == np.dtype('>i8') +def test_signed_int_datatypes(dataset_datatypes_hdf5): + + with pyfive.File(dataset_datatypes_hdf5) as hfile: + + assert isinstance(hfile['int08_little_type'].id, pyfive.h5t.TypeID) + assert hfile['int08_little_type'].id == hfile['int08_little_type2'].id + assert hfile['int08_little_type'].id != hfile['complex64_little_type'].id + + def test_unsigned_int_dataset_datatypes(): with pyfive.File(DATASET_DATATYPES_HDF5_FILE) as hfile: @@ -84,3 +105,57 @@ def test_float_dataset_datatypes(): assert hfile['float32_big'].dtype == np.dtype('>f4') assert hfile['float64_big'].dtype == np.dtype('>f8') + + +def test_complex_dataset_datatypes(dataset_datatypes_hdf5): + + with pyfive.File(dataset_datatypes_hdf5) as hfile: + ref_data = 123+456.j + + assert isinstance(hfile['complex64_little_type'].id, pyfive.h5t.TypeCompoundID) + assert hfile['complex64_little_type'].id == hfile['complex64_little_type2'].id + assert hfile['complex64_little_type'].id != hfile['int08_little_type'].id + + assert hfile["complex64_little_type"].dtype == np.dtype("c8") + assert hfile["complex128_little_type"].dtype == np.dtype("c16") + + assert_array_equal(hfile['complex64_little'][:], ref_data) + assert_array_equal(hfile['complex128_little'][:], ref_data) + + assert_array_equal(hfile['complex64_big'][:], ref_data) + assert_array_equal(hfile['complex128_big'][:], ref_data) + + # check dtype + assert hfile['complex64_little'].dtype == np.dtype('c8') + assert hfile['complex128_big'].dtype == np.dtype('>c16') + + +@pytest.mark.parametrize("base", ["i", "u", "f"]) +@pytest.mark.parametrize("width", ["1", "2", "4", "8"]) +@pytest.mark.parametrize("endian", ["<", ">"]) +def test_vlen_dataset_datatypes(dataset_datatypes_hdf5, base, width, endian): + if base == "f" and width == "1": + pytest.skip("no 1 byte floats") + tstr = "".join([endian, base, width]) + dtype = np.dtype(tstr) + ref_data = np.empty(4, dtype=object) + ref_data[0] = np.array([0], dtype) + ref_data[1] = np.array([0, 1], dtype) + ref_data[2] = np.array([0, 1, 2], dtype) + ref_data[3] = np.array([0, 1, 2, 3], dtype) + + with pyfive.File(dataset_datatypes_hdf5) as hfile: + + assert hfile[f"vlen_{tstr}_type"].dtype == dtype + assert isinstance(hfile[f"vlen_{tstr}_type"].id, pyfive.h5t.TypeID) + + # vlen sequence isn't implemented yet + with pytest.raises(NotImplementedError, match="datatype not implemented - VLEN_SEQUENCE"): + assert_array_equal(hfile[f"vlen_{tstr}"][:], ref_data) + + diff --git a/tests/test_enum_var.py b/tests/test_enum_var.py index 9e2e0f28..8e27a61c 100644 --- a/tests/test_enum_var.py +++ b/tests/test_enum_var.py @@ -1,6 +1,8 @@ """ Unit tests for pyfive dealing with an enum variable """ import os +import sys +import subprocess import pytest import h5py import numpy as np @@ -10,6 +12,16 @@ DIRNAME = os.path.dirname(__file__) ENUMVAR_NC_FILE = os.path.join(DIRNAME, 'enum_variable.nc') ENUMVAR_H5_FILE = os.path.join(DIRNAME, 'enum_variable.hdf5') +MAKE_ENUM_VARIABLE_SCRIPT = os.path.join(DIRNAME, 'make_enum_file.py') + + +@pytest.fixture(scope="module") +def enum_variable_nc(tmp_path_factory): + tmp_dir = tmp_path_factory.mktemp("enum_var") + path = tmp_dir / "enum_variable.nc" + subprocess.run([sys.executable, MAKE_ENUM_VARIABLE_SCRIPT, str(path)], check=True) + return str(path) + def test_read_h5enum_variable(): @@ -54,6 +66,7 @@ def test_enum_dict(): assert np.array_equal(h5_evar[:], p5_evar[:]), "Enum stored values do not match" assert p5_edict == h5_edict, "Enum dictionaries do not match" + def test_enum_datatype(): with h5py.File(ENUMVAR_NC_FILE, 'r') as hfile: @@ -75,6 +88,30 @@ def test_enum_datatype(): assert h5_enum_t.dtype == p5_enum_t.dtype +def test_enum_datatype2(enum_variable_nc): + + with h5py.File(enum_variable_nc, 'r') as hfile: + + h5_enum_t = hfile['enum_t'] + + with pyfive.File(enum_variable_nc) as pfile: + p5_enum_t = pfile['enum_t'] + + assert str(h5_enum_t) == str(p5_enum_t) + + assert p5_enum_t.id.enum_valueof('stratus') == 1 + assert p5_enum_t.id.enum_nameof(1) == 'stratus' + + # none of these work as the h5py methods do not follow their documentation AFAIK + # assert h5_enum_t.id.enum_valueof('stratus') == p5_enum_t.id.enum_valueof('stratus') + # assert h5_enum_t.id.get_member_value(1) == p5_enum_t.id.get_member_value(1) + + assert h5_enum_t.dtype == p5_enum_t.dtype + + assert isinstance(pfile["enum_t"].id, pyfive.h5t.TypeEnumID) + assert pfile["enum_t"].id == pfile["enum2_t"].id + assert pfile["enum_t"].id != pfile["phony_vlen"].id + From b32fa74adc51ca9c9a6ec900e6e4ae0e65232b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Wed, 1 Oct 2025 13:45:04 +0200 Subject: [PATCH 2/3] Update pyfive/h5t.py --- pyfive/h5t.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfive/h5t.py b/pyfive/h5t.py index e03bdb30..8b3eb10f 100644 --- a/pyfive/h5t.py +++ b/pyfive/h5t.py @@ -74,7 +74,7 @@ def check_dtype(**kwds): else: return None - +# todo: refactor the following classes, so TypeEnumID and TypeCompoundID sublass from the base TypeID. class TypeID: """ Used by DataType to expose internal structure of a generic From fd98b7d86a8e3fcd4166f5e4614a404ff738829f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Wed, 1 Oct 2025 13:57:50 +0200 Subject: [PATCH 3/3] fix filename --- tests/test_dataset_datatypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dataset_datatypes.py b/tests/test_dataset_datatypes.py index ce2bd728..eaa4b5f3 100644 --- a/tests/test_dataset_datatypes.py +++ b/tests/test_dataset_datatypes.py @@ -11,7 +11,7 @@ DIRNAME = os.path.dirname(__file__) DATASET_DATATYPES_HDF5_FILE = os.path.join(DIRNAME, 'dataset_datatypes.hdf5') -MAKE_DATASET_DATATYPES_SCRIPT = os.path.join(DIRNAME, 'make_dataset_datatypes.py') +MAKE_DATASET_DATATYPES_SCRIPT = os.path.join(DIRNAME, 'make_dataset_datatypes_file.py') @pytest.fixture(scope="module")