From f23f50b91606439a5c958c4545ba4f73dd2980ea Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 15 Aug 2025 15:46:56 +0100 Subject: [PATCH 1/3] add suggestions by Kai --- doc/optimising.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/optimising.rst b/doc/optimising.rst index f2886fe1..49d1a384 100644 --- a/doc/optimising.rst +++ b/doc/optimising.rst @@ -6,7 +6,7 @@ HDF5 files can be large and complicated, with complex internal structures which These complexities (and the overheads they introduce) can be mitigated by optimising how you access the data, but this requires an understanding of how the data is stored in the file and how the data access library (in this case ``pyfive``) works. -The data storage complexities arise from two main factors: the use of chunking, and the way attributes are stored in the files +The data storage complexities arise from two main factors: the use of chunking, and the way attributes are stored in the files. **Chunking**: HDF5 files can store data in chunks, which allows for more efficient access to large datasets. However, this also means that the library needs to maintain an index (a "b-tree") which relates the position in coordinate space to where each chunk is stored in the file. @@ -20,7 +20,7 @@ Optimising the files themselves ------------------------------- Optimal access to data occurs when the data is chunked in a way that matches the access patterns of your application, and when the -b-tree indexes and attributess are stored contiguously in the file. +b-tree indexes and attributes are stored contiguously in the file. Users of ``pyfive`` will always confront data files which have been created by other software, but if possible, it is worth exploring whether the `h5repack `_ tool can @@ -41,7 +41,7 @@ For example, instead of doing: import pyfive with pyfive.File("data.h5", "r") as f: - variables = [f for var in f] + variables = [var for var in f] print("Variables in file:", variables) temp = variables['temp'] @@ -85,7 +85,7 @@ For example, you can use the `concurrent.futures` module to read data from multi print("Results:", results) -You can do the same thing to parallelise manipuations within the variables, by for example using, ``Dask``, but that is beyond the scope of this document. +You can do the same thing to parallelise manipulations within the variables, by for example using, ``Dask``, but that is beyond the scope of this document. Using pyfive with S3 From 5ac67bd3f1fafaf7cf74567e7c74f115bce57193 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 15 Aug 2025 15:47:12 +0100 Subject: [PATCH 2/3] add suggestions by Kai --- pyfive/h5d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfive/h5d.py b/pyfive/h5d.py index 803f6084..a2f794a8 100644 --- a/pyfive/h5d.py +++ b/pyfive/h5d.py @@ -241,7 +241,7 @@ def index(self): return self._index #### The following method can be used to set pseudo chunking size after the #### file has been closed and before data transactions. This is pyfive specific - def set_psuedo_chunk_size(self, newsize_MB): + def set_pseudo_chunk_size(self, newsize_MB): """ Set pseudo chunking size for contiguous variables. This is a ``pyfive`` API extension. The default value is 4 MB which should be suitable for most applications. From f2f074ab4d74281a9726257ff0b4b1d328116bcb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 15 Aug 2025 15:47:33 +0100 Subject: [PATCH 3/3] correct test for new method name psuedo to pseudo --- tests/test_pseudochunking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pseudochunking.py b/tests/test_pseudochunking.py index 94de5701..aafa4f0a 100644 --- a/tests/test_pseudochunking.py +++ b/tests/test_pseudochunking.py @@ -21,7 +21,7 @@ def setup_data(): with pyfive.File(file_like,'r') as f: var1 = f['var1'] # use 100 KB as the chunk size - var1.id.set_psuedo_chunk_size(0.1) + var1.id.set_pseudo_chunk_size(0.1) return var1, data