From 79792de0bff76d0a98781c3910b31d6cda6f21d0 Mon Sep 17 00:00:00 2001 From: manifest-rules Date: Fri, 23 Feb 2024 09:57:36 +0000 Subject: [PATCH 1/4] TEST: Unit test for loading ASCII-encoded "flat" GIFTI data array. Currently failing --- nibabel/gifti/tests/data/ascii_flat_data.gii | 76 ++++++++++++++++++++ nibabel/gifti/tests/test_parse_gifti_fast.py | 15 +++- 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 nibabel/gifti/tests/data/ascii_flat_data.gii diff --git a/nibabel/gifti/tests/data/ascii_flat_data.gii b/nibabel/gifti/tests/data/ascii_flat_data.gii new file mode 100644 index 0000000000..26a73fba02 --- /dev/null +++ b/nibabel/gifti/tests/data/ascii_flat_data.gii @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1.000000 0.000000 0.000000 0.000000 0.000000 1.000000 0.000000 0.000000 0.000000 0.000000 1.000000 0.000000 0.000000 0.000000 0.000000 1.000000 + + 155.17539978 135.58103943 98.30715179 140.33973694 190.0491333 73.24776459 157.3598938 196.97969055 83.65809631 171.46174622 137.43661499 78.4709549 148.54592896 97.06752777 65.96373749 123.45701599 111.46841431 66.3571167 135.30892944 202.28720093 36.38148499 178.28155518 162.59469604 37.75128937 178.11087036 115.28820038 57.17986679 142.81582642 82.82115173 31.02205276 + + + + + + + + + + + + + 6402 17923 25602 14085 25602 17923 25602 14085 4483 17923 1602 14085 4483 25603 25602 25604 25602 25603 25602 25604 6402 25603 3525 25604 1123 17922 12168 25604 12168 17922 + + diff --git a/nibabel/gifti/tests/test_parse_gifti_fast.py b/nibabel/gifti/tests/test_parse_gifti_fast.py index f08bdd1b17..49f2729f37 100644 --- a/nibabel/gifti/tests/test_parse_gifti_fast.py +++ b/nibabel/gifti/tests/test_parse_gifti_fast.py @@ -39,9 +39,10 @@ DATA_FILE5 = pjoin(IO_DATA_PATH, 'base64bin.gii') DATA_FILE6 = pjoin(IO_DATA_PATH, 'rh.aparc.annot.gii') DATA_FILE7 = pjoin(IO_DATA_PATH, 'external.gii') +DATA_FILE8 = pjoin(IO_DATA_PATH, 'ascii_flat_data.gii') -datafiles = [DATA_FILE1, DATA_FILE2, DATA_FILE3, DATA_FILE4, DATA_FILE5, DATA_FILE6, DATA_FILE7] -numDA = [2, 1, 1, 1, 2, 1, 2] +datafiles = [DATA_FILE1, DATA_FILE2, DATA_FILE3, DATA_FILE4, DATA_FILE5, DATA_FILE6, DATA_FILE7, DATA_FILE8] +numDA = [2, 1, 1, 1, 2, 1, 2, 2] DATA_FILE1_darr1 = np.array( [ @@ -152,6 +153,10 @@ dtype=np.int32, ) +DATA_FILE8_darr1 = np.copy(DATA_FILE5_darr1) + +DATA_FILE8_darr2 = np.copy(DATA_FILE5_darr2) + def assert_default_types(loaded): default = loaded.__class__() @@ -448,3 +453,9 @@ def test_load_compressed(): img7 = load(fn) assert_array_almost_equal(img7.darrays[0].data, DATA_FILE7_darr1) assert_array_almost_equal(img7.darrays[1].data, DATA_FILE7_darr2) + + +def test_load_flat_ascii_data(): + img = load(DATA_FILE8) + assert_array_almost_equal(img.darrays[0].data, DATA_FILE8_darr1) + assert_array_almost_equal(img.darrays[1].data, DATA_FILE8_darr2) From 6ffeeacc158c51111691e91fbb2fbbc303f42cd8 Mon Sep 17 00:00:00 2001 From: manifest-rules Date: Fri, 23 Feb 2024 10:08:14 +0000 Subject: [PATCH 2/4] RF: Make sure that ASCII-encoded DataArrays are returned with expected shape --- nibabel/gifti/parse_gifti_fast.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nibabel/gifti/parse_gifti_fast.py b/nibabel/gifti/parse_gifti_fast.py index 7d8eacb825..af01dd544b 100644 --- a/nibabel/gifti/parse_gifti_fast.py +++ b/nibabel/gifti/parse_gifti_fast.py @@ -74,6 +74,10 @@ def read_data_block(darray, fname, data, mmap): # GIFTI_ENCODING_ASCII c = StringIO(data) da = np.loadtxt(c, dtype=dtype) + # Reshape to dims specified in GiftiDataArray attributes, but preserve + # existing behaviour of loading as 1D for arrays with a dimension of + # length 1 + da = da.reshape(darray.dims).squeeze() return da # independent of the endianness elif enclabel not in ('B64BIN', 'B64GZ', 'External'): return 0 From b46c82946d6bd88b73164904834567b12aadf935 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 23 Feb 2024 10:05:56 -0500 Subject: [PATCH 3/4] RF: Consistently apply data type, shape and index order in GIFTI data blocks --- nibabel/gifti/parse_gifti_fast.py | 70 +++++++++++++------------------ 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/nibabel/gifti/parse_gifti_fast.py b/nibabel/gifti/parse_gifti_fast.py index af01dd544b..ccd608324a 100644 --- a/nibabel/gifti/parse_gifti_fast.py +++ b/nibabel/gifti/parse_gifti_fast.py @@ -68,21 +68,21 @@ def read_data_block(darray, fname, data, mmap): if mmap is True: mmap = 'c' enclabel = gifti_encoding_codes.label[darray.encoding] - dtype = data_type_codes.type[darray.datatype] + if enclabel not in ('ASCII', 'B64BIN', 'B64GZ', 'External'): + raise GiftiParseError(f'Unknown encoding {darray.encoding}') + + # Encode the endianness in the dtype + byteorder = gifti_endian_codes.byteorder[darray.endian] + dtype = data_type_codes.dtype[darray.datatype].newbyteorder(byteorder) + + shape = tuple(darray.dims) + order = array_index_order_codes.npcode[darray.ind_ord] + + # GIFTI_ENCODING_ASCII if enclabel == 'ASCII': - # GIFTI_ENCODING_ASCII - c = StringIO(data) - da = np.loadtxt(c, dtype=dtype) - # Reshape to dims specified in GiftiDataArray attributes, but preserve - # existing behaviour of loading as 1D for arrays with a dimension of - # length 1 - da = da.reshape(darray.dims).squeeze() - return da # independent of the endianness - elif enclabel not in ('B64BIN', 'B64GZ', 'External'): - return 0 - - # GIFTI_ENCODING_EXTBIN + return np.loadtxt(StringIO(data), dtype=dtype, ndmin=1).reshape(shape, order=order) + # We assume that the external data file is raw uncompressed binary, with # the data type/endianness/ordering specified by the other DataArray # attributes @@ -98,12 +98,13 @@ def read_data_block(darray, fname, data, mmap): newarr = None if mmap: try: - newarr = np.memmap( + return np.memmap( ext_fname, dtype=dtype, mode=mmap, offset=darray.ext_offset, - shape=tuple(darray.dims), + shape=shape, + order=order, ) # If the memmap fails, we ignore the error and load the data into # memory below @@ -111,13 +112,12 @@ def read_data_block(darray, fname, data, mmap): pass # mmap=False or np.memmap failed if newarr is None: - # We can replace this with a call to np.fromfile in numpy>=1.17, - # as an "offset" parameter was added in that version. - with open(ext_fname, 'rb') as f: - f.seek(darray.ext_offset) - nbytes = np.prod(darray.dims) * dtype().itemsize - buff = f.read(nbytes) - newarr = np.frombuffer(buff, dtype=dtype) + return np.fromfile( + ext_fname, + dtype=dtype, + count=np.prod(darray.dims), + offset=darray.ext_offset, + ).reshape(shape, order=order) # Numpy arrays created from bytes objects are read-only. # Neither b64decode nor decompress will return bytearrays, and there @@ -125,26 +125,14 @@ def read_data_block(darray, fname, data, mmap): # there is not a simple way to avoid making copies. # If this becomes a problem, we should write a decoding interface with # a tunable chunk size. + dec = base64.b64decode(data.encode('ascii')) + if enclabel == 'B64BIN': + buff = bytearray(dec) else: - dec = base64.b64decode(data.encode('ascii')) - if enclabel == 'B64BIN': - # GIFTI_ENCODING_B64BIN - buff = bytearray(dec) - else: - # GIFTI_ENCODING_B64GZ - buff = bytearray(zlib.decompress(dec)) - del dec - newarr = np.frombuffer(buff, dtype=dtype) - - sh = tuple(darray.dims) - if len(newarr.shape) != len(sh): - newarr = newarr.reshape(sh, order=array_index_order_codes.npcode[darray.ind_ord]) - - # check if we need to byteswap - required_byteorder = gifti_endian_codes.byteorder[darray.endian] - if required_byteorder in ('big', 'little') and required_byteorder != sys.byteorder: - newarr = newarr.byteswap() - return newarr + # GIFTI_ENCODING_B64GZ + buff = bytearray(zlib.decompress(dec)) + del dec + return np.frombuffer(buff, dtype=dtype).reshape(shape, order=order) def _str2int(in_str): From afbcc88d2c3ff83df3acadbff4741a790d2d5647 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 23 Feb 2024 10:08:22 -0500 Subject: [PATCH 4/4] TEST: Expect data arrays to be the advertised shapes --- nibabel/gifti/gifti.py | 2 +- nibabel/gifti/tests/test_parse_gifti_fast.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/nibabel/gifti/gifti.py b/nibabel/gifti/gifti.py index 76bad4677a..7aba877309 100644 --- a/nibabel/gifti/gifti.py +++ b/nibabel/gifti/gifti.py @@ -745,7 +745,7 @@ def agg_data(self, intent_code=None): >>> triangles_2 = surf_img.agg_data('triangle') >>> triangles_3 = surf_img.agg_data(1009) # Numeric code for pointset >>> print(np.array2string(triangles)) - [0 1 2] + [[0 1 2]] >>> np.array_equal(triangles, triangles_2) True >>> np.array_equal(triangles, triangles_3) diff --git a/nibabel/gifti/tests/test_parse_gifti_fast.py b/nibabel/gifti/tests/test_parse_gifti_fast.py index 49f2729f37..f972425679 100644 --- a/nibabel/gifti/tests/test_parse_gifti_fast.py +++ b/nibabel/gifti/tests/test_parse_gifti_fast.py @@ -41,7 +41,16 @@ DATA_FILE7 = pjoin(IO_DATA_PATH, 'external.gii') DATA_FILE8 = pjoin(IO_DATA_PATH, 'ascii_flat_data.gii') -datafiles = [DATA_FILE1, DATA_FILE2, DATA_FILE3, DATA_FILE4, DATA_FILE5, DATA_FILE6, DATA_FILE7, DATA_FILE8] +datafiles = [ + DATA_FILE1, + DATA_FILE2, + DATA_FILE3, + DATA_FILE4, + DATA_FILE5, + DATA_FILE6, + DATA_FILE7, + DATA_FILE8, +] numDA = [2, 1, 1, 1, 2, 1, 2, 2] DATA_FILE1_darr1 = np.array( @@ -51,7 +60,7 @@ [-17.614349, -65.401642, 21.071466], ] ) -DATA_FILE1_darr2 = np.array([0, 1, 2]) +DATA_FILE1_darr2 = np.array([[0, 1, 2]]) DATA_FILE2_darr1 = np.array( [