Merge pull request #268 from matthew-brett/mmap-api

MRG: mmap keyword for loading images

Add mmap keyword to control memory mapping on image load.

API as discussed on the nipy list.

Author: matthew-brett

nibabel/analyze.py

@@ -95,6 +95,7 @@
 from .fileholders import copy_file_map
 from .batteryrunners import Report
 from .arrayproxy import ArrayProxy
+from .keywordonly import kw_only_meth
 
 # Sub-parts of standard analyze header from
 # Mayo dbh.h file
@@ -915,17 +916,38 @@ def set_data_dtype(self, dtype):
         self._header.set_data_dtype(dtype)
 
     @classmethod
-    def from_file_map(klass, file_map):
+    @kw_only_meth(1)
+    def from_file_map(klass, file_map, mmap=True):
         ''' class method to create image from mapping in `file_map ``
+
+        Parameters
+        ----------
+        file_map : dict
+            Mapping with (kay, value) pairs of (``file_type``, FileHolder
+            instance giving file-likes for each file needed for this image
+            type.
+        mmap : {True, False, 'c', 'r'}, optional, keyword only
+            `mmap` controls the use of numpy memory mapping for reading image
+            array data.  If False, do not try numpy ``memmap`` for data array.
+            If one of {'c', 'r'}, try numpy memmap with ``mode=mmap``.  A `mmap`
+            value of True gives the same behavior as ``mmap='c'``.  If image
+            data file cannot be memory-mapped, ignore `mmap` value and read
+            array from file.
+
+        Returns
+        -------
+        img : AnalyzeImage instance
         '''
+        if mmap not in (True, False, 'c', 'r'):
+            raise ValueError("mmap should be one of {True, False, 'c', 'r'}")
         hdr_fh, img_fh = klass._get_fileholders(file_map)
         with hdr_fh.get_prepare_fileobj(mode='rb') as hdrf:
             header = klass.header_class.from_fileobj(hdrf)
         hdr_copy = header.copy()
         imgf = img_fh.fileobj
         if imgf is None:
             imgf = img_fh.filename
-        data = klass.ImageArrayProxy(imgf, hdr_copy)
+        data = klass.ImageArrayProxy(imgf, hdr_copy, mmap=mmap)
         # Initialize without affine to allow header to pass through unmodified
         img = klass(data, None, header, file_map=file_map)
         # set affine from header though
@@ -935,6 +957,34 @@ def from_file_map(klass, file_map):
                            'file_map': copy_file_map(file_map)}
         return img
 
+    @classmethod
+    @kw_only_meth(1)
+    def from_filename(klass, filename, mmap=True):
+        ''' class method to create image from filename `filename`
+
+        Parameters
+        ----------
+        filename : str
+            Filename of image to load
+        mmap : {True, False, 'c', 'r'}, optional, keyword only
+            `mmap` controls the use of numpy memory mapping for reading image
+            array data.  If False, do not try numpy ``memmap`` for data array.
+            If one of {'c', 'r'}, try numpy memmap with ``mode=mmap``.  A `mmap`
+            value of True gives the same behavior as ``mmap='c'``.  If image
+            data file cannot be memory-mapped, ignore `mmap` value and read
+            array from file.
+
+        Returns
+        -------
+        img : Analyze Image instance
+        '''
+        if mmap not in (True, False, 'c', 'r'):
+            raise ValueError("mmap should be one of {True, False, 'c', 'r'}")
+        file_map = klass.filespec_to_file_map(filename)
+        return klass.from_file_map(file_map, mmap=mmap)
+
+    load = from_filename
+
     @staticmethod
     def _get_fileholders(file_map):
         """ Return fileholder for header and image

nibabel/arrayproxy.py

@@ -29,6 +29,7 @@
 
 from .volumeutils import BinOpener, array_from_file, apply_read_scaling
 from .fileslice import fileslice
+from .keywordonly import kw_only_meth
 
 
 class ArrayProxy(object):
@@ -55,12 +56,36 @@ class ArrayProxy(object):
     including Nifti1, and with the MGH format.
 
     Other image types might need more specific classes to implement the API.
-    API.  See :mod:`nibabel.minc1` and :mod:`nibabel.ecat` for examples.
+    See :mod:`nibabel.minc1`, :mod:`nibabel.ecat` and :mod:`nibabel.parrec` for
+    examples.
     """
     # Assume Fortran array memory layout
     order = 'F'
 
-    def __init__(self, file_like, header):
+    @kw_only_meth(2)
+    def __init__(self, file_like, header, mmap=True):
+        """ Initialize array proxy instance
+
+        Parameters
+        ----------
+        file_like : object
+            File-like object or filename. If file-like object, should implement
+            at least ``read`` and ``seek``.
+        header : object
+            Header object implementing ``get_data_shape``, ``get_data_dtype``,
+            ``get_data_offset``, ``get_slope_inter``
+        mmap : {True, False, 'c', 'r'}, optional, keyword only
+            `mmap` controls the use of numpy memory mapping for reading data.
+            If False, do not try numpy ``memmap`` for data array.  If one of
+            {'c', 'r'}, try numpy memmap with ``mode=mmap``.  A `mmap` value of
+            True gives the same behavior as ``mmap='c'``.  If `file_like`
+            cannot be memory-mapped, ignore `mmap` value and read array from
+            file.
+        scaling : {'fp', 'dv'}, optional, keyword only
+            Type of scaling to use - see header ``get_data_scaling`` method.
+        """
+        if mmap not in (True, False, 'c', 'r'):
+            raise ValueError("mmap should be one of {True, False, 'c', 'r'}")
         self.file_like = file_like
         # Copies of values needed to read array
         self._shape = header.get_data_shape()
@@ -69,6 +94,7 @@ def __init__(self, file_like, header):
         self._slope, self._inter = header.get_slope_inter()
         self._slope = 1.0 if self._slope is None else self._slope
         self._inter = 0.0 if self._inter is None else self._inter
+        self._mmap = mmap
         # Reference to original header; we will remove this soon
         self._header = header.copy()
 
@@ -109,7 +135,8 @@ def get_unscaled(self):
                                        self._dtype,
                                        fileobj,
                                        offset=self._offset,
-                                       order=self.order)
+                                       order=self.order,
+                                       mmap=self._mmap)
         return raw_data
 
     def __array__(self):

nibabel/freesurfer/mghformat.py

@@ -13,11 +13,11 @@
 from os.path import splitext
 import numpy as np
 
-from nibabel.volumeutils import (array_to_file, array_from_file, Recoder)
-from nibabel.spatialimages import HeaderDataError, ImageFileError, SpatialImage
-from nibabel.fileholders import FileHolder,  copy_file_map
-from nibabel.filename_parser import types_filenames, TypesFilenamesError
-from nibabel.arrayproxy import ArrayProxy
+from ..volumeutils import (array_to_file, array_from_file, Recoder)
+from ..spatialimages import HeaderDataError, SpatialImage
+from ..fileholders import FileHolder,  copy_file_map
+from ..arrayproxy import ArrayProxy
+from ..keywordonly import kw_only_meth
 
 # mgh header
 # See http://surfer.nmr.mgh.harvard.edu/fswiki/FsTutorial/MghFormat
@@ -468,26 +468,64 @@ def filespec_to_file_map(klass, filespec):
         return super(MGHImage, klass).filespec_to_file_map(filespec)
 
     @classmethod
-    def from_file_map(klass, file_map):
+    @kw_only_meth(1)
+    def from_file_map(klass, file_map, mmap=True):
         '''Load image from `file_map`
 
         Parameters
         ----------
         file_map : None or mapping, optional
            files mapping.  If None (default) use object's ``file_map``
            attribute instead
-        '''
+        mmap : {True, False, 'c', 'r'}, optional, keyword only
+            `mmap` controls the use of numpy memory mapping for reading image
+            array data.  If False, do not try numpy ``memmap`` for data array.
+            If one of {'c', 'r'}, try numpy memmap with ``mode=mmap``.  A `mmap`
+            value of True gives the same behavior as ``mmap='c'``.  If image
+            data file cannot be memory-mapped, ignore `mmap` value and read
+            array from file.
+        '''
+        if not mmap in (True, False, 'c', 'r'):
+            raise ValueError("mmap should be one of {True, False, 'c', 'r'}")
         mghf = file_map['image'].get_prepare_fileobj('rb')
         header = klass.header_class.from_fileobj(mghf)
         affine = header.get_affine()
         hdr_copy = header.copy()
-        data = klass.ImageArrayProxy(mghf, hdr_copy)
+        data = klass.ImageArrayProxy(mghf, hdr_copy, mmap=mmap)
         img = klass(data, affine, header, file_map=file_map)
         img._load_cache = {'header': hdr_copy,
                            'affine': affine.copy(),
                            'file_map': copy_file_map(file_map)}
         return img
 
+    @classmethod
+    @kw_only_meth(1)
+    def from_filename(klass, filename, mmap=True):
+        ''' class method to create image from filename `filename`
+
+        Parameters
+        ----------
+        filename : str
+            Filename of image to load
+        mmap : {True, False, 'c', 'r'}, optional, keyword only
+            `mmap` controls the use of numpy memory mapping for reading image
+            array data.  If False, do not try numpy ``memmap`` for data array.
+            If one of {'c', 'r'}, try numpy memmap with ``mode=mmap``.  A `mmap`
+            value of True gives the same behavior as ``mmap='c'``.  If image
+            data file cannot be memory-mapped, ignore `mmap` value and read
+            array from file.
+
+        Returns
+        -------
+        img : MGHImage instance
+        '''
+        if not mmap in (True, False, 'c', 'r'):
+            raise ValueError("mmap should be one of {True, False, 'c', 'r'}")
+        file_map = klass.filespec_to_file_map(filename)
+        return klass.from_file_map(file_map, mmap=mmap)
+
+    load = from_filename
+
     def to_file_map(self, file_map=None):
         ''' Write image to `file_map` or contained ``self.file_map``
 

nibabel/loadsave.py

@@ -26,20 +26,22 @@
 from .arrayproxy import is_proxy
 
 
-def load(filename):
+def load(filename, **kwargs):
     ''' Load file given filename, guessing at file type
 
     Parameters
     ----------
     filename : string
        specification of file to load
+    \*\*kwargs : keyword arguments
+        Keyword arguments to format-specific load
 
     Returns
     -------
     img : ``SpatialImage``
        Image of guessed type
     '''
-    return guessed_image_type(filename).from_filename(filename)
+    return guessed_image_type(filename).from_filename(filename, **kwargs)
 
 
 def guessed_image_type(filename):