Merge branch 'master' into afni_interfaces

* master: (21 commits)
  doc: updated debug config message.
  ants code cleanup (PEP8, unused variables)
  removed in_file from DTLUTGen docstring
  added changelog message
  removed unused variables
  removed unused imports
  pep8 and reindent
  removed an evil trailing comma from WIMT
  fix: added test to check that inputs are not modified when setting display variable via config
  fix: added tests, actually added display variable to config
  fix: interface environment always overwrites config environ DISPLAY
  doc: change closes nipy#368
  fix: environment inputs not changed by config display variable
  AFNITraitedSpec for Calc
  Renamed NBS interface
  Threshold to a float
  Typo fix
  Replaced CFF option with multiple network output
  Added optional node position network
  Initial NBS commit
  ...

Author: bpinsard

CHANGES

@@ -1,7 +1,10 @@
 Since last release
 ==================
 
-* ENH: New interfaces: MySQLSink, nipy.Similarity, WatershedBEM
+* API: display variable no longer encoded as inputs in commandline interfaces
+
+* ENH: New interfaces: MySQLSink, nipy.Similarity, WatershedBEM,
+       NetworkBasedStatistic
 
 * FIX: Afni outputs should inherit from TraitedSpec
 

doc/users/debug.rst

@@ -17,11 +17,16 @@ performance issues.
 
 #. Use the debug config mode. This can be done by setting::
 
-      import config
+      from nipype import config
       config.enable_debug_mode()
 
    as the first import of your nipype script.
 
+   .. note::
+
+     Turning on debug will rerun your workflows and will rerun them after debugging
+     is turned off.
+
 #. There are several configuration options that can help with debugging. See
    :ref:`config_file` for more details::
 

nipype/interfaces/afni/preprocess.py

@@ -1588,7 +1588,7 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None):
 """
 
 
-class CalcInputSpec(CommandLineInputSpec):
+class CalcInputSpec(AFNITraitedSpec):
     in_file_a = File(desc='input file to 3dcalc',
         argstr='-a %s', position=0, mandatory=True)
     in_file_b = File(desc='operand file to 3dcalc',

nipype/interfaces/ants/__init__.py

@@ -3,6 +3,5 @@
 
 """Top-level namespace for ants."""
 
-from .normalize import (BuildTemplate, WarpImageMultiTransform, 
+from .normalize import (BuildTemplate, WarpImageMultiTransform,
                         WarpTimeSeriesImageMultiTransform, GenWarpFields)
-

nipype/interfaces/ants/normalize.py

@@ -146,7 +146,7 @@ def _list_outputs(self):
                                    self.inputs.out_prefix)
         outputs['subject_outfiles'] = []
         for filename in self.inputs.in_files:
-            pth, base, ext = split_filename(filename)
+            _, base, _ = split_filename(filename)
             temp = glob(os.path.realpath('%s%s*' % (self.inputs.out_prefix,
                                                     base)))
             for file_ in temp:
@@ -222,7 +222,7 @@ def _format_arg(self, opt, spec, val):
                     isdefined(self.inputs.invert_affine):
                     affine_counter += 1
                     if affine_counter in self.inputs.invert_affine:
-                        series += ['-i'],
+                        series += ['-i']
                 series += [transformation]
             return ' '.join(series)
         return super(WarpImageMultiTransform, self)._format_arg(opt, spec, val)

nipype/interfaces/ants/setup.py

@@ -1,6 +1,8 @@
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
 # vi: set ft=python sts=4 ts=4 sw=4 et:
-def configuration(parent_package='',top_path=None):
+
+
+def configuration(parent_package='', top_path=None):
     from numpy.distutils.misc_util import Configuration
 
     config = Configuration('ants', parent_package, top_path)

nipype/interfaces/base.py

@@ -1033,11 +1033,6 @@ def __init__(self, command=None, **inputs):
             raise Exception("Missing command")
         if command:
             self._cmd = command
-        try:
-            display_var = config.get('execution', 'display_variable')
-            self.inputs.environ['DISPLAY'] = display_var
-        except NoOptionError:
-            pass
 
     @property
     def cmd(self):
@@ -1087,7 +1082,16 @@ def _run_interface(self, runtime):
         setattr(runtime, 'stdout', None)
         setattr(runtime, 'stderr', None)
         setattr(runtime, 'cmdline', self.cmdline)
-        runtime.environ.update(self.inputs.environ)
+        out_environ = {}
+        try:
+            display_var = config.get('execution', 'display_variable')
+            out_environ = {'DISPLAY': display_var}
+        except NoOptionError:
+            pass
+        iflogger.info(out_environ)
+        if isdefined(self.inputs.environ):
+            out_environ.update(self.inputs.environ)
+        runtime.environ.update(out_environ)
         if not self._exists_in_path(self.cmd.split()[0]):
             raise IOError("%s could not be found on host %s" % (self.cmd.split()[0],
                                                                 runtime.hostname))

nipype/interfaces/camino/dti.py

@@ -209,7 +209,6 @@ class DTLUTGen(StdOutCommandLine):
     >>> dtl = cmon.DTLUTGen()
     >>> dtl.inputs.snr = 16
     >>> dtl.inputs.scheme_file = 'A.scheme'
-    >>> dtl.inputs.in_file = 'tensor_fitted_data.Bfloat'
     >>> dtl.run()                  # doctest: +SKIP
     """
     _cmd = 'dtlutgen'

nipype/interfaces/cmtk/__init__.py

@@ -2,3 +2,4 @@
 from .nx import NetworkXMetrics, AverageNetworks
 from .parcellation import Parcellate
 from .convert import CFFConverter, MergeCNetworks
+from .nbs import NetworkBasedStatistic

nipype/interfaces/cmtk/nbs.py

@@ -0,0 +1,146 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+from nipype.interfaces.base import (BaseInterface, BaseInterfaceInputSpec, traits,
+                                    File, TraitedSpec, InputMultiPath,
+                                    OutputMultiPath, isdefined)
+import os.path as op
+import numpy as np
+import networkx as nx
+from nipype.utils.misc import package_check
+import warnings
+
+from ... import logging
+iflogger = logging.getLogger('interface')
+
+have_cv = True
+try:
+    package_check('cviewer')
+except Exception, e:
+    have_cv = False
+    warnings.warn('ConnectomeViewer not installed')
+else:
+    import cviewer.libs.pyconto.groupstatistics.nbs as nbs
+
+
+def ntwks_to_matrices(in_files, edge_key):
+    first = nx.read_gpickle(in_files[0])
+    files = len(in_files)
+    nodes = len(first.nodes())
+    matrix = np.zeros((nodes, nodes, files))
+    for idx, name in enumerate(in_files):
+        graph = nx.read_gpickle(name)
+        for u, v, d in graph.edges(data=True):
+            graph[u][v]['weight'] = d[edge_key]  # Setting the edge requested edge value as weight value
+        matrix[:, :, idx] = nx.to_numpy_matrix(graph)  # Retrieve the matrix
+    return matrix
+
+
+class NetworkBasedStatisticInputSpec(BaseInterfaceInputSpec):
+    in_group1 = InputMultiPath(File(exists=True), mandatory=True, desc='Networks for the first group of subjects')
+    in_group2 = InputMultiPath(File(exists=True), mandatory=True, desc='Networks for the second group of subjects')
+    node_position_network = File(desc='An optional network used to position the nodes for the output networks')
+    number_of_permutations = traits.Int(1000, usedefault=True, desc='Number of permutations to perform')
+    threshold = traits.Float(3, usedefault=True, desc='T-statistic threshold')
+    t_tail = traits.Enum('left', 'right', 'both', usedefault=True, desc='Can be one of "left", "right", or "both"')
+    edge_key = traits.Str('number_of_fibers', usedefault=True, desc='Usually "number_of_fibers, "fiber_length_mean", "fiber_length_std" for matrices made with CMTK' \
+     'Sometimes "weight" or "value" for functional networks.')
+    out_nbs_network = File(desc='Output network with edges identified by the NBS')
+    out_nbs_pval_network = File(desc='Output network with p-values to weight the edges identified by the NBS')
+
+
+class NetworkBasedStatisticOutputSpec(TraitedSpec):
+    nbs_network = File(exists=True, desc='Output network with edges identified by the NBS')
+    nbs_pval_network = File(exists=True, desc='Output network with p-values to weight the edges identified by the NBS')
+    network_files = OutputMultiPath(File(exists=True), desc='Output network with edges identified by the NBS')
+
+
+class NetworkBasedStatistic(BaseInterface):
+    """
+    Calculates and outputs the average network given a set of input NetworkX gpickle files
+
+    For documentation of Network-based statistic parameters:
+
+            https://github.com/LTS5/connectomeviewer/blob/master/cviewer/libs/pyconto/groupstatistics/nbs/_nbs.py
+
+    Example
+    -------
+
+    >>> import nipype.interfaces.cmtk as cmtk
+    >>> nbs = cmtk.NetworkBasedStatistic()
+    >>> nbs.inputs.in_group1 = ['subj1.pck', 'subj2.pck'] # doctest: +SKIP
+    >>> nbs.inputs.in_group2 = ['pat1.pck', 'pat2.pck'] # doctest: +SKIP
+    >>> nbs.run()                 # doctest: +SKIP
+    """
+    input_spec = NetworkBasedStatisticInputSpec
+    output_spec = NetworkBasedStatisticOutputSpec
+
+    def _run_interface(self, runtime):
+        THRESH = self.inputs.threshold
+        K = self.inputs.number_of_permutations
+        TAIL = self.inputs.t_tail
+        edge_key = self.inputs.edge_key
+        details = edge_key + '-thresh-' + str(THRESH) + '-k-' + str(K) + '-tail-' + TAIL + '.pck'
+
+        # Fill in the data from the networks
+        X = ntwks_to_matrices(self.inputs.in_group1, edge_key)
+        Y = ntwks_to_matrices(self.inputs.in_group2, edge_key)
+
+        PVAL, ADJ, _ = nbs.compute_nbs(X, Y, THRESH, K, TAIL)
+
+        iflogger.info('p-values:')
+        iflogger.info(PVAL)
+
+        pADJ = ADJ.copy()
+        for idx, _ in enumerate(PVAL):
+            x, y = np.where(ADJ == idx + 1)
+            pADJ[x, y] = PVAL[idx]
+
+        # Create networkx graphs from the adjacency matrix
+        nbsgraph = nx.from_numpy_matrix(ADJ)
+        nbs_pval_graph = nx.from_numpy_matrix(pADJ)
+
+        # Relabel nodes because they should not start at zero for our convention
+        nbsgraph = nx.relabel_nodes(nbsgraph, lambda x: x + 1)
+        nbs_pval_graph = nx.relabel_nodes(nbs_pval_graph, lambda x: x + 1)
+
+        if isdefined(self.inputs.node_position_network):
+            node_ntwk_name = self.inputs.node_position_network
+        else:
+            node_ntwk_name = self.inputs.in_group1[0]
+
+        node_network = nx.read_gpickle(node_ntwk_name)
+        iflogger.info('Populating node dictionaries with attributes from {node}'.format(node=node_ntwk_name))
+
+        for nid, ndata in node_network.nodes_iter(data=True):
+            nbsgraph.node[nid] = ndata
+            nbs_pval_graph.node[nid] = ndata
+
+        path = op.abspath('NBS_Result_' + details)
+        iflogger.info(path)
+        nx.write_gpickle(nbsgraph, path)
+        iflogger.info('Saving output NBS edge network as {out}'.format(out=path))
+
+        pval_path = op.abspath('NBS_P_vals_' + details)
+        iflogger.info(pval_path)
+        nx.write_gpickle(nbs_pval_graph, pval_path)
+        iflogger.info('Saving output p-value network as {out}'.format(out=pval_path))
+        return runtime
+
+    def _list_outputs(self):
+        outputs = self.output_spec().get()
+
+        THRESH = self.inputs.threshold
+        K = self.inputs.number_of_permutations
+        TAIL = self.inputs.t_tail
+        edge_key = self.inputs.edge_key
+        details = edge_key + '-thresh-' + str(THRESH) + '-k-' + str(K) + '-tail-' + TAIL + '.pck'
+        path = op.abspath('NBS_Result_' + details)
+        pval_path = op.abspath('NBS_P_vals_' + details)
+
+        outputs['nbs_network'] = path
+        outputs['nbs_pval_network'] = pval_path
+        outputs['network_files'] = [path, pval_path]
+        return outputs
+
+    def _gen_outfilename(self, name, ext):
+        return name + '.' + ext

nipype/interfaces/fsl/preprocess.py

@@ -133,7 +133,7 @@ def _gen_outfilename(self):
         if not isdefined(out_file) and isdefined(self.inputs.in_file):
             out_file = self._gen_fname(self.inputs.in_file,
                                        suffix='_brain')
-        return out_file
+        return os.path.abspath(out_file)
 
     def _list_outputs(self):
         outputs = self.output_spec().get()
@@ -474,6 +474,7 @@ def _list_outputs(self):
         if not isdefined(outputs['out_file']) and isdefined(self.inputs.in_file):
             outputs['out_file'] = self._gen_fname(self.inputs.in_file,
                                                  suffix='_flirt')
+        outputs['out_file'] = os.path.abspath(self.inputs.out_file)
         outputs['out_matrix_file'] = self.inputs.out_matrix_file
         # Generate an out_matrix file if one is not provided
         if not isdefined(outputs['out_matrix_file']) and \
@@ -904,7 +905,7 @@ def _list_outputs(self):
             outputs['out_file'] = self._gen_fname(self.inputs.in_file,
                                              suffix='_warp')
         else:
-            outputs['out_file'] = self.inputs.out_file
+            outputs['out_file'] = os.path.abspath(self.inputs.out_file)
         return outputs
 
     def _gen_filename(self, name):
@@ -963,7 +964,7 @@ def _list_outputs(self):
         if not isdefined(out_file):
             out_file = self._gen_fname(self.inputs.in_file,
                                       suffix='_st')
-        outputs['slice_time_corrected_file'] = out_file
+        outputs['slice_time_corrected_file'] = os.path.abspath(out_file)
         return outputs
 
     def _gen_filename(self, name):
@@ -1041,7 +1042,7 @@ def _list_outputs(self):
         if not isdefined(out_file):
             out_file = self._gen_fname(self.inputs.in_file,
                                       suffix='_smooth')
-        outputs['smoothed_file'] = out_file
+        outputs['smoothed_file'] = os.path.abspath(out_file)
         return outputs
 
     def _gen_filename(self, name):
@@ -1143,7 +1144,7 @@ def _list_outputs(self):
         if not isdefined(out_file):
             out_file = self._gen_fname(self.inputs.in_file,
                                       suffix='_unwarped')
-        outputs['unwarped_file'] = out_file
+        outputs['unwarped_file'] = os.path.abspath(out_file)
         return outputs
 
     def _gen_filename(self, name):
@@ -1222,7 +1223,7 @@ def _list_outputs(self):
         if not isdefined(out_file):
             out_file = self._gen_fname(self.inputs.in_file,
                                       suffix='_unwrapped')
-        outputs['unwrapped_phase_file'] = out_file
+        outputs['unwrapped_phase_file'] = os.path.abspath(out_file)
         return outputs
 
     def _gen_filename(self, name):

nipype/interfaces/tests/test_base.py

@@ -263,3 +263,18 @@ def _gen_filename(self, name):
     ci6 = DerivedClass(command='cmd')
     yield assert_equal, ci6._parse_inputs()[0], 'filename'
     nib.CommandLine.input_spec = nib.CommandLineInputSpec
+
+
+def test_Commandline_environ():
+    from nipype import config
+    config.set_default_config()
+    ci3 = nib.CommandLine(command='echo')
+    res = ci3.run()
+    yield assert_equal, res.runtime.environ['DISPLAY'], ':1'
+    config.set('execution', 'display_variable', ':3')
+    res = ci3.run()
+    yield assert_false, 'DISPLAY' in ci3.inputs.environ
+    yield assert_equal, res.runtime.environ['DISPLAY'], ':3'
+    ci3.inputs.environ = {'DISPLAY' : ':2'}
+    res = ci3.run()
+    yield assert_equal, res.runtime.environ['DISPLAY'], ':2'

nipype/utils/config.py

@@ -32,6 +32,7 @@
 [execution]
 create_report = true
 crashdump_dir = %s
+display_variable = :1
 hash_method = timestamp
 job_finished_timeout = 5
 keep_inputs = false