Merge remote-tracking branch 'upstream/master' into ref/interface-base

Author: oesteban

CHANGES

@@ -1,11 +1,21 @@
 Upcoming release
 ================
 
-0.14.0 (November 21, 2017)
+
+0.14.0 (November 29, 2017)
 ==========================
 
 ###### [Full changelog](https://github.com/nipy/nipype/milestone/13)
 
+* FIX+MAINT: Revision of the resource monitor (https://github.com/nipy/nipype/pull/2285)
+* FIX: MultiProc mishandling crashes (https://github.com/nipy/nipype/pull/2301)
+* MAINT: Revise use of `subprocess.Popen` (https://github.com/nipy/nipype/pull/2289)
+* ENH: Memorize version checks (https://github.com/nipy/nipype/pull/2274, https://github.com/nipy/nipype/pull/2295)
+
+
+0.14.0rc1 (November 21, 2017)
+-----------------------------
+
 * ENH: Generate Dockerfiles with neurodocker (https://github.com/nipy/nipype/pull/2202)
 * ENH: FLAIR options for recon-all (https://github.com/nipy/nipype/pull/2279)
 * ENH: Config option for setting maxtasksperchild when multiprocessing (https://github.com/nipy/nipype/pull/2284)

doc/users/config_file.rst

@@ -153,14 +153,29 @@ Execution
     crashfiles allow portability across machines and shorter load time.
     (possible values: ``pklz`` and ``txt``; default value: ``pklz``)
 
-*resource_monitor*
+
+Resource Monitor
+~~~~~~~~~~~~~~~~
+
+*enabled*
     Enables monitoring the resources occupation (possible values: ``true`` and
-    ``false``; default value: ``false``)
+    ``false``; default value: ``false``). All the following options will be
+    dismissed if the resource monitor is not enabled.
 
-*resource_monitor_frequency*
+*sample_frequency*
     Sampling period (in seconds) between measurements of resources (memory, cpus)
-    being used by an interface. Requires ``resource_monitor`` to be ``true``.
-    (default value: ``1``)
+    being used by an interface (default value: ``1``)
+
+*summary_file*
+    Indicates where the summary file collecting all profiling information from the
+    resource monitor should be stored after execution of a workflow.
+    The ``summary_file`` does not apply to interfaces run independently.
+    (unset by default, in which case the summary file will be written out to 
+    ``<base_dir>/resource_monitor.json`` of the top-level workflow).
+
+*summary_append*
+    Append to an existing summary file (only applies to workflows).
+    (default value: ``true``, possible values: ``true`` or ``false``).
 
 Example
 ~~~~~~~
@@ -175,6 +190,10 @@ Example
     hash_method = timestamp
     display_variable = :1
 
+    [monitoring]
+    enabled = false
+
+
 Workflow.config property has a form of a nested dictionary reflecting the
 structure of the .cfg file.
 

docker/files/run_examples.sh

@@ -20,8 +20,9 @@ echo '[execution]' >> ${HOME}/.nipype/nipype.cfg
 echo 'crashfile_format = txt' >> ${HOME}/.nipype/nipype.cfg
 
 if [[ "${NIPYPE_RESOURCE_MONITOR:-0}" == "1" ]]; then
-    echo 'resource_monitor = true' >> ${HOME}/.nipype/nipype.cfg
-    echo 'resource_monitor_frequency = 3' >> ${HOME}/.nipype/nipype.cfg
+    echo '[monitoring]' >> ${HOME}/.nipype/nipype.cfg
+    echo 'enabled = true' >> ${HOME}/.nipype/nipype.cfg
+    echo 'sample_frequency = 3' >> ${HOME}/.nipype/nipype.cfg
 fi
 
 # Set up coverage

nipype/algorithms/confounds.py

@@ -22,7 +22,7 @@
 from numpy.polynomial import Legendre
 from scipy import linalg
 
-from .. import logging
+from .. import config, logging
 from ..external.due import BibTeX
 from ..interfaces.base import (traits, TraitedSpec, BaseInterface,
                                BaseInterfaceInputSpec, File, isdefined,
@@ -816,7 +816,7 @@ def plot_confound(tseries, figsize, name, units=None,
 
     """
     import matplotlib
-    matplotlib.use('Agg')
+    matplotlib.use(config.get('execution', 'matplotlib_backend'))
     import matplotlib.pyplot as plt
     from matplotlib.gridspec import GridSpec
     from matplotlib.backends.backend_pdf import FigureCanvasPdf as FigureCanvas

nipype/algorithms/metrics.py

@@ -24,7 +24,7 @@
 from scipy.spatial.distance import cdist, euclidean, dice, jaccard
 from scipy.ndimage.measurements import center_of_mass, label
 
-from .. import logging
+from .. import config, logging
 from ..utils.misc import package_check
 
 from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File,
@@ -138,6 +138,8 @@ def _eucl_mean(self, nii1, nii2, weighted=False):
 
         dist_matrix = cdist(set1_coordinates.T, set2_coordinates.T)
         min_dist_matrix = np.amin(dist_matrix, axis=0)
+        import matplotlib
+        matplotlib.use(config.get('execution', 'matplotlib_backend'))
         import matplotlib.pyplot as plt
         plt.figure()
         plt.hist(min_dist_matrix, 50, normed=1, facecolor='green')

nipype/interfaces/afni/tests/test_auto_Unifize.py

@@ -32,6 +32,8 @@ def test_Unifize_inputs():
     name_source='in_file',
     ),
     outputtype=dict(),
+    rbt=dict(argstr='-rbt %f %f %f',
+    ),
     scale_file=dict(argstr='-ssave %s',
     ),
     t2=dict(argstr='-T2',

nipype/interfaces/afni/utils.py

@@ -2329,6 +2329,14 @@ class UnifizeInputSpec(AFNICommandInputSpec):
         argstr='-EPI',
         requires=['no_duplo', 't2'],
         xor=['gm'])
+    rbt = traits.Tuple(
+        traits.Float(), traits.Float(), traits.Float(),
+        desc='Option for AFNI experts only.'
+             'Specify the 3 parameters for the algorithm:\n'
+             'R = radius; same as given by option \'-Urad\', [default=18.3]\n'
+             'b = bottom percentile of normalizing data range, [default=70.0]\n'
+             'r = top percentile of normalizing data range, [default=80.0]\n',
+        argstr='-rbt %f %f %f')
 
 
 class UnifizeOutputSpec(TraitedSpec):

nipype/interfaces/ants/registration.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
-"""The ants module provides basic functions for interfacing with ants functions.
+"""The ants module provides basic functions for interfacing with ants
+   functions.
 
    Change directory to provide relative paths for doctests
    >>> import os
    >>> filepath = os.path.dirname( os.path.realpath( __file__ ) )
    >>> datadir = os.path.realpath(os.path.join(filepath, '../../testing/data'))
    >>> os.chdir(datadir)
 """
-from __future__ import print_function, division, unicode_literals, absolute_import
+from __future__ import (print_function, division, unicode_literals,
+                        absolute_import)
 from builtins import range, str
 import os
 
@@ -20,17 +22,19 @@ class ANTSInputSpec(ANTSCommandInputSpec):
     dimension = traits.Enum(3, 2, argstr='%d', usedefault=False,
                             position=1, desc='image dimension (2 or 3)')
     fixed_image = InputMultiPath(File(exists=True), mandatory=True,
-                                 desc=('image to which the moving image is warped'))
+                                 desc=('image to which the moving image is '
+                                       'warped'))
     moving_image = InputMultiPath(File(exists=True), argstr='%s',
                                   mandatory=True,
-                                  desc=('image to apply transformation to (generally a coregistered '
+                                  desc=('image to apply transformation to '
+                                        '(generally a coregistered'
                                         'functional)'))
 
 #    Not all metrics are appropriate for all modalities. Also, not all metrics
-#    are efficeint or appropriate at all resolution levels, Some metrics perform
-#    well for gross global registraiton, but do poorly for small changes (i.e.
-#    Mattes), and some metrics do well for small changes but don't work well for
-#    gross level changes (i.e. 'CC').
+#    are efficeint or appropriate at all resolution levels, Some metrics
+#    perform well for gross global registraiton, but do poorly for small
+#    changes (i.e. Mattes), and some metrics do well for small changes but
+#    don't work well for gross level changes (i.e. 'CC').
 #
 #    This is a two stage registration. in the first stage
 #      [ 'Mattes', .................]
@@ -49,10 +53,18 @@ class ANTSInputSpec(ANTSCommandInputSpec):
     metric = traits.List(traits.Enum('CC', 'MI', 'SMI', 'PR', 'SSD',
                                      'MSQ', 'PSE'), mandatory=True, desc='')
 
-    metric_weight = traits.List(traits.Float(), requires=['metric'], desc='')
-    radius = traits.List(traits.Int(), requires=['metric'], desc='')
+    metric_weight = traits.List(traits.Float(), value=[1.0], usedefault=True,
+                                requires=['metric'], mandatory=True,
+                                desc='the metric weight(s) for each stage. '
+                                'The weights must sum to 1 per stage.')
 
-    output_transform_prefix = Str('out', usedefault=True, argstr='--output-naming %s',
+    radius = traits.List(traits.Int(), requires=['metric'],  mandatory=True,
+                         desc='radius of the region (i.e. number of layers'
+                         ' around a voxel point)'
+                         ' that is used for computing cross correlation')
+
+    output_transform_prefix = Str('out', usedefault=True,
+                                  argstr='--output-naming %s',
                                   mandatory=True, desc='')
     transformation_model = traits.Enum('Diff', 'Elast', 'Exp', 'Greedy Exp',
                                        'SyN', argstr='%s', mandatory=True,

nipype/interfaces/ants/tests/test_registration.py

@@ -0,0 +1,22 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+from __future__ import unicode_literals
+from nipype.interfaces.ants import registration
+import os
+import pytest
+
+
+def test_ants_mand(tmpdir):
+    tmpdir.chdir()
+    filepath = os.path.dirname(os.path.realpath(__file__))
+    datadir = os.path.realpath(os.path.join(filepath, '../../../testing/data'))
+
+    ants = registration.ANTS()
+    ants.inputs.transformation_model = "SyN"
+    ants.inputs.moving_image = [os.path.join(datadir, 'resting.nii')]
+    ants.inputs.fixed_image = [os.path.join(datadir, 'T1.nii')]
+    ants.inputs.metric = ['MI']
+
+    with pytest.raises(ValueError) as er:
+        ants.run()
+    assert "ANTS requires a value for input 'radius'" in str(er.value)

nipype/interfaces/base/core.py

@@ -13,8 +13,10 @@
 
 """
 from __future__ import print_function, division, unicode_literals, absolute_import
+
 from builtins import object, open, str, bytes
 
+import gc
 from copy import deepcopy
 from datetime import datetime as dt
 import errno
@@ -28,7 +30,6 @@
 import simplejson as json
 from dateutil.parser import parse as parseutc
 
-
 from ... import config, logging, LooseVersion
 from ...utils.provenance import write_provenance
 from ...utils.misc import trim, str2bool
@@ -657,6 +658,7 @@ class SimpleInterface(BaseInterface):
     >>> os.chdir(old.strpath)
 
     """
+
     def __init__(self, from_file=None, resource_monitor=None, **inputs):
         super(SimpleInterface, self).__init__(
             from_file=from_file, resource_monitor=resource_monitor, **inputs)
@@ -705,6 +707,7 @@ def run_command(runtime, output=None, timeout=0.01):
                     env=env,
                     close_fds=True,
                     )
+
     result = {
         'stdout': [],
         'stderr': [],
@@ -743,37 +746,50 @@ def _process(drain=0):
         temp.sort()
         result['merged'] = [r[1] for r in temp]
 
-    if output == 'allatonce':
-        stdout, stderr = proc.communicate()
-        result['stdout'] = read_stream(stdout, logger=iflogger)
-        result['stderr'] = read_stream(stderr, logger=iflogger)
-
-    elif output.startswith('file'):
+    if output.startswith('file'):
         proc.wait()
         if outfile is not None:
             stdout.flush()
             stdout.close()
             with open(outfile, 'rb') as ofh:
                 stdoutstr = ofh.read()
             result['stdout'] = read_stream(stdoutstr, logger=iflogger)
+            del stdoutstr
 
         if errfile is not None:
             stderr.flush()
             stderr.close()
             with open(errfile, 'rb') as efh:
                 stderrstr = efh.read()
             result['stderr'] = read_stream(stderrstr, logger=iflogger)
+            del stderrstr
 
         if output == 'file':
             result['merged'] = result['stdout']
             result['stdout'] = []
     else:
-        proc.communicate()  # Discard stdout and stderr
+        stdout, stderr = proc.communicate()
+        if output == 'allatonce':  # Discard stdout and stderr otherwise
+            result['stdout'] = read_stream(stdout, logger=iflogger)
+            result['stderr'] = read_stream(stderr, logger=iflogger)
+
+    runtime.returncode = proc.returncode
+    try:
+        proc.terminate()  # Ensure we are done
+    except OSError as error:
+        # Python 2 raises when the process is already gone
+        if error.errno != errno.ESRCH:
+            raise
+
+    # Dereference & force GC for a cleanup
+    del proc
+    del stdout
+    del stderr
+    gc.collect()
 
     runtime.stderr = '\n'.join(result['stderr'])
     runtime.stdout = '\n'.join(result['stdout'])
     runtime.merged = '\n'.join(result['merged'])
-    runtime.returncode = proc.returncode
     return runtime
 
 
@@ -847,6 +863,9 @@ def __init__(self, command=None, terminal_output=None, **inputs):
         # Set command. Input argument takes precedence
         self._cmd = command or getattr(self, '_cmd', None)
 
+        # Store dependencies in runtime object
+        self._ldd = str2bool(config.get('execution', 'get_linked_libs', 'true'))
+
         if self._cmd is None:
             raise Exception("Missing command")
 
@@ -895,8 +914,11 @@ def _get_environ(self):
         return getattr(self.inputs, 'environ', {})
 
     def version_from_command(self, flag='-v', cmd=None):
+        iflogger.warning('version_from_command member of CommandLine was '
+                         'Deprecated in nipype-1.0.0 and deleted in 1.1.0')
         if cmd is None:
             cmd = self.cmd.split()[0]
+
         env = dict(os.environ)
         if which(cmd, env=env):
             out_environ = self._get_environ()
@@ -942,7 +964,8 @@ def _run_interface(self, runtime, correct_return_codes=(0,)):
                     executable_name, runtime.hostname))
 
         runtime.command_path = cmd_path
-        runtime.dependencies = get_dependencies(executable_name, runtime.environ)
+        runtime.dependencies = (get_dependencies(executable_name, runtime.environ)
+                                if self._ldd else '<skipped>')
         runtime = run_command(runtime, output=self.terminal_output)
         if runtime.returncode is None or \
                 runtime.returncode not in correct_return_codes: