Merge branch 'enh/FSLstd2imgcoords' of github.com:oesteban/nipype into enh/FSLstd2imgcoords

Author: oesteban

CHANGES

@@ -2,6 +2,7 @@ Next release
 ============
 
 * ENH: Simple interface to FSL std2imgcoords (https://github.com/nipy/nipype/pull/1398)
+* FIX: Clean up byte/unicode issues using subprocess (https://github.com/nipy/nipype/pull/1394)
 * FIX: Prevent crash when tvtk is loaded - ETS_TOOLKIT=null (https://github.com/nipy/nipype/pull/973)
 * ENH: New interfaces in dipy: RESTORE, EstimateResponseSH, CSD and StreamlineTractography
        (https://github.com/nipy/nipype/pull/1090)
@@ -22,6 +23,7 @@ Next release
 * API: Default model level for the bedpostx workflow has been set to "2" following FSL 5.0.9 lead
 * ENH: New interfaces for interacting with AWS S3: S3DataSink and S3DataGrabber (https://github.com/nipy/nipype/pull/1201)
 * ENH: Interfaces for MINC tools (https://github.com/nipy/nipype/pull/1304)
+* FIX: Use realpath to determine hard link source (https://github.com/nipy/nipype/pull/1388)
 
 Release 0.11.0 (September 15, 2015)
 ============

nipype/interfaces/base.py

@@ -19,6 +19,7 @@
 from copy import deepcopy
 import datetime
 import errno
+import locale
 import os
 import re
 import platform
@@ -47,7 +48,7 @@
 from ..utils.provenance import write_provenance
 from .. import config, logging, LooseVersion
 from .. import __version__
-from ..external.six import string_types
+from ..external.six import string_types, text_type
 
 nipype_version = LooseVersion(__version__)
 
@@ -1205,8 +1206,8 @@ def run_command(runtime, output=None, timeout=0.01, redirect_x=False):
     if output == 'file':
         errfile = os.path.join(runtime.cwd, 'stderr.nipype')
         outfile = os.path.join(runtime.cwd, 'stdout.nipype')
-        stderr = open(errfile, 'wt')  # t=='text'===default
-        stdout = open(outfile, 'wt')
+        stderr = open(errfile, 'wb')  # t=='text'===default
+        stdout = open(outfile, 'wb')
 
         proc = subprocess.Popen(cmdline,
                                 stdout=stdout,
@@ -1256,26 +1257,17 @@ def _process(drain=0):
         result['merged'] = [r[1] for r in temp]
     if output == 'allatonce':
         stdout, stderr = proc.communicate()
-        if stdout and isinstance(stdout, bytes):
-            try:
-                stdout = stdout.decode()
-            except UnicodeDecodeError:
-                stdout = stdout.decode("ISO-8859-1")
-        if stderr and isinstance(stderr, bytes):
-            try:
-                stderr = stderr.decode()
-            except UnicodeDecodeError:
-                stderr = stderr.decode("ISO-8859-1")
-
-        result['stdout'] = str(stdout).split('\n')
-        result['stderr'] = str(stderr).split('\n')
+        stdout = stdout.decode(locale.getdefaultlocale()[1])
+        stderr = stderr.decode(locale.getdefaultlocale()[1])
+        result['stdout'] = stdout.split('\n')
+        result['stderr'] = stderr.split('\n')
         result['merged'] = ''
     if output == 'file':
         ret_code = proc.wait()
         stderr.flush()
         stdout.flush()
-        result['stdout'] = [line.strip() for line in open(outfile).readlines()]
-        result['stderr'] = [line.strip() for line in open(errfile).readlines()]
+        result['stdout'] = [line.decode(locale.getdefaultlocale()[1]).strip() for line in open(outfile, 'rb').readlines()]
+        result['stderr'] = [line.decode(locale.getdefaultlocale()[1]).strip() for line in open(errfile, 'rb').readlines()]
         result['merged'] = ''
     if output == 'none':
         proc.communicate()

nipype/interfaces/spm/tests/test_base.py

@@ -12,7 +12,7 @@
 from nipype.interfaces.spm import no_spm
 import nipype.interfaces.matlab as mlab
 from nipype.interfaces.spm.base import SPMCommandInputSpec
-from nipype.interfaces.base import traits
+from nipype.interfaces.base import traits, text_type
 
 try:
     matlab_cmd = os.environ['MATLABCMD']
@@ -57,7 +57,7 @@ def test_scan_for_fnames():
     def test_spm_path():
         spm_path = spm.Info.version()['path']
         if spm_path is not None:
-            yield assert_equal, type(spm_path), type('')
+            yield assert_equal, type(spm_path), text_type
             yield assert_true, 'spm' in spm_path
 
 

nipype/interfaces/tests/test_base.py

@@ -14,7 +14,7 @@
                             skipif)
 import nipype.interfaces.base as nib
 from nipype.utils.filemanip import split_filename
-from nipype.interfaces.base import Undefined, config
+from nipype.interfaces.base import Undefined, config, text_type
 from traits.testing.nose_tools import skip
 import traits.api as traits
 
@@ -661,7 +661,7 @@ def test_CommandLine_output():
     ci.inputs.terminal_output = 'file'
     res = ci.run()
     yield assert_true, 'stdout.nipype' in res.runtime.stdout
-    yield assert_equal, type(res.runtime.stdout), type('hi')
+    yield assert_equal, type(res.runtime.stdout), text_type
     ci = nib.CommandLine(command='ls -l')
     ci.inputs.terminal_output = 'none'
     res = ci.run()

nipype/pipeline/plugins/slurmgraph.py

@@ -135,7 +135,7 @@ def make_job_name(jobnumber, nodeslist):
                     if self._sbatch_args.count('-o ') == 0:
                         stdoutFile = '-o {outFile}'.format(
                             outFile=batchscriptoutfile)
-                    full_line = '{jobNm}=$(sbatch {outFileOption} {errFileOption} {extraSBatchArgs} {dependantIndex} -J {jobNm} {batchscript} | awk \'{{print $4}}\')\n'.format(
+                    full_line = '{jobNm}=$(sbatch {outFileOption} {errFileOption} {extraSBatchArgs} {dependantIndex} -J {jobNm} {batchscript} | awk \'/^Submitted/ {{print $4}}\')\n'.format(
                         jobNm=jobname,
                         outFileOption=stdoutFile,
                         errFileOption=stderrFile,

nipype/utils/filemanip.py

@@ -42,7 +42,9 @@ def nipype_hardlink_wrapper(raw_src, raw_dst):
     If the hardlink fails, then fall back to using
     a standard copy.
     """
-    src = os.path.normpath(raw_src)
+    # Use realpath to avoid hardlinking symlinks
+    src = os.path.realpath(raw_src)
+    # Use normpath, in case destination is a symlink
     dst = os.path.normpath(raw_dst)
     del raw_src
     del raw_dst
@@ -283,12 +285,15 @@ def copyfile(originalfile, newfile, copy=False, create_new=False,
         matofile = originalfile[:-4] + ".mat"
         if os.path.exists(matofile):
             matnfile = newfile[:-4] + ".mat"
-            copyfile(matofile, matnfile, copy)
-        copyfile(hdrofile, hdrnfile, copy)
+            copyfile(matofile, matnfile, copy, create_new, hashmethod,
+                     use_hardlink)
+        copyfile(hdrofile, hdrnfile, copy, create_new, hashmethod,
+                 use_hardlink)
     elif originalfile.endswith(".BRIK"):
         hdrofile = originalfile[:-5] + ".HEAD"
         hdrnfile = newfile[:-5] + ".HEAD"
-        copyfile(hdrofile, hdrnfile, copy)
+        copyfile(hdrofile, hdrnfile, copy, create_new, hashmethod,
+                 use_hardlink)
 
     return newfile
 

nipype/utils/tests/test_filemanip.py

@@ -132,6 +132,41 @@ def test_copyfiles():
     os.unlink(new_hdr2)
 
 
+def test_linkchain():
+    if os.name is not 'posix':
+        return
+    orig_img, orig_hdr = _temp_analyze_files()
+    pth, fname = os.path.split(orig_img)
+    new_img1 = os.path.join(pth, 'newfile1.img')
+    new_hdr1 = os.path.join(pth, 'newfile1.hdr')
+    new_img2 = os.path.join(pth, 'newfile2.img')
+    new_hdr2 = os.path.join(pth, 'newfile2.hdr')
+    new_img3 = os.path.join(pth, 'newfile3.img')
+    new_hdr3 = os.path.join(pth, 'newfile3.hdr')
+    copyfile(orig_img, new_img1)
+    yield assert_true, os.path.islink(new_img1)
+    yield assert_true, os.path.islink(new_hdr1)
+    copyfile(new_img1, new_img2, copy=True)
+    yield assert_false, os.path.islink(new_img2)
+    yield assert_false, os.path.islink(new_hdr2)
+    yield assert_false, os.path.samefile(orig_img, new_img2)
+    yield assert_false, os.path.samefile(orig_hdr, new_hdr2)
+    copyfile(new_img1, new_img3, copy=True, use_hardlink=True)
+    yield assert_false, os.path.islink(new_img3)
+    yield assert_false, os.path.islink(new_hdr3)
+    yield assert_true, os.path.samefile(orig_img, new_img3)
+    yield assert_true, os.path.samefile(orig_hdr, new_hdr3)
+    os.unlink(new_img1)
+    os.unlink(new_hdr1)
+    os.unlink(new_img2)
+    os.unlink(new_hdr2)
+    os.unlink(new_img3)
+    os.unlink(new_hdr3)
+    # final cleanup
+    os.unlink(orig_img)
+    os.unlink(orig_hdr)
+
+
 def test_filename_to_list():
     x = filename_to_list('foo.nii')
     yield assert_equal, x, ['foo.nii']