Merge pull request #1401 from satra/fix/jobexecute_log

FIX: job execution on systems/approaches where locale is undefined

Author: satra

CHANGES

@@ -1,6 +1,7 @@
 Next release
 ============
 
+* FIX: job execution on systems/approaches where locale is undefined (https://github.com/nipy/nipype/pull/1401)
 * FIX: Clean up byte/unicode issues using subprocess (https://github.com/nipy/nipype/pull/1394)
 * FIX: Prevent crash when tvtk is loaded - ETS_TOOLKIT=null (https://github.com/nipy/nipype/pull/973)
 * ENH: New interfaces in dipy: RESTORE, EstimateResponseSH, CSD and StreamlineTractography

circle.yml

@@ -29,7 +29,7 @@ dependencies:
     - pip install -e git+https://github.com/enthought/etsproxy.git#egg=etsproxy
     - pip install -e git+https://github.com/enthought/ets.git#egg=ets
     - gem install fakes3
-    - if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
+    - if [[ ! -d ~/examples/data ]]; then wget "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
     - if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && mkdir ~/examples/fsl_course_data && tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
     - bash ~/nipype/tools/install_spm_mcr.sh
     - mkdir -p ~/.nipype && echo '[logging]' > ~/.nipype/nipype.cfg && echo 'workflow_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'interface_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'filemanip_level = DEBUG' >> ~/.nipype/nipype.cfg

doc/documentation.rst

@@ -9,7 +9,7 @@ Documentation
    :Release: |version|
    :Date: |today|
 
-Previous versions:  `0.10.0 <http://nipy.org/nipype/0.10.0>`_ `0.9.2 <http://nipy.org/nipype/0.9.2>`_
+Previous versions:  `0.11.0 <http://nipy.org/nipype/0.11.0>`_ `0.10.0 <http://nipy.org/nipype/0.10.0>`_
 
 .. container:: doc2
 

doc/users/pipeline_tutorial.rst

@@ -63,7 +63,7 @@ Step 0
 ~~~~~~
 
 Download and extract the `Pipeline tutorial data (429MB).
-<http://sourceforge.net/projects/nipy/files/nipype/nipype-0.2/nipype-tutorial.tar.bz2/download>`_
+<https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2>`_
 
 (checksum: 56ed4b7e0aac5627d1724e9c10cd26a7)
 

nipype/interfaces/base.py

@@ -1150,6 +1150,9 @@ def __init__(self, name, impl):
         self._buf = ''
         self._rows = []
         self._lastidx = 0
+        self.default_encoding = locale.getdefaultlocale()[1]
+        if self.default_encoding is None:
+            self.default_encoding = 'UTF-8'
 
     def fileno(self):
         "Pass-through for file descriptor."
@@ -1164,7 +1167,7 @@ def read(self, drain=0):
     def _read(self, drain):
         "Read from the file descriptor"
         fd = self.fileno()
-        buf = os.read(fd, 4096).decode()
+        buf = os.read(fd, 4096).decode(self.default_encoding)
         if not buf and not self._buf:
             return None
         if '\n' not in buf:
@@ -1203,6 +1206,9 @@ def run_command(runtime, output=None, timeout=0.01, redirect_x=False):
             raise RuntimeError('Xvfb was not found, X redirection aborted')
         cmdline = 'xvfb-run -a ' + cmdline
 
+    default_encoding = locale.getdefaultlocale()[1]
+    if default_encoding is None:
+        default_encoding = 'UTF-8'
     if output == 'file':
         errfile = os.path.join(runtime.cwd, 'stderr.nipype')
         outfile = os.path.join(runtime.cwd, 'stdout.nipype')
@@ -1257,17 +1263,17 @@ def _process(drain=0):
         result['merged'] = [r[1] for r in temp]
     if output == 'allatonce':
         stdout, stderr = proc.communicate()
-        stdout = stdout.decode(locale.getdefaultlocale()[1])
-        stderr = stderr.decode(locale.getdefaultlocale()[1])
+        stdout = stdout.decode(default_encoding)
+        stderr = stderr.decode(default_encoding)
         result['stdout'] = stdout.split('\n')
         result['stderr'] = stderr.split('\n')
         result['merged'] = ''
     if output == 'file':
         ret_code = proc.wait()
         stderr.flush()
         stdout.flush()
-        result['stdout'] = [line.decode(locale.getdefaultlocale()[1]).strip() for line in open(outfile, 'rb').readlines()]
-        result['stderr'] = [line.decode(locale.getdefaultlocale()[1]).strip() for line in open(errfile, 'rb').readlines()]
+        result['stdout'] = [line.decode(default_encoding).strip() for line in open(outfile, 'rb').readlines()]
+        result['stderr'] = [line.decode(default_encoding).strip() for line in open(errfile, 'rb').readlines()]
         result['merged'] = ''
     if output == 'none':
         proc.communicate()

nipype/pipeline/plugins/base.py

@@ -363,7 +363,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None):
                     self.proc_done[jobid] = True
                     self.proc_pending[jobid] = True
                     # Send job to task manager and add to pending tasks
-                    logger.info('Executing: %s ID: %d' %
+                    logger.info('Submitting: %s ID: %d' %
                                 (self.procs[jobid]._id, jobid))
                     if self._status_callback:
                         self._status_callback(self.procs[jobid], 'start')
@@ -405,7 +405,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None):
                                 self.proc_pending[jobid] = False
                             else:
                                 self.pending_tasks.insert(0, (tid, jobid))
-                    logger.info('Finished executing: %s ID: %d' %
+                    logger.info('Finished submitting: %s ID: %d' %
                                 (self.procs[jobid]._id, jobid))
             else:
                 break

nipype/pipeline/plugins/pbs.py

@@ -7,7 +7,7 @@
 
 from .base import (SGELikeBatchManagerBase, logger, iflogger, logging)
 
-from nipype.interfaces.base import CommandLine
+from ...interfaces.base import CommandLine, text_type
 
 
 class PBSPlugin(SGELikeBatchManagerBase):
@@ -97,7 +97,7 @@ def _submit_batchtask(self, scriptfile, node):
                     iflogger.setLevel(oldlevel)
                     raise RuntimeError('\n'.join((('Could not submit pbs task'
                                                    ' for node %s') % node._id,
-                                                  str(e))))
+                                                  text_type(e))))
             else:
                 break
         iflogger.setLevel(oldlevel)