Skip to content

Commit 44d8c42

Browse files
chrisgorgochrisgorgo
committed
Merge pull request #503 from satra/enh/stdout2
enh: allow commandline interfaces to control terminal output
2 parents b65c5f3 + 5c49fb2 commit 44d8c42

File tree

17 files changed

+183
-54
lines changed

17 files changed

+183
-54
lines changed

‎CHANGES

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ Next release
22
============
33
* ENH: New interfaces: nipy.Trim
44

5+
* ENH: Allow control over terminal output for commandline interfaces
56

67
Release 0.7.0 (Dec 18, 2012)
78
============================

‎nipype/interfaces/afni/base.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ def version():
4242
Version number as string or None if AFNI not found
4343
4444
"""
45-
clout = CommandLine(command='afni_vcheck').run()
45+
clout = CommandLine(command='afni_vcheck',
46+
terminal_output='allatonce').run()
4647
out = clout.runtime.stdout
4748
return out.split('\n')[1]
4849

@@ -87,7 +88,8 @@ def standard_image(img_name):
8788
'''Grab an image from the standard location.
8889
8990
Could be made more fancy to allow for more relocatability'''
90-
clout = CommandLine('which afni').run()
91+
clout = CommandLine('which afni',
92+
terminal_output='allatonce').run()
9193
if clout.runtime.returncode is not 0:
9294
return None
9395

‎nipype/interfaces/base.py

Lines changed: 92 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1024,7 +1024,7 @@ def _read(self, drain):
10241024
self._lastidx = len(self._rows)
10251025

10261026

1027-
def run_command(runtime, timeout=0.01):
1027+
def run_command(runtime, output=None, timeout=0.01):
10281028
"""
10291029
Run a command, read stdout and stderr, prefix with timestamp. The returned
10301030
runtime contains a merged stdout+stderr log with timestamps
@@ -1038,50 +1038,81 @@ def run_command(runtime, timeout=0.01):
10381038
shell=True,
10391039
cwd=runtime.cwd,
10401040
env=runtime.environ)
1041-
streams = [
1042-
Stream('stdout', proc.stdout),
1043-
Stream('stderr', proc.stderr)
1044-
]
1041+
result = {}
1042+
if output == 'stream':
1043+
streams = [
1044+
Stream('stdout', proc.stdout),
1045+
Stream('stderr', proc.stderr)
1046+
]
10451047

1046-
def _process(drain=0):
1047-
try:
1048-
res = select.select(streams, [], [], timeout)
1049-
except select.error, e:
1050-
iflogger.info(str(e))
1051-
if e[0] == errno.EINTR:
1052-
return
1048+
def _process(drain=0):
1049+
try:
1050+
res = select.select(streams, [], [], timeout)
1051+
except select.error, e:
1052+
iflogger.info(str(e))
1053+
if e[0] == errno.EINTR:
1054+
return
1055+
else:
1056+
raise
10531057
else:
1054-
raise
1055-
else:
1056-
for stream in res[0]:
1057-
stream.read(drain)
1058-
1059-
while proc.returncode is None:
1060-
proc.poll()
1061-
_process()
1062-
runtime.returncode = proc.returncode
1063-
_process(drain=1)
1064-
1065-
# collect results, merge and return
1066-
result = {}
1067-
temp = []
1068-
for stream in streams:
1069-
rows = stream._rows
1070-
temp += rows
1071-
result[stream._name] = [r[2] for r in rows]
1072-
temp.sort()
1073-
result['merged'] = [r[1] for r in temp]
1058+
for stream in res[0]:
1059+
stream.read(drain)
1060+
1061+
while proc.returncode is None:
1062+
proc.poll()
1063+
_process()
1064+
_process(drain=1)
1065+
1066+
# collect results, merge and return
1067+
result = {}
1068+
temp = []
1069+
for stream in streams:
1070+
rows = stream._rows
1071+
temp += rows
1072+
result[stream._name] = [r[2] for r in rows]
1073+
temp.sort()
1074+
result['merged'] = [r[1] for r in temp]
1075+
if output == 'allatonce':
1076+
stdout, stderr = proc.communicate()
1077+
result['stdout'] = stdout.split('\n')
1078+
result['stderr'] = stderr.split('\n')
1079+
result['merged'] = ''
1080+
if output == 'file':
1081+
errfile = os.path.join(runtime.cwd, 'stderr.nipype')
1082+
outfile = os.path.join(runtime.cwd, 'stdout.nipype')
1083+
stderr = open(errfile, 'wt')
1084+
stdout = open(outfile, 'wt')
1085+
proc = subprocess.Popen(runtime.cmdline,
1086+
stdout=stdout,
1087+
stderr=stderr,
1088+
shell=True,
1089+
cwd=runtime.cwd,
1090+
env=runtime.environ)
1091+
ret_code = proc.wait()
1092+
stderr.flush()
1093+
stdout.flush()
1094+
result['stdout'] = [line.strip() for line in open(outfile).readlines()]
1095+
result['stderr'] = [line.strip() for line in open(errfile).readlines()]
1096+
result['merged'] = ''
1097+
if output == 'none':
1098+
proc.communicate()
1099+
result['stdout'] = []
1100+
result['stderr'] = []
1101+
result['merged'] = ''
10741102
runtime.stderr = '\n'.join(result['stderr'])
10751103
runtime.stdout = '\n'.join(result['stdout'])
10761104
runtime.merged = result['merged']
1105+
runtime.returncode = proc.returncode
10771106
return runtime
10781107

10791108

10801109
class CommandLineInputSpec(BaseInterfaceInputSpec):
10811110
args = traits.Str(argstr='%s', desc='Additional parameters to the command')
10821111
environ = traits.DictStrStr(desc='Environment variables', usedefault=True,
10831112
nohash=True)
1084-
1113+
terminal_output = traits.Enum('stream', 'allatonce', 'file', 'none',
1114+
desc='Control terminal output', nohash=True,
1115+
mandatory=True)
10851116

10861117
class CommandLine(BaseInterface):
10871118
"""Implements functionality to interact with command line programs
@@ -1107,7 +1138,7 @@ class must be instantiated with a command argument
11071138
'ls -al'
11081139
11091140
>>> cli.inputs.trait_get()
1110-
{'ignore_exception': False, 'args': '-al', 'environ': {'DISPLAY': ':1'}}
1141+
{'ignore_exception': False, 'terminal_output': 'stream', 'environ': {'DISPLAY': ':1'}, 'args': '-al'}
11111142
11121143
>>> cli.inputs.get_hashval()
11131144
({'args': '-al'}, 'a2f45e04a34630c5f33a75ea2a533cdd')
@@ -1117,6 +1148,7 @@ class must be instantiated with a command argument
11171148
input_spec = CommandLineInputSpec
11181149
_cmd = None
11191150
_version = None
1151+
_terminal_output = 'stream'
11201152

11211153
def __init__(self, command=None, **inputs):
11221154
super(CommandLine, self).__init__(**inputs)
@@ -1127,6 +1159,31 @@ def __init__(self, command=None, **inputs):
11271159
raise Exception("Missing command")
11281160
if command:
11291161
self._cmd = command
1162+
self.inputs.on_trait_change(self._terminal_output_update,
1163+
'terminal_output')
1164+
if not isdefined(self.inputs.terminal_output):
1165+
self.inputs.terminal_output = self._terminal_output
1166+
else:
1167+
self._terminal_output_update()
1168+
1169+
def _terminal_output_update(self):
1170+
self._terminal_output = self.inputs.terminal_output
1171+
1172+
@classmethod
1173+
def set_default_terminal_output(cls, output_type):
1174+
"""Set the default output type for FSL classes.
1175+
1176+
This method is used to set the default output type for all fSL
1177+
subclasses. However, setting this will not update the output
1178+
type for any existing instances. For these, assign the
1179+
<instance>.inputs.output_type.
1180+
"""
1181+
1182+
if output_type in ['stream', 'allatonce', 'file', 'none']:
1183+
cls._terminal_output = output_type
1184+
else:
1185+
raise AttributeError('Invalid terminal output_type: %s' %
1186+
output_type)
11301187

11311188
@property
11321189
def cmd(self):
@@ -1207,7 +1264,7 @@ def _run_interface(self, runtime):
12071264
if not self._exists_in_path(self.cmd.split()[0]):
12081265
raise IOError("%s could not be found on host %s" % (self.cmd.split()[0],
12091266
runtime.hostname))
1210-
runtime = run_command(runtime)
1267+
runtime = run_command(runtime, output=self.inputs.terminal_output)
12111268
if runtime.returncode is None or runtime.returncode != 0:
12121269
self.raise_exception(runtime)
12131270

‎nipype/interfaces/diffusion_toolkit/base.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ def version():
4242
Version number as string or None if FSL not found
4343
4444
"""
45-
clout = CommandLine(command='dti_recon').run()
45+
clout = CommandLine(command='dti_recon',
46+
terminal_output='allatonce').run()
4647

4748
if clout.runtime.returncode is not 0:
4849
return None

‎nipype/interfaces/fsl/base.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ def version():
7373
except KeyError:
7474
return None
7575
clout = CommandLine(command='cat',
76-
args='%s/etc/fslversion' % (basedir)).run()
76+
args='%s/etc/fslversion' % (basedir),
77+
terminal_output='allatonce').run()
7778
out = clout.runtime.stdout
7879
return out.strip('\n')
7980

‎nipype/interfaces/fsl/tests/test_preprocess.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,8 @@ def test_flirt():
219219
# Skip mandatory inputs and the trait methods
220220
if key in ('trait_added', 'trait_modified', 'in_file', 'reference',
221221
'environ', 'output_type', 'out_file', 'out_matrix_file',
222-
'in_matrix_file', 'apply_xfm', 'ignore_exception'):
222+
'in_matrix_file', 'apply_xfm', 'ignore_exception',
223+
'terminal_output'):
223224
continue
224225
param = None
225226
value = None

‎nipype/interfaces/matlab.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ def get_matlab_command():
1717
matlab_cmd = 'matlab'
1818

1919
try:
20-
res = CommandLine(command='which', args=matlab_cmd).run()
20+
res = CommandLine(command='which', args=matlab_cmd,
21+
terminal_output='allatonce').run()
2122
matlab_path = res.runtime.stdout.strip()
2223
except Exception, e:
2324
return None
@@ -95,6 +96,9 @@ def __init__(self, matlab_cmd = None, **inputs):
9596
not isdefined(self.inputs.uses_mcr):
9697
if config.getboolean('execution','single_thread_matlab'):
9798
self.inputs.single_comp_thread = True
99+
# For matlab commands force all output to be returned since matlab
100+
# does not have a clean way of notifying an error
101+
self.inputs.terminal_output = 'allatonce'
98102

99103
@classmethod
100104
def set_default_matlab_cmd(cls, matlab_cmd):
@@ -130,6 +134,7 @@ def set_default_paths(cls, paths):
130134
cls._default_paths = paths
131135

132136
def _run_interface(self,runtime):
137+
self.inputs.terminal_output = 'allatonce'
133138
runtime = super(MatlabCommand, self)._run_interface(runtime)
134139
try:
135140
# Matlab can leave the terminal in a barbbled state

‎nipype/interfaces/tests/test_base.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,3 +461,53 @@ def test_Commandline_environ():
461461
ci3.inputs.environ = {'DISPLAY' : ':2'}
462462
res = ci3.run()
463463
yield assert_equal, res.runtime.environ['DISPLAY'], ':2'
464+
465+
def test_CommandLine_output():
466+
tmp_infile = setup_file()
467+
tmpd, name = os.path.split(tmp_infile)
468+
pwd = os.getcwd()
469+
os.chdir(tmpd)
470+
yield assert_true, os.path.exists(tmp_infile)
471+
ci = nib.CommandLine(command='ls -l')
472+
ci.inputs.terminal_output = 'allatonce'
473+
res = ci.run()
474+
yield assert_equal, res.runtime.merged, ''
475+
yield assert_true, name in res.runtime.stdout
476+
ci = nib.CommandLine(command='ls -l')
477+
ci.inputs.terminal_output = 'file'
478+
res = ci.run()
479+
yield assert_true, 'stdout.nipype' in res.runtime.stdout
480+
ci = nib.CommandLine(command='ls -l')
481+
ci.inputs.terminal_output = 'none'
482+
res = ci.run()
483+
yield assert_equal, res.runtime.stdout, ''
484+
ci = nib.CommandLine(command='ls -l')
485+
res = ci.run()
486+
yield assert_true, 'stdout.nipype' in res.runtime.stdout
487+
os.chdir(pwd)
488+
teardown_file(tmpd)
489+
490+
def test_global_CommandLine_output():
491+
tmp_infile = setup_file()
492+
tmpd, name = os.path.split(tmp_infile)
493+
pwd = os.getcwd()
494+
os.chdir(tmpd)
495+
ci = nib.CommandLine(command='ls -l')
496+
res = ci.run()
497+
yield assert_true, name in res.runtime.stdout
498+
yield assert_true, os.path.exists(tmp_infile)
499+
nib.CommandLine.set_default_terminal_output('allatonce')
500+
ci = nib.CommandLine(command='ls -l')
501+
res = ci.run()
502+
yield assert_equal, res.runtime.merged, ''
503+
yield assert_true, name in res.runtime.stdout
504+
nib.CommandLine.set_default_terminal_output('file')
505+
ci = nib.CommandLine(command='ls -l')
506+
res = ci.run()
507+
yield assert_true, 'stdout.nipype' in res.runtime.stdout
508+
nib.CommandLine.set_default_terminal_output('none')
509+
ci = nib.CommandLine(command='ls -l')
510+
res = ci.run()
511+
yield assert_equal, res.runtime.stdout, ''
512+
os.chdir(pwd)
513+
teardown_file(tmpd)

‎nipype/pipeline/plugins/condor.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ def __init__(self, **kwargs):
4545
super(CondorPlugin, self).__init__(template, **kwargs)
4646

4747
def _is_pending(self, taskid):
48-
cmd = CommandLine('condor_q')
48+
cmd = CommandLine('condor_q',
49+
terminal_output='allatonce')
4950
cmd.inputs.args = '%d' % taskid
5051
# check condor cluster
5152
oldlevel = iflogger.level
@@ -57,7 +58,8 @@ def _is_pending(self, taskid):
5758
return False
5859

5960
def _submit_batchtask(self, scriptfile, node):
60-
cmd = CommandLine('condor_qsub', environ=os.environ.data)
61+
cmd = CommandLine('condor_qsub', environ=os.environ.data,
62+
terminal_output='allatonce')
6163
path = os.path.dirname(scriptfile)
6264
qsubargs = ''
6365
if self._qsub_args:

‎nipype/pipeline/plugins/dagman.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ def _submit_graph(self, pyfiles, dependencies, nodes):
9898
% (' '.join([str(i) for i in parents]),
9999
child))
100100
# hand over DAG to condor_dagman
101-
cmd = CommandLine('condor_submit_dag', environ=os.environ.data)
101+
cmd = CommandLine('condor_submit_dag', environ=os.environ.data,
102+
terminal_output='allatonce')
102103
# needs -update_submit or re-running a workflow will fail
103104
cmd.inputs.args = '-update_submit %s %s' % (dagfilename,
104105
self._dagman_args)

0 commit comments

Comments
 (0)