Skip to content

Commit cf7b331

Browse files
tclosetclose
committed
cleaned up insertion of special stdout, stderr and return_code outputs form shell commands
1 parent 328e0b1 commit cf7b331

File tree

3 files changed

+174
-16
lines changed

3 files changed

+174
-16
lines changed

‎pydra/design/base.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,18 @@ def make_outputs_spec(
505505
raise ValueError(
506506
f"{reserved_names} are reserved and cannot be used for output field names"
507507
)
508+
# Add in any fields in base classes that haven't already been converted into attrs
509+
# fields (e.g. stdout, stderr and return_code)
510+
for base in outputs_bases:
511+
base_outputs = {
512+
n: o
513+
for n, o in base.__dict__.items()
514+
if isinstance(o, Out) and n not in outputs
515+
}
516+
for name, field in base_outputs.items():
517+
field.name = name
518+
field.type = base.__annotations__.get(name, ty.Any)
519+
outputs.update(base_outputs)
508520
outputs_klass = type(
509521
spec_name + "Outputs",
510522
tuple(outputs_bases),

‎pydra/design/tests/test_shell.py‎

Lines changed: 154 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,13 @@
66
import cloudpickle as cp
77
from pydra.design import shell
88
from pydra.engine.helpers import list_fields
9-
from pydra.engine.specs import ShellSpec, ShellOutputs
9+
from pydra.engine.specs import (
10+
ShellSpec,
11+
ShellOutputs,
12+
RETURN_CODE_HELP,
13+
STDOUT_HELP,
14+
STDERR_HELP,
15+
)
1016
from fileformats.generic import File, Directory, FsObject
1117
from fileformats import text, image
1218
from pydra.utils.typing import MultiInputObj
@@ -34,7 +40,24 @@ def test_interface_template():
3440
shell.arg(name="in_path", type=FsObject, position=1),
3541
output,
3642
]
37-
assert sorted_fields(SampleInterface.Outputs) == [output]
43+
assert sorted_fields(SampleInterface.Outputs) == [
44+
output,
45+
shell.out(
46+
name="return_code",
47+
type=int,
48+
help_string=RETURN_CODE_HELP,
49+
),
50+
shell.out(
51+
name="stderr",
52+
type=str,
53+
help_string=STDERR_HELP,
54+
),
55+
shell.out(
56+
name="stdout",
57+
type=str,
58+
help_string=STDOUT_HELP,
59+
),
60+
]
3861
intf = SampleInterface(in_path=File.mock("in-path.txt"))
3962
assert intf.executable == "cp"
4063
SampleInterface(in_path=File.mock("in-path.txt"), out_path=Path("./out-path.txt"))
@@ -65,7 +88,24 @@ def test_interface_template_w_types_and_path_template_ext():
6588
shell.arg(name="in_image", type=image.Png, position=1),
6689
output,
6790
]
68-
assert sorted_fields(SampleInterface.Outputs) == [output]
91+
assert sorted_fields(SampleInterface.Outputs) == [
92+
output,
93+
shell.out(
94+
name="return_code",
95+
type=int,
96+
help_string=RETURN_CODE_HELP,
97+
),
98+
shell.out(
99+
name="stderr",
100+
type=str,
101+
help_string=STDERR_HELP,
102+
),
103+
shell.out(
104+
name="stdout",
105+
type=str,
106+
help_string=STDOUT_HELP,
107+
),
108+
]
69109
SampleInterface(in_image=image.Png.mock())
70110
SampleInterface(in_image=image.Png.mock(), out_image=Path("./new_image.png"))
71111
SampleInterface.Outputs(out_image=image.Png.mock())
@@ -93,7 +133,22 @@ def test_interface_template_w_modify():
93133
name="image",
94134
type=image.Png,
95135
callable=shell._InputPassThrough("image"),
96-
)
136+
),
137+
shell.out(
138+
name="return_code",
139+
type=int,
140+
help_string=RETURN_CODE_HELP,
141+
),
142+
shell.out(
143+
name="stderr",
144+
type=str,
145+
help_string=STDERR_HELP,
146+
),
147+
shell.out(
148+
name="stdout",
149+
type=str,
150+
help_string=STDOUT_HELP,
151+
),
97152
]
98153
SampleInterface(image=image.Png.mock())
99154
SampleInterface.Outputs(image=image.Png.mock())
@@ -153,7 +208,24 @@ def test_interface_template_more_complex():
153208
position=6,
154209
),
155210
]
156-
assert sorted_fields(SampleInterface.Outputs) == [output]
211+
assert sorted_fields(SampleInterface.Outputs) == [
212+
output,
213+
shell.out(
214+
name="return_code",
215+
type=int,
216+
help_string=RETURN_CODE_HELP,
217+
),
218+
shell.out(
219+
name="stderr",
220+
type=str,
221+
help_string=STDERR_HELP,
222+
),
223+
shell.out(
224+
name="stdout",
225+
type=str,
226+
help_string=STDOUT_HELP,
227+
),
228+
]
157229
SampleInterface(in_fs_objects=[File.sample(), File.sample(seed=1)])
158230
SampleInterface.Outputs(out_dir=Directory.sample())
159231

@@ -234,7 +306,23 @@ def test_interface_template_with_overrides_and_optionals():
234306
]
235307
+ outargs
236308
)
237-
assert sorted_fields(SampleInterface.Outputs) == outargs
309+
assert sorted_fields(SampleInterface.Outputs) == outargs + [
310+
shell.out(
311+
name="return_code",
312+
type=int,
313+
help_string=RETURN_CODE_HELP,
314+
),
315+
shell.out(
316+
name="stderr",
317+
type=str,
318+
help_string=STDERR_HELP,
319+
),
320+
shell.out(
321+
name="stdout",
322+
type=str,
323+
help_string=STDOUT_HELP,
324+
),
325+
]
238326

239327

240328
def test_interface_template_with_defaults():
@@ -281,7 +369,24 @@ def test_interface_template_with_defaults():
281369
position=6,
282370
),
283371
]
284-
assert sorted_fields(SampleInterface.Outputs) == [output]
372+
assert sorted_fields(SampleInterface.Outputs) == [
373+
output,
374+
shell.out(
375+
name="return_code",
376+
type=int,
377+
help_string=RETURN_CODE_HELP,
378+
),
379+
shell.out(
380+
name="stderr",
381+
type=str,
382+
help_string=STDERR_HELP,
383+
),
384+
shell.out(
385+
name="stdout",
386+
type=str,
387+
help_string=STDOUT_HELP,
388+
),
389+
]
285390
SampleInterface(in_fs_objects=[File.sample(), File.sample(seed=1)])
286391
SampleInterface.Outputs(out_dir=Directory.sample())
287392

@@ -333,7 +438,24 @@ def test_interface_template_with_type_overrides():
333438
position=6,
334439
),
335440
]
336-
assert sorted_fields(SampleInterface.Outputs) == [output]
441+
assert sorted_fields(SampleInterface.Outputs) == [
442+
output,
443+
shell.out(
444+
name="return_code",
445+
type=int,
446+
help_string=RETURN_CODE_HELP,
447+
),
448+
shell.out(
449+
name="stderr",
450+
type=str,
451+
help_string=STDERR_HELP,
452+
),
453+
shell.out(
454+
name="stdout",
455+
type=str,
456+
help_string=STDOUT_HELP,
457+
),
458+
]
337459

338460

339461
@pytest.fixture(params=["static", "dynamic"])
@@ -582,7 +704,12 @@ class Outputs:
582704
)
583705

584706
assert sorted([a.name for a in attrs.fields(A)]) == ["executable", "x", "y"]
585-
assert [a.name for a in attrs.fields(A.Outputs)] == ["y"]
707+
assert sorted(a.name for a in attrs.fields(A.Outputs)) == [
708+
"return_code",
709+
"stderr",
710+
"stdout",
711+
"y",
712+
]
586713
output = shell.outarg(
587714
name="y",
588715
type=File,
@@ -609,7 +736,24 @@ class Outputs:
609736
),
610737
output,
611738
]
612-
assert sorted_fields(A.Outputs) == [output]
739+
assert sorted_fields(A.Outputs) == [
740+
output,
741+
shell.out(
742+
name="return_code",
743+
type=int,
744+
help_string=RETURN_CODE_HELP,
745+
),
746+
shell.out(
747+
name="stderr",
748+
type=str,
749+
help_string=STDERR_HELP,
750+
),
751+
shell.out(
752+
name="stdout",
753+
type=str,
754+
help_string=STDOUT_HELP,
755+
),
756+
]
613757

614758

615759
def test_shell_output_field_name_dynamic():

‎pydra/engine/specs.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -335,15 +335,17 @@ class WorkflowSpec(TaskSpec[WorkflowOutputsType]):
335335
pass
336336

337337

338+
RETURN_CODE_HELP = """The process' exit code."""
339+
STDOUT_HELP = """The standard output stream produced by the command."""
340+
STDERR_HELP = """The standard error stream produced by the command."""
341+
342+
338343
class ShellOutputs(Outputs):
339344
"""Output specification of a generic shell process."""
340345

341-
return_code: int = shell.out()
342-
"""The process' exit code."""
343-
stdout: str = shell.out()
344-
"""The process' standard output."""
345-
stderr: str = shell.out()
346-
"""The process' standard input."""
346+
return_code: int = shell.out(help_string=RETURN_CODE_HELP)
347+
stdout: str = shell.out(help_string=STDOUT_HELP)
348+
stderr: str = shell.out(help_string=STDERR_HELP)
347349

348350
@classmethod
349351
def collect_outputs(

0 commit comments

Comments
 (0)