debugging optional shell outputs

Author: tclose

pydra/design/base.py

@@ -715,9 +715,9 @@ def make_converter(
         field_type, label=checker_label, superclass_auto_cast=True
     )
     converters = []
-    if field.type in (MultiInputObj, MultiInputFile):
+    if field_type in (MultiInputObj, MultiInputFile):
         converters.append(ensure_list)
-    elif field.type in (MultiOutputObj, MultiOutputFile):
+    elif field_type in (MultiOutputObj, MultiOutputFile):
         converters.append(from_list_if_single)
     if field.converter:
         converters.append(field.converter)

pydra/engine/helpers_file.py

@@ -334,7 +334,11 @@ def _single_template_formatting(
         formatted_value = _element_formatting(
             template, val_dict, file_template, keep_extension=field.keep_extension
         )
-    return Path(formatted_value) if formatted_value is not None else formatted_value
+    if isinstance(formatted_value, list):
+        return [Path(val) for val in formatted_value]
+    elif isinstance(formatted_value, str):
+        return Path(formatted_value)
+    return None
 
 
 def _element_formatting(

pydra/engine/specs.py

@@ -32,7 +32,12 @@
 from . import helpers_state as hlpst
 from . import lazy
 from pydra.utils.hash import hash_function, Cache
-from pydra.utils.typing import StateArray, is_multi_input
+from pydra.utils.typing import (
+    StateArray,
+    is_multi_input,
+    MultiOutputObj,
+    MultiOutputFile,
+)
 from pydra.design.base import Field, Arg, Out, RequirementSet, NO_DEFAULT
 from pydra.design import shell
 
@@ -846,7 +851,17 @@ def _from_task(cls, task: "Task[ShellDef]") -> Self:
             else:
                 resolved_value = cls._resolve_value(fld, task)
             # Set the resolved value
-            setattr(outputs, fld.name, resolved_value)
+            try:
+                setattr(outputs, fld.name, resolved_value)
+            except FileNotFoundError as e:
+                if is_optional(fld.type):
+                    setattr(outputs, fld.name, None)
+                else:
+                    e.add_note(
+                        f"file system path provided to {fld.name!r}, {resolved_value}, "
+                        f"does not exist, this is likely due to an error in the task {task}"
+                    )
+                    raise
         return outputs
 
     @classmethod
@@ -1129,7 +1144,7 @@ def _command_pos_args(
             # if False, nothing is added to the command.
             if value is True:
                 cmd_add.append(field.argstr)
-        elif is_multi_input(tp):
+        elif is_multi_input(tp) or tp is MultiOutputObj or tp is MultiOutputFile:
             # if the field is MultiInputObj, it is used to create a list of arguments
             for val in value or []:
                 cmd_add += self._format_arg(field, val)

pydra/engine/tests/test_shelltask.py

@@ -3062,7 +3062,7 @@ def no_fsl():
 
 
 @pytest.mark.skipif(no_fsl(), reason="fsl is not installed")
-def test_fsl(data_tests_dir):
+def test_fsl(data_tests_dir, tmp_path):
     """mandatory field added to fields, value provided"""
 
     _xor_inputs = [
@@ -3232,7 +3232,7 @@ class Shelly(ShellDef["Shelly.Outputs"]):
         input: File = shell.arg(argstr="", help="input file")
 
         class Outputs(ShellOutputs):
-            output: File = shell.outarg(
+            output: File | None = shell.outarg(
                 argstr="",
                 path_template="out.txt",
                 help="dummy output",
@@ -3252,7 +3252,7 @@ class Outputs(ShellOutputs):
 
 def test_shell_cmd_non_existing_outputs_1(tmp_path):
     """Checking that non existing output files do not return a phantom path,
-    but return NOTHING instead"""
+    but return None instead"""
 
     @shell.define
     class Shelly(ShellDef["Shelly.Outputs"]):
@@ -3264,25 +3264,24 @@ class Shelly(ShellDef["Shelly.Outputs"]):
         )
 
         class Outputs(ShellOutputs):
-            out_1: File = shell.outarg(
+            out_1: File | None = shell.out(
                 help="fictional output #1",
-                path_template="{out_name}_1.nii",
+                callable=lambda: "out_1.nii",
             )
-            out_2: File = shell.outarg(
+            out_2: File | None = shell.out(
                 help="fictional output #2",
-                path_template="{out_name}_2.nii",
+                callable=lambda: "out_2.nii",
             )
 
-    shelly = Shelly(
-        out_name="test",
-    )
-    outputs = shelly()
-    assert outputs.out_1 == attr.NOTHING and outputs.out_2 == attr.NOTHING
+    shelly = Shelly(out_name="test")
+    outputs = shelly(cache_dir=tmp_path)
+    assert outputs.out_1 is None
+    assert outputs.out_2 is None
 
 
 def test_shell_cmd_non_existing_outputs_2(tmp_path):
     """Checking that non existing output files do not return a phantom path,
-    but return NOTHING instead. This test has one existing and one non existing output file.
+    but return None instead. This test has one existing and one non existing output file.
     """
 
     @shell.define
@@ -3306,7 +3305,7 @@ class Outputs(ShellOutputs):
             )
 
     shelly = Shelly(out_name="test")
-    outputs = shelly()
+    outputs = shelly(cache_dir=tmp_path)
     # the first output file is created
     assert outputs.out_1.fspath == next(tmp_path.iterdir()) / "test_1.nii"
     assert outputs.out_1.fspath.exists()
@@ -3316,7 +3315,8 @@ class Outputs(ShellOutputs):
 
 def test_shell_cmd_non_existing_outputs_3(tmp_path):
     """Checking that non existing output files do not return a phantom path,
-    but return NOTHING instead. This test has an existing mandatory output and another non existing output file.
+    but return None instead. This test has an existing mandatory output and another
+    non existing output file.
     """
 
     @shell.define
@@ -3331,55 +3331,51 @@ class Shelly(ShellDef["Shelly.Outputs"]):
 
         class Outputs(ShellOutputs):
             out_1: File = shell.outarg(
-                help="fictional output #1",
-                path_template="{out_name}_1.nii",
+                help="real output #1",
+                default="{out_name}_1.nii",
             )
-            out_2: File = shell.outarg(
+            out_2: File | None = shell.outarg(
                 help="fictional output #2",
-                path_template="{out_name}_2.nii",
+                default="{out_name}_2.nii",
             )
 
     shelly = Shelly(out_name="test")
 
-    outputs = shelly()
+    outputs = shelly(cache_dir=tmp_path)
     # the first output file is created
     assert outputs.out_1.fspath == next(tmp_path.iterdir()) / "test_1.nii"
     assert outputs.out_1.fspath.exists()
     # the second output file is not created
-    assert outputs.out_2 == attr.NOTHING
+    assert outputs.out_2 is None
 
 
 def test_shell_cmd_non_existing_outputs_4(tmp_path):
     """Checking that non existing output files do not return a phantom path,
-    but return NOTHING instead. This test has an existing mandatory output and another non existing
+    but return None instead. This test has an existing mandatory output and another non existing
     mandatory output file."""
 
     @shell.define
     class Shelly(ShellDef["Shelly.Outputs"]):
         executable = "touch"
         out_name: str = shell.arg(
-            help="""
-                        base name of the pretend outputs.
-                        """,
+            help="""base name of the pretend outputs.""",
             argstr="{out_name}_1.nii",
         )
 
         class Outputs(ShellOutputs):
             out_1: File = shell.outarg(
                 help="fictional output #1",
-                path_template="{out_name}_1.nii",
+                default="{out_name}_1.nii",
             )
             out_2: File = shell.outarg(
                 help="fictional output #2",
-                path_template="{out_name}_2.nii",
+                default="{out_name}_2.nii",
             )
 
-    shelly = Shelly(
-        out_name="test",
-    )
+    shelly = Shelly(out_name="test")
     # An exception should be raised because the second mandatory output does not exist
     with pytest.raises(Exception) as excinfo:
-        shelly()
+        shelly(cache_dir=tmp_path)
     assert "mandatory output for variable out_2 does not exist" == str(excinfo.value)
     # checking if the first output was created
     assert (next(tmp_path.iterdir()) / "test_1.nii").exists()