debugging test_shelltask

Author: tclose

pydra/design/base.py

@@ -8,7 +8,7 @@
 from typing import Self
 import attrs.validators
 from attrs.converters import default_if_none
-from fileformats.generic import File
+from fileformats.generic import File, FileSet
 from pydra.utils.typing import TypeParser, is_optional, is_fileset_or_union, is_type
 from pydra.engine.helpers import (
     from_list_if_single,
@@ -59,6 +59,8 @@ def convert_default_value(value: ty.Any, self_: "Field") -> ty.Any:
         return value
     if self_.type is ty.Callable and isinstance(value, ty.Callable):
         return value
+    if isinstance(self_, Out) and TypeParser.contains_type(FileSet, self_.type):
+        return value
     return TypeParser[self_.type](self_.type, label=self_.name)(value)
 
 

pydra/design/shell.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 import typing as ty
 import re
+import glob
 from collections import defaultdict
 import inspect
 from copy import copy
@@ -25,6 +26,7 @@
 from pydra.utils.typing import (
     is_fileset_or_union,
     MultiInputObj,
+    TypeParser,
     is_optional,
     optional_type,
 )
@@ -439,6 +441,16 @@ def make(
             if inpt.position is None:
                 inpt.position = position_stack.pop(0)
 
+        # Convert string default values to callables that glob the files in the cwd
+        for outpt in parsed_outputs.values():
+            if (
+                isinstance(outpt, out)
+                and isinstance(outpt.default, str)
+                and TypeParser.contains_type(generic.FileSet, outpt.type)
+            ):
+                outpt.callable = GlobCallable(outpt.default)
+                outpt.default = NO_DEFAULT
+
         defn = make_task_def(
             ShellDef,
             ShellOutputs,
@@ -782,3 +794,16 @@ class _InputPassThrough:
 
     def __call__(self, inputs: ShellDef) -> ty.Any:
         return getattr(inputs, self.name)
+
+
+class GlobCallable:
+    """Callable that can be used to glob files"""
+
+    def __init__(self, pattern: str):
+        self.pattern = pattern
+
+    def __call__(self) -> generic.FileSet:
+        matches = glob.glob(self.pattern)
+        if not matches:
+            raise FileNotFoundError(f"No files found matching pattern: {self.pattern}")
+        return matches

pydra/engine/core.py

@@ -227,15 +227,9 @@ def uid(self):
 
     @property
     def output_names(self):
-        """Get the names of the outputs from the task's output_spec
-        (not everything has to be generated, see _generated_output_names).
-        """
+        """Get the names of the outputs from the task's output_spec"""
         return [f.name for f in attr.fields(self.definition.Outputs)]
 
-    @property
-    def _generated_output_names(self):
-        return self.output_names
-
     @property
     def can_resume(self):
         """Whether the task accepts checkpoint-restart."""
@@ -286,7 +280,7 @@ def inputs(self) -> dict[str, ty.Any]:
                     map_copyfiles[name] = copied_value
         self._inputs.update(
             template_update(
-                self.definition, self.output_dir, map_copyfiles=map_copyfiles
+                self.definition, output_dir=self.output_dir, map_copyfiles=map_copyfiles
             )
         )
         return self._inputs

pydra/engine/environments.py

@@ -1,5 +1,6 @@
 import typing as ty
 import os
+from copy import copy
 from .helpers import execute
 from pathlib import Path
 from fileformats.generic import FileSet
@@ -128,14 +129,14 @@ def get_bindings(
                         f"No support for generating bindings for {type(fileset)} types "
                         f"({fileset})"
                     )
-                copy = fld.copy_mode == FileSet.CopyMode.copy
+                copy_file = fld.copy_mode == FileSet.CopyMode.copy
 
                 host_path, env_path = fileset.parent, Path(f"{root}{fileset.parent}")
 
                 # Default to mounting paths as read-only, but respect existing modes
                 bindings[host_path] = (
                     env_path,
-                    "rw" if copy or isinstance(fld, shell.outarg) else "ro",
+                    "rw" if copy_file or isinstance(fld, shell.outarg) else "ro",
                 )
 
                 # Provide updated in-container paths to the command to be run. If a

pydra/engine/helpers_file.py

@@ -159,7 +159,7 @@ def template_update_single(
     values: dict[str, ty.Any] = None,
     output_dir: Path | None = None,
     spec_type: str = "input",
-) -> Path | None:
+) -> Path | list[Path | None] | None:
     """Update a single template from the input_spec or output_spec
     based on the value from inputs_dict
     (checking the types of the fields, that have "output_file_template)"
@@ -196,18 +196,19 @@ def template_update_single(
             return None
     # inputs_dict[field.name] is True or spec_type is output
     value = _template_formatting(field, definition, values)
-    # changing path so it is in the output_dir
     if output_dir and value is not None:
+        # changing path so it is in the output_dir
         # should be converted to str, it is also used for input fields that should be str
         if type(value) is list:
-            return [output_dir / val.name for val in value]
+            value = [output_dir / val.name for val in value]
         else:
-            return output_dir / value.name
-    else:
-        return None
+            value = output_dir / value.name
+    return value
 
 
-def _template_formatting(field, definition, values):
+def _template_formatting(
+    field: "shell.arg", definition: "ShellDef", values: dict[str, ty.Any]
+) -> Path | list[Path] | None:
     """Formatting the field template based on the values from inputs.
     Taking into account that the field with a template can be a MultiOutputFile
     and the field values needed in the template can be a list -
@@ -226,7 +227,7 @@ def _template_formatting(field, definition, values):
 
     Returns
     -------
-    formatted : str or list
+    formatted : Path or list[Path | None] or None
         formatted template
     """
     # if a template is a function it has to be run first with the inputs as the only arg
@@ -237,6 +238,8 @@ def _template_formatting(field, definition, values):
     # as default, we assume that keep_extension is True
     if isinstance(template, (tuple, list)):
         formatted = [_single_template_formatting(field, t, values) for t in template]
+        if any([val is None for val in formatted]):
+            return None
     else:
         assert isinstance(template, str)
         formatted = _single_template_formatting(field, template, values)

pydra/engine/specs.py

@@ -898,12 +898,27 @@ def _from_task(cls, task: "Task[ShellDef]") -> Self:
                     setattr(outputs, fld.name, None)
                 else:
                     raise ValueError(
-                        f"file system path(s) provided to mandatory field {fld.name!r},"
-                        f"{resolved_value}, does not exist, this is likely due to an "
+                        f"file system path(s) provided to mandatory field {fld.name!r}, "
+                        f"'{resolved_value}', does not exist, this is likely due to an "
                         f"error in the {task.name!r} task"
                     )
         return outputs
 
+    # @classmethod
+    # def _from_defaults(cls) -> Self:
+    #     """Create an output object from the default values of the fields"""
+    #     defaults = {}
+    #     for field in attrs_fields(cls):
+    #         if isinstance(field.default, attrs.Factory):
+    #             defaults[field.name] = field.default.factory()
+    #         elif TypeParser.contains_type(FileSet, field.type):
+    #             # Will be set by the templating code
+    #             defaults[field.name] = attrs.NOTHING
+    #         else:
+    #             defaults[field.name] = field.default
+
+    #     return cls(**defaults)
+
     @classmethod
     def _resolve_default_value(cls, fld: shell.out, output_dir: Path) -> ty.Any:
         """Resolve path and glob expr default values relative to the output dir"""
@@ -991,20 +1006,24 @@ def _resolve_value(
                 call_args_val[argnm] = fld
             elif argnm == "output_dir":
                 call_args_val[argnm] = task.output_dir
+            elif argnm == "executable":
+                call_args_val[argnm] = task.definition.executable
             elif argnm == "inputs":
                 call_args_val[argnm] = task.inputs
             elif argnm == "stdout":
                 call_args_val[argnm] = task.return_values["stdout"]
             elif argnm == "stderr":
                 call_args_val[argnm] = task.return_values["stderr"]
+            elif argnm == "self":
+                pass  # If the callable is a class
             else:
                 try:
                     call_args_val[argnm] = task.inputs[argnm]
                 except KeyError as e:
                     e.add_note(
-                        f"arguments of the callable function from {fld.name} "
+                        f"arguments of the callable function from {fld.name!r} "
                         f"has to be in inputs or be field or output_dir, "
-                        f"but {argnm} is used"
+                        f"but {argnm!r} is used"
                     )
                     raise
         return callable_(**call_args_val)
@@ -1040,7 +1059,7 @@ def cmdline(self) -> str:
         the current working directory."""
         # Skip the executable, which can be a multi-part command, e.g. 'docker run'.
         values = attrs_values(self)
-        values.update(template_update(self))
+        values.update(template_update(self, output_dir=Path.cwd()))
         cmd_args = self._command_args(values=values)
         cmdline = cmd_args[0]
         for arg in cmd_args[1:]:
@@ -1221,22 +1240,6 @@ def _format_arg(self, field: shell.arg, values: dict[str, ty.Any]) -> list[str]:
                     cmd_el_str = ""
         return split_cmd(cmd_el_str)
 
-    def _generated_output_names(self, stdout: str, stderr: str):
-        """Returns a list of all outputs that will be generated by the task.
-        Takes into account the task input and the requires list for the output fields.
-        TODO: should be in all Output specs?
-        """
-        # checking the input (if all mandatory fields are provided, etc.)
-        self._check_rules()
-        output_names = ["return_code", "stdout", "stderr"]
-        for fld in list_fields(self):
-            # assuming that field should have either default or metadata, but not both
-            if is_set(fld.default):
-                output_names.append(fld.name)
-            elif is_set(self.Outputs._resolve_output_value(fld, stdout, stderr)):
-                output_names.append(fld.name)
-        return output_names
-
     def _rule_violations(self) -> list[str]:
 
         errors = super()._rule_violations()