restored functionality from specs

Author: tclose

pydra/design/base.py

@@ -14,18 +14,18 @@
     ensure_list,
     PYDRA_ATTR_METADATA,
     list_fields,
+    is_lazy,
 )
 from pydra.utils.typing import (
     MultiInputObj,
     MultiInputFile,
     MultiOutputObj,
     MultiOutputFile,
 )
-from pydra.engine.workflow.lazy import LazyField
 
 
 if ty.TYPE_CHECKING:
-    from pydra.engine.specs import OutputsSpec
+    from pydra.engine.specs import TaskSpec, OutSpec
     from pydra.engine.core import Task
 
 __all__ = [
@@ -84,7 +84,9 @@ class Field:
         validator=is_type, default=ty.Any, converter=default_if_none(ty.Any)
     )
     help_string: str = ""
-    requires: list | None = None
+    requires: list[str] | list[list[str]] = attrs.field(
+        factory=list, converter=ensure_list
+    )
     converter: ty.Callable | None = None
     validator: ty.Callable | None = None
 
@@ -240,6 +242,8 @@ def get_fields(klass, field_type, auto_attribs, helps) -> dict[str, Field]:
 
 
 def make_task_spec(
+    spec_type: type["TaskSpec"],
+    out_type: type["OutSpec"],
     task_type: type["Task"],
     inputs: dict[str, Arg],
     outputs: dict[str, Out],
@@ -281,14 +285,16 @@ def make_task_spec(
 
     if name is None and klass is not None:
         name = klass.__name__
-    outputs_klass = make_outputs_spec(outputs, outputs_bases, name)
-    if klass is None or not issubclass(klass, TaskSpec):
+    outputs_klass = make_outputs_spec(out_type, outputs, outputs_bases, name)
+    if klass is None or not issubclass(klass, spec_type):
         if name is None:
             raise ValueError("name must be provided if klass is not")
+        if klass is not None and issubclass(klass, TaskSpec):
+            raise ValueError(f"Cannot change type of spec {klass} to {spec_type}")
         bases = tuple(bases)
         # Ensure that TaskSpec is a base class
-        if not any(issubclass(b, TaskSpec) for b in bases):
-            bases = bases + (TaskSpec,)
+        if not any(issubclass(b, spec_type) for b in bases):
+            bases = bases + (spec_type,)
         # If building from a decorated class (as opposed to dynamically from a function
         # or shell-template), add any base classes not already in the bases tuple
         if klass is not None:
@@ -346,8 +352,11 @@ def make_task_spec(
 
 
 def make_outputs_spec(
-    outputs: dict[str, Out], bases: ty.Sequence[type], spec_name: str
-) -> type["OutputsSpec"]:
+    spec_type: type["OutSpec"],
+    outputs: dict[str, Out],
+    bases: ty.Sequence[type],
+    spec_name: str,
+) -> type["OutSpec"]:
     """Create an outputs specification class and its outputs specification class from the
     output fields provided to the decorator/function.
 
@@ -368,10 +377,14 @@ def make_outputs_spec(
     klass : type
         The class created using the attrs package
     """
-    from pydra.engine.specs import OutputsSpec
+    from pydra.engine.specs import OutSpec
 
-    if not any(issubclass(b, OutputsSpec) for b in bases):
-        outputs_bases = bases + (OutputsSpec,)
+    if not any(issubclass(b, spec_type) for b in bases):
+        if out_spec_bases := [b for b in bases if issubclass(b, OutSpec)]:
+            raise ValueError(
+                f"Cannot make {spec_type} output spec from {out_spec_bases} bases"
+            )
+        outputs_bases = bases + (spec_type,)
     if reserved_names := [n for n in outputs if n in RESERVED_OUTPUT_NAMES]:
         raise ValueError(
             f"{reserved_names} are reserved and cannot be used for output field names"
@@ -549,7 +562,7 @@ def make_validator(field: Field, interface_name: str) -> ty.Callable[..., None]
 def allowed_values_validator(_, attribute, value):
     """checking if the values is in allowed_values"""
     allowed = attribute.metadata[PYDRA_ATTR_METADATA].allowed_values
-    if value is attrs.NOTHING or isinstance(value, LazyField):
+    if value is attrs.NOTHING or is_lazy(value):
         pass
     elif value not in allowed:
         raise ValueError(

pydra/design/python.py

@@ -2,7 +2,7 @@
 import inspect
 import attrs
 from pydra.engine.task import FunctionTask
-from pydra.engine.specs import TaskSpec
+from pydra.engine.specs import PythonSpec, PythonOutSpec
 from .base import (
     Arg,
     Out,
@@ -87,7 +87,7 @@ def define(
     bases: ty.Sequence[type] = (),
     outputs_bases: ty.Sequence[type] = (),
     auto_attribs: bool = True,
-) -> TaskSpec:
+) -> PythonSpec:
     """
     Create an interface for a function or a class.
 
@@ -103,7 +103,7 @@ def define(
         Whether to use auto_attribs mode when creating the class.
     """
 
-    def make(wrapped: ty.Callable | type) -> TaskSpec:
+    def make(wrapped: ty.Callable | type) -> PythonSpec:
         if inspect.isclass(wrapped):
             klass = wrapped
             function = klass.function
@@ -139,6 +139,8 @@ def make(wrapped: ty.Callable | type) -> TaskSpec:
         )
 
         interface = make_task_spec(
+            PythonSpec,
+            PythonOutSpec,
             FunctionTask,
             parsed_inputs,
             parsed_outputs,

pydra/design/shell.py

@@ -11,7 +11,7 @@
 from fileformats.core import from_mime
 from fileformats import generic
 from fileformats.core.exceptions import FormatRecognitionError
-from pydra.engine.specs import TaskSpec
+from pydra.engine.specs import ShellSpec, ShellOutSpec
 from .base import (
     Arg,
     Out,
@@ -177,9 +177,8 @@ class outarg(Out, arg):
         inputs (entire inputs will be passed) or any input field name (a specific input
         field will be sent).
     path_template: str, optional
-        If provided, the field is treated also as an output field and it is added to
-        the output spec. The template can use other fields, e.g. {file1}. Used in order
-        to create an output specification.
+        The template used to specify where the output file will be written to can use
+        other fields, e.g. {file1}. Used in order to create an output specification.
     """
 
     path_template: str | None = attrs.field(default=None)
@@ -202,7 +201,7 @@ def define(
     outputs_bases: ty.Sequence[type] = (),
     auto_attribs: bool = True,
     name: str | None = None,
-) -> TaskSpec:
+) -> ShellSpec:
     """Create a task specification for a shell command. Can be used either as a decorator on
     the "canonical" dataclass-form of a task specification or as a function that takes a
     "shell-command template string" of the form
@@ -251,13 +250,13 @@ def define(
 
     Returns
     -------
-    TaskSpec
+    ShellSpec
         The interface for the shell command
     """
 
     def make(
         wrapped: ty.Callable | type | None = None,
-    ) -> TaskSpec:
+    ) -> ShellSpec:
 
         if inspect.isclass(wrapped):
             klass = wrapped
@@ -272,6 +271,14 @@ def make(
                         f"Shell task class {wrapped} must have an `executable` "
                         "attribute that specifies the command to run"
                     ) from None
+            if not isinstance(executable, str) and not (
+                isinstance(executable, ty.Sequence)
+                and all(isinstance(e, str) for e in executable)
+            ):
+                raise ValueError(
+                    "executable must be a string or a sequence of strings"
+                    f", not {executable!r}"
+                )
             class_name = klass.__name__
             check_explicit_fields_are_none(klass, inputs, outputs)
             parsed_inputs, parsed_outputs = extract_fields_from_class(
@@ -309,7 +316,15 @@ def make(
             {o.name: o for o in parsed_outputs.values() if isinstance(o, arg)}
         )
         parsed_inputs["executable"] = arg(
-            name="executable", type=str, argstr="", position=0, default=executable
+            name="executable",
+            type=str | ty.Sequence[str],
+            argstr="",
+            position=0,
+            default=executable,
+            help_string=(
+                "the first part of the command, can be a string, "
+                "e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']"
+            ),
         )
 
         # Set positions for the remaining inputs that don't have an explicit position
@@ -319,6 +334,8 @@ def make(
                 inpt.position = position_stack.pop(0)
 
         interface = make_task_spec(
+            ShellSpec,
+            ShellOutSpec,
             ShellCommandTask,
             parsed_inputs,
             parsed_outputs,

pydra/design/tests/test_workflow.py

@@ -344,7 +344,9 @@ def MyTestWorkflow(a: list[int], b: list[float]) -> list[float]:
     wf = Workflow.construct(MyTestWorkflow(a=[1, 2, 3], b=[1.0, 10.0, 100.0]))
     assert wf["Mul"].splitter == ["Mul.x", "Mul.y"]
     assert wf["Mul"].combiner == ["Mul.x"]
-    assert wf.outputs.out == LazyOutField(node=wf["Sum"], field="out", type=list[float])
+    assert wf.outputs.out == LazyOutField(
+        node=wf["Sum"], field="out", type=list[float], type_checked=True
+    )
 
 
 def test_workflow_split_combine2():

pydra/design/workflow.py

@@ -13,7 +13,7 @@
     check_explicit_fields_are_none,
     extract_fields_from_class,
 )
-from pydra.engine.specs import TaskSpec
+from pydra.engine.specs import TaskSpec, OutSpec, WorkflowSpec, WorkflowOutSpec
 
 
 __all__ = ["define", "add", "this", "arg", "out"]
@@ -154,6 +154,8 @@ def make(wrapped: ty.Callable | type) -> TaskSpec:
             parsed_inputs[inpt_name].lazy = True
 
         interface = make_task_spec(
+            WorkflowSpec,
+            WorkflowOutSpec,
             WorkflowTask,
             parsed_inputs,
             parsed_outputs,
@@ -172,9 +174,6 @@ def make(wrapped: ty.Callable | type) -> TaskSpec:
     return make
 
 
-OutputType = ty.TypeVar("OutputType")
-
-
 def this() -> Workflow:
     """Get the workflow currently being constructed.
 
@@ -186,7 +185,10 @@ def this() -> Workflow:
     return Workflow.under_construction
 
 
-def add(task_spec: TaskSpec[OutputType], name: str = None) -> OutputType:
+OutSpecType = ty.TypeVar("OutSpecType", bound=OutSpec)
+
+
+def add(task_spec: TaskSpec[OutSpecType], name: str = None) -> OutSpecType:
     """Add a node to the workflow currently being constructed
 
     Parameters
@@ -199,7 +201,7 @@ def add(task_spec: TaskSpec[OutputType], name: str = None) -> OutputType:
 
     Returns
     -------
-    OutputType
+    OutSpec
         The outputs specification of the node
     """
     return this().add(task_spec, name=name)