resolved gnarly circular imports

Author: tclose

pydra/design/__init__.py

@@ -1,6 +1,6 @@
-from .base import TaskSpec, list_fields
 from . import python
 from . import shell
+from . import workflow
 
 
-__all__ = ["TaskSpec", "list_fields", "python", "shell"]
+__all__ = ["python", "shell", "workflow"]

pydra/design/base.py

@@ -5,33 +5,35 @@
 import enum
 from pathlib import Path
 from copy import copy
-from typing_extensions import Self
 import attrs.validators
 from attrs.converters import default_if_none
 from fileformats.generic import File
 from pydra.utils.typing import TypeParser, is_optional, is_fileset_or_union
-
-# from pydra.utils.misc import get_undefined_symbols
-from pydra.engine.helpers import from_list_if_single, ensure_list
-from pydra.engine.specs import (
-    LazyField,
+from pydra.engine.helpers import (
+    from_list_if_single,
+    ensure_list,
+    PYDRA_ATTR_METADATA,
+    list_fields,
+)
+from pydra.utils.typing import (
     MultiInputObj,
     MultiInputFile,
     MultiOutputObj,
     MultiOutputFile,
 )
-from pydra.engine.core import Task, AuditFlag
+from pydra.engine.workflow.lazy import LazyField
 
 
+if ty.TYPE_CHECKING:
+    from pydra.engine.specs import OutputsSpec
+    from pydra.engine.core import Task
+
 __all__ = [
     "Field",
     "Arg",
     "Out",
-    "TaskSpec",
-    "OutputsSpec",
     "ensure_field_objects",
     "make_task_spec",
-    "list_fields",
 ]
 
 RESERVED_OUTPUT_NAMES = ("split", "combine")
@@ -154,120 +156,6 @@ class Out(Field):
     pass
 
 
-class OutputsSpec:
-    """Base class for all output specifications"""
-
-    def split(
-        self,
-        splitter: ty.Union[str, ty.List[str], ty.Tuple[str, ...], None] = None,
-        /,
-        overwrite: bool = False,
-        cont_dim: ty.Optional[dict] = None,
-        **inputs,
-    ) -> Self:
-        """
-        Run this task parametrically over lists of split inputs.
-
-        Parameters
-        ----------
-        splitter : str or list[str] or tuple[str] or None
-            the fields which to split over. If splitting over multiple fields, lists of
-            fields are interpreted as outer-products and tuples inner-products. If None,
-            then the fields to split are taken from the keyword-arg names.
-        overwrite : bool, optional
-            whether to overwrite an existing split on the node, by default False
-        cont_dim : dict, optional
-            Container dimensions for specific inputs, used in the splitter.
-            If input name is not in cont_dim, it is assumed that the input values has
-            a container dimension of 1, so only the most outer dim will be used for splitting.
-        **inputs
-            fields to split over, will automatically be wrapped in a StateArray object
-            and passed to the node inputs
-
-        Returns
-        -------
-        self : TaskBase
-            a reference to the task
-        """
-        self._node.split(splitter, overwrite=overwrite, cont_dim=cont_dim, **inputs)
-        return self
-
-    def combine(
-        self,
-        combiner: ty.Union[ty.List[str], str],
-        overwrite: bool = False,  # **kwargs
-    ) -> Self:
-        """
-        Combine inputs parameterized by one or more previous tasks.
-
-        Parameters
-        ----------
-        combiner : list[str] or str
-            the field or list of inputs to be combined (i.e. not left split) after the
-            task has been run
-        overwrite : bool
-            whether to overwrite an existing combiner on the node
-        **kwargs : dict[str, Any]
-            values for the task that will be "combined" before they are provided to the
-            node
-
-        Returns
-        -------
-        self : Self
-            a reference to the outputs object
-        """
-        self._node.combine(combiner, overwrite=overwrite)
-        return self
-
-
-OutputType = ty.TypeVar("OutputType", bound=OutputsSpec)
-
-
-class TaskSpec(ty.Generic[OutputType]):
-    """Base class for all task specifications"""
-
-    Task: ty.Type[Task]
-
-    def __call__(
-        self,
-        name: str | None = None,
-        audit_flags: AuditFlag = AuditFlag.NONE,
-        cache_dir=None,
-        cache_locations=None,
-        inputs: ty.Text | File | dict[str, ty.Any] | None = None,
-        cont_dim=None,
-        messenger_args=None,
-        messengers=None,
-        rerun=False,
-        **kwargs,
-    ):
-        self._check_for_unset_values()
-        task = self.Task(
-            self,
-            name=name,
-            audit_flags=audit_flags,
-            cache_dir=cache_dir,
-            cache_locations=cache_locations,
-            inputs=inputs,
-            cont_dim=cont_dim,
-            messenger_args=messenger_args,
-            messengers=messengers,
-            rerun=rerun,
-        )
-        return task(**kwargs)
-
-    def _check_for_unset_values(self):
-        if unset := [
-            k
-            for k, v in attrs.asdict(self, recurse=False).items()
-            if v is attrs.NOTHING
-        ]:
-            raise ValueError(
-                f"The following values {unset} in the {self!r} interface need to be set "
-                "before the workflow can be constructed"
-            )
-
-
 def extract_fields_from_class(
     klass: type,
     arg_type: type[Arg],
@@ -352,7 +240,7 @@ def get_fields(klass, field_type, auto_attribs, helps) -> dict[str, Field]:
 
 
 def make_task_spec(
-    task_type: type[Task],
+    task_type: type["Task"],
     inputs: dict[str, Arg],
     outputs: dict[str, Out],
     klass: type | None = None,
@@ -389,6 +277,8 @@ def make_task_spec(
     klass : type
         The class created using the attrs package
     """
+    from pydra.engine.specs import TaskSpec
+
     if name is None and klass is not None:
         name = klass.__name__
     outputs_klass = make_outputs_spec(outputs, outputs_bases, name)
@@ -457,7 +347,7 @@ def make_task_spec(
 
 def make_outputs_spec(
     outputs: dict[str, Out], bases: ty.Sequence[type], spec_name: str
-) -> type[OutputsSpec]:
+) -> type["OutputsSpec"]:
     """Create an outputs specification class and its outputs specification class from the
     output fields provided to the decorator/function.
 
@@ -478,6 +368,8 @@ def make_outputs_spec(
     klass : type
         The class created using the attrs package
     """
+    from pydra.engine.specs import OutputsSpec
+
     if not any(issubclass(b, OutputsSpec) for b in bases):
         outputs_bases = bases + (OutputsSpec,)
     if reserved_names := [n for n in outputs if n in RESERVED_OUTPUT_NAMES]:
@@ -880,16 +772,6 @@ def split_block(string: str) -> ty.Generator[str, None, None]:
         yield block.strip()
 
 
-def list_fields(interface: TaskSpec) -> list[Field]:
-    if not attrs.has(interface):
-        return []
-    return [
-        f.metadata[PYDRA_ATTR_METADATA]
-        for f in attrs.fields(interface)
-        if PYDRA_ATTR_METADATA in f.metadata
-    ]
-
-
 def check_explicit_fields_are_none(klass, inputs, outputs):
     if inputs is not None:
         raise ValueError(
@@ -918,5 +800,3 @@ def nothing_factory():
 
 
 white_space_re = re.compile(r"\s+")
-
-PYDRA_ATTR_METADATA = "__PYDRA_METADATA__"

pydra/design/python.py

@@ -2,12 +2,12 @@
 import inspect
 import attrs
 from pydra.engine.task import FunctionTask
+from pydra.engine.specs import TaskSpec
 from .base import (
     Arg,
     Out,
     ensure_field_objects,
     make_task_spec,
-    TaskSpec,
     parse_doc_string,
     extract_function_inputs_and_outputs,
     check_explicit_fields_are_none,

pydra/design/shell.py

@@ -11,20 +11,20 @@
 from fileformats.core import from_mime
 from fileformats import generic
 from fileformats.core.exceptions import FormatRecognitionError
+from pydra.engine.specs import TaskSpec
 from .base import (
     Arg,
     Out,
     check_explicit_fields_are_none,
     extract_fields_from_class,
     ensure_field_objects,
-    TaskSpec,
     make_task_spec,
     EMPTY,
 )
-from pydra.utils.typing import is_fileset_or_union
-from pydra.engine.specs import MultiInputObj
+from pydra.utils.typing import is_fileset_or_union, MultiInputObj
 from pydra.engine.task import ShellCommandTask
 
+
 __all__ = ["arg", "out", "outarg", "define"]
 
 
@@ -180,7 +180,6 @@ class outarg(Out, arg):
         If provided, the field is treated also as an output field and it is added to
         the output spec. The template can use other fields, e.g. {file1}. Used in order
         to create an output specification.
-
     """
 
     path_template: str | None = attrs.field(default=None)
@@ -204,7 +203,35 @@ def define(
     auto_attribs: bool = True,
     name: str | None = None,
 ) -> TaskSpec:
-    """Create a shell command interface
+    """Create a task specification for a shell command. Can be used either as a decorator on
+    the "canonical" dataclass-form of a task specification or as a function that takes a
+    "shell-command template string" of the form
+
+    ```
+    shell.define("command <input1> <input2> --output <out|output1>")
+    ```
+
+    Fields are inferred from the template if not provided. In the template, inputs are
+    specified with `<fieldname>` and outputs with `<out:fieldname>`.
+
+    ```
+    my_command <myinput> <out|myoutput2>
+    ```
+
+    The types of the fields can be specified using their MIME like (see fileformats.core.from_mime), e.g.
+
+    ```
+    my_command <myinput:text/csv> <out|myoutput2:image/png>
+    ```
+
+    The template can also specify options with `-` or `--` followed by the option name
+    and arguments with `<argname:type>`. The type is optional and will default to
+    `generic/fs-object` if not provided for arguments and `field/text` for
+    options. The file-formats namespace can be dropped for generic and field formats, e.g.
+
+    ```
+    another-command <input1:directory> <input2:int> --output <out|output1:text/csv>
+    ```
 
     Parameters
     ----------
@@ -221,6 +248,11 @@ def define(
         as they appear in the template
     name: str | None
         The name of the returned class
+
+    Returns
+    -------
+    TaskSpec
+        The interface for the shell command
     """
 
     def make(
@@ -331,9 +363,10 @@ def parse_command_line_template(
     outputs: list[str | Out] | dict[str, Out | type] | None = None,
 ) -> ty.Tuple[str, dict[str, Arg | type], dict[str, Out | type]]:
     """Parses a command line template into a name and input and output fields. Fields
-    are inferred from the template if not provided, where inputs are specified with `<fieldname>`
-    and outputs with `<out:fieldname>`. The types of the fields can be specified using their
-    MIME like (see fileformats.core.from_mime), e.g.
+    are inferred from the template if not explicitly provided.
+
+    In the template, inputs are specified with `<fieldname>` and outputs with `<out:fieldname>`.
+    The types of the fields can be specified using their MIME like (see fileformats.core.from_mime), e.g.
 
     ```
     my_command <myinput> <out|myoutput2>
@@ -345,7 +378,7 @@ def parse_command_line_template(
     options. The file-formats namespace can be dropped for generic and field formats, e.g.
 
     ```
-    another-command <input1:directory> <input2:integer> --output <out|output1:text/csv>
+    another-command <input1:directory> <input2:int> --output <out|output1:text/csv>
     ```
 
     Parameters
@@ -365,6 +398,13 @@ def parse_command_line_template(
         The input fields of the command line template
     outputs : dict[str, Out | type]
         The output fields of the command line template
+
+    Raises
+    ------
+    ValueError
+        If an unknown token is found in the command line template
+    TypeError
+        If an unknown type is found in the command line template
     """
     if isinstance(inputs, list):
         inputs = {arg.name: arg for arg in inputs}
@@ -437,9 +477,9 @@ def from_type_str(type_str) -> type:
                     try:
                         type_ = from_mime(f"generic/{tp}")
                     except FormatRecognitionError:
-                        raise ValueError(
+                        raise TypeError(
                             f"Found unknown type, {tp!r}, in command template: {template!r}"
-                        )
+                        ) from None
             types.append(type_)
         if len(types) == 2 and types[1] == "...":
             type_ = MultiInputObj[types[0]]

pydra/design/tests/test_python.py

@@ -3,7 +3,8 @@
 from decimal import Decimal
 import attrs
 import pytest
-from pydra.design import list_fields, TaskSpec
+from pydra.engine.helpers import list_fields
+from pydra.engine.specs import TaskSpec
 from pydra.design import python
 from pydra.engine.task import FunctionTask