cleaned up specs.py so that it works with new syntax

Author: tclose

pydra/design/base.py

@@ -36,8 +36,6 @@
     "make_task_spec",
 ]
 
-RESERVED_OUTPUT_NAMES = ("split", "combine")
-
 
 class _Empty(enum.Enum):
 
@@ -58,6 +56,11 @@ def is_type(_, __, val: ty.Any) -> bool:
     return inspect.isclass(val) or ty.get_origin(val)
 
 
+def convert_default_value(value: ty.Any, self_: "Field") -> ty.Any:
+    """Ensure the default value has been coerced into the correct type"""
+    return TypeParser[self_.type](self_.type, label=self_.name)(value)
+
+
 @attrs.define(kw_only=True)
 class Field:
     """Base class for input and output fields to task specifications
@@ -66,9 +69,11 @@ class Field:
     ----------
     name: str, optional
         The name of the field, used when specifying a list of fields instead of a mapping
-        from name to field, by default it is None
     type: type, optional
         The type of the field, by default it is Any
+        from name to field, by default it is None
+    default : Any, optional
+        the default value for the field, by default it is EMPTY
     help_string: str, optional
         A short description of the input field.
     requires: list, optional
@@ -83,6 +88,9 @@ class Field:
     type: ty.Type[ty.Any] = attrs.field(
         validator=is_type, default=ty.Any, converter=default_if_none(ty.Any)
     )
+    default: ty.Any = attrs.field(
+        default=EMPTY, converter=attrs.Converter(convert_default_value, with_self=True)
+    )
     help_string: str = ""
     requires: list[str] | list[list[str]] = attrs.field(
         factory=list, converter=ensure_list
@@ -97,10 +105,15 @@ class Arg(Field):
 
     Parameters
     ----------
+    name: str, optional
+        The name of the field, used when specifying a list of fields instead of a mapping
+        from name to field, by default it is None
+    type: type, optional
+        The type of the field, by default it is Any
+    default : Any, optional
+        the default value for the field, by default it is EMPTY
     help_string: str
         A short description of the input field.
-    default : Any, optional
-        the default value for the argument
     allowed_values: list, optional
         List of allowed values for the field.
     requires: list, optional
@@ -118,14 +131,8 @@ class Arg(Field):
         If True the input field can’t be provided by the user but it aggregates other
         input fields (for example the fields with argstr: -o {fldA} {fldB}), by default
         it is False
-    type: type, optional
-        The type of the field, by default it is Any
-    name: str, optional
-        The name of the field, used when specifying a list of fields instead of a mapping
-        from name to field, by default it is None
     """
 
-    default: ty.Any = EMPTY
     allowed_values: list | None = None
     xor: list | None = None
     copy_mode: File.CopyMode = File.CopyMode.any
@@ -145,6 +152,8 @@ class Out(Field):
         from name to field, by default it is None
     type: type, optional
         The type of the field, by default it is Any
+    default : Any, optional
+        the default value for the field, by default it is EMPTY
     help_string: str, optional
         A short description of the input field.
     requires: list, optional
@@ -385,7 +394,7 @@ def make_outputs_spec(
                 f"Cannot make {spec_type} output spec from {out_spec_bases} bases"
             )
         outputs_bases = bases + (spec_type,)
-    if reserved_names := [n for n in outputs if n in RESERVED_OUTPUT_NAMES]:
+    if reserved_names := [n for n in outputs if n in spec_type.RESERVED_FIELD_NAMES]:
         raise ValueError(
             f"{reserved_names} are reserved and cannot be used for output field names"
         )

pydra/design/shell.py

@@ -12,6 +12,7 @@
 from fileformats import generic
 from fileformats.core.exceptions import FormatRecognitionError
 from pydra.engine.specs import ShellSpec, ShellOutSpec
+from pydra.engine.helpers import attrs_values
 from .base import (
     Arg,
     Out,
@@ -470,7 +471,7 @@ def add_arg(name, field_type, kwds, is_option=False):
                 kwds["type"] = field
                 field = field_type(name=name, **kwds)
             elif not isinstance(field, field_type):  # If field type is outarg not out
-                field = field_type(**attrs.asdict(field, recurse=False))
+                field = field_type(**attrs_values(field))
             field.name = name
             type_ = kwds.pop("type", field.type)
             if field.type is ty.Any:

pydra/engine/audit.py

@@ -2,8 +2,8 @@
 
 import os
 import json
-import attr
 from pydra.utils.messenger import send_message, make_message, gen_uuid, now, AuditFlag
+from pydra.engine.helpers import attrs_values
 from fileformats.core import FileSet
 from pydra.utils.hash import hash_function
 
@@ -104,7 +104,7 @@ def finalize_audit(self, result):
                 )
                 # audit resources/runtime information
                 self.eid = f"uid:{gen_uuid()}"
-                entity = attr.asdict(result.runtime, recurse=False)
+                entity = attrs_values(result.runtime)
                 entity.update(
                     **{
                         "@id": self.eid,
@@ -180,12 +180,12 @@ def audit_check(self, flag):
 
     def audit_task(self, task):
         import subprocess as sp
-        from .helpers import attr_fields
+        from .helpers import attrs_fields
 
         label = task.name
 
         command = task.cmdline if hasattr(task.inputs, "executable") else None
-        attr_list = attr_fields(task.inputs)
+        attr_list = attrs_fields(task.inputs)
         for attrs in attr_list:
             input_name = attrs.name
             value = getattr(task.inputs, input_name)

pydra/engine/boutiques.py

@@ -7,7 +7,7 @@
 
 from pydra.utils.messenger import AuditFlag
 from pydra.engine.task import ShellCommandTask
-from pydra.engine.specs import SpecInfo, ShellSpec, ShellOutSpec, File, attr_fields
+from pydra.engine.specs import SpecInfo, ShellSpec, ShellOutSpec, File, attrs_fields
 from .helpers_file import is_local_file
 
 
@@ -192,7 +192,7 @@ def _command_args_single(self, state_ind=None, index=None):
     def _bosh_invocation_file(self, state_ind=None, index=None):
         """creating bosh invocation file - json file with inputs values"""
         input_json = {}
-        for f in attr_fields(self.inputs, exclude_names=("executable", "args")):
+        for f in attrs_fields(self.inputs, exclude_names=("executable", "args")):
             if self.state and f"{self.name}.{f.name}" in state_ind:
                 value = getattr(self.inputs, f.name)[state_ind[f"{self.name}.{f.name}"]]
             else:

pydra/engine/core.py

@@ -26,7 +26,8 @@
 from .workflow.lazy import is_lazy
 from .helpers import (
     create_checksum,
-    attr_fields,
+    attrs_fields,
+    attrs_values,
     print_help,
     load_result,
     save,
@@ -342,7 +343,7 @@ def generated_output_names(self):
         The results depends on the input provided to the task
         """
         output_klass = self.interface.Outputs
-        if hasattr(output_klass, "generated_output_names"):
+        if hasattr(output_klass, "_generated_output_names"):
             output = output_klass(
                 **{f.name: attr.NOTHING for f in attr.fields(output_klass)}
             )
@@ -352,7 +353,7 @@ def generated_output_names(self):
             if modified_inputs:
                 _inputs = attr.evolve(_inputs, **modified_inputs)
 
-            return output.generated_output_names(
+            return output._generated_output_names(
                 inputs=_inputs, output_dir=self.output_dir
             )
         else:
@@ -461,9 +462,7 @@ def _modify_inputs(self):
         from pydra.utils.typing import TypeParser
 
         orig_inputs = {
-            k: v
-            for k, v in attr.asdict(self.inputs, recurse=False).items()
-            if not k.startswith("_")
+            k: v for k, v in attrs_values(self.inputs).items() if not k.startswith("_")
         }
         map_copyfiles = {}
         input_fields = attr.fields(type(self.inputs))
@@ -754,7 +753,7 @@ def result(self, state_index=None, return_inputs=False):
 
     def _reset(self):
         """Reset the connections between inputs and LazyFields."""
-        for field in attr_fields(self.inputs):
+        for field in attrs_fields(self.inputs):
             if field.name in self.inp_lf:
                 setattr(self.inputs, field.name, self.inp_lf[field.name])
         if is_workflow(self):
@@ -979,7 +978,7 @@ def create_connections(self, task, detailed=False):
         """
         # TODO: create connection is run twice
         other_states = {}
-        for field in attr_fields(task.inputs):
+        for field in attrs_fields(task.inputs):
             val = getattr(task.inputs, field.name)
             if is_lazy(val):
                 # saving all connections with LazyFields

pydra/engine/helpers.py

@@ -26,11 +26,18 @@
 PYDRA_ATTR_METADATA = "__PYDRA_METADATA__"
 
 
-def attr_fields(spec, exclude_names=()):
+def attrs_fields(spec, exclude_names=()) -> list[attrs.Attribute]:
+    """Get the fields of a spec, excluding some names."""
     return [field for field in spec.__attrs_attrs__ if field.name not in exclude_names]
 
 
+def attrs_values(obj, **kwargs) -> dict[str, ty.Any]:
+    """Get the values of an attrs object."""
+    return attrs.asdict(obj, recurse=False, **kwargs)
+
+
 def list_fields(interface: "TaskSpec") -> list["Field"]:
+    """List the fields of a task specification"""
     if not attrs.has(interface):
         return []
     return [
@@ -43,7 +50,7 @@ def list_fields(interface: "TaskSpec") -> list["Field"]:
 # from .specs import MultiInputFile, MultiInputObj, MultiOutputObj, MultiOutputFile
 
 
-def from_list_if_single(obj):
+def from_list_if_single(obj: ty.Any) -> ty.Any:
     """Converts a list to a single item if it is of length == 1"""
 
     if obj is attrs.NOTHING:
@@ -109,7 +116,7 @@ def load_result(checksum, cache_locations):
     return None
 
 
-def save(task_path: Path, result=None, task=None, name_prefix=None):
+def save(task_path: Path, result=None, task=None, name_prefix=None) -> None:
     """
     Save a :class:`~pydra.engine.core.TaskBase` object and/or results.
 
@@ -147,7 +154,7 @@ def save(task_path: Path, result=None, task=None, name_prefix=None):
 
 def copyfile_workflow(wf_path: os.PathLike, result):
     """if file in the wf results, the file will be copied to the workflow directory"""
-    for field in attr_fields(result.output):
+    for field in attrs_fields(result.output):
         value = getattr(result.output, field.name)
         # if the field is a path or it can contain a path _copyfile_single_value is run
         # to move all files and directories to the workflow directory
@@ -375,38 +382,6 @@ def get_open_loop():
     return loop
 
 
-# def output_from_inputfields(interface: "Interface"):
-#     """
-#     Collect values from output from input fields.
-#     If names_only is False, the output_spec is updated,
-#     if names_only is True only the names are returned
-
-#     Parameters
-#     ----------
-#     output_spec :
-#         TODO
-#     input_spec :
-#         TODO
-
-#     """
-#     current_output_spec_names = [f.name for f in attrs.fields(interface.Outputs)]
-#     new_fields = []
-#     for fld in attrs.fields(interface):
-#         if "output_file_template" in fld.metadata:
-#             if "output_field_name" in fld.metadata:
-#                 field_name = fld.metadata["output_field_name"]
-#             else:
-#                 field_name = fld.name
-#             # not adding if the field already in the output_spec
-#             if field_name not in current_output_spec_names:
-#                 # TODO: should probably remove some of the keys
-#                 new_fields.append(
-#                     (field_name, attrs.field(type=File, metadata=fld.metadata))
-#                 )
-#     output_spec.fields += new_fields
-#     return output_spec
-
-
 def get_available_cpus():
     """
     Return the number of CPUs available to the current process or, if that is not
@@ -658,7 +633,7 @@ def is_lazy(obj):
     if is_lazy(obj):
         return True
 
-    for f in attr_fields(obj):
+    for f in attrs_fields(obj):
         if isinstance(getattr(obj, f.name), LazyField):
             return True
     return False

pydra/engine/helpers_file.py

@@ -10,7 +10,7 @@
 from contextlib import contextmanager
 import attr
 from fileformats.core import FileSet
-from pydra.engine.helpers import is_lazy
+from pydra.engine.helpers import is_lazy, attrs_values
 
 
 logger = logging.getLogger("pydra")
@@ -105,7 +105,7 @@ def template_update(inputs, output_dir, state_ind=None, map_copyfiles=None):
 
     """
 
-    inputs_dict_st = attr.asdict(inputs, recurse=False)
+    inputs_dict_st = attrs_values(inputs)
     if map_copyfiles is not None:
         inputs_dict_st.update(map_copyfiles)
 
@@ -114,12 +114,12 @@ def template_update(inputs, output_dir, state_ind=None, map_copyfiles=None):
             k = k.split(".")[1]
             inputs_dict_st[k] = inputs_dict_st[k][v]
 
-    from .specs import attr_fields
+    from .specs import attrs_fields
 
     # Collect templated inputs for which all requirements are satisfied.
     fields_templ = [
         field
-        for field in attr_fields(inputs)
+        for field in attrs_fields(inputs)
         if field.metadata.get("output_file_template")
         and getattr(inputs, field.name) is not False
         and all(
@@ -155,7 +155,7 @@ def template_update_single(
     from pydra.engine.specs import OUTPUT_TEMPLATE_TYPES
 
     if inputs_dict_st is None:
-        inputs_dict_st = attr.asdict(inputs, recurse=False)
+        inputs_dict_st = attrs_values(inputs)
 
     if spec_type == "input":
         inp_val_set = inputs_dict_st[field.name]

pydra/engine/helpers_state.py

@@ -1,11 +1,10 @@
 """Additional functions used mostly by the State class."""
 
-import attr
 import itertools
 from copy import deepcopy
 import logging
 import typing as ty
-from .helpers import ensure_list
+from .helpers import ensure_list, attrs_values
 
 logger = logging.getLogger("pydra")
 
@@ -622,9 +621,7 @@ def map_splits(split_iter, inputs, cont_dim=None):
 def inputs_types_to_dict(name, inputs):
     """Convert type.Inputs to dictionary."""
     # dj: any better option?
-    input_names = [
-        field for field in attr.asdict(inputs, recurse=False) if field != "_func"
-    ]
+    input_names = [field for field in attrs_values(inputs) if field != "_func"]
     inputs_dict = {}
     for field in input_names:
         inputs_dict[f"{name}.{field}"] = getattr(inputs, field)