expanded print_help to be more helpful and updated related tests

Author: tclose

.github/workflows/ci-cd.yml

@@ -277,10 +277,10 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-    - name: Install Pandoc for NBSphinx
+    - name: Install Pandoc for NBSphinx and graphviz for workflow plotting (dot)
       run: |
         sudo apt-get update
-        sudo apt-get install -y pandoc
+        sudo apt-get install -y pandoc graphviz
     - name: Install Dependencies for virtual notifications in Adv.-Exec Tutorial
       run: |
           sudo apt update

docs/source/tutorial/6-workflow.ipynb

@@ -63,6 +63,7 @@
     "    return mul.out\n",
     "\n",
     "\n",
+    "print_help(BasicWorkflow)\n",
     "show_workflow(BasicWorkflow, figsize=(2, 2.5))"
    ]
   },
@@ -116,6 +117,7 @@
     "    return output_video  # test implicit detection of output name\n",
     "\n",
     "\n",
+    "print_help(ShellWorkflow)\n",
     "show_workflow(ShellWorkflow, figsize=(2.5, 3))"
    ]
   },
@@ -149,6 +151,7 @@
     "    return sum.out\n",
     "\n",
     "\n",
+    "print_help(SplitWorkflow)\n",
     "show_workflow(SplitWorkflow, figsize=(2, 2.5))"
    ]
   },
@@ -173,6 +176,7 @@
     "    return sum.out\n",
     "\n",
     "\n",
+    "print_help(SplitThenCombineWorkflow)\n",
     "show_workflow(SplitThenCombineWorkflow, figsize=(3, 3.5))"
    ]
   },
@@ -230,6 +234,7 @@
     "    return output_video  # test implicit detection of output name\n",
     "\n",
     "\n",
+    "print_help(ConditionalWorkflow)\n",
     "show_workflow(ConditionalWorkflow(watermark_dims=(10, 10)), figsize=(2.5, 3))"
    ]
   },
@@ -349,6 +354,7 @@
     "    return handbrake.out_video\n",
     "\n",
     "\n",
+    "print_help(TypeErrorWorkflow)\n",
     "show_workflow(TypeErrorWorkflow, plot_type=\"detailed\")"
    ]
   },
@@ -404,6 +410,7 @@
     "    return mul.out, divide.divided\n",
     "\n",
     "\n",
+    "print_help(DirectAccesWorkflow)\n",
     "show_workflow(DirectAccesWorkflow(b=1), plot_type=\"detailed\")"
    ]
   },
@@ -447,6 +454,7 @@
     "    wf.outputs.out2 = divide.divided\n",
     "\n",
     "\n",
+    "print_help(SetOutputsOfWorkflow)\n",
     "show_workflow(SetOutputsOfWorkflow(b=3), plot_type=\"detailed\")"
    ]
   },

docs/source/tutorial/7-canonical-form.ipynb

@@ -195,6 +195,7 @@
    "outputs": [],
    "source": [
     "from pydra.compose import python, workflow\n",
+    "from pydra.utils import print_help, show_workflow\n",
     "\n",
     "\n",
     "# Example python tasks\n",
@@ -233,7 +234,8 @@
     "        out: float\n",
     "\n",
     "\n",
-    "print_help(CanonicalWorkflowTask)"
+    "print_help(CanonicalWorkflowTask)\n",
+    "show_workflow(CanonicalWorkflowTask)"
    ]
   }
  ],

pydra/compose/base/builder.py

@@ -5,7 +5,6 @@
 import attrs.validators
 from pydra.utils.typing import TypeParser, is_optional, is_fileset_or_union
 import attrs
-from .field import NO_DEFAULT
 from .task import Task, Outputs
 from pydra.utils.hash import hash_function
 from pydra.utils.general import (
@@ -123,12 +122,12 @@ def build_task_class(
             if getattr(arg, "path_template", False):
                 if is_optional(arg.type):
                     field_type = Path | bool | None
-                    if arg.default is NO_DEFAULT:
+                    if arg.mandatory:  # provide default if one is not provided
                         attrs_kwargs["default"] = True if arg.requires else None
                         del attrs_kwargs["factory"]
                 else:
                     field_type = Path | bool
-                    if arg.default is NO_DEFAULT:
+                    if arg.mandatory:  # provide default if one is not provided
                         attrs_kwargs["default"] = True  # use the template by default
                         del attrs_kwargs["factory"]
             elif is_optional(arg.type):
@@ -316,7 +315,7 @@ def allowed_values_validator(_, attribute, value):
 
 def _get_attrs_kwargs(field: Field) -> dict[str, ty.Any]:
     kwargs = {}
-    if field.default is not NO_DEFAULT:
+    if not field.mandatory:
         kwargs["default"] = field.default
     # elif is_optional(field.type):
     #     kwargs["default"] = None

pydra/compose/base/field.py

@@ -3,9 +3,10 @@
 from typing import Self
 import attrs.validators
 from attrs.converters import default_if_none
+from fileformats.core import to_mime
 from fileformats.generic import File, FileSet
-from pydra.utils.typing import TypeParser, is_optional, is_type
-from pydra.utils.general import task_fields
+from pydra.utils.typing import TypeParser, is_optional, is_type, is_union
+from pydra.utils.general import task_fields, wrap_text
 import attrs
 
 if ty.TYPE_CHECKING:
@@ -234,6 +235,64 @@ def _requires_validator(self, _, value):
                 f"with None) or boolean, not type {self.type} ({self!r})"
             )
 
+    def markdown_listing(
+        self, line_width: int = 79, help_indent: int = 4, **kwargs
+    ) -> str:
+        """Get the listing for the field in markdown-like format
+
+        Parameters
+        ----------
+        line_width: int
+            The maximum line width for the output, by default it is 79
+        help_indent: int
+            The indentation for the help text, by default it is 4
+
+        Returns
+        -------
+        str
+            The listing for the field in markdown-like format
+        """
+
+        def type_to_str(type_: ty.Type[ty.Any]) -> str:
+            if type_ is type(None):
+                return "None"
+            if is_union(type_):
+                return " | ".join(
+                    type_to_str(t) for t in ty.get_args(type_) if t is not None
+                )
+            try:
+                type_str = to_mime(type_, official=False)
+            except Exception:
+                if origin := ty.get_origin(type_):
+                    type_str = f"{origin.__name__}[{', '.join(map(type_to_str, ty.get_args(type_)))}]"
+                else:
+                    try:
+                        type_str = type_.__name__
+                    except AttributeError:
+                        type_str = str(type_)
+            return type_str
+
+        s = f"- {self.name}: {type_to_str(self.type)}"
+        if isinstance(self.default, attrs.Factory):
+            s += f"; default-factory = {self.default.factory.__name__}()"
+        elif callable(self.default):
+            s += f"; default = {self.default.__name__}()"
+        elif not self.mandatory:
+            s += f"; default = {self.default!r}"
+        if self._additional_descriptors(**kwargs):
+            s += f" ({', '.join(self._additional_descriptors(**kwargs))})"
+        if self.help:
+            s += f"\n{wrap_text(self.help, width=line_width, indent_size=help_indent)}"
+        return s
+
+    def _additional_descriptors(self, **kwargs) -> list[str]:
+        """Get additional descriptors for the field"""
+        return []
+
+    def __lt__(self, other: "Field") -> bool:
+        """Compare two fields based on their position"""
+        return self.name < other.name
+
 
 @attrs.define(kw_only=True)
 class Arg(Field):
@@ -273,6 +332,13 @@ class Arg(Field):
     copy_ext_decomp: File.ExtensionDecomposition = File.ExtensionDecomposition.single
     readonly: bool = False
 
+    def _additional_descriptors(self, **kwargs) -> list[str]:
+        """Get additional descriptors for the field"""
+        descriptors = super()._additional_descriptors(**kwargs)
+        if self.allowed_values:
+            descriptors.append(f"allowed_values={self.allowed_values}")
+        return descriptors
+
 
 @attrs.define(kw_only=True, slots=False)
 class Out(Field):

pydra/compose/base/helpers.py

@@ -208,7 +208,7 @@ def extract_function_inputs_and_outputs(
     for inpt_name, default in input_defaults.items():
         inpt = inputs[inpt_name]
         if isinstance(inpt, arg_type):
-            if inpt.default is NO_DEFAULT:
+            if inpt.mandatory:
                 inpt.default = default
         elif inspect.isclass(inpt) or ty.get_origin(inpt):
             inputs[inpt_name] = arg_type(type=inpt, default=default)
@@ -283,10 +283,10 @@ def parse_doc_string(doc_str: str) -> tuple[dict[str, str], dict[str, str] | lis
     for return_val, return_help in re.findall(r":return (\w+): (.*)", doc_str):
         output_helps[return_val] = return_help
     google_args_match = re.match(
-        r".*\n\s*Args:\n(.*)", doc_str, flags=re.DOTALL | re.MULTILINE
+        r"(?:.*\n)?\s*Args:\n(.*)", doc_str, flags=re.DOTALL | re.MULTILINE
     )
     google_returns_match = re.match(
-        r".*\n\s*Returns:\n(.*)", doc_str, flags=re.DOTALL | re.MULTILINE
+        r"(?:.*\n)?\s*Returns:\n(.*)", doc_str, flags=re.DOTALL | re.MULTILINE
     )
     if google_args_match:
         args_str = google_args_match.group(1)
@@ -303,12 +303,14 @@ def parse_doc_string(doc_str: str) -> tuple[dict[str, str], dict[str, str] | lis
             return_help = white_space_re.sub(" ", return_help).strip()
             output_helps[return_name] = return_help
     numpy_args_match = re.match(
-        r".*\n\s+Parameters\n\s*----------\s*\n(.*)",
+        r"(?:.*\n)?\s+Parameters\n\s*----------\s*\n(.*)",
         doc_str,
         flags=re.DOTALL | re.MULTILINE,
     )
     numpy_returns_match = re.match(
-        r".*\n\s+Returns\n\s*-------\s*\n(.*)", doc_str, flags=re.DOTALL | re.MULTILINE
+        r"(?:.*\n)?\s+Returns\n\s*-------\s*\n(.*)",
+        doc_str,
+        flags=re.DOTALL | re.MULTILINE,
     )
     if numpy_args_match:
         args_str = numpy_args_match.group(1)

pydra/compose/base/task.py

@@ -19,7 +19,7 @@
     attrs_values,
 )
 from pydra.utils.hash import Cache, hash_single, register_serializer
-from .field import Field, Arg, Out, NO_DEFAULT
+from .field import Field, Arg, Out
 
 
 if ty.TYPE_CHECKING:
@@ -62,7 +62,7 @@ def _from_task(cls, job: "Job[TaskType]") -> Self:
         """
         defaults = {}
         for output in task_fields(cls):
-            if output.default is NO_DEFAULT:
+            if output.mandatory:
                 default = attrs.NOTHING
             elif isinstance(output.default, attrs.Factory):
                 default = output.default.factory()
@@ -598,7 +598,7 @@ def _check_resolved(self):
 
 
 # def set_none_default_if_optional(field: Field) -> None:
-#     if is_optional(field.type) and field.default is NO_DEFAULT:
+#     if is_optional(field.type) and field.mandatory:
 #         field.default = None
 
 

pydra/compose/shell/builder.py

@@ -33,12 +33,6 @@
 from .task import Task, Outputs
 
 
-EXECUTABLE_HELP_STRING = (
-    "the first part of the command, can be a string, "
-    "e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']"
-)
-
-
 @dataclass_transform(
     kw_only_default=True,
     field_specifiers=(field.out, field.outarg),
@@ -210,7 +204,7 @@ def make(
             position=0,
             default=executable,
             validator=attrs.validators.min_len(1),
-            help=EXECUTABLE_HELP_STRING,
+            help=Task.EXECUTABLE_HELP,
         )
 
         # Set positions for the remaining inputs that don't have an explicit position