use full import paths for tests/utils.py files

Author: tclose

.github/workflows/ci-cd.yml

@@ -73,7 +73,7 @@ jobs:
         run:  echo "NO_ET=TRUE" >> $GITHUB_ENV
       - name: Pytest
         run: |
-          pytest -vs -n auto pydra --doctest-modules --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --rootdir pydra
+          pytest -vs -n auto pydra --doctest-modules --import-mode=importlib --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --rootdir pydra
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
         with:
@@ -133,7 +133,7 @@ jobs:
       - name: Install pydra (test)
         run: pip install -e ".[test]"
       - name: Pytest
-        run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py pydra/engine/tests/test_environments.py --rootdir pydra
+        run: pytest -vs --import-mode=importlib --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py pydra/engine/tests/test_environments.py --rootdir pydra
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
         with:
@@ -181,7 +181,7 @@ jobs:
         docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.utils.__version__)'"
     - name: Run pytest
       run: |
-        docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_submitter.py --rootdir /pydra/pydra --only-worker=slurm --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
+        docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_submitter.py --import-mode=importlib --rootdir /pydra/pydra --only-worker=slurm --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2
       with:
@@ -261,7 +261,7 @@ jobs:
   #       pip install --upgrade pip && pip install -e .[test] && python -c 'import pydra.engine; print(pydra.utils.__version__)'
   #   - name: Run pytest
   #     run: |
-  #       pytest pydra/engine/tests/test_submitter.py --rootdir . --only-worker=sge --color=yes -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
+  #       pytest pydra/engine/tests/test_submitter.py --import-mode=importlib --rootdir . --only-worker=sge --color=yes -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
   #   - name: Upload coverage to Codecov
   #     uses: codecov/codecov-action@v2
   #     with:

docs/source/tutorial/7-canonical-form.ipynb

@@ -194,7 +194,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import typing as ty\n",
+    "import re\n",
     "from pydra.compose import python, workflow\n",
+    "from pydra.compose.base import is_set\n",
     "from pydra.utils import print_help, show_workflow\n",
     "\n",
     "\n",
@@ -213,16 +216,13 @@
     "class CanonicalWorkflowTask(workflow.Task[\"CanonicalWorkflowTask.Outputs\"]):\n",
     "\n",
     "    @staticmethod\n",
-    "    def a_converter(value):\n",
-    "        if value is None:\n",
-    "            return value\n",
-    "        return float(value)\n",
+    "    def str2num(value: ty.Any) -> float | int:\n",
+    "        if isinstance(value, str) and re.match(r\"^\\d+(\\.\\d+)?$\", value):\n",
+    "            return eval(value)  # use eval to convert string to number\n",
+    "        return value\n",
     "\n",
     "    a: int\n",
-    "    b: float = workflow.arg(\n",
-    "        help=\"A float input\",\n",
-    "        converter=a_converter,\n",
-    "    )\n",
+    "    b: float = workflow.arg(help=\"A float input\", converter=str2num)\n",
     "\n",
     "    @staticmethod\n",
     "    def constructor(a, b):\n",

docs/source/tutorial/tst.py

@@ -1,30 +1,43 @@
-import os
-from pathlib import Path
-from fileformats.generic import File
-from pydra.compose import shell
-from pydra.utils import print_help
-
-
-# Arguments to the callable function can be one of
-def get_file_size(out_file: Path) -> int:
-    """Calculate the file size"""
-    result = os.stat(out_file)
-    return result.st_size
-
-
-ACommand = shell.define(
-    "a-command",
-    inputs={
-        "in_file": shell.arg(type=File, help="output file", argstr="", position=-2)
-    },
-    outputs={
-        "out_file": shell.outarg(type=File, help="output file", argstr="", position=-1),
-        "out_file_size": {
-            "type": int,
-            "help": "size of the output directory",
-            "callable": get_file_size,
-        },
-    },
-)
-
-print_help(ACommand)
+from pydra.compose import python, workflow
+from pydra.utils import print_help, show_workflow
+
+
+# Example python tasks
+@python.define
+def Add(a, b):
+    return a + b
+
+
+@python.define
+def Mul(a, b):
+    return a * b
+
+
+@workflow.define
+class CanonicalWorkflowTask(workflow.Task["CanonicalWorkflowTask.Outputs"]):
+
+    @staticmethod
+    def a_converter(value):
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return value
+
+    a: int
+    b: float = workflow.arg(
+        help="A float input",
+        converter=a_converter,
+    )
+
+    @staticmethod
+    def constructor(a, b):
+        add = workflow.add(Add(a=a, b=b))
+        mul = workflow.add(Mul(a=add.out, b=b))
+        return mul.out
+
+    class Outputs(workflow.Outputs):
+        out: float
+
+
+print_help(CanonicalWorkflowTask)
+show_workflow(CanonicalWorkflowTask)

pydra/compose/base/task.py

@@ -24,8 +24,8 @@
 
 if ty.TYPE_CHECKING:
     from pydra.engine.job import Job
-    from pydra.environments.base_environment import Environment
-    from pydra.workers.base_worker import Worker
+    from pydra.environments.base import Environment
+    from pydra.workers.base import Worker
     from pydra.engine.result import Result
     from pydra.engine.hooks import TaskHooks
 

pydra/compose/python.py

@@ -196,10 +196,10 @@ def make(wrapped: ty.Callable | type) -> Task:
 
 
 @attrs.define(kw_only=True, auto_attribs=False, eq=False, repr=False)
-class Outputs(base.Outputs):
+class PythonOutputs(base.Outputs):
 
     @classmethod
-    def _from_task(cls, job: "Job[Task]") -> ty.Self:
+    def _from_task(cls, job: "Job[PythonTask]") -> ty.Self:
         """Collect the outputs of a job from a combination of the provided inputs,
         the objects in the output directory, and the stdout and stderr of the process.
 
@@ -221,15 +221,15 @@ def _from_task(cls, job: "Job[Task]") -> ty.Self:
         return outputs
 
 
-PythonOutputsType = ty.TypeVar("OutputType", bound=Outputs)
+PythonOutputsType = ty.TypeVar("OutputType", bound=PythonOutputs)
 
 
 @attrs.define(kw_only=True, auto_attribs=False, eq=False, repr=False)
-class Task(base.Task[PythonOutputsType]):
+class PythonTask(base.Task[PythonOutputsType]):
 
     _task_type = "python"
 
-    def _run(self, job: "Job[Task]", rerun: bool = True) -> None:
+    def _run(self, job: "Job[PythonTask]", rerun: bool = True) -> None:
         # Prepare the inputs to the function
         inputs = attrs_values(self)
         del inputs["function"]
@@ -252,3 +252,8 @@ def _run(self, job: "Job[Task]", rerun: bool = True) -> None:
             raise RuntimeError(
                 f"expected {len(return_names)} elements, but {returned} were returned"
             )
+
+
+# Alias ShellTask to Task so we can refer to it by shell.Task
+Task = PythonTask
+Outputs = PythonOutputs

pydra/compose/shell/task.py

@@ -230,7 +230,7 @@ def additional_args_converter(value: ty.Any) -> list[str]:
 
 
 @attrs.define(kw_only=True, auto_attribs=False, eq=False, repr=False)
-class Task(base.Task[ShellOutputsType]):
+class ShellTask(base.Task[ShellOutputsType]):
 
     _task_type = "shell"
 
@@ -252,7 +252,7 @@ class Task(base.Task[ShellOutputsType]):
 
     RESERVED_FIELD_NAMES = base.Task.RESERVED_FIELD_NAMES + ("cmdline",)
 
-    def _run(self, job: "Job[Task]", rerun: bool = True) -> None:
+    def _run(self, job: "Job[ShellTask]", rerun: bool = True) -> None:
         """Run the shell command."""
         job.return_values = job.environment.execute(job)
 
@@ -504,3 +504,7 @@ def split_cmd(cmd: str | None):
         else:
             cmd_args.append(arg)
     return cmd_args
+
+
+# Alias ShellTask to Task so we can refer to it by shell.Task
+Task = ShellTask

pydra/compose/workflow.py

@@ -20,11 +20,11 @@
     from pydra.engine.lazy import LazyOutField
     from pydra.engine.graph import DiGraph
     from pydra.engine.submitter import NodeExecution
-    from pydra.environments.base_environment import Environment
+    from pydra.environments.base import Environment
     from pydra.engine.hooks import TaskHooks
 
 
-__all__ = ["define", "add", "this", "arg", "out"]
+__all__ = ["define", "add", "this", "arg", "out", "Task", "Outputs", "cast"]
 
 
 @attrs.define
@@ -284,10 +284,10 @@ def cast(field: ty.Any, new_type: type[U]) -> U:
 
 
 @attrs.define(kw_only=True, auto_attribs=False, eq=False, repr=False)
-class Outputs(base.Outputs):
+class WorkflowOutputs(base.Outputs):
 
     @classmethod
-    def _from_task(cls, job: "Job[Task]") -> ty.Self:
+    def _from_task(cls, job: "Job[WorkflowTask]") -> ty.Self:
         """Collect the outputs of a workflow job from the outputs of the nodes in the
 
         Parameters
@@ -342,23 +342,23 @@ def _from_task(cls, job: "Job[Task]") -> ty.Self:
         return outputs
 
 
-WorkflowOutputsType = ty.TypeVar("OutputType", bound=Outputs)
+WorkflowOutputsType = ty.TypeVar("OutputType", bound=WorkflowOutputs)
 
 
 @attrs.define(kw_only=True, auto_attribs=False, eq=False, repr=False)
-class Task(base.Task[WorkflowOutputsType]):
+class WorkflowTask(base.Task[WorkflowOutputsType]):
 
     _task_type = "workflow"
 
     RESERVED_FIELD_NAMES = base.Task.RESERVED_FIELD_NAMES + ("construct",)
 
     _constructed = attrs.field(default=None, init=False, repr=False, eq=False)
 
-    def _run(self, job: "Job[Task]", rerun: bool) -> None:
+    def _run(self, job: "Job[WorkflowTask]", rerun: bool) -> None:
         """Run the workflow."""
         job.submitter.expand_workflow(job, rerun)
 
-    async def _run_async(self, job: "Job[Task]", rerun: bool) -> None:
+    async def _run_async(self, job: "Job[WorkflowTask]", rerun: bool) -> None:
         """Run the workflow asynchronously."""
         await job.submitter.expand_workflow_async(job, rerun)
 
@@ -369,3 +369,8 @@ def construct(self) -> "Workflow":
             return self._constructed
         self._constructed = Workflow.construct(self)
         return self._constructed
+
+
+# Alias WorkflowTask to Task so we can refer to it as workflow.Task
+Task = WorkflowTask
+Outputs = WorkflowOutputs

pydra/engine/job.py

@@ -36,7 +36,7 @@
 from pydra.utils.typing import copy_nested_files
 from pydra.compose.shell.templating import template_update
 from pydra.utils.messenger import AuditFlag
-from pydra.environments.base_environment import Environment
+from pydra.environments.base import Environment
 
 logger = logging.getLogger("pydra")
 

pydra/engine/node.py

@@ -9,7 +9,7 @@
 
 if ty.TYPE_CHECKING:
     from pydra.engine.workflow import Workflow
-    from pydra.environments.base_environment import Environment
+    from pydra.environments.base import Environment
     from pydra.compose import base
     from pydra.engine.hooks import TaskHooks
 

pydra/engine/submitter.py

@@ -25,7 +25,7 @@
 from pydra.utils.general import default_run_cache_dir
 from pydra.compose import workflow
 from pydra.engine.state import State
-from pydra.workers.base_worker import Worker
+from pydra.workers.base import Worker
 from pydra.compose.base import Task, Outputs
 
 logger = logging.getLogger("pydra.submitter")
@@ -35,7 +35,7 @@
     from pydra.engine.result import Result
     from pydra.engine.hooks import TaskHooks
     from pydra.engine.workflow import Workflow
-    from pydra.environments.base_environment import Environment
+    from pydra.environments.base import Environment
 
 
 TaskType = ty.TypeVar("TaskType", bound="Task")
@@ -198,7 +198,7 @@ def __call__(
         result : Any
             The result of the job
         """
-        from pydra.environments.base_environment import Environment
+        from pydra.environments.base import Environment
 
         if raise_errors is None:
             raise_errors = self.worker.plugin_name() == "debug"