split-out unittests into new sub-packages/modules

Author: tclose

.github/workflows/ci-cd.yml

@@ -181,7 +181,7 @@ jobs:
         docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.utils.__version__)'"
     - name: Run pytest
       run: |
-        docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_submitter.py --import-mode=importlib --rootdir /pydra/pydra --only-worker=slurm --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
+        docker exec slurm bash -c "pytest /pydra/pydra/workers/tests/test_worker.py --import-mode=importlib --rootdir /pydra/pydra --only-worker=slurm --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2
       with:
@@ -261,7 +261,7 @@ jobs:
   #       pip install --upgrade pip && pip install -e .[test] && python -c 'import pydra.engine; print(pydra.utils.__version__)'
   #   - name: Run pytest
   #     run: |
-  #       pytest pydra/engine/tests/test_submitter.py --import-mode=importlib --rootdir . --only-worker=sge --color=yes -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
+  #       pytest pydra/workers/tests/test_worker.py --import-mode=importlib --rootdir . --only-worker=sge --color=yes -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
   #   - name: Upload coverage to Codecov
   #     uses: codecov/codecov-action@v2
   #     with:

pydra/compose/shell/task.py

@@ -43,7 +43,7 @@
 
 
 @attrs.define(kw_only=True, auto_attribs=False, eq=False, repr=False)
-class Outputs(base.Outputs):
+class ShellOutputs(base.Outputs):
     """Output task of a generic shell process."""
 
     BASE_NAMES = ["return_code", "stdout", "stderr"]
@@ -216,7 +216,7 @@ def _resolve_value(
         return callable_(**call_args_val)
 
 
-ShellOutputsType = ty.TypeVar("OutputType", bound=Outputs)
+ShellOutputsType = ty.TypeVar("OutputType", bound=ShellOutputs)
 
 
 @state_array_support
@@ -508,3 +508,4 @@ def split_cmd(cmd: str | None):
 
 # Alias ShellTask to Task so we can refer to it by shell.Task
 Task = ShellTask
+Outputs = ShellOutputs

pydra/engine/tests/conftest.py renamed to pydra/compose/shell/tests/conftest.py

@@ -5,7 +5,7 @@
 import pytest
 import cloudpickle as cp
 from pydra.compose import shell
-from pydra.utils.general import task_fields
+from pydra.utils.general import task_fields, task_help, wrap_text
 from pydra.compose.shell.builder import _InputPassThrough
 from fileformats.generic import File, Directory, FsObject
 from fileformats import text, image
@@ -984,6 +984,49 @@ def test_shell_missing_executable_dynamic():
         )
 
 
+def test_shell_help1():
+
+    Shelly = shell.define(
+        "shelly <in_file:generic/file> <out|out_file:generic/file> --arg1 <arg1:int> "
+        "--arg2 <arg2:float?> --opt-out <out|opt_out:File?>"
+    )
+
+    assert task_help(Shelly) == [
+        "----------------------------",
+        "Help for Shell task 'shelly'",
+        "----------------------------",
+        "",
+        "Inputs:",
+        "- executable: str | Sequence[str]; default = 'shelly'",
+        "    the first part of the command, can be a string, e.g. 'ls', or a list, e.g.",
+        "    ['ls', '-l', 'dirname']",
+        "- in_file: generic/file",
+        "- out_file: Path | bool; default = True",
+    ] + wrap_text(shell.outarg.PATH_TEMPLATE_HELP).split("\n") + [
+        "- arg1: int ('--arg1')",
+        "- arg2: float | None; default = None ('--arg2')",
+        "- opt_out: Path | bool | None; default = None ('--opt-out')",
+    ] + wrap_text(
+        shell.outarg.OPTIONAL_PATH_TEMPLATE_HELP
+    ).split(
+        "\n"
+    ) + [
+        "- additional_args: list[str | generic/file]; default-factory = list()",
+        "    Additional free-form arguments to append to the end of the command.",
+        "",
+        "Outputs:",
+        "- out_file: generic/file",
+        "- opt_out: generic/file | None; default = None",
+        "- return_code: int",
+        "    " + shell.Outputs.RETURN_CODE_HELP,
+        "- stdout: str",
+        "    " + shell.Outputs.STDOUT_HELP,
+        "- stderr: str",
+        "    " + shell.Outputs.STDERR_HELP,
+        "",
+    ]
+
+
 def list_entries(stdout):
     return stdout.split("\n")[:-1]
 

pydra/engine/tests/test_shelltask_inputspec.py renamed to pydra/compose/shell/tests/test_shell_cmdline.py

@@ -12,7 +12,6 @@
     File,
     Directory,
 )
-from pydra.utils.general import task_help, wrap_text
 from pydra.utils.typing import (
     MultiOutputFile,
     MultiInputObj,
@@ -3576,44 +3575,24 @@ class Outputs(shell.Outputs):
     )
 
 
-def test_shell_help1():
+@no_win
+def test_shell_cmd(tmpdir):
+    cmd = ["echo", "hail", "pydra"]
 
+    # all args given as executable
+    Shelly = shell.define(" ".join(cmd))
+    shelly = Shelly()
+    assert shelly.cmdline == " ".join(cmd)
+    outputs = shelly()
+    assert outputs.stdout == " ".join(cmd[1:]) + "\n"
+
+    # separate command into exec + args
     Shelly = shell.define(
-        "shelly <in_file:generic/file> <out|out_file:generic/file> --arg1 <arg1:int> "
-        "--arg2 <arg2:float?> --opt-out <out|opt_out:File?>"
+        cmd[0], inputs=[shell.arg(name=a, default=a) for a in cmd[1:]]
     )
-
-    assert task_help(Shelly) == [
-        "----------------------------",
-        "Help for Shell task 'shelly'",
-        "----------------------------",
-        "",
-        "Inputs:",
-        "- executable: str | Sequence[str]; default = 'shelly'",
-        "    the first part of the command, can be a string, e.g. 'ls', or a list, e.g.",
-        "    ['ls', '-l', 'dirname']",
-        "- in_file: generic/file",
-        "- out_file: Path | bool; default = True",
-    ] + wrap_text(shell.outarg.PATH_TEMPLATE_HELP).split("\n") + [
-        "- arg1: int ('--arg1')",
-        "- arg2: float | None; default = None ('--arg2')",
-        "- opt_out: Path | bool | None; default = None ('--opt-out')",
-    ] + wrap_text(
-        shell.outarg.OPTIONAL_PATH_TEMPLATE_HELP
-    ).split(
-        "\n"
-    ) + [
-        "- additional_args: list[str | generic/file]; default-factory = list()",
-        "    Additional free-form arguments to append to the end of the command.",
-        "",
-        "Outputs:",
-        "- out_file: generic/file",
-        "- opt_out: generic/file | None; default = None",
-        "- return_code: int",
-        "    " + shell.Outputs.RETURN_CODE_HELP,
-        "- stdout: str",
-        "    " + shell.Outputs.STDOUT_HELP,
-        "- stderr: str",
-        "    " + shell.Outputs.STDERR_HELP,
-        "",
-    ]
+    shelly = Shelly()
+    assert shelly.executable == "echo"
+    assert shelly.cmdline == " ".join(cmd)
+    outputs = shelly()
+    assert outputs.return_code == 0
+    assert outputs.stdout == " ".join(cmd[1:]) + "\n"

pydra/compose/tests/test_shell.py renamed to pydra/compose/shell/tests/test_shell_fields.py

@@ -0,0 +1,109 @@
+import typing as ty
+from pydra.compose import shell
+from pydra.compose.shell.templating import argstr_formatting
+from pathlib import Path
+from unittest.mock import Mock
+from pydra.compose.shell.templating import template_update_single
+import os
+import shutil
+from pathlib import Path
+import random
+import platform
+import typing as ty
+import pytest
+import cloudpickle as cp
+from pydra.engine.submitter import Submitter
+from pydra.engine.job import Job
+from pydra.compose import workflow
+from fileformats.generic import Directory, File
+from pydra.engine.tests.utils import Multiply, RaiseXeq1
+from pydra.utils.general import position_sort
+from pydra.compose.shell.templating import parse_format_string
+from pydra.engine.job import save, load_and_run
+from pydra.workers.cf import get_available_cpus
+from pydra.utils.hash import hash_function
+
+
+@pytest.mark.parametrize(
+    "pos_args",
+    [
+        [(2, "b"), (1, "a"), (3, "c")],
+        [(-2, "b"), (1, "a"), (-1, "c")],
+        [(None, "b"), (1, "a"), (-1, "c")],
+        [(-3, "b"), (None, "a"), (-1, "c")],
+        [(None, "b"), (1, "a"), (None, "c")],
+    ],
+)
+def test_position_sort(pos_args):
+    final_args = position_sort(pos_args)
+    assert final_args == ["a", "b", "c"]
+
+
+def test_parse_format_string1():
+    assert parse_format_string("{a}") == {"a"}
+
+
+def test_parse_format_string2():
+    assert parse_format_string("{abc}") == {"abc"}
+
+
+def test_parse_format_string3():
+    assert parse_format_string("{a:{b}}") == {"a", "b"}
+
+
+def test_parse_format_string4():
+    assert parse_format_string("{a:{b[2]}}") == {"a", "b"}
+
+
+def test_parse_format_string5():
+    assert parse_format_string("{a.xyz[somekey].abc:{b[a][b].d[0]}}") == {"a", "b"}
+
+
+def test_parse_format_string6():
+    assert parse_format_string("{a:05{b[a 2][b].e}}") == {"a", "b"}
+
+
+def test_parse_format_string7():
+    assert parse_format_string(
+        "{a1_field} {b2_field:02f} -test {c3_field[c]} -me {d4_field[0]}"
+    ) == {"a1_field", "b2_field", "c3_field", "d4_field"}
+
+
+def test_argstr_formatting():
+    @shell.define
+    class Shelly(shell.Task["Shelly.Outputs"]):
+        a1_field: str
+        b2_field: float
+        c3_field: ty.Dict[str, str]
+        d4_field: ty.List[str] = shell.arg(sep=" ")
+        executable = "dummy"
+
+        class Outputs(shell.Outputs):
+            pass
+
+    values = dict(a1_field="1", b2_field=2.0, c3_field={"c": "3"}, d4_field=["4"])
+    assert (
+        argstr_formatting(
+            "{a1_field} {b2_field:02f} -test {c3_field[c]} -me {d4_field[0]}",
+            values,
+        )
+        == "1 2.000000 -test 3 -me 4"
+    )
+
+
+def test_template_formatting(tmp_path: Path):
+    field = Mock()
+    field.name = "grad"
+    field.argstr = "--grad"
+    field.path_template = ("{in_file}.bvec", "{in_file}.bval")
+    field.keep_extension = False
+    task = Mock()
+    values = {"in_file": Path("/a/b/c/file.txt"), "grad": True}
+
+    assert template_update_single(
+        field,
+        task,
+        values=values,
+        output_dir=tmp_path,
+        spec_type="input",
+    ) == [tmp_path / "file.bvec", tmp_path / "file.bval"]

pydra/engine/tests/test_shelltask.py renamed to pydra/compose/shell/tests/test_shell_run.py

@@ -400,3 +400,27 @@ class Outputs:
             @staticmethod
             def function(a):
                 return a + 1
+
+
+def test_task_repr():
+    @python.define(outputs=["x", "y", "z"])
+    def IdentityN3(x: int, y: int = 1, z: int = 2) -> tuple[int, int, int]:
+        return x, y, z
+
+    assert repr(IdentityN3(x=1, y=2)) == "IdentityN3(x=1, y=2)"
+
+
+@attrs.define(auto_attribs=True)
+class A:
+    x: int
+
+
+def test_object_input():
+    """Test function tasks with object inputs"""
+
+    @python.define
+    def TestFunc(a: A):
+        return a.x
+
+    outputs = TestFunc(a=A(x=7))()
+    assert outputs.out == 7

pydra/compose/shell/tests/test_shell_templating.py

@@ -93,3 +93,16 @@ def test_task_numpyinput_2(tmp_path: Path):
     outputs = nn(cache_dir=tmp_path)
     assert outputs.out[0] == np.array(["VAL1"], dtype=object)
     assert outputs.out[1] == np.array(["VAL2"], dtype=object)
+
+
+def test_numpy_fft():
+    """checking if mark.task works for numpy functions"""
+    np = pytest.importorskip("numpy")
+    FFT = python.define(inputs={"a": np.ndarray}, outputs={"out": np.ndarray})(
+        np.fft.fft
+    )
+
+    arr = np.array([[1, 10], [2, 20]])
+    fft = FFT(a=arr)
+    outputs = fft()
+    assert np.allclose(np.fft.fft(arr), outputs.out)