debugged test_dockertask

Author: tclose

pydra/design/shell.py

@@ -4,7 +4,6 @@
 import typing as ty
 import re
 from collections import defaultdict
-import shlex
 import inspect
 from copy import copy
 import attrs
@@ -23,7 +22,7 @@
     make_task_def,
     NO_DEFAULT,
 )
-from pydra.utils.typing import is_fileset_or_union, MultiInputObj
+from pydra.utils.typing import is_fileset_or_union, MultiInputObj, non_optional_type
 
 if ty.TYPE_CHECKING:
     from pydra.engine.specs import ShellDef
@@ -108,6 +107,7 @@ def _validate_sep(self, _, sep):
             tp = ty.get_args(self.type)[0]
         else:
             tp = self.type
+        tp = non_optional_type(tp)
         origin = ty.get_origin(tp) or tp
         if (
             inspect.isclass(origin)
@@ -522,7 +522,7 @@ def parse_command_line_template(
     if isinstance(template, list):
         tokens = template
     else:
-        tokens = shlex.split(template)
+        tokens = template.split()
     executable = []
     start_args_index = 0
     for part in tokens:
@@ -631,7 +631,9 @@ def from_type_str(type_str) -> type:
                 kwds["default"] = attrs.Factory(list)
             elif "=" in name:
                 name, default = name.split("=")
-                kwds["default"] = eval(default)
+                kwds["default"] = (
+                    default[1:-1] if re.match(r"('|\").*\1", default) else eval(default)
+                )
             elif "$" in name:
                 name, path_template = name.split("$")
                 kwds["path_template"] = path_template

pydra/design/tests/test_shell.py

@@ -162,7 +162,7 @@ def test_interface_template_more_complex():
 
     Cp = shell.define(
         (
-            "cp <in_fs_objects:fs-object,...> <out|out_dir:directory> "
+            "cp <in_fs_objects:fs-object+> <out|out_dir:directory> "
             "-R<recursive> "
             "--text-arg <text_arg?> "
             "--int-arg <int_arg:int?> "
@@ -187,7 +187,9 @@ def test_interface_template_more_complex():
             help=shell.EXECUTABLE_HELP_STRING,
         ),
         shell.arg(
-            name="in_fs_objects", type=MultiInputObj[FsObject], position=1, sep=" "
+            name="in_fs_objects",
+            type=MultiInputObj[FsObject],
+            position=1,
         ),
         output,
         shell.arg(name="recursive", argstr="-R", type=bool, default=False, position=3),
@@ -209,6 +211,7 @@ def test_interface_template_more_complex():
             name="tuple_arg",
             argstr="--tuple-arg",
             type=tuple[int, str] | None,
+            sep=" ",
             default=None,
             position=6,
         ),
@@ -245,7 +248,7 @@ def test_interface_template_with_overrides_and_optionals():
 
     Cp = shell.define(
         (
-            "cp <in_fs_objects:fs-object,...> <out|out_dir:directory> <out|out_file:file?> "
+            "cp <in_fs_objects:fs-object+> <out|out_dir:directory> <out|out_file:file?> "
             "-R<recursive> "
             "--text-arg <text_arg> "
             "--int-arg <int_arg:int?> "
@@ -283,9 +286,7 @@ def test_interface_template_with_overrides_and_optionals():
             position=0,
             help=shell.EXECUTABLE_HELP_STRING,
         ),
-        shell.arg(
-            name="in_fs_objects", type=MultiInputObj[FsObject], position=1, sep=" "
-        ),
+        shell.arg(name="in_fs_objects", type=MultiInputObj[FsObject], position=1),
         shell.arg(
             name="recursive",
             argstr="-R",
@@ -307,6 +308,7 @@ def test_interface_template_with_overrides_and_optionals():
             argstr="--tuple-arg",
             type=tuple[int, str],
             position=5,
+            sep=" ",
         ),
     ] + outargs + [ShellDef.additional_args]
     assert sorted_fields(Cp.Outputs) == outargs + [
@@ -332,7 +334,7 @@ def test_interface_template_with_defaults():
 
     Cp = shell.define(
         (
-            "cp <in_fs_objects:fs-object,...> <out|out_dir:directory> "
+            "cp <in_fs_objects:fs-object+> <out|out_dir:directory> "
             "-R<recursive=True> "
             "--text-arg <text_arg='foo'> "
             "--int-arg <int_arg:int=99> "
@@ -356,9 +358,7 @@ def test_interface_template_with_defaults():
             position=0,
             help=shell.EXECUTABLE_HELP_STRING,
         ),
-        shell.arg(
-            name="in_fs_objects", type=MultiInputObj[FsObject], position=1, sep=" "
-        ),
+        shell.arg(name="in_fs_objects", type=MultiInputObj[FsObject], position=1),
         output,
         shell.arg(name="recursive", argstr="-R", type=bool, default=True, position=3),
         shell.arg(
@@ -371,6 +371,7 @@ def test_interface_template_with_defaults():
             type=tuple[int, str],
             default=(1, "bar"),
             position=6,
+            sep=" ",
         ),
         ShellDef.additional_args,
     ]
@@ -400,7 +401,7 @@ def test_interface_template_with_type_overrides():
 
     Cp = shell.define(
         (
-            "cp <in_fs_objects:fs-object,...> <out|out_dir:directory> "
+            "cp <in_fs_objects:fs-object+> <out|out_dir:directory> "
             "-R<recursive> "
             "--text-arg <text_arg> "
             "--int-arg <int_arg> "
@@ -425,9 +426,7 @@ def test_interface_template_with_type_overrides():
             position=0,
             help=shell.EXECUTABLE_HELP_STRING,
         ),
-        shell.arg(
-            name="in_fs_objects", type=MultiInputObj[FsObject], position=1, sep=" "
-        ),
+        shell.arg(name="in_fs_objects", type=MultiInputObj[FsObject], position=1),
         output,
         shell.arg(name="recursive", argstr="-R", type=bool, default=False, position=3),
         shell.arg(name="text_arg", argstr="--text-arg", type=str, position=4),
@@ -443,6 +442,7 @@ def test_interface_template_with_type_overrides():
             argstr="--tuple-arg",
             type=tuple[int, str],
             position=6,
+            sep=" ",
         ),
         ShellDef.additional_args,
     ]

pydra/engine/core.py

@@ -773,11 +773,17 @@ def add(
         OutputType
             The outputs definition of the node
         """
+        from pydra.engine.environments import Native
+
         if name is None:
             name = type(task_def).__name__
         if name in self._nodes:
             raise ValueError(f"Node with name {name!r} already exists in the workflow")
-        if environment and task_def._task_type != "shell":
+        if (
+            environment
+            and not isinstance(environment, Native)
+            and task_def._task_type != "shell"
+        ):
             raise ValueError(
                 "Environments can only be used with 'shell' tasks not "
                 f"{task_def._task_type!r} tasks ({task_def!r})"

pydra/engine/tests/test_dockertask.py

@@ -545,7 +545,7 @@ def Workflow(file):
 
     wf = Workflow().split(file=[file_1, file_2])
 
-    outputs = wf()
+    outputs = wf(cache_dir=tmp_path)
 
     assert outputs.out[0].strip() == "hello from pydra"
     assert outputs.out[1].strip() == "have a nice one"
@@ -589,5 +589,5 @@ def Workflow(file):
 
     wf = Workflow().split(file=[str(file_1), str(file_2)])
 
-    outputs = wf()
+    outputs = wf(cache_dir=tmp_path)
     assert outputs.out == ["hello from pydra", "have a nice one"]

pydra/engine/tests/test_singularity.py

@@ -390,7 +390,7 @@ class Outputs(ShellOutputs):
                 help="output file",
             )
 
-    singu = Singu(orig_file=str(file))
+    singu = Singu(orig_file=file)
 
     outputs = singu(environment=Singularity(image=image), cache_dir=tmp_path)
     assert outputs.stdout == ""