test_dockertask tests pass

tclose · tclose · commit 761d97a714c4 · 2025-02-24T16:06:04.000+11:00
diff --git a/new-docs/source/tutorial/5-shell.ipynb b/new-docs/source/tutorial/5-shell.ipynb
@@ -109,11 +109,6 @@
     "print(trim_png.cmdline)"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -180,6 +175,68 @@
     "print(f\"'--int-arg' default: {fields_dict(Cp)['int_arg'].default}\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Path templates for output files\n",
+    "\n",
+    "By default, when an output file argument is defined, a `path_template` attribute will\n",
+    "be assigned to the field based on its name and extension (if applicable). For example,\n",
+    "the `zipped` output field in the following Gzip command will be assigned a\n",
+    "`path_template` of `out_file.gz`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydra.design import shell\n",
+    "from fileformats.generic import File\n",
+    "\n",
+    "Gzip = shell.define(\"gzip <out|out_file:application/gzip> <in_files+>\")\n",
+    "gzip = Gzip(in_files=File.mock(\"/a/file.txt\"))\n",
+    "print(gzip.cmdline)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "However, if this needs to be specified it can be by using the `$` operator, e.g."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Gzip = shell.define(\"gzip <out|out_file:application/gzip$zipped.gz> <in_files+>\")\n",
+    "gzip = Gzip(in_files=File.mock(\"/a/file.txt\"))\n",
+    "print(gzip.cmdline)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To give the field a path_template of `archive.gz` when it is written on the command line.\n",
+    "Note that this value can always be overridden when the task is initialised, e.g."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gzip = Gzip(in_files=File.mock(\"/a/file.txt\"), out_file=\"/path/to/archive.gz\")\n",
+    "print(gzip.cmdline)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
diff --git a/pydra/design/shell.py b/pydra/design/shell.py
@@ -537,7 +537,7 @@ def parse_command_line_template(
     tokens = tokens[start_args_index:]
     if not tokens:
         return executable, inputs, outputs
-    arg_pattern = r"<([:a-zA-Z0-9_,\|\-\.\/\+\*]+(?:\?|=[^>]+)?)>"
+    arg_pattern = r"<([:a-zA-Z0-9_,\|\-\.\/\+\*]+(?:\?|(?:=|\$)[^>]+)?)>"
     opt_pattern = r"--?[a-zA-Z0-9_]+"
     arg_re = re.compile(arg_pattern)
     opt_re = re.compile(opt_pattern)
@@ -632,6 +632,13 @@ def from_type_str(type_str) -> type:
             elif "=" in name:
                 name, default = name.split("=")
                 kwds["default"] = eval(default)
+            elif "$" in name:
+                name, path_template = name.split("$")
+                kwds["path_template"] = path_template
+                if field_type is not outarg:
+                    raise ValueError(
+                        f"Path templates can only be used with output fields, not {token}"
+                    )
             if ":" in name:
                 name, type_str = name.split(":")
                 type_ = from_type_str(type_str)
@@ -649,7 +656,7 @@ def from_type_str(type_str) -> type:
                 # Add field to outputs with the same name as the input
                 add_arg(name, out, {"type": type_, "callable": _InputPassThrough(name)})
             # If name contains a '.', treat it as a file template and strip it from the name
-            if field_type is outarg:
+            if field_type is outarg and "path_template" not in kwds:
                 path_template = name
                 if is_fileset_or_union(type_):
                     if ty.get_origin(type_):
diff --git a/pydra/engine/environments.py b/pydra/engine/environments.py
@@ -92,7 +92,7 @@ def bind(self, loc, mode="ro"):
         loc_abs = Path(loc).absolute()
         return f"{loc_abs}:{self.root}{loc_abs}:{mode}"
 
-    def _get_bindings(
+    def get_bindings(
         self, task: "Task", root: str | None = None
     ) -> tuple[dict[str, tuple[str, str]], dict[str, tuple[Path, ...]]]:
         """Return bindings necessary to run task in an alternative root.
@@ -156,13 +156,14 @@ class Docker(Container):
     def execute(self, task: "Task[ShellDef]") -> dict[str, ty.Any]:
         docker_img = f"{self.image}:{self.tag}"
         # mounting all input locations
-        mounts, input_updates = self._get_bindings(task=task, root=self.root)
+        mounts, input_updates = self.get_bindings(task=task, root=self.root)
+
+        # add the cache directory to the list of mounts
+        mounts[task.cache_dir] = (f"{self.root}{task.cache_dir}", "rw")
 
         docker_args = [
             "docker",
             "run",
-            "-v",
-            self.bind(task.cache_dir, "rw"),
             *self.xargs,
         ]
         docker_args.extend(
@@ -195,7 +196,7 @@ class Singularity(Container):
     def execute(self, task: "Task[ShellDef]") -> dict[str, ty.Any]:
         singularity_img = f"{self.image}:{self.tag}"
         # mounting all input locations
-        mounts, input_updates = self._get_bindings(task=task, root=self.root)
+        mounts, input_updates = self.get_bindings(task=task, root=self.root)
 
         # todo adding xargsy etc
         singularity_args = [
diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py
@@ -1,3 +1,4 @@
+import attrs
 import pytest
 from pydra.engine.submitter import Submitter
 from pydra.engine.specs import ShellDef, ShellOutputs
@@ -100,7 +101,10 @@ def test_docker_st_1(run_function, plugin, tmp_path):
     docky = Docky().split(executable=cmd)
 
     outputs = run_function(docky, tmp_path, plugin, environment=Docker(image="busybox"))
-    assert outputs.stdout[0] == f"/mnt/pydra{docky.output_dir[0]}\n"
+    assert (
+        outputs.stdout[0]
+        == f"/mnt/pydra{tmp_path}/{attrs.evolve(docky, executable=cmd[0])._checksum}\n"
+    )
     assert outputs.stdout[1] == "root\n"
     assert outputs.return_code[0] == outputs.return_code[1] == 0
 
@@ -115,8 +119,7 @@ def test_docker_outputspec_1(plugin, tmp_path):
     customised output_spec, adding files to the output, providing specific pathname
     output_path is automatically added to the bindings
     """
-    outputs = [shell.out(name="newfile", type=File, help="new file")]
-    Docky = shell.define("touch newfile_tmp.txt", outputs=outputs)
+    Docky = shell.define("touch <out|newfile$newfile_tmp.txt>")
     docky = Docky()
 
     outputs = docky(plugin=plugin, environment=Docker(image="ubuntu"))
@@ -136,23 +139,22 @@ def test_docker_inputspec_1(tmp_path):
 
     cmd = "cat"
 
-    inputs = [
-        shell.arg(
-            name="file",
-            type=File,
-            position=1,
-            argstr="",
-            help="input file",
-        )
-    ]
-
-    docky = shell.define(cmd, inputs=inputs)(
-        environment=Docker(image="busybox"),
-        file=filename,
-        strip=True,
+    Docky = shell.define(
+        cmd,
+        inputs=[
+            shell.arg(
+                name="file",
+                type=File,
+                position=1,
+                argstr="",
+                help="input file",
+            )
+        ],
     )
 
-    outputs = docky()
+    docky = Docky(file=filename)
+
+    outputs = docky(environment=Docker(image="busybox"), cache_dir=tmp_path)
     assert outputs.stdout.strip() == "hello from pydra"
 
 
@@ -168,23 +170,23 @@ def test_docker_inputspec_1a(tmp_path):
 
     cmd = "cat"
 
-    inputs = [
-        shell.arg(
-            name="file",
-            type=File,
-            default=filename,
-            position=1,
-            argstr="",
-            help="input file",
-        )
-    ]
-
-    docky = shell.define(cmd, inputs=inputs)(
-        environment=Docker(image="busybox"),
-        strip=True,
+    Docky = shell.define(
+        cmd,
+        inputs=[
+            shell.arg(
+                name="file",
+                type=File,
+                default=filename,
+                position=1,
+                argstr="",
+                help="input file",
+            )
+        ],
     )
 
-    outputs = docky()
+    docky = Docky()
+
+    outputs = docky(environment=Docker(image="busybox"), cache_dir=tmp_path)
     assert outputs.stdout.strip() == "hello from pydra"
 
 
@@ -476,33 +478,33 @@ def test_docker_wf_inputspec_1(plugin, tmp_path):
 
     cmd = "cat"
 
-    inputs = [
-        shell.arg(
-            name="file",
-            type=File,
-            position=1,
-            argstr="",
-            help="input file",
-        )
-    ]
+    Docky = shell.define(
+        cmd,
+        inputs=[
+            shell.arg(
+                name="file",
+                type=File,
+                position=1,
+                argstr="",
+                help="input file",
+            )
+        ],
+    )
 
     @workflow.define
-    def Workflow(cmd, file):
+    def Workflow(file):
 
         docky = workflow.add(
-            shell.define(cmd, inputs=inputs)(
-                file=file,
-                environment=Docker(image="busybox"),
-                strip=True,
-            )
+            Docky(file=file),
+            environment=Docker(image="busybox"),
         )
 
         return docky.stdout
 
-    wf = Workflow(cmd=cmd, file=filename)
+    wf = Workflow(file=filename)
 
     outputs = wf()
-    assert outputs.ou.strip() == "hello from pydra"
+    assert outputs.out.strip() == "hello from pydra"
 
 
 @no_win
@@ -532,7 +534,7 @@ def test_docker_wf_state_inputspec_1(plugin, tmp_path):
     )
 
     @workflow.define
-    def Workflow(cmd, file):
+    def Workflow(file):
 
         docky = workflow.add(
             Docky(file=file),
@@ -541,9 +543,9 @@ def Workflow(cmd, file):
 
         return docky.stdout
 
-    wf = Workflow(cmd=cmd)
+    wf = Workflow().split(file=[file_1, file_2])
 
-    outputs = wf(split={"file": [file_1, file_2]})
+    outputs = wf()
 
     assert outputs.out[0].strip() == "hello from pydra"
     assert outputs.out[1].strip() == "have a nice one"
@@ -562,30 +564,30 @@ def test_docker_wf_ndst_inputspec_1(plugin, tmp_path):
 
     cmd = "cat"
 
-    inputs = [
-        shell.arg(
-            name="file",
-            type=File,
-            position=1,
-            argstr="",
-            help="input file",
-        )
-    ]
+    Docky = shell.define(
+        cmd,
+        inputs=[
+            shell.arg(
+                name="file",
+                type=File,
+                position=1,
+                argstr="",
+                help="input file",
+            )
+        ],
+    )
 
     @workflow.define
-    def Workflow(cmd, file):
+    def Workflow(file):
 
         docky = workflow.add(
-            shell.define(cmd, inputs=inputs)(
-                environment=Docker(image="busybox"),
-                file=file,
-                strip=True,
-            )
+            Docky(file=file),
+            environment=Docker(image="busybox"),
         )
 
         return docky.stdout
 
-    wf = Workflow(cmd=cmd)
+    wf = Workflow().split(file=[str(file_1), str(file_2)])
 
-    outputs = wf(split={"file": [str(file_1), str(file_2)]})
+    outputs = wf()
     assert outputs.out == ["hello from pydra", "have a nice one"]
