fixed up toy environments example, all docs should build now

Author: tclose

new-docs/source/tutorial/6-workflow.ipynb

@@ -435,13 +435,36 @@
    "outputs": [],
    "source": [
     "import tempfile\n",
+    "from pathlib import Path\n",
     "import numpy as np\n",
     "from fileformats.medimage import Nifti1\n",
     "import fileformats.medimage_mrtrix3 as mrtrix3\n",
     "from pydra.engine.environments import Docker\n",
     "from pydra.design import workflow, python\n",
     "from pydra.tasks.mrtrix3.v3_0 import MrConvert, MrThreshold\n",
     "\n",
+    "MRTRIX2NUMPY_DTYPES = {\n",
+    "    \"Int8\": np.dtype(\"i1\"),\n",
+    "    \"UInt8\": np.dtype(\"u1\"),\n",
+    "    \"Int16LE\": np.dtype(\"<i2\"),\n",
+    "    \"Int16BE\": np.dtype(\">i2\"),\n",
+    "    \"UInt16LE\": np.dtype(\"<u2\"),\n",
+    "    \"UInt16BE\": np.dtype(\">u2\"),\n",
+    "    \"Int32LE\": np.dtype(\"<i4\"),\n",
+    "    \"Int32BE\": np.dtype(\">i4\"),\n",
+    "    \"UInt32LE\": np.dtype(\"<u4\"),\n",
+    "    \"UInt32BE\": np.dtype(\">u4\"),\n",
+    "    \"Float32LE\": np.dtype(\"<f4\"),\n",
+    "    \"Float32BE\": np.dtype(\">f4\"),\n",
+    "    \"Float64LE\": np.dtype(\"<f8\"),\n",
+    "    \"Float64BE\": np.dtype(\">f8\"),\n",
+    "    \"CFloat32LE\": np.dtype(\"<c8\"),\n",
+    "    \"CFloat32BE\": np.dtype(\">c8\"),\n",
+    "    \"CFloat64LE\": np.dtype(\"<c16\"),\n",
+    "    \"CFloat64BE\": np.dtype(\">c16\"),\n",
+    "}\n",
+    "\n",
+    "\n",
     "@workflow.define(outputs=[\"out_image\"])\n",
     "def ToyMedianThreshold(in_image: Nifti1) -> mrtrix3.ImageFormat:\n",
     "    \"\"\"A toy example workflow that\n",
@@ -457,23 +480,17 @@
     "    )\n",
     "\n",
     "    @python.define\n",
-    "    def SelectDataFile(in_file: mrtrix3.ImageHeader) -> mrtrix3.ImageDataFile:\n",
-    "        return in_file.data_file\n",
-    "\n",
-    "    select_data = workflow.add(SelectDataFile(in_file=input_conversion.out_file))\n",
-    "\n",
-    "    @python.define\n",
-    "    def Median(data_file: mrtrix3.ImageDataFile) -> float:\n",
-    "        data = np.load(data_file)\n",
+    "    def Median(mih: mrtrix3.ImageHeader) -> float:\n",
+    "        \"\"\"A bespoke function that reads the separate data file in the MRTrix3 image\n",
+    "        header format (i.e. .mih) and calculates the median value.\"\"\"\n",
+    "        dtype = MRTRIX2NUMPY_DTYPES[mih.metadata[\"datatype\"].strip()]\n",
+    "        data = np.frombuffer(Path.read_bytes(mih.data_file), dtype=dtype)\n",
     "        return np.median(data)\n",
     "\n",
-    "    median = workflow.add(Median(data_file=select_data.out))\n",
+    "    median = workflow.add(Median(mih=input_conversion.out_file))\n",
     "    threshold = workflow.add(\n",
-    "        MrThreshold(\n",
-    "            in_file=in_image,\n",
-    "            abs=median.out\n",
-    "        ),                     \n",
-    "        environment=Docker(\"mrtrix3/mrtrix3\", tag=\"\")\n",
+    "        MrThreshold(in_file=in_image, out_file=\"binary.mif\", abs=median.out),\n",
+    "        environment=Docker(\"mrtrix3/mrtrix3\", tag=\"latest\"),\n",
     "    )\n",
     "\n",
     "    output_conversion = workflow.add(\n",

new-docs/source/tutorial/tst.py

@@ -1,11 +1,33 @@
 import tempfile
+from pathlib import Path
 import numpy as np
 from fileformats.medimage import Nifti1
 import fileformats.medimage_mrtrix3 as mrtrix3
 from pydra.engine.environments import Docker
 from pydra.design import workflow, python
 from pydra.tasks.mrtrix3.v3_0 import MrConvert, MrThreshold
 
+MRTRIX2NUMPY_DTYPES = {
+    "Int8": np.dtype("i1"),
+    "UInt8": np.dtype("u1"),
+    "Int16LE": np.dtype("<i2"),
+    "Int16BE": np.dtype(">i2"),
+    "UInt16LE": np.dtype("<u2"),
+    "UInt16BE": np.dtype(">u2"),
+    "Int32LE": np.dtype("<i4"),
+    "Int32BE": np.dtype(">i4"),
+    "UInt32LE": np.dtype("<u4"),
+    "UInt32BE": np.dtype(">u4"),
+    "Float32LE": np.dtype("<f4"),
+    "Float32BE": np.dtype(">f4"),
+    "Float64LE": np.dtype("<f8"),
+    "Float64BE": np.dtype(">f8"),
+    "CFloat32LE": np.dtype("<c8"),
+    "CFloat32BE": np.dtype(">c8"),
+    "CFloat64LE": np.dtype("<c16"),
+    "CFloat64BE": np.dtype(">c16"),
+}
+
 
 @workflow.define(outputs=["out_image"])
 def ToyMedianThreshold(in_image: Nifti1) -> mrtrix3.ImageFormat:
@@ -22,20 +44,17 @@ def ToyMedianThreshold(in_image: Nifti1) -> mrtrix3.ImageFormat:
     )
 
     @python.define
-    def SelectDataFile(in_file: mrtrix3.ImageHeader) -> mrtrix3.ImageDataFile:
-        return in_file.data_file
-
-    select_data = workflow.add(SelectDataFile(in_file=input_conversion.out_file))
-
-    @python.define
-    def Median(data_file: mrtrix3.ImageDataFile) -> float:
-        data = np.load(data_file)
+    def Median(mih: mrtrix3.ImageHeader) -> float:
+        """A bespoke function that reads the separate data file in the MRTrix3 image
+        header format (i.e. .mih) and calculates the median value."""
+        dtype = MRTRIX2NUMPY_DTYPES[mih.metadata["datatype"].strip()]
+        data = np.frombuffer(Path.read_bytes(mih.data_file), dtype=dtype)
         return np.median(data)
 
-    median = workflow.add(Median(data_file=select_data.out))
+    median = workflow.add(Median(mih=input_conversion.out_file))
     threshold = workflow.add(
-        MrThreshold(in_file=in_image, abs=median.out),
-        environment=Docker("mrtrix3/mrtrix3", tag=""),
+        MrThreshold(in_file=in_image, out_file="binary.mif", abs=median.out),
+        environment=Docker("mrtrix3/mrtrix3", tag="latest"),
     )
 
     output_conversion = workflow.add(

pydra/engine/environments.py

@@ -94,7 +94,7 @@ class Docker(Container):
     def execute(self, task: "Task[ShellDef]") -> dict[str, ty.Any]:
         docker_img = f"{self.image}:{self.tag}"
         # mounting all input locations
-        mounts, inputs_mod_root = task.definition._get_bindings(root=self.root)
+        mounts, input_updates = task.definition._get_bindings(root=self.root)
 
         docker_args = [
             "docker",
@@ -112,7 +112,11 @@ def execute(self, task: "Task[ShellDef]") -> dict[str, ty.Any]:
         keys = ["return_code", "stdout", "stderr"]
 
         values = execute(
-            docker_args + [docker_img] + task.definition._command_args(root=self.root),
+            docker_args
+            + [docker_img]
+            + task.definition._command_args(
+                root=self.root, input_updates=input_updates
+            ),
         )
         output = dict(zip(keys, values))
         if output["return_code"]:
@@ -129,7 +133,7 @@ class Singularity(Container):
     def execute(self, task: "Task[ShellDef]") -> dict[str, ty.Any]:
         singularity_img = f"{self.image}:{self.tag}"
         # mounting all input locations
-        mounts, inputs_mod_root = task.definition._get_bindings(root=self.root)
+        mounts, input_updates = task.definition._get_bindings(root=self.root)
 
         # todo adding xargsy etc
         singularity_args = [
@@ -150,7 +154,9 @@ def execute(self, task: "Task[ShellDef]") -> dict[str, ty.Any]:
         values = execute(
             singularity_args
             + [singularity_img]
-            + task.definition._command_args(root=self.root),
+            + task.definition._command_args(
+                root=self.root, input_updates=input_updates
+            ),
         )
         output = dict(zip(keys, values))
         if output["return_code"]:

pydra/engine/specs.py

@@ -16,7 +16,7 @@
 import attrs
 from attrs.converters import default_if_none
 import cloudpickle as cp
-from fileformats.core import FileSet
+from fileformats.generic import FileSet
 from pydra.utils.messenger import AuditFlag, Messenger
 from pydra.utils.typing import TypeParser, is_optional, non_optional_type
 from .helpers import (
@@ -1224,7 +1224,7 @@ def _get_bindings(
           Mapping from paths in the host environment to the target environment
         """
         bindings: dict[str, tuple[str, str]] = {}
-        inputs_mod_root: dict[str, tuple[Path, ...]] = {}
+        input_updates: dict[str, tuple[Path, ...]] = {}
         if root is None:
             return bindings
         fld: Arg
@@ -1233,10 +1233,10 @@ def _get_bindings(
                 fileset: FileSet | None = self[fld.name]
                 if fileset is None:
                     continue
-                if not isinstance(fileset, FileSet):
+                if not isinstance(fileset, (os.PathLike, FileSet)):
                     raise NotImplementedError(
-                        "Generating environment bindings for nested FileSets are not "
-                        "yet supported"
+                        "Generating environment bindings for nested FileSets is not "
+                        "supported yet"
                     )
                 copy = fld.copy_mode == FileSet.CopyMode.copy
 
@@ -1245,11 +1245,17 @@ def _get_bindings(
                 # Default to mounting paths as read-only, but respect existing modes
                 bindings[host_path] = (env_path, "rw" if copy else "ro")
 
-                # Provide in-container paths without type-checking
-                inputs_mod_root[fld.name] = tuple(
-                    env_path / rel for rel in fileset.relative_fspaths
+                # Provide updated in-container paths to the command to be run. If a
+                # fs-object, which resolves to a single path, just pass in the name of
+                # that path relative to the location in the mount point in the container.
+                # If it is a more complex file-set with multiple paths, then it is converted
+                # into a tuple of paths relative to the base of the fileset.
+                input_updates[fld.name] = (
+                    env_path / fileset.name
+                    if isinstance(fileset, os.PathLike)
+                    else tuple(env_path / rel for rel in fileset.relative_fspaths)
                 )
-        return bindings, inputs_mod_root
+        return bindings, input_updates
 
     def _generated_output_names(self, stdout: str, stderr: str):
         """Returns a list of all outputs that will be generated by the task.

pydra/utils/typing.py

@@ -261,6 +261,11 @@ def __call__(self, obj: ty.Any) -> T:
             try:
                 coerced = self.coerce(obj)
             except TypeError as e:
+                if obj is None:
+                    raise TypeError(
+                        f"Mandatory field{self.label_str} of type {self.tp} was not "
+                        "provided a value (i.e. a value that wasn't None) "
+                    ) from None
                 raise TypeError(
                     f"Incorrect type for field{self.label_str}: {obj!r} is not of type "
                     f"{self.tp} (and cannot be coerced to it)"