Merge pull request #289 from djarecka/mnt/input_spec

extending command_args to include more flexible argstr, and sep

Author: djarecka

pydra/engine/helpers.py

@@ -598,3 +598,41 @@ def load_task(task_pkl, ind=None):
         task.inputs = attr.evolve(task.inputs, **inputs_dict)
         task.state = None
     return task
+
+
+def position_adjustment(pos_args):
+    """
+    sorting elements with the first element - position,
+    the negative positions should go to the end of the list
+    everything that has no position (i.e. it's None),
+    should go between elements with positive positions an with negative pos.
+    Returns a list of sorted args.
+    """
+    # sorting all elements of the command
+    try:
+        pos_args.sort()
+    except TypeError:  # if some positions are None
+        pos_args_none = []
+        pos_args_int = []
+        for el in pos_args:
+            if el[0] is None:
+                pos_args_none.append(el)
+            else:
+                pos_args_int.append(el)
+            pos_args_int.sort()
+        last_el = pos_args_int[-1][0]
+        for el_none in pos_args_none:
+            last_el += 1
+            pos_args_int.append((last_el, el_none[1]))
+        pos_args = pos_args_int
+
+    # if args available, they should be moved at the of the list
+    while pos_args[0][0] < 0:
+        pos_args.append(pos_args.pop(0))
+
+    # dropping the position index
+    cmd_args = []
+    for el in pos_args:
+        cmd_args += el[1]
+
+    return cmd_args

pydra/engine/helpers_file.py

@@ -10,6 +10,7 @@
 from builtins import str, bytes, open
 import logging
 from pathlib import Path
+import typing as ty
 
 related_filetype_sets = [(".hdr", ".img", ".mat"), (".nii", ".mat"), (".BRIK", ".HEAD")]
 """List of neuroimaging file types that are to be interpreted together."""
@@ -512,23 +513,54 @@ def template_update(inputs, map_copyfiles=None):
 
     from .specs import attr_fields
 
-    fields = attr_fields(inputs)
-    # TODO: Create a dependency graph first and then traverse it
-    for fld in fields:
-        if getattr(inputs, fld.name) is not attr.NOTHING:
-            continue
-        if fld.metadata.get("output_file_template"):
-            if fld.type is str:
-                value = fld.metadata["output_file_template"].format(**dict_)
-                dict_[fld.name] = str(value)
-            else:
-                raise Exception(
-                    f"output_file_template metadata for "
-                    "{fld.name} should be a string"
-                )
+    fields_templ = [
+        fld for fld in attr_fields(inputs) if fld.metadata.get("output_file_template")
+    ]
+    for fld in fields_templ:
+        if fld.type not in [str, ty.Union[str, bool]]:
+            raise Exception(
+                f"fields with output_file_template"
+                "has to be a string or Union[str, bool]"
+            )
+        inp_val_set = getattr(inputs, fld.name)
+        if inp_val_set is not attr.NOTHING and not isinstance(inp_val_set, (str, bool)):
+            raise Exception(f"{fld.name} has to be str or bool, but {inp_val_set} set")
+        if isinstance(inp_val_set, bool) and fld.type is str:
+            raise Exception(
+                f"type of {fld.name} is str, consider using Union[str, bool]"
+            )
+
+        if isinstance(inp_val_set, str):
+            dict_[fld.name] = inp_val_set
+        elif inp_val_set is False:
+            # if False, the field should not be used, so setting attr.NOTHING
+            dict_[fld.name] = attr.NOTHING
+        else:  # True or attr.NOTHING
+            template = fld.metadata["output_file_template"]
+            value = template.format(**dict_)
+            value = removing_nothing(value)
+            dict_[fld.name] = value
     return {k: v for k, v in dict_.items() if getattr(inputs, k) is not v}
 
 
+def removing_nothing(template_str):
+    """ removing all fields that had NOTHING"""
+    if "NOTHING" not in template_str:
+        return template_str
+    regex = re.compile("[^a-zA-Z_\-]")
+    fields_str = regex.sub(" ", template_str)
+    for fld in fields_str.split():
+        if "NOTHING" in fld:
+            template_str = template_str.replace(fld, "")
+    return (
+        template_str.replace("[ ", "[")
+        .replace(" ]", "]")
+        .replace(",]", "]")
+        .replace("[,", "[")
+        .strip()
+    )
+
+
 def is_local_file(f):
     from .specs import File
 

pydra/engine/specs.py

@@ -207,12 +207,14 @@ def check_metadata(self):
             "copyfile",
             "help_string",
             "mandatory",
+            "readonly",
             "output_field_name",
             "output_file_template",
             "position",
             "requires",
             "separate_ext",
             "xor",
+            "sep",
         }
         # special inputs, don't have to follow rules for standard inputs
         special_input = ["_func", "_graph_checksums"]
@@ -222,22 +224,23 @@ def check_metadata(self):
             mdata = fld.metadata
             # checking keys from metadata
             if set(mdata.keys()) - supported_keys:
-                raise Exception(
+                raise AttributeError(
                     f"only these keys are supported {supported_keys}, but "
                     f"{set(mdata.keys()) - supported_keys} provided"
                 )
             # checking if the help string is provided (required field)
             if "help_string" not in mdata:
-                raise Exception(f"{fld.name} doesn't have help_string field")
-
+                raise AttributeError(f"{fld.name} doesn't have help_string field")
             # assuming that fields with output_file_template shouldn't have default
-            if not fld.default == attr.NOTHING and mdata.get("output_file_template"):
-                raise Exception(
+            if fld.default not in [attr.NOTHING, True, False] and mdata.get(
+                "output_file_template"
+            ):
+                raise AttributeError(
                     "default value should not be set together with output_file_template"
                 )
             # not allowing for default if the field is mandatory
             if not fld.default == attr.NOTHING and mdata.get("mandatory"):
-                raise Exception(
+                raise AttributeError(
                     "default value should not be set when the field is mandatory"
                 )
             # setting default if value not provided and default is available
@@ -260,15 +263,17 @@ def check_fields_input_spec(self):
             # checking if the mandatory field is provided
             if getattr(self, fld.name) is attr.NOTHING:
                 if mdata.get("mandatory"):
-                    raise Exception(f"{fld.name} is mandatory, but no value provided")
+                    raise AttributeError(
+                        f"{fld.name} is mandatory, but no value provided"
+                    )
                 else:
                     continue
             names.append(fld.name)
 
             # checking if fields meet the xor and requires are
             if "xor" in mdata:
                 if [el for el in mdata["xor"] if el in names]:
-                    raise Exception(
+                    raise AttributeError(
                         f"{fld.name} is mutually exclusive with {mdata['xor']}"
                     )
 
@@ -283,15 +288,15 @@ def check_fields_input_spec(self):
         for nm, required in require_to_check.items():
             required_notfound = [el for el in required if el not in names]
             if required_notfound:
-                raise Exception(f"{nm} requires {required_notfound}")
+                raise AttributeError(f"{nm} requires {required_notfound}")
 
             # TODO: types might be checked here
         self._type_checking()
 
     def _file_check(self, field):
         file = Path(getattr(self, field.name))
         if not file.exists():
-            raise Exception(f"the file from the {field.name} input does not exist")
+            raise AttributeError(f"the file from the {field.name} input does not exist")
 
     def _type_checking(self):
         """Use fld.type to check the types TODO.
@@ -327,7 +332,9 @@ def collect_additional_outputs(self, input_spec, inputs, output_dir):
                     if (
                         fld.default is None or fld.default == attr.NOTHING
                     ) and not fld.metadata:  # TODO: is it right?
-                        raise Exception("File has to have default value or metadata")
+                        raise AttributeError(
+                            "File has to have default value or metadata"
+                        )
                     elif not fld.default == attr.NOTHING:
                         additional_out[fld.name] = self._field_defaultvalue(
                             fld, output_dir
@@ -343,7 +350,7 @@ def collect_additional_outputs(self, input_spec, inputs, output_dir):
     def _field_defaultvalue(self, fld, output_dir):
         """Collect output file if the default value specified."""
         if not isinstance(fld.default, (str, Path)):
-            raise Exception(
+            raise AttributeError(
                 f"{fld.name} is a File, so default value "
                 f"should be a string or a Path, "
                 f"{fld.default} provided"
@@ -358,15 +365,15 @@ def _field_defaultvalue(self, fld, output_dir):
             if default.exists():
                 return default
             else:
-                raise Exception(f"file {default} does not exist")
+                raise AttributeError(f"file {default} does not exist")
         else:
             all_files = list(Path(default.parent).expanduser().glob(default.name))
             if len(all_files) > 1:
                 return all_files
             elif len(all_files) == 1:
                 return all_files[0]
             else:
-                raise Exception(f"no file matches {default.name}")
+                raise AttributeError(f"no file matches {default.name}")
 
     def _field_metadata(self, fld, inputs, output_dir):
         """Collect output file if metadata specified."""