debugging interface converter

Author: tclose

nipype2pydra/cli/convert.py

@@ -87,9 +87,9 @@ def convert(
             spec = yaml.safe_load(f)
         converter.add_interface_from_spec(
             spec=spec,
-            callables_file=(
-                fspath.parent / (fspath.name[: -len(".yaml")] + "_callables.py")
-            ),
+            # callables_file=(
+            #     fspath.parent / (fspath.name[: -len(".yaml")] + "_callables.py")
+            # ),
         )
 
     # Load workflow specs

nipype2pydra/cli/pkg_gen.py

@@ -185,10 +185,10 @@ def pkg_gen(
                 with open(interfaces_spec_dir / (spec_name + ".yaml"), "w") as f:
                     f.write(yaml_spec)
 
-                callables_fspath = interfaces_spec_dir / f"{spec_name}_callables.py"
+                # callables_fspath = interfaces_spec_dir / f"{spec_name}_callables.py"
 
-                with open(callables_fspath, "w") as f:
-                    f.write(parsed.generate_callables(nipype_interface))
+                # with open(callables_fspath, "w") as f:
+                #     f.write(parsed.generate_callables(nipype_interface))
 
         if "functions" in spec:
             functions_spec_dir = spec_dir / "functions"

nipype2pydra/helpers.py

@@ -93,8 +93,10 @@ class BaseHelperConverter:
         factory=list,
         converter=from_list_to_imports,
         metadata={
-            "help": """list import statements required by the test, with each list item
-                consisting of 'module', 'name', and optionally 'alias' keys"""
+            "help": (
+                "list import statements required by the test, with each list item"
+                "consisting of 'module', 'name', and optionally 'alias' keys"
+            )
         },
     )
     package: "nipype2pydra.package.PackageConverter" = attrs.field(

nipype2pydra/interface/base.py

@@ -339,10 +339,10 @@ class BaseInterfaceConverter(metaclass=ABCMeta):
         factory=OutputsConverter,
         converter=from_dict_to_outputs,
     )
-    callables_module: ModuleType = attrs.field(
-        converter=import_module_from_path,
-        default=None,
-    )
+    # callables_module: ModuleType = attrs.field(
+    #     converter=import_module_from_path,
+    #     default=None,
+    # )
     tests: ty.List[TestGenerator] = attrs.field(  # type: ignore
         factory=list, converter=from_list_to_tests
     )
@@ -884,7 +884,7 @@ def _converted_test(self):
                                 else:
                                     value = attrs.NOTHING
                     if value is not attrs.NOTHING:
-                        spec_str += f"    task.inputs.{nm} = {value}\n"
+                        spec_str += f"    task.{nm} = {value}\n"
             if hasattr(self.nipype_interface, "_cmd"):
                 spec_str += r'    print(f"CMDLINE: {task.cmdline}\n\n")' + "\n"
             spec_str += "    res = task(worker=PassAfterTimeoutWorker)\n"

nipype2pydra/interface/shell.py

@@ -22,7 +22,13 @@
 
 logger = logging.getLogger("nipype2pydra")
 
-CALLABLES_ARGS = ["inputs", "stdout", "stderr", "output_dir"]
+OUT_FUNC_ARGS = ["callable", "formatter"]  # arguments to shell.out that are functions
+CALLABLE_ARGS = [
+    "inputs",
+    "stdout",
+    "stderr",
+    "output_dir",
+]  # Arguments for callable methods
 
 
 @attrs.define(slots=False)
@@ -93,13 +99,26 @@ def generate_code(self, input_fields, nonstd_types, output_fields) -> str:
 
         # Pull out xor fields into task-level xor_sets
         xor_sets = set()
+        has_zero_pos = False
         for inpt in input_fields:
             if len(inpt) == 3:
                 name, _, mdata = inpt
             else:
                 name, _, __, mdata = inpt
             if "xor" in mdata:
-                xor_sets.add(frozenset(mdata["xor"] + [name]))
+                xor_sets.add(frozenset(list(mdata["xor"]) + [name]))
+            if mdata.get("position", None) == 0:
+                has_zero_pos = True
+
+        # Increment positions if there is a zero position
+        if has_zero_pos:
+            for inpt in input_fields:
+                if len(inpt) == 3:
+                    name, _, mdata = inpt
+                else:
+                    name, _, __, mdata = inpt
+                if "position" in mdata and mdata["position"] >= 0:
+                    mdata["position"] = mdata.pop("position") + 1
 
         input_fields_str = ""
         output_fields_str = ""
@@ -126,26 +145,24 @@ def generate_code(self, input_fields, nonstd_types, output_fields) -> str:
             mdata.pop("xor", None)
             args_str = ", ".join(f"{k}={v!r}" for k, v in mdata.items())
             if "path_template" in mdata:
-                output_fields_str = (
+                output_fields_str += (
                     f"        {name}: {type_str} = shell.outarg({args_str})\n"
                 )
             else:
                 input_fields_str += f"    {name}: {type_str} = shell.arg({args_str})\n"
 
-        callable_fields = set(n for n, _, __ in self.callable_output_fields)
+        # callable_fields = set(n for n, _, __ in self.callable_output_fields)
 
         for outpt in output_fields:
             name, type_, mdata = outpt
-            cllble = mdata.pop(
-                "callable", f"{name}_callable" if name in callable_fields else None
-            )
-            args_str = ", ".join(f"{k}={v!r}" for k, v in mdata.items())
-            if args_str:
-                args_str += ", "
-            if cllble:
-                args_str += f"callable={cllble}"
+            func_args = []
+            for func_arg in OUT_FUNC_ARGS:
+                if func_arg in mdata:
+                    func_args.append(f"{func_arg}={mdata[func_arg]}")
+                    mdata.pop(func_arg)
+            args = [f"{k}={v!r}" for k, v in mdata.items()] + func_args
             output_fields_str += (
-                f"        {name}: {type_to_str(type_)} = shell.out({args_str})\n"
+                f"        {name}: {type_to_str(type_)} = shell.out({', '.join(args)})\n"
             )
 
         spec_str = (
@@ -182,7 +199,7 @@ def generate_code(self, input_fields, nonstd_types, output_fields) -> str:
                 s[0] == self.nipype_interface._list_outputs
                 for s in self.used.method_stacks[m.__name__]
             ):
-                additional_args = CALLABLES_ARGS
+                additional_args = CALLABLE_ARGS
             else:
                 additional_args = []
             method_str = self.process_method(
@@ -418,7 +435,7 @@ def callables_code(self):
             if not agg_body.strip():
                 return ""
             agg_body = self.unwrap_nested_methods(
-                agg_body, additional_args=CALLABLES_ARGS, inputs_as_dict=True
+                agg_body, additional_args=CALLABLE_ARGS, inputs_as_dict=True
             )
             agg_body = self.replace_supers(
                 agg_body,
@@ -477,13 +494,15 @@ def callables_code(self):
                 )
                 lo_body = self._process_inputs(lo_body)
                 lo_body = re.sub(
-                    r"(\w+) = self\.output_spec\(\).(?:trait_)get\(\)", r"\1 = {}", lo_body
+                    r"(\w+) = self\.output_spec\(\).(?:trait_)get\(\)",
+                    r"\1 = {}",
+                    lo_body,
                 )
 
                 if not lo_body.strip():
                     return ""
                 lo_body = self.unwrap_nested_methods(
-                    lo_body, additional_args=CALLABLES_ARGS, inputs_as_dict=True
+                    lo_body, additional_args=CALLABLE_ARGS, inputs_as_dict=True
                 )
                 lo_body = self.replace_supers(
                     lo_body,

nipype2pydra/interface/tests/test_interface.py

@@ -13,21 +13,26 @@
     INBUILT_NIPYPE_TRAIT_NAMES,
 )
 from nipype2pydra.package import PackageConverter
+from nipype2pydra.symbols import clear_caches
 from conftest import EXAMPLE_INTERFACES_DIR
 
 
 logging.basicConfig(level=logging.INFO)
 
 
 XFAIL_INTERFACES = [
-    "fsl-prob_track_x2",
-    "fsl-flameo",
-    "fsl-make_dyadic_vectors",
-    "fsl-dual_regression",
-    "fsl-epi_de_warp",
+    "ants-interfaces-ai",
+    "ants-interfaces-measure_image_similarity",
+    "ants-interfaces-threshold_image",
+    "fsl-interfaces-prob_track_x2",
+    "fsl-interfaces-flameo",
+    "fsl-interfaces-make_dyadic_vectors",
+    "fsl-interfaces-dual_regression",
+    "fsl-interfaces-epi_de_warp",
 ]
 
 XFAIL_INTERFACES_IN_COMBINED = [
+    "ants-ai",
     "freesurfer-smooth",
     "freesurfer-apply_mask",
     "afni-merge",
@@ -42,7 +47,11 @@
 @pytest.fixture(
     params=[
         str(p.relative_to(EXAMPLE_INTERFACES_DIR)).replace("/", "-")[:-5]
-        for p in (EXAMPLE_INTERFACES_DIR).glob("**/*.yaml")
+        for p in (EXAMPLE_INTERFACES_DIR).glob("**/interfaces/*.yaml")
+        if (
+            str(p.relative_to(EXAMPLE_INTERFACES_DIR)).replace("/", "-")[:-5]
+            not in XFAIL_INTERFACES
+        )
     ]
 )
 def interface_spec_file(request):
@@ -54,6 +63,8 @@ def interface_spec_file(request):
 def test_interface_convert(
     interface_spec_file, cli_runner, work_dir, gen_test_conftest
 ):
+    # Clear UsedSymbol caches
+    clear_caches()
 
     try:
         with open(interface_spec_file) as f:
@@ -74,8 +85,8 @@ def test_interface_convert(
 
         converter = pkg_converter.add_interface_from_spec(
             spec=interface_spec,
-            callables_file=interface_spec_file.parent
-            / (interface_spec_file.stem + "_callables.py"),
+            # callables_file=interface_spec_file.parent
+            # / (interface_spec_file.stem + "_callables.py"),
         )
 
         converter.write(pkg_root)
@@ -89,7 +100,7 @@ def test_interface_convert(
                 nipype_ports.append(pkg_converter.nipype_port_converters[address])
         for _, func in converter.used.imported_funcs:
             if full_address(func) not in list(pkg_converter.workflows):
-                intra_pkg_modules[func.__module__].add(func)               
+                intra_pkg_modules[func.__module__].add(func)
 
         already_converted = set()
         for converter in tqdm(

nipype2pydra/package.py

@@ -507,6 +507,10 @@ def translate_submodule(
     def untranslate_submodule(self, pydra_module_name: str) -> str:
         """Translates a module name from the Nipype package to the Pydra package"""
         relpath = ImportStatement.get_relative_package(pydra_module_name, self.name)
+        if relpath.startswith(".auto"):
+            relpath = relpath[5:]
+        if relpath.startswith(".nipype_ports"):
+            return "nipype" + relpath[13:]
         if relpath == self.nipype_name:
             raise ValueError(
                 f"Module {pydra_module_name} is not in the nipype package {self.name}"
@@ -741,7 +745,9 @@ def nipype_port_converters(self) -> ty.Dict[str, interface.BaseInterfaceConverte
     )
 
     def add_interface_from_spec(
-        self, spec: ty.Dict[str, ty.Any], callables_file: Path
+        self,
+        spec: ty.Dict[str, ty.Any],
+        # callables_file: Path
     ) -> interface.BaseInterfaceConverter:
         output_module = self.translate_submodule(
             spec["nipype_module"], sub_pkg="auto" if self.interface_only else None
@@ -750,7 +756,7 @@ def add_interface_from_spec(
         converter = self.interfaces[f"{spec['nipype_module']}.{spec['task_name']}"] = (
             interface.get_converter(
                 output_module=output_module,
-                callables_module=callables_file,
+                # callables_module=callables_file,
                 package=self,
                 **spec,
             )

nipype2pydra/pkg_gen/__init__.py

@@ -446,6 +446,8 @@ def _gen_tests(
 
         for doctest_str in doctest_blocks:
             if ">>>" in doctest_str:
+
+                doctest_str = re.sub(r"\n\.\.\.\s+", " ", doctest_str)
                 try:
                     cmdline, inpts, directive, imports = extract_doctest_inputs(
                         doctest_str, self.name
@@ -641,7 +643,7 @@ def initialise_task_repo(
     """Copy the task template to the output directory and customise it for the given
     package name and return the created package directory"""
 
-    pkg_dir = output_dir / f"pydra-{pkg}"
+    pkg_dir = output_dir / f"pydra-tasks-{pkg}"
 
     def copy_ignore(_, names):
         return [n for n in names if n in (".git", "__pycache__", ".pytest_cache")]

nipype2pydra/symbols.py

@@ -199,6 +199,7 @@ def find(
             tuple(f.__name__ if not isinstance(f, str) else f for f in function_bodies),
             collapse_intra_pkg,
             pull_out_inline_imports,
+            tuple(package.all_import_translations),
         )
         try:
             return cls._cache[cache_key]
@@ -216,6 +217,16 @@ def find(
         )
         return used
 
+    @classmethod
+    def clear_cache(cls):
+        cls._cache = {}
+        cls._cache = {}
+        cls._stmts_cache = {}
+        cls._imports_cache = {}
+        cls._funcs_cache = {}
+        cls._classes_cache = {}
+        cls._constants_cache = {}
+
     @classmethod
     def _module_statements(cls, module) -> list:
         try:
@@ -667,6 +678,12 @@ class UsedClassSymbols(UsedSymbols):
 
     _class_attrs_cache = {}
 
+    @classmethod
+    def clear_cache(cls):
+        """Clear the cache for the class attributes"""
+        cls._class_attrs_cache = {}
+        super().clear_cache()
+
     @classmethod
     def find(
         cls,
@@ -721,6 +738,7 @@ def find(
             pull_out_inline_imports,
             absolute_imports,
             tuple(always_include),
+            tuple(package.all_import_translations),
         )
         try:
             return cls._cache[cache_key]
@@ -933,3 +951,8 @@ def local_class_attrs(cls, klass) -> ty.Dict[str, str]:
         class_attrs = dict(class_attrs)
         cls._constants_cache[cache_key] = class_attrs
         return class_attrs
+
+
+def clear_caches():
+    UsedSymbols.clear_cache()
+    UsedClassSymbols.clear_cache()