fix ict->clt conversion for directory (PolusAI#290)

* fix cwl_inline_runtag(name) and post_compile functions

* fix ict->clt io_type when specified directory

---------

Co-authored-by: Vasu Jaganath <[email protected]>

Author: vjaganat90

src/sophios/api/http/restapi.py

@@ -101,7 +101,7 @@ async def compile_wf(request: Request) -> Json:
     # schema non-preserving
     workflow_temp = converter.wfb_to_wic(wfb_payload)
     wkflw_name = "generic_workflow"
-    args = get_args(wkflw_name, ['--inline_cwl_runtag', '--generate_cwl_workflow'])
+    args = get_args(wkflw_name, ['--cwl_inline_runtag', '--generate_cwl_workflow'])
 
     # Build canonical workflow object
     workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp)
@@ -134,7 +134,7 @@ async def compile_wf(request: Request) -> Json:
 
     rose_tree = compiler_info.rose
     input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
-    cwl_inline_runtag(args, rose_tree)
+    rose_tree = cwl_inline_runtag(args, rose_tree)
     # ======== OUTPUT PROCESSING ================
     # ========= PROCESS COMPILED OBJECT =========
     sub_node_data: NodeData = rose_tree.data

src/sophios/api/pythonapi.py

@@ -757,7 +757,7 @@ def run(self) -> None:
         rose_tree: RoseTree = compiler_info.rose
 
         post_compile.cwl_docker_extract(args, self.process_name)
-        post_compile.remove_entrypoints(args, rose_tree)
+        rose_tree = post_compile.remove_entrypoints(args, rose_tree)
 
         # Do NOT capture stdout and/or stderr and pipe warnings and errors into a black hole.
         retval = run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True)

src/sophios/api/utils/ict/ict_spec/io/objects.py

@@ -128,6 +128,12 @@ def _output_to_cwl(self, inputs: Any) -> dict:
                     == "file"  # pylint: disable=unsubscriptable-object
                 ):
                     cwl_type = "File"
+                elif (
+                    isinstance(self.io_format, list)
+                    and len(self.io_format) == 1
+                    and self.io_format[0].lower() == 'directory'
+                ):
+                    cwl_type = "Directory"
                 else:
                     cwl_type = "File"
 

src/sophios/cli.py

@@ -39,7 +39,7 @@
                     (You should only disable provenance if absolutely necessary.)''')
 parser.add_argument('--copy_output_files', default=False, action="store_true",
                     help='Copies output files from the cachedir to outdir/ (automatically enabled with --run_local)')
-parser.add_argument('--inline_cwl_runtag', default=False, action="store_true",
+parser.add_argument('--cwl_inline_runtag', default=False, action="store_true",
                     help='Copies cwl adapter file contents inline into the final .cwl in autogenerated/')
 # This is a hidden flag
 parser.add_argument('--ignore_dir_path', type=bool,

src/sophios/main.py

@@ -171,7 +171,7 @@ def main() -> None:
 
     io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
-    pc.cwl_inline_runtag(args, rose_tree)
+    rose_tree = pc.cwl_inline_runtag(args, rose_tree)
     io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
     if args.graphviz:
@@ -194,7 +194,8 @@ def main() -> None:
 
     if args.run_local or args.generate_run_script:
         pc.cwl_docker_extract(args, yaml_stem)
-        pc.remove_entrypoints(args, rose_tree)
+        rose_tree = pc.remove_entrypoints(args, rose_tree)
+        io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
         run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False)
 

src/sophios/plugins.py

@@ -279,13 +279,13 @@ def cwl_update_inline_runtag_rosetree(rose_tree: RoseTree, path: Path, relative_
     """
     n_d: NodeData = rose_tree.data
     if n_d.compiled_cwl['class'] == 'Workflow':
-        outputs_inline_cwl_runtag = cwl_update_inline_runtag(n_d.compiled_cwl, path, relative_run_path)
+        outputs_cwl_inline_runtag = cwl_update_inline_runtag(n_d.compiled_cwl, path, relative_run_path)
     else:
-        outputs_inline_cwl_runtag = n_d.compiled_cwl
+        outputs_cwl_inline_runtag = n_d.compiled_cwl
 
     sub_trees_path = [cwl_update_inline_runtag_rosetree(sub_rose_tree, path, relative_run_path) for
                       sub_rose_tree in rose_tree.sub_trees]
-    node_data_path = NodeData(n_d.namespaces, n_d.name, n_d.yml, outputs_inline_cwl_runtag, n_d.tool,
+    node_data_path = NodeData(n_d.namespaces, n_d.name, n_d.yml, outputs_cwl_inline_runtag, n_d.tool,
                               n_d.workflow_inputs_file, n_d.explicit_edge_defs, n_d.explicit_edge_calls,
                               n_d.graph, n_d.inputs_workflow, n_d.step_name_1)
     return RoseTree(node_data_path, sub_trees_path)

src/sophios/post_compile.py

@@ -3,7 +3,6 @@
 import subprocess as sub
 
 from . import plugins
-from . import input_output as io
 from .wic_types import RoseTree
 
 
@@ -22,15 +21,16 @@ def cwl_docker_extract(args: argparse.Namespace, file_name: str) -> None:
     sub.run(cmd, check=True)
 
 
-def cwl_inline_runtag(args: argparse.Namespace, rose_tree: RoseTree) -> None:
+def cwl_inline_runtag(args: argparse.Namespace, rose_tree: RoseTree) -> RoseTree:
     """Transform with cwl inline runtag"""
     # this has to happen after at least one write
     # so we can copy from local cwl_dapters in autogenerated/
-    if args.inline_cwl_runtag:
+    if args.cwl_inline_runtag:
         rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True)
+    return rose_tree
 
 
-def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> None:
+def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> RoseTree:
     """Remove entry points"""
     if args.docker_remove_entrypoints:
         # Requires root, so guard behind CLI option
@@ -40,4 +40,4 @@ def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> None:
             plugins.remove_entrypoints_podman()
 
         rose_tree = plugins.dockerPull_append_noentrypoint_rosetree(rose_tree)
-        io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
+    return rose_tree

tests/test_examples.py

@@ -22,7 +22,7 @@
 from sophios import auto_gen_header
 from sophios.cli import get_args
 from sophios.utils_yaml import wic_loader
-from sophios.post_compile import cwl_docker_extract
+from sophios.post_compile import cwl_docker_extract, remove_entrypoints
 from sophios.wic_types import NodeData, StepId, Yaml, YamlTree, Json
 from sophios.utils_graphs import get_graph_reps
 
@@ -216,15 +216,8 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp
         cwl_docker_extract(args, Path(yml_path).stem)
         return
 
-    if args.docker_remove_entrypoints:
-        # Requires root, so guard behind CLI option
-        if args.container_engine == 'docker':
-            sophios.plugins.remove_entrypoints_docker()
-        if args.container_engine == 'podman':
-            sophios.plugins.remove_entrypoints_podman()
-
-        rose_tree = sophios.plugins.dockerPull_append_noentrypoint_rosetree(rose_tree)
-        sophios.input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
+    rose_tree = remove_entrypoints(args, rose_tree)
+    sophios.input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
     if args.partial_failure_enable:
         rose_tree = sophios.plugins.cwl_update_outputs_optional_rosetree(rose_tree)