add mechanism for passthrough flags for Toil (PolusAI#323)

Co-authored-by: Vasu Jaganath <[email protected]>

Author: vjaganat90

src/sophios/api/pythonapi.py

@@ -13,7 +13,7 @@
 
 from sophios import compiler, input_output, plugins, utils_cwl, post_compile
 from sophios import run_local as run_local_module
-from sophios.cli import get_args
+from sophios.cli import get_args, get_known_and_unknown_args
 from sophios.utils_graphs import get_graph_reps
 from sophios.wic_types import CompilerInfo, RoseTree, StepId, Tool, Tools, YamlTree, Json
 
@@ -785,15 +785,19 @@ def run(self) -> None:
         logger.info(f"Running {self.process_name}")
         plugins.logging_filters()
         compiler_info = self.compile(write_to_disk=True)
-        args = get_args(self.process_name, self.user_args)  # Use mock CLI args
+        args, unknown_args = get_known_and_unknown_args(self.process_name, self.user_args)  # Use mock CLI args
         rose_tree: RoseTree = compiler_info.rose
 
         post_compile.cwl_docker_extract(args.container_engine, args.pull_dir, self.process_name)
         if args.docker_remove_entrypoints:
             rose_tree = post_compile.remove_entrypoints(args.container_engine, rose_tree)
         post_compile.find_and_create_output_dirs(rose_tree)
         # Do NOT capture stdout and/or stderr and pipe warnings and errors into a black hole.
-        retval = run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True)
+        if args.toil_passthrough_flags == 'yes':
+            run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner,
+                                       True, passthrough_args=unknown_args)
+        else:
+            run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True)
 
         # Finally, since there is an output file copying bug in cwltool,
         # we need to copy the output files manually. See comment above.

src/sophios/cli.py

@@ -1,6 +1,7 @@
 import argparse
 import sys
 from pathlib import Path
+from typing import List, Tuple
 from unittest.mock import patch
 
 from . import _version
@@ -124,6 +125,11 @@
                     help='Changees the color of the fonts and edges from white to black.')
 parser.add_argument('--custom_net', type=str, required=False,
                     help='Passes --custom-net flag to cwltool.')
+parser.add_argument('--toil_passthrough_flags', type=str, default='no', required=False,
+                    help='''Indicates that the user is passing flags to the Toil backend.
+                    No checks are done on the flags or values passed. User must verify that they are sending correct flags.
+                    only two valid values of this flag 'yes' or 'no'.
+                    If set to 'no' (default) passthrough flags won't be sent to the Toil backend.''')
 
 
 def get_args(yaml_path: str = '', suppliedargs: list[str] = []) -> argparse.Namespace:
@@ -137,3 +143,16 @@ def get_args(yaml_path: str = '', suppliedargs: list[str] = []) -> argparse.Name
     with patch.object(sys, 'argv', testargs):
         args = parser.parse_args()
     return args
+
+
+def get_known_and_unknown_args(yaml_path: str = '', suppliedargs: list[str] = []) -> Tuple[argparse.Namespace, List[str]]:
+    """This is used to get mock command line arguments, default + suppled args
+
+    Returns:
+        argparse.Namespace: The mocked command line arguments
+    """
+    defaultargs = ['sophios', '--yaml', yaml_path]  # ignore --yaml
+    testargs = defaultargs + suppliedargs
+    with patch.object(sys, 'argv', testargs):
+        known_args, unknown_args = parser.parse_known_args()
+    return known_args, unknown_args

src/sophios/main.py

@@ -19,7 +19,7 @@
 
 def main() -> None:
     """See docs/userguide.md"""
-    args = cli.parser.parse_args()
+    args, unknown_args = cli.parser.parse_known_args()
     plugins.logging_filters()
 
     # User may specify a different homedir
@@ -199,7 +199,10 @@ def main() -> None:
             rose_tree = pc.remove_entrypoints(args.container_engine, rose_tree)
         io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
         pc.find_and_create_output_dirs(rose_tree)
-        run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False)
+        if args.toil_passthrough_flags == 'yes':
+            run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False, passthrough_args=unknown_args)
+        else:
+            run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False)
 
         # Finally, since there is an output file copying bug in cwltool,
         # we need to copy the output files manually. See comment above.

src/sophios/run_local.py

@@ -50,7 +50,7 @@ def generate_run_script(cmdline: str) -> None:
 
 
 def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[str], cwl_runner: str,
-              use_subprocess: bool) -> int:
+              use_subprocess: bool, passthrough_args: List[str] = []) -> int:
     """This function runs the compiled workflow locally.
 
     Args:
@@ -250,9 +250,8 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[
         date_time = now.strftime("%Y%m%d%H%M%S")
         cmd += ['--outdir', f'outdir_toil_{yaml_stem}_{date_time}',
                 '--jobStore', f'file:./jobStore_{yaml_stem}',  # NOTE: This is the equivalent of --cachedir
-                # TODO: Check --clean, --cleanWorkDir, --restart
-                '--clean', 'always',  # This effectively disables caching, but is reproducible
                 f'autogenerated/{yaml_stem}.cwl', f'autogenerated/{yaml_stem}_inputs.yml']
+        cmd += passthrough_args
         cmdline = ' '.join(cmd)
 
         if args.generate_run_script: