simplify args dependencies (PolusAI#318)

Co-authored-by: Vasu Jaganath <[email protected]>

Author: vjaganat90

src/sophios/api/http/restapi.py

@@ -94,7 +94,7 @@ async def compile_wf(request: Request) -> Json:
     print('---------- Compile Workflow! ---------')
     # ========= PROCESS REQUEST OBJECT ==========
     req: Json = await request.json()
-    suppliedargs = ['--cwl_inline_runtag', '--generate_cwl_workflow']
+    suppliedargs = ['--generate_cwl_workflow']
     # clean up and convert the incoming object
     # schema preserving
     req = converter.update_payload_missing_inputs_outputs(req)
@@ -138,8 +138,11 @@ async def compile_wf(request: Request) -> Json:
 
     rose_tree = compiler_info.rose
     input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
-    rose_tree = cwl_inline_runtag(args, rose_tree)
-    rose_tree = remove_entrypoints(args, rose_tree)
+    # generating cwl inline within the 'run' tag is post compile
+    # and always on when compiling and preparing REST return payload
+    rose_tree = cwl_inline_runtag(rose_tree)
+    if args.docker_remove_entrypoints:
+        rose_tree = remove_entrypoints(args.container_engine, rose_tree)
     # ======== OUTPUT PROCESSING ================
     # ========= PROCESS COMPILED OBJECT =========
     sub_node_data: NodeData = rose_tree.data

src/sophios/api/pythonapi.py

@@ -752,7 +752,6 @@ def get_cwl_workflow(self) -> Json:
             Json: Contains the compiled CWL and yaml inputs to the workflow.
         """
         compiler_info = self.compile(write_to_disk=False)
-        self.user_args.append('--cwl_inline_runtag')
         args = get_args(self.process_name, self.user_args)
         rose_tree = compiler_info.rose
 
@@ -761,7 +760,7 @@ def get_cwl_workflow(self) -> Json:
         # then remove this write_to_disk call here
         input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
-        rose_tree = post_compile.cwl_inline_runtag(args, rose_tree)
+        rose_tree = post_compile.cwl_inline_runtag(rose_tree)
         sub_node_data = rose_tree.data
         cwl_ast = sub_node_data.compiled_cwl
 
@@ -789,8 +788,9 @@ def run(self) -> None:
         args = get_args(self.process_name, self.user_args)  # Use mock CLI args
         rose_tree: RoseTree = compiler_info.rose
 
-        post_compile.cwl_docker_extract(args, self.process_name)
-        rose_tree = post_compile.remove_entrypoints(args, rose_tree)
+        post_compile.cwl_docker_extract(args.container_engine, args.pull_dir, self.process_name)
+        if args.docker_remove_entrypoints:
+            rose_tree = post_compile.remove_entrypoints(args.container_engine, rose_tree)
         post_compile.find_and_create_output_dirs(rose_tree)
         # Do NOT capture stdout and/or stderr and pipe warnings and errors into a black hole.
         retval = run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True)

src/sophios/cli.py

@@ -28,8 +28,8 @@
 parser.add_argument('--homedir', type=str, required=False, default=str(Path().home()),
                     help='''The users home directory.
                     This is necessary because CWL clears environment variables (e.g. HOME)''')
-parser.add_argument('--singularity_pull_dir', type=str, required=False, default=str(Path().cwd()),
-                    help='''The user specified pull directory for singularity image pull.
+parser.add_argument('--pull_dir', type=str, required=False, default=str(Path().cwd()),
+                    help='''The user specified pull directory for (singularity) image pull.
                     The default is the current working directory i.e. `pwd`''')
 parser.add_argument('--insert_steps_automatically', default=False, action="store_true",
                     help='''Attempt to fix inference failures by speculatively

src/sophios/main.py

@@ -171,7 +171,8 @@ def main() -> None:
 
     io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
-    rose_tree = pc.cwl_inline_runtag(args, rose_tree)
+    if args.cwl_inline_runtag:
+        rose_tree = pc.cwl_inline_runtag(rose_tree)
     io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
     if args.graphviz:
@@ -193,8 +194,9 @@ def main() -> None:
             print("but not the graphviz system package.)")
 
     if args.run_local or args.generate_run_script:
-        pc.cwl_docker_extract(args, yaml_stem)
-        rose_tree = pc.remove_entrypoints(args, rose_tree)
+        pc.cwl_docker_extract(args.container_engine, args.pull_dir, yaml_stem)
+        if args.docker_remove_entrypoints:
+            rose_tree = pc.remove_entrypoints(args.container_engine, rose_tree)
         io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
         pc.find_and_create_output_dirs(rose_tree)
         run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False)

src/sophios/post_compile.py

@@ -1,4 +1,3 @@
-import argparse
 from pathlib import Path
 import subprocess as sub
 from typing import Dict, Union
@@ -8,11 +7,11 @@
 
 def find_output_dirs(data: Union[RoseTree, Dict, list]) -> list:
     """
-    Recursively searches through a nested structure and finds all dictionaries 
+    Recursively searches through a nested structure and finds all dictionaries
     that contain the key 'location', and a key 'class' with a value of 'Directory'.
 
     Args:
-        data (any): The data to search through, which can be a dictionary, list, 
+        data (any): The data to search through, which can be a dictionary, list,
         or any other structure.
 
     Returns:
@@ -53,38 +52,33 @@ def find_and_create_output_dirs(rose_tree: RoseTree, basepath: str = 'autogenera
     create_output_dirs(output_dirs, basepath)
 
 
-def cwl_docker_extract(args: argparse.Namespace, file_name: str) -> None:
+def cwl_docker_extract(container_engine: str, pull_dir: str, file_name: str) -> None:
     """Helper function to do the cwl_docker_extract"""
     # cwl-docker-extract recursively `docker pull`s all images in all subworkflows.
     # This is important because cwltool only uses `docker run` when executing
     # workflows, and if there is a local image available,
     # `docker run` will NOT query the remote repository for the latest image!
     # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel.
-    if args.container_engine == 'singularity':
+    if container_engine == 'singularity':
         cmd = ['cwl-docker-extract', '-s', '--dir',
-               f'{args.singularity_pull_dir}', f'autogenerated/{file_name}.cwl']
+               f'{pull_dir}', f'autogenerated/{file_name}.cwl']
     else:
         cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{file_name}.cwl']
     sub.run(cmd, check=True)
 
 
-def cwl_inline_runtag(args: argparse.Namespace, rose_tree: RoseTree) -> RoseTree:
+def cwl_inline_runtag(rose_tree: RoseTree) -> RoseTree:
     """Transform with cwl inline runtag"""
     # this has to happen after at least one write
     # so we can copy from local cwl_dapters in autogenerated/
-    if args.cwl_inline_runtag:
-        rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True)
-    return rose_tree
+    return plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True)
 
 
-def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> RoseTree:
+def remove_entrypoints(container_engine: str, rose_tree: RoseTree) -> RoseTree:
     """Remove entry points"""
-    if args.docker_remove_entrypoints:
-        # Requires root, so guard behind CLI option
-        if args.container_engine == 'docker':
-            plugins.remove_entrypoints_docker()
-        if args.container_engine == 'podman':
-            plugins.remove_entrypoints_podman()
-
-        rose_tree = plugins.dockerPull_append_noentrypoint_rosetree(rose_tree)
-    return rose_tree
+    # Requires root, so guard behind CLI option
+    if container_engine == 'docker':
+        plugins.remove_entrypoints_docker()
+    if container_engine == 'podman':
+        plugins.remove_entrypoints_podman()
+    return plugins.dockerPull_append_noentrypoint_rosetree(rose_tree)

tests/test_examples.py

@@ -213,10 +213,11 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp
     sophios.input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
     if docker_pull_only:
-        cwl_docker_extract(args, Path(yml_path).stem)
+        cwl_docker_extract(args.container_engine, args.pull_dir, Path(yml_path).stem)
         return
 
-    rose_tree = remove_entrypoints(args, rose_tree)
+    if args.docker_remove_entrypoints:
+        rose_tree = remove_entrypoints(args.container_engine, rose_tree)
     sophios.input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
 
     if args.partial_failure_enable: