add missing dependency and merging of namespaces in inlined runtag workflows

Author: Vasu Jaganath

pyproject.toml

@@ -44,7 +44,8 @@ dependencies = [
     "fastapi",
     "python-jose",
     "uvicorn",
-    "referencing"
+    "referencing",
+    "aiofiles"
 ]
 
 [project.readme]

src/sophios/api/pythonapi.py

@@ -769,7 +769,7 @@ def run(self, compile_args_dict: Dict[str, str] = {}, run_args_dict: Dict[str, s
         compiler_info = self.compile(
             args_dict=compile_args_dict, write_to_disk=False)
         rose_tree: RoseTree = compiler_info.rose
-        # rose_tree = post_compile.cwl_inline_runtag(rose_tree)
+        rose_tree = post_compile.cwl_inline_runtag(rose_tree)
         post_compile.find_and_create_output_dirs(rose_tree)
         # only write out after all the transformations
         input_output.write_to_disk(rose_tree, Path(

src/sophios/compiler.py

@@ -207,8 +207,6 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
     graph_nx = graph.networkx
     graphdata = graph.graphdata
 
-    # plugin_ns = wic['wic'].get('namespace', 'global')
-
     tools_lst: List[Tool] = []
 
     for i, step_key in enumerate(steps_keys):
@@ -267,8 +265,6 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
                 print('AST modified', step_key)
                 wic_steps[f'({i+1}, {step_key})'] = {'wic': sub_node_data.yml.get('wic', {})}
                 wic_step_i = wic_steps.get(f'({i+1}, {step_key})', {})
-                # import yaml
-                # print(yaml.dump(wic_steps))
 
             # Add arguments to the compiled subworkflow (if any), being careful
             # to remove any child wic: metadata annotations. Post-compilation
@@ -355,11 +351,6 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
         # should be written to subdirectories or if everything should be
         # written to autogenerated/
         if relative_run_path:
-            # if step_key in subkeys:
-            #     run_path = step_name_or_key + '/' + run_path
-            # else:
-            #     run_path = os.path.relpath(run_path, 'autogenerated/')
-            #     run_path = ('../' * len(namespaces)) + run_path
             run_path = step_name_or_key + '/' + run_path
         else:
             if step_key in subkeys:
@@ -781,8 +772,6 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
                                        inputs_file_workflow, vars_workflow_output_internal,
                                        explicit_edge_defs_copy, explicit_edge_calls_copy)
                     compiler_info = CompilerInfo(rose_tree, env_data)
-                    # node_data_dummy = NodeData(None, None, yaml_tree_mod, None, None, None, None, None, None, None)
-                    # compiler_info_dummy = CompilerInfo(RoseTree(node_data_dummy, None), None)
                     return compiler_info
 
         # Add CommandLineTool/Subworkflow outputs tags to workflow out tags.
@@ -791,7 +780,6 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
         out_keyvals = {}
         for out_key, out_dict in tool_i.cwl['outputs'].items():
             out_keyvals[out_key] = utils_cwl.copy_cwl_input_output_dict(out_dict)
-            # print(out_key, out_keyvals[out_key])
         if not out_keyvals:  # FYI out_keyvals should never be {}
             print(f'Error! no outputs for step {step_key}')
         outputs_workflow.append(out_keyvals)
@@ -864,7 +852,6 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
     def populate_scalar_val(in_dict: dict) -> Any:
         newval: Any = ()
         if 'File' == in_dict['type']:
-            # path = Path(in_dict['value']).name # NOTE: Use .name ?
             newval = {'class': 'File', 'path': in_dict['value']}
             if 'format' in in_dict:
                 in_format = in_dict['format']
@@ -904,8 +891,6 @@ def populate_scalar_val(in_dict: dict) -> Any:
             new_keyval = {key: val_list}
         else:
             new_keyval = {key: populate_scalar_val(in_dict)}
-        # else:
-        #    raise Exception(f"Error! Unknown type: {in_dict['type']}")
         yaml_inputs.update(new_keyval)
 
     if not testing:

src/sophios/cwl_subinterpreter.py

@@ -7,7 +7,6 @@
 import time
 from pathlib import Path
 from typing import Dict, List
-from unittest.mock import patch
 
 import graphviz
 import networkx as nx

src/sophios/post_compile.py

@@ -88,9 +88,14 @@ def cwl_inline_runtag(rose_tree: RoseTree) -> RoseTree:
         for sub_rose_tree in rose_tree_mod.sub_trees:
             sub_node_data: NodeData = sub_rose_tree.data
             sub_step_name = sub_node_data.namespaces[-1]
-            step_to_update = next(item for item in cwl_tree['steps'] if item.get('id') == sub_step_name)
+            step_to_update = next(
+                item for item in cwl_tree['steps'] if item.get('id') == sub_step_name)
             step_to_update['run'] = sub_node_data.compiled_cwl
-            # The below two lines needed to avoid parsing errors of the run tag
+            # merge the steps/clt namespaces to global namespaces
+            # as the run tag can't have namespaces and schemas
+            cwl_tree['$namespaces'] = cwl_tree['$namespaces'] | step_to_update['run'].get(
+                '$namespaces', {})
+            # and then get rid off $namespaces and $schemas in the run tag
             step_to_update['run'].pop('$namespaces', None)
             step_to_update['run'].pop('$schemas', None)
             sub_rose_tree = cwl_inline_runtag(sub_rose_tree)

src/sophios/run_local_async.py

@@ -40,8 +40,6 @@ async def run_cwl_workflow(
         cwl_runner (str): The command for the CWL runner (e.g., 'cwltool', 'toil-cwl-runner').
         container_cmd (str): The command for the container engine (e.g., 'docker', 'podman').
         user_env (Dict[str, str]): A dictionary of environment variables to set for the subprocess.
-        supported_runners (Optional[List[str]]): A list of explicitly supported CWL runner commands.
-                                                 If None, defaults to ['cwltool', 'toil-cwl-runner'].
 
     Returns:
         Optional[int]: The exit code of the executed workflow process, or None if an