Skip to content

Commit 206ed49

Browse files
vjaganat90Vasu Jaganathsameeul
authored
feature: add get_cwl_workflow() method to Python API (PolusAI#316)
* add toJson method to python API and an example * Update src/sophios/api/pythonapi.py * Update examples/scripts/multistep1_toJson_pyapi.py --------- Co-authored-by: Vasu Jaganath <[email protected]> Co-authored-by: Sameeul Bashir Samee <[email protected]>
1 parent 6ef44b1 commit 206ed49

File tree

3 files changed

+240
-2
lines changed

3 files changed

+240
-2
lines changed

‎examples/scripts/ground_truth_multistep1.json‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
{
2+
"name": "multistep1_toJson_pyapi_py",
3+
"yaml_inputs": {
4+
"multistep1_toJson_pyapi_py__step__1__touch___filename": "empty.txt",
5+
"multistep1_toJson_pyapi_py__step__2__append___str": "Hello"
6+
},
7+
"steps": [
8+
{
9+
"id": "multistep1_toJson_pyapi_py__step__1__touch",
10+
"in": {
11+
"filename": {
12+
"source": "multistep1_toJson_pyapi_py__step__1__touch___filename"
13+
}
14+
},
15+
"out": [
16+
"file"
17+
],
18+
"run": {
19+
"cwlVersion": "v1.0",
20+
"class": "CommandLineTool",
21+
"requirements": {
22+
"DockerRequirement": {
23+
"dockerPull": "docker.io/bash:4.4"
24+
},
25+
"InlineJavascriptRequirement": {}
26+
},
27+
"baseCommand": "touch",
28+
"inputs": {
29+
"filename": {
30+
"type": "string",
31+
"inputBinding": {
32+
"position": 1
33+
}
34+
}
35+
},
36+
"outputs": {
37+
"file": {
38+
"type": "File",
39+
"outputBinding": {
40+
"glob": "$(inputs.filename)"
41+
}
42+
}
43+
}
44+
}
45+
},
46+
{
47+
"id": "multistep1_toJson_pyapi_py__step__2__append",
48+
"in": {
49+
"file": {
50+
"source": "multistep1_toJson_pyapi_py__step__1__touch/file"
51+
},
52+
"str": {
53+
"source": "multistep1_toJson_pyapi_py__step__2__append___str"
54+
}
55+
},
56+
"out": [
57+
"file"
58+
],
59+
"run": {
60+
"class": "CommandLineTool",
61+
"cwlVersion": "v1.0",
62+
"requirements": {
63+
"ShellCommandRequirement": {},
64+
"InlineJavascriptRequirement": {},
65+
"InitialWorkDirRequirement": {
66+
"listing": [
67+
"$(inputs.file)"
68+
]
69+
}
70+
},
71+
"inputs": {
72+
"str": {
73+
"type": "string",
74+
"inputBinding": {
75+
"shellQuote": false,
76+
"position": 1,
77+
"prefix": "echo"
78+
}
79+
},
80+
"file": {
81+
"type": "File",
82+
"inputBinding": {
83+
"shellQuote": false,
84+
"position": 2,
85+
"prefix": ">>"
86+
}
87+
}
88+
},
89+
"outputs": {
90+
"file": {
91+
"type": "File",
92+
"outputBinding": {
93+
"glob": "$(inputs.file.basename)"
94+
}
95+
}
96+
}
97+
}
98+
},
99+
{
100+
"id": "multistep1_toJson_pyapi_py__step__3__cat",
101+
"in": {
102+
"file": {
103+
"source": "multistep1_toJson_pyapi_py__step__2__append/file"
104+
}
105+
},
106+
"out": [
107+
"output"
108+
],
109+
"run": {
110+
"class": "CommandLineTool",
111+
"cwlVersion": "v1.0",
112+
"requirements": {
113+
"DockerRequirement": {
114+
"dockerPull": "docker.io/bash:4.4"
115+
},
116+
"InlineJavascriptRequirement": {}
117+
},
118+
"baseCommand": [
119+
"cat"
120+
],
121+
"inputs": {
122+
"file": {
123+
"type": "File"
124+
}
125+
},
126+
"outputs": {
127+
"output": {
128+
"type": "string",
129+
"outputBinding": {
130+
"glob": "output",
131+
"loadContents": true,
132+
"outputEval": "$(self[0].contents)"
133+
}
134+
}
135+
},
136+
"stdin": "$(inputs.file.path)",
137+
"stdout": "output"
138+
}
139+
}
140+
],
141+
"cwlVersion": "v1.2",
142+
"class": "Workflow",
143+
"$namespaces": {
144+
"edam": "https://edamontology.org/"
145+
},
146+
"$schemas": [
147+
"https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl"
148+
],
149+
"inputs": {
150+
"multistep1_toJson_pyapi_py__step__1__touch___filename": {
151+
"type": "string"
152+
},
153+
"multistep1_toJson_pyapi_py__step__2__append___str": {
154+
"type": "string"
155+
}
156+
},
157+
"outputs": {
158+
"multistep1_toJson_pyapi_py__step__1__touch___file": {
159+
"type": "File",
160+
"outputSource": "multistep1_toJson_pyapi_py__step__1__touch/file"
161+
},
162+
"multistep1_toJson_pyapi_py__step__2__append___file": {
163+
"type": "File",
164+
"outputSource": "multistep1_toJson_pyapi_py__step__2__append/file"
165+
},
166+
"multistep1_toJson_pyapi_py__step__3__cat___output": {
167+
"type": "string",
168+
"outputSource": "multistep1_toJson_pyapi_py__step__3__cat/output"
169+
}
170+
}
171+
}

‎examples/scripts/multistep1_toJson_pyapi.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
from pathlib import Path
2+
import json
3+
from sophios.api.pythonapi import Step, Workflow
4+
5+
6+
def workflow() -> Workflow:
7+
# step echo
8+
touch = Step(clt_path='../../cwl_adapters/touch.cwl')
9+
touch.filename = 'empty.txt'
10+
append = Step(clt_path='../../cwl_adapters/append.cwl')
11+
append.file = touch.file
12+
append.str = 'Hello'
13+
cat = Step(clt_path='../../cwl_adapters/cat.cwl')
14+
cat.file = append.file
15+
# arrange steps
16+
steps = [touch, append, cat]
17+
18+
# create workflow
19+
filename = 'multistep1_toJson_pyapi_py'
20+
wkflw = Workflow(steps, filename)
21+
return wkflw
22+
23+
# Do NOT .run() here
24+
25+
26+
if __name__ == '__main__':
27+
multistep1 = workflow()
28+
workflow_json = multistep1.get_cwl_workflow() # .run() here inside main
29+
fname = 'workflow.json'
30+
paren_dir = Path(__file__).parent
31+
with open(paren_dir / 'ground_truth_multistep1.json', 'r', encoding='utf-8') as file:
32+
ground_truth = json.load(file)
33+
assert ground_truth == workflow_json

‎src/sophios/api/pythonapi.py‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from sophios import run_local as run_local_module
1616
from sophios.cli import get_args
1717
from sophios.utils_graphs import get_graph_reps
18-
from sophios.wic_types import CompilerInfo, RoseTree, StepId, Tool, Tools, YamlTree
18+
from sophios.wic_types import CompilerInfo, RoseTree, StepId, Tool, Tools, YamlTree, Json, Cwl, NodeData
1919

2020
from ._types import ScatterMethod
2121

@@ -746,12 +746,46 @@ def compile(self, write_to_disk: bool = False) -> CompilerInfo:
746746

747747
return compiler_info
748748

749+
def get_cwl_workflow(self) -> Json:
750+
"""Return the compiled Cwl and its inputs in one payload Json
751+
Returns:
752+
Json: Contains the compiled CWL and yaml inputs to the workflow.
753+
"""
754+
compiler_info = self.compile(write_to_disk=False)
755+
self.user_args.append('--cwl_inline_runtag')
756+
args = get_args(self.process_name, self.user_args)
757+
rose_tree = compiler_info.rose
758+
759+
# this is unfortunately necessary for now
760+
# TODO: uncouple dumping to file and getting cwl steps
761+
# then remove this write_to_disk call here
762+
input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
763+
764+
rose_tree = post_compile.cwl_inline_runtag(args, rose_tree)
765+
sub_node_data = rose_tree.data
766+
cwl_ast = sub_node_data.compiled_cwl
767+
768+
# Copy samee's workaround for duplicate outs
769+
for step in cwl_ast['steps']:
770+
out_vars = step['out']
771+
out_vars_unique = list(set(out_vars))
772+
out_vars_unique.sort()
773+
step['out'] = out_vars_unique
774+
775+
yaml_inputs = sub_node_data.workflow_inputs_file
776+
workflow_json: Json = {}
777+
workflow_json = {
778+
"name": self.process_name,
779+
"yaml_inputs": yaml_inputs,
780+
**cwl_ast
781+
}
782+
return workflow_json
783+
749784
def run(self) -> None:
750785
"""Run compiled workflow."""
751786
logger.info(f"Running {self.process_name}")
752787
plugins.logging_filters()
753788
compiler_info = self.compile(write_to_disk=True)
754-
755789
args = get_args(self.process_name, self.user_args) # Use mock CLI args
756790
rose_tree: RoseTree = compiler_info.rose
757791

0 commit comments

Comments
 (0)