Workflow step source and scatter (#20)

Made possible by workflows donated by the 4D Nucleome Data Coordination and Integration Center http://dcic.4dnucleome.org/

* improve order, fix remaining descr→doc

Author: mr-c

cwlupgrader/main.py

@@ -8,7 +8,7 @@
 from typing import (Any, Dict, List, Optional,  # pylint:disable=unused-import
                     Text, Union)
 import ruamel.yaml
-
+from ruamel.yaml.comments import CommentedMap  # for consistent sort order
 
 def main(args=None):  # type: (Optional[List[str]]) -> int
     """Main function."""
@@ -21,14 +21,15 @@ def main(args=None):  # type: (Optional[List[str]]) -> int
             if ('cwlVersion' in document
                     and (document['cwlVersion'] == 'cwl:draft-3'
                          or document['cwlVersion'] == 'draft-3')):
-                draft3_to_v1_0(document)
+                document = draft3_to_v1_0(document)
             else:
                 print("Skipping non draft-3 CWL document", file=sys.stderr)
-            print(ruamel.yaml.dump(document, default_flow_style=False))
+            print(ruamel.yaml.round_trip_dump(
+                document, default_flow_style=False))
     return 0
 
 
-def draft3_to_v1_0(document):  # type: (Dict[Text, Any]) -> None
+def draft3_to_v1_0(document):  # type: (Dict[Text, Any]) -> Dict
     """Transformation loop."""
     _draft3_to_v1_0(document)
     if isinstance(document, MutableMapping):
@@ -40,6 +41,7 @@ def draft3_to_v1_0(document):  # type: (Dict[Text, Any]) -> None
                     if isinstance(entry, MutableMapping):
                         value[index] = _draft3_to_v1_0(entry)
     document['cwlVersion'] = 'v1.0'
+    return sort_v1_0(document)
 
 
 def _draft3_to_v1_0(document):
@@ -53,6 +55,11 @@ def _draft3_to_v1_0(document):
         elif document["class"] == "CommandLineTool":
             input_output_clean(document)
             hints_and_requirements_clean(document)
+            if isinstance(document["baseCommand"], list) and \
+                    len(document["baseCommand"]) == 1:
+                document["baseCommand"] = document["baseCommand"][0]
+            if "arguments" in document and not document["arguments"]:
+                del document["arguments"]
     clean_secondary_files(document)
 
     if "description" in document:
@@ -62,38 +69,46 @@ def _draft3_to_v1_0(document):
 
 
 def workflow_clean(document):  # type: (MutableMapping[Text, Any]) -> None
-    """Transform draft-3 style Workflows to idiomatic v1.0"""
+    """Transform draft-3 style Workflows to more idiomatic v1.0"""
     input_output_clean(document)
     hints_and_requirements_clean(document)
     outputs = document['outputs']
     for output_id in outputs:
         outputs[output_id]["outputSource"] = \
             outputs[output_id].pop("source").lstrip('#').replace(".", "/")
-    new_steps = {}
+    new_steps = CommentedMap()
     for step in document["steps"]:
-        new_step = copy.deepcopy(step)  # type: Dict[Text, Any]
-        del new_step["id"]
-        new_step["out"] = [outp["id"][len(step["id"])+1:] for outp in
-                           step["outputs"]]
-        del new_step["outputs"]
-        ins = {}
+        new_step = CommentedMap()
+        new_step.update(step)
+        step = new_step
+        step_id = step.pop("id")
+        step_id_len = len(step_id)+1
+        step["out"] = [outp["id"][step_id_len:] for outp in
+                       step["outputs"]]
+        del step["outputs"]
+        ins = CommentedMap()
         for inp in step["inputs"]:
-            ident = inp["id"][len(step["id"])+1:]  # remove step id prefix
+            ident = inp["id"][step_id_len:]  # remove step id prefix
             if 'source' in inp:
-                inp["source"] = inp["source"].lstrip('#')
+                inp["source"] = inp["source"].lstrip('#').replace(".", "/")
             del inp["id"]
             if len(inp) > 1:
                 ins[ident] = inp
             elif len(inp) == 1:
                 ins[ident] = inp.popitem()[1]
             else:
                 ins[ident] = {}
-        new_step["in"] = ins
-        del new_step["inputs"]
+        step["in"] = ins
+        del step["inputs"]
         if "scatter" in step:
-            new_step["scatter"] = step["scatter"][  # remove step prefix
-                len(step["id"])*2+3:]
-        new_steps[step["id"].lstrip('#')] = new_step
+            if len(step["scatter"]) == 1:
+                step["scatter"] = step["scatter"][step_id_len:]
+            else:
+                step["scatter"] = [source[step_id_len:] for
+                                   source in step["scatter"]]
+        if "description" in step:
+            step["doc"] = step.pop("description")
+        new_steps[step_id.lstrip('#')] = step
     document["steps"] = new_steps
 
 
@@ -102,13 +117,15 @@ def input_output_clean(document):  # type: (MutableMapping[Text, Any]) -> None
     for param_type in ['inputs', 'outputs']:
         if param_type not in document:
             break
-        new_section = {}
+        new_section = CommentedMap()
         for param in document[param_type]:
             param_id = param.pop('id').lstrip('#')
             if 'type' in param:
                 param['type'] = shorten_type(param['type'])
+            if 'description' in param:
+                param['doc'] = param.pop('description')
             if len(param) > 1:
-                new_section[param_id] = param
+                new_section[param_id] = sort_input_or_output(param)
             else:
                 new_section[param_id] = param.popitem()[1]
         document[param_type] = new_section
@@ -143,15 +160,19 @@ def shorten_type(type_obj):  # type: (List[Any]) -> Union[Text, List[Any]]
     for entry in type_obj:  # find arrays that we can shorten and do so
         if isinstance(entry, Mapping):
             if (entry['type'] == 'array' and
-                    isinstance(entry['items'], Text)):
+                    isinstance(entry['items'], (str, Text))):
                 entry = entry['items'] + '[]'
+            elif entry['type'] == 'enum':
+                entry = sort_enum(entry)
         new_type.extend([entry])
     if len(new_type) == 2:
         if 'null' in new_type:
             type_copy = copy.deepcopy(new_type)
             type_copy.remove('null')
             if isinstance(type_copy[0], (str, Text)):
                 return type_copy[0] + '?'
+    if len(new_type) == 1:
+        return new_type[0]
     return new_type
 
 
@@ -165,5 +186,33 @@ def clean_secondary_files(document):
                     '"path"', '"location"').replace(".path", ".location")
 
 
+def sort_v1_0(document):  # type: (Dict) -> Dict
+    """Sort the sections of the CWL document in a more meaningful order."""
+    keyorder = ['cwlVersion', 'class', 'id', 'label', 'doc', 'requirements',
+                'hints', 'inputs', 'stdin', 'baseCommand', 'steps',
+                'expression', 'arguments', 'stderr', 'stdout', 'outputs',
+                'successCodes', 'temporaryFailCodes', 'permanentFailCodes']
+    return CommentedMap(
+        sorted(document.items(), key=lambda i: keyorder.index(i[0])
+               if i[0] in keyorder else 100))
+
+
+def sort_enum(enum):  # type: (Mapping) -> Dict
+    """Sort the enum type definitions in a more meaningful order."""
+    keyorder = ['type', 'name', 'label', 'symbols', 'inputBinding']
+    return CommentedMap(
+        sorted(enum.items(), key=lambda i: keyorder.index(i[0])
+               if i[0] in keyorder else 100))
+
+
+def sort_input_or_output(io_def):  # type: (Dict) -> Dict
+    """Sort the input definitions in a more meaningful order."""
+    keyorder = ['label', 'doc', 'type', 'format', 'secondaryFiles',
+                'default', 'inputBinding', 'outputBinding', 'streamable']
+    return CommentedMap(
+        sorted(io_def.items(), key=lambda i: keyorder.index(i[0])
+               if i[0] in keyorder else 100))
+
+
 if __name__ == "__main__":
-    sys.exit(main(sys.argv[:1]))
+    sys.exit(main())

tests/draft-3-wf-v1.0.cwl

@@ -1,28 +1,28 @@
-class: Workflow
 cwlVersion: v1.0
-inputs:
-  input_file: File?
-outputs:
-  md5_report:
-    outputSource: md5/report
-    type: File?
-  validatefiles_report:
-    outputSource: validatefiles/report
-    type: File?
+class: Workflow
 requirements:
   InlineJavascriptRequirement: {}
+inputs:
+  input_file: File?
 steps:
   md5:
+    run: md5.cwl
+    out:
+    - report
     in:
       input_file: input_file
+  validatefiles:
+    run: validate.cwl
     out:
     - report
-    run: md5.cwl
-  validatefiles:
     in:
       input_file: input_file
       type: {}
-    out:
-    - report
-    run: validate.cwl
+outputs:
+  validatefiles_report:
+    type: File?
+    outputSource: validatefiles/report
+  md5_report:
+    type: File?
+    outputSource: md5/report
 

tests/test_complete.py

@@ -1,5 +1,4 @@
 import filecmp
-import os
 from cwlupgrader.main import main
 from .util import get_data
 
@@ -11,6 +10,6 @@ def test_draft3_workflow(tmpdir, capsys):
         outfile.write(capsys.readouterr().out)
         outfile.flush()
         outfile.close()
-    result = filecmp.cmp('tests/draft-3-wf-v1.0.cwl', str(test_path),
+    result = filecmp.cmp(get_data('tests/draft-3-wf-v1.0.cwl'), str(test_path),
                          shallow=False)
     assert result