Codegen changes to support cwltool --fast-parse (#156)

* Update parsers based on latest codegen, with mypy fixes
* adjust expression refactor logic for the fixed refscope
* adjust cite-extract and docker-extract for getting concrete classes from `hints`

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <[email protected]>

Co-authored-by: Michael R. Crusoe <[email protected]>

Author: tetron

README.rst

@@ -148,13 +148,13 @@ Regenerate parsers
 To regenerate install the ``schema_salad`` package and run:
 
 ``cwl_utils/parser/cwl_v1_0.py`` was created via
-``schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/main/v1.0/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_0"``
+``schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/codegen/v1.0/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_0" > cwl_utils/parser/cwl_v1_0.py``
 
 ``cwl_utils/parser/cwl_v1_1.py`` was created via
-``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.1/raw/main/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_1"``
+``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.1/raw/codegen/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_1" > cwl_utils/parser/cwl_v1_1.py``
 
 ``cwl_utils/parser/cwl_v1_2.py`` was created via
-``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.2/raw/1.2.1_proposed/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_2"``
+``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.2/raw/1.2.1_proposed/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_2" > cwl_utils/parser/cwl_v1_2.py``
 
 Release
 ~~~~~~~

cwl_utils/cite_extract.py

@@ -55,7 +55,10 @@ def extract_software_reqs(
                 yield req
     if process.hints:
         for req in process.hints:
-            if req["class"] == "SoftwareRequirement":
+            if isinstance(req, cwl.ProcessRequirement):
+                if isinstance(req, cwl.SoftwareRequirement):
+                    yield req
+            elif req["class"] == "SoftwareRequirement":
                 yield cwl.load_field(
                     req,
                     cwl.SoftwareRequirementLoader,

cwl_utils/cwl_v1_2_expression_refactor.py

@@ -702,14 +702,9 @@ def process_workflow_inputs_and_outputs(
                     target_type.name = None
                 target = cwl.WorkflowInputParameter(id=None, type=target_type)
                 if not isinstance(param2.outputSource, list):
-                    sources: Union[List[str], str] = "/".join(
-                        param2.outputSource.split("#")[-1].split("/")[1:]
-                    )
+                    sources: Union[List[str], str] = param2.outputSource.split("#")[-1]
                 else:
-                    sources = [
-                        "/".join(s.split("#")[-1].split("/")[1:])
-                        for s in param2.outputSource
-                    ]
+                    sources = [s.split("#")[-1] for s in param2.outputSource]
                 source_type_items = utils.type_for_source(workflow, sources)
                 if "null" not in source_type_items:
                     if isinstance(source_type_items, list):

cwl_utils/docker_extract.py

@@ -76,7 +76,10 @@ def extract_docker_reqs(process: ProcessType) -> Iterator[cwl.DockerRequirement]
                 yield req
     if process.hints:
         for req in process.hints:
-            if req["class"] == "DockerRequirement":
+            if isinstance(req, cwl.ProcessRequirement):
+                if isinstance(req, cwl.DockerRequirement):
+                    yield req
+            elif req["class"] == "DockerRequirement":
                 yield cwl.load_field(
                     req,
                     cwl.DockerRequirementLoader,

cwl_utils/expression_refactor.py

@@ -13,6 +13,7 @@
     Callable,
     Dict,
     List,
+    MutableMapping,
     MutableSequence,
     Optional,
     Tuple,
@@ -43,15 +44,17 @@
 )
 from cwl_utils.parser import cwl_v1_0, cwl_v1_1, cwl_v1_2
 
-save_type = Union[Dict[str, str], List[Union[Dict[str, str], List[Any], None]], None]
+save_type = Optional[
+    Union[MutableMapping[str, Any], MutableSequence[Any], int, float, bool, str]
+]
 
 
 class saveCWL(Protocol):
     """Shortcut type for CWL v1.x parse.save()."""
 
     def __call__(
         self,
-        val: Optional[Union[Any, MutableSequence[Any]]],
+        val: Any,
         top: bool = True,
         base_url: str = "",
         relative_uris: bool = True,

cwl_utils/parser/__init__.py

@@ -14,7 +14,7 @@
 LoadingOptions = Union[
     cwl_v1_0.LoadingOptions, cwl_v1_1.LoadingOptions, cwl_v1_2.LoadingOptions
 ]
-Savable = Union[cwl_v1_0.Savable, cwl_v1_1.Savable, cwl_v1_2.Savable]
+Saveable = Union[cwl_v1_0.Saveable, cwl_v1_1.Saveable, cwl_v1_2.Saveable]
 Workflow = Union[cwl_v1_0.Workflow, cwl_v1_1.Workflow, cwl_v1_2.Workflow]
 WorkflowTypes = (cwl_v1_0.Workflow, cwl_v1_1.Workflow, cwl_v1_2.Workflow)
 WorkflowStep = Union[
@@ -157,16 +157,16 @@ def load_document_by_yaml(
 
 
 def save(
-    val: Optional[Union[Savable, MutableSequence[Savable]]],
+    val: Optional[Union[Saveable, MutableSequence[Saveable]]],
     top: bool = True,
     base_url: str = "",
     relative_uris: bool = True,
 ) -> Any:
     """Convert a given CWL object into a built-in typed object."""
     if (
-        isinstance(val, cwl_v1_0.Savable)
-        or isinstance(val, cwl_v1_1.Savable)
-        or isinstance(val, cwl_v1_2.Savable)
+        isinstance(val, cwl_v1_0.Saveable)
+        or isinstance(val, cwl_v1_1.Saveable)
+        or isinstance(val, cwl_v1_2.Saveable)
     ):
         return val.save(top=top, base_url=base_url, relative_uris=relative_uris)
     if isinstance(val, MutableSequence):