Merge branch 'master' into random_output_dir

Author: mr-c

.gitignore

@@ -9,6 +9,7 @@ eggs/
 *.egg-info/
 *.egg
 .tox/
+.pytest_cache
 
 # Editor Temps
 .*.sw?
@@ -44,4 +45,5 @@ output.txt
 pydocstyle_report.txt
 response.txt
 test.txt
+time.txt
 value

Makefile

@@ -59,7 +59,12 @@ install-deb-dep:
 
 ## install     : install the ${MODULE} module and schema-salad-tool
 install: FORCE
-	pip install .
+	pip install .[deps]
+
+## dev     : install the ${MODULE} module in dev mode
+dev: install-dep
+	pip install -e .[deps]
+
 
 ## dist        : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz

appveyor.yml

@@ -41,7 +41,7 @@ build_script:
 test_script:
   - |
     %PYTHON%\\python.exe setup.py test --addopts "--cov-report xml --cov cwltool -p no:cacheprovider --junit-xml=tests.xml"
-  - "%PYTHON%\\python.exe -m codecov"
+  - "%PYTHON%\\python.exe -m codecov --file coverage.xml"
 
 on_finish:
   - ps: |

cwltool/__main__.py

@@ -1,3 +1,4 @@
+"""Default entrypoint for the cwltool module."""
 from __future__ import absolute_import
 
 from . import main

cwltool/argparser.py

@@ -1,14 +1,15 @@
+"""Command line argument parsing for cwltool."""
 from __future__ import absolute_import, print_function
 
 import argparse
 import os
-from typing import Any, AnyStr, Dict, List, Optional, Sequence, Union, cast
+from typing import (Any, AnyStr, Dict, List, MutableMapping, MutableSequence,
+                    Optional, Sequence, Union, cast)
 
+from schema_salad.ref_resolver import file_uri
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
-from schema_salad.ref_resolver import file_uri
-
 from .loghandler import _logger
 from .process import Process, shortname  # pylint: disable=unused-import
 from .resolver import ga4gh_tool_registries
@@ -381,7 +382,7 @@ def add_argument(toolparser, name, inptype, records, description="",
         flag = "--"
 
     required = True
-    if isinstance(inptype, list):
+    if isinstance(inptype, MutableSequence):
         if inptype[0] == "null":
             required = False
             if len(inptype) == 2:
@@ -398,16 +399,16 @@ def add_argument(toolparser, name, inptype, records, description="",
         action = cast(argparse.Action, FileAction)
     elif inptype == "Directory":
         action = cast(argparse.Action, DirectoryAction)
-    elif isinstance(inptype, dict) and inptype["type"] == "array":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
         if inptype["items"] == "File":
             action = cast(argparse.Action, FileAppendAction)
         elif inptype["items"] == "Directory":
             action = cast(argparse.Action, DirectoryAppendAction)
         else:
             action = "append"
-    elif isinstance(inptype, dict) and inptype["type"] == "enum":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
         atype = Text
-    elif isinstance(inptype, dict) and inptype["type"] == "record":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
         records.append(name)
         for field in inptype['fields']:
             fieldname = name + "." + shortname(field['name'])

cwltool/builder.py

@@ -2,27 +2,31 @@
 
 import copy
 import logging
-from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple
-from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
+                    Optional, Set, Tuple, Union)
 
 from rdflib import Graph, URIRef  # pylint: disable=unused-import
 from rdflib.namespace import OWL, RDFS
-import schema_salad.schema  # pylint: disable=unused-import
+from ruamel.yaml.comments import CommentedMap
 from schema_salad import validate
-from schema_salad.schema import AvroSchemaFromJSONData
+from schema_salad.schema import AvroSchemaFromJSONData, Names
 from schema_salad.sourceline import SourceLine
 from six import iteritems, string_types
+from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
+                               Text, Type)
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from . import expression
 from .errors import WorkflowException
 from .loghandler import _logger
 from .mutation import MutationManager  # pylint: disable=unused-import
-from .pathmapper import (PathMapper,  # pylint: disable=unused-import
-                         get_listing, normalizeFilesDirs, visit_class)
+from .pathmapper import PathMapper  # pylint: disable=unused-import
+from .pathmapper import get_listing, normalizeFilesDirs, visit_class
 from .stdfsaccess import StdFsAccess  # pylint: disable=unused-import
-from .utils import (aslist, docker_windows_path_adjust,
-                    json_dumps, onWindows)
+from .utils import aslist, docker_windows_path_adjust, json_dumps, onWindows
+
+
+
 if TYPE_CHECKING:
     from .provenance import CreateProvProfile  # pylint: disable=unused-import
 CONTENT_LIMIT = 64 * 1024
@@ -109,7 +113,7 @@ def __init__(self,
                  files=None,                # type: List[Dict[Text, Text]]
                  bindings=None,             # type: List[Dict[Text, Any]]
                  schemaDefs=None,           # type: Dict[Text, Dict[Text, Any]]
-                 names=None,                # type: schema_salad.schema.Names
+                 names=None,                # type: Names
                  requirements=None,         # type: List[Dict[Text, Any]]
                  hints=None,                # type: List[Dict[Text, Any]]
                  timeout=None,              # type: float
@@ -129,7 +133,7 @@ def __init__(self,
                 ):  # type: (...) -> None
 
         if names is None:
-            self.names = schema_salad.schema.Names()
+            self.names = Names()
         else:
             self.names = names
 
@@ -194,22 +198,23 @@ def build_job_script(self, commands):
             return None
 
     def bind_input(self,
-                   schema,                   # type: Dict[Text, Any]
+                   schema,                   # type: MutableMapping[Text, Any]
                    datum,                    # type: Any
                    discover_secondaryFiles,  # type: bool
                    lead_pos=None,            # type: Optional[Union[int, List[int]]]
                    tail_pos=None,            # type: Optional[List[int]]
-                  ):  # type: (...) -> List[Dict[Text, Any]]
+                  ):  # type: (...) -> List[MutableMapping[Text, Any]]
 
         if tail_pos is None:
             tail_pos = []
         if lead_pos is None:
             lead_pos = []
-        bindings = []  # type: List[Dict[Text,Text]]
-        binding = None  # type: Optional[Dict[Text,Any]]
+        bindings = []  # type: List[MutableMapping[Text, Text]]
+        binding = None  # type: Optional[MutableMapping[Text,Any]]
         value_from_expression = False
-        if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
-            binding = copy.copy(schema["inputBinding"])
+        if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
+            binding = CommentedMap(schema["inputBinding"].items())
+            assert binding is not None
 
             if "position" in binding:
                 binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos)
@@ -221,12 +226,12 @@ def bind_input(self,
                 value_from_expression = True
 
         # Handle union types
-        if isinstance(schema["type"], list):
+        if isinstance(schema["type"], MutableSequence):
             bound_input = False
             for t in schema["type"]:
                 if isinstance(t, string_types) and self.names.has_name(t, ""):
                     avsc = self.names.get_name(t, "")
-                elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
+                elif isinstance(t, MutableMapping) and "name" in t and self.names.has_name(t["name"], ""):
                     avsc = self.names.get_name(t["name"], "")
                 else:
                     avsc = AvroSchemaFromJSONData(t, self.names)
@@ -240,7 +245,7 @@ def bind_input(self,
                         bound_input = True
             if not bound_input:
                 raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
-        elif isinstance(schema["type"], dict):
+        elif isinstance(schema["type"], MutableMapping):
             st = copy.deepcopy(schema["type"])
             if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding:
                 st["inputBinding"] = {}
@@ -289,7 +294,7 @@ def bind_input(self,
                     if "secondaryFiles" not in datum:
                         datum["secondaryFiles"] = []
                     for sf in aslist(schema["secondaryFiles"]):
-                        if isinstance(sf, dict) or "$(" in sf or "${" in sf:
+                        if isinstance(sf, MutableMapping) or "$(" in sf or "${" in sf:
                             sfpath = self.do_eval(sf, context=datum)
                         else:
                             sfpath = substitute(datum["basename"], sf)
@@ -301,7 +306,7 @@ def bind_input(self,
                                 if d["basename"] == sfname:
                                     found = True
                             if not found:
-                                if isinstance(sfname, dict):
+                                if isinstance(sfname, MutableMapping):
                                     datum["secondaryFiles"].append(sfname)
                                 elif discover_secondaryFiles:
                                     datum["secondaryFiles"].append({
@@ -344,7 +349,7 @@ def _capture_files(f):
         return bindings
 
     def tostr(self, value):  # type: (Any) -> Text
-        if isinstance(value, dict) and value.get("class") in ("File", "Directory"):
+        if isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
             if "path" not in value:
                 raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))
 
@@ -372,8 +377,8 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
             with SourceLine(binding, "separate", WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
                 raise WorkflowException("'separate' option can not be specified without prefix")
 
-        argl = []  # type: List[Dict[Text,Text]]
-        if isinstance(value, list):
+        argl = []  # type: MutableSequence[MutableMapping[Text, Text]]
+        if isinstance(value, MutableSequence):
             if binding.get("itemSeparator") and value:
                 argl = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
             elif binding.get("valueFrom"):
@@ -383,9 +388,9 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
                 return [prefix]
             else:
                 return []
-        elif isinstance(value, dict) and value.get("class") in ("File", "Directory"):
+        elif isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
             argl = [value]
-        elif isinstance(value, dict):
+        elif isinstance(value, MutableMapping):
             return [prefix] if prefix else []
         elif value is True and prefix:
             return [prefix]
@@ -407,10 +412,10 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
     def do_eval(self, ex, context=None, recursive=False, strip_whitespace=True):
         # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
         if recursive:
-            if isinstance(ex, dict):
+            if isinstance(ex, MutableMapping):
                 return {k: self.do_eval(v, context, recursive)
                         for k, v in iteritems(ex)}
-            if isinstance(ex, list):
+            if isinstance(ex, MutableSequence):
                 return [self.do_eval(v, context, recursive)
                         for v in ex]
 

cwltool/checker.py

@@ -1,11 +1,12 @@
+"""Static checking of CWL workflow connectivity."""
 from collections import namedtuple
-from typing import Any, Dict, List, Optional
-from typing_extensions import Text  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from typing import Any, Dict, List, MutableMapping, MutableSequence, Optional
 
+import six
 from schema_salad import validate
 from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno
-import six
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from .errors import WorkflowException
 from .loghandler import _logger
@@ -15,7 +16,7 @@
 
 def _get_type(tp):
     # type: (Any) -> Any
-    if isinstance(tp, dict):
+    if isinstance(tp, MutableMapping):
         if tp.get("type") not in ("array", "record", "enum"):
             return tp["type"]
     return tp
@@ -46,9 +47,9 @@ def merge_flatten_type(src):
     """Return the merge flattened type of the source type
     """
 
-    if isinstance(src, list):
+    if isinstance(src, MutableSequence):
         return [merge_flatten_type(t) for t in src]
-    if isinstance(src, dict) and src.get("type") == "array":
+    if isinstance(src, MutableMapping) and src.get("type") == "array":
         return src
     return {"items": src, "type": "array"}
 
@@ -65,7 +66,7 @@ def can_assign_src_to_sink(src, sink, strict=False):  # type: (Any, Any, bool) -
 
     if src == "Any" or sink == "Any":
         return True
-    if isinstance(src, dict) and isinstance(sink, dict):
+    if isinstance(src, MutableMapping) and isinstance(sink, MutableMapping):
         if sink.get("not_connected") and strict:
             return False
         if src["type"] == "array" and sink["type"] == "array":
@@ -79,35 +80,33 @@ def can_assign_src_to_sink(src, sink, strict=False):  # type: (Any, Any, bool) -
                         return False
             return True
         return can_assign_src_to_sink(src["type"], sink["type"], strict)
-    elif isinstance(src, list):
+    if isinstance(src, MutableSequence):
         if strict:
-            for t in src:
-                if not can_assign_src_to_sink(t, sink):
+            for this_src in src:
+                if not can_assign_src_to_sink(this_src, sink):
                     return False
             return True
-        else:
-            for t in src:
-                if can_assign_src_to_sink(t, sink):
-                    return True
-            return False
-    elif isinstance(sink, list):
-        for t in sink:
-            if can_assign_src_to_sink(src, t):
+        for this_src in src:
+            if can_assign_src_to_sink(this_src, sink):
+                return True
+        return False
+    if isinstance(sink, MutableSequence):
+        for this_sink in sink:
+            if can_assign_src_to_sink(src, this_sink):
                 return True
         return False
-    else:
-        return src == sink
+    return src == sink
 
 
 def _compare_records(src, sink, strict=False):
-    # type: (Dict[Text, Any], Dict[Text, Any], bool) -> bool
+    # type: (MutableMapping[Text, Any], MutableMapping[Text, Any], bool) -> bool
     """Compare two records, ensuring they have compatible fields.
 
     This handles normalizing record names, which will be relative to workflow
     step, so that they can be compared.
     """
 
-    def _rec_fields(rec):  # type: (Dict[Text, Any]) -> Dict[Text, Any]
+    def _rec_fields(rec):  # type: (MutableMapping[Text, Any]) -> MutableMapping[Text, Any]
         out = {}
         for field in rec["fields"]:
             name = shortname(field["name"])
@@ -227,7 +226,7 @@ def check_all_types(src_dict, sinks, sourceField):
     for sink in sinks:
         if sourceField in sink:
             valueFrom = sink.get("valueFrom")
-            if isinstance(sink[sourceField], list):
+            if isinstance(sink[sourceField], MutableSequence):
                 srcs_of_sink = [src_dict[parm_id] for parm_id in sink[sourceField]]
                 linkMerge = sink.get("linkMerge", ("merge_nested"
                                                    if len(sink[sourceField]) > 1 else None))