cwltool: misc improvements in type annotations and six lib integration

Author: manu-chroma

cwltool/builder.py

@@ -56,7 +56,7 @@ def __init__(self):  # type: () -> None
 
         # One of "no_listing", "shallow_listing", "deep_listing"
         # Will be default "no_listing" for CWL v1.1
-        self.loadListing = "deep_listing"  # type: Union[None, Text]
+        self.loadListing = "deep_listing"  # type: Union[None, str]
 
     def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
         # type: (Dict[Text, Any], Any, Union[int, List[int]], List[int]) -> List[Dict[Text, Any]]
@@ -130,7 +130,7 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
                 self.files.append(datum)
                 if binding:
                     if binding.get("loadContents"):
-                        with self.fs_access.open(datum["location"], str("rb")) as f:
+                        with self.fs_access.open(datum["location"], "rb") as f:
                             datum["contents"] = f.read(CONTENT_LIMIT)
 
                 if "secondaryFiles" in schema:

cwltool/main.py

@@ -37,7 +37,6 @@
 from .stdfsaccess import StdFsAccess
 from .update import ALLUPDATES, UPDATES
 
-
 _logger = logging.getLogger("cwltool")
 
 defaultStreamHandler = logging.StreamHandler()

cwltool/pack.py

@@ -28,7 +28,7 @@ def find_run(d, loadref, runs):  # type: (Any, Callable[[Text, Text], Union[Dict
         for s in d:
             find_run(s, loadref, runs)
     elif isinstance(d, dict):
-        if "run" in d and isinstance(d["run"], (str, six.text_type)):
+        if "run" in d and isinstance(d["run"], six.string_types)):
             if d["run"] not in runs:
                 runs.add(d["run"])
                 find_run(loadref(None, d["run"]), loadref, runs)
@@ -42,7 +42,7 @@ def find_ids(d, ids):  # type: (Any, Set[Text]) -> None
             find_ids(s, ids)
     elif isinstance(d, dict):
         for i in ("id", "name"):
-            if i in d and isinstance(d[i], (str, six.text_type)):
+            if i in d and isinstance(d[i], six.string_types)):
                 ids.add(d[i])
         for s in d.values():
             find_ids(s, ids)
@@ -52,7 +52,7 @@ def replace_refs(d, rewrite, stem, newstem):
     # type: (Any, Dict[Text, Text], Text, Text) -> None
     if isinstance(d, list):
         for s, v in enumerate(d):
-            if isinstance(v, (str, six.text_type)):
+            if isinstance(v, six.string_types):
                 if v in rewrite:
                     d[s] = rewrite[v]
                 elif v.startswith(stem):
@@ -61,7 +61,7 @@ def replace_refs(d, rewrite, stem, newstem):
                 replace_refs(v, rewrite, stem, newstem)
     elif isinstance(d, dict):
         for s, v in d.items():
-            if isinstance(v, (str, six.text_type)):
+            if isinstance(v, six.string_types):
                 if v in rewrite:
                     d[s] = rewrite[v]
                 elif v.startswith(stem):

cwltool/process.py

@@ -30,7 +30,7 @@
 from schema_salad.sourceline import SourceLine
 from six.moves import urllib
 
-from cwltool.utils import cmp_like_py2
+from .utils import cmp_like_py2
 from .builder import Builder
 from .errors import UnsupportedRequirement, WorkflowException
 from .pathmapper import (PathMapper, adjustDirObjs, get_listing,
@@ -603,8 +603,13 @@ def _init_job(self, joborder, **kwargs):
                     builder.bindings.append(cm)
 
         # use python2 like sorting of heterogeneous lists
-        # (containing str and int types)
-        builder.bindings.sort(key=cmp_to_key(cmp_like_py2))
+        # (containing str and int types),
+        # TODO: unify for both runtime
+        if six.PY3:
+            key = cmp_to_key(cmp_like_py2)
+        else:  # PY2
+            key = lambda dict: dict["position"]
+        builder.bindings.sort(key=key)
         builder.resources = self.evalResources(builder, kwargs)
 
         return builder

cwltool/sandboxjs.py

@@ -29,7 +29,7 @@ class JavascriptException(Exception):
 minimum_node_version_str = '0.10.26'
 
 def check_js_threshold_version(working_alias):
-    # type: (Text) -> bool
+    # type: (str) -> bool
 
     """Checks if the nodeJS engine version on the system
     with the allowed minimum version.

cwltool/update.py

@@ -9,7 +9,6 @@
 
 import six
 from six.moves import urllib
-from six import u
 import schema_salad.validate
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.ref_resolver import Loader
@@ -62,7 +61,7 @@ def _draft2toDraft3dev1(doc, loader, baseuri, update_steps=True):
                 else:
                     raise Exception("Unexpected code path.")
                 r["id"] = imp
-                _, frag = u(urllib.parse.urldefrag(imp))
+                _, frag = urllib.parse.urldefrag(imp)
                 if frag:
                     frag = "#" + frag
                     r = findId(r, frag)
@@ -217,9 +216,7 @@ def _draftDraft3dev2toDev3(doc, loader, baseuri):
                     else:
                         raise Exception("Unexpected code path.")
                     r["id"] = imp
-                    # explicit conversion of urldfrag to unicode,
-                    # to avoid mypy error
-                    frag = u(urllib.parse.urldefrag(imp)[1])
+                    frag = urllib.parse.urldefrag(imp)[1]
                     if frag:
                         frag = "#" + frag
                         r = findId(r, frag)
@@ -267,7 +264,7 @@ def traverseImport(doc, loader, baseuri, func):
             else:
                 raise Exception("Unexpected code path.")
             r["id"] = imp
-            _, frag = urllib.parse.urldefrag(imp).decode('ascii')
+            _, frag = urllib.parse.urldefrag(imp)
             if frag:
                 frag = "#" + frag
                 r = findId(r, frag)
@@ -354,7 +351,7 @@ def _draft3toDraft4dev1(doc, loader, baseuri):
     if isinstance(doc, dict):
         if "class" in doc and doc["class"] == "Workflow":
             def fixup(f):  # type: (Text) -> Text
-                doc, frg = u(urllib.parse.urldefrag(f))
+                doc, frg = urllib.parse.urldefrag(f)
                 frg = '/'.join(frg.rsplit('.', 1))
                 return doc + "#" + frg
 

cwltool/workflow.py

@@ -676,7 +676,7 @@ def __init__(self, toolpath_object, pos, **kwargs):
         for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
             toolpath_object[toolfield] = []
             for n, step_entry in enumerate(toolpath_object[stepfield]):
-                if isinstance(step_entry, (str, six.text_type)):
+                if isinstance(step_entry, six.string_types):
                     param = CommentedMap()  # type: CommentedMap
                     inputid = step_entry
                 else: