fix for Python 3.5.{01} (#860)

Typing.Text doesn't exist in this version of Python

And many pylint inspired cleanups

Author: mr-c

cwltool/__main__.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-import sys
-
 from . import main
 
 main.run()

cwltool/argparser.py

@@ -2,8 +2,10 @@
 
 import argparse
 import os
-from typing import (Any, AnyStr, Dict, List,  # pylint: disable=unused-import
-                    Optional, Sequence, Text, Union, cast)
+from typing import Any, AnyStr, Dict, List, Optional, Sequence, Union, cast
+
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from schema_salad.ref_resolver import file_uri
 
@@ -25,44 +27,43 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         help="[experimental] Run jobs in parallel. ")
     envgroup = parser.add_mutually_exclusive_group()
     envgroup.add_argument("--preserve-environment", type=Text, action="append",
-                        help="Preserve specific environment variable when "
-                        "running CommandLineTools.  May be provided multiple "
-                        "times.", metavar="ENVVAR", default=["PATH"],
-                        dest="preserve_environment")
+                          help="Preserve specific environment variable when "
+                          "running CommandLineTools.  May be provided multiple "
+                          "times.", metavar="ENVVAR", default=["PATH"],
+                          dest="preserve_environment")
     envgroup.add_argument("--preserve-entire-environment", action="store_true",
-                        help="Preserve all environment variable when running "
-                        "CommandLineTools.", default=False,
-                        dest="preserve_entire_environment")
+                          help="Preserve all environment variable when running "
+                          "CommandLineTools.", default=False,
+                          dest="preserve_entire_environment")
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--rm-container", action="store_true", default=True,
                          help="Delete Docker container used by jobs after they exit (default)",
                          dest="rm_container")
 
-    exgroup.add_argument("--leave-container", action="store_false",
-                         default=True, help="Do not delete Docker container used by jobs after they exit",
-                         dest="rm_container")
+    exgroup.add_argument(
+        "--leave-container", action="store_false", default=True,
+        help="Do not delete Docker container used by jobs after they exit",
+        dest="rm_container")
 
-    cidgroup = parser.add_argument_group("Options for recording the Docker "
-        "container identifier into a file")
+    cidgroup = parser.add_argument_group(
+        "Options for recording the Docker container identifier into a file")
     cidgroup.add_argument("--record-container-id", action="store_true",
                           default=False,
                           help="If enabled, store the Docker container ID into a file. "
                           "See --cidfile-dir to specify the directory.",
                           dest="record_container_id")
 
-    cidgroup.add_argument("--cidfile-dir", type=Text,
-                        help="Directory for storing the Docker container ID file. "
-                             "The default is the current directory",
-                        default="",
-                        dest="cidfile_dir")
+    cidgroup.add_argument(
+        "--cidfile-dir", type=Text, help="Directory for storing the Docker "
+        "container ID file. The default is the current directory",
+        default="", dest="cidfile_dir")
 
-    cidgroup.add_argument("--cidfile-prefix", type=Text,
-                        help="Specify a prefix to the container ID filename. "
-                             "Final file name will be followed by a timestamp. "
-                             "The default is no prefix.",
-                        default="",
-                        dest="cidfile_prefix")
+    cidgroup.add_argument(
+        "--cidfile-prefix", type=Text,
+        help="Specify a prefix to the container ID filename. "
+        "Final file name will be followed by a timestamp. The default is no prefix.",
+        default="", dest="cidfile_prefix")
 
     parser.add_argument("--tmpdir-prefix", type=Text,
                         help="Path prefix for temporary directories",
@@ -73,8 +74,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                          help="Path prefix for intermediate output directories",
                          default=DEFAULT_TMP_PREFIX)
 
-    exgroup.add_argument("--cachedir", type=Text, default="",
-                         help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")
+    exgroup.add_argument(
+        "--cachedir", type=Text, default="",
+        help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--rm-tmpdir", action="store_true", default=True,
@@ -86,9 +88,10 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                          dest="rm_tmpdir")
 
     exgroup = parser.add_mutually_exclusive_group()
-    exgroup.add_argument("--move-outputs", action="store_const", const="move", default="move",
-                         help="Move output files to the workflow output directory and delete intermediate output directories (default).",
-                         dest="move_outputs")
+    exgroup.add_argument(
+        "--move-outputs", action="store_const", const="move", default="move",
+        help="Move output files to the workflow output directory and delete "
+        "intermediate output directories (default).", dest="move_outputs")
 
     exgroup.add_argument("--leave-outputs", action="store_const", const="leave", default="move",
                          help="Leave output files in intermediate output directories.",

cwltool/builder.py

@@ -2,13 +2,14 @@
 
 import copy
 import logging
-from typing import (Any, Callable, Dict, List,  # pylint: disable=unused-import
-                    Optional, Set, Text, Type, Union, Tuple, TYPE_CHECKING)
+from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple
+from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from rdflib import Graph, URIRef  # pylint: disable=unused-import
 from rdflib.namespace import OWL, RDFS
 import schema_salad.schema  # pylint: disable=unused-import
-import schema_salad.validate as validate
+from schema_salad import validate
 from schema_salad.schema import AvroSchemaFromJSONData
 from schema_salad.sourceline import SourceLine
 from six import iteritems, string_types

cwltool/checker.py

@@ -1,9 +1,9 @@
-import logging
 from collections import namedtuple
-from typing import (Any, Callable, Dict,  # pylint: disable=unused-import
-                    Generator, Iterable, List, Optional, Text, Union, cast)
+from typing import Any, Dict, List, Optional
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
-import schema_salad.validate as validate
+from schema_salad import validate
 from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno
 import six
 
@@ -27,19 +27,18 @@ def check_types(srctype, sinktype, linkMerge, valueFrom):
 
     if valueFrom:
         return "pass"
-    elif not linkMerge:
+    if not linkMerge:
         if can_assign_src_to_sink(srctype, sinktype, strict=True):
             return "pass"
-        elif can_assign_src_to_sink(srctype, sinktype, strict=False):
+        if can_assign_src_to_sink(srctype, sinktype, strict=False):
             return "warning"
-        else:
-            return "exception"
-    elif linkMerge == "merge_nested":
-        return check_types({"items": _get_type(srctype), "type": "array"}, _get_type(sinktype), None, None)
-    elif linkMerge == "merge_flattened":
+        return "exception"
+    if linkMerge == "merge_nested":
+        return check_types({"items": _get_type(srctype), "type": "array"},
+                           _get_type(sinktype), None, None)
+    if linkMerge == "merge_flattened":
         return check_types(merge_flatten_type(_get_type(srctype)), _get_type(sinktype), None, None)
-    else:
-        raise WorkflowException(u"Unrecognized linkMerge enu_m '%s'" % linkMerge)
+    raise WorkflowException(u"Unrecognized linkMerge enu_m '{}'".format(linkMerge))
 
 
 def merge_flatten_type(src):
@@ -49,10 +48,9 @@ def merge_flatten_type(src):
 
     if isinstance(src, list):
         return [merge_flatten_type(t) for t in src]
-    elif isinstance(src, dict) and src.get("type") == "array":
+    if isinstance(src, dict) and src.get("type") == "array":
         return src
-    else:
-        return {"items": src, "type": "array"}
+    return {"items": src, "type": "array"}
 
 
 def can_assign_src_to_sink(src, sink, strict=False):  # type: (Any, Any, bool) -> bool
@@ -72,16 +70,15 @@ def can_assign_src_to_sink(src, sink, strict=False):  # type: (Any, Any, bool) -
             return False
         if src["type"] == "array" and sink["type"] == "array":
             return can_assign_src_to_sink(src["items"], sink["items"], strict)
-        elif src["type"] == "record" and sink["type"] == "record":
+        if src["type"] == "record" and sink["type"] == "record":
             return _compare_records(src, sink, strict)
-        elif src["type"] == "File" and sink["type"] == "File":
+        if src["type"] == "File" and sink["type"] == "File":
             for sinksf in sink.get("secondaryFiles", []):
                 if not [1 for srcsf in src.get("secondaryFiles", []) if sinksf == srcsf]:
                     if strict:
                         return False
             return True
-        else:
-            return can_assign_src_to_sink(src["type"], sink["type"], strict)
+        return can_assign_src_to_sink(src["type"], sink["type"], strict)
     elif isinstance(src, list):
         if strict:
             for t in src:
@@ -122,11 +119,11 @@ def _rec_fields(rec):  # type: (Dict[Text, Any]) -> Dict[Text, Any]
     for key in six.iterkeys(sinkfields):
         if (not can_assign_src_to_sink(
                 srcfields.get(key, "null"), sinkfields.get(key, "null"), strict)
-            and sinkfields.get(key) is not None):
+                and sinkfields.get(key) is not None):
             _logger.info("Record comparison failure for %s and %s\n"
-                         "Did not match fields for %s: %s and %s" %
-                         (src["name"], sink["name"], key, srcfields.get(key),
-                          sinkfields.get(key)))
+                         "Did not match fields for %s: %s and %s",
+                         src["name"], sink["name"], key, srcfields.get(key),
+                         sinkfields.get(key))
             return False
     return True
 
@@ -156,7 +153,8 @@ def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs,
         src = warning.src
         sink = warning.sink
         linkMerge = warning.linkMerge
-        if sink.get("secondaryFiles") and sorted(sink.get("secondaryFiles",[])) != sorted(src.get("secondaryFiles",[])):
+        if sink.get("secondaryFiles") and sorted(
+                sink.get("secondaryFiles", [])) != sorted(src.get("secondaryFiles", [])):
             msg1 = "Sink '%s'" % (shortname(sink["id"]))
             msg2 = SourceLine(sink.get("_tool_entry", sink), "secondaryFiles").makeError(
                 "expects secondaryFiles: %s but" % (sink.get("secondaryFiles")))
@@ -202,7 +200,7 @@ def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs,
 
     for sink in step_inputs:
         if ('null' != sink["type"] and 'null' not in sink["type"]
-            and "source" not in sink and "default" not in sink and "valueFrom" not in sink):
+                and "source" not in sink and "default" not in sink and "valueFrom" not in sink):
             msg = SourceLine(sink).makeError(
                 "Required parameter '%s' does not have source, default, or valueFrom expression"
                 % shortname(sink["id"]))
@@ -212,7 +210,7 @@ def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs,
     all_exception_msg = strip_dup_lineno("\n".join(exception_msgs))
 
     if warnings:
-        _logger.warning("Workflow checker warning:\n%s" % all_warning_msg)
+        _logger.warning("Workflow checker warning:\n%s", all_warning_msg)
     if exceptions:
         raise validate.ValidationException(all_exception_msg)
 

cwltool/command_line_tool.py

@@ -11,11 +11,12 @@
 import tempfile
 import threading
 from functools import cmp_to_key, partial
-from typing import (Any, Callable, Dict,  # pylint: disable=unused-import
-                    Generator, List, Optional, Set, Text, Type, TYPE_CHECKING,
-                    Union, cast)
+from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Union,
+                    cast)
+from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
-import schema_salad.validate as validate
+from schema_salad import validate
 from schema_salad.ref_resolver import file_uri, uri_file_path
 from schema_salad.sourceline import SourceLine
 import shellescape
@@ -92,9 +93,9 @@ def run(self, runtimeContext):  # type: (RuntimeContext) -> None
                 ev = self.builder.do_eval(self.script)
                 normalizeFilesDirs(ev)
                 self.output_callback(ev, "success")
-            except Exception as e:
+            except Exception as err:
                 _logger.warning(u"Failed to evaluate expression:\n%s",
-                             e, exc_info=runtimeContext.debug)
+                                err, exc_info=runtimeContext.debug)
                 self.output_callback({}, "permanentFail")
 
     def job(self,
@@ -238,21 +239,22 @@ def make_job_runner(self,
                         "dockerPull": default_container
                     })
                     dockerReq = self.requirements[0]
-                    if default_container == windows_default_container_id and runtimeContext.use_container and onWindows():
-                        _logger.warning(DEFAULT_CONTAINER_MSG % (windows_default_container_id, windows_default_container_id))
+                    if default_container == windows_default_container_id \
+                            and runtimeContext.use_container and onWindows():
+                        _logger.warning(
+                            DEFAULT_CONTAINER_MSG, windows_default_container_id,
+                            windows_default_container_id)
 
         if dockerReq and runtimeContext.use_container:
             if runtimeContext.singularity:
                 return SingularityCommandLineJob
-            else:
-                return DockerCommandLineJob
-        else:
-            for t in reversed(self.requirements):
-                if t["class"] == "DockerRequirement":
-                    raise UnsupportedRequirement(
-                        "--no-container, but this CommandLineTool has "
-                        "DockerRequirement under 'requirements'.")
-            return CommandLineJob
+            return DockerCommandLineJob
+        for t in reversed(self.requirements):
+            if t["class"] == "DockerRequirement":
+                raise UnsupportedRequirement(
+                    "--no-container, but this CommandLineTool has "
+                    "DockerRequirement under 'requirements'.")
+        return CommandLineJob
 
     def make_path_mapper(self, reffiles, stagedir, runtimeContext, separateDirs):
         # type: (List[Any], Text, RuntimeContext, bool) -> PathMapper

cwltool/context.py

@@ -1,20 +1,19 @@
 import copy
 import threading  # pylint: disable=unused-import
+from typing import Any, Callable, Dict, Iterable, List, Optional
+from typing_extensions import Text, TYPE_CHECKING  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from schema_salad.ref_resolver import (  # pylint: disable=unused-import
+    ContextType, Fetcher, Loader)
+from schema_salad import schema
 
 from .utils import DEFAULT_TMP_PREFIX
 from .stdfsaccess import StdFsAccess
-from typing import (Any, Callable, Dict,  # pylint: disable=unused-import
-                    Generator, Iterable, List, Optional, Text, Union, AnyStr)
-from schema_salad.ref_resolver import (  # pylint: disable=unused-import
-    ContextType, Fetcher, Loader)
-import schema_salad.schema as schema
 from .builder import Builder, HasReqsHints
 from .mutation import MutationManager
 from .software_requirements import DependenciesConfiguration
 from .secrets import SecretStore
-import six
 
-from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from .process import Process
     from .provenance import (ResearchObject,  # pylint: disable=unused-import

cwltool/cwlrdf.py

@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 
-from typing import IO, Any, Dict, Text
+from typing import IO, Any, Dict
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from rdflib import Graph
 from schema_salad.jsonld_context import makerdf
@@ -33,8 +35,7 @@ def lastpart(uri):  # type: (Any) -> Text
     uri = Text(uri)
     if "/" in uri:
         return uri[uri.rindex("/") + 1:]
-    else:
-        return uri
+    return uri
 
 
 def dot_with_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
@@ -45,7 +46,7 @@ def dot_with_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
               ?run rdf:type ?runtype .
            }""")
 
-    for step, run, runtype in qres:
+    for step, run, _ in qres:
         stdout.write(u'"%s" [label="%s"]\n' % (lastpart(step), "%s (%s)" % (lastpart(step), lastpart(run))))
 
     qres = g.query(

cwltool/docker.py

@@ -8,8 +8,9 @@
 import tempfile
 import threading
 from io import open  # pylint: disable=redefined-builtin
-from typing import (Any, Dict, List,  # pylint: disable=unused-import
-                    MutableMapping, Optional, Set, Text)
+from typing import Dict, List, MutableMapping, Optional, Set
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 import requests
 
@@ -132,7 +133,7 @@ def get_image(docker_requirement,     # type: Dict[Text, Text]
             elif "dockerFile" in docker_requirement:
                 dockerfile_dir = str(tempfile.mkdtemp(prefix=tmp_outdir_prefix))
                 with open(os.path.join(
-                    dockerfile_dir, "Dockerfile"), "wb") as dfile:
+                        dockerfile_dir, "Dockerfile"), "wb") as dfile:
                     dfile.write(docker_requirement["dockerFile"].encode('utf-8'))
                 cmd = ["docker", "build", "--tag=%s" %
                        str(docker_requirement["dockerImageId"]), dockerfile_dir]