Merge branch 'master' into rm-intermediates

Author: keysmashes

cwltool/__main__.py

@@ -1,3 +1,4 @@
+"""Default entrypoint for the cwltool module."""
 from __future__ import absolute_import
 
 from . import main

cwltool/argparser.py

@@ -1,15 +1,15 @@
+"""Command line argument parsing for cwltool."""
 from __future__ import absolute_import, print_function
 
 import argparse
 import os
-from typing import (Any, AnyStr, Dict, List, Optional, Sequence,
-                    Union, cast, MutableMapping, MutableSequence)
+from typing import (Any, AnyStr, Dict, List, MutableMapping, MutableSequence,
+                    Optional, Sequence, Union, cast)
 
+from schema_salad.ref_resolver import file_uri
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
-from schema_salad.ref_resolver import file_uri
-
 from .loghandler import _logger
 from .process import Process, shortname  # pylint: disable=unused-import
 from .resolver import ga4gh_tool_registries

cwltool/builder.py

@@ -2,29 +2,31 @@
 
 import copy
 import logging
-from typing import (Any, Callable, Dict, List, Optional, Set, Union,
-                    Tuple, MutableMapping, MutableSequence)
-from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
+                    Optional, Set, Tuple, Union)
 
 from rdflib import Graph, URIRef  # pylint: disable=unused-import
 from rdflib.namespace import OWL, RDFS
 from ruamel.yaml.comments import CommentedMap
-import schema_salad.schema  # pylint: disable=unused-import
 from schema_salad import validate
-from schema_salad.schema import AvroSchemaFromJSONData
+from schema_salad.schema import AvroSchemaFromJSONData, Names
 from schema_salad.sourceline import SourceLine
 from six import iteritems, string_types
+from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
+                               Text, Type)
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from . import expression
 from .errors import WorkflowException
 from .loghandler import _logger
 from .mutation import MutationManager  # pylint: disable=unused-import
-from .pathmapper import (PathMapper,  # pylint: disable=unused-import
-                         get_listing, normalizeFilesDirs, visit_class)
+from .pathmapper import PathMapper  # pylint: disable=unused-import
+from .pathmapper import get_listing, normalizeFilesDirs, visit_class
 from .stdfsaccess import StdFsAccess  # pylint: disable=unused-import
-from .utils import (aslist, docker_windows_path_adjust,
-                    json_dumps, onWindows)
+from .utils import aslist, docker_windows_path_adjust, json_dumps, onWindows
+
+
+
 if TYPE_CHECKING:
     from .provenance import CreateProvProfile  # pylint: disable=unused-import
 CONTENT_LIMIT = 64 * 1024
@@ -111,7 +113,7 @@ def __init__(self,
                  files=None,                # type: List[Dict[Text, Text]]
                  bindings=None,             # type: List[Dict[Text, Any]]
                  schemaDefs=None,           # type: Dict[Text, Dict[Text, Any]]
-                 names=None,                # type: schema_salad.schema.Names
+                 names=None,                # type: Names
                  requirements=None,         # type: List[Dict[Text, Any]]
                  hints=None,                # type: List[Dict[Text, Any]]
                  timeout=None,              # type: float
@@ -131,7 +133,7 @@ def __init__(self,
                 ):  # type: (...) -> None
 
         if names is None:
-            self.names = schema_salad.schema.Names()
+            self.names = Names()
         else:
             self.names = names
 

cwltool/checker.py

@@ -1,11 +1,12 @@
+"""Static checking of CWL workflow connectivity."""
 from collections import namedtuple
-from typing import Any, Dict, List, Optional, MutableMapping, MutableSequence
-from typing_extensions import Text  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from typing import Any, Dict, List, MutableMapping, MutableSequence, Optional
 
+import six
 from schema_salad import validate
 from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno
-import six
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from .errors import WorkflowException
 from .loghandler import _logger
@@ -79,24 +80,22 @@ def can_assign_src_to_sink(src, sink, strict=False):  # type: (Any, Any, bool) -
                         return False
             return True
         return can_assign_src_to_sink(src["type"], sink["type"], strict)
-    elif isinstance(src, MutableSequence):
+    if isinstance(src, MutableSequence):
         if strict:
-            for t in src:
-                if not can_assign_src_to_sink(t, sink):
+            for this_src in src:
+                if not can_assign_src_to_sink(this_src, sink):
                     return False
             return True
-        else:
-            for t in src:
-                if can_assign_src_to_sink(t, sink):
-                    return True
-            return False
-    elif isinstance(sink, MutableSequence):
-        for t in sink:
-            if can_assign_src_to_sink(src, t):
+        for this_src in src:
+            if can_assign_src_to_sink(this_src, sink):
+                return True
+        return False
+    if isinstance(sink, MutableSequence):
+        for this_sink in sink:
+            if can_assign_src_to_sink(src, this_sink):
                 return True
         return False
-    else:
-        return src == sink
+    return src == sink
 
 
 def _compare_records(src, sink, strict=False):

cwltool/command_line_tool.py

@@ -1,3 +1,4 @@
+"""Implementation of CommandLineTool."""
 from __future__ import absolute_import
 
 import copy
@@ -11,20 +12,24 @@
 import tempfile
 import threading
 from functools import cmp_to_key, partial
-from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Union,
-                    cast, MutableMapping, MutableSequence)
-from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from typing import (Any, Callable, Dict, Generator, List, MutableMapping,
+                    MutableSequence, Optional, Set, Union, cast)
 
+import shellescape
 from schema_salad import validate
 from schema_salad.ref_resolver import file_uri, uri_file_path
 from schema_salad.sourceline import SourceLine
-import shellescape
 from six import string_types
+
 from six.moves import map, urllib
+from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
+                               Text, Type)
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from .builder import (CONTENT_LIMIT, Builder,  # pylint: disable=unused-import
                       substitute)
+from .context import LoadingContext  # pylint: disable=unused-import
+from .context import RuntimeContext, getdefault
 from .docker import DockerCommandLineJob
 from .errors import WorkflowException
 from .flatten import flatten
@@ -43,8 +48,6 @@
 from .utils import (aslist, convert_pathsep_to_unix,
                     docker_windows_path_adjust, json_dumps, onWindows,
                     windows_default_container_id)
-from .context import (LoadingContext,  # pylint: disable=unused-import
-                      RuntimeContext, getdefault)
 if TYPE_CHECKING:
     from .provenance import CreateProvProfile  # pylint: disable=unused-import
 

cwltool/context.py

@@ -1,19 +1,22 @@
+"""Shared context objects that replace use of kwargs."""
 import copy
 import threading  # pylint: disable=unused-import
-from typing import Any, Callable, Dict, Iterable, List, MutableMapping, Optional
-from typing_extensions import Text, TYPE_CHECKING  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
-from schema_salad.ref_resolver import (  # pylint: disable=unused-import
-    ContextType, Fetcher, Loader)
-from schema_salad import schema
+from typing import (Any, Callable, Dict, Iterable, List, MutableMapping,
+                    Optional)
 
-from .utils import DEFAULT_TMP_PREFIX
-from .stdfsaccess import StdFsAccess
+from schema_salad import schema
+from schema_salad.ref_resolver import (ContextType,  # pylint: disable=unused-import
+                                       Fetcher, Loader)
+from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
+                               Text)
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 from .builder import Builder, HasReqsHints
 from .mutation import MutationManager
-from .software_requirements import DependenciesConfiguration
-from .secrets import SecretStore
 from .pathmapper import PathMapper
+from .secrets import SecretStore
+from .software_requirements import DependenciesConfiguration
+from .stdfsaccess import StdFsAccess
+from .utils import DEFAULT_TMP_PREFIX
 
 if TYPE_CHECKING:
     from .process import Process

cwltool/cwlrdf.py

@@ -1,13 +1,13 @@
 from __future__ import absolute_import
 
 from typing import IO, Any, Dict
-from typing_extensions import Text  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from rdflib import Graph
 from schema_salad.jsonld_context import makerdf
 from schema_salad.ref_resolver import ContextType
 from six.moves import urllib
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from .process import Process
 

cwltool/docker.py

@@ -1,3 +1,4 @@
+"""Enables Docker software containers via the {dx-,u,}docker runtimes."""
 from __future__ import absolute_import
 
 import datetime
@@ -9,21 +10,22 @@
 import threading
 from io import open  # pylint: disable=redefined-builtin
 from typing import Dict, List, MutableMapping, Optional, Set
-from typing_extensions import Text  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 import requests
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
+from .context import RuntimeContext  # pylint: disable=unused-import
 from .docker_id import docker_vm_id
 from .errors import WorkflowException
 from .job import ContainerCommandLineJob
 from .loghandler import _logger
-from .pathmapper import (PathMapper,  # pylint: disable=unused-import
-                         ensure_writable)
+from .pathmapper import PathMapper  # pylint: disable=unused-import
+from .pathmapper import ensure_writable
 from .secrets import SecretStore  # pylint: disable=unused-import
 from .utils import (DEFAULT_TMP_PREFIX, docker_windows_path_adjust, onWindows,
                     subprocess)
-from .context import RuntimeContext  # pylint: disable=unused-import
+
 
 _IMAGES = set()  # type: Set[Text]
 _IMAGES_LOCK = threading.Lock()

cwltool/docker_id.py

@@ -1,6 +1,8 @@
+"""Helper functions for docker."""
 from __future__ import absolute_import, print_function
 
 from typing import List, Optional, Tuple  # pylint: disable=unused-import
+
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported