mypyc prep

Author: mr-c

cwltool/builder.py

@@ -19,12 +19,13 @@
 from rdflib import Graph, URIRef
 from rdflib.namespace import OWL, RDFS
 from ruamel.yaml.comments import CommentedMap
+from typing_extensions import TYPE_CHECKING, Type  # pylint: disable=unused-import
+
 from schema_salad import validate
 from schema_salad.avro.schema import Schema, make_avsc_object
 from schema_salad.schema import Names, convert_to_dict
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
-from typing_extensions import TYPE_CHECKING, Type  # pylint: disable=unused-import
 
 from . import expression
 from .errors import WorkflowException

cwltool/command_line_tool.py

@@ -29,12 +29,14 @@
 )
 
 import shellescape
+from ruamel.yaml.comments import CommentedMap
+from typing_extensions import TYPE_CHECKING, Type
+
 from schema_salad import validate
 from schema_salad.avro.schema import Schema
 from schema_salad.ref_resolver import file_uri, uri_file_path
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
-from typing_extensions import TYPE_CHECKING, Type
 
 from .builder import Builder, content_limit_respected_read_bytes, substitute
 from .context import LoadingContext, RuntimeContext, getdefault
@@ -327,7 +329,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: Dict[str, Any]) -> None:
 
 class CommandLineTool(Process):
     def __init__(
-        self, toolpath_object: MutableMapping[str, Any], loadingContext: LoadingContext
+        self, toolpath_object: CommentedMap, loadingContext: LoadingContext
     ) -> None:
         """Initialize this CommandLineTool."""
         super(CommandLineTool, self).__init__(toolpath_object, loadingContext)

cwltool/context.py

@@ -5,10 +5,11 @@
 
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 from ruamel.yaml.comments import CommentedMap
-from schema_salad import schema
-from schema_salad.ref_resolver import Loader
 from typing_extensions import TYPE_CHECKING
 
+from schema_salad.ref_resolver import Loader
+from schema_salad.schema import Names
+
 from .builder import Builder, HasReqsHints
 from .mutation import MutationManager
 from .pathmapper import PathMapper
@@ -35,9 +36,8 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
 
 
 def make_tool_notimpl(
-    toolpath_object,  # type: MutableMapping[str, Any]
-    loadingContext,  # type: LoadingContext
-):  # type: (...) -> Process
+    toolpath_object: CommentedMap, loadingContext: "LoadingContext"
+) -> "Process":
     raise NotImplementedError()
 
 
@@ -53,7 +53,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.hints = None  # type: Optional[List[Dict[str, Any]]]
         self.overrides_list = []  # type: List[Dict[str, Any]]
         self.loader = None  # type: Optional[Loader]
-        self.avsc_names = None  # type: Optional[schema.Names]
+        self.avsc_names = None  # type: Optional[Names]
         self.disable_js_validation = False  # type: bool
         self.js_hint_options_file = None
         self.do_validate = True  # type: bool

cwltool/cwlrdf.py

@@ -2,6 +2,8 @@
 from typing import IO, Any, Dict, MutableMapping, cast
 
 from rdflib import Graph
+from ruamel.yaml.comments import CommentedMap
+
 from schema_salad.jsonld_context import makerdf
 from schema_salad.ref_resolver import ContextType
 
@@ -11,7 +13,7 @@
 def gather(tool, ctx):  # type: (Process, ContextType) -> Graph
     g = Graph()
 
-    def visitor(t):  # type: (MutableMapping[str, Any]) -> None
+    def visitor(t):  # type: (CommentedMap) -> None
         makerdf(t["id"], t, ctx, graph=g)
 
     tool.visit(visitor)

cwltool/executors.py

@@ -20,6 +20,7 @@
 )
 
 import psutil
+
 from schema_salad.sourceline import SourceLine
 from schema_salad.validate import ValidationException
 

cwltool/job.py

@@ -34,9 +34,10 @@
 import psutil
 import shellescape
 from prov.model import PROV
+from typing_extensions import TYPE_CHECKING
+
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dump, json_dumps
-from typing_extensions import TYPE_CHECKING
 
 from .builder import Builder, HasReqsHints
 from .context import RuntimeContext, getdefault

cwltool/load_tool.py

@@ -21,8 +21,17 @@
 
 import requests.sessions
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from schema_salad import schema
-from schema_salad.ref_resolver import ContextType, Fetcher, Loader, SubLoader, file_uri
+
+from schema_salad.ref_resolver import (
+    ContextType,
+    Fetcher,
+    FetcherCallableType,
+    IdxResultType,
+    Loader,
+    ResolveType,
+    file_uri,
+)
+from schema_salad.schema import validate_doc
 from schema_salad.sourceline import SourceLine, cmap
 from schema_salad.utils import json_dumps
 from schema_salad.validate import ValidationException
@@ -58,14 +67,11 @@
 }  # type: ContextType
 
 
-FetcherConstructorType = Callable[
-    [Dict[str, Union[str, bool]], requests.sessions.Session], Fetcher
-]
 ResolverType = Callable[[Loader, Union[str, Dict[str, Any]]], str]
 
 
 def default_loader(fetcher_constructor=None, enable_dev=False, doc_cache=True):
-    # type: (Optional[FetcherConstructorType], bool, bool) -> Loader
+    # type: (Optional[FetcherCallableType], bool, bool) -> Loader
     return Loader(
         jobloaderctx,
         fetcher_constructor=fetcher_constructor,
@@ -77,7 +83,7 @@ def default_loader(fetcher_constructor=None, enable_dev=False, doc_cache=True):
 def resolve_tool_uri(
     argsworkflow: str,
     resolver: Optional[ResolverType] = None,
-    fetcher_constructor: Optional[FetcherConstructorType] = None,
+    fetcher_constructor: Optional[FetcherCallableType] = None,
     document_loader: Optional[Loader] = None,
 ) -> Tuple[str, str]:
 
@@ -126,7 +132,7 @@ def fetch_document(
             resolver=loadingContext.resolver,
             document_loader=loadingContext.loader,
         )
-        workflowobj = loadingContext.loader.fetch(fileuri)
+        workflowobj = cast(CommentedMap, loadingContext.loader.fetch(fileuri))
         return loadingContext, workflowobj, uri
     if isinstance(argsworkflow, dict):
         uri = argsworkflow["id"] if argsworkflow.get("id") else "_:" + str(uuid.uuid4())
@@ -318,7 +324,7 @@ def resolve_and_validate_document(
     if isinstance(avsc_names, Exception):
         raise avsc_names
 
-    processobj = None  # type: Union[CommentedMap, CommentedSeq, str, None]
+    processobj = None  # type: Optional[ResolveType]
     document_loader = Loader(
         sch_document_loader.ctx,
         schemagraph=sch_document_loader.graph,
@@ -359,9 +365,7 @@ def resolve_and_validate_document(
         return loadingContext, uri
 
     if loadingContext.do_validate:
-        schema.validate_doc(
-            avsc_names, processobj, document_loader, loadingContext.strict
-        )
+        validate_doc(avsc_names, processobj, document_loader, loadingContext.strict)
 
     # None means default behavior (do update)
     if loadingContext.do_update in (True, None):
@@ -436,7 +440,7 @@ def load_tool(
 
 
 def resolve_overrides(
-    ov,  # type: CommentedMap
+    ov,  # type: IdxResultType
     ov_uri,  # type: str
     baseurl,  # type: str
 ):  # type: (...) -> List[Dict[str, Any]]

cwltool/main.py

@@ -12,7 +12,7 @@
 import time
 import urllib
 from codecs import StreamWriter, getwriter
-from collections.abc import Iterable, MutableSequence, Sequence
+from collections.abc import Iterable, MutableMapping, MutableSequence, Sequence
 from typing import (
     IO,
     Any,
@@ -34,10 +34,11 @@
 import pkg_resources  # part of setuptools
 from ruamel import yaml
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from schema_salad import validate
+
 from schema_salad.ref_resolver import Fetcher, Loader, file_uri, uri_file_path
-from schema_salad.sourceline import cmap, strip_dup_lineno
+from schema_salad.sourceline import strip_dup_lineno
 from schema_salad.utils import json_dumps
+from schema_salad.validate import ValidationException
 
 from . import command_line_tool, workflow
 from .argparser import arg_parser, generate_parser, get_default_args
@@ -737,7 +738,7 @@ def choose_target(
     else:
         _logger.error("Can only use --target on Workflows")
         return None
-    if isinstance(loadingContext.loader.idx, CommentedMap):
+    if isinstance(loadingContext.loader.idx, MutableMapping):
         loadingContext.loader.idx[extracted["id"]] = extracted
         tool = make_tool(extracted["id"], loadingContext)
     else:
@@ -989,7 +990,7 @@ def main(
                 )
                 return 0
 
-        except (validate.ValidationException) as exc:
+        except (ValidationException) as exc:
             _logger.error(
                 "Tool definition failed validation:\n%s", str(exc), exc_info=args.debug
             )
@@ -1158,7 +1159,7 @@ def loc_to_path(obj):  # type: (Dict[str, Any]) -> None
             _logger.info("Final process status is %s", status)
             return 0
 
-        except (validate.ValidationException) as exc:
+        except (ValidationException) as exc:
             _logger.error(
                 "Input object failed validation:\n%s", str(exc), exc_info=args.debug
             )

cwltool/pack.py

@@ -16,8 +16,8 @@
 )
 
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from schema_salad.ref_resolver import Loader, SubLoader
-from schema_salad.sourceline import cmap
+
+from schema_salad.ref_resolver import Loader, ResolveType, SubLoader
 
 from .context import LoadingContext
 from .load_tool import fetch_document, resolve_and_validate_document
@@ -37,9 +37,7 @@ def flatten_deps(d, files):  # type: (Any, Set[str]) -> None
             flatten_deps(d["listing"], files)
 
 
-LoadRefType = Callable[
-    [Optional[str], str], Union[Dict[str, Any], List[Dict[str, Any]], str, None]
-]
+LoadRefType = Callable[[Optional[str], str], ResolveType]
 
 
 def find_run(
@@ -165,7 +163,7 @@ def pack(
         document_loader.idx[metadata["id"]] = CommentedMap(metadata.items())
 
     def loadref(base, uri):
-        # type: (Optional[str], str) -> Union[Dict[str, Any], List[Dict[str, Any]], str, None]
+        # type: (Optional[str], str) -> ResolveType
         return document_loader.resolve_ref(uri, base_url=base)[0]
 
     ids = set()  # type: Set[str]

cwltool/pathmapper.py

@@ -23,9 +23,10 @@
 import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches import FileCache
-from schema_salad import validate
+
 from schema_salad.ref_resolver import uri_file_path
 from schema_salad.sourceline import SourceLine
+from schema_salad.validate import ValidationException
 
 from .loghandler import _logger
 from .stdfsaccess import StdFsAccess, abspath
@@ -71,13 +72,13 @@ def normalizeFilesDirs(
     def addLocation(d):  # type: (Dict[str, Any]) -> None
         if "location" not in d:
             if d["class"] == "File" and ("contents" not in d):
-                raise validate.ValidationException(
+                raise ValidationException(
                     "Anonymous file object must have 'contents' and 'basename' fields."
                 )
             if d["class"] == "Directory" and (
                 "listing" not in d or "basename" not in d
             ):
-                raise validate.ValidationException(
+                raise ValidationException(
                     "Anonymous directory object must have 'listing' and 'basename' fields."
                 )
             d["location"] = "_:" + str(uuid.uuid4())
@@ -89,7 +90,7 @@ def addLocation(d):  # type: (Dict[str, Any]) -> None
         # strip trailing slash
         if path.endswith("/"):
             if d["class"] != "Directory":
-                raise validate.ValidationException(
+                raise ValidationException(
                     "location '%s' ends with '/' but is not a Directory" % d["location"]
                 )
             path = path.rstrip("/")
@@ -120,7 +121,7 @@ def addLocation(d):  # type: (Dict[str, Any]) -> None
             contents = d.get("contents")
             if contents and len(contents) > CONTENT_LIMIT:
                 if len(contents) > CONTENT_LIMIT:
-                    raise validate.ValidationException(
+                    raise ValidationException(
                         "File object contains contents with number of bytes that exceeds CONTENT_LIMIT length (%d)"
                         % CONTENT_LIMIT
                     )
@@ -336,7 +337,7 @@ def visit(self, obj, stagedir, basedir, copy=False, staged=False):
                 with SourceLine(
                     obj,
                     "location",
-                    validate.ValidationException,
+                    ValidationException,
                     _logger.isEnabledFor(logging.DEBUG),
                 ):
                     deref = ab