Merge pull request #163 from common-workflow-language/update-types

update types

Author: mr-c

MANIFEST.in

@@ -1 +1 @@
-include gittaggers.py ez_setup.py
+include gittaggers.py ez_setup.py Makefile

Makefile

@@ -146,4 +146,15 @@ list-author-emails:
 	@echo 'name, E-Mail Address'
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
+
+mypy: ${PYSOURCES}
+	rm -Rf typeshed/2.7/ruamel/yaml
+	ln -s $(shell python -c 'import ruamel.yaml; import os.path; print os.path.dirname(ruamel.yaml.__file__)') \
+		typeshed/2.7/ruamel/yaml
+	rm -Rf typeshed/2.7/schema_salad
+	ln -s $(shell python -c 'import schema_salad; import os.path; print os.path.dirname(schema_salad.__file__)') \
+		typeshed/2.7/schema_salad
+	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
+		 --warn-redundant-casts --warn-unused-ignores cwltool
+
 FORCE:

cwltool/builder.py

@@ -3,7 +3,7 @@
 from . import expression
 import avro
 import schema_salad.validate as validate
-from typing import Any, Union, AnyStr, Callable
+from typing import Any, Union, AnyStr, Callable, Type
 from .errors import WorkflowException
 from .stdfsaccess import StdFsAccess
 from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs, normalizeFilesDirs
@@ -33,9 +33,10 @@ def __init__(self):  # type: () -> None
         self.timeout = None  # type: int
         self.pathmapper = None  # type: PathMapper
         self.stagedir = None  # type: unicode
+        self.make_fs_access = None  # type: Type[StdFsAccess]
 
     def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
-        # type: (Dict[unicode, Any], Any, List[int], List[int]) -> List[Dict[str, Any]]
+        # type: (Dict[unicode, Any], Any, Union[int, List[int]], List[int]) -> List[Dict[str, Any]]
         bindings = []  # type: List[Dict[str,str]]
         binding = None  # type: Dict[str,Any]
         if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
@@ -84,8 +85,11 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
                     if binding:
                         b2 = copy.deepcopy(binding)
                         b2["datum"] = item
-                    bindings.extend(self.bind_input({"type": schema["items"], "inputBinding": b2},
-                                                    item, lead_pos=n, tail_pos=tail_pos))
+                    bindings.extend(
+                            self.bind_input(
+                                {"type": schema["items"],
+                                 "inputBinding": b2},
+                                item, lead_pos=n, tail_pos=tail_pos))
                 binding = None
 
             if schema["type"] == "File":

cwltool/cwltest.py

@@ -141,8 +141,7 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
         failed = True
 
     if outdir:
-        shutil.rmtree(outdir, True)  # type: ignore
-        # Weird AnyStr != basestring issue
+        shutil.rmtree(outdir, True)
 
     if failed:
         return 1

cwltool/draft2tool.py

@@ -23,6 +23,7 @@
 from .builder import CONTENT_LIMIT, substitute, Builder, adjustFileObjs, adjustDirObjs
 from .pathmapper import PathMapper
 from .job import CommandLineJob
+from .stdfsaccess import StdFsAccess
 
 ACCEPTLIST_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
 
@@ -53,7 +54,8 @@ def run(self, **kwargs):  # type: (**Any) -> None
                 normalizeFilesDirs(ev)
                 self.output_callback(ev, "success")
             except Exception as e:
-                _logger.warn(u"Failed to evaluate expression:\n%s", e, exc_info=(e if kwargs.get('debug') else False))
+                _logger.warn(u"Failed to evaluate expression:\n%s",
+                        e, exc_info=kwargs.get('debug'))
                 self.output_callback({}, "permanentFail")
 
     def job(self, joborder, output_callback, **kwargs):
@@ -362,7 +364,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
         yield j
 
     def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
-        # type: (Set[Dict[str,Any]], Builder, str) -> Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
+        # type: (Set[Dict[str,Any]], Builder, str, bool) -> Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
         try:
             ret = {}  # type: Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
             fs_access = builder.make_fs_access(outdir)
@@ -377,8 +379,13 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
                     try:
                         ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum)
                     except Exception as e:
-                        _logger.debug(u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=e)
-                        raise WorkflowException(u"Error collecting output for parameter '%s': %s" % (shortname(port["id"]), e))
+                        _logger.debug(
+                                u"Error collecting output for parameter '%s'"
+                                % shortname(port["id"]), exc_info=True)
+                        raise WorkflowException(
+                                u"Error collecting output for " \
+                                        "parameter '%s': %s" %
+                                        (shortname(port["id"]), e))
 
             if ret:
                 adjustFileObjs(ret,
@@ -397,7 +404,7 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
             raise WorkflowException("Error validating output record, " + str(e) + "\n in " + json.dumps(ret, indent=4))
 
     def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=True):
-        # type: (Dict[str,Any], Builder, str) -> Union[Dict[unicode, Any], List[Union[Dict[unicode, Any], unicode]]]
+        # type: (Dict[str,Any], Builder, str, StdFsAccess, bool) -> Union[Dict[unicode, Any], List[Union[Dict[unicode, Any], unicode]]]
         r = []  # type: List[Any]
         if "outputBinding" in schema:
             binding = schema["outputBinding"]

cwltool/job.py

@@ -16,7 +16,8 @@
 import shellescape
 from .docker_uid import docker_vm_uid
 from .builder import Builder
-from typing import Union, Iterable, Callable, Any, Mapping, IO, cast, Tuple
+from typing import (Any, Callable, Union, Iterable, Mapping, MutableMapping,
+        IO, cast, Tuple)
 from .pathmapper import PathMapper
 import functools
 
@@ -57,7 +58,7 @@ def __init__(self):  # type: () -> None
         self.output_callback = None  # type: Callable[[Any, Any], Any]
         self.outdir = None  # type: str
         self.tmpdir = None  # type: str
-        self.environment = None  # type: Dict[str,str]
+        self.environment = None  # type: MutableMapping[str, str]
         self.generatefiles = None  # type: Dict[unicode, Union[List[Dict[str, str]], Dict[str,str], str]]
         self.stagedir = None  # type: unicode
 
@@ -82,6 +83,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
                 % (knownfile, self.pathmapper.mapper(knownfile)[0]))
 
         img_id = None
+        env = None  # type: MutableMapping[str, str]
         if docker_req and kwargs.get("use_container") is not False:
             env = os.environ
             img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)

cwltool/main.py

@@ -15,7 +15,7 @@
 import functools
 
 import rdflib
-from typing import Union, Any, cast, Callable, Dict, Tuple, IO
+from typing import Union, Any, cast, Callable, Dict, Tuple, Type, IO
 
 from schema_salad.ref_resolver import Loader
 import schema_salad.validate as validate
@@ -353,7 +353,7 @@ def generate_parser(toolparser, tool, namemap):
 
 def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
                    stdout=sys.stdout, make_fs_access=None):
-    # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]]
+    # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Type[StdFsAccess]) -> Union[int,Tuple[Dict[str,Any],str]]
 
     job_order_object = None
 
@@ -383,7 +383,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
         try:
             job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
         except Exception as e:
-            _logger.error(str(e), exc_info=(e if args.debug else False))
+            _logger.error(str(e), exc_info=args.debug)
             return 1
         toolparser = None
     else:
@@ -401,7 +401,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
                     input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
                     job_order_object = loader.resolve_ref(cmd_line["job_order"])
                 except Exception as e:
-                    _logger.error(str(e), exc_info=(e if args.debug else False))
+                    _logger.error(str(e), exc_info=args.debug)
                     return 1
             else:
                 job_order_object = {"id": args.workflow}
@@ -575,7 +575,7 @@ def main(argsl=None,
          versionfunc=versionstring,
          job_order_object=None,
          make_fs_access=StdFsAccess):
-    # type: (List[str], argparse.Namespace, Callable[..., Union[str, Dict[str, str]]], Callable[..., Process], Callable[[Dict[str, int]], Dict[str, int]], IO[Any], IO[Any], IO[Any], Callable[[], unicode], Union[int, Tuple[Dict[str, Any], str]]) -> int
+    # type: (List[str], argparse.Namespace, Callable[..., Union[str, Dict[str, str]]], Callable[..., Process], Callable[[Dict[str, int]], Dict[str, int]], IO[Any], IO[Any], IO[Any], Callable[[], unicode], Union[int, Tuple[Dict[str, Any], str]], Type[StdFsAccess]) -> int
 
     _logger.removeHandler(defaultStreamHandler)
     stderr_handler = logging.StreamHandler(stderr)
@@ -661,18 +661,18 @@ def main(argsl=None,
                     makeTool, {})
         except (validate.ValidationException) as exc:
             _logger.error(u"Tool definition failed validation:\n%s", exc,
-                          exc_info=(exc if args.debug else False))
+                          exc_info=args.debug)
             return 1
         except (RuntimeError, WorkflowException) as exc:
             _logger.error(u"Tool definition failed initialization:\n%s", exc,
-                          exc_info=(exc if args.debug else False))
+                          exc_info=args.debug)
             return 1
         except Exception as exc:
             _logger.error(
                 u"I'm sorry, I couldn't load this CWL file%s",
                 ", try again with --debug for more information.\nThe error was: "
                 "%s" % exc if not args.debug else ".  The error was:",
-                exc_info=(exc if args.debug else False))
+                exc_info=args.debug)
             return 1
 
         if isinstance(tool, int):
@@ -738,24 +738,23 @@ def locToPath(p):
             else:
                 return 1
         except (validate.ValidationException) as exc:
-            _logger.error(
-                u"Input object failed validation:\n%s", exc,
-                exc_info=(exc if args.debug else False))
+            _logger.error(u"Input object failed validation:\n%s", exc,
+                    exc_info=args.debug)
             return 1
         except UnsupportedRequirement as exc:
             _logger.error(
                 u"Workflow or tool uses unsupported feature:\n%s", exc,
-                exc_info=(exc if args.debug else False))
+                exc_info=args.debug)
             return 33
         except WorkflowException as exc:
             _logger.error(
                 u"Workflow error, try again with --debug for more "
-                "information:\n  %s", exc, exc_info=(exc if args.debug else False))
+                "information:\n  %s", exc, exc_info=args.debug)
             return 1
         except Exception as exc:
             _logger.error(
                 u"Unhandled error, try again with --debug for more information:\n"
-                "  %s", exc, exc_info=(exc if args.debug else False))
+                "  %s", exc, exc_info=args.debug)
             return 1
 
         return 0

cwltool/process.py

@@ -170,7 +170,7 @@ def getListing(fs_access, rec):
         rec["listing"] = listing
 
 def stageFiles(pm, stageFunc, ignoreWritable=False):
-    # type: (PathMapper, Callable[..., Any]) -> None
+    # type: (PathMapper, Callable[..., Any], bool) -> None
     for f, p in pm.items():
         if not os.path.exists(os.path.dirname(p.target)):
             os.makedirs(os.path.dirname(p.target), 0755)
@@ -374,9 +374,9 @@ def __init__(self, toolpath_object, **kwargs):
                     c["type"] = c["type"]
                 c["type"] = avroize_type(c["type"], c["name"])
                 if key == "inputs":
-                    self.inputs_record_schema["fields"].append(c)  # type: ignore
+                    self.inputs_record_schema["fields"].append(c)
                 elif key == "outputs":
-                    self.outputs_record_schema["fields"].append(c)  # type: ignore
+                    self.outputs_record_schema["fields"].append(c)
 
         try:
             self.inputs_record_schema = schema_salad.schema.make_valid_avro(self.inputs_record_schema, {}, set())

cwltool/stdfsaccess.py

@@ -30,8 +30,8 @@ def isdir(self, fn):  # type: (unicode) -> bool
     def listdir(self, fn):  # type: (unicode) -> List[unicode]
         return [abspath(l, fn) for l in os.listdir(self._abs(fn))]
 
-    def join(self, path, *paths): # type: (unicode, *unicode) -> unicode
+    def join(self, path, *paths):  # type: (unicode, *unicode) -> unicode
         return os.path.join(path, *paths)
 
-    def realpath(self, path):
+    def realpath(self, path):  # type: (str) -> str
         return os.path.realpath(path)

cwltool/update.py

@@ -191,6 +191,7 @@ def _draftDraft3dev2toDev3(doc, loader, baseuri):
                 else:
                     imp = urlparse.urljoin(baseuri, doc["@import"])
                     impLoaded = loader.fetch(imp)
+                    r = {}  # type: Dict[str, Any]
                     if isinstance(impLoaded, list):
                         r = {"@graph": impLoaded}
                     elif isinstance(impLoaded, dict):
@@ -236,6 +237,7 @@ def traverseImport(doc, loader, baseuri, func):
         else:
             imp = urlparse.urljoin(baseuri, doc["$import"])
             impLoaded = loader.fetch(imp)
+            r = {}  # type: Dict[str, Any]
             if isinstance(impLoaded, list):
                 r = {"$graph": impLoaded}
             elif isinstance(impLoaded, dict):