Skip to content

Commit 0a0c90c

Browse files
mr-cmr-c
committed
update types
1 parent d6532bb commit 0a0c90c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+2737
-38
lines changed

‎MANIFEST.in

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
include gittaggers.py ez_setup.py
1+
include gittaggers.py ez_setup.py Makefile

‎Makefile

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,4 +146,15 @@ list-author-emails:
146146
@echo 'name, E-Mail Address'
147147
@git log --format='%aN,%aE' | sort -u | grep -v 'root'
148148

149+
150+
mypy: ${PYSOURCES}
151+
rm -Rf typeshed/2.7/ruamel/yaml
152+
ln -s $(shell python -c 'import ruamel.yaml; import os.path; print os.path.dirname(ruamel.yaml.__file__)') \
153+
typeshed/2.7/ruamel/yaml
154+
rm -Rf typeshed/2.7/schema_salad
155+
ln -s $(shell python -c 'import schema_salad; import os.path; print os.path.dirname(schema_salad.__file__)') \
156+
typeshed/2.7/schema_salad
157+
MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
158+
--warn-redundant-casts --warn-unused-ignores cwltool
159+
149160
FORCE:

‎cwltool/builder.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from . import expression
44
import avro
55
import schema_salad.validate as validate
6-
from typing import Any, Union, AnyStr, Callable
6+
from typing import Any, Union, AnyStr, Callable, Type
77
from .errors import WorkflowException
88
from .stdfsaccess import StdFsAccess
99
from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs, normalizeFilesDirs
@@ -33,9 +33,10 @@ def __init__(self): # type: () -> None
3333
self.timeout = None # type: int
3434
self.pathmapper = None # type: PathMapper
3535
self.stagedir = None # type: unicode
36+
self.make_fs_access = None # type: Type[StdFsAccess]
3637

3738
def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
38-
# type: (Dict[unicode, Any], Any, List[int], List[int]) -> List[Dict[str, Any]]
39+
# type: (Dict[unicode, Any], Any, Union[int, List[int]], List[int]) -> List[Dict[str, Any]]
3940
bindings = [] # type: List[Dict[str,str]]
4041
binding = None # type: Dict[str,Any]
4142
if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
@@ -84,8 +85,11 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
8485
if binding:
8586
b2 = copy.deepcopy(binding)
8687
b2["datum"] = item
87-
bindings.extend(self.bind_input({"type": schema["items"], "inputBinding": b2},
88-
item, lead_pos=n, tail_pos=tail_pos))
88+
bindings.extend(
89+
self.bind_input(
90+
{"type": schema["items"],
91+
"inputBinding": b2},
92+
item, lead_pos=n, tail_pos=tail_pos))
8993
binding = None
9094

9195
if schema["type"] == "File":

‎cwltool/cwltest.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,7 @@ def run_test(args, i, t): # type: (argparse.Namespace, Any, Dict[str,str]) -> i
141141
failed = True
142142

143143
if outdir:
144-
shutil.rmtree(outdir, True) # type: ignore
145-
# Weird AnyStr != basestring issue
144+
shutil.rmtree(outdir, True)
146145

147146
if failed:
148147
return 1

‎cwltool/draft2tool.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from .builder import CONTENT_LIMIT, substitute, Builder, adjustFileObjs, adjustDirObjs
2424
from .pathmapper import PathMapper
2525
from .job import CommandLineJob
26+
from .stdfsaccess import StdFsAccess
2627

2728
ACCEPTLIST_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
2829

@@ -53,7 +54,8 @@ def run(self, **kwargs): # type: (**Any) -> None
5354
normalizeFilesDirs(ev)
5455
self.output_callback(ev, "success")
5556
except Exception as e:
56-
_logger.warn(u"Failed to evaluate expression:\n%s", e, exc_info=(e if kwargs.get('debug') else False))
57+
_logger.warn(u"Failed to evaluate expression:\n%s",
58+
e, exc_info=kwargs.get('debug'))
5759
self.output_callback({}, "permanentFail")
5860

5961
def job(self, joborder, output_callback, **kwargs):
@@ -362,7 +364,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
362364
yield j
363365

364366
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
365-
# type: (Set[Dict[str,Any]], Builder, str) -> Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
367+
# type: (Set[Dict[str,Any]], Builder, str, bool) -> Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
366368
try:
367369
ret = {} # type: Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
368370
fs_access = builder.make_fs_access(outdir)
@@ -377,8 +379,13 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
377379
try:
378380
ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum)
379381
except Exception as e:
380-
_logger.debug(u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=e)
381-
raise WorkflowException(u"Error collecting output for parameter '%s': %s" % (shortname(port["id"]), e))
382+
_logger.debug(
383+
u"Error collecting output for parameter '%s'"
384+
% shortname(port["id"]), exc_info=True)
385+
raise WorkflowException(
386+
u"Error collecting output for " \
387+
"parameter '%s': %s" %
388+
(shortname(port["id"]), e))
382389

383390
if ret:
384391
adjustFileObjs(ret,
@@ -397,7 +404,7 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
397404
raise WorkflowException("Error validating output record, " + str(e) + "\n in " + json.dumps(ret, indent=4))
398405

399406
def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=True):
400-
# type: (Dict[str,Any], Builder, str) -> Union[Dict[unicode, Any], List[Union[Dict[unicode, Any], unicode]]]
407+
# type: (Dict[str,Any], Builder, str, StdFsAccess, bool) -> Union[Dict[unicode, Any], List[Union[Dict[unicode, Any], unicode]]]
401408
r = [] # type: List[Any]
402409
if "outputBinding" in schema:
403410
binding = schema["outputBinding"]

‎cwltool/job.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
import shellescape
1717
from .docker_uid import docker_vm_uid
1818
from .builder import Builder
19-
from typing import Union, Iterable, Callable, Any, Mapping, IO, cast, Tuple
19+
from typing import (Any, Callable, Union, Iterable, Mapping, MutableMapping,
20+
IO, cast, Tuple)
2021
from .pathmapper import PathMapper
2122
import functools
2223

@@ -57,7 +58,7 @@ def __init__(self): # type: () -> None
5758
self.output_callback = None # type: Callable[[Any, Any], Any]
5859
self.outdir = None # type: str
5960
self.tmpdir = None # type: str
60-
self.environment = None # type: Dict[str,str]
61+
self.environment = None # type: MutableMapping[str, str]
6162
self.generatefiles = None # type: Dict[unicode, Union[List[Dict[str, str]], Dict[str,str], str]]
6263
self.stagedir = None # type: unicode
6364

@@ -82,6 +83,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
8283
% (knownfile, self.pathmapper.mapper(knownfile)[0]))
8384

8485
img_id = None
86+
env = None # type: MutableMapping[str, str]
8587
if docker_req and kwargs.get("use_container") is not False:
8688
env = os.environ
8789
img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)

‎cwltool/main.py

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import functools
1616

1717
import rdflib
18-
from typing import Union, Any, cast, Callable, Dict, Tuple, IO
18+
from typing import Union, Any, cast, Callable, Dict, Tuple, Type, IO
1919

2020
from schema_salad.ref_resolver import Loader
2121
import schema_salad.validate as validate
@@ -353,7 +353,7 @@ def generate_parser(toolparser, tool, namemap):
353353

354354
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
355355
stdout=sys.stdout, make_fs_access=None):
356-
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]]
356+
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Type[StdFsAccess]) -> Union[int,Tuple[Dict[str,Any],str]]
357357

358358
job_order_object = None
359359

@@ -383,7 +383,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
383383
try:
384384
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
385385
except Exception as e:
386-
_logger.error(str(e), exc_info=(e if args.debug else False))
386+
_logger.error(str(e), exc_info=args.debug)
387387
return 1
388388
toolparser = None
389389
else:
@@ -401,7 +401,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
401401
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
402402
job_order_object = loader.resolve_ref(cmd_line["job_order"])
403403
except Exception as e:
404-
_logger.error(str(e), exc_info=(e if args.debug else False))
404+
_logger.error(str(e), exc_info=args.debug)
405405
return 1
406406
else:
407407
job_order_object = {"id": args.workflow}
@@ -575,7 +575,7 @@ def main(argsl=None,
575575
versionfunc=versionstring,
576576
job_order_object=None,
577577
make_fs_access=StdFsAccess):
578-
# type: (List[str], argparse.Namespace, Callable[..., Union[str, Dict[str, str]]], Callable[..., Process], Callable[[Dict[str, int]], Dict[str, int]], IO[Any], IO[Any], IO[Any], Callable[[], unicode], Union[int, Tuple[Dict[str, Any], str]]) -> int
578+
# type: (List[str], argparse.Namespace, Callable[..., Union[str, Dict[str, str]]], Callable[..., Process], Callable[[Dict[str, int]], Dict[str, int]], IO[Any], IO[Any], IO[Any], Callable[[], unicode], Union[int, Tuple[Dict[str, Any], str]], Type[StdFsAccess]) -> int
579579

580580
_logger.removeHandler(defaultStreamHandler)
581581
stderr_handler = logging.StreamHandler(stderr)
@@ -661,18 +661,18 @@ def main(argsl=None,
661661
makeTool, {})
662662
except (validate.ValidationException) as exc:
663663
_logger.error(u"Tool definition failed validation:\n%s", exc,
664-
exc_info=(exc if args.debug else False))
664+
exc_info=args.debug)
665665
return 1
666666
except (RuntimeError, WorkflowException) as exc:
667667
_logger.error(u"Tool definition failed initialization:\n%s", exc,
668-
exc_info=(exc if args.debug else False))
668+
exc_info=args.debug)
669669
return 1
670670
except Exception as exc:
671671
_logger.error(
672672
u"I'm sorry, I couldn't load this CWL file%s",
673673
", try again with --debug for more information.\nThe error was: "
674674
"%s" % exc if not args.debug else ". The error was:",
675-
exc_info=(exc if args.debug else False))
675+
exc_info=args.debug)
676676
return 1
677677

678678
if isinstance(tool, int):
@@ -738,24 +738,23 @@ def locToPath(p):
738738
else:
739739
return 1
740740
except (validate.ValidationException) as exc:
741-
_logger.error(
742-
u"Input object failed validation:\n%s", exc,
743-
exc_info=(exc if args.debug else False))
741+
_logger.error(u"Input object failed validation:\n%s", exc,
742+
exc_info=args.debug)
744743
return 1
745744
except UnsupportedRequirement as exc:
746745
_logger.error(
747746
u"Workflow or tool uses unsupported feature:\n%s", exc,
748-
exc_info=(exc if args.debug else False))
747+
exc_info=args.debug)
749748
return 33
750749
except WorkflowException as exc:
751750
_logger.error(
752751
u"Workflow error, try again with --debug for more "
753-
"information:\n %s", exc, exc_info=(exc if args.debug else False))
752+
"information:\n %s", exc, exc_info=args.debug)
754753
return 1
755754
except Exception as exc:
756755
_logger.error(
757756
u"Unhandled error, try again with --debug for more information:\n"
758-
" %s", exc, exc_info=(exc if args.debug else False))
757+
" %s", exc, exc_info=args.debug)
759758
return 1
760759

761760
return 0

‎cwltool/process.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ def getListing(fs_access, rec):
170170
rec["listing"] = listing
171171

172172
def stageFiles(pm, stageFunc, ignoreWritable=False):
173-
# type: (PathMapper, Callable[..., Any]) -> None
173+
# type: (PathMapper, Callable[..., Any], bool) -> None
174174
for f, p in pm.items():
175175
if not os.path.exists(os.path.dirname(p.target)):
176176
os.makedirs(os.path.dirname(p.target), 0755)
@@ -374,9 +374,9 @@ def __init__(self, toolpath_object, **kwargs):
374374
c["type"] = c["type"]
375375
c["type"] = avroize_type(c["type"], c["name"])
376376
if key == "inputs":
377-
self.inputs_record_schema["fields"].append(c) # type: ignore
377+
self.inputs_record_schema["fields"].append(c)
378378
elif key == "outputs":
379-
self.outputs_record_schema["fields"].append(c) # type: ignore
379+
self.outputs_record_schema["fields"].append(c)
380380

381381
try:
382382
self.inputs_record_schema = schema_salad.schema.make_valid_avro(self.inputs_record_schema, {}, set())

‎cwltool/stdfsaccess.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ def isdir(self, fn): # type: (unicode) -> bool
3030
def listdir(self, fn): # type: (unicode) -> List[unicode]
3131
return [abspath(l, fn) for l in os.listdir(self._abs(fn))]
3232

33-
def join(self, path, *paths): # type: (unicode, *unicode) -> unicode
33+
def join(self, path, *paths): # type: (unicode, *unicode) -> unicode
3434
return os.path.join(path, *paths)
3535

36-
def realpath(self, path):
36+
def realpath(self, path): # type: (str) -> str
3737
return os.path.realpath(path)

‎cwltool/update.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ def _draftDraft3dev2toDev3(doc, loader, baseuri):
191191
else:
192192
imp = urlparse.urljoin(baseuri, doc["@import"])
193193
impLoaded = loader.fetch(imp)
194+
r = {} # type: Dict[str, Any]
194195
if isinstance(impLoaded, list):
195196
r = {"@graph": impLoaded}
196197
elif isinstance(impLoaded, dict):
@@ -236,6 +237,7 @@ def traverseImport(doc, loader, baseuri, func):
236237
else:
237238
imp = urlparse.urljoin(baseuri, doc["$import"])
238239
impLoaded = loader.fetch(imp)
240+
r = {} # type: Dict[str, Any]
239241
if isinstance(impLoaded, list):
240242
r = {"$graph": impLoaded}
241243
elif isinstance(impLoaded, dict):

0 commit comments

Comments
 (0)