Skip to content

Commit a907840

Browse files
mr-cmr-c
authored
mypy 0.720 & strict types (#1171)
* increase test coverage, fix a bug
1 parent 665141f commit a907840

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+1486
-1143
lines changed

‎Makefile

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -175,9 +175,6 @@ mypy2: ${PYSOURCES}
175175
rm -Rf typeshed/2and3/ruamel/yaml
176176
ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
177177
typeshed/2and3/ruamel/yaml
178-
rm -Rf typeshed/2and3/schema_salad
179-
ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
180-
typeshed/2and3/schema_salad
181178
MYPYPATH=$$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
182179
--warn-redundant-casts \
183180
cwltool
@@ -186,9 +183,6 @@ mypy3: ${PYSOURCES}
186183
rm -Rf typeshed/2and3/ruamel/yaml
187184
ln -s $(shell python3 -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
188185
typeshed/2and3/ruamel/yaml
189-
rm -Rf typeshed/2and3/schema_salad
190-
ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
191-
typeshed/2and3/schema_salad
192186
MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
193187
--warn-redundant-casts \
194188
cwltool

‎cwltool/argparser.py

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -340,10 +340,14 @@ def __init__(self, option_strings, dest, nargs=None, **kwargs):
340340
raise ValueError("nargs not allowed")
341341
super(FSAction, self).__init__(option_strings, dest, **kwargs)
342342

343-
def __call__(self, parser, namespace, values, option_string=None):
344-
# type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None
343+
def __call__(self,
344+
parser, # type: argparse.ArgumentParser
345+
namespace, # type: argparse.Namespace
346+
values, # type: Union[AnyStr, Sequence[Any], None]
347+
option_string=None # type: Optional[Text]
348+
): # type: (...) -> None
345349
setattr(namespace,
346-
self.dest, # type: ignore
350+
self.dest,
347351
{"class": self.objclass,
348352
"location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})
349353

@@ -358,16 +362,16 @@ def __init__(self, option_strings, dest, nargs=None, **kwargs):
358362
raise ValueError("nargs not allowed")
359363
super(FSAppendAction, self).__init__(option_strings, dest, **kwargs)
360364

361-
def __call__(self, parser, namespace, values, option_string=None):
362-
# type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None
363-
g = getattr(namespace,
364-
self.dest # type: ignore
365-
)
365+
def __call__(self,
366+
parser, # type: argparse.ArgumentParser
367+
namespace, # type: argparse.Namespace
368+
values, # type: Union[AnyStr, Sequence[Any], None]
369+
option_string=None # type: Optional[Text]
370+
): # type: (...) -> None
371+
g = getattr(namespace, self.dest)
366372
if not g:
367373
g = []
368-
setattr(namespace,
369-
self.dest, # type: ignore
370-
g)
374+
setattr(namespace, self.dest, g)
371375
g.append(
372376
{"class": self.objclass,
373377
"location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})

‎cwltool/builder.py

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,14 @@
3939
from .provenance import ProvenanceProfile # pylint: disable=unused-import
4040

4141

42-
def content_limit_respected_read_bytes(f): # type: (IO) -> bytes
42+
def content_limit_respected_read_bytes(f): # type: (IO[bytes]) -> bytes
4343
contents = f.read(CONTENT_LIMIT + 1)
4444
if len(contents) > CONTENT_LIMIT:
4545
raise WorkflowException("loadContents handling encountered buffer that is exceeds maximum lenght of %d bytes" % CONTENT_LIMIT)
4646
return contents
4747

4848

49-
def content_limit_respected_read(f): # type: (IO) -> Text
49+
def content_limit_respected_read(f): # type: (IO[bytes]) -> Text
5050
return content_limit_respected_read_bytes(f).decode("utf-8")
5151

5252

@@ -92,7 +92,7 @@ def formatSubclassOf(fmt, cls, ontology, visited):
9292

9393
return False
9494

95-
def check_format(actual_file, # type: Union[Dict[Text, Any], List, Text]
95+
def check_format(actual_file, # type: Union[Dict[Text, Any], List[Dict[Text, Any]], Text]
9696
input_formats, # type: Union[List[Text], Text]
9797
ontology # type: Optional[Graph]
9898
): # type: (...) -> None
@@ -113,10 +113,10 @@ def check_format(actual_file, # type: Union[Dict[Text, Any], List, Text]
113113
json_dumps(afile, indent=4)))
114114

115115
class HasReqsHints(object):
116-
def __init__(self):
116+
def __init__(self): # type: () -> None
117117
"""Initialize this reqs decorator."""
118-
self.requirements = [] # List[Dict[Text, Any]]
119-
self.hints = [] # List[Dict[Text, Any]]
118+
self.requirements = [] # type: List[Dict[Text, Any]]
119+
self.hints = [] # type: List[Dict[Text, Any]]
120120

121121
def get_requirement(self,
122122
feature # type: Text
@@ -131,7 +131,7 @@ def get_requirement(self,
131131

132132
class Builder(HasReqsHints):
133133
def __init__(self,
134-
job, # type: Dict[Text, Union[Dict[Text, Any], List, Text, None]]
134+
job, # type: Dict[Text, expression.JSON]
135135
files, # type: List[Dict[Text, Text]]
136136
bindings, # type: List[Dict[Text, Any]]
137137
schemaDefs, # type: Dict[Text, Dict[Text, Any]]
@@ -208,7 +208,7 @@ def bind_input(self,
208208
lead_pos = []
209209

210210
bindings = [] # type: List[MutableMapping[Text, Text]]
211-
binding = None # type: Optional[MutableMapping[Text,Any]]
211+
binding = {} # type: Union[MutableMapping[Text, Text], CommentedMap]
212212
value_from_expression = False
213213
if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
214214
binding = CommentedMap(schema["inputBinding"].items())
@@ -254,8 +254,7 @@ def bind_input(self,
254254
raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
255255
elif isinstance(schema["type"], MutableMapping):
256256
st = copy.deepcopy(schema["type"])
257-
if binding is not None\
258-
and "inputBinding" not in st\
257+
if binding and "inputBinding" not in st\
259258
and "type" in st\
260259
and st["type"] == "array"\
261260
and "itemSeparator" not in binding:
@@ -281,7 +280,7 @@ def bind_input(self,
281280
if schema["type"] == "array":
282281
for n, item in enumerate(datum):
283282
b2 = None
284-
if binding is not None:
283+
if binding:
285284
b2 = copy.deepcopy(binding)
286285
b2["datum"] = item
287286
itemschema = {
@@ -293,9 +292,9 @@ def bind_input(self,
293292
itemschema[k] = schema[k]
294293
bindings.extend(
295294
self.bind_input(itemschema, item, lead_pos=n, tail_pos=tail_pos, discover_secondaryFiles=discover_secondaryFiles))
296-
binding = None
295+
binding = {}
297296

298-
def _capture_files(f):
297+
def _capture_files(f): # type: (Dict[Text, Text]) -> Dict[Text, Text]
299298
self.files.append(f)
300299
return f
301300

@@ -365,14 +364,14 @@ def _capture_files(f):
365364
visit_class(datum, ("File", "Directory"), _capture_files)
366365

367366
# Position to front of the sort key
368-
if binding is not None:
367+
if binding:
369368
for bi in bindings:
370369
bi["position"] = binding["position"] + bi["position"]
371370
bindings.append(binding)
372371

373372
return bindings
374373

375-
def tostr(self, value): # type: (Any) -> Text
374+
def tostr(self, value): # type: (Union[MutableMapping[Text, Text], Any]) -> Text
376375
if isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
377376
if "path" not in value:
378377
raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))

‎cwltool/checker.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
9595
if can_assign_src_to_sink(src, this_sink):
9696
return True
9797
return False
98-
return src == sink
98+
return bool(src == sink)
9999

100100

101101
def _compare_records(src, sink, strict=False):
@@ -126,8 +126,7 @@ def _rec_fields(rec): # type: (MutableMapping[Text, Any]) -> MutableMapping[Tex
126126
return False
127127
return True
128128

129-
def missing_subset(fullset, subset):
130-
# type: (List, List) -> List
129+
def missing_subset(fullset, subset): # type: (List[Any], List[Any]) -> List[Any]
131130
missing = []
132131
for i in subset:
133132
if i not in fullset:

‎cwltool/command_line_tool.py

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
import tempfile
1313
import threading
1414
from functools import cmp_to_key, partial
15-
from typing import (Any, Callable, Dict, Generator, List, Mapping, MutableMapping,
16-
MutableSequence, Optional, Set, Union, cast)
15+
from typing import (Any, Callable, Dict, Generator, IO, List, Mapping,
16+
MutableMapping, MutableSequence, Optional, Set, Union, cast)
1717

1818
from typing_extensions import Text, Type, TYPE_CHECKING # pylint: disable=unused-import
1919
# move to a regular typing import when Python 3.3-3.6 is no longer supported
@@ -83,8 +83,8 @@ def __init__(self,
8383
builder, # type: Builder
8484
script, # type: Dict[Text, Text]
8585
output_callback, # type: Callable[[Any, Any], Any]
86-
requirements, # type: Dict[Text, Text]
87-
hints, # type: Dict[Text, Text]
86+
requirements, # type: List[Dict[Text, Text]]
87+
hints, # type: List[Dict[Text, Text]]
8888
outdir=None, # type: Optional[Text]
8989
tmpdir=None, # type: Optional[Text]
9090
): # type: (...) -> None
@@ -101,7 +101,7 @@ def __init__(self,
101101

102102
def run(self,
103103
runtimeContext, # type: RuntimeContext
104-
tmpdir_lock=None # type: threading.Lock
104+
tmpdir_lock=None # type: Optional[threading.Lock]
105105
): # type: (...) -> None
106106
try:
107107
normalizeFilesDirs(self.builder.job)
@@ -194,8 +194,10 @@ def __init__(self, job, output_callback, cachebuilder, jobcache):
194194
self.outdir = jobcache
195195
self.prov_obj = None # type: Optional[ProvenanceProfile]
196196

197-
def run(self, runtimeContext):
198-
# type: (RuntimeContext) -> None
197+
def run(self,
198+
runtimeContext, # type: RuntimeContext
199+
tmpdir_lock=None # type: Optional[threading.Lock]
200+
): # type: (...) -> None
199201
self.output_callback(self.job.collect_output_ports(
200202
self.job.tool["outputs"],
201203
self.cachebuilder,
@@ -232,7 +234,7 @@ def check_adjust(builder, file_o):
232234
file_o["basename"]))
233235
return file_o
234236

235-
def check_valid_locations(fs_access, ob):
237+
def check_valid_locations(fs_access, ob): # type: (StdFsAccess, Dict[Text, Any]) -> None
236238
if ob["location"].startswith("_:"):
237239
pass
238240
if ob["class"] == "File" and not fs_access.isfile(ob["location"]):
@@ -285,7 +287,7 @@ def make_path_mapper(self, reffiles, stagedir, runtimeContext, separateDirs):
285287
return PathMapper(reffiles, runtimeContext.basedir, stagedir, separateDirs)
286288

287289
def updatePathmap(self, outdir, pathmap, fn):
288-
# type: (Text, PathMapper, Dict) -> None
290+
# type: (Text, PathMapper, Dict[Text, Any]) -> None
289291
if "location" in fn and fn["location"] in pathmap:
290292
pathmap.update(fn["location"], pathmap.mapper(fn["location"]).resolved,
291293
os.path.join(outdir, fn["basename"]),
@@ -298,7 +300,7 @@ def updatePathmap(self, outdir, pathmap, fn):
298300
def job(self,
299301
job_order, # type: Mapping[Text, Text]
300302
output_callbacks, # type: Callable[[Any, Any], Any]
301-
runtimeContext # RuntimeContext
303+
runtimeContext # type: RuntimeContext
302304
):
303305
# type: (...) -> Generator[Union[JobBase, CallbackJob], None, None]
304306

@@ -332,9 +334,9 @@ def job(self,
332334
if dockerimg is not None:
333335
cmdline = ["docker", "run", dockerimg] + cmdline
334336
# not really run using docker, just for hashing purposes
335-
keydict = {u"cmdline": cmdline}
337+
keydict = {u"cmdline": cmdline} # type: Dict[Text, Union[Dict[Text, Any], List[Any]]]
336338

337-
for shortcut in ["stdout", "stderr"]: # later, add "stdin"
339+
for shortcut in ["stdin", "stdout", "stderr"]:
338340
if shortcut in self.tool:
339341
keydict[shortcut] = self.tool[shortcut]
340342

@@ -409,8 +411,12 @@ def job(self,
409411
runtimeContext = runtimeContext.copy()
410412
runtimeContext.outdir = jobcache
411413

412-
def update_status_output_callback(output_callbacks, jobcachelock,
413-
outputs, processStatus):
414+
def update_status_output_callback(
415+
output_callbacks, # type: Callable[[List[Dict[Text, Any]], Text], None]
416+
jobcachelock, # type: IO[Any]
417+
outputs, # type: List[Dict[Text, Any]]
418+
processStatus # type: Text
419+
): # type: (...) -> None
414420
# save status to the lockfile then release the lock
415421
jobcachelock.seek(0)
416422
jobcachelock.truncate()
@@ -556,13 +562,15 @@ def update_status_output_callback(output_callbacks, jobcachelock,
556562
muts = set() # type: Set[Text]
557563

558564
if builder.mutation_manager is not None:
559-
def register_mut(f):
565+
def register_mut(f): # type: (Dict[Text, Any]) -> None
566+
mm = cast(MutationManager, builder.mutation_manager)
560567
muts.add(f["location"])
561-
builder.mutation_manager.register_mutation(j.name, f)
568+
mm.register_mutation(j.name, f)
562569

563-
def register_reader(f):
570+
def register_reader(f): # type: (Dict[Text, Any]) -> None
571+
mm = cast(MutationManager, builder.mutation_manager)
564572
if f["location"] not in muts:
565-
builder.mutation_manager.register_reader(j.name, f)
573+
mm.register_reader(j.name, f)
566574
readers[f["location"]] = copy.deepcopy(f)
567575

568576
for li in j.generatefiles["listing"]:
@@ -628,7 +636,7 @@ def collect_output_ports(self,
628636
rcode, # type: int
629637
compute_checksum=True, # type: bool
630638
jobname="", # type: Text
631-
readers=None # type: Dict[Text, Any]
639+
readers=None # type: Optional[Dict[Text, Any]]
632640
): # type: (...) -> OutputPorts
633641
ret = {} # type: OutputPorts
634642
debug = _logger.isEnabledFor(logging.DEBUG)
@@ -647,11 +655,12 @@ def collect_output_ports(self,
647655
json_dumps(ret, indent=4))
648656
else:
649657
for i, port in enumerate(ports):
650-
def makeWorkflowException(msg):
651-
return WorkflowException(
652-
u"Error collecting output for parameter '%s':\n%s"
653-
% (shortname(port["id"]), msg))
654-
with SourceLine(ports, i, makeWorkflowException, debug):
658+
class ParameterOutputWorkflowException(WorkflowException):
659+
def __init__(self, msg, **kwargs): # type: (Text, **Any) -> None
660+
super(ParameterOutputWorkflowException, self).__init__(
661+
u"Error collecting output for parameter '%s':\n%s"
662+
% (shortname(port["id"]), msg), kwargs)
663+
with SourceLine(ports, i, ParameterOutputWorkflowException, debug):
655664
fragment = shortname(port["id"])
656665
ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
657666
compute_checksum=compute_checksum)

‎cwltool/context.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ def __init__(self, kwargs=None): # type: (Optional[Dict[str, Any]]) -> None
4545
"""Initialize the LoadingContext from the kwargs."""
4646
self.debug = False # type: bool
4747
self.metadata = {} # type: Dict[Text, Any]
48-
self.requirements = None
49-
self.hints = None
48+
self.requirements = None # type: Optional[List[Dict[Text, Any]]]
49+
self.hints = None # type: Optional[List[Dict[Text, Any]]]
5050
self.overrides_list = [] # type: List[Dict[Text, Any]]
5151
self.loader = None # type: Optional[Loader]
5252
self.avsc_names = None # type: Optional[schema.Names]

‎cwltool/cwlrdf.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from __future__ import absolute_import
22

3-
from typing import IO, Any, Dict
3+
from typing import cast, IO, Any, Dict, MutableMapping
44

55
from rdflib import Graph
66
from schema_salad.jsonld_context import makerdf
@@ -15,26 +15,26 @@
1515
def gather(tool, ctx): # type: (Process, ContextType) -> Graph
1616
g = Graph()
1717

18-
def visitor(t):
18+
def visitor(t): # type: (MutableMapping[Text, Any]) -> None
1919
makerdf(t["id"], t, ctx, graph=g)
2020

2121
tool.visit(visitor)
2222
return g
2323

2424

25-
def printrdf(wflow, ctx, style): # type: (Process, ContextType, Text) -> Text
25+
def printrdf(wflow, ctx, style): # type: (Process, ContextType, str) -> Text
2626
"""Serialize the CWL document into a string, ready for printing."""
2727
rdf = gather(wflow, ctx).serialize(format=style, encoding='utf-8')
2828
if not rdf:
2929
return u""
30-
return rdf.decode('utf-8')
30+
return cast(Text, rdf.decode('utf-8'))
3131

3232

3333
def lastpart(uri): # type: (Any) -> Text
34-
uri = Text(uri)
35-
if "/" in uri:
36-
return uri[uri.rindex("/") + 1:]
37-
return uri
34+
uri2 = Text(uri)
35+
if "/" in uri2:
36+
return uri2[uri2.rindex("/") + 1:]
37+
return uri2
3838

3939

4040
def dot_with_parameters(g, stdout): # type: (Graph, IO[Any]) -> None

0 commit comments

Comments
 (0)