Skip to content

Commit 9d490f1

Browse files
FarahZKhanFarahZKhan
committed
Merge branch 'cwlprov-add_output' of https://github.com/common-workflow-language/cwltool into cwlprov-add_output
2 parents d49ebc1 + 709712e commit 9d490f1

24 files changed

+1266
-246
lines changed

‎cwltool/argparser.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
import argparse
44
import os
5-
from typing import Any, AnyStr, Dict, List, Optional, Sequence, Union, cast
5+
from typing import (Any, AnyStr, Dict, List, Optional, Sequence,
6+
Union, cast, MutableMapping, MutableSequence)
67

78
from typing_extensions import Text # pylint: disable=unused-import
89
# move to a regular typing import when Python 3.3-3.6 is no longer supported
@@ -381,7 +382,7 @@ def add_argument(toolparser, name, inptype, records, description="",
381382
flag = "--"
382383

383384
required = True
384-
if isinstance(inptype, list):
385+
if isinstance(inptype, MutableSequence):
385386
if inptype[0] == "null":
386387
required = False
387388
if len(inptype) == 2:
@@ -398,16 +399,16 @@ def add_argument(toolparser, name, inptype, records, description="",
398399
action = cast(argparse.Action, FileAction)
399400
elif inptype == "Directory":
400401
action = cast(argparse.Action, DirectoryAction)
401-
elif isinstance(inptype, dict) and inptype["type"] == "array":
402+
elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
402403
if inptype["items"] == "File":
403404
action = cast(argparse.Action, FileAppendAction)
404405
elif inptype["items"] == "Directory":
405406
action = cast(argparse.Action, DirectoryAppendAction)
406407
else:
407408
action = "append"
408-
elif isinstance(inptype, dict) and inptype["type"] == "enum":
409+
elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
409410
atype = Text
410-
elif isinstance(inptype, dict) and inptype["type"] == "record":
411+
elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
411412
records.append(name)
412413
for field in inptype['fields']:
413414
fieldname = name + "." + shortname(field['name'])

‎cwltool/builder.py

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22

33
import copy
44
import logging
5-
from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple
5+
from typing import (Any, Callable, Dict, List, Optional, Set, Union,
6+
Tuple, MutableMapping, MutableSequence)
67
from typing_extensions import Text, Type, TYPE_CHECKING # pylint: disable=unused-import
78
# move to a regular typing import when Python 3.3-3.6 is no longer supported
89

910
from rdflib import Graph, URIRef # pylint: disable=unused-import
1011
from rdflib.namespace import OWL, RDFS
12+
from ruamel.yaml.comments import CommentedMap
1113
import schema_salad.schema # pylint: disable=unused-import
1214
from schema_salad import validate
1315
from schema_salad.schema import AvroSchemaFromJSONData
@@ -194,22 +196,23 @@ def build_job_script(self, commands):
194196
return None
195197

196198
def bind_input(self,
197-
schema, # type: Dict[Text, Any]
199+
schema, # type: MutableMapping[Text, Any]
198200
datum, # type: Any
199201
discover_secondaryFiles, # type: bool
200202
lead_pos=None, # type: Optional[Union[int, List[int]]]
201203
tail_pos=None, # type: Optional[List[int]]
202-
): # type: (...) -> List[Dict[Text, Any]]
204+
): # type: (...) -> List[MutableMapping[Text, Any]]
203205

204206
if tail_pos is None:
205207
tail_pos = []
206208
if lead_pos is None:
207209
lead_pos = []
208-
bindings = [] # type: List[Dict[Text,Text]]
209-
binding = None # type: Optional[Dict[Text,Any]]
210+
bindings = [] # type: List[MutableMapping[Text, Text]]
211+
binding = None # type: Optional[MutableMapping[Text,Any]]
210212
value_from_expression = False
211-
if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
212-
binding = copy.copy(schema["inputBinding"])
213+
if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
214+
binding = CommentedMap(schema["inputBinding"].items())
215+
assert binding is not None
213216

214217
if "position" in binding:
215218
binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos)
@@ -221,12 +224,12 @@ def bind_input(self,
221224
value_from_expression = True
222225

223226
# Handle union types
224-
if isinstance(schema["type"], list):
227+
if isinstance(schema["type"], MutableSequence):
225228
bound_input = False
226229
for t in schema["type"]:
227230
if isinstance(t, string_types) and self.names.has_name(t, ""):
228231
avsc = self.names.get_name(t, "")
229-
elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
232+
elif isinstance(t, MutableMapping) and "name" in t and self.names.has_name(t["name"], ""):
230233
avsc = self.names.get_name(t["name"], "")
231234
else:
232235
avsc = AvroSchemaFromJSONData(t, self.names)
@@ -240,7 +243,7 @@ def bind_input(self,
240243
bound_input = True
241244
if not bound_input:
242245
raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
243-
elif isinstance(schema["type"], dict):
246+
elif isinstance(schema["type"], MutableMapping):
244247
st = copy.deepcopy(schema["type"])
245248
if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding:
246249
st["inputBinding"] = {}
@@ -289,7 +292,7 @@ def bind_input(self,
289292
if "secondaryFiles" not in datum:
290293
datum["secondaryFiles"] = []
291294
for sf in aslist(schema["secondaryFiles"]):
292-
if isinstance(sf, dict) or "$(" in sf or "${" in sf:
295+
if isinstance(sf, MutableMapping) or "$(" in sf or "${" in sf:
293296
sfpath = self.do_eval(sf, context=datum)
294297
else:
295298
sfpath = substitute(datum["basename"], sf)
@@ -301,7 +304,7 @@ def bind_input(self,
301304
if d["basename"] == sfname:
302305
found = True
303306
if not found:
304-
if isinstance(sfname, dict):
307+
if isinstance(sfname, MutableMapping):
305308
datum["secondaryFiles"].append(sfname)
306309
elif discover_secondaryFiles:
307310
datum["secondaryFiles"].append({
@@ -344,7 +347,7 @@ def _capture_files(f):
344347
return bindings
345348

346349
def tostr(self, value): # type: (Any) -> Text
347-
if isinstance(value, dict) and value.get("class") in ("File", "Directory"):
350+
if isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
348351
if "path" not in value:
349352
raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))
350353

@@ -372,8 +375,8 @@ def generate_arg(self, binding): # type: (Dict[Text, Any]) -> List[Text]
372375
with SourceLine(binding, "separate", WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
373376
raise WorkflowException("'separate' option can not be specified without prefix")
374377

375-
argl = [] # type: List[Dict[Text,Text]]
376-
if isinstance(value, list):
378+
argl = [] # type: MutableSequence[MutableMapping[Text, Text]]
379+
if isinstance(value, MutableSequence):
377380
if binding.get("itemSeparator") and value:
378381
argl = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
379382
elif binding.get("valueFrom"):
@@ -383,9 +386,9 @@ def generate_arg(self, binding): # type: (Dict[Text, Any]) -> List[Text]
383386
return [prefix]
384387
else:
385388
return []
386-
elif isinstance(value, dict) and value.get("class") in ("File", "Directory"):
389+
elif isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
387390
argl = [value]
388-
elif isinstance(value, dict):
391+
elif isinstance(value, MutableMapping):
389392
return [prefix] if prefix else []
390393
elif value is True and prefix:
391394
return [prefix]
@@ -407,10 +410,10 @@ def generate_arg(self, binding): # type: (Dict[Text, Any]) -> List[Text]
407410
def do_eval(self, ex, context=None, recursive=False, strip_whitespace=True):
408411
# type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
409412
if recursive:
410-
if isinstance(ex, dict):
413+
if isinstance(ex, MutableMapping):
411414
return {k: self.do_eval(v, context, recursive)
412415
for k, v in iteritems(ex)}
413-
if isinstance(ex, list):
416+
if isinstance(ex, MutableSequence):
414417
return [self.do_eval(v, context, recursive)
415418
for v in ex]
416419

‎cwltool/checker.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from collections import namedtuple
2-
from typing import Any, Dict, List, Optional
2+
from typing import Any, Dict, List, Optional, MutableMapping, MutableSequence
33
from typing_extensions import Text # pylint: disable=unused-import
44
# move to a regular typing import when Python 3.3-3.6 is no longer supported
55

@@ -15,7 +15,7 @@
1515

1616
def _get_type(tp):
1717
# type: (Any) -> Any
18-
if isinstance(tp, dict):
18+
if isinstance(tp, MutableMapping):
1919
if tp.get("type") not in ("array", "record", "enum"):
2020
return tp["type"]
2121
return tp
@@ -46,9 +46,9 @@ def merge_flatten_type(src):
4646
"""Return the merge flattened type of the source type
4747
"""
4848

49-
if isinstance(src, list):
49+
if isinstance(src, MutableSequence):
5050
return [merge_flatten_type(t) for t in src]
51-
if isinstance(src, dict) and src.get("type") == "array":
51+
if isinstance(src, MutableMapping) and src.get("type") == "array":
5252
return src
5353
return {"items": src, "type": "array"}
5454

@@ -65,7 +65,7 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
6565

6666
if src == "Any" or sink == "Any":
6767
return True
68-
if isinstance(src, dict) and isinstance(sink, dict):
68+
if isinstance(src, MutableMapping) and isinstance(sink, MutableMapping):
6969
if sink.get("not_connected") and strict:
7070
return False
7171
if src["type"] == "array" and sink["type"] == "array":
@@ -79,7 +79,7 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
7979
return False
8080
return True
8181
return can_assign_src_to_sink(src["type"], sink["type"], strict)
82-
elif isinstance(src, list):
82+
elif isinstance(src, MutableSequence):
8383
if strict:
8484
for t in src:
8585
if not can_assign_src_to_sink(t, sink):
@@ -90,7 +90,7 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
9090
if can_assign_src_to_sink(t, sink):
9191
return True
9292
return False
93-
elif isinstance(sink, list):
93+
elif isinstance(sink, MutableSequence):
9494
for t in sink:
9595
if can_assign_src_to_sink(src, t):
9696
return True
@@ -100,14 +100,14 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
100100

101101

102102
def _compare_records(src, sink, strict=False):
103-
# type: (Dict[Text, Any], Dict[Text, Any], bool) -> bool
103+
# type: (MutableMapping[Text, Any], MutableMapping[Text, Any], bool) -> bool
104104
"""Compare two records, ensuring they have compatible fields.
105105
106106
This handles normalizing record names, which will be relative to workflow
107107
step, so that they can be compared.
108108
"""
109109

110-
def _rec_fields(rec): # type: (Dict[Text, Any]) -> Dict[Text, Any]
110+
def _rec_fields(rec): # type: (MutableMapping[Text, Any]) -> MutableMapping[Text, Any]
111111
out = {}
112112
for field in rec["fields"]:
113113
name = shortname(field["name"])
@@ -227,7 +227,7 @@ def check_all_types(src_dict, sinks, sourceField):
227227
for sink in sinks:
228228
if sourceField in sink:
229229
valueFrom = sink.get("valueFrom")
230-
if isinstance(sink[sourceField], list):
230+
if isinstance(sink[sourceField], MutableSequence):
231231
srcs_of_sink = [src_dict[parm_id] for parm_id in sink[sourceField]]
232232
linkMerge = sink.get("linkMerge", ("merge_nested"
233233
if len(sink[sourceField]) > 1 else None))

‎cwltool/command_line_tool.py

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import threading
1313
from functools import cmp_to_key, partial
1414
from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Union,
15-
cast)
15+
cast, MutableMapping, MutableSequence)
1616
from typing_extensions import Text, Type, TYPE_CHECKING # pylint: disable=unused-import
1717
# move to a regular typing import when Python 3.3-3.6 is no longer supported
1818

@@ -99,7 +99,7 @@ def run(self, runtimeContext): # type: (RuntimeContext) -> None
9999
self.output_callback({}, "permanentFail")
100100

101101
def job(self,
102-
job_order, # type: Dict[Text, Text]
102+
job_order, # type: MutableMapping[Text, Text]
103103
output_callbacks, # type: Callable[[Any, Any], Any]
104104
runtimeContext # type: RuntimeContext
105105
):
@@ -222,7 +222,7 @@ def check_valid_locations(fs_access, ob):
222222

223223
class CommandLineTool(Process):
224224
def __init__(self, toolpath_object, loadingContext):
225-
# type: (Dict[Text, Any], LoadingContext) -> None
225+
# type: (MutableMapping[Text, Any], LoadingContext) -> None
226226
super(CommandLineTool, self).__init__(toolpath_object, loadingContext)
227227
self.prov_obj = loadingContext.prov_obj
228228

@@ -272,7 +272,7 @@ def updatePathmap(self, outdir, pathmap, fn):
272272
self.updatePathmap(os.path.join(outdir, fn["basename"]), pathmap, ls)
273273

274274
def job(self,
275-
job_order, # type: Dict[Text, Text]
275+
job_order, # type: MutableMapping[Text, Text]
276276
output_callbacks, # type: Callable[[Any, Any], Any]
277277
runtimeContext # RuntimeContext
278278
):
@@ -510,7 +510,7 @@ def register_mut(f):
510510
def register_reader(f):
511511
if f["location"] not in muts:
512512
builder.mutation_manager.register_reader(j.name, f)
513-
readers[f["location"]] = copy.copy(f)
513+
readers[f["location"]] = copy.deepcopy(f)
514514

515515
for li in j.generatefiles["listing"]:
516516
li = cast(Dict[Text, Any], li)
@@ -693,20 +693,18 @@ def collect_output(self,
693693
if binding.get("loadContents") or compute_checksum:
694694
contents = f.read(CONTENT_LIMIT)
695695
if binding.get("loadContents"):
696-
files["contents"] = contents.decode("utf-8")
696+
files["contents"] = contents.decode("utf-8")
697697
if compute_checksum:
698698
checksum = hashlib.sha1()
699699
while contents != b"":
700700
checksum.update(contents)
701701
contents = f.read(1024 * 1024)
702702
files["checksum"] = "sha1$%s" % checksum.hexdigest()
703-
f.seek(0, 2)
704-
filesize = f.tell()
705-
files["size"] = filesize
703+
files["size"] = fs_access.size(rfile["location"])
706704

707705
optional = False
708706
single = False
709-
if isinstance(schema["type"], list):
707+
if isinstance(schema["type"], MutableSequence):
710708
if "null" in schema["type"]:
711709
optional = True
712710
if "File" in schema["type"] or "Directory" in schema["type"]:
@@ -724,7 +722,7 @@ def collect_output(self,
724722
raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns))
725723
elif not r and optional:
726724
pass
727-
elif isinstance(r, list):
725+
elif isinstance(r, MutableSequence):
728726
if len(r) > 1:
729727
raise WorkflowException("Multiple matches for output item that is a single file.")
730728
else:
@@ -733,11 +731,11 @@ def collect_output(self,
733731
if "secondaryFiles" in schema:
734732
with SourceLine(schema, "secondaryFiles", WorkflowException, debug):
735733
for primary in aslist(r):
736-
if isinstance(primary, dict):
734+
if isinstance(primary, MutableMapping):
737735
primary.setdefault("secondaryFiles", [])
738736
pathprefix = primary["path"][0:primary["path"].rindex("/")+1]
739737
for sf in aslist(schema["secondaryFiles"]):
740-
if isinstance(sf, dict) or "$(" in sf or "${" in sf:
738+
if isinstance(sf, MutableMapping) or "$(" in sf or "${" in sf:
741739
sfpath = builder.do_eval(sf, context=primary)
742740
subst = False
743741
else:
@@ -768,7 +766,7 @@ def collect_output(self,
768766
if not r and optional:
769767
return None
770768

771-
if (not empty_and_optional and isinstance(schema["type"], dict)
769+
if (not empty_and_optional and isinstance(schema["type"], MutableMapping)
772770
and schema["type"]["type"] == "record"):
773771
out = {}
774772
for f in schema["type"]["fields"]:

0 commit comments

Comments
 (0)