Skip to content

Commit da3f1e7

Browse files
mr-cmr-c
authored
Merge pull request #864 from common-workflow-language/MultableMap
Support ruamel 0.15.52+
2 parents 4c31b01 + 1903992 commit da3f1e7

23 files changed

+1254
-236
lines changed

‎cwltool/argparser.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
import argparse
44
import os
5-
from typing import Any, AnyStr, Dict, List, Optional, Sequence, Union, cast
5+
from typing import (Any, AnyStr, Dict, List, Optional, Sequence,
6+
Union, cast, MutableMapping, MutableSequence)
67

78
from typing_extensions import Text # pylint: disable=unused-import
89
# move to a regular typing import when Python 3.3-3.6 is no longer supported
@@ -381,7 +382,7 @@ def add_argument(toolparser, name, inptype, records, description="",
381382
flag = "--"
382383

383384
required = True
384-
if isinstance(inptype, list):
385+
if isinstance(inptype, MutableSequence):
385386
if inptype[0] == "null":
386387
required = False
387388
if len(inptype) == 2:
@@ -398,16 +399,16 @@ def add_argument(toolparser, name, inptype, records, description="",
398399
action = cast(argparse.Action, FileAction)
399400
elif inptype == "Directory":
400401
action = cast(argparse.Action, DirectoryAction)
401-
elif isinstance(inptype, dict) and inptype["type"] == "array":
402+
elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
402403
if inptype["items"] == "File":
403404
action = cast(argparse.Action, FileAppendAction)
404405
elif inptype["items"] == "Directory":
405406
action = cast(argparse.Action, DirectoryAppendAction)
406407
else:
407408
action = "append"
408-
elif isinstance(inptype, dict) and inptype["type"] == "enum":
409+
elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
409410
atype = Text
410-
elif isinstance(inptype, dict) and inptype["type"] == "record":
411+
elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
411412
records.append(name)
412413
for field in inptype['fields']:
413414
fieldname = name + "." + shortname(field['name'])

‎cwltool/builder.py

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22

33
import copy
44
import logging
5-
from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple
5+
from typing import (Any, Callable, Dict, List, Optional, Set, Union,
6+
Tuple, MutableMapping, MutableSequence)
67
from typing_extensions import Text, Type, TYPE_CHECKING # pylint: disable=unused-import
78
# move to a regular typing import when Python 3.3-3.6 is no longer supported
89

910
from rdflib import Graph, URIRef # pylint: disable=unused-import
1011
from rdflib.namespace import OWL, RDFS
12+
from ruamel.yaml.comments import CommentedMap
1113
import schema_salad.schema # pylint: disable=unused-import
1214
from schema_salad import validate
1315
from schema_salad.schema import AvroSchemaFromJSONData
@@ -194,22 +196,23 @@ def build_job_script(self, commands):
194196
return None
195197

196198
def bind_input(self,
197-
schema, # type: Dict[Text, Any]
199+
schema, # type: MutableMapping[Text, Any]
198200
datum, # type: Any
199201
discover_secondaryFiles, # type: bool
200202
lead_pos=None, # type: Optional[Union[int, List[int]]]
201203
tail_pos=None, # type: Optional[List[int]]
202-
): # type: (...) -> List[Dict[Text, Any]]
204+
): # type: (...) -> List[MutableMapping[Text, Any]]
203205

204206
if tail_pos is None:
205207
tail_pos = []
206208
if lead_pos is None:
207209
lead_pos = []
208-
bindings = [] # type: List[Dict[Text,Text]]
209-
binding = None # type: Optional[Dict[Text,Any]]
210+
bindings = [] # type: List[MutableMapping[Text, Text]]
211+
binding = None # type: Optional[MutableMapping[Text,Any]]
210212
value_from_expression = False
211-
if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
212-
binding = copy.copy(schema["inputBinding"])
213+
if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
214+
binding = CommentedMap(schema["inputBinding"].items())
215+
assert binding is not None
213216

214217
if "position" in binding:
215218
binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos)
@@ -221,12 +224,12 @@ def bind_input(self,
221224
value_from_expression = True
222225

223226
# Handle union types
224-
if isinstance(schema["type"], list):
227+
if isinstance(schema["type"], MutableSequence):
225228
bound_input = False
226229
for t in schema["type"]:
227230
if isinstance(t, string_types) and self.names.has_name(t, ""):
228231
avsc = self.names.get_name(t, "")
229-
elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
232+
elif isinstance(t, MutableMapping) and "name" in t and self.names.has_name(t["name"], ""):
230233
avsc = self.names.get_name(t["name"], "")
231234
else:
232235
avsc = AvroSchemaFromJSONData(t, self.names)
@@ -240,7 +243,7 @@ def bind_input(self,
240243
bound_input = True
241244
if not bound_input:
242245
raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
243-
elif isinstance(schema["type"], dict):
246+
elif isinstance(schema["type"], MutableMapping):
244247
st = copy.deepcopy(schema["type"])
245248
if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding:
246249
st["inputBinding"] = {}
@@ -289,7 +292,7 @@ def bind_input(self,
289292
if "secondaryFiles" not in datum:
290293
datum["secondaryFiles"] = []
291294
for sf in aslist(schema["secondaryFiles"]):
292-
if isinstance(sf, dict) or "$(" in sf or "${" in sf:
295+
if isinstance(sf, MutableMapping) or "$(" in sf or "${" in sf:
293296
sfpath = self.do_eval(sf, context=datum)
294297
else:
295298
sfpath = substitute(datum["basename"], sf)
@@ -301,7 +304,7 @@ def bind_input(self,
301304
if d["basename"] == sfname:
302305
found = True
303306
if not found:
304-
if isinstance(sfname, dict):
307+
if isinstance(sfname, MutableMapping):
305308
datum["secondaryFiles"].append(sfname)
306309
elif discover_secondaryFiles:
307310
datum["secondaryFiles"].append({
@@ -344,7 +347,7 @@ def _capture_files(f):
344347
return bindings
345348

346349
def tostr(self, value): # type: (Any) -> Text
347-
if isinstance(value, dict) and value.get("class") in ("File", "Directory"):
350+
if isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
348351
if "path" not in value:
349352
raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))
350353

@@ -372,8 +375,8 @@ def generate_arg(self, binding): # type: (Dict[Text, Any]) -> List[Text]
372375
with SourceLine(binding, "separate", WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
373376
raise WorkflowException("'separate' option can not be specified without prefix")
374377

375-
argl = [] # type: List[Dict[Text,Text]]
376-
if isinstance(value, list):
378+
argl = [] # type: MutableSequence[MutableMapping[Text, Text]]
379+
if isinstance(value, MutableSequence):
377380
if binding.get("itemSeparator") and value:
378381
argl = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
379382
elif binding.get("valueFrom"):
@@ -383,9 +386,9 @@ def generate_arg(self, binding): # type: (Dict[Text, Any]) -> List[Text]
383386
return [prefix]
384387
else:
385388
return []
386-
elif isinstance(value, dict) and value.get("class") in ("File", "Directory"):
389+
elif isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
387390
argl = [value]
388-
elif isinstance(value, dict):
391+
elif isinstance(value, MutableMapping):
389392
return [prefix] if prefix else []
390393
elif value is True and prefix:
391394
return [prefix]
@@ -407,10 +410,10 @@ def generate_arg(self, binding): # type: (Dict[Text, Any]) -> List[Text]
407410
def do_eval(self, ex, context=None, recursive=False, strip_whitespace=True):
408411
# type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
409412
if recursive:
410-
if isinstance(ex, dict):
413+
if isinstance(ex, MutableMapping):
411414
return {k: self.do_eval(v, context, recursive)
412415
for k, v in iteritems(ex)}
413-
if isinstance(ex, list):
416+
if isinstance(ex, MutableSequence):
414417
return [self.do_eval(v, context, recursive)
415418
for v in ex]
416419

‎cwltool/checker.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from collections import namedtuple
2-
from typing import Any, Dict, List, Optional
2+
from typing import Any, Dict, List, Optional, MutableMapping, MutableSequence
33
from typing_extensions import Text # pylint: disable=unused-import
44
# move to a regular typing import when Python 3.3-3.6 is no longer supported
55

@@ -15,7 +15,7 @@
1515

1616
def _get_type(tp):
1717
# type: (Any) -> Any
18-
if isinstance(tp, dict):
18+
if isinstance(tp, MutableMapping):
1919
if tp.get("type") not in ("array", "record", "enum"):
2020
return tp["type"]
2121
return tp
@@ -46,9 +46,9 @@ def merge_flatten_type(src):
4646
"""Return the merge flattened type of the source type
4747
"""
4848

49-
if isinstance(src, list):
49+
if isinstance(src, MutableSequence):
5050
return [merge_flatten_type(t) for t in src]
51-
if isinstance(src, dict) and src.get("type") == "array":
51+
if isinstance(src, MutableMapping) and src.get("type") == "array":
5252
return src
5353
return {"items": src, "type": "array"}
5454

@@ -65,7 +65,7 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
6565

6666
if src == "Any" or sink == "Any":
6767
return True
68-
if isinstance(src, dict) and isinstance(sink, dict):
68+
if isinstance(src, MutableMapping) and isinstance(sink, MutableMapping):
6969
if sink.get("not_connected") and strict:
7070
return False
7171
if src["type"] == "array" and sink["type"] == "array":
@@ -79,7 +79,7 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
7979
return False
8080
return True
8181
return can_assign_src_to_sink(src["type"], sink["type"], strict)
82-
elif isinstance(src, list):
82+
elif isinstance(src, MutableSequence):
8383
if strict:
8484
for t in src:
8585
if not can_assign_src_to_sink(t, sink):
@@ -90,7 +90,7 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
9090
if can_assign_src_to_sink(t, sink):
9191
return True
9292
return False
93-
elif isinstance(sink, list):
93+
elif isinstance(sink, MutableSequence):
9494
for t in sink:
9595
if can_assign_src_to_sink(src, t):
9696
return True
@@ -100,14 +100,14 @@ def can_assign_src_to_sink(src, sink, strict=False): # type: (Any, Any, bool) -
100100

101101

102102
def _compare_records(src, sink, strict=False):
103-
# type: (Dict[Text, Any], Dict[Text, Any], bool) -> bool
103+
# type: (MutableMapping[Text, Any], MutableMapping[Text, Any], bool) -> bool
104104
"""Compare two records, ensuring they have compatible fields.
105105
106106
This handles normalizing record names, which will be relative to workflow
107107
step, so that they can be compared.
108108
"""
109109

110-
def _rec_fields(rec): # type: (Dict[Text, Any]) -> Dict[Text, Any]
110+
def _rec_fields(rec): # type: (MutableMapping[Text, Any]) -> MutableMapping[Text, Any]
111111
out = {}
112112
for field in rec["fields"]:
113113
name = shortname(field["name"])
@@ -227,7 +227,7 @@ def check_all_types(src_dict, sinks, sourceField):
227227
for sink in sinks:
228228
if sourceField in sink:
229229
valueFrom = sink.get("valueFrom")
230-
if isinstance(sink[sourceField], list):
230+
if isinstance(sink[sourceField], MutableSequence):
231231
srcs_of_sink = [src_dict[parm_id] for parm_id in sink[sourceField]]
232232
linkMerge = sink.get("linkMerge", ("merge_nested"
233233
if len(sink[sourceField]) > 1 else None))

‎cwltool/command_line_tool.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import threading
1313
from functools import cmp_to_key, partial
1414
from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Union,
15-
cast)
15+
cast, MutableMapping, MutableSequence)
1616
from typing_extensions import Text, Type, TYPE_CHECKING # pylint: disable=unused-import
1717
# move to a regular typing import when Python 3.3-3.6 is no longer supported
1818

@@ -99,7 +99,7 @@ def run(self, runtimeContext): # type: (RuntimeContext) -> None
9999
self.output_callback({}, "permanentFail")
100100

101101
def job(self,
102-
job_order, # type: Dict[Text, Text]
102+
job_order, # type: MutableMapping[Text, Text]
103103
output_callbacks, # type: Callable[[Any, Any], Any]
104104
runtimeContext # type: RuntimeContext
105105
):
@@ -222,7 +222,7 @@ def check_valid_locations(fs_access, ob):
222222

223223
class CommandLineTool(Process):
224224
def __init__(self, toolpath_object, loadingContext):
225-
# type: (Dict[Text, Any], LoadingContext) -> None
225+
# type: (MutableMapping[Text, Any], LoadingContext) -> None
226226
super(CommandLineTool, self).__init__(toolpath_object, loadingContext)
227227
self.prov_obj = loadingContext.prov_obj
228228

@@ -272,7 +272,7 @@ def updatePathmap(self, outdir, pathmap, fn):
272272
self.updatePathmap(os.path.join(outdir, fn["basename"]), pathmap, ls)
273273

274274
def job(self,
275-
job_order, # type: Dict[Text, Text]
275+
job_order, # type: MutableMapping[Text, Text]
276276
output_callbacks, # type: Callable[[Any, Any], Any]
277277
runtimeContext # RuntimeContext
278278
):
@@ -510,7 +510,7 @@ def register_mut(f):
510510
def register_reader(f):
511511
if f["location"] not in muts:
512512
builder.mutation_manager.register_reader(j.name, f)
513-
readers[f["location"]] = copy.copy(f)
513+
readers[f["location"]] = copy.deepcopy(f)
514514

515515
for li in j.generatefiles["listing"]:
516516
li = cast(Dict[Text, Any], li)
@@ -704,7 +704,7 @@ def collect_output(self,
704704

705705
optional = False
706706
single = False
707-
if isinstance(schema["type"], list):
707+
if isinstance(schema["type"], MutableSequence):
708708
if "null" in schema["type"]:
709709
optional = True
710710
if "File" in schema["type"] or "Directory" in schema["type"]:
@@ -722,7 +722,7 @@ def collect_output(self,
722722
raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns))
723723
elif not r and optional:
724724
pass
725-
elif isinstance(r, list):
725+
elif isinstance(r, MutableSequence):
726726
if len(r) > 1:
727727
raise WorkflowException("Multiple matches for output item that is a single file.")
728728
else:
@@ -731,11 +731,11 @@ def collect_output(self,
731731
if "secondaryFiles" in schema:
732732
with SourceLine(schema, "secondaryFiles", WorkflowException, debug):
733733
for primary in aslist(r):
734-
if isinstance(primary, dict):
734+
if isinstance(primary, MutableMapping):
735735
primary.setdefault("secondaryFiles", [])
736736
pathprefix = primary["path"][0:primary["path"].rindex("/")+1]
737737
for sf in aslist(schema["secondaryFiles"]):
738-
if isinstance(sf, dict) or "$(" in sf or "${" in sf:
738+
if isinstance(sf, MutableMapping) or "$(" in sf or "${" in sf:
739739
sfpath = builder.do_eval(sf, context=primary)
740740
subst = False
741741
else:
@@ -766,7 +766,7 @@ def collect_output(self,
766766
if not r and optional:
767767
return None
768768

769-
if (not empty_and_optional and isinstance(schema["type"], dict)
769+
if (not empty_and_optional and isinstance(schema["type"], MutableMapping)
770770
and schema["type"]["type"] == "record"):
771771
out = {}
772772
for f in schema["type"]["fields"]:

‎cwltool/context.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import copy
22
import threading # pylint: disable=unused-import
3-
from typing import Any, Callable, Dict, Iterable, List, Optional
3+
from typing import Any, Callable, Dict, Iterable, List, MutableMapping, Optional
44
from typing_extensions import Text, TYPE_CHECKING # pylint: disable=unused-import
55
# move to a regular typing import when Python 3.3-3.6 is no longer supported
66
from schema_salad.ref_resolver import ( # pylint: disable=unused-import
@@ -28,13 +28,13 @@ def __init__(self, kwargs=None):
2828
if hasattr(self, k):
2929
setattr(self, k, v)
3030

31-
def make_tool_notimpl(toolpath_object, # type: Dict[Text, Any]
31+
def make_tool_notimpl(toolpath_object, # type: MutableMapping[Text, Any]
3232
loadingContext # type: LoadingContext
3333
): # type: (...) -> Process
3434
raise NotImplementedError()
3535

3636

37-
default_make_tool = make_tool_notimpl # type: Callable[[Dict[Text, Any], LoadingContext], Process]
37+
default_make_tool = make_tool_notimpl # type: Callable[[MutableMapping[Text, Any], LoadingContext], Process]
3838

3939
class LoadingContext(ContextBase):
4040

@@ -115,7 +115,7 @@ def __init__(self, kwargs=None):
115115
self.job_script_provider = None # type: Optional[DependenciesConfiguration]
116116
self.select_resources = None # type: Optional[select_resources_callable]
117117
self.eval_timeout = 20 # type: float
118-
self.postScatterEval = None # type: Optional[Callable[[Dict[Text, Any]], Dict[Text, Any]]]
118+
self.postScatterEval = None # type: Optional[Callable[[MutableMapping[Text, Any]], Dict[Text, Any]]]
119119
self.on_error = "stop" # type: Text
120120
self.strict_memory_limit = False # type: bool
121121

0 commit comments

Comments
 (0)