Support ruamel.yaml 0.15.52+

Use isinstance(..., MutableMapping) instead of isinstance(..., dict)
because CommentedMap no longer inherits from ordereddict.

Also convert some instances of copy.copy() to copy.deepcopy() because
using copy.copy() to make a shallow copy of CommentedMap no longer has
the desired behavior.

Author: Peter Amstutz

cwltool/argparser.py

@@ -2,7 +2,7 @@
 
 import argparse
 import os
-from typing import Any, AnyStr, Dict, List, Optional, Sequence, Union, cast
+from typing import Any, AnyStr, Dict, List, Optional, Sequence, Union, cast, MutableMapping
 
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
@@ -398,16 +398,16 @@ def add_argument(toolparser, name, inptype, records, description="",
         action = cast(argparse.Action, FileAction)
     elif inptype == "Directory":
         action = cast(argparse.Action, DirectoryAction)
-    elif isinstance(inptype, dict) and inptype["type"] == "array":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
         if inptype["items"] == "File":
             action = cast(argparse.Action, FileAppendAction)
         elif inptype["items"] == "Directory":
             action = cast(argparse.Action, DirectoryAppendAction)
         else:
             action = "append"
-    elif isinstance(inptype, dict) and inptype["type"] == "enum":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
         atype = Text
-    elif isinstance(inptype, dict) and inptype["type"] == "record":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
         records.append(name)
         for field in inptype['fields']:
             fieldname = name + "." + shortname(field['name'])

cwltool/builder.py

@@ -2,7 +2,7 @@
 
 import copy
 import logging
-from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple
+from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple, MutableMapping
 from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -208,8 +208,8 @@ def bind_input(self,
         bindings = []  # type: List[Dict[Text,Text]]
         binding = None  # type: Optional[Dict[Text,Any]]
         value_from_expression = False
-        if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
-            binding = copy.copy(schema["inputBinding"])
+        if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
+            binding = copy.deepcopy(schema["inputBinding"])
 
             if "position" in binding:
                 binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos)
@@ -226,7 +226,7 @@ def bind_input(self,
             for t in schema["type"]:
                 if isinstance(t, string_types) and self.names.has_name(t, ""):
                     avsc = self.names.get_name(t, "")
-                elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
+                elif isinstance(t, MutableMapping) and "name" in t and self.names.has_name(t["name"], ""):
                     avsc = self.names.get_name(t["name"], "")
                 else:
                     avsc = AvroSchemaFromJSONData(t, self.names)
@@ -240,7 +240,7 @@ def bind_input(self,
                         bound_input = True
             if not bound_input:
                 raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
-        elif isinstance(schema["type"], dict):
+        elif isinstance(schema["type"], MutableMapping):
             st = copy.deepcopy(schema["type"])
             if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding:
                 st["inputBinding"] = {}
@@ -289,7 +289,7 @@ def bind_input(self,
                     if "secondaryFiles" not in datum:
                         datum["secondaryFiles"] = []
                     for sf in aslist(schema["secondaryFiles"]):
-                        if isinstance(sf, dict) or "$(" in sf or "${" in sf:
+                        if isinstance(sf, MutableMapping) or "$(" in sf or "${" in sf:
                             sfpath = self.do_eval(sf, context=datum)
                         else:
                             sfpath = substitute(datum["basename"], sf)
@@ -301,7 +301,7 @@ def bind_input(self,
                                 if d["basename"] == sfname:
                                     found = True
                             if not found:
-                                if isinstance(sfname, dict):
+                                if isinstance(sfname, MutableMapping):
                                     datum["secondaryFiles"].append(sfname)
                                 elif discover_secondaryFiles:
                                     datum["secondaryFiles"].append({
@@ -344,7 +344,7 @@ def _capture_files(f):
         return bindings
 
     def tostr(self, value):  # type: (Any) -> Text
-        if isinstance(value, dict) and value.get("class") in ("File", "Directory"):
+        if isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
             if "path" not in value:
                 raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))
 
@@ -383,9 +383,9 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
                 return [prefix]
             else:
                 return []
-        elif isinstance(value, dict) and value.get("class") in ("File", "Directory"):
+        elif isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
             argl = [value]
-        elif isinstance(value, dict):
+        elif isinstance(value, MutableMapping):
             return [prefix] if prefix else []
         elif value is True and prefix:
             return [prefix]
@@ -407,7 +407,7 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
     def do_eval(self, ex, context=None, recursive=False, strip_whitespace=True):
         # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
         if recursive:
-            if isinstance(ex, dict):
+            if isinstance(ex, MutableMapping):
                 return {k: self.do_eval(v, context, recursive)
                         for k, v in iteritems(ex)}
             if isinstance(ex, list):

cwltool/command_line_tool.py

@@ -12,7 +12,7 @@
 import threading
 from functools import cmp_to_key, partial
 from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Union,
-                    cast)
+                    cast, MutableMapping)
 from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -510,7 +510,7 @@ def register_mut(f):
             def register_reader(f):
                 if f["location"] not in muts:
                     builder.mutation_manager.register_reader(j.name, f)
-                    readers[f["location"]] = copy.copy(f)
+                    readers[f["location"]] = copy.deepcopy(f)
 
             for li in j.generatefiles["listing"]:
                 li = cast(Dict[Text, Any], li)
@@ -693,7 +693,7 @@ def collect_output(self,
                             if binding.get("loadContents") or compute_checksum:
                                 contents = f.read(CONTENT_LIMIT)
                             if binding.get("loadContents"):
-                                files["contents"] = contents.decode("utf-8") 
+                                files["contents"] = contents.decode("utf-8")
                             if compute_checksum:
                                 checksum = hashlib.sha1()
                                 while contents != b"":
@@ -733,11 +733,11 @@ def collect_output(self,
             if "secondaryFiles" in schema:
                 with SourceLine(schema, "secondaryFiles", WorkflowException, debug):
                     for primary in aslist(r):
-                        if isinstance(primary, dict):
+                        if isinstance(primary, MutableMapping):
                             primary.setdefault("secondaryFiles", [])
                             pathprefix = primary["path"][0:primary["path"].rindex("/")+1]
                             for sf in aslist(schema["secondaryFiles"]):
-                                if isinstance(sf, dict) or "$(" in sf or "${" in sf:
+                                if isinstance(sf, MutableMapping) or "$(" in sf or "${" in sf:
                                     sfpath = builder.do_eval(sf, context=primary)
                                     subst = False
                                 else:
@@ -768,7 +768,7 @@ def collect_output(self,
             if not r and optional:
                 return None
 
-        if (not empty_and_optional and isinstance(schema["type"], dict)
+        if (not empty_and_optional and isinstance(schema["type"], MutableMapping)
                 and schema["type"]["type"] == "record"):
             out = {}
             for f in schema["type"]["fields"]:

cwltool/expression.py

@@ -2,7 +2,7 @@
 
 import copy
 import re
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, MutableMapping
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 import six
@@ -132,7 +132,7 @@ def next_seg(parsed_string, remaining_string, current_value):  # type: (Text, Te
         if key:
             if isinstance(current_value, list) and key == "length" and not remaining_string[m.end(0):]:
                 return len(current_value)
-            if not isinstance(current_value, dict):
+            if not isinstance(current_value, MutableMapping):
                 raise WorkflowException("%s is a %s, cannot index on string '%s'" % (parsed_string, type(current_value).__name__, key))
             if key not in current_value:
                 raise WorkflowException("%s does not contain key '%s'" % (parsed_string, key))
@@ -257,7 +257,7 @@ def do_eval(ex,                       # type: Union[Text, Dict]
             strip_whitespace=True     # type: bool
            ):  # type: (...) -> Any
 
-    runtime = copy.copy(resources)  # type: Dict[str, Any]
+    runtime = copy.deepcopy(resources)  # type: Dict[str, Any]
     runtime["tmpdir"] = docker_windows_path_adjust(tmpdir)
     runtime["outdir"] = docker_windows_path_adjust(outdir)
 

cwltool/job.py

@@ -107,7 +107,7 @@
 
 
 def deref_links(outputs):  # type: (Any) -> None
-    if isinstance(outputs, dict):
+    if isinstance(outputs, MutableMapping):
         if outputs.get("class") == "File":
             st = os.lstat(outputs["path"])
             if stat.S_ISLNK(st.st_mode):

cwltool/load_tool.py

@@ -6,7 +6,7 @@
 import os
 import re
 import uuid
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast, MutableMapping
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -111,7 +111,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
         uri, fileuri = resolve_tool_uri(argsworkflow, resolver=resolver,
                                         document_loader=document_loader)
         workflowobj = document_loader.fetch(fileuri)
-    elif isinstance(argsworkflow, dict):
+    elif isinstance(argsworkflow, MutableMapping):
         uri = "#" + Text(id(argsworkflow))
         workflowobj = cast(CommentedMap, cmap(argsworkflow, fn=uri))
     else:
@@ -124,7 +124,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
 def _convert_stdstreams_to_files(workflowobj):
     # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None
 
-    if isinstance(workflowobj, dict):
+    if isinstance(workflowobj, MutableMapping):
         if workflowobj.get('class') == 'CommandLineTool':
             with SourceLine(workflowobj, "outputs", ValidationException,
                             _logger.isEnabledFor(logging.DEBUG)):
@@ -179,9 +179,9 @@ def _convert_stdstreams_to_files(workflowobj):
 def _add_blank_ids(workflowobj):
     # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None
 
-    if isinstance(workflowobj, dict):
+    if isinstance(workflowobj, MutableMapping):
         if ("run" in workflowobj and
-                isinstance(workflowobj["run"], dict) and
+                isinstance(workflowobj["run"], MutableMapping) and
                 "id" not in workflowobj["run"] and
                 "$import" not in workflowobj["run"]):
             workflowobj["run"]["id"] = Text(uuid.uuid4())
@@ -211,7 +211,7 @@ def validate_document(document_loader,  # type: Loader
             "$graph": workflowobj
         }, fn=uri)
 
-    if not isinstance(workflowobj, dict):
+    if not isinstance(workflowobj, MutableMapping):
         raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
             type(workflowobj), workflowobj))
 
@@ -329,7 +329,7 @@ def make_tool(document_loader,    # type: Loader
                 "one of #%s" % ", #".join(
                     urllib.parse.urldefrag(i["id"])[1] for i in resolveduri
                     if "id" in i))
-    elif isinstance(resolveduri, dict):
+    elif isinstance(resolveduri, MutableMapping):
         processobj = resolveduri
     else:
         raise Exception("Must resolve to list or dict")

cwltool/pack.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 
 import copy
-from typing import Any, Callable, Dict, List, Optional, Set, Union, cast
+from typing import Any, Callable, Dict, List, Optional, Set, Union, cast, MutableMapping
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -18,7 +18,7 @@ def flatten_deps(d, files):  # type: (Any, Set[Text]) -> None
     if isinstance(d, list):
         for s in d:
             flatten_deps(s, files)
-    elif isinstance(d, dict):
+    elif isinstance(d, MutableMapping):
         if d["class"] == "File":
             files.add(d["location"])
         if "secondaryFiles" in d:
@@ -36,7 +36,7 @@ def find_run(d,        # type: Any
     if isinstance(d, list):
         for s in d:
             find_run(s, loadref, runs)
-    elif isinstance(d, dict):
+    elif isinstance(d, MutableMapping):
         if "run" in d and isinstance(d["run"], string_types):
             if d["run"] not in runs:
                 runs.add(d["run"])
@@ -49,7 +49,7 @@ def find_ids(d, ids):  # type: (Any, Set[Text]) -> None
     if isinstance(d, list):
         for s in d:
             find_ids(s, ids)
-    elif isinstance(d, dict):
+    elif isinstance(d, MutableMapping):
         for i in ("id", "name"):
             if i in d and isinstance(d[i], string_types):
                 ids.add(d[i])
@@ -69,7 +69,7 @@ def replace_refs(d, rewrite, stem, newstem):
                     rewrite[v] = d[s]
             else:
                 replace_refs(v, rewrite, stem, newstem)
-    elif isinstance(d, dict):
+    elif isinstance(d, MutableMapping):
         for s, v in d.items():
             if isinstance(v, string_types):
                 if v in rewrite:
@@ -89,7 +89,7 @@ def import_embed(d, seen):
     if isinstance(d, list):
         for v in d:
             import_embed(v, seen)
-    elif isinstance(d, dict):
+    elif isinstance(d, MutableMapping):
         for n in ("id", "name"):
             if n in d:
                 if d[n] in seen:
@@ -114,7 +114,7 @@ def pack(document_loader,  # type: Loader
 
     document_loader = SubLoader(document_loader)
     document_loader.idx = {}
-    if isinstance(processobj, dict):
+    if isinstance(processobj, MutableMapping):
         document_loader.idx[processobj["id"]] = CommentedMap(iteritems(processobj))
     elif isinstance(processobj, list):
         _, frag = urllib.parse.urldefrag(uri)

cwltool/pathmapper.py

@@ -31,7 +31,7 @@
 def adjustFiles(rec, op):  # type: (Any, Union[Callable[..., Any], partial[Any]]) -> None
     """Apply a mapping function to each File path in the object `rec`."""
 
-    if isinstance(rec, dict):
+    if isinstance(rec, MutableMapping):
         if rec.get("class") == "File":
             rec["path"] = op(rec["path"])
         for d in rec: