Fixing tests WIP

Author: Peter Amstutz

cwltool/argparser.py

@@ -2,7 +2,8 @@
 
 import argparse
 import os
-from typing import Any, AnyStr, Dict, List, Optional, Sequence, Union, cast, MutableMapping
+from typing import (Any, AnyStr, Dict, List, Optional, Sequence,
+                    Union, cast, MutableMapping, MutableSequence)
 
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
@@ -381,7 +382,7 @@ def add_argument(toolparser, name, inptype, records, description="",
         flag = "--"
 
     required = True
-    if isinstance(inptype, list):
+    if isinstance(inptype, MutableSequence):
         if inptype[0] == "null":
             required = False
             if len(inptype) == 2:

cwltool/builder.py

@@ -2,7 +2,8 @@
 
 import copy
 import logging
-from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple, MutableMapping
+from typing import (Any, Callable, Dict, List, Optional, Set, Union,
+                    Tuple, MutableMapping, MutableSequence)
 from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -221,7 +222,7 @@ def bind_input(self,
                 value_from_expression = True
 
         # Handle union types
-        if isinstance(schema["type"], list):
+        if isinstance(schema["type"], MutableSequence):
             bound_input = False
             for t in schema["type"]:
                 if isinstance(t, string_types) and self.names.has_name(t, ""):
@@ -373,7 +374,7 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
                 raise WorkflowException("'separate' option can not be specified without prefix")
 
         argl = []  # type: List[Dict[Text,Text]]
-        if isinstance(value, list):
+        if isinstance(value, MutableSequence):
             if binding.get("itemSeparator") and value:
                 argl = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
             elif binding.get("valueFrom"):
@@ -410,7 +411,7 @@ def do_eval(self, ex, context=None, recursive=False, strip_whitespace=True):
             if isinstance(ex, MutableMapping):
                 return {k: self.do_eval(v, context, recursive)
                         for k, v in iteritems(ex)}
-            if isinstance(ex, list):
+            if isinstance(ex, MutableSequence):
                 return [self.do_eval(v, context, recursive)
                         for v in ex]
 

cwltool/checker.py

@@ -1,5 +1,5 @@
 from collections import namedtuple
-from typing import Any, Dict, List, Optional, MutableMapping
+from typing import Any, Dict, List, Optional, MutableMapping, MutableSequence
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -46,7 +46,7 @@ def merge_flatten_type(src):
     """Return the merge flattened type of the source type
     """
 
-    if isinstance(src, list):
+    if isinstance(src, MutableSequence):
         return [merge_flatten_type(t) for t in src]
     if isinstance(src, MutableMapping) and src.get("type") == "array":
         return src
@@ -79,7 +79,7 @@ def can_assign_src_to_sink(src, sink, strict=False):  # type: (Any, Any, bool) -
                         return False
             return True
         return can_assign_src_to_sink(src["type"], sink["type"], strict)
-    elif isinstance(src, list):
+    elif isinstance(src, MutableSequence):
         if strict:
             for t in src:
                 if not can_assign_src_to_sink(t, sink):
@@ -90,7 +90,7 @@ def can_assign_src_to_sink(src, sink, strict=False):  # type: (Any, Any, bool) -
                 if can_assign_src_to_sink(t, sink):
                     return True
             return False
-    elif isinstance(sink, list):
+    elif isinstance(sink, MutableSequence):
         for t in sink:
             if can_assign_src_to_sink(src, t):
                 return True
@@ -227,7 +227,7 @@ def check_all_types(src_dict, sinks, sourceField):
     for sink in sinks:
         if sourceField in sink:
             valueFrom = sink.get("valueFrom")
-            if isinstance(sink[sourceField], list):
+            if isinstance(sink[sourceField], MutableSequence):
                 srcs_of_sink = [src_dict[parm_id] for parm_id in sink[sourceField]]
                 linkMerge = sink.get("linkMerge", ("merge_nested"
                                                    if len(sink[sourceField]) > 1 else None))

cwltool/command_line_tool.py

@@ -12,7 +12,7 @@
 import threading
 from functools import cmp_to_key, partial
 from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Union,
-                    cast, MutableMapping)
+                    cast, MutableMapping, MutableSequence)
 from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -706,7 +706,7 @@ def collect_output(self,
 
             optional = False
             single = False
-            if isinstance(schema["type"], list):
+            if isinstance(schema["type"], MutableSequence):
                 if "null" in schema["type"]:
                     optional = True
                 if "File" in schema["type"] or "Directory" in schema["type"]:
@@ -724,7 +724,7 @@ def collect_output(self,
                         raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns))
                 elif not r and optional:
                     pass
-                elif isinstance(r, list):
+                elif isinstance(r, MutableSequence):
                     if len(r) > 1:
                         raise WorkflowException("Multiple matches for output item that is a single file.")
                     else:

cwltool/expression.py

@@ -2,7 +2,7 @@
 
 import copy
 import re
-from typing import Any, Dict, List, Optional, Union, MutableMapping
+from typing import Any, Dict, List, Optional, Union, MutableMapping, MutableSequence
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 import six
@@ -130,7 +130,7 @@ def next_seg(parsed_string, remaining_string, current_value):  # type: (Text, Te
             key = next_segment_str[2:-2].replace("\\'", "'").replace('\\"', '"')
 
         if key:
-            if isinstance(current_value, list) and key == "length" and not remaining_string[m.end(0):]:
+            if isinstance(current_value, MutableSequence) and key == "length" and not remaining_string[m.end(0):]:
                 return len(current_value)
             if not isinstance(current_value, MutableMapping):
                 raise WorkflowException("%s is a %s, cannot index on string '%s'" % (parsed_string, type(current_value).__name__, key))
@@ -141,7 +141,7 @@ def next_seg(parsed_string, remaining_string, current_value):  # type: (Text, Te
                 key = int(next_segment_str[1:-1])
             except ValueError as v:
                 raise WorkflowException(u(str(v)))
-            if not isinstance(current_value, list):
+            if not isinstance(current_value, MutableSequence):
                 raise WorkflowException("%s is a %s, cannot index on int '%s'" % (parsed_string, type(current_value).__name__, key))
             if key >= len(current_value):
                 raise WorkflowException("%s list index %i out of range" % (parsed_string, key))

cwltool/job.py

@@ -13,8 +13,8 @@
 from threading import Timer
 from abc import ABCMeta, abstractmethod
 from io import IOBase, open  # pylint: disable=redefined-builtin
-from typing import (IO, Any, AnyStr, Callable, Dict, Iterable, List, MutableMapping,
-                    Optional, Union, cast)
+from typing import (IO, Any, AnyStr, Callable, Dict, Iterable, List,
+                    Optional, Union, cast, MutableMapping, MutableSequence)
 
 from typing_extensions import Text, TYPE_CHECKING  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
@@ -116,7 +116,7 @@ def deref_links(outputs):  # type: (Any) -> None
         else:
             for v in outputs.values():
                 deref_links(v)
-    if isinstance(outputs, list):
+    if isinstance(outputs, MutableSequence):
         for output in outputs:
             deref_links(output)
 
@@ -523,6 +523,7 @@ def _job_popen(
         if stderr_path is not None:
             stderr = open(stderr_path, "wb")
 
+        print("ZZZ", commands)
         sproc = subprocess.Popen(commands,
                                  shell=False,
                                  close_fds=not onWindows(),

cwltool/load_tool.py

@@ -6,7 +6,8 @@
 import os
 import re
 import uuid
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast, MutableMapping
+from typing import (Any, Callable, Dict, List, Optional, Tuple,
+                    Union, cast, MutableMapping, MutableSequence)
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -172,7 +173,7 @@ def _convert_stdstreams_to_files(workflowobj):
         else:
             for entry in itervalues(workflowobj):
                 _convert_stdstreams_to_files(entry)
-    if isinstance(workflowobj, list):
+    if isinstance(workflowobj, MutableSequence):
         for entry in workflowobj:
             _convert_stdstreams_to_files(entry)
 
@@ -187,7 +188,7 @@ def _add_blank_ids(workflowobj):
             workflowobj["run"]["id"] = Text(uuid.uuid4())
         for entry in itervalues(workflowobj):
             _add_blank_ids(entry)
-    if isinstance(workflowobj, list):
+    if isinstance(workflowobj, MutableSequence):
         for entry in workflowobj:
             _add_blank_ids(entry)
 
@@ -206,7 +207,7 @@ def validate_document(document_loader,  # type: Loader
     # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
     """Validate a CWL document."""
 
-    if isinstance(workflowobj, list):
+    if isinstance(workflowobj, MutableSequence):
         workflowobj = cmap({
             "$graph": workflowobj
         }, fn=uri)
@@ -318,7 +319,7 @@ def make_tool(document_loader,    # type: Loader
     resolveduri = document_loader.resolve_ref(uri)[0]
 
     processobj = None
-    if isinstance(resolveduri, list):
+    if isinstance(resolveduri, MutableSequence):
         for obj in resolveduri:
             if obj['id'].endswith('#main'):
                 processobj = obj

cwltool/pack.py

@@ -1,7 +1,8 @@
 from __future__ import absolute_import
 
 import copy
-from typing import Any, Callable, Dict, List, Optional, Set, Union, cast, MutableMapping
+from typing import (Any, Callable, Dict, List, Optional, Set,
+                    Union, cast, MutableMapping, MutableSequence)
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -15,7 +16,7 @@
 
 
 def flatten_deps(d, files):  # type: (Any, Set[Text]) -> None
-    if isinstance(d, list):
+    if isinstance(d, MutableSequence):
         for s in d:
             flatten_deps(s, files)
     elif isinstance(d, MutableMapping):
@@ -33,7 +34,7 @@ def find_run(d,        # type: Any
              loadref,  # type: LoadRefType
              runs      # type: Set[Text]
             ):  # type: (...) -> None
-    if isinstance(d, list):
+    if isinstance(d, MutableSequence):
         for s in d:
             find_run(s, loadref, runs)
     elif isinstance(d, MutableMapping):
@@ -46,7 +47,7 @@ def find_run(d,        # type: Any
 
 
 def find_ids(d, ids):  # type: (Any, Set[Text]) -> None
-    if isinstance(d, list):
+    if isinstance(d, MutableSequence):
         for s in d:
             find_ids(s, ids)
     elif isinstance(d, MutableMapping):
@@ -59,7 +60,7 @@ def find_ids(d, ids):  # type: (Any, Set[Text]) -> None
 
 def replace_refs(d, rewrite, stem, newstem):
     # type: (Any, Dict[Text, Text], Text, Text) -> None
-    if isinstance(d, list):
+    if isinstance(d, MutableSequence):
         for s, v in enumerate(d):
             if isinstance(v, string_types):
                 if v in rewrite:
@@ -86,7 +87,7 @@ def replace_refs(d, rewrite, stem, newstem):
 
 def import_embed(d, seen):
     # type: (Any, Set[Text]) -> None
-    if isinstance(d, list):
+    if isinstance(d, MutableSequence):
         for v in d:
             import_embed(v, seen)
     elif isinstance(d, MutableMapping):
@@ -116,7 +117,7 @@ def pack(document_loader,  # type: Loader
     document_loader.idx = {}
     if isinstance(processobj, MutableMapping):
         document_loader.idx[processobj["id"]] = CommentedMap(iteritems(processobj))
-    elif isinstance(processobj, list):
+    elif isinstance(processobj, MutableSequence):
         _, frag = urllib.parse.urldefrag(uri)
         for po in processobj:
             if not frag:

cwltool/pathmapper.py

@@ -7,8 +7,8 @@
 import uuid
 from functools import partial  # pylint: disable=unused-import
 from tempfile import NamedTemporaryFile
-from typing import (Any, Callable, Dict, List, MutableMapping, Optional, Set, Tuple,
-                    Union)
+from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
+                    Optional, Set, Tuple, Union)
 from typing_extensions import Text  # pylint: disable=unused-import
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
@@ -36,7 +36,7 @@ def adjustFiles(rec, op):  # type: (Any, Union[Callable[..., Any], partial[Any]]
             rec["path"] = op(rec["path"])
         for d in rec:
             adjustFiles(rec[d], op)
-    if isinstance(rec, list):
+    if isinstance(rec, MutableSequence):
         for d in rec:
             adjustFiles(d, op)