Merge remote-tracking branch 'origin/master' into bad_type_error

Author: mr-c

cwltool/builder.py

@@ -76,12 +76,12 @@ def checkFormat(actualFile, inputFormats, ontology):
         if not af:
             continue
         if "format" not in af:
-            raise validate.ValidationException(u"Missing required 'format' for File %s" % af)
+            raise validate.ValidationException(u"File has no 'format' defined: %s" % json.dumps(af, indent=4))
         for inpf in aslist(inputFormats):
             if af["format"] == inpf or formatSubclassOf(af["format"], inpf, ontology, set()):
                 return
         raise validate.ValidationException(
-            u"Incompatible file format, expected format(s) %s but file object is: %s" % (inputFormats, json.dumps(af, indent=4)))
+            u"File has an incompatible format: %s" % json.dumps(af, indent=4))
 
 class Builder(object):
     def __init__(self):  # type: () -> None
@@ -237,7 +237,10 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None, discover_secon
                     normalizeFilesDirs(datum["secondaryFiles"])
 
                 if "format" in schema:
-                    checkFormat(datum, self.do_eval(schema["format"]), self.formatgraph)
+                    try:
+                        checkFormat(datum, self.do_eval(schema["format"]), self.formatgraph)
+                    except validate.ValidationException as ve:
+                        raise WorkflowException("Expected value of '%s' to have format %s but\n  %s" % (schema["name"], schema["format"], ve))
 
                 def _capture_files(f):
                     self.files.append(f)

cwltool/docker.py

@@ -4,22 +4,25 @@
 import os
 import re
 import shutil
-import subprocess
 import sys
 import tempfile
 from io import open
-
 import datetime
+import threading
+
 import requests
 from typing import (Dict, List, Text, Any, MutableMapping, Set)
-import threading
 
 from .docker_id import docker_vm_id
 from .errors import WorkflowException
 from .job import ContainerCommandLineJob
 from .pathmapper import PathMapper, ensure_writable
 from .secrets import SecretStore
 from .utils import docker_windows_path_adjust, onWindows
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
 
 _logger = logging.getLogger("cwltool")
 

cwltool/docker_id.py

@@ -1,8 +1,12 @@
 from __future__ import print_function
 from __future__ import absolute_import
-
-import subprocess
+import os
+import sys
 from typing import List, Text, Tuple
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
 
 
 def docker_vm_id():  # type: () -> Tuple[int, int]

cwltool/executors.py

@@ -6,6 +6,7 @@
 from abc import ABCMeta, abstractmethod
 
 from typing import Dict, Text, Any, Tuple, Set, List
+from schema_salad.validate import ValidationException
 
 from .builder import Builder
 from .errors import WorkflowException
@@ -14,6 +15,7 @@
 from .process import relocateOutputs, cleanIntermediate, Process
 from . import loghandler
 
+
 _logger = logging.getLogger("cwltool")
 
 class JobExecutor(object):
@@ -107,7 +109,7 @@ def run_jobs(self,
                 else:
                     logger.error("Workflow cannot make any more progress.")
                     break
-        except WorkflowException:
+        except (ValidationException, WorkflowException):
             raise
         except Exception as e:
             logger.exception("Got workflow error")

cwltool/job.py

@@ -9,7 +9,6 @@
 import re
 import shutil
 import stat
-import subprocess
 import sys
 import tempfile
 from abc import ABCMeta, abstractmethod
@@ -28,6 +27,11 @@
 from .secrets import SecretStore
 from .utils import bytes2str_in_dicts
 from .utils import copytree_with_merge, onWindows
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
+
 
 _logger = logging.getLogger("cwltool")
 

cwltool/resolver.py

@@ -1,29 +1,38 @@
 from __future__ import absolute_import
 import logging
 import os
+import sys
 
+if sys.version_info < (3, 4):
+    from pathlib2 import Path
+else:
+    from pathlib import Path
 from six.moves import urllib
 
-from schema_salad.ref_resolver import file_uri
 
 _logger = logging.getLogger("cwltool")
 
 
 def resolve_local(document_loader, uri):
-    if uri.startswith("/"):
-        return None
-    shares = [os.environ.get("XDG_DATA_HOME", os.path.join(os.path.expanduser('~'), ".local", "share"))]
-    shares.extend(os.environ.get("XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":"))
-    shares = [os.path.join(s, "commonwl", uri) for s in shares]
-    shares.insert(0, os.path.join(os.getcwd(), uri))
+    if uri.startswith("/") and os.path.exists(uri):
+        return Path(uri).as_uri()
+    if os.path.exists(urllib.parse.urlparse(
+            urllib.parse.urldefrag(
+                "{}/{}".format(Path.cwd().as_uri(), uri))[0])[2]):
+        return "{}/{}".format(Path.cwd().as_uri(), uri)
+    sharepaths = [os.environ.get("XDG_DATA_HOME", os.path.join(
+        os.path.expanduser('~'), ".local", "share"))]
+    sharepaths.extend(os.environ.get(
+        "XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":"))
+    shares = [os.path.join(s, "commonwl", uri) for s in sharepaths]
 
     _logger.debug("Search path is %s", shares)
 
-    for s in shares:
-        if os.path.exists(s):
-            return file_uri(s)
-        if os.path.exists("%s.cwl" % s):
-            return file_uri(s)
+    for path in shares:
+        if os.path.exists(path):
+            return Path(uri).as_uri()
+        if os.path.exists("{}.cwl".format(path)):
+            return Path("{}.cwl".format(path)).as_uri()
     return None
 
 
@@ -32,7 +41,6 @@ def tool_resolver(document_loader, uri):
         ret = r(document_loader, uri)
         if ret is not None:
             return ret
-    return file_uri(os.path.abspath(uri), split_frag=True)
 
 
 ga4gh_tool_registries = ["https://dockstore.org:8443"]

cwltool/sandboxjs.py

@@ -5,20 +5,23 @@
 import os
 import re
 import select
-import subprocess
 import threading
 import sys
 from io import BytesIO
 from typing import Any, Dict, List, Mapping, Text, Tuple, Union
-from .utils import onWindows
-from pkg_resources import resource_stream
-
 import six
+from pkg_resources import resource_stream
+from .utils import onWindows
 
 try:
     import queue  # type: ignore
 except ImportError:
     import Queue as queue  # type: ignore
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
+
 
 class JavascriptException(Exception):
     pass
@@ -194,7 +197,7 @@ def terminate():
 
     PROCESS_FINISHED_STR = "r1cepzbhUTxtykz5XTC4\n"
 
-    def process_finished(): # type: () -> bool
+    def process_finished():  # type: () -> bool
         return stdout_buf.getvalue().decode('utf-8').endswith(PROCESS_FINISHED_STR) and \
             stderr_buf.getvalue().decode('utf-8').endswith(PROCESS_FINISHED_STR)
 
@@ -363,4 +366,4 @@ def fn_linenum():  # type: () -> Text
         return json.loads(stdout)
     except ValueError as e:
         raise JavascriptException(u"%s\nscript was:\n%s\nstdout was: '%s'\nstderr was: '%s'\n" %
-                                    (e, fn_linenum(), stdout, stderr))
+                                  (e, fn_linenum(), stdout, stderr))