Merge branch 'master' into singularity-support

Author: Anton Khodak

cwltool/builder.py

@@ -217,6 +217,9 @@ def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
 
         prefix = binding.get("prefix")
         sep = binding.get("separate", True)
+        if prefix is None and not sep:
+            with SourceLine(binding, "separate", WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
+                raise WorkflowException("'separate' option can not be specified without prefix")
 
         l = []  # type: List[Dict[Text,Text]]
         if isinstance(value, list):
@@ -249,8 +252,8 @@ def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
 
         return [a for a in args if a is not None]
 
-    def do_eval(self, ex, context=None, pull_image=True, recursive=False):
-        # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
+    def do_eval(self, ex, context=None, pull_image=True, recursive=False, strip_whitespace=True):
+        # type: (Union[Dict[Text, Text], Text], Any, bool, bool, bool) -> Any
         if recursive:
             if isinstance(ex, dict):
                 return {k: self.do_eval(v, context, pull_image, recursive) for k, v in iteritems(ex)}
@@ -265,4 +268,5 @@ def do_eval(self, ex, context=None, pull_image=True, recursive=False):
                                   timeout=self.timeout,
                                   debug=self.debug,
                                   js_console=self.js_console,
-                                  force_docker_pull=self.force_docker_pull)
+                                  force_docker_pull=self.force_docker_pull,
+                                  strip_whitespace=strip_whitespace)

cwltool/command_line_tool.py

@@ -71,7 +71,8 @@ def execute(self, t,  # type: Process
         if self.final_output and self.final_output[0] and finaloutdir:
             self.final_output[0] = relocateOutputs(self.final_output[0], finaloutdir,
                                                    self.output_dirs, kwargs.get("move_outputs"),
-                                                   kwargs["make_fs_access"](""))
+                                                   kwargs["make_fs_access"](""),
+                                                   kwargs["compute_checksum"])
 
         if kwargs.get("rm_tmpdir"):
             cleanIntermediate(self.output_dirs)

cwltool/draft2tool.py

@@ -173,9 +173,10 @@ def evaluator(ex, jslib, obj, fullJS=False, timeout=None, force_docker_pull=Fals
 
 def interpolate(scan, rootvars,
                 timeout=None, fullJS=None, jslib="", force_docker_pull=False,
-                debug=False, js_console=False):
-    # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool, bool, bool) -> JSON
-    scan = scan.strip()
+                debug=False, js_console=False, strip_whitespace=True):
+    # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool, bool, bool, bool) -> JSON
+    if strip_whitespace:
+        scan = scan.strip()
     parts = []
     w = scanner(scan)
     while w:
@@ -202,8 +203,9 @@ def interpolate(scan, rootvars,
 
 
 def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
-            context=None, pull_image=True, timeout=None, force_docker_pull=False, debug=False, js_console=False):
-    # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool, bool, bool) -> Any
+            context=None, pull_image=True, timeout=None, force_docker_pull=False,
+            debug=False, js_console=False, strip_whitespace=True):
+    # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool, bool, bool, bool) -> Any
 
     runtime = copy.copy(resources)
     runtime["tmpdir"] = docker_windows_path_adjust(tmpdir)
@@ -231,7 +233,8 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
                                jslib=jslib,
                                force_docker_pull=force_docker_pull,
                                debug=debug,
-                               js_console=js_console)
+                               js_console=js_console,
+                               strip_whitespace=strip_whitespace)
 
         except Exception as e:
             raise WorkflowException("Expression evaluation error:\n%s" % e)

cwltool/executors.py

@@ -20,7 +20,7 @@
 from avro.schema import Names
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.ref_resolver import ContextType, Fetcher, Loader, file_uri
-from schema_salad.sourceline import cmap
+from schema_salad.sourceline import cmap, SourceLine
 from schema_salad.validate import ValidationException
 
 from . import process, update
@@ -124,6 +124,9 @@ def _convert_stdstreams_to_files(workflowobj):
     if isinstance(workflowobj, dict):
         if workflowobj.get('class') == 'CommandLineTool':
             for out in workflowobj.get('outputs', []):
+                if type(out) is not CommentedMap:
+                    with SourceLine(workflowobj, "outputs", ValidationException, _logger.isEnabledFor(logging.DEBUG)):
+                        raise ValidationException("Output '%s' is not a valid OutputParameter." % out)
                 for streamtype in ['stdout', 'stderr']:
                     if out.get('type') == streamtype:
                         if 'outputBinding' in out:
@@ -215,9 +218,10 @@ def validate_document(document_loader,  # type: Loader
         if metadata and 'cwlVersion' in metadata:
             workflowobj['cwlVersion'] = metadata['cwlVersion']
         else:
-            raise ValidationException("No cwlVersion found."
-                "Use the following syntax in your CWL document to declare "
-                "the version: cwlVersion: <version>")
+            raise ValidationException(
+                  "No cwlVersion found. "
+                  "Use the following syntax in your CWL document to declare the version: cwlVersion: <version>.\n"
+                  "Note: if this is a CWL draft-2 (pre v1.0) document then it will need to be upgraded first.")
 
     if not isinstance(workflowobj["cwlVersion"], (str, Text)):
         raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"]))
@@ -227,16 +231,14 @@ def validate_document(document_loader,  # type: Loader
         workflowobj["cwlVersion"])
     if workflowobj["cwlVersion"] not in list(ALLUPDATES):
         # print out all the Supported Versions of cwlVersion
-        versions = list(ALLUPDATES)  # ALLUPDATES is a dict
+        versions = []
+        for version in list(ALLUPDATES):
+            if "dev" in version:
+                version += " (with --enable-dev flag only)"
+            versions.append(version)
         versions.sort()
-        raise ValidationException("'cwlVersion' not valid. Supported CWL versions are: \n{}".format("\n".join(versions)))
-
-    if workflowobj["cwlVersion"] == "draft-2":
-        workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1(
-            workflowobj, document_loader, uri, update_steps=False)))
-        if "@graph" in workflowobj:
-            workflowobj["$graph"] = workflowobj["@graph"]
-            del workflowobj["@graph"]
+        raise ValidationException("The CWL reference runner no longer supports pre CWL v1.0 documents. "
+                                  "Supported versions are: \n{}".format("\n".join(versions)))
 
     (sch_document_loader, avsc_names) = \
         process.get_schema(workflowobj["cwlVersion"])[:2]
@@ -257,8 +259,6 @@ def validate_document(document_loader,  # type: Loader
         raise ValidationException("Workflow must be a dict or list.")
 
     if not new_metadata:
-        if not isinstance(processobj, dict):
-            raise ValidationException("Draft-2 workflows must be a dict.")
         new_metadata = cast(CommentedMap, cmap(
             {"$namespaces": processobj.get("$namespaces", {}),
              "$schemas": processobj.get("$schemas", []),

cwltool/expression.py

@@ -21,7 +21,7 @@
 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.sourceline import strip_dup_lineno
 
-from . import draft2tool, workflow
+from . import command_line_tool, workflow
 from .argparser import arg_parser, generate_parser, DEFAULT_TMP_PREFIX
 from .cwlrdf import printdot, printrdf
 from .errors import UnsupportedRequirement, WorkflowException
@@ -419,7 +419,7 @@ def main(argsl=None,  # type: List[str]
                 arg_parser().print_help()
                 return 1
         if args.relax_path_checks:
-            draft2tool.ACCEPTLIST_RE = draft2tool.ACCEPTLIST_EN_RELAXED_RE
+            command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE
 
         if args.ga4gh_tool_registries:
             ga4gh_tool_registries[:] = args.ga4gh_tool_registries

cwltool/load_tool.py

@@ -256,8 +256,8 @@ def collectFilesAndDirs(obj, out):
             collectFilesAndDirs(l, out)
 
 
-def relocateOutputs(outputObj, outdir, output_dirs, action, fs_access):
-    # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]], Text, Set[Text], Text, StdFsAccess) -> Union[Dict[Text, Any], List[Dict[Text, Any]]]
+def relocateOutputs(outputObj, outdir, output_dirs, action, fs_access, compute_checksum):
+    # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]], Text, Set[Text], Text, StdFsAccess, bool) -> Union[Dict[Text, Any], List[Dict[Text, Any]]]
     adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True))
 
     if action not in ("move", "copy"):
@@ -299,8 +299,8 @@ def _check_adjust(f):
         return f
 
     visit_class(outputObj, ("File", "Directory"), _check_adjust)
-
-    visit_class(outputObj, ("File",), functools.partial(compute_checksums, fs_access))
+    if compute_checksum:
+        visit_class(outputObj, ("File",), functools.partial(compute_checksums, fs_access))
 
     # If there are symlinks to intermediate output directories, we want to move
     # the real files into the final output location.  If a file is linked more than once,