Merge remote-tracking branch 'origin/parallel-exec' into parallel-exec

Author: Anton Khodak

README.rst

@@ -65,7 +65,7 @@ Running tests locally
 To run the basis tests after installing `cwltool` execute the following:
 
 .. code:: bash
-  
+
   pip install pytest mock
   py.test --ignore cwltool/schemas/ --pyarg cwltool
 
@@ -113,20 +113,31 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``::
 .. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-Running user-space implementations of Docker
---------------------------------------------
+Using user-space replacements for Docker
+----------------------------------------
 
-Some compute environments disallow user-space installation of Docker due to incompatiblities in libraries or to meet security requirements. The CWL reference supports using a user space implementation with the `--user-space-docker-cmd` option.
+Some shared computing environments don't support Docker software containers for technical or policy reasons.
+As a work around, the CWL reference runner supports using a alternative ``docker`` implementations on Linux
+with the ``--user-space-docker-cmd`` option.
 
-Example using `dx-docker` (https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images):
+One such "user space" friendly docker replacement is ``udocker`` https://github.com/indigo-dc/udocker and another
+is ``dx-docker`` https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images
 
-For use on Linux, install the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions).
+udocker installation: https://github.com/indigo-dc/udocker/blob/master/doc/installation_manual.md#22-install-from-indigo-datacloud-repositories
+
+dx-docker installation: start with the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions).
 
 Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests:
 
 .. code:: bash
 
-  cwltool --user-space-docker-cmd=dx-docker --outdir=/tmp/tmpidytmp v1.0/test-cwl-out2.cwl v1.0/empty.json
+  cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
+  
+or 
+
+.. code:: bash
+
+  cwltool --user-space-docker-cmd=dx-docker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
 
 Tool or workflow loading from remote or local locations
 -------------------------------------------------------
@@ -224,7 +235,7 @@ the correct module environment before executing the above tool would simply be:
 
 .. code:: yaml
 
-  - type: module
+  - type: modules
 
 The outer list indicates that one plugin is being enabled, the plugin parameters are
 defined as a dictionary for this one list item. There is only one required parameter
@@ -399,17 +410,16 @@ environment or with a particular dataset.  To avoid the need to modify the
 underlying workflow, cwltool supports requirement "overrides".
 
 The format of the "overrides" object is a mapping of item identifier (workflow,
-workflow step, or command line tool) followed by a list of ProcessRequirements
-that should be applied.
+workflow step, or command line tool) to the process requirements that should be applied.
 
 .. code:: yaml
 
   cwltool:overrides:
     echo.cwl:
-      - class: EnvVarRequirement
-        envDef:
-          MESSAGE: override_value
-
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
 
 Overrides can be specified either on the command line, or as part of the job
 input document.  Workflow steps are identified using the name of the workflow
@@ -426,9 +436,10 @@ Override identifiers are relative to the toplevel workflow document.
   input_parameter2: value2
   cwltool:overrides:
     workflow.cwl#step1:
-      - class: EnvVarRequirement
-        envDef:
-          MESSAGE: override_value
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
 
 .. code:: bash
 

cwltool/job.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import codecs
 import functools
 import io
 import json
@@ -553,14 +554,14 @@ def _job_popen(
             stdin_path=stdin_path,
         )
         with open(os.path.join(job_dir, "job.json"), "wb") as f:
-            json.dump(job_description, f)
+            json.dump(job_description, codecs.getwriter('utf-8')(f), ensure_ascii=False) # type: ignore
         try:
             job_script = os.path.join(job_dir, "run_job.bash")
             with open(job_script, "wb") as f:
                 f.write(job_script_contents.encode('utf-8'))
             job_run = os.path.join(job_dir, "run_job.py")
             with open(job_run, "wb") as f:
-                f.write(PYTHON_RUN_SCRIPT)
+                f.write(PYTHON_RUN_SCRIPT.encode('utf-8'))
             sp = subprocess.Popen(
                 ["bash", job_script.encode("utf-8")],
                 shell=False,

cwltool/load_tool.py

@@ -45,10 +45,9 @@
     u"overrides": {
         "@id": "cwltool:overrides",
         "mapSubject": "overrideTarget",
-        "mapPredicate": "override"
     },
-    u"override": {
-        "@id": "cwltool:override",
+    "requirements": {
+        "@id": "https://w3id.org/cwl/cwl#requirements",
         "mapSubject": "class"
     }
 }  # type: ContextType

cwltool/pack.py

@@ -95,8 +95,8 @@ def import_embed(d, seen):
             import_embed(d[k], seen)
 
 
-def pack(document_loader, processobj, uri, metadata):
-    # type: (Loader, Union[Dict[Text, Any], List[Dict[Text, Any]]], Text, Dict[Text, Text]) -> Dict[Text, Any]
+def pack(document_loader, processobj, uri, metadata, rewrite_out=None):
+    # type: (Loader, Union[Dict[Text, Any], List[Dict[Text, Any]]], Text, Dict[Text, Text], Dict[Text, Text]) -> Dict[Text, Any]
 
     document_loader = SubLoader(document_loader)
     document_loader.idx = {}
@@ -114,15 +114,20 @@ def loadref(b, u):
         # type: (Text, Text) -> Union[Dict, List, Text]
         return document_loader.resolve_ref(u, base_url=b)[0]
 
+    ids = set()  # type: Set[Text]
+    find_ids(processobj, ids)
+
     runs = {uri}
     find_run(processobj, loadref, runs)
 
-    ids = set()  # type: Set[Text]
     for f in runs:
         find_ids(document_loader.resolve_ref(f)[0], ids)
 
     names = set()  # type: Set[Text]
-    rewrite = {}  # type: Dict[Text, Text]
+    if rewrite_out is None:
+        rewrite = {}  # type: Dict[Text, Text]
+    else:
+        rewrite = rewrite_out
 
     mainpath, _ = urllib.parse.urldefrag(uri)
 
@@ -131,8 +136,10 @@ def rewrite_id(r, mainuri):
         if r == mainuri:
             rewrite[r] = "#main"
         elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"):
-            path, frag = urllib.parse.urldefrag(r)
-            rewrite[r] = "#"+frag
+            if r[len(mainuri):].startswith("#main/"):
+                rewrite[r] = "#" + uniquename(r[len(mainuri)+1:], names)
+            else:
+                rewrite[r] = "#" + uniquename("main/"+r[len(mainuri)+1:], names)
         else:
             path, frag = urllib.parse.urldefrag(r)
             if path == mainpath:
@@ -144,8 +151,7 @@ def rewrite_id(r, mainuri):
     sortedids = sorted(ids)
 
     for r in sortedids:
-        if r in document_loader.idx:
-            rewrite_id(r, uri)
+        rewrite_id(r, uri)
 
     packed = {"$graph": [], "cwlVersion": metadata["cwlVersion"]
               }  # type: Dict[Text, Any]

cwltool/process.py

@@ -422,13 +422,13 @@ def avroize_type(field_type, name_prefix=""):
             avroize_type(field_type["items"], name_prefix)
     return field_type
 
-def get_overrides(overrides, toolid):  # type: (List[Dict[Text, Any]], Text) -> List[Dict[Text, Any]]
-    req = []  # type: List[Dict[Text, Any]]
+def get_overrides(overrides, toolid):  # type: (List[Dict[Text, Any]], Text) -> Dict[Text, Any]
+    req = {}  # type: Dict[Text, Any]
     if not isinstance(overrides, list):
         raise validate.ValidationException("Expected overrides to be a list, but was %s" % type(overrides))
     for ov in overrides:
         if ov["overrideTarget"] == toolid:
-            req.extend(ov["override"])
+            req.update(ov)
     return req
 
 class Process(six.with_metaclass(abc.ABCMeta, object)):
@@ -467,7 +467,7 @@ def __init__(self, toolpath_object, **kwargs):
         self.tool = toolpath_object
         self.requirements = (kwargs.get("requirements", []) +
                              self.tool.get("requirements", []) +
-                             get_overrides(kwargs.get("overrides", []), self.tool["id"]))
+                             get_overrides(kwargs.get("overrides", []), self.tool["id"]).get("requirements", []))
         self.hints = kwargs.get("hints", []) + self.tool.get("hints", [])
         self.formatgraph = None  # type: Graph
         if "loader" in kwargs:

cwltool/workflow.py

@@ -210,13 +210,13 @@ def object_from_state(state, parms, frag_only, supportsMultipleInput, sourceFiel
         if frag_only:
             iid = shortname(iid)
         if sourceField in inp:
-            if (isinstance(inp[sourceField], list) and not
-            supportsMultipleInput):
+            connections = aslist(inp[sourceField])
+            if (len(connections) > 1 and
+                not supportsMultipleInput):
                 raise WorkflowException(
                     "Workflow contains multiple inbound links to a single "
                     "parameter but MultipleInputFeatureRequirement is not "
                     "declared.")
-            connections = aslist(inp[sourceField])
             for src in connections:
                 if src in state and state[src] is not None and (state[src].success == "success" or incomplete):
                     if not match_types(
@@ -673,7 +673,7 @@ def __init__(self, toolpath_object, pos, **kwargs):
 
         kwargs["requirements"] = (kwargs.get("requirements", []) +
                                   toolpath_object.get("requirements", []) +
-                                  get_overrides(kwargs.get("overrides", []), self.id))
+                                  get_overrides(kwargs.get("overrides", []), self.id).get("requirements", []))
         kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get("hints", [])
 
         try:
@@ -710,7 +710,14 @@ def __init__(self, toolpath_object, pos, **kwargs):
                 for tool_entry in self.embedded_tool.tool[toolfield]:
                     frag = shortname(tool_entry["id"])
                     if frag == shortinputid:
+                        #if the case that the step has a default for a parameter,
+                        #we do not want the default of the tool to override it
+                        step_default = None
+                        if "default" in param and "default" in tool_entry:
+                            step_default = param["default"]
                         param.update(tool_entry)
+                        if step_default is not None:
+                            param["default"] = step_default
                         found = True
                         bound.add(frag)
                         break

tests/override/echo-job-ov.yml

@@ -1,6 +1,7 @@
 m1: zing
 cwltool:overrides:
   echo.cwl:
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello3
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello3

tests/override/echo-job-ov2.yml

@@ -1,7 +1,8 @@
 m1: zing
 cwltool:overrides:
   echo.cwl:
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello4
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello4
 cwl:tool: echo.cwl

tests/override/ov.yml

@@ -1,5 +1,6 @@
 cwltool:overrides:
   echo.cwl:
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello2
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello2

tests/override/ov2.yml

@@ -1,5 +1,6 @@
 cwltool:overrides:
   "echo-wf.cwl#step1":
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello5
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello5