Merge branch 'master' into parallel-exec

Author: Anton Khodak

MANIFEST.in

@@ -2,6 +2,7 @@ include gittaggers.py Makefile cwltool.py
 include tests/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/wf/*
+include tests/override/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/draft-2/*.yml
 include cwltool/schemas/draft-3/*.yml

Makefile

@@ -89,7 +89,7 @@ pydocstyle_report.txt: $(PYSOURCES)
 	pydocstyle setup.py $^ > pydocstyle_report.txt 2>&1 || true
 
 diff_pydocstyle_report: pydocstyle_report.txt
-	diff-quality --violations=pep8 $^
+	diff-quality --violations=pycodestyle $^
 
 ## autopep8    : fix most Python code indentation and formatting
 autopep8: $(PYSOURCES)
@@ -160,7 +160,7 @@ mypy2: ${PYSOURCES}
 	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
-	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
+	MYPYPATH=$$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
 		 --warn-redundant-casts \
 		 cwltool
 
@@ -171,7 +171,7 @@ mypy3: ${PYSOURCES}
 	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
-	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+	MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
 		 --warn-redundant-casts \
 		 cwltool
 

README.rst

@@ -31,7 +31,7 @@ It is highly recommended to setup virtual environment before installing `cwltool
   virtualenv -p python2 venv   # Create a virtual environment, can use `python3` as well
   source venv/bin/activate     # Activate environment before installing `cwltool`
 
-1. Installing the official package from PyPi (will install "cwltool" package as
+Installing the official package from PyPi (will install "cwltool" package as
 well)
 
 .. code:: bash
@@ -44,7 +44,7 @@ If installing alongside another CWL implementation then
 
   pip install cwltool
 
-2. To install from source
+Or you can install from source:
 
 .. code:: bash
 
@@ -62,9 +62,16 @@ Running tests locally
 
 -  Running basic tests ``(/tests)``:
 
-We use `tox <https://github.com/common-workflow-language/cwltool/tree/master/tox.ini>`_
-to run various tests in all supported Python environments.
-You can run the test suite by simply running the following in the terminal:
+To run the basis tests after installing `cwltool` execute the following:
+
+.. code:: bash
+  
+  pip install pytest mock
+  py.test --ignore cwltool/schemas/ --pyarg cwltool
+
+To run various tests in all supported Python environments we use `tox <https://github.com/common-workflow-language/cwltool/tree/master/tox.ini>`_. To run the test suite in all supported Python environments
+first downloading the complete code repository (see the ``git clone`` instructions above) and then run
+the following in the terminal:
 ``pip install tox; tox``
 
 List of all environment can be seen using:
@@ -106,6 +113,21 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``::
 .. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
+Running user-space implementations of Docker
+--------------------------------------------
+
+Some compute environments disallow user-space installation of Docker due to incompatiblities in libraries or to meet security requirements. The CWL reference supports using a user space implementation with the `--user-space-docker-cmd` option.
+
+Example using `dx-docker` (https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images):
+
+For use on Linux, install the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions).
+
+Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests:
+
+.. code:: bash
+
+  cwltool --user-space-docker-cmd=dx-docker --outdir=/tmp/tmpidytmp v1.0/test-cwl-out2.cwl v1.0/empty.json
+
 Tool or workflow loading from remote or local locations
 -------------------------------------------------------
 
@@ -369,6 +391,50 @@ at the following links:
 - `Specifications - Implementation <https://github.com/galaxyproject/galaxy/commit/81d71d2e740ee07754785306e4448f8425f890bc>`__
 - `Initial cwltool Integration Pull Request <https://github.com/common-workflow-language/cwltool/pull/214>`__
 
+Overriding workflow requirements at load time
+---------------------------------------------
+
+Sometimes a workflow needs additional requirements to run in a particular
+environment or with a particular dataset.  To avoid the need to modify the
+underlying workflow, cwltool supports requirement "overrides".
+
+The format of the "overrides" object is a mapping of item identifier (workflow,
+workflow step, or command line tool) followed by a list of ProcessRequirements
+that should be applied.
+
+.. code:: yaml
+
+  cwltool:overrides:
+    echo.cwl:
+      - class: EnvVarRequirement
+        envDef:
+          MESSAGE: override_value
+
+
+Overrides can be specified either on the command line, or as part of the job
+input document.  Workflow steps are identified using the name of the workflow
+file followed by the step name as a document fragment identifier "#id".
+Override identifiers are relative to the toplevel workflow document.
+
+.. code:: bash
+
+  cwltool --overrides overrides.yml my-tool.cwl my-job.yml
+
+.. code:: yaml
+
+  input_parameter1: value1
+  input_parameter2: value2
+  cwltool:overrides:
+    workflow.cwl#step1:
+      - class: EnvVarRequirement
+        envDef:
+          MESSAGE: override_value
+
+.. code:: bash
+
+  cwltool my-tool.cwl my-job-with-overrides.yml
+
+
 CWL Tool Control Flow
 ---------------------
 
@@ -500,18 +566,3 @@ logger_handler
       logging.Handler
 
   Handler object for logging.
-
-Running user-space implementations of Docker
---------------------------------------------
-
-Some compute environments disallow user-space installation of Docker due to incompatiblities in libraries or to meet security requirements. The CWL reference supports using a user space implementation with the `--user-space-docker-cmd` option.
-
-Example using `dx-docker` (https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images):
-
-For use on Linux, install the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions).
-
-Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests:
-
-```
-cwltool --user-space-docker-cmd=dx-docker --outdir=/tmp/tmpidytmp v1.0/test-cwl-out2.cwl v1.0/empty.json
-```

cwltool/draft2tool.py

@@ -1,14 +1,16 @@
 from __future__ import absolute_import
 import copy
 import hashlib
+import locale
 import json
 import logging
 import os
 import re
 import shutil
 import tempfile
-from functools import partial
-from typing import Any, Callable, Dict, Generator, List, Optional, Set, Text, Union, cast
+from functools import partial, cmp_to_key
+from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Text,
+    Union, cast)
 
 from six import string_types, u
 
@@ -208,7 +210,7 @@ def makeJobRunner(self, use_container=True, **kwargs):  # type: (Optional[bool],
                     })
                     dockerReq = self.requirements[0]
                     if default_container == windows_default_container_id and use_container and onWindows():
-                        _logger.warning(DEFAULT_CONTAINER_MSG%(windows_default_container_id, windows_default_container_id))
+                        _logger.warning(DEFAULT_CONTAINER_MSG % (windows_default_container_id, windows_default_container_id))
 
         if dockerReq and use_container:
             return DockerCommandLineJob()
@@ -523,8 +525,8 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jo
                 for i, port in enumerate(ports):
                     def makeWorkflowException(msg):
                         return WorkflowException(
-                                u"Error collecting output for parameter '%s':\n%s"
-                                % (shortname(port["id"]), msg))
+                            u"Error collecting output for parameter '%s':\n%s"
+                            % (shortname(port["id"]), msg))
                     with SourceLine(ports, i, makeWorkflowException, debug):
                         fragment = shortname(port["id"])
                         ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
@@ -575,16 +577,25 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                         elif gb == ".":
                             gb = outdir
                         elif gb.startswith("/"):
-                            raise WorkflowException("glob patterns must not start with '/'")
+                            raise WorkflowException(
+                                "glob patterns must not start with '/'")
                         try:
                             prefix = fs_access.glob(outdir)
                             r.extend([{"location": g,
-                                       "path": fs_access.join(builder.outdir, g[len(prefix[0])+1:]),
+                                       "path": fs_access.join(builder.outdir,
+                                           g[len(prefix[0])+1:]),
                                        "basename": os.path.basename(g),
-                                       "nameroot": os.path.splitext(os.path.basename(g))[0],
-                                       "nameext": os.path.splitext(os.path.basename(g))[1],
-                                       "class": "File" if fs_access.isfile(g) else "Directory"}
-                                      for g in fs_access.glob(fs_access.join(outdir, gb))])
+                                       "nameroot": os.path.splitext(
+                                           os.path.basename(g))[0],
+                                       "nameext": os.path.splitext(
+                                           os.path.basename(g))[1],
+                                       "class": "File" if fs_access.isfile(g)
+                                       else "Directory"}
+                                      for g in sorted(fs_access.glob(
+                                          fs_access.join(outdir, gb)),
+                                          key=cmp_to_key(cast(
+                                              Callable[[Text, Text],
+                                                  int], locale.strcoll)))])
                         except (OSError, IOError) as e:
                             _logger.warning(Text(e))
                         except: