Merge branch 'master' into bad_type_error

Author: mr-c

README.rst

@@ -7,16 +7,16 @@ CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Cov
 
 .. |Conformance Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
    :target: https://ci.commonwl.org/job/cwltool-conformance/
-   
+
 .. |Linux Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/master.svg?label=Linux%20builds
    :target: https://travis-ci.org/common-workflow-language/cwltool
-   
+
 .. |Windows Status| image:: https://img.shields.io/appveyor/ci/mr-c/cwltool/master.svg?label=Windows%20builds
    :target: https://ci.appveyor.com/project/mr-c/cwltool
 
 .. |Coverage Status| image:: https://img.shields.io/codecov/c/github/common-workflow-language/cwltool.svg
   :target: https://codecov.io/gh/common-workflow-language/cwltool
-   
+
 This is the reference implementation of the Common Workflow Language.  It is
 intended to feature complete and provide comprehensive validation of CWL
 files as well as provide other tools related to working with CWL.
@@ -88,6 +88,8 @@ List of all environment can be seen using:
 ``tox --listenvs``
 and running a specfic test env using:
 ``tox -e <env name>``
+and additionally run a specific test using this format:
+``tox -e py36-unit -- tests/test_examples.py::TestParamMatching``
 
 -  Running the entire suite of CWL conformance tests:
 
@@ -139,8 +141,8 @@ Run `cwltool` just as you normally would, but with the new option, e.g. from the
 .. code:: bash
 
   cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
-  
-or 
+
+or
 
 .. code:: bash
 

cwltool/argparser.py

@@ -214,6 +214,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--custom-net", type=Text,
                         help="Will be passed to `docker run` as the '--net' "
                              "parameter. Implies '--enable-net'.")
+    parser.add_argument("--disable-validate", dest="do_validate",
+                        action="store_false", default=True,
+                        help=argparse.SUPPRESS)
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--enable-ga4gh-tool-registry", action="store_true", help="Enable resolution using GA4GH tool registry API",

cwltool/docker.py

@@ -11,7 +11,8 @@
 
 import datetime
 import requests
-from typing import (Dict, List, Text, Any, MutableMapping)
+from typing import (Dict, List, Text, Any, MutableMapping, Set)
+import threading
 
 from .docker_id import docker_vm_id
 from .errors import WorkflowException
@@ -22,17 +23,23 @@
 
 _logger = logging.getLogger("cwltool")
 
+found_images = set()  # type: Set[Text]
+found_images_lock = threading.Lock()
 
 class DockerCommandLineJob(ContainerCommandLineJob):
 
     @staticmethod
-    def get_image(dockerRequirement, pull_image, dry_run=False):
-        # type: (Dict[Text, Text], bool, bool) -> bool
+    def get_image(dockerRequirement, pull_image, dry_run=False, force_pull=False):
+        # type: (Dict[Text, Text], bool, bool, bool) -> bool
         found = False
 
         if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
             dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
 
+        with found_images_lock:
+            if dockerRequirement["dockerImageId"] in found_images:
+                return True
+
         for ln in subprocess.check_output(
                 ["docker", "images", "--no-trunc", "--all"]).decode('utf-8').splitlines():
             try:
@@ -58,7 +65,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             except ValueError:
                 pass
 
-        if not found and pull_image:
+        if (force_pull or not found) and pull_image:
             cmd = []  # type: List[Text]
             if "dockerPull" in dockerRequirement:
                 cmd = ["docker", "pull", str(dockerRequirement["dockerPull"])]
@@ -106,10 +113,14 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                     subprocess.check_call(cmd, stdout=sys.stderr)
                     found = True
 
+        if found:
+            with found_images_lock:
+                found_images.add(dockerRequirement["dockerImageId"])
+
         return found
 
-    def get_from_requirements(self, r, req, pull_image, dry_run=False):
-        # type: (Dict[Text, Text], bool, bool, bool) -> Text
+    def get_from_requirements(self, r, req, pull_image, dry_run=False, force_pull=False):
+        # type: (Dict[Text, Text], bool, bool, bool, bool) -> Text
         if r:
             errmsg = None
             try:
@@ -125,7 +136,7 @@ def get_from_requirements(self, r, req, pull_image, dry_run=False):
                 else:
                     return None
 
-            if self.get_image(r, pull_image, dry_run):
+            if self.get_image(r, pull_image, dry_run, force_pull=force_pull):
                 return r["dockerImageId"]
             else:
                 if req:

cwltool/job.py

@@ -337,8 +337,8 @@ class ContainerCommandLineJob(JobBase):
     __metaclass__ = ABCMeta
 
     @abstractmethod
-    def get_from_requirements(self, r, req, pull_image, dry_run=False):
-        # type: (Dict[Text, Text], bool, bool, bool) -> Text
+    def get_from_requirements(self, r, req, pull_image, dry_run=False, force_pull=False):
+        # type: (Dict[Text, Text], bool, bool, bool, bool) -> Text
         pass
 
     @abstractmethod
@@ -373,7 +373,7 @@ def run(self, pull_image=True, rm_container=True,
             try:
                 env = cast(MutableMapping[Text, Text], os.environ)
                 if docker_req and kwargs.get("use_container"):
-                    img_id = str(self.get_from_requirements(docker_req, True, pull_image))
+                    img_id = str(self.get_from_requirements(docker_req, True, pull_image, force_pull=kwargs.get("force_docker_pull")))
                 if img_id is None:
                     if self.builder.find_default_container:
                         default_container = self.builder.find_default_container()

cwltool/load_tool.py

@@ -193,7 +193,8 @@ def validate_document(document_loader,  # type: Loader
                       fetcher_constructor=None,  # type: FetcherConstructorType
                       skip_schemas=None,  # type: bool
                       overrides=None,  # type: List[Dict]
-                      metadata=None  # type: Optional[Dict]
+                      metadata=None,  # type: Optional[Dict]
+                      do_validate=True
                       ):
     # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
     """Validate a CWL document."""
@@ -209,7 +210,7 @@ def validate_document(document_loader,  # type: Loader
     jobobj = None
     if "cwl:tool" in workflowobj:
         job_loader = default_loader(fetcher_constructor)  # type: ignore
-        jobobj, _ = job_loader.resolve_all(workflowobj, uri)
+        jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate)
         uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
         del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
 
@@ -262,7 +263,7 @@ def validate_document(document_loader,  # type: Loader
     _add_blank_ids(workflowobj)
 
     workflowobj["id"] = fileuri
-    processobj, new_metadata = document_loader.resolve_all(workflowobj, fileuri)
+    processobj, new_metadata = document_loader.resolve_all(workflowobj, fileuri, checklinks=do_validate)
     if not isinstance(processobj, (CommentedMap, CommentedSeq)):
         raise ValidationException("Workflow must be a dict or list.")
 
@@ -277,7 +278,8 @@ def validate_document(document_loader,  # type: Loader
     if preprocess_only:
         return document_loader, avsc_names, processobj, new_metadata, uri
 
-    schema.validate_doc(avsc_names, processobj, document_loader, strict)
+    if do_validate:
+        schema.validate_doc(avsc_names, processobj, document_loader, strict)
 
     if new_metadata.get("cwlVersion") != update.LATEST:
         processobj = cast(CommentedMap, cmap(update.update(

cwltool/main.py

@@ -396,7 +396,8 @@ def main(argsl=None,  # type: List[str]
                      'ga4gh_tool_registries': [],
                      'find_default_container': None,
                                    'make_template': False,
-                                   'overrides': None
+                                   'overrides': None,
+                                   'do_validate': True
         }):
             if not hasattr(args, k):
                 setattr(args, k, v)
@@ -478,7 +479,8 @@ def main(argsl=None,  # type: List[str]
                                     preprocess_only=args.print_pre or args.pack,
                                     fetcher_constructor=fetcher_constructor,
                                     skip_schemas=args.skip_schemas,
-                                    overrides=overrides)
+                                    overrides=overrides,
+                                    do_validate=args.do_validate)
 
             if args.print_pre:
                 stdout.write(json.dumps(processobj, indent=4))
@@ -498,6 +500,7 @@ def main(argsl=None,  # type: List[str]
 
             make_tool_kwds["find_default_container"] = functools.partial(find_default_container, args)
             make_tool_kwds["overrides"] = overrides
+            make_tool_kwds["disable_js_validation"] = args.disable_js_validation or (not args.do_validate)
 
             tool = make_tool(document_loader, avsc_names, metadata, uri,
                              makeTool, make_tool_kwds)

cwltool/singularity.py

@@ -69,9 +69,13 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
 
         return found
 
-    def get_from_requirements(self, r, req, pull_image, dry_run=False):
-        # type: (Dict[Text, Text], bool, bool, bool) -> Text
+    def get_from_requirements(self, r, req, pull_image, dry_run=False, force_pull=False):
+        # type: (Dict[Text, Text], bool, bool, bool, bool) -> Text
         # returns the filename of the Singularity image (e.g. hello-world-latest.img)
+
+        if force_pull:
+            _logger.warn("--force-docker-pull currently not supported for singularity")
+
         if r:
             errmsg = None
             try:

cwltool/workflow.py

@@ -557,7 +557,8 @@ def __init__(self, toolpath_object, **kwargs):
             step_inputs.extend(step.tool["inputs"])
             step_outputs.extend(step.tool["outputs"])
 
-        static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs)
+        if kwargs.get("do_validate", True):
+            static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs)
 
 
     def job(self,

tox.ini

@@ -24,7 +24,7 @@ deps =
   py{27,34,35,36}-lint: flake8
 
 commands =
-  py{27,34,35,36}-unit: python setup.py test --addopts "--cov-report xml --cov cwltool"
+  py{27,34,35,36}-unit: python setup.py test --addopts "--cov-report xml --cov cwltool {posargs}"
   py{27,34,35,36}-unit: codecov
   py{27,34,35,36}-lint: flake8 schema_salad setup.py