Merge branch 'master' of https://github.com/common-workflow-language/cwltool

Author: michael-kotliar

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+This issue tracker is for reporting bugs in cwltool (the CWL reference implementation) itself. Please use the [Gitter channel](https://gitter.im/common-workflow-language/common-workflow-language) or [Biostars forum](https://www.biostars.org/) for general questions about using cwltool to create/run workflows or issues not related to cwltool. Don't forget to use cwl tag when posting on Biostars Forum.
+
+If you'd like to report a bug, fill out the template below and provide any extra information that may be useful / related to your problem. Ideally, you create an [a Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) reproducing the problem before opening an issue to ensure it's not caused by something in your code.
+
+Before you submit, please delete this help text above the ---
+Thanks!
+---
+
+## Expected Behavior
+Tell us what should happen
+
+## Actual Behavior
+Tell us what happens instead
+
+## Workflow Code
+```
+Paste the template code (ideally a minimal example) that causes the issue
+
+```
+
+## Full Traceback
+```pytb
+Paste the full traceback in case there is an exception
+Run the workflow with ``--debug`` flag for more verbose logging 
+```
+
+## Your Environment
+* cwltool version: 
+Check using ``cwltool --version``

MANIFEST.in

@@ -1,5 +1,6 @@
 include gittaggers.py Makefile cwltool.py
 include tests/*
+include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/wf/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/draft-2/*.yml

Makefile

@@ -26,7 +26,7 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest
+DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest isort
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8
 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
@@ -65,6 +65,11 @@ clean: FORCE
 	rm -Rf .coverage
 	rm -f diff-cover.html
 
+# Linting and code style related targets
+## sorting imports using isort: https://github.com/timothycrosley/isort
+sort_imports:
+	isort ${MODULE}/*.py tests/*.py setup.py
+
 ## pep8        : check Python code style
 pep8: $(PYSOURCES)
 	pep8 --exclude=_version.py  --show-source --show-pep8 $^ || true

cwltool/builder.py

@@ -1,14 +1,17 @@
 import copy
+from typing import Any, Callable, Text, Type, Union
+
+from six import iteritems, string_types
 
 import avro
 import schema_salad.validate as validate
 from schema_salad.sourceline import SourceLine
-from typing import Any, Callable, Text, Type, Union
-from six import string_types, iteritems
 
 from . import expression
 from .errors import WorkflowException
-from .pathmapper import PathMapper, normalizeFilesDirs, get_listing, visit_class
+from .mutation import MutationManager
+from .pathmapper import (PathMapper, get_listing, normalizeFilesDirs,
+                         visit_class)
 from .stdfsaccess import StdFsAccess
 from .utils import aslist
 
@@ -41,6 +44,7 @@ def __init__(self):  # type: () -> None
         self.make_fs_access = None  # type: Type[StdFsAccess]
         self.build_job_script = None  # type: Callable[[List[str]], Text]
         self.debug = False  # type: bool
+        self.mutation_manager = None  # type: MutationManager
 
         # One of "no_listing", "shallow_listing", "deep_listing"
         # Will be default "no_listing" for CWL v1.1

cwltool/cwlrdf.py

@@ -1,7 +1,9 @@
+from typing import IO, Any, Dict, Text
+
 from rdflib import Graph
+
 from schema_salad.jsonld_context import makerdf
 from schema_salad.ref_resolver import ContextType
-from typing import Any, Dict, IO, Text
 from six.moves import urllib
 
 from .process import Process

cwltool/docker.py

@@ -4,9 +4,9 @@
 import subprocess
 import sys
 import tempfile
+from typing import Text
 
 import requests
-from typing import Text
 
 from .errors import WorkflowException
 
@@ -27,6 +27,17 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             sp = dockerRequirement["dockerImageId"].split(":")
             if len(sp) == 1:
                 sp.append("latest")
+            elif len(sp) == 2:
+                #  if sp[1] doesn't  match valid tag names, it is a part of repository
+                if not re.match(r'[\w][\w.-]{0,127}', sp[1]):
+                    sp[0] = sp[0] + ":" + sp[1]
+                    sp[1] = "latest"
+            elif len(sp) == 3:
+                if re.match(r'[\w][\w.-]{0,127}', sp[2]):
+                    sp[0] = sp[0] + ":" + sp[1]
+                    sp[1] = sp[2]
+                    del sp[2]
+
             # check for repository:tag match or image id match
             if ((sp[0] == m.group(1) and sp[1] == m.group(2)) or dockerRequirement["dockerImageId"] == m.group(3)):
                 found = True

cwltool/docker_uid.py

@@ -1,6 +1,6 @@
 from __future__ import print_function
-import subprocess
 
+import subprocess
 from typing import Text
 
 

cwltool/draft2tool.py

@@ -6,30 +6,33 @@
 import re
 import shutil
 import tempfile
-from six.moves import urllib
-from six import string_types, u
 from functools import partial
+from typing import Any, Callable, Dict, Generator, Optional, Text, Union, cast
+
+from six import string_types, u
 
 import schema_salad.validate as validate
 import shellescape
 from schema_salad.ref_resolver import file_uri, uri_file_path
 from schema_salad.sourceline import SourceLine, indent
-from typing import Any, Callable, cast, Generator, Text, Union
+from six.moves import urllib
 
-from .builder import CONTENT_LIMIT, substitute, Builder
-from .pathmapper import adjustFileObjs, adjustDirObjs, visit_class
+from .builder import CONTENT_LIMIT, Builder, substitute
 from .errors import WorkflowException
-from .job import CommandLineJob
-from .pathmapper import PathMapper, get_listing, trim_listing
-from .process import Process, shortname, uniquename, normalizeFilesDirs, compute_checksums
+from .flatten import flatten
+from .job import CommandLineJob, DockerCommandLineJob, JobBase
+from .pathmapper import (PathMapper, adjustDirObjs, adjustFileObjs,
+                         get_listing, trim_listing, visit_class)
+from .process import (Process, UnsupportedRequirement,
+                      _logger_validation_warnings, compute_checksums,
+                      normalizeFilesDirs, shortname, uniquename)
 from .stdfsaccess import StdFsAccess
 from .utils import aslist
 
 ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
 ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")  # Accept anything
 ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE
 
-from .flatten import flatten
 
 _logger = logging.getLogger("cwltool")
 
@@ -150,6 +153,7 @@ def run(self, **kwargs):
 # walk over input as implicit reassignment doesn't reach everything in builder.bindings
 def check_adjust(builder, f):
     # type: (Builder, Dict[Text, Any]) -> Dict[Text, Any]
+
     f["path"] = builder.pathmapper.mapper(f["location"])[1]
     f["dirname"], f["basename"] = os.path.split(f["path"])
     if f["class"] == "File":
@@ -171,20 +175,28 @@ def __init__(self, toolpath_object, **kwargs):
         # type: (Dict[Text, Any], **Any) -> None
         super(CommandLineTool, self).__init__(toolpath_object, **kwargs)
 
-    def makeJobRunner(self):  # type: () -> CommandLineJob
-        return CommandLineJob()
+    def makeJobRunner(self, use_container=True):  # type: (Optional[bool]) -> JobBase
+        dockerReq, _ = self.get_requirement("DockerRequirement")
+        if dockerReq and use_container:
+            return DockerCommandLineJob()
+        else:
+            for t in reversed(self.requirements):
+                if t["class"] == "DockerRequirement":
+                    raise UnsupportedRequirement(
+                        "--no-container, but this CommandLineTool has "
+                        "DockerRequirement under 'requirements'.")
+            return CommandLineJob()
 
     def makePathMapper(self, reffiles, stagedir, **kwargs):
         # type: (List[Any], Text, **Any) -> PathMapper
-        dockerReq, _ = self.get_requirement("DockerRequirement")
         return PathMapper(reffiles, kwargs["basedir"], stagedir)
 
     def job(self,
             job_order,  # type: Dict[Text, Text]
             output_callbacks,  # type: Callable[[Any, Any], Any]
             **kwargs  # type: Any
             ):
-        # type: (...) -> Generator[Union[CommandLineJob, CallbackJob], None, None]
+        # type: (...) -> Generator[Union[JobBase, CallbackJob], None, None]
 
         jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job"))))
 
@@ -199,9 +211,9 @@ def job(self,
                                                  cachebuilder.stagedir,
                                                  separateDirs=False)
             _check_adjust = partial(check_adjust, cachebuilder)
-
             visit_class([cachebuilder.files, cachebuilder.bindings],
                        ("File", "Directory"), _check_adjust)
+
             cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings))
             (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
             if docker_req and kwargs.get("use_container") is not False:
@@ -264,7 +276,7 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
 
         reffiles = copy.deepcopy(builder.files)
 
-        j = self.makeJobRunner()
+        j = self.makeJobRunner(kwargs.get("use_container"))
         j.builder = builder
         j.joborder = builder.job
         j.stdin = None
@@ -368,8 +380,39 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
                         ls[i] = t["entry"]
             j.generatefiles[u"listing"] = ls
 
+        inplaceUpdateReq = self.get_requirement("http://commonwl.org/cwltool#InplaceUpdateRequirement")[0]
+
+        if inplaceUpdateReq:
+            j.inplace_update = inplaceUpdateReq["inplaceUpdate"]
         normalizeFilesDirs(j.generatefiles)
 
+        readers = {}
+        muts = set()
+
+        if builder.mutation_manager:
+            def register_mut(f):
+                muts.add(f["location"])
+                builder.mutation_manager.register_mutation(j.name, f)
+
+            def register_reader(f):
+                if f["location"] not in muts:
+                    builder.mutation_manager.register_reader(j.name, f)
+                    readers[f["location"]] = f
+
+            for li in j.generatefiles["listing"]:
+                li = cast(Dict[Text, Any], li)
+                if li.get("writable") and j.inplace_update:
+                    adjustFileObjs(li, register_mut)
+                    adjustDirObjs(li, register_mut)
+                else:
+                    adjustFileObjs(li, register_reader)
+                    adjustDirObjs(li, register_reader)
+
+            adjustFileObjs(builder.files, register_reader)
+            adjustFileObjs(builder.bindings, register_reader)
+            adjustDirObjs(builder.files, register_reader)
+            adjustDirObjs(builder.bindings, register_reader)
+
         j.environment = {}
         evr = self.get_requirement("EnvVarRequirement")[0]
         if evr:
@@ -391,16 +434,17 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
         j.pathmapper = builder.pathmapper
         j.collect_outputs = partial(
             self.collect_output_ports, self.tool["outputs"], builder,
-            compute_checksum=kwargs.get("compute_checksum", True))
+            compute_checksum=kwargs.get("compute_checksum", True),
+            jobname=jobname,
+            readers=readers)
         j.output_callback = output_callbacks
 
         yield j
 
-    def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
-        # type: (Set[Dict[Text, Any]], Builder, Text, bool) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
+    def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jobname="", readers=None):
+        # type: (Set[Dict[Text, Any]], Builder, Text, bool, Text, Dict[Text, Any]) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
         ret = {}  # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
         try:
-
             fs_access = builder.make_fs_access(outdir)
             custom_output = fs_access.join(outdir, "cwl.output.json")
             if fs_access.exists(custom_output):
@@ -429,14 +473,22 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
                 visit_class(ret, ("File", "Directory"), cast(Callable[[Any], Any], revmap))
                 visit_class(ret, ("File", "Directory"), remove_path)
                 normalizeFilesDirs(ret)
+                if builder.mutation_manager:
+                    adjustFileObjs(ret, builder.mutation_manager.set_generation)
                 visit_class(ret, ("File", "Directory"), partial(check_valid_locations, fs_access))
+
                 if compute_checksum:
                     adjustFileObjs(ret, partial(compute_checksums, fs_access))
 
-            validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret)
+            validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret,
+                                 strict=False, logger=_logger_validation_warnings)
             return ret if ret is not None else {}
         except validate.ValidationException as e:
             raise WorkflowException("Error validating output record. " + Text(e) + "\n in " + json.dumps(ret, indent=4))
+        finally:
+            if builder.mutation_manager and readers:
+                for r in readers.values():
+                    builder.mutation_manager.release_reader(jobname, r)
 
     def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=True):
         # type: (Dict[Text, Any], Builder, Text, StdFsAccess, bool) -> Union[Dict[Text, Any], List[Union[Dict[Text, Any], Text]]]

cwltool/expression.py

@@ -2,8 +2,8 @@
 import json
 import logging
 import re
+from typing import Any, AnyStr, Dict, List, Text, Union
 
-from typing import Any, AnyStr, Union, Text, Dict, List
 from six import u
 
 from . import sandboxjs

cwltool/extensions.yml

@@ -19,4 +19,18 @@ $graph:
       type:
         - type: enum
           name: LoadListingEnum
-          symbols: [no_listing, shallow_listing, deep_listing]
+          symbols: [no_listing, shallow_listing, deep_listing]
+
+- name: InplaceUpdateRequirement
+  type: record
+  inVocab: false
+  extends: cwl:ProcessRequirement
+  fields:
+    class:
+      type: string
+      doc: "Always 'InplaceUpdateRequirement'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    inplaceUpdate:
+      type: boolean