Merge branch 'master' into py3

Author: mr-c

MANIFEST.in

@@ -1 +1,19 @@
-include gittaggers.py ez_setup.py Makefile cwltool.py
+include gittaggers.py Makefile cwltool.py
+include tests/*
+include tests/wf/*
+include cwltool/schemas/v1.0/*.yml
+include cwltool/schemas/draft-2/*.yml
+include cwltool/schemas/draft-3/*.yml
+include cwltool/schemas/draft-3/*.md
+include cwltool/schemas/draft-3/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/draft-3/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.0/*.yml
+include cwltool/schemas/v1.0/*.md
+include cwltool/schemas/v1.0/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.0/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.1.0-dev1/*.yml
+include cwltool/schemas/v1.1.0-dev1/*.md
+include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.md
+include cwltool/cwlNodeEngine.js
+global-exclude *.pyc

Makefile

@@ -26,7 +26,7 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8
+DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8
 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
@@ -50,7 +50,7 @@ install-deb-dep:
 
 ## install     : install the ${MODULE} module and schema-salad-tool
 install: FORCE
-	./setup.py build install
+	pip install .
 
 ## dist        : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz

cwltool/draft2tool.py

@@ -26,7 +26,7 @@
 from .utils import aslist
 
 ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
-ACCEPTLIST_EN_RELAXED_RE = re.compile(r"^[ a-zA-Z0-9._+-]+$")  # with spaces
+ACCEPTLIST_EN_RELAXED_RE = re.compile(r"^[ #a-zA-Z0-9._+-]+$")  # with spaces and hashes
 ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE
 
 from .flatten import flatten
@@ -355,9 +355,10 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
                             "writable": t.get("writable")
                         }
                     else:
-                        if t["entryname"]:
+                        if t["entryname"] or t["writable"]:
                             t = copy.deepcopy(t)
-                            t["entry"]["basename"] = t["entryname"]
+                            if t["entryname"]:
+                                t["entry"]["basename"] = t["entryname"]
                             t["entry"]["writable"] = t.get("writable")
                         ls[i] = t["entry"]
             j.generatefiles[u"listing"] = ls
@@ -462,6 +463,9 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                                       for g in fs_access.glob(fs_access.join(outdir, gb))])
                         except (OSError, IOError) as e:
                             _logger.warn(Text(e))
+                        except:
+                            _logger.error("Unexpected error from fs_access", exc_info=True)
+                            raise
 
                 for files in r:
                     if files["class"] == "Directory" and "listing" not in files:

cwltool/pathmapper.py

@@ -148,21 +148,21 @@ def __init__(self, referenced_files, basedir, stagedir, separateDirs=True):
         self.separateDirs = separateDirs
         self.setup(dedup(referenced_files), basedir)
 
-    def visitlisting(self, listing, stagedir, basedir):
-        # type: (List[Dict[Text, Any]], Text, Text) -> None
+    def visitlisting(self, listing, stagedir, basedir, copy=False):
+        # type: (List[Dict[Text, Any]], Text, Text, bool) -> None
         for ld in listing:
             tgt = os.path.join(stagedir, ld["basename"])
             if ld["class"] == "Directory":
-                self.visit(ld, stagedir, basedir, copy=ld.get("writable", False))
+                self.visit(ld, stagedir, basedir, copy=ld.get("writable", copy))
             else:
-                self.visit(ld, stagedir, basedir, copy=ld.get("writable", False))
+                self.visit(ld, stagedir, basedir, copy=ld.get("writable", copy))
 
     def visit(self, obj, stagedir, basedir, copy=False):
         # type: (Dict[Text, Any], Text, Text, bool) -> None
         tgt = os.path.join(stagedir, obj["basename"])
         if obj["class"] == "Directory":
             self._pathmap[obj["location"]] = MapperEnt(obj["location"], tgt, "Directory")
-            self.visitlisting(obj.get("listing", []), tgt, basedir)
+            self.visitlisting(obj.get("listing", []), tgt, basedir, copy=copy)
         elif obj["class"] == "File":
             path = obj["location"]
             if path in self._pathmap:
@@ -185,7 +185,7 @@ def visit(self, obj, stagedir, basedir, copy=False):
                             st = os.lstat(deref)
 
                     self._pathmap[path] = MapperEnt(deref, tgt, "File")
-                self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir)
+                self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir, copy=copy)
 
     def setup(self, referenced_files, basedir):
         # type: (List[Any], Text) -> None

cwltool/resolver.py

@@ -29,4 +29,4 @@ def tool_resolver(document_loader, uri):
         ret = r(document_loader, uri)
         if ret is not None:
             return ret
-    return file_uri(os.path.abspath(uri))
+    return file_uri(os.path.abspath(uri), split_frag=True)

cwltool/stdfsaccess.py

@@ -1,5 +1,6 @@
 import glob
 import os
+import urllib
 
 from schema_salad.ref_resolver import file_uri
 from typing import BinaryIO, Text
@@ -30,7 +31,7 @@ def isdir(self, fn):  # type: (Text) -> bool
         return os.path.isdir(self._abs(fn))
 
     def listdir(self, fn):  # type: (Text) -> List[Text]
-        return [abspath(l, fn) for l in os.listdir(self._abs(fn))]
+        return [abspath(urllib.quote(str(l)), fn) for l in os.listdir(self._abs(fn))]
 
     def join(self, path, *paths):  # type: (Text, *Text) -> Text
         return os.path.join(path, *paths)

cwltool/workflow.py

@@ -19,7 +19,7 @@
 
 _logger = logging.getLogger("cwltool")
 
-WorkflowStateItem = namedtuple('WorkflowStateItem', ['parameter', 'value'])
+WorkflowStateItem = namedtuple('WorkflowStateItem', ['parameter', 'value', 'success'])
 
 
 def defaultMakeTool(toolpath_object,  # type: Dict[Text, Any]
@@ -161,7 +161,7 @@ def object_from_state(state, parms, frag_only, supportsMultipleInput, sourceFiel
                     "declared.")
             connections = aslist(inp[sourceField])
             for src in connections:
-                if src in state and state[src] is not None:
+                if src in state and state[src] is not None and (state[src].success == "success" or incomplete):
                     if not match_types(
                             inp["type"], state[src], iid, inputobj,
                             inp.get("linkMerge", ("merge_nested"
@@ -232,7 +232,7 @@ def receive_output(self, step, outputparms, jobout, processStatus):
         for i in outputparms:
             if "id" in i:
                 if i["id"] in jobout:
-                    self.state[i["id"]] = WorkflowStateItem(i, jobout[i["id"]])
+                    self.state[i["id"]] = WorkflowStateItem(i, jobout[i["id"]], processStatus)
                 else:
                     _logger.error(u"[%s] Output is missing expected field %s", step.name, i["id"])
                     processStatus = "permanentFail"
@@ -359,9 +359,9 @@ def job(self, joborder, output_callback, **kwargs):
             with SourceLine(self.tool["inputs"], e, WorkflowException):
                 iid = shortname(i["id"])
                 if iid in joborder:
-                    self.state[i["id"]] = WorkflowStateItem(i, copy.deepcopy(joborder[iid]))
+                    self.state[i["id"]] = WorkflowStateItem(i, copy.deepcopy(joborder[iid]), "success")
                 elif "default" in i:
-                    self.state[i["id"]] = WorkflowStateItem(i, copy.deepcopy(i["default"]))
+                    self.state[i["id"]] = WorkflowStateItem(i, copy.deepcopy(i["default"]), "success")
                 else:
                     raise WorkflowException(
                         u"Input '%s' not in input object and does not have a default value." % (i["id"]))

cwltool_module.Dockerfile

@@ -8,7 +8,8 @@ RUN apt-get update -qq && apt-get install -qqy \
     python-setuptools gcc python-dev
 
 # Install cwltool
-ADD setup.py README.rst ez_setup.py cwltool/ /root/cwltool/
+ADD setup.py README.rst cwltool/ /root/cwltool/
 ADD cwltool/ /root/cwltool/cwltool
 ADD cwltool/schemas/ /root/cwltool/cwltool/schemas
-RUN cd /root/cwltool && ./ez_setup.py && easy_install .
+ADD tests/ /root/cwltool/tests
+RUN cd /root/cwltool && python setup.py install