Merge branch 'master' into argparser-require

Author: Peter Amstutz

.travis.yml

@@ -20,18 +20,12 @@ install:
   - pip install tox-travis
 jobs:
   include:
-    - stage: test
-      python: "2.7"
-    - stage: test
-      python: "3.5"
-    - stage: test
-      python: "3.6"
-    - stage: test
-      python: "3.7"
-    #- stage: test
-    #  python: "3.8-dev"  # ruamel.yaml doesn't support 3.8 yet
-    - stage: release-test
-      python: "2.7"
+    - python: "2.7"
+    - python: "3.5"
+    - python: "3.6"
+    - python: "3.7"
+    #- python: "3.8-dev"  # ruamel.yaml doesn't support 3.8 yet
+    - python: "3.7"
       script: RELEASE_SKIP=head ${TRAVIS_BUILD_DIR}/release-test.sh
 script: tox
 branches:

Makefile

@@ -34,12 +34,6 @@ VERSION=1.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
 	--format=format:%cd --date=format-local:%Y%m%d%H%M%S)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 UNAME_S=$(shell uname -s)
-ifeq ($(UNAME_S),Linux)
-	nproc=$(shell nproc)
-endif
-ifeq ($(UNAME_S),Darwin)
-	nproc=$(shell sysctl -n hw.physicalcpu)
-endif
 
 ## all         : default task
 all:
@@ -120,11 +114,11 @@ format: autopep8
 ## pylint      : run static code analysis on Python code
 pylint: $(PYSOURCES)
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
-                $^ -j$(nproc)|| true
+                $^ -j0|| true
 
 pylint_report.txt: ${PYSOURCES}
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
-		$^ -j$(nproc)> $@ || true
+		$^ -j0> $@ || true
 
 diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
@@ -154,11 +148,11 @@ diff-cover.html:  coverage.xml
 
 ## test        : run the ${MODULE} test suite
 test: $(pysources)
-	python setup.py test --addopts "-n$(nproc) --dist=loadfile"
+	python setup.py test --addopts "-n auto --dist=loadfile"
 
 ## testcov     : run the ${MODULE} test suite and collect coverage
 testcov: $(pysources)
-	python setup.py test --addopts "--cov cwltool -n$(nproc) --dist=loadfile"
+	python setup.py test --addopts "--cov cwltool -n auto --dist=loadfile"
 
 sloccount.sc: ${PYSOURCES} Makefile
 	sloccount --duplicates --wide --details $^ > $@

cwltool/builder.py

@@ -426,7 +426,7 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
             if sep:
                 args.extend([prefix, self.tostr(j)])
             else:
-                args.append("" if not prefix else prefix + self.tostr(j))
+                args.append(self.tostr(j) if prefix is None else prefix + self.tostr(j))
 
         return [a for a in args if a is not None]
 

cwltool/job.py

@@ -621,6 +621,16 @@ def run(self,
                 img_id = str(docker_req["dockerImageId"])
             elif 'dockerPull' in docker_req:
                 img_id = str(docker_req["dockerPull"])
+                cmd = [user_space_docker_cmd, "pull", img_id]
+                _logger.info(Text(cmd))
+                try:
+                    subprocess.check_call(cmd, stdout=sys.stderr)
+                except OSError:
+                    raise WorkflowException(SourceLine(docker_req).makeError(
+                        "Either Docker container {} is not available with "
+                        "user space docker implementation {} or {} is missing "
+                        "or broken.".format(img_id, user_space_docker_cmd,
+                                            user_space_docker_cmd)))
             else:
                 raise WorkflowException(SourceLine(docker_req).makeError(
                     "Docker image must be specified as 'dockerImageId' or "

cwltool/update.py

@@ -5,6 +5,8 @@
 from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
                     Optional, Tuple, Union)
 
+from functools import partial
+
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad import validate
 from schema_salad.ref_resolver import Loader  # pylint: disable=unused-import
@@ -61,8 +63,8 @@ def rewrite_requirements(t):  # type: (MutableMapping[Text, Union[Text, Dict[Tex
                             type(s), s))
 
 
-    def update_secondaryFiles(t):
-        # type: (Any) -> Union[MutableSequence[MutableMapping[Text, Text]], MutableMapping[Text, Text]]
+    def update_secondaryFiles(t, top=False):
+        # type: (Any, bool) -> Union[MutableSequence[MutableMapping[Text, Text]], MutableMapping[Text, Text]]
         if isinstance(t, CommentedSeq):
             new_seq = copy.deepcopy(t)
             for index, entry in enumerate(t):
@@ -72,6 +74,8 @@ def update_secondaryFiles(t):
             return CommentedSeq([update_secondaryFiles(p) for p in t])
         elif isinstance(t, MutableMapping):
             return t
+        elif top:
+            return CommentedSeq([CommentedMap([("pattern", t)])])
         else:
             return CommentedMap([("pattern", t)])
 
@@ -88,7 +92,7 @@ def fix_inputBinding(t):  # type: (Dict[Text, Any]) -> None
 
     visit_class(doc, ("CommandLineTool","Workflow"), rewrite_requirements)
     visit_class(doc, ("ExpressionTool","Workflow"), fix_inputBinding)
-    visit_field(doc, "secondaryFiles", update_secondaryFiles)
+    visit_field(doc, "secondaryFiles", partial(update_secondaryFiles, top=True))
 
     upd = doc
     if isinstance(upd, MutableMapping) and "$graph" in upd:

release-test.sh

@@ -9,22 +9,13 @@ package=cwltool
 module=cwltool
 slug=${TRAVIS_PULL_REQUEST_SLUG:=common-workflow-language/cwltool}
 repo=https://github.com/${slug}.git
-parallel=""
-if [[ "${OSTYPE}" == linux* ]]
-then
-	parallel=-n$(( $(nproc) / 2 ))
-fi
-if [[ "${OSTYPE}" == darwin* ]]
-then
-	parallel=-n$(( $(sysctl -n hw.physicalcpu) / 2 ))
-fi
 test_prefix=""
 run_tests() {
 	local mod_loc
 	mod_loc=$(pip show ${package} | 
 		grep ^Location | awk '{print $2}')/${module}
 	${test_prefix}bin/py.test "--ignore=${mod_loc}/schemas/" \
-		--pyarg -x ${module} ${parallel} --dist=loadfile
+		--pyarg -x ${module} -n auto --dist=loadfile
 }
 pipver=7.0.2 # minimum required version of pip
 setuptoolsver=24.2.0 # required to generate correct metadata for

tests/arg-empty-prefix-separate-false.cwl

@@ -0,0 +1,18 @@
+class: CommandLineTool
+cwlVersion: v1.0
+baseCommand: []
+inputs:
+  echo: boolean
+outputs: []
+arguments:
+- position: 1
+  shellQuote: false
+  prefix: ''
+  valueFrom: "non_existing_app"
+- position: 0
+  prefix: ''
+  separate: false
+  shellQuote: false
+  valueFrom: "echo"
+requirements:
+- class: ShellCommandRequirement

tests/test_examples.py

@@ -954,7 +954,7 @@ def test_bad_userspace_runtime(factor):
         "--user-space-docker-cmd=quaquioN", "--default-container=debian",
         get_data(test_file), get_data(job_file)])
     error_code, stdout, stderr = get_main_output(commands)
-    assert "'quaquioN' not found:" in stderr, stderr
+    assert "or quaquioN is missing or broken" in stderr, stderr
     assert error_code == 1
 
 @windows_needs_docker
@@ -1018,3 +1018,11 @@ def test_env_filtering(factor):
     else:
         target = 4
     assert json.loads(stdout)['env_count'] == target, (error_code, stdout, stderr)
+
+@windows_needs_docker
+def test_v1_0_arg_empty_prefix_separate_false():
+    test_file = "tests/arg-empty-prefix-separate-false.cwl"
+    error_code, stdout, stderr = get_main_output(
+        ['--debug', get_data(test_file), "--echo"])
+    assert "completed success" in stderr
+    assert error_code == 0

tests/test_udocker.py

@@ -16,23 +16,23 @@ class TestUdocker:
 
     @classmethod
     def setup_class(cls):
-        install_cmds = [
-            "curl https://raw.githubusercontent.com/indigo-dc/udocker/master/udocker.py -o ./udocker",
-            "chmod u+rx ./udocker",
-            "./udocker install"]
-
         test_cwd = os.getcwd()
         test_environ = os.environ.copy()
-
         cls.docker_install_dir = tempfile.mkdtemp()
         os.chdir(cls.docker_install_dir)
 
+        url="https://download.ncg.ingrid.pt/webdav/udocker/udocker-1.1.3.tar.gz"
+        install_cmds = [
+            ["curl", url, "-o", "./udocker-tarball.tgz"],
+            ["tar", "xzvf", "udocker-tarball.tgz", "udocker"],
+            ["bash", "-c", "UDOCKER_TARBALL={}/udocker-tarball.tgz ./udocker install".format(cls.docker_install_dir)]]
+
         os.environ['UDOCKER_DIR'] = os.path.join(cls.docker_install_dir, ".udocker")
         os.environ['HOME'] = cls.docker_install_dir
 
         results = []
         for _ in range(3):
-              results = [subprocess.call(cmd.split()) for cmd in install_cmds]
+              results = [subprocess.call(cmds) for cmds in install_cmds]
               if sum(results) == 0:
                   break
               subprocess.call(["rm", "./udocker"])