Merge branch 'singularity23_fix' of https://github.com/drkennetz/cwltool-singularity23-fix into singularity23_fix

necessary to perform a PR

Author: Dennis Kennetz

.travis.yml

@@ -1,5 +1,4 @@
-dist: trusty
-sudo: required
+dist: xenial
 services:
   - docker
 
@@ -23,22 +22,20 @@ jobs:
   include:
     - stage: test
       python: "2.7"
-    - stage: test
-      python: "3.4"
     - stage: test
       python: "3.5"
     - stage: test
       python: "3.6"
     - stage: test
       python: "3.7"
-      dist: xenial
+    #- stage: test
+    #  python: "3.8-dev"  # ruamel.yaml doesn't support 3.8 yet
     - stage: release-test
       python: "2.7"
       script: RELEASE_SKIP=head ${TRAVIS_BUILD_DIR}/release-test.sh
 script: tox
 branches:
   only:
     - master
-    - singularity23_fix
 notifications:
   email: false

CONTRIBUTING.md

@@ -1,11 +1,11 @@
 Style guide:
 PEP-8
-Python 2.7 + Python 3.4+ compatible code
+Python 2.7 + Python 3.5+ compatible code
 PEP-484 type hints, in comments for Python 2.7 compatability
 Vertically align the type hints in function definitions
 
 In order to contribute to the development of ``cwltool``, you need to install cwltool from source (preferably in a virtual environment):
-Here's a rough 
+Here's a rough guide (improvements are welcome!) 
 - Install virtualenv via pip: ``pip install virtualenv``
 - Clone the cwltool: ``git clone https://github.com/common-workflow-language/cwltool.git``
 - Switch to cwltool directory: ``cd cwltool``

MANIFEST.in

@@ -9,6 +9,7 @@ include tests/wf/*
 include tests/override/*
 include tests/checker_wf/*
 include tests/subgraph/*
+include tests/trs/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.md

Makefile

@@ -27,7 +27,7 @@ MODULE=cwltool
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
 DEVPKGS=pycodestyle diff_cover autopep8 pylint coverage pydocstyle flake8 \
-	pytest pytest-xdist isort wheel
+	pytest-xdist==1.27.0 isort wheel -rtest-requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
 	   python-flake8 python-mock shellcheck
 VERSION=1.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
@@ -51,7 +51,7 @@ help: Makefile
 
 ## install-dep : install most of the development dependencies via pip
 install-dep:
-	pip install --upgrade $(DEVPKGS) -rtest-requirements.txt
+	pip install --upgrade $(DEVPKGS)
 
 ## install-deb-dep: install most of the dev dependencies via apt-get
 install-deb-dep:

README.rst

@@ -25,7 +25,7 @@ intended to be feature complete and provide comprehensive validation of CWL
 files as well as provide other tools related to working with CWL.
 
 This is written and tested for
-`Python <https://www.python.org/>`_ ``2.7 and 3.x {x = 4, 5, 6, 7}``
+`Python <https://www.python.org/>`_ ``2.7 and 3.x {x = 5, 6, 7}``
 
 The reference implementation consists of two packages.  The ``cwltool`` package
 is the primary Python module containing the reference implementation in the
@@ -498,7 +498,7 @@ Running tests locally
 
 -  Running basic tests ``(/tests)``:
 
-To run the basis tests after installing `cwltool` execute the following:
+To run the basic tests after installing `cwltool` execute the following:
 
 .. code:: bash
 

appveyor.yml

@@ -12,10 +12,6 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python34-x64"
-      PYTHON_VERSION: "3.4.x"
-      PYTHON_ARCH: "64"
-
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "64"
@@ -32,12 +28,12 @@ install:
   - ps: 'Install-Product node 0.12 x64'
   - "set PATH=%PYTHON%\\Scripts;%PATH%"
   - "%PYTHON%\\python.exe -m pip install -U pip setuptools^>=20.3 wheel"
-  - "%PYTHON%\\python.exe -m pip install -U codecov pytest-xdist pytest-cov galaxy-lib -rtest-requirements.txt"
+  - "%PYTHON%\\python.exe -m pip install -U codecov -rtest-requirements.txt pytest-xdist==1.27 "
     # Note the use of a `^` to escape the `>`
 
 build_script:
   - "%PYTHON%\\python.exe -m pip install -rrequirements.txt"
-  - "%PYTHON%\\python.exe -m pip install -e ."
+  - "%PYTHON%\\python.exe -m pip install -e .[deps]"
 
 test_script:
   - |

cwl-docker.sh

@@ -1,2 +1,3 @@
 #!/bin/sh
+# Changing the line below? Update https://cloud.docker.com/u/commonworkflowlanguage/repository/docker/commonworkflowlanguage/cwltool
 exec docker run -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v "$PWD":"$PWD" -w="$PWD" commonworkflowlanguage/cwltool "$@"

cwltool/argparser.py

@@ -105,10 +105,10 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--enable-pull", default=True, action="store_true",
-                         help="Try to pull Docker images", dest="enable_pull")
+                         help="Try to pull Docker images", dest="pull_image")
 
     exgroup.add_argument("--disable-pull", default=True, action="store_false",
-                         help="Do not try to pull Docker images", dest="enable_pull")
+                         help="Do not try to pull Docker images", dest="pull_image")
 
     parser.add_argument("--rdf-serializer",
                         help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",

cwltool/builder.py

@@ -50,8 +50,12 @@ def content_limit_respected_read(f):  # type: (IO) -> Text
 
 
 def substitute(value, replace):  # type: (Text, Text) -> Text
-    if replace[0] == "^":
-        return substitute(value[0:value.rindex('.')], replace[1:])
+    if replace.startswith("^"):
+        try:
+            return substitute(value[0:value.rindex('.')], replace[1:])
+        except ValueError:
+            # No extension to remove
+            return value + replace.lstrip("^")
     return value + replace
 
 def formatSubclassOf(fmt, cls, ontology, visited):
@@ -207,13 +211,12 @@ def bind_input(self,
         value_from_expression = False
         if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
             binding = CommentedMap(schema["inputBinding"].items())
-            assert binding is not None
 
             bp = list(aslist(lead_pos))
             if "position" in binding:
                 position = binding["position"]
-                if isinstance(position, str):  # no need to test the CWL Version
-                                               # the schema for v1.0 only allow ints
+                if isinstance(position, str):   # no need to test the CWL Version
+                                                # the schema for v1.0 only allow ints
                     binding['position'] = self.do_eval(position, context=datum)
                     bp.append(binding['position'])
                 else:
@@ -238,7 +241,6 @@ def bind_input(self,
                     avsc = self.names.get_name(t["name"], "")
                 if not avsc:
                     avsc = make_avsc_object(convert_to_dict(t), self.names)
-                assert avsc is not None
                 if validate.validate(avsc, datum):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t
@@ -379,7 +381,6 @@ def tostr(self, value):  # type: (Any) -> Text
                 # docker_req is none only when there is no dockerRequirement
                 # mentioned in hints and Requirement
                 path = docker_windows_path_adjust(value["path"])
-                assert path is not None
                 return path
             return value["path"]
         else:
@@ -424,8 +425,7 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
             if sep:
                 args.extend([prefix, self.tostr(j)])
             else:
-                assert prefix is not None
-                args.append(prefix + self.tostr(j))
+                args.append("" if not prefix else prefix + self.tostr(j))
 
         return [a for a in args if a is not None]
 

cwltool/command_line_tool.py

@@ -157,7 +157,8 @@ def revmap_file(builder, outdir, f):
         if "basename" not in f:
             f["basename"] = os.path.basename(path)
 
-        assert builder.pathmapper is not None
+        if not builder.pathmapper:
+            raise ValueError("Do not call revmap_file using a builder that doesn't have a pathmapper.")
         revmap_f = builder.pathmapper.reversemap(path)
 
         if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"):
@@ -204,7 +205,8 @@ def check_adjust(builder, file_o):
     doesn't reach everything in builder.bindings
     """
 
-    assert builder.pathmapper is not None
+    if not builder.pathmapper:
+            raise ValueError("Do not call check_adjust using a builder that doesn't have a pathmapper.")
     file_o["path"] = docker_windows_path_adjust(
         builder.pathmapper.mapper(file_o["location"])[1])
     dn, bn = os.path.split(file_o["path"])
@@ -300,7 +302,7 @@ def job(self,
         if runtimeContext.cachedir and enableReuse:
             cachecontext = runtimeContext.copy()
             cachecontext.outdir = "/out"
-            cachecontext.tmpdir = "/tmp"
+            cachecontext.tmpdir = "/tmp"  # nosec
             cachecontext.stagedir = "/stage"
             cachebuilder = self._init_job(job_order, cachecontext)
             cachebuilder.pathmapper = PathMapper(cachebuilder.files,
@@ -354,7 +356,8 @@ def job(self,
 
             keydictstr = json_dumps(keydict, separators=(',', ':'),
                                     sort_keys=True)
-            cachekey = hashlib.md5(keydictstr.encode('utf-8')).hexdigest()
+            cachekey = hashlib.md5(  # nosec
+                keydictstr.encode('utf-8')).hexdigest()
 
             _logger.debug("[job %s] keydictstr is %s -> %s", jobname,
                           keydictstr, cachekey)
@@ -477,24 +480,24 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
         if self.tool.get("stdin"):
             with SourceLine(self.tool, "stdin", validate.ValidationException, debug):
                 j.stdin = builder.do_eval(self.tool["stdin"])
-                assert j.stdin is not None
-                reffiles.append({"class": "File", "path": j.stdin})
+                if j.stdin:
+                    reffiles.append({"class": "File", "path": j.stdin})
 
         if self.tool.get("stderr"):
             with SourceLine(self.tool, "stderr", validate.ValidationException, debug):
                 j.stderr = builder.do_eval(self.tool["stderr"])
-                assert j.stderr is not None
-                if os.path.isabs(j.stderr) or ".." in j.stderr:
-                    raise validate.ValidationException(
-                        "stderr must be a relative path, got '%s'" % j.stderr)
+                if j.stderr:
+                    if os.path.isabs(j.stderr) or ".." in j.stderr:
+                        raise validate.ValidationException(
+                            "stderr must be a relative path, got '%s'" % j.stderr)
 
         if self.tool.get("stdout"):
             with SourceLine(self.tool, "stdout", validate.ValidationException, debug):
                 j.stdout = builder.do_eval(self.tool["stdout"])
-                assert j.stdout is not None
-                if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
-                    raise validate.ValidationException(
-                        "stdout must be a relative path, got '%s'" % j.stdout)
+                if j.stdout:
+                    if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
+                        raise validate.ValidationException(
+                            "stdout must be a relative path, got '%s'" % j.stdout)
 
         if debug:
             _logger.debug(u"[job %s] command line bindings is %s", j.name,
@@ -674,8 +677,8 @@ def collect_output(self,
                             globpatterns.extend(aslist(gb))
 
                     for gb in globpatterns:
-                        if gb.startswith(outdir):
-                            gb = gb[len(outdir) + 1:]
+                        if gb.startswith(builder.outdir):
+                            gb = gb[len(builder.outdir) + 1:]
                         elif gb == ".":
                             gb = outdir
                         elif gb.startswith("/"):
@@ -717,7 +720,7 @@ def collect_output(self,
                                 files["contents"] = content_limit_respected_read_bytes(f).decode("utf-8")
                         if compute_checksum:
                             with fs_access.open(rfile["location"], "rb") as f:
-                                checksum = hashlib.sha1()
+                                checksum = hashlib.sha1()  # nosec
                                 contents = f.read(1024 * 1024)
                                 while contents != b"":
                                     checksum.update(contents)