Merge remote-tracking branch 'origin/master' into non_ASCII_testing

Author: mr-c

.gitignore

@@ -9,6 +9,7 @@ eggs/
 *.egg-info/
 *.egg
 .tox/
+.pytest_cache
 
 # Editor Temps
 .*.sw?
@@ -44,4 +45,5 @@ output.txt
 pydocstyle_report.txt
 response.txt
 test.txt
+time.txt
 value

.travis.yml

@@ -3,9 +3,6 @@ sudo: required
 services:
   - docker
 
-env:
-  - SINGVER=2.5.1
-
 before_install:
 - wget -O- http://neuro.debian.net/lists/trusty.us-nh.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
 - sudo apt-key add .travis.singularity_key.txt

MANIFEST.in

@@ -1,9 +1,11 @@
 include MANIFEST.in
+include *requirements.txt
 include gittaggers.py Makefile cwltool.py
 include tests/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/wf/*
 include tests/override/*
+include tests/checker_wf/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/draft-2/*.yml
 include cwltool/schemas/draft-3/*.yml

Makefile

@@ -59,7 +59,12 @@ install-deb-dep:
 
 ## install     : install the ${MODULE} module and schema-salad-tool
 install: FORCE
-	pip install .
+	pip install .[deps]
+
+## dev     : install the ${MODULE} module in dev mode
+dev: install-dep
+	pip install -e .[deps]
+
 
 ## dist        : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz
@@ -143,11 +148,11 @@ diff-cover.html:  coverage.xml
 
 ## test        : run the ${MODULE} test suite
 test: $(pysources)
-	python setup.py test --addopt "-n$(shell expr $(nproc) / 2)"
+	python setup.py test --addopts "-n$(nproc) --dist=loadfile"
 
 ## testcov     : run the ${MODULE} test suite and collect coverage
 testcov: $(pysources)
-	python setup.py test --addopts "--cov cwltool"
+	python setup.py test --addopts "--cov cwltool -n$(nproc) --dist=loadfile"
 
 sloccount.sc: ${PYSOURCES} Makefile
 	sloccount --duplicates --wide --details $^ > sloccount.sc

appveyor.yml

@@ -35,19 +35,13 @@ install:
   - "%PYTHON%\\python.exe -m pip install -U codecov pytest-xdist pytest-cov galaxy-lib -rtest-requirements.txt"
     # Note the use of a `^` to escape the `>`
 
-for:
-  - matrix:
-      only: [ { PYTHON_VERSION: "3.7.x" } ]
-    before_build:
-      - "%PYTHON%\\python.exe -m pip install https://download.lfd.uci.edu/pythonlibs/l8ulg3xw/lxml-4.2.3-cp37-cp37m-win_amd64.whl"
-
 build_script:
   - "%PYTHON%\\python.exe -m pip install -rrequirements.txt"
 
 test_script:
   - |
     %PYTHON%\\python.exe setup.py test --addopts "--cov-report xml --cov cwltool -p no:cacheprovider --junit-xml=tests.xml"
-  - "%PYTHON%\\python.exe -m codecov"
+  - "%PYTHON%\\python.exe -m codecov --file coverage.xml"
 
 on_finish:
   - ps: |

build-cwl-docker.sh

@@ -1,11 +1,11 @@
-#!/bin/sh
+#!/bin/bash
 set -e
-docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool-module --target module .
+docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool_module --target module .
 docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool .
 
 version=$(git describe --tags)
 echo $version | grep -vq '\-' >& /dev/null
 if [ $? -eq 0 ];then
-    docker tag commonworkflowlanguage/cwltool-module commonworkflowlanguage/cwltool-module:$version
+    docker tag commonworkflowlanguage/cwltool_module commonworkflowlanguage/cwltool_module:$version
     docker tag commonworkflowlanguage/cwltool commonworkflowlanguage/cwltool:$version
 fi

cwltool.Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.6-alpine as builder
 
-RUN apk add --no-cache git
+RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linux-headers
 
 WORKDIR /cwltool
 COPY . .
@@ -12,13 +12,13 @@ RUN pip install --no-index --no-warn-script-location --root=/pythonroot/ /wheels
 FROM python:3.6-alpine as module
 LABEL maintainer [email protected]
 
-RUN apk add --no-cache docker nodejs
+RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt
 COPY --from=builder /pythonroot/ /
 
 FROM python:3.6-alpine
 LABEL maintainer [email protected]
 
-RUN apk add --no-cache docker nodejs
+RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt
 COPY --from=builder /pythonroot/ /
 COPY cwltool-in-docker.sh /cwltool-in-docker.sh
 

cwltool.py

@@ -11,4 +11,4 @@
 from cwltool import main
 
 if __name__ == "__main__":
-    sys.exit(main.main(sys.argv[1:]))
+    main.run(sys.argv[1:])

cwltool/__main__.py

@@ -1,7 +1,6 @@
+"""Default entrypoint for the cwltool module."""
 from __future__ import absolute_import
 
-import sys
-
 from . import main
 
-sys.exit(main.main())
+main.run()

cwltool/argparser.py

@@ -1,11 +1,14 @@
+"""Command line argument parsing for cwltool."""
 from __future__ import absolute_import, print_function
 
 import argparse
 import os
-from typing import (Any, AnyStr, Dict, List,  # pylint: disable=unused-import
-                    Optional, Sequence, Text, Union, cast)
+from typing import (Any, AnyStr, Dict, List, MutableMapping, MutableSequence,
+                    Optional, Sequence, Union, cast)
 
 from schema_salad.ref_resolver import file_uri
+from typing_extensions import Text  # pylint: disable=unused-import
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from .loghandler import _logger
 from .process import Process, shortname  # pylint: disable=unused-import
@@ -25,44 +28,43 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         help="[experimental] Run jobs in parallel. ")
     envgroup = parser.add_mutually_exclusive_group()
     envgroup.add_argument("--preserve-environment", type=Text, action="append",
-                        help="Preserve specific environment variable when "
-                        "running CommandLineTools.  May be provided multiple "
-                        "times.", metavar="ENVVAR", default=["PATH"],
-                        dest="preserve_environment")
+                          help="Preserve specific environment variable when "
+                          "running CommandLineTools.  May be provided multiple "
+                          "times.", metavar="ENVVAR", default=["PATH"],
+                          dest="preserve_environment")
     envgroup.add_argument("--preserve-entire-environment", action="store_true",
-                        help="Preserve all environment variable when running "
-                        "CommandLineTools.", default=False,
-                        dest="preserve_entire_environment")
+                          help="Preserve all environment variable when running "
+                          "CommandLineTools.", default=False,
+                          dest="preserve_entire_environment")
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--rm-container", action="store_true", default=True,
                          help="Delete Docker container used by jobs after they exit (default)",
                          dest="rm_container")
 
-    exgroup.add_argument("--leave-container", action="store_false",
-                         default=True, help="Do not delete Docker container used by jobs after they exit",
-                         dest="rm_container")
+    exgroup.add_argument(
+        "--leave-container", action="store_false", default=True,
+        help="Do not delete Docker container used by jobs after they exit",
+        dest="rm_container")
 
-    cidgroup = parser.add_argument_group("Options for recording the Docker "
-        "container identifier into a file")
+    cidgroup = parser.add_argument_group(
+        "Options for recording the Docker container identifier into a file")
     cidgroup.add_argument("--record-container-id", action="store_true",
                           default=False,
                           help="If enabled, store the Docker container ID into a file. "
                           "See --cidfile-dir to specify the directory.",
                           dest="record_container_id")
 
-    cidgroup.add_argument("--cidfile-dir", type=Text,
-                        help="Directory for storing the Docker container ID file. "
-                             "The default is the current directory",
-                        default="",
-                        dest="cidfile_dir")
+    cidgroup.add_argument(
+        "--cidfile-dir", type=Text, help="Directory for storing the Docker "
+        "container ID file. The default is the current directory",
+        default="", dest="cidfile_dir")
 
-    cidgroup.add_argument("--cidfile-prefix", type=Text,
-                        help="Specify a prefix to the container ID filename. "
-                             "Final file name will be followed by a timestamp. "
-                             "The default is no prefix.",
-                        default="",
-                        dest="cidfile_prefix")
+    cidgroup.add_argument(
+        "--cidfile-prefix", type=Text,
+        help="Specify a prefix to the container ID filename. "
+        "Final file name will be followed by a timestamp. The default is no prefix.",
+        default="", dest="cidfile_prefix")
 
     parser.add_argument("--tmpdir-prefix", type=Text,
                         help="Path prefix for temporary directories",
@@ -73,8 +75,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                          help="Path prefix for intermediate output directories",
                          default=DEFAULT_TMP_PREFIX)
 
-    exgroup.add_argument("--cachedir", type=Text, default="",
-                         help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")
+    exgroup.add_argument(
+        "--cachedir", type=Text, default="",
+        help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--rm-tmpdir", action="store_true", default=True,
@@ -86,9 +89,10 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                          dest="rm_tmpdir")
 
     exgroup = parser.add_mutually_exclusive_group()
-    exgroup.add_argument("--move-outputs", action="store_const", const="move", default="move",
-                         help="Move output files to the workflow output directory and delete intermediate output directories (default).",
-                         dest="move_outputs")
+    exgroup.add_argument(
+        "--move-outputs", action="store_const", const="move", default="move",
+        help="Move output files to the workflow output directory and delete "
+        "intermediate output directories (default).", dest="move_outputs")
 
     exgroup.add_argument("--leave-outputs", action="store_const", const="leave", default="move",
                          help="Leave output files in intermediate output directories.",
@@ -378,7 +382,7 @@ def add_argument(toolparser, name, inptype, records, description="",
         flag = "--"
 
     required = True
-    if isinstance(inptype, list):
+    if isinstance(inptype, MutableSequence):
         if inptype[0] == "null":
             required = False
             if len(inptype) == 2:
@@ -395,16 +399,16 @@ def add_argument(toolparser, name, inptype, records, description="",
         action = cast(argparse.Action, FileAction)
     elif inptype == "Directory":
         action = cast(argparse.Action, DirectoryAction)
-    elif isinstance(inptype, dict) and inptype["type"] == "array":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
         if inptype["items"] == "File":
             action = cast(argparse.Action, FileAppendAction)
         elif inptype["items"] == "Directory":
             action = cast(argparse.Action, DirectoryAppendAction)
         else:
             action = "append"
-    elif isinstance(inptype, dict) and inptype["type"] == "enum":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
         atype = Text
-    elif isinstance(inptype, dict) and inptype["type"] == "record":
+    elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
         records.append(name)
         for field in inptype['fields']:
             fieldname = name + "." + shortname(field['name'])