Merge branch 'main' of https://github.com/common-workflow-language/cwltool into mount-only-dirs

Author: DailyDreaming

Makefile

@@ -92,7 +92,7 @@ pep257: pydocstyle
 pydocstyle: $(PYSOURCES)
 	pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true
 
-pydocstyle_report.txt: $(PYSOURCES)
+pydocstyle_report.txt: $(filter-out tests/%,${PYSOURCES})
 	pydocstyle setup.py $^ > $@ 2>&1 || true
 
 diff_pydocstyle_report: pydocstyle_report.txt

README.rst

@@ -21,7 +21,7 @@ intended to be feature complete and provide comprehensive validation of CWL
 files as well as provide other tools related to working with CWL.
 
 This is written and tested for
-`Python <https://www.python.org/>`_ ``3.x {x = 6, 7, 8}``
+`Python <https://www.python.org/>`_ ``3.x {x = 6, 7, 8, 9}``
 
 The reference implementation consists of two packages.  The ``cwltool`` package
 is the primary Python module containing the reference implementation in the
@@ -31,61 +31,81 @@ The ``cwlref-runner`` package is optional and provides an additional entry point
 under the alias ``cwl-runner``, which is the implementation-agnostic name for the
 default CWL interpreter installed on a host.
 
-``cwltool`` is provided by the CWL project, `a member project of Software Freedom Conservancy <https://sfconservancy.org/news/2018/apr/11/cwl-new-member-project/>`_ and our `many contributors <https://github.com/common-workflow-language/cwltool/graphs/contributors>`_.
+``cwltool`` is provided by the CWL project, `a member project of Software Freedom Conservancy <https://sfconservancy.org/news/2018/apr/11/cwl-new-member-project/>`_
+and our `many contributors <https://github.com/common-workflow-language/cwltool/graphs/contributors>`_.
 
 Install
 -------
 
-Your operating system may offer cwltool directly. For `Debian <https://tracker.debian.org/pkg/cwltool>`_  or `Ubuntu <https://launchpad.net/ubuntu/+source/cwltool>`_ try
+``cwltool`` packages
+^^^^^^^^^^^^^^^^^^^^
+
+Your operating system may offer cwltool directly. For `Debian <https://tracker.debian.org/pkg/cwltool>`_, `Ubuntu <https://launchpad.net/ubuntu/+source/cwltool>`_,
+and similar Linux distribution try
 
 .. code:: bash
 
-  apt-get install cwltool
+  sudo apt-get install cwltool
 
-For MacOS X, other UNIXes or Windows packages prepared by the conda-forge project. Please follow instructions of conda-forge (https://conda-forge.org/#about) for its installation, then perform: 
+If you are running MacOS X or other UNIXes and you want to use packages prepared by the conda-forge project, then
+please follow the install instructions for `conda-forge <https://conda-forge.org/#about>`_ (if you haven't already) and then 
 
 .. code:: bash
 
    conda install -c conda-forge cwltool
-   
-Under the hood, conda setups virtual environments before installing `cwltool` to
-avoid conflicting versions of the same library. When installing cwltool directly,
-it is recommended to do the same manually:
+
+All of the above methods of installing ``cwltool`` use packages which might contain bugs already fixed in newer versions, or be missing features that you desire.
+If the packaged version of ``cwltool`` available to you is too old, then we recommend installing using ``pip`` and ``venv`::
 
 .. code:: bash
 
-  virtualenv -p python3 venv   # Create a virtual environment
-  source venv/bin/activate     # Activate environment before installing `cwltool`
+  python3 -m venv env      # Create a virtual environment named 'env' in the current directory
+  source env/bin/activate  # Activate environment before installing `cwltool`
 
-Installing the official package from PyPi (will install "cwltool" package as
+Then install the latest ``cwlref-runner`` package from PyPi (which will install the latest ``cwltool`` package as
 well)
 
 .. code:: bash
 
   pip install cwlref-runner
 
-If installing alongside another CWL implementation then
+If installing alongside another CWL implementation (like ``toil-cwl-runner`` or ``arvados-cwl-runner``) then instead run
 
 .. code:: bash
 
   pip install cwltool
 
-Or you can install from source:
+MS Windows users
+^^^^^^^^^^^^^^^^
+
+1.  Install `"Windows Subsystem for Linux 2" (WSL2) and Docker Desktop <https://docs.docker.com/docker-for-windows/wsl/#prerequisites>`_
+2.  Install `Debian from the Microsoft Store <https://www.microsoft.com/en-us/p/debian/9msvkqc78pk6>`_
+3.  Set Debian as your default WSL 2 distro: ``wsl --set-default debian``
+4.  Reboot if you have not yet already.
+5.  Launch Debian and follow the Linux instructions above (``apt-get install cwltool`` or use the ``venv`` method)
+
+``cwltool`` development version
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Or you can skip the direct ``pip`` commands above and install the latest development version of ``cwltool``:
 
 .. code:: bash
 
-  git clone https://github.com/common-workflow-language/cwltool.git # clone cwltool repo
-  cd cwltool         # Switch to source directory
-  pip install .      # Install `cwltool` from source
+  git clone https://github.com/common-workflow-language/cwltool.git # clone (copy) the cwltool git repository
+  cd cwltool         # Change to source directory that git clone just downloaded
+  pip install .      # Installs ``cwltool`` from source
   cwltool --version  # Check if the installation works correctly
 
 Remember, if co-installing multiple CWL implementations then you need to
 maintain which implementation ``cwl-runner`` points to via a symbolic file
 system link or `another facility <https://wiki.debian.org/DebianAlternatives>`_.
 
+Recommended Software
+^^^^^^^^^^^^^^^^^^^^
+
 You may also want to have the following installed: 
-node.js
-Docker, udocker, or Singularity (optional)
+- `node.js <https://nodejs.org/en/download/>`_
+- Docker, udocker, or Singularity (optional)
 
 Without these, some examples in the CWL tutorials at http://www.commonwl.org/user_guide/ may not work. 
 
@@ -97,7 +117,7 @@ Simple command::
   cwl-runner [tool-or-workflow-description] [input-job-settings]
 
 Or if you have multiple CWL implementations installed and you want to override
-the default cwl-runner use::
+the default cwl-runner then use::
 
   cwltool [tool-or-workflow-description] [input-job-settings]
 
@@ -106,8 +126,8 @@ these will be inserted at the beginning of the command line::
 
   export CWLTOOL_OPTIONS="--debug"
 
-Use with boot2docker
---------------------
+Use with boot2docker on macOS
+-----------------------------
 boot2docker runs Docker inside a virtual machine and it only mounts ``Users``
 on it. The default behavior of CWL is to create temporary directories under e.g.
 ``/Var`` which is not accessible to Docker containers.
@@ -117,8 +137,8 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``::
 
     $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json
 
-Using user-space replacements for Docker
-----------------------------------------
+Using uDocker
+-------------
 
 Some shared computing environments don't support Docker software containers for technical or policy reasons.
 As a work around, the CWL reference runner supports using alternative ``docker`` implementations on Linux
@@ -132,9 +152,9 @@ Run `cwltool` just as you normally would, but with the new option, e.g. from the
 
 .. code:: bash
 
-  cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json
+  cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/raw/main/v1.0/v1.0/empty.json
 
-``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ version 2.6.1
+``cwltool`` can also use `Singularity <https://github.com/hpcng/singularity/releases/>`_ version 2.6.1
 or later as a Docker container runtime.
 ``cwltool`` with Singularity will run software containers specified in
 ``DockerRequirement`` and therefore works with Docker images only, native
@@ -143,7 +163,7 @@ runtime, provide ``--singularity`` command line option to ``cwltool``.
 With Singularity, ``cwltool`` can pass all CWL v1.0 conformance tests, except
 those involving Docker container ENTRYPOINTs.
 
-
+Example:
 .. code:: bash
 
   cwltool --singularity https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/v1.0/cat3-tool-mediumcut.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/cat-job.json

cwltool/executors.py

@@ -32,11 +32,10 @@
 from .mutation import MutationManager
 from .process import Process, cleanIntermediate, relocateOutputs
 from .provenance_profile import ProvenanceProfile
+from .task_queue import TaskQueue
 from .utils import CWLObjectType, JobsType
 from .workflow import Workflow
 from .workflow_job import WorkflowJob, WorkflowJobStep
-from .task_queue import TaskQueue
-
 
 TMPDIR_LOCK = Lock()
 

cwltool/task_queue.py

@@ -5,7 +5,6 @@
 
 import queue
 import threading
-
 from typing import Callable, Optional
 
 from .loghandler import _logger

requirements.txt

@@ -1,8 +1,8 @@
 requests>=2.4.3
 ruamel.yaml>=0.12.4,<=0.16.5
-rdflib>=4.2.2,<4.3
+rdflib>=4.2.2,<5.1
 shellescape>=3.4.1,<3.5
-schema-salad>=7,<8
+schema-salad>=7.0.20210124093443,<8
 prov==1.5.1
 bagit==1.6.4
 mypy-extensions

setup.py

@@ -95,9 +95,9 @@
         "requests >= 2.6.1",  # >= 2.6.1 to workaround
         # https://github.com/ionrock/cachecontrol/issues/137
         "ruamel.yaml >= 0.12.4, <= 0.16.5",
-        "rdflib >= 4.2.2, < 4.3.0",
+        "rdflib >= 4.2.2, < 5.1.0",
         "shellescape >= 3.4.1, < 3.5",
-        "schema-salad >= 7, < 8",
+        "schema-salad >= 7.0.20210124093443, < 8",
         "mypy-extensions",
         "psutil",
         "prov == 1.5.1",
@@ -121,7 +121,7 @@
     setup_requires=PYTEST_RUNNER,
     test_suite="tests",
     tests_require=[
-        "pytest < 7",
+        "pytest == 6.0.2",
         "mock >= 2.0.0",
         "pytest-mock >= 1.10.0",
         "arcp >= 0.2.0",

test-requirements.txt

@@ -1,4 +1,4 @@
-pytest < 7
+pytest == 6.0.2
 mock >= 2.0.0
 pytest-mock >= 1.10.0
 pytest-cov

tests/test_docker.py

@@ -1,27 +1,27 @@
 from distutils import spawn
-
-import py.path
+from pathlib import Path
 
 from cwltool.main import main
 
 from .util import get_data, get_main_output, needs_docker
 
 
 @needs_docker
-def test_docker_workflow(tmpdir: py.path.local) -> None:
+def test_docker_workflow(tmp_path: Path) -> None:
+    """Basic test for docker with a CWL Workflow."""
     result_code, _, stderr = get_main_output(
         [
             "--default-container",
             "debian",
             "--outdir",
-            str(tmpdir),
+            str(tmp_path),
             get_data("tests/wf/hello-workflow.cwl"),
             "--usermessage",
             "hello",
         ]
     )
     assert "completed success" in stderr
-    assert (tmpdir / "response.txt").read_text("utf-8") == "hello"
+    assert (tmp_path / "response.txt").read_text("utf-8") == "hello"
     assert result_code == 0
 
 

tests/test_empty_input.py

@@ -1,19 +1,25 @@
 from io import StringIO
+from pathlib import Path
 
 from cwltool.main import main
 
-from .util import get_data, temp_dir, windows_needs_docker
+from .util import get_data, windows_needs_docker
 
 
 @windows_needs_docker
-def test_empty_input() -> None:
+def test_empty_input(tmp_path: Path) -> None:
+    """Affirm that an empty input works."""
     empty_json = "{}"
     empty_input = StringIO(empty_json)
 
-    with temp_dir() as tmpdir:
-        params = ["--outdir", tmpdir, get_data("tests/wf/no-parameters-echo.cwl"), "-"]
+    params = [
+        "--outdir",
+        str(tmp_path),
+        get_data("tests/wf/no-parameters-echo.cwl"),
+        "-",
+    ]
 
-        try:
-            assert main(params, stdin=empty_input) == 0
-        except SystemExit as err:
-            assert err.code == 0
+    try:
+        assert main(params, stdin=empty_input) == 0
+    except SystemExit as err:
+        assert err.code == 0