Deps testing (#874)

* fix error with bioconda and add test
* attempt to support bioconda on windows
* Don't set close_fds on windows
* env must not be unicode on MS Windows
* Don't test bioconda on MS Windows for now
* skip galaxy-lib during release testing
* improve parallel execution of tests

Author: mr-c

Makefile

@@ -59,7 +59,7 @@ install-deb-dep:
 
 ## install     : install the ${MODULE} module and schema-salad-tool
 install: FORCE
-	pip install .
+	pip install .[deps]
 
 ## dist        : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz

cwltool/job.py

@@ -20,6 +20,7 @@
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 import shellescape
 from schema_salad.sourceline import SourceLine
+import six
 from six import with_metaclass
 from prov.model import PROV
 
@@ -41,11 +42,11 @@
 
 FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1"
 
-SHELL_COMMAND_TEMPLATE = """#!/bin/bash
+SHELL_COMMAND_TEMPLATE = u"""#!/bin/bash
 python "run_job.py" "job.json"
 """
 
-PYTHON_RUN_SCRIPT = """
+PYTHON_RUN_SCRIPT = u"""
 import json
 import os
 import sys
@@ -78,9 +79,15 @@
         stderr = open(stderr_path, "wb")
     else:
         stderr = sys.stderr
+    if os.name == 'nt':
+        close_fds = False
+        for key, value in env.items():
+            env[key] = str(value)
+    else:
+        close_fds = True
     sp = subprocess.Popen(commands,
                           shell=False,
-                          close_fds=True,
+                          close_fds=close_fds,
                           stdin=stdin,
                           stdout=stdout,
                           stderr=stderr,
@@ -564,17 +571,21 @@ def terminate():
         for key in env:
             env_copy[key] = env[key]
 
-        job_description = dict(
-            commands=commands,
-            cwd=cwd,
-            env=env_copy,
-            stdout_path=stdout_path,
-            stderr_path=stderr_path,
-            stdin_path=stdin_path,
-        )
-        with open(os.path.join(job_dir, "job.json"), encoding='utf-8',
-                  mode="wb") as job_file:
-            json_dump(job_description, job_file, ensure_ascii=False)
+        job_description = {
+            u"commands": commands,
+            u"cwd": cwd,
+            u"env": env_copy,
+            u"stdout_path": stdout_path,
+            u"stderr_path": stderr_path,
+            u"stdin_path": stdin_path}
+
+        if six.PY2:
+            with open(os.path.join(job_dir, "job.json"), mode="wb") as job_file:
+                json_dump(job_description, job_file, ensure_ascii=False)
+        else:
+            with open(os.path.join(job_dir, "job.json"), mode="w",
+                      encoding='utf-8') as job_file:
+                json_dump(job_description, job_file, ensure_ascii=False)
         try:
             job_script = os.path.join(job_dir, "run_job.bash")
             with open(job_script, "wb") as _:

cwltool/main.py

@@ -261,7 +261,7 @@ def load_job_order(args,                 # type: argparse.Namespace
 
 def init_job_order(job_order_object,        # type: Optional[MutableMapping[Text, Any]]
                    args,                    # type: argparse.Namespace
-                   t,                       # type: Process
+                   process,                 # type: Process
                    loader,                  # type: Loader
                    stdout,                  # type: Union[TextIO, StreamWriter]
                    print_input_deps=False,  # type: bool
@@ -270,12 +270,12 @@ def init_job_order(job_order_object,        # type: Optional[MutableMapping[Text
                    input_basedir="",        # type: Text
                    secret_store=None        # type: SecretStore
                   ):  # type: (...) -> MutableMapping[Text, Any]
-    secrets_req, _ = t.get_requirement("http://commonwl.org/cwltool#Secrets")
+    secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets")
     if not job_order_object:
         namemap = {}  # type: Dict[Text, Text]
         records = []  # type: List[Text]
         toolparser = generate_parser(
-            argparse.ArgumentParser(prog=args.workflow), t, namemap, records)
+            argparse.ArgumentParser(prog=args.workflow), process, namemap, records)
         if toolparser:
             if args.tool_help:
                 toolparser.print_help()
@@ -295,8 +295,8 @@ def init_job_order(job_order_object,        # type: Optional[MutableMapping[Text
                 try:
                     job_order_object = cast(
                         MutableMapping, loader.resolve_ref(cmd_line["job_order"])[0])
-                except Exception as e:
-                    _logger.error(Text(e), exc_info=args.debug)
+                except Exception as err:
+                    _logger.error(Text(err), exc_info=args.debug)
                     exit(1)
             else:
                 job_order_object = {"id": args.workflow}
@@ -315,15 +315,15 @@ def init_job_order(job_order_object,        # type: Optional[MutableMapping[Text
         else:
             job_order_object = None
 
-    for inp in t.tool["inputs"]:
+    for inp in process.tool["inputs"]:
         if "default" in inp and (
                 not job_order_object or shortname(inp["id"]) not in job_order_object):
             if not job_order_object:
                 job_order_object = {}
             job_order_object[shortname(inp["id"])] = inp["default"]
 
     if not job_order_object:
-        if len(t.tool["inputs"]) > 0:
+        if process.tool["inputs"]:
             if toolparser:
                 print(u"\nOptions for {} ".format(args.workflow))
                 toolparser.print_help()
@@ -344,7 +344,7 @@ def path_to_loc(p):
             del p["path"]
 
     ns = {}  # type: Dict[Text, Union[Dict[Any, Any], Text, Iterable[Text]]]
-    ns.update(t.metadata.get("$namespaces", {}))
+    ns.update(process.metadata.get("$namespaces", {}))
     ld = Loader(ns)
 
     def expand_formats(p):
@@ -655,7 +655,7 @@ def my_represent_none(self, data):  # pylint: disable=unused-argument
         for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"):
             if getattr(runtimeContext, dirprefix) and getattr(runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX:
                 sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \
-                        else ""
+                    else ""
                 setattr(runtimeContext, dirprefix,
                         os.path.abspath(getattr(runtimeContext, dirprefix)) + sl)
                 if not os.path.exists(os.path.dirname(getattr(runtimeContext, dirprefix))):
@@ -701,11 +701,12 @@ def my_represent_none(self, data):  # pylint: disable=unused-argument
 
             if conf_file or use_conda_dependencies:
                 runtimeContext.job_script_provider = DependenciesConfiguration(args)
+            else:
+                runtimeContext.find_default_container = functools.partial(
+                    find_default_container,
+                    default_container=runtimeContext.default_container,
+                    use_biocontainers=args.beta_use_biocontainers)
 
-            runtimeContext.find_default_container = functools.partial(
-                find_default_container,
-                default_container=runtimeContext.default_container,
-                use_biocontainers=args.beta_use_biocontainers)
             (out, status) = executor(tool,
                                      initialized_job_order_object,
                                      runtimeContext,

release-test.sh

@@ -9,7 +9,7 @@ package=cwltool
 module=cwltool
 slug=${TRAVIS_PULL_REQUEST_SLUG:=common-workflow-language/cwltool}
 repo=https://github.com/${slug}.git
-run_tests="bin/py.test --ignore ${module}/schemas/ --pyarg cwltool -n$(nproc) --dist=loadfile"
+run_tests="bin/py.test --ignore ${module}/schemas/ --pyarg -x cwltool -n$(nproc) --dist=loadfile"
 pipver=7.0.2 # minimum required version of pip
 setuptoolsver=24.2.0 # required to generate correct metadata for
                      # python_requires
@@ -29,6 +29,7 @@ then
 		&& pip install --force-reinstall -U pip==${pipver} \
 	        && pip install setuptools==${setuptoolsver} wheel
 	make install-dep
+	#pip install 'galaxy-lib>=17.09.3'
 	make test
 	pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 	mkdir testenv1/not-${module}
@@ -52,7 +53,8 @@ source bin/activate
 rm lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
-pip install -e "git+${repo}@${HEAD}#egg=${package}"
+#pip install 'galaxy-lib==17.09.3'
+pip install -e "git+${repo}@${HEAD}#egg=${package}"  #[deps]
 cd src/${package}
 make install-dep
 make dist
@@ -72,8 +74,10 @@ source bin/activate
 rm lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
-pip install ${package}*tar.gz
+package_tar=${package}*tar.gz
 pip install "-r${DIR}/test-requirements.txt"
+#pip install 'galaxy-lib==17.09.3'
+pip install ${package_tar}  # [deps]
 mkdir out
 tar --extract --directory=out -z -f ${package}*.tar.gz
 cd out/${package}*

tests/test_check.py

@@ -2,11 +2,6 @@
 
 import unittest
 
-import cwltool.expression as expr
-import cwltool.factory
-import cwltool.pathmapper
-import cwltool.process
-import cwltool.workflow
 from cwltool.main import main
 
 from .util import get_data, needs_docker

tests/test_cwl_version.py

@@ -7,10 +7,10 @@
 from .util import get_data
 
 
-class CWL_Version_Checks(unittest.TestCase):
-    # no cwlVersion in the workflow
+class CWLVersionChecks(unittest.TestCase):
     def test_missing_cwl_version(self):
+        """no cwlVersion in the workflow"""
         self.assertEqual(main([get_data('tests/wf/missing_cwlVersion.cwl')]), 1)
-    # using cwlVersion: v0.1 in the workflow
     def test_incorrect_cwl_version(self):
+        """using cwlVersion: v0.1 in the workflow"""
         self.assertEqual(main([get_data('tests/wf/wrong_cwlVersion.cwl')]), 1)

tests/test_default_path.py

@@ -8,8 +8,8 @@
 
 
 class TestDefaultPath(unittest.TestCase):
-    # Testing that error is not raised when default path is not present
     def test_default_path(self):
+        """Testing that error is not raised when default path is not present"""
         document_loader, workflowobj, uri = fetch_document(
             get_data("tests/wf/default_path.cwl"))
         document_loader, avsc_names, processobj, metadata, uri = validate_document(

tests/test_dependencies.py

@@ -0,0 +1,41 @@
+from __future__ import absolute_import
+
+from cwltool.utils import onWindows
+
+from .util import get_data, needs_docker
+from .test_examples import TestCmdLine
+
+import pytest
+
+try:
+    from galaxy.tools.deps.requirements import ToolRequirement, ToolRequirements
+    from galaxy.tools import deps
+except ImportError:
+    deps = None
+
+class TestBetaDependenciesResolver(TestCmdLine):
+
+    @needs_docker
+    def test_biocontainers(self):
+        if not deps:
+            pytest.skip("galaxy-lib is not installed.")
+        wflow = get_data("tests/seqtk_seq.cwl")
+        job = get_data("tests/seqtk_seq_job.json")
+        error_code, stdout, stderr = self.get_main_output(
+            ["--beta-use-biocontainers", wflow, job])
+        print(stderr)
+        print(stdout)
+        assert error_code is 0
+
+    def test_bioconda(self):
+        if onWindows() or not deps:
+            pytest.skip("bioconda currently not working on MS Windows.")
+        elif not deps:
+            pytest.skip("galaxy-lib is not installed.")
+        wflow = get_data("tests/seqtk_seq.cwl")
+        job = get_data("tests/seqtk_seq_job.json")
+        error_code, stdout, stderr = self.get_main_output(
+            ["--beta-conda-dependencies", "--debug", wflow, job])
+        print(stderr)
+        print(stdout)
+        assert error_code is 0

tests/test_docker_warning.py

@@ -20,7 +20,7 @@ def __init__(self, **kwargs):
                 self.requirements=[]
                 self.hints=[]
 
-            def find_default_container(args, builder):
+            def find_default_container(self, args, builder):
                 return windows_default_container_id
 
         TestObject = TestCommandLineTool()

tests/test_provenance.py

@@ -13,6 +13,7 @@
 
 # Module to be tested
 from cwltool import provenance
+from cwltool import load_tool
 
 from .util import get_data
 import bagit
@@ -43,6 +44,7 @@ class TestProvenance(unittest.TestCase):
 
 
     def cwltool(self, *args):
+        load_tool.loaders = {}
         new_args = ['--no-container',
             '--provenance',
             self.folder]