test timeout feature

Author: mr-c

.coveragerc

@@ -0,0 +1,14 @@
+[run]
+branch = True
+source = cwltest
+omit = cwltest/tests/*
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+ignore_errors = True
+omit =
+    cwltest/tests/*

Makefile

@@ -112,7 +112,7 @@ diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
 
 .coverage: $(PYSOURCES) all
-	python setup.py test --addopts "--cov --cov-config=.coveragerc --cov-report= -n auto ${PYTEST_EXTRA}"
+	python setup.py test --addopts "--cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}"
 
 coverage.xml: .coverage
 	coverage xml

cwltest/tests/mock_cwl_runner.py

@@ -1,12 +1,15 @@
 import argparse
+import sys
+from time import sleep
 
 from cwltest import UNSUPPORTED_FEATURE
 
 UNSUPPORTED_FEATURE_TOOL = "return-unsupported.cwl"
 ERROR_TOOL = "return-1.cwl"
+TIMEOUT_TOOL = "timeout.cwl"
 
 
-def main():  # type: ()->int
+def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("processfile")
     parser.add_argument("jobfile")
@@ -20,6 +23,11 @@ def main():  # type: ()->int
         exit(UNSUPPORTED_FEATURE)
     elif args.processfile.endswith(ERROR_TOOL):
         exit(1)
+    elif args.processfile.endswith(TIMEOUT_TOOL):
+        print("timeout stderr", file=sys.stderr)
+        sys.stderr.flush()
+        sleep(100000)
+        exit(1)
 
     exit(0)
 

cwltest/tests/test-data/timeout.cwl

@@ -0,0 +1,5 @@
+class: CommandLineTool
+cwlVersion: v1.0
+inputs: []
+outputs: []
+baseCommand: [sleep, "15"]

cwltest/tests/test-data/timeout.yml

@@ -0,0 +1,9 @@
+- job: v1.0/empty.json
+  output:
+    output_txt:
+      class: File
+      checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b
+      location: output.txt
+      size: 13
+  tool: timeout.cwl
+  doc: Test of timeout stdout and stderr capture

cwltest/tests/test-data/v1.0/empty.json

@@ -0,0 +1 @@
+{}

cwltest/tests/test_timeout.py

@@ -0,0 +1,42 @@
+import re
+import os
+from os import linesep as n
+from os import sep as p
+from pathlib import Path
+
+import defusedxml.ElementTree as ET
+
+from .util import run_with_mock_cwl_runner, get_data
+import schema_salad.ref_resolver
+
+
+def test_timeout_stderr_stdout(tmp_path):
+    junit_xml_report = tmp_path / "junit-report.xml"
+
+    args = [
+        "--test",
+        schema_salad.ref_resolver.file_uri(get_data("tests/test-data/timeout.yml")),
+        "--timeout",
+        "5",
+        "--junit-xml",
+        str(junit_xml_report),
+    ]
+    try:
+        cwd = os.getcwd()
+        os.chdir(get_data("tests/test-data/"))
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+    finally:
+        os.chdir(cwd)
+
+    assert error_code == 1
+    assert "Test 1 timed out" in stderr
+    tree = ET.parse(junit_xml_report)
+    try:
+        root = tree.getroot()
+        timeout_text = root.find("testsuite").find("testcase").find("failure").text
+        timeout_stderr = root.find("testsuite").find("testcase").find("system-err").text
+        assert "Test timed out" in timeout_text
+        assert "timeout stderr" in timeout_stderr
+    except AttributeError as e:
+        print(junit_xml_report.read_text())
+        raise e

cwltest/tests/util.py

@@ -1,5 +1,4 @@
 import os
-import sys
 
 import subprocess  # nosec
 
@@ -11,9 +10,9 @@
 
 
 def get_data(filename):
-    filename = os.path.normpath(
-        filename
-    )  # normalizing path depending on OS or else it will cause problem when joining path
+    filename = os.path.normpath(filename)
+    # normalizing path depending on OS or else it will cause problem when
+    # joining path
     filepath = None
     try:
         filepath = resource_filename(Requirement.parse("cwltest"), filename)

test-requirements.txt

@@ -1,3 +1,2 @@
 pytest >= 6.2, < 6.3
 pytest-cov
-pytest-xdist

tox.ini

@@ -12,7 +12,7 @@ skipsdist = True
 skip_missing_interpreters = True
 
 [pytest]
-addopts=-n auto --pyargs cwltest
+addopts = --pyargs cwltest
 testpaths = cwltest/tests
 
 [gh-actions]