Merge pull request #93 from common-workflow-language/required-unsupported

Report when a required test returns unsupported feature

Author: Peter Amstutz

cwltest/__init__.py

@@ -126,13 +126,16 @@ def run_test(args,         # type: argparse.Namespace
         _logger.error(outstr)
         _logger.error(outerr)
     except subprocess.CalledProcessError as err:
-        if err.returncode == UNSUPPORTED_FEATURE:
+        if err.returncode == UNSUPPORTED_FEATURE and REQUIRED not in test.get("tags", ["required"]):
             return TestResult(UNSUPPORTED_FEATURE, outstr, outerr, duration, args.classname)
         if test.get("should_fail", False):
             return TestResult(0, outstr, outerr, duration, args.classname)
         _logger.error(u"""Test %i failed: %s""", test_number, " ".join([quote(tc) for tc in test_command]))
         _logger.error(test.get("doc"))
-        _logger.error(u"Returned non-zero")
+        if err.returncode == UNSUPPORTED_FEATURE:
+            _logger.error(u"Does not support required feature")
+        else:
+            _logger.error(u"Returned non-zero")
         _logger.error(outerr)
         return TestResult(1, outstr, outerr, duration, args.classname, str(err))
     except (yamlscanner.ScannerError, TypeError) as err:
@@ -273,9 +276,9 @@ def main():  # type: () -> int
     if args.l:
         for i, t in enumerate(tests):
             if t.get("short_name"):
-                print(u"[%i] %s: %s" % (i + 1, t["short_name"], t["doc"].strip()))
+                print(u"[%i] %s: %s" % (i + 1, t["short_name"], t.get("doc", "").strip()))
             else:
-                print(u"[%i] %s" % (i + 1, t["doc"].strip()))
+                print(u"[%i] %s" % (i + 1, t.get("doc", "").strip()))
 
         return 0
 

cwltest/tests/test_categories.py

@@ -1,7 +1,8 @@
 import unittest
-
+import re
 import os
 from os import linesep as n
+from os import sep as p
 
 from .util import run_with_mock_cwl_runner, get_data
 import xml.etree.ElementTree as ET
@@ -13,11 +14,26 @@ def test_unsupported_with_required_tests(self):
         args = ["--test", get_data("tests/test-data/required-unsupported.yml")]
         error_code, stdout, stderr = run_with_mock_cwl_runner(args)
         self.assertEqual(error_code, 1)
+        print(stderr)
+        stderr = re.sub(r" '?--outdir=[^ ]*", '', stderr)
+        if os.name == 'nt':
+            q = "'"
+        else:
+            q = ""
         self.assertEqual(
-            "Test [1/2] Required test that is unsupported (without tags){n}{n}"
-            "Test [2/2] Required test that is unsupported (with tags){n}{n}"
-            "0 tests passed, 2 failures, 0 unsupported "
-            "features{n}".format(n=n), stderr)
+            "Test [1/2] Required test that is unsupported (without tags){n}"
+            "{n}"
+            "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}"
+            "Required test that is unsupported (without tags){n}"
+            "Does not support required feature{n}"
+            "{n}"
+            "Test [2/2] Required test that is unsupported (with tags){n}"
+            "{n}"
+            "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}"
+            "Required test that is unsupported (with tags){n}"
+            "Does not support required feature{n}"
+            "{n}"
+            "0 tests passed, 2 failures, 0 unsupported features{n}".format(n=n, p=p, q=q), stderr)
 
     def test_unsupported_with_optional_tests(self):
         args = ["--test", get_data("tests/test-data/optional-unsupported.yml")]