Merge pull request #8 from bosonogi/junit-xml-improvements

jUnit XML improvements

Author: mr-c

cwltest/__init__.py

@@ -7,6 +7,7 @@
 import sys
 import shutil
 import tempfile
+import junit_xml
 import ruamel.yaml as yaml
 import ruamel.yaml.scanner as yamlscanner
 import pipes
@@ -35,6 +36,22 @@ def format(cls, expected, actual, cause=None):
         return cls(message)
 
 
+class TestResult(object):
+
+    """Encapsulate relevant test result data."""
+
+    def __init__(self, return_code, standard_output, error_output):
+        # type: (int, str, str) -> None
+        self.return_code = return_code
+        self.standard_output = standard_output
+        self.error_output = error_output
+
+    def create_test_case(self, test):
+        # type: (Dict[Text, Any]) -> junit_xml.TestCase
+        doc = test.get(u'doc', 'N/A').strip()
+        return junit_xml.TestCase(doc, stdout=self.standard_output, stderr=self.error_output)
+
+
 def compare_file(expected, actual):
     # type: (Dict[str,Any], Dict[str,Any]) -> None
     if "path" in expected:
@@ -127,9 +144,9 @@ def compare(expected, actual):  # type: (Any, Any) -> None
         raise CompareFail(str(e))
 
 
-def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str, str]]) -> int
+def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str, str]]) -> TestResult
     out = {}  # type: Dict[str,Any]
-    outdir, outstr, test_command = None, None, None
+    outdir = outstr = outerr = test_command = None
     t = tests[i]
     try:
         test_command = [args.tool]
@@ -148,23 +165,32 @@ def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str,
 
         sys.stderr.write("\rTest [%i/%i] " % (i + 1, len(tests)))
         sys.stderr.flush()
-        outstr = subprocess.check_output(test_command)
+
+        process = subprocess.Popen(test_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        outstr, outerr = process.communicate()
+        return_code = process.poll()
+        if return_code:
+            raise subprocess.CalledProcessError(return_code, " ".join(test_command))
+
         out = json.loads(outstr)
     except ValueError as v:
         _logger.error(str(v))
         _logger.error(outstr)
+        _logger.error(outerr)
     except subprocess.CalledProcessError as err:
         if err.returncode == UNSUPPORTED_FEATURE:
-            return UNSUPPORTED_FEATURE
+            return TestResult(UNSUPPORTED_FEATURE, outstr, outerr)
         else:
             _logger.error(u"""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
             _logger.error(t.get("doc"))
             _logger.error("Returned non-zero")
-            return 1
+            _logger.error(outerr)
+            return TestResult(1, outstr, outerr)
     except (yamlscanner.ScannerError, TypeError) as e:
         _logger.error(u"""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
         _logger.error(outstr)
         _logger.error(u"Parse error %s", str(e))
+        _logger.error(outerr)
     except KeyboardInterrupt:
         _logger.error(u"""Test interrupted: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
         raise
@@ -182,10 +208,7 @@ def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str,
     if outdir:
         shutil.rmtree(outdir, True)
 
-    if failed:
-        return 1
-    else:
-        return 0
+    return TestResult((1 if failed else 0), outstr, outerr)
 
 
 def main():  # type: () -> int
@@ -215,7 +238,8 @@ def main():  # type: () -> int
     failures = 0
     unsupported = 0
     passed = 0
-    xml_lines = []  # type: List[Text]
+    suite_name, _ = os.path.splitext(os.path.basename(args.test))
+    report = junit_xml.TestSuite(suite_name, [])
 
     if args.only_tools:
         alltests = tests
@@ -251,28 +275,26 @@ def main():  # type: () -> int
                 for i in ntest]
         try:
             for i, job in zip(ntest, jobs):
-                rt = job.result()
+                test_result = job.result()
+                test_case = test_result.create_test_case(tests[i])
                 total += 1
-                if rt == 1:
+                if test_result.return_code == 1:
                     failures += 1
-                elif rt == UNSUPPORTED_FEATURE:
+                    test_case.add_failure_info("N/A")
+                elif test_result.return_code == UNSUPPORTED_FEATURE:
                     unsupported += 1
+                    test_case.add_skipped_info("Unsupported")
                 else:
                     passed += 1
-                xml_lines += make_xml_lines(tests[i], rt, args.test)
+                report.test_cases.append(test_case)
         except KeyboardInterrupt:
             for job in jobs:
                 job.cancel()
             _logger.error("Tests interrupted")
 
     if args.junit_xml:
         with open(args.junit_xml, 'w') as fp:
-            fp.write('<testsuites>\n')
-            fp.write('  <testsuite name="%s" tests="%s" failures="%s" skipped="%s">\n' % (
-                args.tool, len(ntest), failures, unsupported))
-            fp.writelines(xml_lines)
-            fp.write('  </testsuite>\n')
-            fp.write('</testsuites>\n')
+            junit_xml.TestSuite.to_file(fp, [report])
 
     if failures == 0 and unsupported == 0:
         _logger.info("All tests passed")
@@ -285,25 +307,5 @@ def main():  # type: () -> int
         return 1
 
 
-def make_xml_lines(test, rt, test_case_group='N/A'):
-    # type: (Dict[Text, Any], int, Text) -> List[Text]
-    doc = test.get('doc', 'N/A').strip()
-    test_case_group = test_case_group.replace(".yaml", "").replace(".yml", "")
-    elem = '    <testcase name="%s" classname="%s"' % (doc, test_case_group.replace(".", "_"))
-    if rt == 0:
-        return [ elem + '/>\n' ]
-    if rt == UNSUPPORTED_FEATURE:
-        return [
-            elem + '>\n',
-            '      <skipped/>\n'
-            '</testcase>\n',
-        ]
-    return [
-        elem + '>\n',
-        '      <failure message="N/A">N/A</failure>\n'
-        '</testcase>\n',
-    ]
-
-
 if __name__ == "__main__":
     sys.exit(main())

requirements.txt

@@ -1,3 +1,4 @@
 schema-salad >= 1.14
 typing >= 3.5.2
 futures >= 3.0.5; python_version == '2.7'
+junit-xml >= 1.7

setup.py

@@ -26,7 +26,8 @@
       packages=["cwltest"],
       install_requires=[
           'schema-salad >= 1.14',
-          'typing >= 3.5.2'
+          'typing >= 3.5.2',
+          'junit-xml >= 1.7'
       ],
       extras_require={
           ':python_version == "2.7"': [

typeshed/2.7/__builtin__.pyi

@@ -911,7 +911,7 @@ class file(BinaryIO):
     @overload
     def __init__(self, file: unicode, mode: str = 'r', buffering: int = ...) -> None: ...
     @overload
-    def __init__(file: int, mode: str = 'r', buffering: int = ...) -> None: ...
+    def __init__(self, file: int, mode: str = 'r', buffering: int = ...) -> None: ...
     def __iter__(self) -> Iterator[str]: ...
     def read(self, n: int = ...) -> str: ...
     def __enter__(self) -> BinaryIO: ...

typeshed/2.7/junit_xml.pyi

@@ -0,0 +1,44 @@
+# Stubs for junit_xml (Python 3.5)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any, Optional
+
+unichr = ...  # type: Any
+
+def decode(var, encoding): ...
+
+class TestSuite:
+    name = ...  # type: Any
+    test_cases = ...  # type: Any
+    hostname = ...  # type: Any
+    id = ...  # type: Any
+    package = ...  # type: Any
+    timestamp = ...  # type: Any
+    properties = ...  # type: Any
+    def __init__(self, name, test_cases: Optional[Any] = ..., hostname: Optional[Any] = ..., id: Optional[Any] = ..., package: Optional[Any] = ..., timestamp: Optional[Any] = ..., properties: Optional[Any] = ...) -> None: ...
+    def build_xml_doc(self, encoding: Optional[Any] = ...): ...
+    @staticmethod
+    def to_xml_string(test_suites, prettyprint: bool = ..., encoding: Optional[Any] = ...): ...
+    @staticmethod
+    def to_file(file_descriptor, test_suites, prettyprint: bool = ..., encoding: Optional[Any] = ...): ...
+
+class TestCase:
+    name = ...  # type: Any
+    elapsed_sec = ...  # type: Any
+    stdout = ...  # type: Any
+    stderr = ...  # type: Any
+    classname = ...  # type: Any
+    error_message = ...  # type: Any
+    error_output = ...  # type: Any
+    failure_message = ...  # type: Any
+    failure_output = ...  # type: Any
+    skipped_message = ...  # type: Any
+    skipped_output = ...  # type: Any
+    def __init__(self, name, classname: Optional[Any] = ..., elapsed_sec: Optional[Any] = ..., stdout: Optional[Any] = ..., stderr: Optional[Any] = ...) -> None: ...
+    def add_error_info(self, message: Optional[Any] = ..., output: Optional[Any] = ...): ...
+    def add_failure_info(self, message: Optional[Any] = ..., output: Optional[Any] = ...): ...
+    def add_skipped_info(self, message: Optional[Any] = ..., output: Optional[Any] = ...): ...
+    def is_failure(self): ...
+    def is_error(self): ...
+    def is_skipped(self): ...