Add tests for short names & use junit_xml file attribute

* instead of non-existent `short_name` attribute

Author: Anton Khodak

cwltest/__init__.py

@@ -22,7 +22,7 @@
 from six.moves import zip
 from typing import Any, Dict, List
 
-import cwltest.cwl_junit_xml as cwl_junit_xml
+import junit_xml
 from cwltest.utils import compare, CompareFail, TestResult, REQUIRED, get_test_number_by_key
 
 _logger = logging.getLogger("cwltest")
@@ -198,7 +198,7 @@ def main():  # type: () -> int
     unsupported = 0
     passed = 0
     suite_name, _ = os.path.splitext(os.path.basename(args.test))
-    report = cwl_junit_xml.CWLTestSuite(suite_name, [])
+    report = junit_xml.TestSuite(suite_name, [])
 
     if args.only_tools:
         alltests = tests
@@ -267,7 +267,7 @@ def main():  # type: () -> int
 
     if args.junit_xml:
         with open(args.junit_xml, 'w') as fp:
-            cwl_junit_xml.CWLTestSuite.to_file(fp, [report])
+            junit_xml.TestSuite.to_file(fp, [report])
 
     if failures == 0 and unsupported == 0:
         _logger.info("All tests passed")

cwltest/cwl_junit_xml.py

@@ -3,8 +3,7 @@
 from six.moves import range
 from typing import Any, Dict, Set, Text, List, Optional
 
-from cwltest import cwl_junit_xml
-
+import junit_xml
 REQUIRED = "required"
 
 
@@ -22,15 +21,15 @@ def __init__(self, return_code, standard_output, error_output, duration, classna
         self.classname = classname
 
     def create_test_case(self, test):
-        # type: (Dict[Text, Any]) -> cwl_junit_xml.TestCase
+        # type: (Dict[Text, Any]) -> junit_xml.TestCase
         doc = test.get(u'doc', 'N/A').strip()
         if test.get("tags"):
             category = ", ".join(test['tags'])
         else:
             category = REQUIRED
         short_name = test.get(u'short_name')
-        case = cwl_junit_xml.TestCase(
-            doc, elapsed_sec=self.duration, short_name=short_name,
+        case = junit_xml.TestCase(
+            doc, elapsed_sec=self.duration, file=short_name,
             category=category, stdout=self.standard_output, stderr=self.error_output,
         )
         if self.return_code > 0:

cwltest/utils.py

@@ -0,0 +1,7 @@
+- job: v1.0/cat-job.json
+  output: {}
+  tool: return-0.cwl
+  doc: Test with a short name
+  short_name: opt-error
+  tags: [ js, init_work_dir ]
+

tests/test-data/short-names.yml

@@ -0,0 +1,9 @@
+- job: v1.0/cat1-job.json
+  output: {}
+  tool: return-0.cwl
+  doc: Test without a short name
+- job: v1.0/cat2-job.json
+  output: {}
+  tool: return-0.cwl
+  doc: Test with a short name
+  short_name: opt-error

tests/test-data/with-and-without-short-names.yml

@@ -12,14 +12,15 @@ def test_unsupported_with_required_tests(self):
         args = ["--test", get_data("tests/test-data/required-unsupported.yml")]
         error_code, stdout, stderr = run_with_mock_cwl_runner(args)
         self.assertEquals(error_code, 1)
-        self.assertEquals("Test [1/2] \n\nTest [2/2] \n\n"
+        self.assertEquals("Test [1/2] Required test that is unsupported (without tags)\n\n"
+                          "Test [2/2] Required test that is unsupported (with tags)\n\n"
                           "0 tests passed, 2 failures, 0 unsupported features\n", stderr)
 
     def test_unsupported_with_optional_tests(self):
         args = ["--test", get_data("tests/test-data/optional-unsupported.yml")]
         error_code, stdout, stderr = run_with_mock_cwl_runner(args)
         self.assertEquals(error_code, 0)
-        self.assertEquals("Test [1/1] \n\n"
+        self.assertEquals("Test [1/1] Optional test that is unsupported\n\n"
                           "0 tests passed, 1 unsupported features\n", stderr)
 
     def test_error_with_optional_tests(self):

tests/test_categories.py

@@ -0,0 +1,37 @@
+import unittest
+
+import os
+
+from .util import run_with_mock_cwl_runner, get_data
+import xml.etree.ElementTree as ET
+
+
+class TestShortNames(unittest.TestCase):
+
+    def test_stderr_output(self):
+        args = ["--test", get_data("tests/test-data/short-names.yml")]
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+        self.assertIn("Test [1/1] opt-error: Test with a short name\n", stderr)
+
+    def test_run_by_short_name(self):
+        short_name = "opt-error"
+        args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-s", short_name]
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+        self.assertIn("Test [2/2] opt-error: Test with a short name", stderr)
+        self.assertNotIn("Test [1/2]", stderr)
+
+    def test_list_tests(self):
+        args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-l"]
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+        self.assertEquals("[1] Test without a short name\n"
+                          "[2] opt-error: Test with a short name\n", stdout)
+
+    def test_short_name_in_junit_xml(self):
+        junit_xml_report = get_data("tests/test-data/junit-report.xml")
+        args = ["--test", get_data("tests/test-data/short-names.yml"), "--junit-xml", junit_xml_report]
+        run_with_mock_cwl_runner(args)
+        tree = ET.parse(junit_xml_report)
+        root = tree.getroot()
+        category = root.find("testsuite").find("testcase").attrib['file']
+        self.assertEquals(category, "opt-error")
+        os.remove(junit_xml_report)