twister: Add support for Cpputest

Similar to gTest, CppuTest is a CPP framework for unit tests.
This commit adds support to detect Cpputest console output
when verbose mode is enabled (-v).

Signed-off-by: Victor Chavez <[email protected]>

Author: vChavezB

doc/develop/test/twister.rst

@@ -435,15 +435,17 @@ harness: <string>
     - pytest
     - gtest
     - robot
-
-    Harnesses ``ztest``, ``gtest`` and ``console`` are based on parsing of the
-    output and matching certain phrases. ``ztest`` and ``gtest`` harnesses look
-    for pass/fail/etc. frames defined in those frameworks. Use ``gtest``
-    harness if you've already got tests written in the gTest framework and do
-    not wish to update them to zTest. The ``console`` harness tells Twister to
-    parse a test's text output for a regex defined in the test's YAML file.
-    The ``robot`` harness is used to execute Robot Framework test suites
-    in the Renode simulation framework.
+	- cpputest
+
+    Harnesses ``ztest``, ``gtest``, ``cpputest`` and ``console`` are based on parsing of the
+    output and matching certain phrases. ``ztest``, ``gtest`` and ``cpputest`` harnesses look
+    for pass/fail/etc. frames defined in those frameworks. Use ``gtest`` or ``cpputest``
+    harness if you've already got tests written with either of these framework and do
+    not wish to update them to zTest. If using ``cpputest`` be sure to pass the
+    verbose argument ``-v`` to `CommandLineTestRunner::RunAllTests`.
+    The ``console`` harness tells Twister to parse a test's text output
+    for a regex defined in the test's YAML file. The ``robot`` harness is used
+    to execute Robot Framework test suites in the Renode simulation framework.
 
     Some widely used harnesses that are not supported yet:
 

scripts/pylib/twister/twisterlib/harness.py

@@ -718,6 +718,82 @@ def handle(self, line):
                 tc.reason = "Test failure"
 
 
+class Cpputest(Harness):
+    TEST_START_PATTERN = r".*(?<!Failure in )TEST\((?P<suite_name>[^,]+), (?P<test_name>[^\)]+)\)"
+    TEST_FAIL_PATTERN = r".*Failure in TEST\((?P<suite_name>[^,]+), (?P<test_name>[^\)]+)\).*"
+    FINISHED_PATTERN = r".*(OK|Errors) \(\d+ tests, \d+ ran, \d+ checks, \d+ ignored, \d+ filtered out, \d+ ms\)"
+
+    def __init__(self):
+        super().__init__()
+        self.tc = None
+        self.has_failures = False
+
+    def handle(self, line):
+        if self.state:
+            return
+
+        # Check if a new test starts
+        test_start_match = re.search(self.TEST_START_PATTERN, line)
+        if test_start_match:
+            # If a new test starts and there is an unfinished test, mark it as passed
+            if self.tc is not None:
+                self.tc.status = "passed"
+                self.tc.output = self.testcase_output
+                self.testcase_output = ""
+                self.tc = None
+
+            suite_name = test_start_match.group("suite_name")
+            test_name = test_start_match.group("test_name")
+            if suite_name not in self.detected_suite_names:
+                self.detected_suite_names.append(suite_name)
+
+            name = "{}.{}.{}".format(self.id, suite_name, test_name)
+
+            tc = self.instance.get_case_by_name(name)
+            assert tc is None, "CppUTest error, {} running twice".format(name)
+
+            tc = self.instance.get_case_or_create(name)
+            self.tc = tc
+            self.tc.status = "started"
+            self.testcase_output += line + "\n"
+            self._match = True
+
+        # Check if a test failure occurred
+        test_fail_match = re.search(self.TEST_FAIL_PATTERN, line)
+        if test_fail_match:
+            suite_name = test_fail_match.group("suite_name")
+            test_name = test_fail_match.group("test_name")
+            name = "{}.{}.{}".format(self.id, suite_name, test_name)
+
+            tc = self.instance.get_case_by_name(name)
+            if tc is not None:
+                tc.status = "failed"
+                self.has_failures = True
+                tc.output = self.testcase_output
+                self.testcase_output = ""
+                self.tc = None
+            return
+
+        # Check if the test run finished
+        finished_match = re.search(self.FINISHED_PATTERN, line)
+        if finished_match:
+            # No need to check result if previously there was a failure
+            # or no tests were run
+            if self.has_failures or self.tc is None:
+                return
+
+            tc = self.instance.get_case_or_create(self.tc.name)
+
+            finish_result = finished_match.group(1)
+            if finish_result == "OK":
+                self.state = "passed"
+                tc.status = "passed"
+            else:
+                self.state = "failed"
+                tc.status = "failed"
+            return
+
+
 class Ztest(Test):
     pass
 

scripts/pylib/twister/twisterlib/testinstance.py

@@ -176,7 +176,7 @@ def get_case_or_create(self, name):
     def testsuite_runnable(testsuite, fixtures):
         can_run = False
         # console harness allows us to run the test and capture data.
-        if testsuite.harness in [ 'console', 'ztest', 'pytest', 'test', 'gtest', 'robot']:
+        if testsuite.harness in [ 'console', 'ztest', 'pytest', 'test', 'gtest', 'robot','cpputest']:
             can_run = True
             # if we have a fixture that is also being supplied on the
             # command-line, then we need to run the test, not just build it.

scripts/tests/twister/test_harness.py

@@ -16,7 +16,7 @@
 ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
 sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
 
-from twisterlib.harness import Gtest, Bsim
+from twisterlib.harness import Gtest, Bsim, Cpputest
 from twisterlib.harness import Harness
 from twisterlib.harness import Robot
 from twisterlib.harness import Test
@@ -38,6 +38,17 @@
     "[00:00:00.000,000] <inf> label:  [==========] Done running all tests."
 )
 
+SAMPLE_CPPUTEST_NO_TESTS = (
+    "Errors (ran nothing, 0 tests, 0 ran, 0 checks, 0 ignored, 0 filtered out, 0 ms)")
+SAMPLE_CPPUTEST_START_FMT = "[00:00:00.000,000] <inf> label:  TEST({suite}, {test})"
+SAMPLE_CPPUTEST_END_PASS_FMT = "[00:00:00.000,000] <inf> label:  OK ({tests} tests" \
+                               ", {ran} ran, {checks} checks, {ignored} ignored," \
+                               " {filtered} filtered out, {time} ms)"
+SAMPLE_CPPUTEST_FAIL_FMT = "[00:00:00.000,000] <inf> label:  Failure in TEST({suite}, {test})"
+SAMPLE_CPPUTEST_END_FAIL_FMT = "[00:00:00.000,000] <inf> label:  Errors({failures} failures" \
+                               ", {tests} tests, {ran} ran, {checks} checks, {ignored} ignored," \
+                               " {filtered} filtered out, {time} ms)"
+
 
 def process_logs(harness, logs):
     for line in logs:
@@ -795,6 +806,124 @@ def test_gtest_repeated_run(gtest):
         )
 
 
+@pytest.fixture
+def cpputest(tmp_path):
+    mock_platform = mock.Mock()
+    mock_platform.name = "mock_platform"
+    mock_platform.normalized_name = "mock_platform"
+    mock_testsuite = mock.Mock()
+    mock_testsuite.name = "mock_testsuite"
+    mock_testsuite.detailed_test_id = True
+    mock_testsuite.id = "id"
+    mock_testsuite.testcases = []
+    mock_testsuite.harness_config = {}
+    outdir = tmp_path / 'cpputest_out'
+    outdir.mkdir()
+
+    instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir=outdir)
+
+    harness = Cpputest()
+    harness.configure(instance)
+    return harness
+
+
+def test_cpputest_start_test_no_suites_detected(cpputest):
+    process_logs(cpputest, [SAMPLE_CPPUTEST_NO_TESTS])
+    assert len(cpputest.detected_suite_names) == 0
+    assert cpputest.state is None
+
+
+def test_cpputest_start_test(cpputest):
+    process_logs(
+        cpputest,
+        [
+            SAMPLE_CPPUTEST_START_FMT.format(
+                suite="suite_name", test="test_name"
+            ),
+        ],
+    )
+    assert cpputest.state is None
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name") is not None
+    assert (
+        cpputest.instance.get_case_by_name("id.suite_name.test_name").status == "started"
+    )
+
+
+def test_cpputest_one_test_passed(cpputest):
+    process_logs(
+        cpputest,
+        [
+            SAMPLE_CPPUTEST_START_FMT.format(
+                suite="suite_name", test="test_name"
+            ),
+            SAMPLE_CPPUTEST_END_PASS_FMT.format(
+                tests=1, ran=1, checks=5, ignored=0, filtered=0, time=10
+            )
+        ],
+    )
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name") is not None
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name").status == "passed"
+
+
+def test_cpputest_multiple_test_passed(cpputest):
+    logs = []
+    total_passed_tests = 5
+    for i in range(0, total_passed_tests):
+        logs.append(SAMPLE_CPPUTEST_START_FMT.format(suite="suite_name",
+                                                     test="test_name_%d" % i))
+    logs.append(SAMPLE_CPPUTEST_END_PASS_FMT.format(
+        tests=total_passed_tests, ran=total_passed_tests, checks=5, ignored=0, filtered=0, time=10
+    ))
+    process_logs(cpputest, logs)
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    for i in range(0, total_passed_tests):
+        test_name = "id.suite_name.test_name_%d" % i
+        assert cpputest.instance.get_case_by_name(test_name) is not None
+        assert cpputest.instance.get_case_by_name(test_name).status == "passed"
+
+
+def test_cpputest_test_failed(cpputest):
+    process_logs(
+        cpputest,
+        [
+            SAMPLE_CPPUTEST_START_FMT.format(
+                suite="suite_name", test="test_name"
+            ),
+            SAMPLE_CPPUTEST_FAIL_FMT.format(
+                suite="suite_name", test="test_name"
+            )
+        ],
+    )
+    assert cpputest.state is None
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name") is not None
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name").status == "failed"
+
+
+def test_cpputest_test_repeated(cpputest):
+    with pytest.raises(
+        AssertionError,
+        match=r"CppUTest error, id.suite_name.test_name running twice",
+    ):
+        process_logs(
+            cpputest,
+            [
+                SAMPLE_CPPUTEST_START_FMT.format(
+                    suite="suite_name", test="test_name"
+                ),
+                SAMPLE_CPPUTEST_START_FMT.format(
+                    suite="suite_name", test="test_name"
+                ),
+            ],
+        )
+
+
 def test_bsim_build(monkeypatch, tmp_path):
     mocked_instance = mock.Mock()
     build_dir = tmp_path / 'build_dir'