twister: Add support for Cpputest

vChavezB · vChavezB · commit f399e8784550 · 2024-11-09T15:23:53.000+01:00
Similar to gTest, CppuTest is a CPP framework for unit tests.
This commit adds support based on the patterns for test when
CppUTest verbose mode is enabled.

Signed-off-by: Victor Chavez &lt;vchavezb@protonmail.com&gt;
diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst
@@ -482,15 +482,19 @@ harness: <string>
     - pytest
     - gtest
     - robot
-
-    Harnesses ``ztest``, ``gtest`` and ``console`` are based on parsing of the
-    output and matching certain phrases. ``ztest`` and ``gtest`` harnesses look
-    for pass/fail/etc. frames defined in those frameworks. Use ``gtest``
-    harness if you've already got tests written in the gTest framework and do
-    not wish to update them to zTest. The ``console`` harness tells Twister to
-    parse a test's text output for a regex defined in the test's YAML file.
-    The ``robot`` harness is used to execute Robot Framework test suites
-    in the Renode simulation framework.
+    - cpputest
+
+    Harnesses ``ztest``, ``gtest``, ``cpputest`` and ``console`` are based on parsing of the
+    output and matching certain phrases. ``ztest``, ``gtest`` and ``cpputest`` harnesses look
+    for pass/fail/etc. frames defined in those frameworks. Use ``gtest`` or ``cpputest``
+    harness if you've already got tests written with either of these framework and do
+    If you are using ``cpputest``, you will need to enable verbose within your ``cpputest``
+    application when calling ``CommandLineTestRunner::RunAllTests``.
+    The reason for this is that by default ``cpputest`` does not output any information
+    about individual test cases.
+    The ``console`` harness tells Twister to parse a test's text output
+    for a regex defined in the test's YAML file. The ``robot`` harness is used
+    to execute Robot Framework test suites in the Renode simulation framework.
 
     Some widely used harnesses that are not supported yet:
 
diff --git a/scripts/pylib/twister/twisterlib/harness.py b/scripts/pylib/twister/twisterlib/harness.py
@@ -782,6 +782,86 @@ def handle(self, line):
                 tc.reason = "Test failure"
 
 
+class Cpputest(Harness):
+    TEST_START_PATTERN = r".*(?<!Failure in )TEST\((?P<suite_name>[^,]+), (?P<test_name>[^\)]+)\)"
+    TEST_FAIL_PATTERN = r".*Failure in TEST\((?P<suite_name>[^,]+), (?P<test_name>[^\)]+)\).*"
+    FINISHED_PATTERN = r".*(OK|Errors) \(\d+ tests, \d+ ran, \d+ checks, \d+ ignored, \d+ filtered out, \d+ ms\)"
+
+    def __init__(self):
+        super().__init__()
+        self.tc = None
+        self.has_failures = False
+        self.started = False
+
+    def handle(self, line):
+        if not self.started:
+            self.instance.testcases = []
+            self.started = True
+        if self.status != TwisterStatus.NONE:
+            return
+
+        # Check if a new test starts
+        test_start_match = re.search(self.TEST_START_PATTERN, line)
+        if test_start_match:
+            # If a new test starts and there is an unfinished test, mark it as passed
+            if self.tc is not None:
+                self.tc.status = TwisterStatus.PASS
+                self.tc.output = self.testcase_output
+                self.testcase_output = ""
+                self.tc = None
+
+            suite_name = test_start_match.group("suite_name")
+            test_name = test_start_match.group("test_name")
+            if suite_name not in self.detected_suite_names:
+                self.detected_suite_names.append(suite_name)
+
+            name = "{}.{}.{}".format(self.id, suite_name, test_name)
+
+            tc = self.instance.get_case_by_name(name)
+            assert tc is None, "CppUTest error, {} running twice".format(name)
+
+            tc = self.instance.get_case_or_create(name)
+            self.tc = tc
+            self.tc.status = TwisterStatus.STARTED
+            self.testcase_output += line + "\n"
+            self._match = True
+
+        # Check if a test failure occurred
+        test_fail_match = re.search(self.TEST_FAIL_PATTERN, line)
+        if test_fail_match:
+            suite_name = test_fail_match.group("suite_name")
+            test_name = test_fail_match.group("test_name")
+            name = "{}.{}.{}".format(self.id, suite_name, test_name)
+
+            tc = self.instance.get_case_by_name(name)
+            if tc is not None:
+                tc.status = TwisterStatus.FAIL
+                self.has_failures = True
+                tc.output = self.testcase_output
+                self.testcase_output = ""
+                self.tc = None
+            return
+
+        # Check if the test run finished
+        finished_match = re.search(self.FINISHED_PATTERN, line)
+        if finished_match:
+            # No need to check result if previously there was a failure
+            # or no tests were run
+            if self.has_failures or self.tc is None:
+                return
+
+            tc = self.instance.get_case_or_create(self.tc.name)
+
+            finish_result = finished_match.group(1)
+            if finish_result == "OK":
+                self.status = TwisterStatus.PASS
+                tc.status = TwisterStatus.PASS
+            else:
+                self.status = TwisterStatus.FAIL
+                tc.status = TwisterStatus.FAIL
+            return
+
+
 class Ztest(Test):
     pass
 
diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py
@@ -194,7 +194,7 @@ def get_case_or_create(self, name):
     def testsuite_runnable(testsuite, fixtures):
         can_run = False
         # console harness allows us to run the test and capture data.
-        if testsuite.harness in [ 'console', 'ztest', 'pytest', 'test', 'gtest', 'robot']:
+        if testsuite.harness in [ 'console', 'ztest', 'pytest', 'test', 'gtest', 'robot', 'cpputest']:
             can_run = True
             # if we have a fixture that is also being supplied on the
             # command-line, then we need to run the test, not just build it.
diff --git a/scripts/tests/twister/test_harness.py b/scripts/tests/twister/test_harness.py
@@ -22,6 +22,7 @@
     Bsim,
     Console,
     Gtest,
+    Cpputest,
     Harness,
     HarnessImporter,
     Pytest,
@@ -53,6 +54,17 @@
     "[00:00:00.000,000] [0m<inf> label:  [----------] Global test environment tear-down[0m"
 )
 
+SAMPLE_CPPUTEST_NO_TESTS = (
+    "Errors (ran nothing, 0 tests, 0 ran, 0 checks, 0 ignored, 0 filtered out, 0 ms)")
+SAMPLE_CPPUTEST_START_FMT = "[00:00:00.000,000] [0m<inf> label:  TEST({suite}, {test})[0m"
+SAMPLE_CPPUTEST_END_PASS_FMT = "[00:00:00.000,000] [0m<inf> label:  OK ({tests} tests" \
+                               ", {ran} ran, {checks} checks, {ignored} ignored," \
+                               " {filtered} filtered out, {time} ms)[0m"
+SAMPLE_CPPUTEST_FAIL_FMT = "[00:00:00.000,000] [0m<inf> label:  Failure in TEST({suite}, {test})[0m"
+SAMPLE_CPPUTEST_END_FAIL_FMT = "[00:00:00.000,000] [0m<inf> label:  Errors({failures} failures" \
+                               ", {tests} tests, {ran} ran, {checks} checks, {ignored} ignored," \
+                               " {filtered} filtered out, {time} ms)[0m"
+
 
 def process_logs(harness, logs):
     for line in logs:
@@ -1077,6 +1089,124 @@ def test_gtest_repeated_run(gtest):
         )
 
 
+@pytest.fixture
+def cpputest(tmp_path):
+    mock_platform = mock.Mock()
+    mock_platform.name = "mock_platform"
+    mock_platform.normalized_name = "mock_platform"
+    mock_testsuite = mock.Mock()
+    mock_testsuite.name = "mock_testsuite"
+    mock_testsuite.detailed_test_id = True
+    mock_testsuite.id = "id"
+    mock_testsuite.testcases = []
+    mock_testsuite.harness_config = {}
+    outdir = tmp_path / 'cpputest_out'
+    outdir.mkdir()
+
+    instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir=outdir)
+
+    harness = Cpputest()
+    harness.configure(instance)
+    return harness
+
+
+def test_cpputest_start_test_no_suites_detected(cpputest):
+    process_logs(cpputest, [SAMPLE_CPPUTEST_NO_TESTS])
+    assert len(cpputest.detected_suite_names) == 0
+    assert cpputest.status == TwisterStatus.NONE
+
+
+def test_cpputest_start_test(cpputest):
+    process_logs(
+        cpputest,
+        [
+            SAMPLE_CPPUTEST_START_FMT.format(
+                suite="suite_name", test="test_name"
+            ),
+        ],
+    )
+    assert cpputest.status == TwisterStatus.NONE
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name") is not None
+    assert (
+        cpputest.instance.get_case_by_name("id.suite_name.test_name").status == TwisterStatus.STARTED
+    )
+
+
+def test_cpputest_one_test_passed(cpputest):
+    process_logs(
+        cpputest,
+        [
+            SAMPLE_CPPUTEST_START_FMT.format(
+                suite="suite_name", test="test_name"
+            ),
+            SAMPLE_CPPUTEST_END_PASS_FMT.format(
+                tests=1, ran=1, checks=5, ignored=0, filtered=0, time=10
+            )
+        ],
+    )
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name") != TwisterStatus.NONE
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name").status == TwisterStatus.PASS
+
+
+def test_cpputest_multiple_test_passed(cpputest):
+    logs = []
+    total_passed_tests = 5
+    for i in range(0, total_passed_tests):
+        logs.append(SAMPLE_CPPUTEST_START_FMT.format(suite="suite_name",
+                                                     test="test_name_%d" % i))
+    logs.append(SAMPLE_CPPUTEST_END_PASS_FMT.format(
+        tests=total_passed_tests, ran=total_passed_tests, checks=5, ignored=0, filtered=0, time=10
+    ))
+    process_logs(cpputest, logs)
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    for i in range(0, total_passed_tests):
+        test_name = "id.suite_name.test_name_%d" % i
+        assert cpputest.instance.get_case_by_name(test_name) != TwisterStatus.NONE
+        assert cpputest.instance.get_case_by_name(test_name).status == TwisterStatus.PASS
+
+
+def test_cpputest_test_failed(cpputest):
+    process_logs(
+        cpputest,
+        [
+            SAMPLE_CPPUTEST_START_FMT.format(
+                suite="suite_name", test="test_name"
+            ),
+            SAMPLE_CPPUTEST_FAIL_FMT.format(
+                suite="suite_name", test="test_name"
+            )
+        ],
+    )
+    assert cpputest.status == TwisterStatus.NONE
+    assert len(cpputest.detected_suite_names) == 1
+    assert cpputest.detected_suite_names[0] == "suite_name"
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name") != TwisterStatus.NONE
+    assert cpputest.instance.get_case_by_name("id.suite_name.test_name").status == TwisterStatus.FAIL
+
+
+def test_cpputest_test_repeated(cpputest):
+    with pytest.raises(
+        AssertionError,
+        match=r"CppUTest error, id.suite_name.test_name running twice",
+    ):
+        process_logs(
+            cpputest,
+            [
+                SAMPLE_CPPUTEST_START_FMT.format(
+                    suite="suite_name", test="test_name"
+                ),
+                SAMPLE_CPPUTEST_START_FMT.format(
+                    suite="suite_name", test="test_name"
+                ),
+            ],
+        )
+
+
 def test_bsim_build(monkeypatch, tmp_path):
     mocked_instance = mock.Mock()
     build_dir = tmp_path / "build_dir"
diff --git a/tests/cpputest/base/CMakeLists.txt b/tests/cpputest/base/CMakeLists.txt
@@ -0,0 +1,25 @@
+# Copyright (c) 2024, Victor Chavez (vchavezb@protonmail.com)
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(cpputest_sample)
+
+include(FetchContent)
+FetchContent_Declare(
+    CppUTest
+    GIT_REPOSITORY https://github.com/cpputest/cpputest.git
+    GIT_TAG        v4.0
+)
+set(TESTS OFF CACHE BOOL "Switch off CppUTest Test build")
+
+FetchContent_MakeAvailable(CppUTest)
+
+
+target_sources(app PRIVATE  src/main.cpp
+                            src/test_suite.cpp
+)
+
+target_link_libraries(app PRIVATE
+                            CppUTest
+                            CppUTestExt)
diff --git a/tests/cpputest/base/prj.conf b/tests/cpputest/base/prj.conf
@@ -0,0 +1,4 @@
+CONFIG_CPP=y
+CONFIG_LOG=y
+CONFIG_STD_CPP17=y
+CONFIG_REQUIRES_FULL_LIBCPP=y
diff --git a/tests/cpputest/base/src/main.cpp b/tests/cpputest/base/src/main.cpp
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2024, Victor Chavez (vchavezb@protonmail.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <CppUTest/CommandLineTestRunner.h>
+#include <posix_board_if.h> // posix_exit
+
+int main(void)
+{
+	/* Enable cpputest verbose mode (-v) to get the name
+	 * of tests that have passed. Otherwise only a dot
+	 * is printed per test.
+	 */
+	const char *cppu_test_args[] = {__FILE__, "-v", "-c"};
+	int num_args = std::size(cppu_test_args);
+	int test_Res = CommandLineTestRunner::RunAllTests(num_args, cppu_test_args);
+	/* Exit before main ends as zephyr idle thread runs in the background  */
+	posix_exit(test_Res);
+	return 0;
+}
diff --git a/tests/cpputest/base/src/test_suite.cpp b/tests/cpputest/base/src/test_suite.cpp
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2024, Victor Chavez (vchavezb@protonmail.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <zephyr/logging/log.h>
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTestExt/MockSupport.h>
+
+LOG_MODULE_REGISTER(test_suite, CONFIG_LOG_DEFAULT_LEVEL);
+
+TEST_GROUP(my_test_group) {
+    void setup() final
+	{
+    }
+    void teardown() final
+	{
+    }
+};
+
+TEST(my_test_group, test_1) {
+
+}
diff --git a/tests/cpputest/base/testcase.yaml b/tests/cpputest/base/testcase.yaml
@@ -0,0 +1,9 @@
+common:
+  tags:
+    - test_framework
+
+tests:
+  base.my_test_group:
+    platform_allow:
+      - native_sim/native/64
+    harness: "cpputest"
