twister: fix Ztest C++ test names extraction from ELF

Fix Ztest test function name extraction from ELF symbols
for C++ compiled binaries where symbol names need additional
'demangling' to match with corresponding test names.

The `c++filt` utility (part of binutils) is called for
demangling when it is needed.

Twister test suite extension and adjustment.

Signed-off-by: Dmitrii Golovanov <[email protected]>

Author: golowanow

scripts/pylib/twister/twisterlib/runner.py

@@ -814,6 +814,10 @@ def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs)
         self.env = env
         self.duts = None
 
+    @property
+    def trace(self) -> bool:
+        return self.options.verbose > 2
+
     def log_info(self, filename, inline_logs, log_testcases=False):
         filename = os.path.abspath(os.path.realpath(filename))
         if inline_logs:
@@ -1087,6 +1091,18 @@ def process(self, pipeline, done, message, lock, results):
                 self.instance.reason = reason
                 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
 
+    def demangle(self, symbol_name):
+        if symbol_name[:2] == '_Z':
+            try:
+                cpp_filt = subprocess.run('c++filt', input=symbol_name, text=True, check=True,
+                                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                if self.trace:
+                    logger.debug(f"Demangle: '{symbol_name}'==>'{cpp_filt.stdout}'")
+                return cpp_filt.stdout.strip()
+            except Exception as e:
+                logger.error(f"Failed to demangle '{symbol_name}': {e}")
+        return symbol_name
+
     def determine_testcases(self, results):
         yaml_testsuite_name = self.instance.testsuite.id
         logger.debug(f"Determine test cases for test suite: {yaml_testsuite_name}")
@@ -1102,19 +1118,22 @@ def determine_testcases(self, results):
                 for sym in section.iter_symbols():
                     # It is only meant for new ztest fx because only new ztest fx exposes test functions
                     # precisely.
-
+                    m_ = new_ztest_unit_test_regex.search(sym.name)
+                    if not m_:
+                        continue
+                    # Demangle C++ symbols
+                    m_ = new_ztest_unit_test_regex.search(self.demangle(sym.name))
+                    if not m_:
+                        continue
                     # The 1st capture group is new ztest suite name.
                     # The 2nd capture group is new ztest unit test name.
-                    matches = new_ztest_unit_test_regex.findall(sym.name)
-                    if matches:
-                        for m in matches:
-                            new_ztest_suite = m[0]
-                            if new_ztest_suite not in self.instance.testsuite.ztest_suite_names:
-                                logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' "
-                                               f"not present in: {self.instance.testsuite.ztest_suite_names}")
-                            test_func_name = m[1].replace("test_", "", 1)
-                            testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}"
-                            detected_cases.append(testcase_id)
+                    new_ztest_suite = m_[1]
+                    if new_ztest_suite not in self.instance.testsuite.ztest_suite_names:
+                        logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' "
+                                       f"not present in: {self.instance.testsuite.ztest_suite_names}")
+                    test_func_name = m_[2].replace("test_", "", 1)
+                    testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}"
+                    detected_cases.append(testcase_id)
 
         if detected_cases:
             logger.debug(f"Detected Ztest cases: [{', '.join(detected_cases)}] in {elf_file}")

scripts/tests/twister/test_runner.py

@@ -56,6 +56,7 @@ def mocked_instance(tmp_path):
 def mocked_env():
     env = mock.Mock()
     options = mock.Mock()
+    options.verbose = 2
     env.options = options
     return env
 
@@ -1571,6 +1572,24 @@ def mock_determine_testcases(res):
          ('dummy_id.dummy_suite2_name.dummy_name2')
         ]
     ),
+    (
+        [
+            'z_ztest_unit_test__dummy_suite2_name__test_dummy_name2',
+            'z_ztest_unit_test__bad_suite3_name_no_test',
+            '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name4E',
+            '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_bad_name1E',
+            '_ZN12_GLOBAL__N_1L51z_ztest_unit_test_dummy_suite3_name__test_bad_name2E',
+            '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name5E',
+            '_ZN15foobarnamespaceL54z_ztest_unit_test__dummy_suite3_name__test_dummy_name6E',
+        ],
+        [
+         ('dummy_id.dummy_suite2_name.dummy_name2'),
+         ('dummy_id.dummy_suite3_name.dummy_name4'),
+         ('dummy_id.dummy_suite3_name.bad_name1E'),
+         ('dummy_id.dummy_suite3_name.dummy_name5'),
+         ('dummy_id.dummy_suite3_name.dummy_name6'),
+        ]
+    ),
     (
         ['no match'],
         []
@@ -1580,10 +1599,11 @@ def mock_determine_testcases(res):
 @pytest.mark.parametrize(
     'symbols_names, added_tcs',
     TESTDATA_7,
-    ids=['two hits, one miss', 'nothing']
+    ids=['two hits, one miss', 'demangle', 'nothing']
 )
 def test_projectbuilder_determine_testcases(
     mocked_jobserver,
+    mocked_env,
     symbols_names,
     added_tcs
 ):
@@ -1603,9 +1623,8 @@ def test_projectbuilder_determine_testcases(
     instance_mock.testcases = []
     instance_mock.testsuite.id = 'dummy_id'
     instance_mock.testsuite.ztest_suite_names = []
-    env_mock = mock.Mock()
 
-    pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver)
+    pb = ProjectBuilder(instance_mock, mocked_env, mocked_jobserver)
 
     with mock.patch('twisterlib.runner.ELFFile', elf_mock), \
          mock.patch('builtins.open', mock.mock_open()):

scripts/tests/twister_blackbox/test_data/test_config.yaml

@@ -6,9 +6,10 @@ levels:
     description: >
       A plan to be used verifying basic features
     adds:
-      - dummy.agnostic.*
+      - dummy.agnostic\..*
   - name: acceptance
     description: >
       More coverage
     adds:
-      - dummy.*
+      - dummy.agnostic\..*
+      - dummy.device\..*

scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(integration)
+
+FILE(GLOB app_sources src/*.cpp)
+target_sources(app PRIVATE ${app_sources})

scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf

@@ -0,0 +1 @@
+CONFIG_ZTEST=y

scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2023-2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/ztest.h>
+
+
+// global namespace
+
+ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL);
+
+/**
+ * @brief Test Asserts
+ *
+ * This test verifies various assert macros provided by ztest.
+ *
+ */
+ZTEST(a1_1_tests, test_assert)
+{
+	zassert_true(1, "1 was false");
+	zassert_false(0, "0 was true");
+	zassert_is_null(NULL, "NULL was not NULL");
+	zassert_not_null("foo", "\"foo\" was NULL");
+	zassert_equal(1, 1, "1 was not equal to 1");
+	zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
+}

scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml

@@ -0,0 +1,10 @@
+tests:
+  dummy.agnostic_cpp.group1.subgroup1:
+    platform_allow:
+      - native_sim
+    integration_platforms:
+      - native_sim
+    tags:
+      - agnostic
+      - cpp
+      - subgrouped

scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(integration)
+
+FILE(GLOB app_sources src/*.cpp)
+target_sources(app PRIVATE ${app_sources})

scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf

@@ -0,0 +1 @@
+CONFIG_ZTEST=y

scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2023-2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/ztest.h>
+
+// global namespace
+
+ZTEST_SUITE(a1_2_tests, NULL, NULL, NULL, NULL, NULL);
+
+/**
+ * @brief Test Asserts
+ *
+ * This test verifies various assert macros provided by ztest.
+ *
+ */
+ZTEST(a1_2_tests, test_assert)
+{
+	zassert_true(1, "1 was false");
+	zassert_false(0, "0 was true");
+	zassert_is_null(NULL, "NULL was not NULL");
+	zassert_not_null("foo", "\"foo\" was NULL");
+	zassert_equal(1, 1, "1 was not equal to 1");
+	zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
+}