From 2b84965bfd5fb236197d7f93e286cc815cb5b63c Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Thu, 17 Oct 2024 09:02:35 +0200 Subject: [PATCH 1/3] twister: ztest: harness: Fix missed TestCase statuses Fix a problem of Ztest suite names not taken into account by Twister to identify a TestCase, so in some situations a Ztest test's status was not assigned to the proper TestCase and it remains 'None' whereas the actual status value lost, eventually the resulting total execution counters not correct. The issue was observed in these situations: * Ztest application with multiple test suites having same test names. * Ztest suite is 'skipped' entirely on execution with all its tests. The proposed solution extends Twister test case name for Ztest to include Ztest suite name, so the resulting identifier looks like: `..` The above naming scheme now requires ztest_suite_name part to be provided for `--sub-test` command line option. Testcase identifiers in twister.json and testplan.json will also include ztest_suite_name component. The Twister Ztest(Test) Harness is improved to track all state changes known from the test application's log for Ztest suites and test cases, so now it parses log output from a Ztest application more scurpulously. Regular expressions to match log records are extended and optimized to compile them only once and, in some cases, fixed (suite summary). Signed-off-by: Dmitrii Golovanov --- .../pylib/twister/twisterlib/environment.py | 24 ++- scripts/pylib/twister/twisterlib/harness.py | 159 ++++++++++++++---- scripts/pylib/twister/twisterlib/runner.py | 11 +- scripts/pylib/twister/twisterlib/testsuite.py | 12 +- scripts/tests/twister/test_harness.py | 39 ++++- scripts/tests/twister/test_runner.py | 16 +- scripts/tests/twister/test_testplan.py | 5 +- scripts/tests/twister/test_testsuite.py | 2 +- scripts/tests/twister_blackbox/test_config.py | 7 +- .../tests/dummy/agnostic/group2/src/main.c | 7 + scripts/tests/twister_blackbox/test_filter.py | 16 +- .../tests/twister_blackbox/test_platform.py | 4 +- .../tests/twister_blackbox/test_printouts.py | 28 +-- scripts/tests/twister_blackbox/test_report.py | 6 +- scripts/tests/twister_blackbox/test_runner.py | 4 +- .../tests/twister_blackbox/test_shuffle.py | 10 +- .../tests/twister_blackbox/test_testlist.py | 3 +- .../tests/twister_blackbox/test_testplan.py | 11 +- 18 files changed, 254 insertions(+), 110 deletions(-) diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index 2b330caf2ab33..9f41d24b541aa 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # vim: set syntax=python ts=4 : # -# Copyright (c) 2018 Intel Corporation +# Copyright (c) 2018-2024 Intel Corporation # Copyright 2022 NXP # Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved. # @@ -149,7 +149,8 @@ def add_parse_arguments(parser = None): test_plan_report_xor.add_argument("--list-tests", action="store_true", help="""List of all sub-test functions recursively found in all --testsuite-root arguments. Note different sub-tests can share - the same section name and come from different directories. + the same test scenario identifier (section.subsection) + and come from different directories. The output is flattened and reports --sub-test names only, not their directories. For instance net.socket.getaddrinfo_ok and net.socket.fd_set belong to different directories. @@ -239,17 +240,22 @@ def add_parse_arguments(parser = None): test_xor_subtest.add_argument( "-s", "--test", "--scenario", action="append", type = norm_path, - help="Run only the specified testsuite scenario. These are named by " - "") + help="""Run only the specified test suite scenario. These are named by + 'path/relative/to/Zephyr/base/section.subsection_in_testcase_yaml', + or just 'section.subsection' identifier. With '--testsuite-root' option + the scenario will be found faster. + """) test_xor_subtest.add_argument( "--sub-test", action="append", - help="""Recursively find sub-test functions and run the entire - test section where they were found, including all sibling test + help="""Recursively find sub-test functions (test cases) and run the entire + test scenario (section.subsection) where they were found, including all sibling test functions. Sub-tests are named by: - section.name.in.testcase.yaml.function_name_without_test_prefix - Example: In kernel.fifo.fifo_loop: 'kernel.fifo' is a section name - and 'fifo_loop' is a name of a function found in main.c without test prefix. + 'section.subsection_in_testcase_yaml.ztest_suite.ztest_without_test_prefix'. + Example_1: 'kernel.fifo.fifo_api_1cpu.fifo_loop' where 'kernel.fifo' is a test scenario + name (section.subsection) and 'fifo_api_1cpu.fifo_loop' is + a Ztest suite_name.test_name identificator. + Example_2: 'debug.coredump.logging_backend' is a standalone test scenario name. """) parser.add_argument( diff --git a/scripts/pylib/twister/twisterlib/harness.py b/scripts/pylib/twister/twisterlib/harness.py index 2629cdfb83faf..cd2655e927f93 100644 --- a/scripts/pylib/twister/twisterlib/harness.py +++ b/scripts/pylib/twister/twisterlib/harness.py @@ -31,7 +31,6 @@ _WINDOWS = platform.system() == 'Windows' -result_re = re.compile(r".*(PASS|FAIL|SKIP) - (test_)?(\S*) in (\d*[.,]?\d*) seconds") class Harness: GCOV_START = "GCOV_COVERAGE_DUMP_START" GCOV_END = "GCOV_COVERAGE_DUMP_END" @@ -59,12 +58,19 @@ def __init__(self): self.ztest = False self.detected_suite_names = [] self.run_id = None + self.started_suites = {} + self.started_cases = {} self.matched_run_id = False self.run_id_exists = False self.instance: TestInstance | None = None self.testcase_output = "" self._match = False + + @property + def trace(self) -> bool: + return self.instance.handler.options.verbose > 2 + @property def status(self) -> TwisterStatus: return self._status @@ -710,42 +716,124 @@ def _check_result(self, line): class Test(Harness): __test__ = False # for pytest to skip this class when collects tests - RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL" - RUN_FAILED = "PROJECT EXECUTION FAILED" - test_suite_start_pattern = r"Running TESTSUITE (?P.*)" - ZTEST_START_PATTERN = r"START - (test_)?([a-zA-Z0-9_-]+)" - def handle(self, line): - test_suite_match = re.search(self.test_suite_start_pattern, line) - if test_suite_match: - suite_name = test_suite_match.group("suite_name") + test_suite_start_pattern = re.compile(r"Running TESTSUITE (?P\S*)") + test_suite_end_pattern = re.compile(r"TESTSUITE (?P\S*)\s+(?Psucceeded|failed)") + test_case_start_pattern = re.compile(r"START - (test_)?([a-zA-Z0-9_-]+)") + test_case_end_pattern = re.compile(r".*(PASS|FAIL|SKIP) - (test_)?(\S*) in (\d*[.,]?\d*) seconds") + test_suite_summary_pattern = re.compile(r"SUITE (?P\S*) - .* \[(?P\S*)\]: .* duration = (\d*[.,]?\d*) seconds") + test_case_summary_pattern = re.compile(r" - (PASS|FAIL|SKIP) - \[([^\.]*).(test_)?(\S*)\] duration = (\d*[.,]?\d*) seconds") + + + def get_testcase(self, tc_name, phase, ts_name=None): + """ Search a Ztest case among detected in the test image binary + expecting the same test names as already known from the ELF. + Track suites and cases unexpectedly found in the log. + """ + ts_names = self.started_suites.keys() + if ts_name: + if ts_name not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"On {phase}: unexpected Ztest suite '{ts_name}' " + f"not present among: {self.instance.testsuite.ztest_suite_names}") + if ts_name not in self.detected_suite_names: + if self.trace: + logger.debug(f"On {phase}: detected new Ztest suite '{ts_name}'") + self.detected_suite_names.append(ts_name) + ts_names = [ ts_name ] if ts_name in ts_names else [] + + # Firstly try to match the test case ID to the first running Ztest suite with this test name. + for ts_name_ in ts_names: + if self.started_suites[ts_name_]['count'] < (0 if phase == 'TS_SUM' else 1): + continue + tc_fq_id = "{}.{}.{}".format(self.id, ts_name_, tc_name) + if tc := self.instance.get_case_by_name(tc_fq_id): + if self.trace: + logger.debug(f"On {phase}: Ztest case '{tc_name}' matched to '{tc_fq_id}") + return tc + logger.debug(f"On {phase}: Ztest case '{tc_name}' is not known in {self.started_suites} running suite(s).") + tc_id = "{}.{}".format(self.id, tc_name) + return self.instance.get_case_or_create(tc_id) + + def start_suite(self, suite_name): + if suite_name not in self.detected_suite_names: self.detected_suite_names.append(suite_name) + if suite_name not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"Unexpected Ztest suite '{suite_name}'") + if suite_name in self.started_suites: + if self.started_suites[suite_name]['count'] > 0: + logger.warning(f"Already STARTED '{suite_name}':{self.started_suites[suite_name]}") + elif self.trace: + logger.debug(f"START suite '{suite_name}'") + self.started_suites[suite_name]['count'] += 1 + self.started_suites[suite_name]['repeat'] += 1 + else: + self.started_suites[suite_name] = { 'count': 1, 'repeat': 0 } + + def end_suite(self, suite_name, phase='', suite_status=None): + if suite_name in self.started_suites: + if phase == 'TS_SUM' and self.started_suites[suite_name]['count'] == 0: + return + if self.started_suites[suite_name]['count'] < 1: + logger.error(f"Already ENDED {phase} suite '{suite_name}':{self.started_suites[suite_name]}") + elif self.trace: + logger.debug(f"END {phase} suite '{suite_name}':{self.started_suites[suite_name]}") + self.started_suites[suite_name]['count'] -= 1 + elif suite_status == 'SKIP': + self.start_suite(suite_name) # register skipped suites at their summary end + self.started_suites[suite_name]['count'] -= 1 + else: + logger.warning(f"END {phase} suite '{suite_name}' without START detected") - testcase_match = re.search(self.ZTEST_START_PATTERN, line) - if testcase_match: - name = "{}.{}".format(self.id, testcase_match.group(2)) - tc = self.instance.get_case_or_create(name) + def start_case(self, tc_name): + if tc_name in self.started_cases: + if self.started_cases[tc_name]['count'] > 0: + logger.warning(f"Already STARTED '{tc_name}':{self.started_cases[tc_name]}") + self.started_cases[tc_name]['count'] += 1 + else: + self.started_cases[tc_name] = { 'count': 1 } + + def end_case(self, tc_name, phase=''): + if tc_name in self.started_cases: + if phase == 'TS_SUM' and self.started_cases[tc_name]['count'] == 0: + return + if self.started_cases[tc_name]['count'] < 1: + logger.error(f"Already ENDED {phase} case '{tc_name}':{self.started_cases[tc_name]}") + elif self.trace: + logger.debug(f"END {phase} case '{tc_name}':{self.started_cases[tc_name]}") + self.started_cases[tc_name]['count'] -= 1 + elif phase != 'TS_SUM': + logger.warning(f"END {phase} case '{tc_name}' without START detected") + + + def handle(self, line): + testcase_match = None + if self._match: + self.testcase_output += line + "\n" + + if test_suite_start_match := re.search(self.test_suite_start_pattern, line): + self.start_suite(test_suite_start_match.group("suite_name")) + elif test_suite_end_match := re.search(self.test_suite_end_pattern, line): + suite_name=test_suite_end_match.group("suite_name") + self.end_suite(suite_name, 'TS_END') + elif testcase_match := re.search(self.test_case_start_pattern, line): + tc_name = testcase_match.group(2) + tc = self.get_testcase(tc_name, 'TC_START') + self.start_case(tc.name) # Mark the test as started, if something happens here, it is mostly # due to this tests, for example timeout. This should in this case # be marked as failed and not blocked (not run). tc.status = TwisterStatus.STARTED - - if testcase_match or self._match: - self.testcase_output += line + "\n" - self._match = True - - result_match = result_re.match(line) + if not self._match: + self.testcase_output += line + "\n" + self._match = True # some testcases are skipped based on predicates and do not show up # during test execution, however they are listed in the summary. Parse # the summary for status and use that status instead. - - summary_re = re.compile(r"- (PASS|FAIL|SKIP) - \[([^\.]*).(test_)?(\S*)\] duration = (\d*[.,]?\d*) seconds") - summary_match = summary_re.match(line) - - if result_match: + elif result_match := self.test_case_end_pattern.match(line): matched_status = result_match.group(1) - name = "{}.{}".format(self.id, result_match.group(3)) - tc = self.instance.get_case_or_create(name) + tc_name = result_match.group(3) + tc = self.get_testcase(tc_name, 'TC_END') + self.end_case(tc.name) tc.status = TwisterStatus[matched_status] if tc.status == TwisterStatus.SKIP: tc.reason = "ztest skip" @@ -755,15 +843,22 @@ def handle(self, line): self.testcase_output = "" self._match = False self.ztest = True - elif summary_match: - matched_status = summary_match.group(1) - self.detected_suite_names.append(summary_match.group(2)) - name = "{}.{}".format(self.id, summary_match.group(4)) - tc = self.instance.get_case_or_create(name) + elif test_suite_summary_match := self.test_suite_summary_pattern.match(line): + suite_name=test_suite_summary_match.group("suite_name") + suite_status=test_suite_summary_match.group("suite_status") + self._match = False + self.ztest = True + self.end_suite(suite_name, 'TS_SUM', suite_status=suite_status) + elif test_case_summary_match := self.test_case_summary_pattern.match(line): + matched_status = test_case_summary_match.group(1) + suite_name = test_case_summary_match.group(2) + tc_name = test_case_summary_match.group(4) + tc = self.get_testcase(tc_name, 'TS_SUM', suite_name) + self.end_case(tc.name, 'TS_SUM') tc.status = TwisterStatus[matched_status] if tc.status == TwisterStatus.SKIP: tc.reason = "ztest skip" - tc.duration = float(summary_match.group(5)) + tc.duration = float(test_case_summary_match.group(5)) if tc.status == TwisterStatus.FAIL: tc.output = self.testcase_output self.testcase_output = "" diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 128bc598ed9c4..83ca94f9fb5f2 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -1,6 +1,6 @@ # vim: set syntax=python ts=4 : # -# Copyright (c) 20180-2022 Intel Corporation +# Copyright (c) 2018-2024 Intel Corporation # Copyright 2022 NXP # SPDX-License-Identifier: Apache-2.0 @@ -1108,13 +1108,16 @@ def determine_testcases(self, results): matches = new_ztest_unit_test_regex.findall(sym.name) if matches: for m in matches: - # new_ztest_suite = m[0] # not used for now + new_ztest_suite = m[0] + if new_ztest_suite not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' " + f"not present in: {self.instance.testsuite.ztest_suite_names}") test_func_name = m[1].replace("test_", "", 1) - testcase_id = f"{yaml_testsuite_name}.{test_func_name}" + testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}" detected_cases.append(testcase_id) if detected_cases: - logger.debug(f"{', '.join(detected_cases)} in {elf_file}") + logger.debug(f"Detected Ztest cases: [{', '.join(detected_cases)}] in {elf_file}") tc_keeper = {tc.name: {'status': tc.status, 'reason': tc.reason} for tc in self.instance.testcases} self.instance.testcases.clear() self.instance.testsuite.testcases.clear() diff --git a/scripts/pylib/twister/twisterlib/testsuite.py b/scripts/pylib/twister/twisterlib/testsuite.py index 3522c5bb218e0..01b91f4b876d8 100644 --- a/scripts/pylib/twister/twisterlib/testsuite.py +++ b/scripts/pylib/twister/twisterlib/testsuite.py @@ -1,6 +1,6 @@ # vim: set syntax=python ts=4 : # -# Copyright (c) 2018-2022 Intel Corporation +# Copyright (c) 2018-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 from enum import Enum @@ -248,14 +248,16 @@ def _find_ztest_testcases(search_area, testcase_regex): testcase_regex_matches = \ [m for m in testcase_regex.finditer(search_area)] testcase_names = \ - [m.group("testcase_name") for m in testcase_regex_matches] - testcase_names = [name.decode("UTF-8") for name in testcase_names] + [(m.group("suite_name") if m.groupdict().get("suite_name") else b'', m.group("testcase_name")) \ + for m in testcase_regex_matches] + testcase_names = [(ts_name.decode("UTF-8"), tc_name.decode("UTF-8")) for ts_name, tc_name in testcase_names] warnings = None for testcase_name in testcase_names: - if not testcase_name.startswith("test_"): + if not testcase_name[1].startswith("test_"): warnings = "Found a test that does not start with test_" testcase_names = \ - [tc_name.replace("test_", "", 1) for tc_name in testcase_names] + [(ts_name + '.' if ts_name else '') + f"{tc_name.replace('test_', '', 1)}" \ + for (ts_name, tc_name) in testcase_names] return testcase_names, warnings diff --git a/scripts/tests/twister/test_harness.py b/scripts/tests/twister/test_harness.py index c0a135fb24105..7e0fca79677ab 100644 --- a/scripts/tests/twister/test_harness.py +++ b/scripts/tests/twister/test_harness.py @@ -597,31 +597,48 @@ def test_get_harness(name): "", "Running TESTSUITE suite_name", ["suite_name"], + { 'suite_name': { 'count': 1, 'repeat': 0 } }, + {}, TwisterStatus.NONE, True, TwisterStatus.NONE, ), - ("", "START - test_testcase", [], TwisterStatus.STARTED, True, TwisterStatus.NONE), ( - "", + "On TC_START: Ztest case 'testcase' is not known in {} running suite(s)", + "START - test_testcase", + [], + {}, + { 'test_id.testcase': { 'count': 1 } }, + TwisterStatus.STARTED, + True, + TwisterStatus.NONE + ), + ( + "On TC_END: Ztest case 'example' is not known in {} running suite(s)", "PASS - test_example in 0 seconds", [], + {}, + {}, TwisterStatus.PASS, True, TwisterStatus.NONE, ), ( - "", + "On TC_END: Ztest case 'example' is not known in {} running suite(s)", "SKIP - test_example in 0 seconds", [], + {}, + {}, TwisterStatus.SKIP, True, TwisterStatus.NONE, ), ( - "", + "On TC_END: Ztest case 'example' is not known in {} running suite(s)", "FAIL - test_example in 0 seconds", [], + {}, + {}, TwisterStatus.FAIL, True, TwisterStatus.NONE, @@ -630,6 +647,8 @@ def test_get_harness(name): "not a ztest and no state for test_id", "START - test_testcase", [], + {}, + { 'test_id.testcase': { 'count': 1 } }, TwisterStatus.PASS, False, TwisterStatus.PASS, @@ -638,6 +657,8 @@ def test_get_harness(name): "not a ztest and no state for test_id", "START - test_testcase", [], + {}, + { 'test_id.testcase': { 'count': 1 } }, TwisterStatus.FAIL, False, TwisterStatus.FAIL, @@ -646,12 +667,14 @@ def test_get_harness(name): @pytest.mark.parametrize( - "exp_out, line, exp_suite_name, exp_status, ztest, state", + "exp_out, line, exp_suite_name, exp_started_suites, exp_started_cases, exp_status, ztest, state", TEST_DATA_7, ids=["testsuite", "testcase", "pass", "skip", "failed", "ztest pass", "ztest fail"], ) def test_test_handle( - tmp_path, caplog, exp_out, line, exp_suite_name, exp_status, ztest, state + tmp_path, caplog, exp_out, line, + exp_suite_name, exp_started_suites, exp_started_cases, + exp_status, ztest, state ): # Arrange line = line @@ -662,6 +685,7 @@ def test_test_handle( mock_testsuite = mock.Mock(id="id", testcases=[]) mock_testsuite.name = "mock_testsuite" mock_testsuite.harness_config = {} + mock_testsuite.ztest_suite_names = [] outdir = tmp_path / "gtest_out" outdir.mkdir() @@ -681,6 +705,9 @@ def test_test_handle( # Assert assert test_obj.detected_suite_names == exp_suite_name + assert test_obj.started_suites == exp_started_suites + assert test_obj.started_cases == exp_started_cases + assert exp_out in caplog.text if not "Running" in line and exp_out == "": assert test_obj.instance.testcases[0].status == exp_status diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index 3ab7de2fb935f..24e5e4fb77998 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -1562,11 +1562,14 @@ def mock_determine_testcases(res): TESTDATA_7 = [ ( [ - 'z_ztest_unit_test__dummy_suite_name__dummy_test_name', - 'z_ztest_unit_test__dummy_suite_name__test_dummy_name', + 'z_ztest_unit_test__dummy_suite1_name__dummy_test_name1', + 'z_ztest_unit_test__dummy_suite2_name__test_dummy_name2', 'no match' ], - ['dummy_id.dummy_name', 'dummy_id.dummy_name'] + [ + ('dummy_id.dummy_suite1_name.dummy_name1'), + ('dummy_id.dummy_suite2_name.dummy_name2') + ] ), ( ['no match'], @@ -1599,6 +1602,7 @@ def test_projectbuilder_determine_testcases( instance_mock = mock.Mock() instance_mock.testcases = [] instance_mock.testsuite.id = 'dummy_id' + instance_mock.testsuite.ztest_suite_names = [] env_mock = mock.Mock() pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) @@ -2137,13 +2141,11 @@ def test_projectbuilder_cmake(): instance_mock = mock.Mock() instance_mock.handler = 'dummy handler' instance_mock.build_dir = os.path.join('build', 'dir') - instance_mock.platform.name = 'frdm_k64f' env_mock = mock.Mock() pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) pb.build_dir = 'build_dir' - pb.testsuite.platform = instance_mock.platform - pb.testsuite.extra_args = ['some', 'platform:frdm_k64f:args'] + pb.testsuite.extra_args = ['some', 'args'] pb.testsuite.extra_conf_files = ['some', 'files1'] pb.testsuite.extra_overlay_confs = ['some', 'files2'] pb.testsuite.extra_dtc_overlay_files = ['some', 'files3'] @@ -2156,7 +2158,7 @@ def test_projectbuilder_cmake(): assert res == cmake_res_mock pb.cmake_assemble_args.assert_called_once_with( - ['some', 'args'], + pb.testsuite.extra_args, pb.instance.handler, pb.testsuite.extra_conf_files, pb.testsuite.extra_overlay_confs, diff --git a/scripts/tests/twister/test_testplan.py b/scripts/tests/twister/test_testplan.py index eea3d3e9abefb..b00d69ab06113 100644 --- a/scripts/tests/twister/test_testplan.py +++ b/scripts/tests/twister/test_testplan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2020-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 @@ -85,7 +85,8 @@ def test_get_all_testsuites_short(class_testplan, all_testsuites_dict): 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2', 'test_d.check_1.unit_1a', 'test_d.check_1.unit_1b', - 'test_e.check_1.1a', 'test_e.check_1.1b', + 'test_e.check_1.feature5.1a', + 'test_e.check_1.feature5.1b', 'test_config.main'] assert sorted(plan.get_all_tests()) == sorted(expected_tests) diff --git a/scripts/tests/twister/test_testsuite.py b/scripts/tests/twister/test_testsuite.py index e297b6b6d9cd6..8d20902a5ebaf 100644 --- a/scripts/tests/twister/test_testsuite.py +++ b/scripts/tests/twister/test_testsuite.py @@ -165,7 +165,7 @@ def test_scanpathresults_dunders(original, provided, expected): ), ScanPathResult( warnings=None, - matches=['1a', '1b'], + matches=['feature5.1a', 'feature5.1b'], has_registered_test_suites=False, has_run_registered_test_suites=True, has_test_main=False, diff --git a/scripts/tests/twister_blackbox/test_config.py b/scripts/tests/twister_blackbox/test_config.py index c05d18cdaa780..2cad497055f5b 100644 --- a/scripts/tests/twister_blackbox/test_config.py +++ b/scripts/tests/twister_blackbox/test_config.py @@ -13,6 +13,7 @@ import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -55,13 +56,13 @@ def test_alt_config_root(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 3 + assert len(filtered_j) == 4 @pytest.mark.parametrize( 'level, expected_tests', [ - ('smoke', 5), - ('acceptance', 6), + ('smoke', 6), + ('acceptance', 7), ], ids=['smoke', 'acceptance'] ) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c index 55c375965aeb9..798fd9756a8c1 100644 --- a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c @@ -9,6 +9,8 @@ ZTEST_SUITE(a2_tests, NULL, NULL, NULL, NULL, NULL); +ZTEST_SUITE(a3_tests, NULL, NULL, NULL, NULL, NULL); + /** * @brief Test Asserts * @@ -34,3 +36,8 @@ ZTEST(a2_tests, test_assert2) zassert_equal(1, 1, "1 was not equal to 1"); zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); } + +ZTEST(a3_tests, test_assert1) +{ + zassert_true(1, "1 was false"); +} diff --git a/scripts/tests/twister_blackbox/test_filter.py b/scripts/tests/twister_blackbox/test_filter.py index 90ea95e643075..d8dfd3575e407 100644 --- a/scripts/tests/twister_blackbox/test_filter.py +++ b/scripts/tests/twister_blackbox/test_filter.py @@ -14,6 +14,7 @@ import json import re +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -83,11 +84,12 @@ def teardown_class(cls): @pytest.mark.parametrize( 'tag, expected_test_count', [ - ('device', 5), # dummy.agnostic.group1.subgroup1.assert - # dummy.agnostic.group1.subgroup2.assert - # dummy.agnostic.group2.assert1 - # dummy.agnostic.group2.assert2 - # dummy.agnostic.group2.assert3 + ('device', 6), # dummy.agnostic.group1.subgroup1.a1_1_tests.assert + # dummy.agnostic.group1.subgroup2.a2_2_tests.assert + # dummy.agnostic.group2.a2_tests.assert1 + # dummy.agnostic.group2.a2_tests.assert2 + # dummy.agnostic.group2.a2_tests.assert3 + # dummy.agnostic.group2.a3_tests.assert1 ('agnostic', 1) # dummy.device.group.assert ], ids=['no device', 'no agnostic'] @@ -144,7 +146,7 @@ def test_enable_slow(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 5 + assert len(filtered_j) == 6 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_enable_slow_only(self, out_path): @@ -172,7 +174,7 @@ def test_enable_slow_only(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 3 + assert len(filtered_j) == 4 @pytest.mark.parametrize( 'arch, expected', diff --git a/scripts/tests/twister_blackbox/test_platform.py b/scripts/tests/twister_blackbox/test_platform.py index 43ff3346cf8af..61c9f4103b1cb 100644 --- a/scripts/tests/twister_blackbox/test_platform.py +++ b/scripts/tests/twister_blackbox/test_platform.py @@ -36,7 +36,7 @@ class TestPlatform: 'built_configurations': 2, 'failed_configurations': 0, 'errored_configurations': 0, - 'executed_test_cases': 8, + 'executed_test_cases': 10, 'skipped_test_cases': 2, 'platform_count': 3, 'executed_on_platform': 4, @@ -129,7 +129,7 @@ def test_force_platform(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 12 + assert len(filtered_j) == 14 def test_platform(self, out_path): path = os.path.join(TEST_DATA, 'tests', 'dummy') diff --git a/scripts/tests/twister_blackbox/test_printouts.py b/scripts/tests/twister_blackbox/test_printouts.py index 3f65549b8ea9c..853797354f44c 100644 --- a/scripts/tests/twister_blackbox/test_printouts.py +++ b/scripts/tests/twister_blackbox/test_printouts.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 """ @@ -41,17 +41,18 @@ class TestPrintOuts: ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), [ - 'dummy.agnostic.group1.subgroup1.assert', - 'dummy.agnostic.group1.subgroup2.assert', - 'dummy.agnostic.group2.assert1', - 'dummy.agnostic.group2.assert2', - 'dummy.agnostic.group2.assert3' + 'dummy.agnostic.group1.subgroup1.a1_1_tests.assert', + 'dummy.agnostic.group1.subgroup2.a1_2_tests.assert', + 'dummy.agnostic.group2.a2_tests.assert1', + 'dummy.agnostic.group2.a2_tests.assert2', + 'dummy.agnostic.group2.a3_tests.assert1', + 'dummy.agnostic.group2.a2_tests.assert3' ] ), ( os.path.join(TEST_DATA, 'tests', 'dummy', 'device'), [ - 'dummy.device.group.assert' + 'dummy.device.group.d_tests.assert' ] ), ] @@ -64,11 +65,12 @@ class TestPrintOuts: '└── Tests\n' \ ' └── dummy\n' \ ' └── agnostic\n' \ - ' ├── dummy.agnostic.group1.subgroup1.assert\n' \ - ' ├── dummy.agnostic.group1.subgroup2.assert\n' \ - ' ├── dummy.agnostic.group2.assert1\n' \ - ' ├── dummy.agnostic.group2.assert2\n' \ - ' └── dummy.agnostic.group2.assert3\n' + ' ├── dummy.agnostic.group1.subgroup1.a1_1_tests.assert\n' \ + ' ├── dummy.agnostic.group1.subgroup2.a1_2_tests.assert\n' \ + ' ├── dummy.agnostic.group2.a2_tests.assert1\n' \ + ' ├── dummy.agnostic.group2.a2_tests.assert2\n' \ + ' ├── dummy.agnostic.group2.a2_tests.assert3\n' \ + ' └── dummy.agnostic.group2.a3_tests.assert1\n' ), ( os.path.join(TEST_DATA, 'tests', 'dummy', 'device'), @@ -77,7 +79,7 @@ class TestPrintOuts: '└── Tests\n' ' └── dummy\n' ' └── device\n' - ' └── dummy.device.group.assert\n' + ' └── dummy.device.group.d_tests.assert\n' ), ] diff --git a/scripts/tests/twister_blackbox/test_report.py b/scripts/tests/twister_blackbox/test_report.py index 2db1006bc5aa9..3a145fd59b37c 100644 --- a/scripts/tests/twister_blackbox/test_report.py +++ b/scripts/tests/twister_blackbox/test_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 """ @@ -350,12 +350,12 @@ def test_log_file(self, capfd, test_path, test_platforms, out_path, file_name): ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['--detailed-skipped-report'], - {'qemu_x86/atom': 5, 'intel_adl_crb/alder_lake': 1} + {'qemu_x86/atom': 6, 'intel_adl_crb/alder_lake': 1} ), ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['--detailed-skipped-report', '--report-filtered'], - {'qemu_x86/atom': 6, 'intel_adl_crb/alder_lake': 6} + {'qemu_x86/atom': 7, 'intel_adl_crb/alder_lake': 7} ), ], ids=['dummy tests', 'dummy tests with filtered'] diff --git a/scripts/tests/twister_blackbox/test_runner.py b/scripts/tests/twister_blackbox/test_runner.py index 1ab50522c5ebf..16240a04f915d 100644 --- a/scripts/tests/twister_blackbox/test_runner.py +++ b/scripts/tests/twister_blackbox/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 """ @@ -54,7 +54,7 @@ class TestRunner: 'built_configurations': 0, 'failed_configurations': 0, 'errored_configurations': 0, - 'executed_test_cases': 8, + 'executed_test_cases': 10, 'skipped_test_cases': 0, 'platform_count': 0, 'executed_on_platform': 4, diff --git a/scripts/tests/twister_blackbox/test_shuffle.py b/scripts/tests/twister_blackbox/test_shuffle.py index ade1267b48227..412d97a619bf9 100644 --- a/scripts/tests/twister_blackbox/test_shuffle.py +++ b/scripts/tests/twister_blackbox/test_shuffle.py @@ -10,10 +10,10 @@ import mock import os import pytest -import re import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -65,14 +65,8 @@ def test_shuffle_tests(self, out_path, seed, ratio, expected_order): with open(os.path.join(out_path, 'testplan.json')) as f: j = json.load(f) - filtered_j = [ - (ts['platform'], ts['name'], tc['identifier']) \ - for ts in j['testsuites'] \ - for tc in ts['testcases'] if 'reason' not in tc - ] - testcases = [re.sub(r'\.assert[^\.]*?$', '', j[2]) for j in filtered_j] - testsuites = list(dict.fromkeys(testcases)) + testsuites = [os.path.basename(ts['name']) for ts in j['testsuites']] assert testsuites == expected_order diff --git a/scripts/tests/twister_blackbox/test_testlist.py b/scripts/tests/twister_blackbox/test_testlist.py index 1ef93f072b6b5..ad8eaeddfaf55 100644 --- a/scripts/tests/twister_blackbox/test_testlist.py +++ b/scripts/tests/twister_blackbox/test_testlist.py @@ -13,6 +13,7 @@ import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test from twisterlib.testplan import TestPlan @@ -71,4 +72,4 @@ def test_save_tests(self, out_path): for tc in ts['testcases'] if 'reason' not in tc ] - assert len(filtered_j) == 5 + assert len(filtered_j) == 6 diff --git a/scripts/tests/twister_blackbox/test_testplan.py b/scripts/tests/twister_blackbox/test_testplan.py index 915653a33e15e..8834e03ead039 100644 --- a/scripts/tests/twister_blackbox/test_testplan.py +++ b/scripts/tests/twister_blackbox/test_testplan.py @@ -13,6 +13,7 @@ import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan from twisterlib.error import TwisterRuntimeError @@ -20,7 +21,7 @@ class TestTestPlan: TESTDATA_1 = [ - ('dummy.agnostic.group2.assert1', SystemExit, 3), + ('dummy.agnostic.group2.a2_tests.assert1', SystemExit, 4), ( os.path.join('scripts', 'tests', 'twister_blackbox', 'test_data', 'tests', 'dummy', 'agnostic', 'group1', 'subgroup1', @@ -30,12 +31,12 @@ class TestTestPlan: ), ] TESTDATA_2 = [ - ('buildable', 6), - ('runnable', 4), + ('buildable', 7), + ('runnable', 5), ] TESTDATA_3 = [ (True, 1), - (False, 6), + (False, 7), ] @classmethod @@ -52,7 +53,7 @@ def teardown_class(cls): @pytest.mark.parametrize( 'test, expected_exception, expected_subtest_count', TESTDATA_1, - ids=['valid', 'invalid'] + ids=['valid', 'not found'] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_subtest(self, out_path, test, expected_exception, expected_subtest_count): From 42a3b95e2126d9ef8afc701ede502e3f3479abb5 Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Sat, 19 Oct 2024 18:51:29 +0200 Subject: [PATCH 2/3] doc: twister: Update test naming and application diagram Update Test Application diagram as well as Test Scenario and Test Case naming conventions to make them more clear and aligned to Ztest suite name included as a part of Test Case name. Signed-off-by: Dmitrii Golovanov --- .../test/figures/twister_test_project.svg | 2 +- doc/develop/test/twister.rst | 65 ++++++++++--------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/doc/develop/test/figures/twister_test_project.svg b/doc/develop/test/figures/twister_test_project.svg index f21a45c2971ae..75440d6bda601 100644 --- a/doc/develop/test/figures/twister_test_project.svg +++ b/doc/develop/test/figures/twister_test_project.svg @@ -1,4 +1,4 @@ -
Test Framework (ZTEST)
Test Framework (ZTEST)
Test Suite (ZTEST_SUITE)
Test Suite (ZTEST_SUITE)
+ suite_name = "foo_bar_feature_aspect"
+ suite_name = "foo_bar_feature_aspect"
Test (ZTEST*)
Test (ZTEST*)
+ test_name = "test_buzz_and_blink"
+ test_name = "test_buzz_and_blink"
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
tests:
tests:
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:

+ harness:
+ harness_config:

+ tags:...
CMakeList.txt
CMakeList.txt
+ project(foo-bar-feature)

+ project(foo-bar-feature)
./src (Test Project Implementation)
./src (Test Project Implementation)
prj.conf
prj.conf
+ Kconfig:
+ Kconfig:
Test Application binary
Test Application binary
Twister
Twister
Test Instance
Test Instance
+ outdir
+ outdir
+ testsuite:
+ testsuite:
+ platform:
+ platform:
+ testcases: []
+ testcases: []
+ status:
+ status:
+ reason:
+ reason:
+ execution_time:
+ execution_time:
+ retries:
+ retries:
Test Suite (test specification)
Test Suite (test specification)
+ yamlfile
+ yamlfile
+ id
+ id
+ testcases: []
+ testcases: []
+ ztest_suite_names: []
+ ztest_suite_names: []
+ name
+ name
Test Case
Test Case
+ testsuite:
+ testsuite:
+ status:
+ status:
+ reason:
+ reason:
+ output:
+ output:
+ duration:
+ duration:
+ name:
+ name:
Legend:
Legend:
Test Instance (Test Application) 
Test Instance (Test Application) 
Execution time-specific Test properties
Execution time-specific Test properties
Elementary Test
Elementary Test
tests/foo/bar/feature/name_section.name_subsection
tests/foo/bar/feature/name_section.name_subsection
name_section.name_subsection.buzz_and_blink
name_section.name_subsection.buzz_and_blink
Zephyr Test Application Project (tests/foo/bar/feature)
Zephyr Test Application Project (tests/foo/bar/feature)
Test Configuration (testcase.yaml)
Test Configuration (testcase.yaml)
build & run
build & run
Twister parameters:

--arch
--platform
--level

... etc. 
Twister parameters...
select
select
ELF symbols and application log parsing
ELF symbols and application log parsingText is not SVG - cannot display \ No newline at end of file +
Zephyr Test Framework (Ztest)
Zephyr Test Framework (Ztest)
Ztest Test Suite (ZTEST_SUITE)
Ztest Test Suite (ZTEST_SUITE)
+ suite_name = "foo_bar_aspect"
+ suite_name = "foo_bar_aspect"
Ztest Test (ZTEST*)
Ztest Test (ZTEST*)
+ test_name = "test_buzz_and_blink"
+ test_name = "test_buzz_and_blink"
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
tests:
tests:
Test Scenario
(section_name.subsection_name)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:

+ harness:
+ harness_config:

+ tags:...
CMakeList.txt
CMakeList.txt
+ project(foo-bar-feature)

+ project(foo-bar-feature)
./src (Test Project Implementation)
./src (Test Project Implementation)
prj.conf
prj.conf
+ Kconfig:
+ Kconfig:
Test Application binary
Test Application binary
Twister
Twister
Test Instance
Test Instance
+ outdir
+ outdir
+ testsuite:
+ testsuite:
+ platform:
+ platform:
+ testcases: []
+ testcases: []
+ status:
+ status:
+ reason:
+ reason:
+ execution_time:
+ execution_time:
+ retries:
+ retries:
Test Suite (test specification)
Test Suite (test specification)
+ yamlfile
+ yamlfile
+ testcases: []
+ testcases: []
+ id
+ id
+ name
+ name
+ ztest_suite_names: []
+ ztest_suite_names: []
Test Case
Test Case
+ testsuite:
+ testsuite:
+ status:
+ status:
+ reason:
+ reason:
+ output:
+ output:
+ duration:
+ duration:
+ name:
+ name:
Legend:
Legend:
Test Instance (Test Application) 
Test Instance (Test Application) 
Execution time-specific Test properties
Execution time-specific Test properties
Elementary Test
Elementary Test
tests/foo/bar/feature/section_name.subsection_name
tests/foo/bar/feature/section_name.subsection_name
section_name.subsection_name.foo_bar_aspect.buzz_and_blink
section_name.subsection_name.foo_bar_aspect.buzz_and_blink
Zephyr Test Application Project (tests/foo/bar/feature)
Zephyr Test Application Project (tests/foo/bar/feature)
Test Configuration (testcase.yaml)
Test Configuration (testcase.yaml)
build & run
build & run
Twister parameters:

--arch
--platform
--level

... etc. 
Twister parameters...
select
select
ELF symbols and application log parsing
ELF symbols and application log parsing
section_name.subsection_name
section_name.subsection_nameText is not SVG - cannot display \ No newline at end of file diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index f535648dd6e91..1ec1f87369f43 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -219,57 +219,60 @@ Tests Tests are detected by the presence of a ``testcase.yaml`` or a ``sample.yaml`` files in the application's project directory. This test application -configuration file may contain one or more entries in the tests section each -identifying a test scenario. +configuration file may contain one or more entries in the ``tests:`` section each +identifying a Test Scenario. .. _twister_test_project_diagram: .. figure:: figures/twister_test_project.svg - :alt: Twister and a Test applications' project. + :alt: Twister and a Test application project. :figclass: align-center - Twister and a Test applications' project. + Twister and a Test application project. Test application configurations are written using the YAML syntax and share the same structure as samples. -A test scenario is a set of conditions or variables, defined in test scenario -entry, under which a set of test suites will be executed. Can be used -interchangeably with test scenario entry. +A Test Scenario is a set of conditions and variables defined in a Test Scenario +entry, under which a set of Test Suites will be built and executed. -A test suite is a collection of test cases that are intended to be used to test -a software program to ensure it meets certain requirements. The test cases in a -test suite are often related or meant to be executed together. +A Test Suite is a collection of Test Cases which are intended to be used to test +a software program to ensure it meets certain requirements. The Test Cases in a +Test Suite are either related or meant to be executed together. -The name of each test scenario needs to be unique in the context of the overall +The name of each Test Scenario needs to be unique in the context of the overall test application and has to follow basic rules: -#. The format of the test scenario identifier shall be a string without any spaces or +#. The format of the Test Scenario identifier shall be a string without any spaces or special characters (allowed characters: alphanumeric and [\_=]) consisting - of multiple sections delimited with a dot (.). + of multiple sections delimited with a dot (``.``). -#. Each test scenario identifier shall start with a section followed by a - subsection separated by a dot. For example, a test scenario that covers - semaphores in the kernel shall start with ``kernel.semaphore``. +#. Each Test Scenario identifier shall start with a section name followed by a + subsection names delimited with a dot (``.``). For example, a test scenario + that covers semaphores in the kernel shall start with ``kernel.semaphore``. -#. All test scenario identifiers within a ``testcase.yaml`` file need to be unique. For - example a ``testcase.yaml`` file covering semaphores in the kernel can have: +#. All Test Scenario identifiers within a ``testcase.yaml`` file need to be unique. + For example a ``testcase.yaml`` file covering semaphores in the kernel can have: * ``kernel.semaphore``: For general semaphore tests * ``kernel.semaphore.stress``: Stress testing semaphores in the kernel. -#. Depending on the nature of the test, an identifier can consist of at least - two sections: +#. The full canonical name of a Test Suite is: + ``/`` - * Ztest tests: The individual test cases in the ztest testsuite will be - concatenated by dot (``.``) to the identifier in the ``testcase.yaml`` file - generating unique identifiers for every test case in the suite. +#. Depending on the Test Suite implementation, its Test Case identifiers consist + of **at least three sections** delimited with a dot (``.``): - * Standalone tests and samples: This type of test should at least have 3 - sections concatnated by dot (``.``) in the test scenario identifier in the - ``testcase.yaml`` (or ``sample.yaml``) file. - The last section of the name shall signify the test case itself. + * **Ztest tests**: + a Test Scenario identifier from the corresponding ``testcase.yaml`` file, + a Ztest suite name, and a Ztest test name: + ``..`` + + * **Standalone tests and samples**: + a Test Scenario identifier from the corresponding ``testcase.yaml`` (or + ``sample.yaml``) file where the last section signifies the standalone + Test Case name, for example: ``debug.coredump.logging_backend``. The following is an example test configuration with a few options that are @@ -312,12 +315,10 @@ related to the sample and what is being demonstrated: tags: tests min_ram: 16 -The full canonical name for each test scenario is:``/`` - -A test scenario entry is a a block or entry starting with test scenario -identifier in the YAML files. +A Test Scenario entry in the ``tests:`` YAML dictionary has its Test Scenario +identifier as a key. -Each test scenario entry in the test application configuration can define the +Each Test Scenario entry in the Test Application configuration can define the following key/value pairs: .. _test_config_args: From 0a563af1d0c0fe126166912eebca615867a16c35 Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Fri, 1 Nov 2024 12:07:02 +0100 Subject: [PATCH 3/3] twister: fix Ztest C++ test names extraction from ELF Fix Ztest test function name extraction from ELF symbols for C++ compiled binaries where symbol names need additional 'demangling' to match with corresponding test names. The `c++filt` utility (part of binutils) is called for demangling when it is needed. Twister test suite extension and adjustment. Signed-off-by: Dmitrii Golovanov --- scripts/pylib/twister/twisterlib/runner.py | 41 +++++++++++----- scripts/tests/twister/test_runner.py | 25 ++++++++-- .../test_data/test_config.yaml | 5 +- .../group1/subgroup1/CMakeLists.txt | 8 ++++ .../agnostic_cpp/group1/subgroup1/prj.conf | 1 + .../group1/subgroup1/src/main.cpp | 28 +++++++++++ .../group1/subgroup1/test_data.yaml | 10 ++++ .../group1/subgroup2/CMakeLists.txt | 8 ++++ .../agnostic_cpp/group1/subgroup2/prj.conf | 1 + .../group1/subgroup2/src/main.cpp | 27 +++++++++++ .../group1/subgroup2/test_data.yaml | 11 +++++ .../dummy/agnostic_cpp/group2/CMakeLists.txt | 8 ++++ .../tests/dummy/agnostic_cpp/group2/prj.conf | 1 + .../dummy/agnostic_cpp/group2/src/main.cpp | 48 +++++++++++++++++++ .../dummy/agnostic_cpp/group2/src/submain.cpp | 28 +++++++++++ .../dummy/agnostic_cpp/group2/test_data.yaml | 9 ++++ scripts/tests/twister_blackbox/test_filter.py | 15 +++--- .../tests/twister_blackbox/test_platform.py | 24 +++++++++- scripts/tests/twister_blackbox/test_report.py | 4 +- scripts/tests/twister_blackbox/test_runner.py | 2 +- .../tests/twister_blackbox/test_shuffle.py | 19 ++++---- 21 files changed, 288 insertions(+), 35 deletions(-) create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 83ca94f9fb5f2..172add5367edf 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -814,6 +814,10 @@ def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs) self.env = env self.duts = None + @property + def trace(self) -> bool: + return self.options.verbose > 2 + def log_info(self, filename, inline_logs, log_testcases=False): filename = os.path.abspath(os.path.realpath(filename)) if inline_logs: @@ -1087,6 +1091,18 @@ def process(self, pipeline, done, message, lock, results): self.instance.reason = reason self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason) + def demangle(self, symbol_name): + if symbol_name[:2] == '_Z': + try: + cpp_filt = subprocess.run('c++filt', input=symbol_name, text=True, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if self.trace: + logger.debug(f"Demangle: '{symbol_name}'==>'{cpp_filt.stdout}'") + return cpp_filt.stdout.strip() + except Exception as e: + logger.error(f"Failed to demangle '{symbol_name}': {e}") + return symbol_name + def determine_testcases(self, results): yaml_testsuite_name = self.instance.testsuite.id logger.debug(f"Determine test cases for test suite: {yaml_testsuite_name}") @@ -1102,19 +1118,22 @@ def determine_testcases(self, results): for sym in section.iter_symbols(): # It is only meant for new ztest fx because only new ztest fx exposes test functions # precisely. - + m_ = new_ztest_unit_test_regex.search(sym.name) + if not m_: + continue + # Demangle C++ symbols + m_ = new_ztest_unit_test_regex.search(self.demangle(sym.name)) + if not m_: + continue # The 1st capture group is new ztest suite name. # The 2nd capture group is new ztest unit test name. - matches = new_ztest_unit_test_regex.findall(sym.name) - if matches: - for m in matches: - new_ztest_suite = m[0] - if new_ztest_suite not in self.instance.testsuite.ztest_suite_names: - logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' " - f"not present in: {self.instance.testsuite.ztest_suite_names}") - test_func_name = m[1].replace("test_", "", 1) - testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}" - detected_cases.append(testcase_id) + new_ztest_suite = m_[1] + if new_ztest_suite not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' " + f"not present in: {self.instance.testsuite.ztest_suite_names}") + test_func_name = m_[2].replace("test_", "", 1) + testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}" + detected_cases.append(testcase_id) if detected_cases: logger.debug(f"Detected Ztest cases: [{', '.join(detected_cases)}] in {elf_file}") diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index 24e5e4fb77998..29459d585498d 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -56,6 +56,7 @@ def mocked_instance(tmp_path): def mocked_env(): env = mock.Mock() options = mock.Mock() + options.verbose = 2 env.options = options return env @@ -1571,6 +1572,24 @@ def mock_determine_testcases(res): ('dummy_id.dummy_suite2_name.dummy_name2') ] ), + ( + [ + 'z_ztest_unit_test__dummy_suite2_name__test_dummy_name2', + 'z_ztest_unit_test__bad_suite3_name_no_test', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name4E', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_bad_name1E', + '_ZN12_GLOBAL__N_1L51z_ztest_unit_test_dummy_suite3_name__test_bad_name2E', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name5E', + '_ZN15foobarnamespaceL54z_ztest_unit_test__dummy_suite3_name__test_dummy_name6E', + ], + [ + ('dummy_id.dummy_suite2_name.dummy_name2'), + ('dummy_id.dummy_suite3_name.dummy_name4'), + ('dummy_id.dummy_suite3_name.bad_name1E'), + ('dummy_id.dummy_suite3_name.dummy_name5'), + ('dummy_id.dummy_suite3_name.dummy_name6'), + ] + ), ( ['no match'], [] @@ -1580,10 +1599,11 @@ def mock_determine_testcases(res): @pytest.mark.parametrize( 'symbols_names, added_tcs', TESTDATA_7, - ids=['two hits, one miss', 'nothing'] + ids=['two hits, one miss', 'demangle', 'nothing'] ) def test_projectbuilder_determine_testcases( mocked_jobserver, + mocked_env, symbols_names, added_tcs ): @@ -1603,9 +1623,8 @@ def test_projectbuilder_determine_testcases( instance_mock.testcases = [] instance_mock.testsuite.id = 'dummy_id' instance_mock.testsuite.ztest_suite_names = [] - env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, mocked_env, mocked_jobserver) with mock.patch('twisterlib.runner.ELFFile', elf_mock), \ mock.patch('builtins.open', mock.mock_open()): diff --git a/scripts/tests/twister_blackbox/test_data/test_config.yaml b/scripts/tests/twister_blackbox/test_data/test_config.yaml index d7e4828350cf9..6bfe24a4e819e 100644 --- a/scripts/tests/twister_blackbox/test_data/test_config.yaml +++ b/scripts/tests/twister_blackbox/test_data/test_config.yaml @@ -6,9 +6,10 @@ levels: description: > A plan to be used verifying basic features adds: - - dummy.agnostic.* + - dummy.agnostic\..* - name: acceptance description: > More coverage adds: - - dummy.* + - dummy.agnostic\..* + - dummy.device\..* diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt new file mode 100644 index 0000000000000..3ffe630f1ad98 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf new file mode 100644 index 0000000000000..9467c2926896d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp new file mode 100644 index 0000000000000..b37d02646cfcb --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + + +// global namespace + +ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a1_1_tests, test_assert) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml new file mode 100644 index 0000000000000..7d8a642054144 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml @@ -0,0 +1,10 @@ +tests: + dummy.agnostic_cpp.group1.subgroup1: + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp + - subgrouped diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt new file mode 100644 index 0000000000000..3ffe630f1ad98 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf new file mode 100644 index 0000000000000..9467c2926896d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp new file mode 100644 index 0000000000000..0ac835a959c04 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +// global namespace + +ZTEST_SUITE(a1_2_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a1_2_tests, test_assert) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml new file mode 100644 index 0000000000000..f0146d3e19344 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml @@ -0,0 +1,11 @@ +tests: + dummy.agnostic_cpp.group1.subgroup2: + build_only: true + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp + - subgrouped diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt new file mode 100644 index 0000000000000..3ffe630f1ad98 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf new file mode 100644 index 0000000000000..9467c2926896d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp new file mode 100644 index 0000000000000..1c86457e76a0d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + + +namespace +{ + +ZTEST_SUITE(a2_tests, NULL, NULL, NULL, NULL, NULL); + +ZTEST_SUITE(a3_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a2_tests, test_assert1) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +ZTEST(a2_tests, test_assert2) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +ZTEST(a3_tests, test_assert1) +{ + zassert_true(1, "1 was false"); +} + +} // namsespace diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp new file mode 100644 index 0000000000000..bf9c90536b2d2 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +namespace foo_namespace +{ + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a2_tests, test_assert3) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +} // foo_namespace diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml new file mode 100644 index 0000000000000..1a93b591460f6 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml @@ -0,0 +1,9 @@ +tests: + dummy.agnostic_cpp.group2: + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp diff --git a/scripts/tests/twister_blackbox/test_filter.py b/scripts/tests/twister_blackbox/test_filter.py index d8dfd3575e407..3558296d40b11 100644 --- a/scripts/tests/twister_blackbox/test_filter.py +++ b/scripts/tests/twister_blackbox/test_filter.py @@ -82,24 +82,27 @@ def teardown_class(cls): pass @pytest.mark.parametrize( - 'tag, expected_test_count', + 'tags, expected_test_count', [ - ('device', 6), # dummy.agnostic.group1.subgroup1.a1_1_tests.assert + (['device', 'cpp'], 6), + # dummy.agnostic.group1.subgroup1.a1_1_tests.assert # dummy.agnostic.group1.subgroup2.a2_2_tests.assert # dummy.agnostic.group2.a2_tests.assert1 # dummy.agnostic.group2.a2_tests.assert2 # dummy.agnostic.group2.a2_tests.assert3 # dummy.agnostic.group2.a3_tests.assert1 - ('agnostic', 1) # dummy.device.group.assert + (['agnostic'], 1) # dummy.device.group.assert ], - ids=['no device', 'no agnostic'] + ids=['no device, no cpp', 'no agnostic'] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) - def test_exclude_tag(self, out_path, tag, expected_test_count): + def test_exclude_tag(self, out_path, tags, expected_test_count): test_platforms = ['qemu_x86', 'intel_adl_crb'] path = os.path.join(TEST_DATA, 'tests', 'dummy') args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \ - ['--exclude-tag', tag] + \ + [val for pair in zip( + ['--exclude-tag'] * len(tags), tags + ) for val in pair] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms ) for val in pair] diff --git a/scripts/tests/twister_blackbox/test_platform.py b/scripts/tests/twister_blackbox/test_platform.py index 61c9f4103b1cb..f76dab4738a53 100644 --- a/scripts/tests/twister_blackbox/test_platform.py +++ b/scripts/tests/twister_blackbox/test_platform.py @@ -64,6 +64,27 @@ class TestPlatform: 'only_built': 0 } ), + ( + os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic_cpp'), + ['native_sim'], + { + 'selected_test_scenarios': 3, + 'selected_test_instances': 3, + 'executed_test_instances': 3, + 'skipped_configurations': 0, + 'skipped_by_static_filter': 0, + 'skipped_at_runtime': 0, + 'passed_configurations': 2, + 'built_configurations': 1, + 'failed_configurations': 0, + 'errored_configurations': 0, + 'executed_test_cases': 5, + 'skipped_test_cases': 0, + 'platform_count': 1, + 'executed_on_platform': 2, + 'only_built': 1 + } + ), ] @classmethod @@ -129,7 +150,7 @@ def test_force_platform(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 14 + assert len(filtered_j) == 26 def test_platform(self, out_path): path = os.path.join(TEST_DATA, 'tests', 'dummy') @@ -250,6 +271,7 @@ def test_exclude_platform(self, capfd, out_path, test_path, test_platforms, expe ids=[ 'emulation_only tests/dummy/agnostic', 'emulation_only tests/dummy/device', + 'native_sim_only tests/dummy/agnostic_cpp', ] ) def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expected): diff --git a/scripts/tests/twister_blackbox/test_report.py b/scripts/tests/twister_blackbox/test_report.py index 3a145fd59b37c..bac6ddbbbfa0b 100644 --- a/scripts/tests/twister_blackbox/test_report.py +++ b/scripts/tests/twister_blackbox/test_report.py @@ -355,7 +355,7 @@ def test_log_file(self, capfd, test_path, test_platforms, out_path, file_name): ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['--detailed-skipped-report', '--report-filtered'], - {'qemu_x86/atom': 7, 'intel_adl_crb/alder_lake': 7} + {'qemu_x86/atom': 13, 'intel_adl_crb/alder_lake': 13} ), ], ids=['dummy tests', 'dummy tests with filtered'] @@ -392,7 +392,7 @@ def test_detailed_skipped_report(self, out_path, test_path, flags, expected_test 'test_path, report_filtered, expected_filtered_count', [ (os.path.join(TEST_DATA, 'tests', 'dummy'), False, 0), - (os.path.join(TEST_DATA, 'tests', 'dummy'), True, 4), + (os.path.join(TEST_DATA, 'tests', 'dummy'), True, 10), ], ids=['no filtered', 'with filtered'] ) diff --git a/scripts/tests/twister_blackbox/test_runner.py b/scripts/tests/twister_blackbox/test_runner.py index 16240a04f915d..e85f8abdf0445 100644 --- a/scripts/tests/twister_blackbox/test_runner.py +++ b/scripts/tests/twister_blackbox/test_runner.py @@ -591,7 +591,7 @@ def test_tag(self, capfd, out_path, test_path, test_platforms, tags, expected): sys.stderr.write(err) for line in expected: - assert re.search(line, err) + assert re.search(line, err), f"no expected:'{line}' in '{err}'" assert str(sys_exit.value) == '0' diff --git a/scripts/tests/twister_blackbox/test_shuffle.py b/scripts/tests/twister_blackbox/test_shuffle.py index 412d97a619bf9..f45724b7b55e7 100644 --- a/scripts/tests/twister_blackbox/test_shuffle.py +++ b/scripts/tests/twister_blackbox/test_shuffle.py @@ -33,20 +33,21 @@ def teardown_class(cls): @pytest.mark.parametrize( 'seed, ratio, expected_order', [ - ('123', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']), - ('123', '2/2', ['dummy.agnostic.group2', 'dummy.device.group']), - ('321', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']), - ('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), - ('123', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']), + ('123', '1/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), + ('123', '2/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup1']), + ('321', '1/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']), + ('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup1']), + ('123', '1/3', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), ('123', '2/3', ['dummy.agnostic.group2']), - ('123', '3/3', ['dummy.device.group']), - ('321', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']), + ('123', '3/3', ['dummy.agnostic.group1.subgroup1']), + ('321', '1/3', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']), ('321', '2/3', ['dummy.device.group']), - ('321', '3/3', ['dummy.agnostic.group1.subgroup2']) + ('321', '3/3', ['dummy.agnostic.group1.subgroup1']) ], ids=['first half, 123', 'second half, 123', 'first half, 321', 'second half, 321', 'first third, 123', 'middle third, 123', 'last third, 123', - 'first third, 321', 'middle third, 321', 'last third, 321'] + 'first third, 321', 'middle third, 321', 'last third, 321' +] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_shuffle_tests(self, out_path, seed, ratio, expected_order):