diff --git a/doc/develop/test/figures/twister_test_project.svg b/doc/develop/test/figures/twister_test_project.svg index f21a45c2971ae..75440d6bda601 100644 --- a/doc/develop/test/figures/twister_test_project.svg +++ b/doc/develop/test/figures/twister_test_project.svg @@ -1,4 +1,4 @@ -
Test Framework (ZTEST)
Test Framework (ZTEST)
Test Suite (ZTEST_SUITE)
Test Suite (ZTEST_SUITE)
+ suite_name = "foo_bar_feature_aspect"
+ suite_name = "foo_bar_feature_aspect"
Test (ZTEST*)
Test (ZTEST*)
+ test_name = "test_buzz_and_blink"
+ test_name = "test_buzz_and_blink"
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
tests:
tests:
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:

+ harness:
+ harness_config:

+ tags:...
CMakeList.txt
CMakeList.txt
+ project(foo-bar-feature)

+ project(foo-bar-feature)
./src (Test Project Implementation)
./src (Test Project Implementation)
prj.conf
prj.conf
+ Kconfig:
+ Kconfig:
Test Application binary
Test Application binary
Twister
Twister
Test Instance
Test Instance
+ outdir
+ outdir
+ testsuite:
+ testsuite:
+ platform:
+ platform:
+ testcases: []
+ testcases: []
+ status:
+ status:
+ reason:
+ reason:
+ execution_time:
+ execution_time:
+ retries:
+ retries:
Test Suite (test specification)
Test Suite (test specification)
+ yamlfile
+ yamlfile
+ id
+ id
+ testcases: []
+ testcases: []
+ ztest_suite_names: []
+ ztest_suite_names: []
+ name
+ name
Test Case
Test Case
+ testsuite:
+ testsuite:
+ status:
+ status:
+ reason:
+ reason:
+ output:
+ output:
+ duration:
+ duration:
+ name:
+ name:
Legend:
Legend:
Test Instance (Test Application) 
Test Instance (Test Application) 
Execution time-specific Test properties
Execution time-specific Test properties
Elementary Test
Elementary Test
tests/foo/bar/feature/name_section.name_subsection
tests/foo/bar/feature/name_section.name_subsection
name_section.name_subsection.buzz_and_blink
name_section.name_subsection.buzz_and_blink
Zephyr Test Application Project (tests/foo/bar/feature)
Zephyr Test Application Project (tests/foo/bar/feature)
Test Configuration (testcase.yaml)
Test Configuration (testcase.yaml)
build & run
build & run
Twister parameters:

--arch
--platform
--level

... etc. 
Twister parameters...
select
select
ELF symbols and application log parsing
ELF symbols and application log parsingText is not SVG - cannot display \ No newline at end of file +
Zephyr Test Framework (Ztest)
Zephyr Test Framework (Ztest)
Ztest Test Suite (ZTEST_SUITE)
Ztest Test Suite (ZTEST_SUITE)
+ suite_name = "foo_bar_aspect"
+ suite_name = "foo_bar_aspect"
Ztest Test (ZTEST*)
Ztest Test (ZTEST*)
+ test_name = "test_buzz_and_blink"
+ test_name = "test_buzz_and_blink"
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
Test Scenario
(name_section.name_subsection)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:
+ harness:
+ harness_config:

+ tags:...
tests:
tests:
Test Scenario
(section_name.subsection_name)
Test Scenario...

+ tags:
+ levels: 
+ filter:
+ required_*:
+ arch_*:
+ platform_*: 
+ extra_*:

+ harness:
+ harness_config:

+ tags:...
CMakeList.txt
CMakeList.txt
+ project(foo-bar-feature)

+ project(foo-bar-feature)
./src (Test Project Implementation)
./src (Test Project Implementation)
prj.conf
prj.conf
+ Kconfig:
+ Kconfig:
Test Application binary
Test Application binary
Twister
Twister
Test Instance
Test Instance
+ outdir
+ outdir
+ testsuite:
+ testsuite:
+ platform:
+ platform:
+ testcases: []
+ testcases: []
+ status:
+ status:
+ reason:
+ reason:
+ execution_time:
+ execution_time:
+ retries:
+ retries:
Test Suite (test specification)
Test Suite (test specification)
+ yamlfile
+ yamlfile
+ testcases: []
+ testcases: []
+ id
+ id
+ name
+ name
+ ztest_suite_names: []
+ ztest_suite_names: []
Test Case
Test Case
+ testsuite:
+ testsuite:
+ status:
+ status:
+ reason:
+ reason:
+ output:
+ output:
+ duration:
+ duration:
+ name:
+ name:
Legend:
Legend:
Test Instance (Test Application) 
Test Instance (Test Application) 
Execution time-specific Test properties
Execution time-specific Test properties
Elementary Test
Elementary Test
tests/foo/bar/feature/section_name.subsection_name
tests/foo/bar/feature/section_name.subsection_name
section_name.subsection_name.foo_bar_aspect.buzz_and_blink
section_name.subsection_name.foo_bar_aspect.buzz_and_blink
Zephyr Test Application Project (tests/foo/bar/feature)
Zephyr Test Application Project (tests/foo/bar/feature)
Test Configuration (testcase.yaml)
Test Configuration (testcase.yaml)
build & run
build & run
Twister parameters:

--arch
--platform
--level

... etc. 
Twister parameters...
select
select
ELF symbols and application log parsing
ELF symbols and application log parsing
section_name.subsection_name
section_name.subsection_nameText is not SVG - cannot display \ No newline at end of file diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index f535648dd6e91..1ec1f87369f43 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -219,57 +219,60 @@ Tests Tests are detected by the presence of a ``testcase.yaml`` or a ``sample.yaml`` files in the application's project directory. This test application -configuration file may contain one or more entries in the tests section each -identifying a test scenario. +configuration file may contain one or more entries in the ``tests:`` section each +identifying a Test Scenario. .. _twister_test_project_diagram: .. figure:: figures/twister_test_project.svg - :alt: Twister and a Test applications' project. + :alt: Twister and a Test application project. :figclass: align-center - Twister and a Test applications' project. + Twister and a Test application project. Test application configurations are written using the YAML syntax and share the same structure as samples. -A test scenario is a set of conditions or variables, defined in test scenario -entry, under which a set of test suites will be executed. Can be used -interchangeably with test scenario entry. +A Test Scenario is a set of conditions and variables defined in a Test Scenario +entry, under which a set of Test Suites will be built and executed. -A test suite is a collection of test cases that are intended to be used to test -a software program to ensure it meets certain requirements. The test cases in a -test suite are often related or meant to be executed together. +A Test Suite is a collection of Test Cases which are intended to be used to test +a software program to ensure it meets certain requirements. The Test Cases in a +Test Suite are either related or meant to be executed together. -The name of each test scenario needs to be unique in the context of the overall +The name of each Test Scenario needs to be unique in the context of the overall test application and has to follow basic rules: -#. The format of the test scenario identifier shall be a string without any spaces or +#. The format of the Test Scenario identifier shall be a string without any spaces or special characters (allowed characters: alphanumeric and [\_=]) consisting - of multiple sections delimited with a dot (.). + of multiple sections delimited with a dot (``.``). -#. Each test scenario identifier shall start with a section followed by a - subsection separated by a dot. For example, a test scenario that covers - semaphores in the kernel shall start with ``kernel.semaphore``. +#. Each Test Scenario identifier shall start with a section name followed by a + subsection names delimited with a dot (``.``). For example, a test scenario + that covers semaphores in the kernel shall start with ``kernel.semaphore``. -#. All test scenario identifiers within a ``testcase.yaml`` file need to be unique. For - example a ``testcase.yaml`` file covering semaphores in the kernel can have: +#. All Test Scenario identifiers within a ``testcase.yaml`` file need to be unique. + For example a ``testcase.yaml`` file covering semaphores in the kernel can have: * ``kernel.semaphore``: For general semaphore tests * ``kernel.semaphore.stress``: Stress testing semaphores in the kernel. -#. Depending on the nature of the test, an identifier can consist of at least - two sections: +#. The full canonical name of a Test Suite is: + ``/`` - * Ztest tests: The individual test cases in the ztest testsuite will be - concatenated by dot (``.``) to the identifier in the ``testcase.yaml`` file - generating unique identifiers for every test case in the suite. +#. Depending on the Test Suite implementation, its Test Case identifiers consist + of **at least three sections** delimited with a dot (``.``): - * Standalone tests and samples: This type of test should at least have 3 - sections concatnated by dot (``.``) in the test scenario identifier in the - ``testcase.yaml`` (or ``sample.yaml``) file. - The last section of the name shall signify the test case itself. + * **Ztest tests**: + a Test Scenario identifier from the corresponding ``testcase.yaml`` file, + a Ztest suite name, and a Ztest test name: + ``..`` + + * **Standalone tests and samples**: + a Test Scenario identifier from the corresponding ``testcase.yaml`` (or + ``sample.yaml``) file where the last section signifies the standalone + Test Case name, for example: ``debug.coredump.logging_backend``. The following is an example test configuration with a few options that are @@ -312,12 +315,10 @@ related to the sample and what is being demonstrated: tags: tests min_ram: 16 -The full canonical name for each test scenario is:``/`` - -A test scenario entry is a a block or entry starting with test scenario -identifier in the YAML files. +A Test Scenario entry in the ``tests:`` YAML dictionary has its Test Scenario +identifier as a key. -Each test scenario entry in the test application configuration can define the +Each Test Scenario entry in the Test Application configuration can define the following key/value pairs: .. _test_config_args: diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index 2b330caf2ab33..9f41d24b541aa 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # vim: set syntax=python ts=4 : # -# Copyright (c) 2018 Intel Corporation +# Copyright (c) 2018-2024 Intel Corporation # Copyright 2022 NXP # Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved. # @@ -149,7 +149,8 @@ def add_parse_arguments(parser = None): test_plan_report_xor.add_argument("--list-tests", action="store_true", help="""List of all sub-test functions recursively found in all --testsuite-root arguments. Note different sub-tests can share - the same section name and come from different directories. + the same test scenario identifier (section.subsection) + and come from different directories. The output is flattened and reports --sub-test names only, not their directories. For instance net.socket.getaddrinfo_ok and net.socket.fd_set belong to different directories. @@ -239,17 +240,22 @@ def add_parse_arguments(parser = None): test_xor_subtest.add_argument( "-s", "--test", "--scenario", action="append", type = norm_path, - help="Run only the specified testsuite scenario. These are named by " - "") + help="""Run only the specified test suite scenario. These are named by + 'path/relative/to/Zephyr/base/section.subsection_in_testcase_yaml', + or just 'section.subsection' identifier. With '--testsuite-root' option + the scenario will be found faster. + """) test_xor_subtest.add_argument( "--sub-test", action="append", - help="""Recursively find sub-test functions and run the entire - test section where they were found, including all sibling test + help="""Recursively find sub-test functions (test cases) and run the entire + test scenario (section.subsection) where they were found, including all sibling test functions. Sub-tests are named by: - section.name.in.testcase.yaml.function_name_without_test_prefix - Example: In kernel.fifo.fifo_loop: 'kernel.fifo' is a section name - and 'fifo_loop' is a name of a function found in main.c without test prefix. + 'section.subsection_in_testcase_yaml.ztest_suite.ztest_without_test_prefix'. + Example_1: 'kernel.fifo.fifo_api_1cpu.fifo_loop' where 'kernel.fifo' is a test scenario + name (section.subsection) and 'fifo_api_1cpu.fifo_loop' is + a Ztest suite_name.test_name identificator. + Example_2: 'debug.coredump.logging_backend' is a standalone test scenario name. """) parser.add_argument( diff --git a/scripts/pylib/twister/twisterlib/harness.py b/scripts/pylib/twister/twisterlib/harness.py index 2629cdfb83faf..cd2655e927f93 100644 --- a/scripts/pylib/twister/twisterlib/harness.py +++ b/scripts/pylib/twister/twisterlib/harness.py @@ -31,7 +31,6 @@ _WINDOWS = platform.system() == 'Windows' -result_re = re.compile(r".*(PASS|FAIL|SKIP) - (test_)?(\S*) in (\d*[.,]?\d*) seconds") class Harness: GCOV_START = "GCOV_COVERAGE_DUMP_START" GCOV_END = "GCOV_COVERAGE_DUMP_END" @@ -59,12 +58,19 @@ def __init__(self): self.ztest = False self.detected_suite_names = [] self.run_id = None + self.started_suites = {} + self.started_cases = {} self.matched_run_id = False self.run_id_exists = False self.instance: TestInstance | None = None self.testcase_output = "" self._match = False + + @property + def trace(self) -> bool: + return self.instance.handler.options.verbose > 2 + @property def status(self) -> TwisterStatus: return self._status @@ -710,42 +716,124 @@ def _check_result(self, line): class Test(Harness): __test__ = False # for pytest to skip this class when collects tests - RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL" - RUN_FAILED = "PROJECT EXECUTION FAILED" - test_suite_start_pattern = r"Running TESTSUITE (?P.*)" - ZTEST_START_PATTERN = r"START - (test_)?([a-zA-Z0-9_-]+)" - def handle(self, line): - test_suite_match = re.search(self.test_suite_start_pattern, line) - if test_suite_match: - suite_name = test_suite_match.group("suite_name") + test_suite_start_pattern = re.compile(r"Running TESTSUITE (?P\S*)") + test_suite_end_pattern = re.compile(r"TESTSUITE (?P\S*)\s+(?Psucceeded|failed)") + test_case_start_pattern = re.compile(r"START - (test_)?([a-zA-Z0-9_-]+)") + test_case_end_pattern = re.compile(r".*(PASS|FAIL|SKIP) - (test_)?(\S*) in (\d*[.,]?\d*) seconds") + test_suite_summary_pattern = re.compile(r"SUITE (?P\S*) - .* \[(?P\S*)\]: .* duration = (\d*[.,]?\d*) seconds") + test_case_summary_pattern = re.compile(r" - (PASS|FAIL|SKIP) - \[([^\.]*).(test_)?(\S*)\] duration = (\d*[.,]?\d*) seconds") + + + def get_testcase(self, tc_name, phase, ts_name=None): + """ Search a Ztest case among detected in the test image binary + expecting the same test names as already known from the ELF. + Track suites and cases unexpectedly found in the log. + """ + ts_names = self.started_suites.keys() + if ts_name: + if ts_name not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"On {phase}: unexpected Ztest suite '{ts_name}' " + f"not present among: {self.instance.testsuite.ztest_suite_names}") + if ts_name not in self.detected_suite_names: + if self.trace: + logger.debug(f"On {phase}: detected new Ztest suite '{ts_name}'") + self.detected_suite_names.append(ts_name) + ts_names = [ ts_name ] if ts_name in ts_names else [] + + # Firstly try to match the test case ID to the first running Ztest suite with this test name. + for ts_name_ in ts_names: + if self.started_suites[ts_name_]['count'] < (0 if phase == 'TS_SUM' else 1): + continue + tc_fq_id = "{}.{}.{}".format(self.id, ts_name_, tc_name) + if tc := self.instance.get_case_by_name(tc_fq_id): + if self.trace: + logger.debug(f"On {phase}: Ztest case '{tc_name}' matched to '{tc_fq_id}") + return tc + logger.debug(f"On {phase}: Ztest case '{tc_name}' is not known in {self.started_suites} running suite(s).") + tc_id = "{}.{}".format(self.id, tc_name) + return self.instance.get_case_or_create(tc_id) + + def start_suite(self, suite_name): + if suite_name not in self.detected_suite_names: self.detected_suite_names.append(suite_name) + if suite_name not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"Unexpected Ztest suite '{suite_name}'") + if suite_name in self.started_suites: + if self.started_suites[suite_name]['count'] > 0: + logger.warning(f"Already STARTED '{suite_name}':{self.started_suites[suite_name]}") + elif self.trace: + logger.debug(f"START suite '{suite_name}'") + self.started_suites[suite_name]['count'] += 1 + self.started_suites[suite_name]['repeat'] += 1 + else: + self.started_suites[suite_name] = { 'count': 1, 'repeat': 0 } + + def end_suite(self, suite_name, phase='', suite_status=None): + if suite_name in self.started_suites: + if phase == 'TS_SUM' and self.started_suites[suite_name]['count'] == 0: + return + if self.started_suites[suite_name]['count'] < 1: + logger.error(f"Already ENDED {phase} suite '{suite_name}':{self.started_suites[suite_name]}") + elif self.trace: + logger.debug(f"END {phase} suite '{suite_name}':{self.started_suites[suite_name]}") + self.started_suites[suite_name]['count'] -= 1 + elif suite_status == 'SKIP': + self.start_suite(suite_name) # register skipped suites at their summary end + self.started_suites[suite_name]['count'] -= 1 + else: + logger.warning(f"END {phase} suite '{suite_name}' without START detected") - testcase_match = re.search(self.ZTEST_START_PATTERN, line) - if testcase_match: - name = "{}.{}".format(self.id, testcase_match.group(2)) - tc = self.instance.get_case_or_create(name) + def start_case(self, tc_name): + if tc_name in self.started_cases: + if self.started_cases[tc_name]['count'] > 0: + logger.warning(f"Already STARTED '{tc_name}':{self.started_cases[tc_name]}") + self.started_cases[tc_name]['count'] += 1 + else: + self.started_cases[tc_name] = { 'count': 1 } + + def end_case(self, tc_name, phase=''): + if tc_name in self.started_cases: + if phase == 'TS_SUM' and self.started_cases[tc_name]['count'] == 0: + return + if self.started_cases[tc_name]['count'] < 1: + logger.error(f"Already ENDED {phase} case '{tc_name}':{self.started_cases[tc_name]}") + elif self.trace: + logger.debug(f"END {phase} case '{tc_name}':{self.started_cases[tc_name]}") + self.started_cases[tc_name]['count'] -= 1 + elif phase != 'TS_SUM': + logger.warning(f"END {phase} case '{tc_name}' without START detected") + + + def handle(self, line): + testcase_match = None + if self._match: + self.testcase_output += line + "\n" + + if test_suite_start_match := re.search(self.test_suite_start_pattern, line): + self.start_suite(test_suite_start_match.group("suite_name")) + elif test_suite_end_match := re.search(self.test_suite_end_pattern, line): + suite_name=test_suite_end_match.group("suite_name") + self.end_suite(suite_name, 'TS_END') + elif testcase_match := re.search(self.test_case_start_pattern, line): + tc_name = testcase_match.group(2) + tc = self.get_testcase(tc_name, 'TC_START') + self.start_case(tc.name) # Mark the test as started, if something happens here, it is mostly # due to this tests, for example timeout. This should in this case # be marked as failed and not blocked (not run). tc.status = TwisterStatus.STARTED - - if testcase_match or self._match: - self.testcase_output += line + "\n" - self._match = True - - result_match = result_re.match(line) + if not self._match: + self.testcase_output += line + "\n" + self._match = True # some testcases are skipped based on predicates and do not show up # during test execution, however they are listed in the summary. Parse # the summary for status and use that status instead. - - summary_re = re.compile(r"- (PASS|FAIL|SKIP) - \[([^\.]*).(test_)?(\S*)\] duration = (\d*[.,]?\d*) seconds") - summary_match = summary_re.match(line) - - if result_match: + elif result_match := self.test_case_end_pattern.match(line): matched_status = result_match.group(1) - name = "{}.{}".format(self.id, result_match.group(3)) - tc = self.instance.get_case_or_create(name) + tc_name = result_match.group(3) + tc = self.get_testcase(tc_name, 'TC_END') + self.end_case(tc.name) tc.status = TwisterStatus[matched_status] if tc.status == TwisterStatus.SKIP: tc.reason = "ztest skip" @@ -755,15 +843,22 @@ def handle(self, line): self.testcase_output = "" self._match = False self.ztest = True - elif summary_match: - matched_status = summary_match.group(1) - self.detected_suite_names.append(summary_match.group(2)) - name = "{}.{}".format(self.id, summary_match.group(4)) - tc = self.instance.get_case_or_create(name) + elif test_suite_summary_match := self.test_suite_summary_pattern.match(line): + suite_name=test_suite_summary_match.group("suite_name") + suite_status=test_suite_summary_match.group("suite_status") + self._match = False + self.ztest = True + self.end_suite(suite_name, 'TS_SUM', suite_status=suite_status) + elif test_case_summary_match := self.test_case_summary_pattern.match(line): + matched_status = test_case_summary_match.group(1) + suite_name = test_case_summary_match.group(2) + tc_name = test_case_summary_match.group(4) + tc = self.get_testcase(tc_name, 'TS_SUM', suite_name) + self.end_case(tc.name, 'TS_SUM') tc.status = TwisterStatus[matched_status] if tc.status == TwisterStatus.SKIP: tc.reason = "ztest skip" - tc.duration = float(summary_match.group(5)) + tc.duration = float(test_case_summary_match.group(5)) if tc.status == TwisterStatus.FAIL: tc.output = self.testcase_output self.testcase_output = "" diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 128bc598ed9c4..172add5367edf 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -1,6 +1,6 @@ # vim: set syntax=python ts=4 : # -# Copyright (c) 20180-2022 Intel Corporation +# Copyright (c) 2018-2024 Intel Corporation # Copyright 2022 NXP # SPDX-License-Identifier: Apache-2.0 @@ -814,6 +814,10 @@ def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs) self.env = env self.duts = None + @property + def trace(self) -> bool: + return self.options.verbose > 2 + def log_info(self, filename, inline_logs, log_testcases=False): filename = os.path.abspath(os.path.realpath(filename)) if inline_logs: @@ -1087,6 +1091,18 @@ def process(self, pipeline, done, message, lock, results): self.instance.reason = reason self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason) + def demangle(self, symbol_name): + if symbol_name[:2] == '_Z': + try: + cpp_filt = subprocess.run('c++filt', input=symbol_name, text=True, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if self.trace: + logger.debug(f"Demangle: '{symbol_name}'==>'{cpp_filt.stdout}'") + return cpp_filt.stdout.strip() + except Exception as e: + logger.error(f"Failed to demangle '{symbol_name}': {e}") + return symbol_name + def determine_testcases(self, results): yaml_testsuite_name = self.instance.testsuite.id logger.debug(f"Determine test cases for test suite: {yaml_testsuite_name}") @@ -1102,19 +1118,25 @@ def determine_testcases(self, results): for sym in section.iter_symbols(): # It is only meant for new ztest fx because only new ztest fx exposes test functions # precisely. - + m_ = new_ztest_unit_test_regex.search(sym.name) + if not m_: + continue + # Demangle C++ symbols + m_ = new_ztest_unit_test_regex.search(self.demangle(sym.name)) + if not m_: + continue # The 1st capture group is new ztest suite name. # The 2nd capture group is new ztest unit test name. - matches = new_ztest_unit_test_regex.findall(sym.name) - if matches: - for m in matches: - # new_ztest_suite = m[0] # not used for now - test_func_name = m[1].replace("test_", "", 1) - testcase_id = f"{yaml_testsuite_name}.{test_func_name}" - detected_cases.append(testcase_id) + new_ztest_suite = m_[1] + if new_ztest_suite not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' " + f"not present in: {self.instance.testsuite.ztest_suite_names}") + test_func_name = m_[2].replace("test_", "", 1) + testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}" + detected_cases.append(testcase_id) if detected_cases: - logger.debug(f"{', '.join(detected_cases)} in {elf_file}") + logger.debug(f"Detected Ztest cases: [{', '.join(detected_cases)}] in {elf_file}") tc_keeper = {tc.name: {'status': tc.status, 'reason': tc.reason} for tc in self.instance.testcases} self.instance.testcases.clear() self.instance.testsuite.testcases.clear() diff --git a/scripts/pylib/twister/twisterlib/testsuite.py b/scripts/pylib/twister/twisterlib/testsuite.py index 3522c5bb218e0..01b91f4b876d8 100644 --- a/scripts/pylib/twister/twisterlib/testsuite.py +++ b/scripts/pylib/twister/twisterlib/testsuite.py @@ -1,6 +1,6 @@ # vim: set syntax=python ts=4 : # -# Copyright (c) 2018-2022 Intel Corporation +# Copyright (c) 2018-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 from enum import Enum @@ -248,14 +248,16 @@ def _find_ztest_testcases(search_area, testcase_regex): testcase_regex_matches = \ [m for m in testcase_regex.finditer(search_area)] testcase_names = \ - [m.group("testcase_name") for m in testcase_regex_matches] - testcase_names = [name.decode("UTF-8") for name in testcase_names] + [(m.group("suite_name") if m.groupdict().get("suite_name") else b'', m.group("testcase_name")) \ + for m in testcase_regex_matches] + testcase_names = [(ts_name.decode("UTF-8"), tc_name.decode("UTF-8")) for ts_name, tc_name in testcase_names] warnings = None for testcase_name in testcase_names: - if not testcase_name.startswith("test_"): + if not testcase_name[1].startswith("test_"): warnings = "Found a test that does not start with test_" testcase_names = \ - [tc_name.replace("test_", "", 1) for tc_name in testcase_names] + [(ts_name + '.' if ts_name else '') + f"{tc_name.replace('test_', '', 1)}" \ + for (ts_name, tc_name) in testcase_names] return testcase_names, warnings diff --git a/scripts/tests/twister/test_harness.py b/scripts/tests/twister/test_harness.py index c0a135fb24105..7e0fca79677ab 100644 --- a/scripts/tests/twister/test_harness.py +++ b/scripts/tests/twister/test_harness.py @@ -597,31 +597,48 @@ def test_get_harness(name): "", "Running TESTSUITE suite_name", ["suite_name"], + { 'suite_name': { 'count': 1, 'repeat': 0 } }, + {}, TwisterStatus.NONE, True, TwisterStatus.NONE, ), - ("", "START - test_testcase", [], TwisterStatus.STARTED, True, TwisterStatus.NONE), ( - "", + "On TC_START: Ztest case 'testcase' is not known in {} running suite(s)", + "START - test_testcase", + [], + {}, + { 'test_id.testcase': { 'count': 1 } }, + TwisterStatus.STARTED, + True, + TwisterStatus.NONE + ), + ( + "On TC_END: Ztest case 'example' is not known in {} running suite(s)", "PASS - test_example in 0 seconds", [], + {}, + {}, TwisterStatus.PASS, True, TwisterStatus.NONE, ), ( - "", + "On TC_END: Ztest case 'example' is not known in {} running suite(s)", "SKIP - test_example in 0 seconds", [], + {}, + {}, TwisterStatus.SKIP, True, TwisterStatus.NONE, ), ( - "", + "On TC_END: Ztest case 'example' is not known in {} running suite(s)", "FAIL - test_example in 0 seconds", [], + {}, + {}, TwisterStatus.FAIL, True, TwisterStatus.NONE, @@ -630,6 +647,8 @@ def test_get_harness(name): "not a ztest and no state for test_id", "START - test_testcase", [], + {}, + { 'test_id.testcase': { 'count': 1 } }, TwisterStatus.PASS, False, TwisterStatus.PASS, @@ -638,6 +657,8 @@ def test_get_harness(name): "not a ztest and no state for test_id", "START - test_testcase", [], + {}, + { 'test_id.testcase': { 'count': 1 } }, TwisterStatus.FAIL, False, TwisterStatus.FAIL, @@ -646,12 +667,14 @@ def test_get_harness(name): @pytest.mark.parametrize( - "exp_out, line, exp_suite_name, exp_status, ztest, state", + "exp_out, line, exp_suite_name, exp_started_suites, exp_started_cases, exp_status, ztest, state", TEST_DATA_7, ids=["testsuite", "testcase", "pass", "skip", "failed", "ztest pass", "ztest fail"], ) def test_test_handle( - tmp_path, caplog, exp_out, line, exp_suite_name, exp_status, ztest, state + tmp_path, caplog, exp_out, line, + exp_suite_name, exp_started_suites, exp_started_cases, + exp_status, ztest, state ): # Arrange line = line @@ -662,6 +685,7 @@ def test_test_handle( mock_testsuite = mock.Mock(id="id", testcases=[]) mock_testsuite.name = "mock_testsuite" mock_testsuite.harness_config = {} + mock_testsuite.ztest_suite_names = [] outdir = tmp_path / "gtest_out" outdir.mkdir() @@ -681,6 +705,9 @@ def test_test_handle( # Assert assert test_obj.detected_suite_names == exp_suite_name + assert test_obj.started_suites == exp_started_suites + assert test_obj.started_cases == exp_started_cases + assert exp_out in caplog.text if not "Running" in line and exp_out == "": assert test_obj.instance.testcases[0].status == exp_status diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index 3ab7de2fb935f..29459d585498d 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -56,6 +56,7 @@ def mocked_instance(tmp_path): def mocked_env(): env = mock.Mock() options = mock.Mock() + options.verbose = 2 env.options = options return env @@ -1562,11 +1563,32 @@ def mock_determine_testcases(res): TESTDATA_7 = [ ( [ - 'z_ztest_unit_test__dummy_suite_name__dummy_test_name', - 'z_ztest_unit_test__dummy_suite_name__test_dummy_name', + 'z_ztest_unit_test__dummy_suite1_name__dummy_test_name1', + 'z_ztest_unit_test__dummy_suite2_name__test_dummy_name2', 'no match' ], - ['dummy_id.dummy_name', 'dummy_id.dummy_name'] + [ + ('dummy_id.dummy_suite1_name.dummy_name1'), + ('dummy_id.dummy_suite2_name.dummy_name2') + ] + ), + ( + [ + 'z_ztest_unit_test__dummy_suite2_name__test_dummy_name2', + 'z_ztest_unit_test__bad_suite3_name_no_test', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name4E', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_bad_name1E', + '_ZN12_GLOBAL__N_1L51z_ztest_unit_test_dummy_suite3_name__test_bad_name2E', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name5E', + '_ZN15foobarnamespaceL54z_ztest_unit_test__dummy_suite3_name__test_dummy_name6E', + ], + [ + ('dummy_id.dummy_suite2_name.dummy_name2'), + ('dummy_id.dummy_suite3_name.dummy_name4'), + ('dummy_id.dummy_suite3_name.bad_name1E'), + ('dummy_id.dummy_suite3_name.dummy_name5'), + ('dummy_id.dummy_suite3_name.dummy_name6'), + ] ), ( ['no match'], @@ -1577,10 +1599,11 @@ def mock_determine_testcases(res): @pytest.mark.parametrize( 'symbols_names, added_tcs', TESTDATA_7, - ids=['two hits, one miss', 'nothing'] + ids=['two hits, one miss', 'demangle', 'nothing'] ) def test_projectbuilder_determine_testcases( mocked_jobserver, + mocked_env, symbols_names, added_tcs ): @@ -1599,9 +1622,9 @@ def test_projectbuilder_determine_testcases( instance_mock = mock.Mock() instance_mock.testcases = [] instance_mock.testsuite.id = 'dummy_id' - env_mock = mock.Mock() + instance_mock.testsuite.ztest_suite_names = [] - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, mocked_env, mocked_jobserver) with mock.patch('twisterlib.runner.ELFFile', elf_mock), \ mock.patch('builtins.open', mock.mock_open()): @@ -2137,13 +2160,11 @@ def test_projectbuilder_cmake(): instance_mock = mock.Mock() instance_mock.handler = 'dummy handler' instance_mock.build_dir = os.path.join('build', 'dir') - instance_mock.platform.name = 'frdm_k64f' env_mock = mock.Mock() pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) pb.build_dir = 'build_dir' - pb.testsuite.platform = instance_mock.platform - pb.testsuite.extra_args = ['some', 'platform:frdm_k64f:args'] + pb.testsuite.extra_args = ['some', 'args'] pb.testsuite.extra_conf_files = ['some', 'files1'] pb.testsuite.extra_overlay_confs = ['some', 'files2'] pb.testsuite.extra_dtc_overlay_files = ['some', 'files3'] @@ -2156,7 +2177,7 @@ def test_projectbuilder_cmake(): assert res == cmake_res_mock pb.cmake_assemble_args.assert_called_once_with( - ['some', 'args'], + pb.testsuite.extra_args, pb.instance.handler, pb.testsuite.extra_conf_files, pb.testsuite.extra_overlay_confs, diff --git a/scripts/tests/twister/test_testplan.py b/scripts/tests/twister/test_testplan.py index eea3d3e9abefb..b00d69ab06113 100644 --- a/scripts/tests/twister/test_testplan.py +++ b/scripts/tests/twister/test_testplan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2020-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 @@ -85,7 +85,8 @@ def test_get_all_testsuites_short(class_testplan, all_testsuites_dict): 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2', 'test_d.check_1.unit_1a', 'test_d.check_1.unit_1b', - 'test_e.check_1.1a', 'test_e.check_1.1b', + 'test_e.check_1.feature5.1a', + 'test_e.check_1.feature5.1b', 'test_config.main'] assert sorted(plan.get_all_tests()) == sorted(expected_tests) diff --git a/scripts/tests/twister/test_testsuite.py b/scripts/tests/twister/test_testsuite.py index e297b6b6d9cd6..8d20902a5ebaf 100644 --- a/scripts/tests/twister/test_testsuite.py +++ b/scripts/tests/twister/test_testsuite.py @@ -165,7 +165,7 @@ def test_scanpathresults_dunders(original, provided, expected): ), ScanPathResult( warnings=None, - matches=['1a', '1b'], + matches=['feature5.1a', 'feature5.1b'], has_registered_test_suites=False, has_run_registered_test_suites=True, has_test_main=False, diff --git a/scripts/tests/twister_blackbox/test_config.py b/scripts/tests/twister_blackbox/test_config.py index c05d18cdaa780..2cad497055f5b 100644 --- a/scripts/tests/twister_blackbox/test_config.py +++ b/scripts/tests/twister_blackbox/test_config.py @@ -13,6 +13,7 @@ import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -55,13 +56,13 @@ def test_alt_config_root(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 3 + assert len(filtered_j) == 4 @pytest.mark.parametrize( 'level, expected_tests', [ - ('smoke', 5), - ('acceptance', 6), + ('smoke', 6), + ('acceptance', 7), ], ids=['smoke', 'acceptance'] ) diff --git a/scripts/tests/twister_blackbox/test_data/test_config.yaml b/scripts/tests/twister_blackbox/test_data/test_config.yaml index d7e4828350cf9..6bfe24a4e819e 100644 --- a/scripts/tests/twister_blackbox/test_data/test_config.yaml +++ b/scripts/tests/twister_blackbox/test_data/test_config.yaml @@ -6,9 +6,10 @@ levels: description: > A plan to be used verifying basic features adds: - - dummy.agnostic.* + - dummy.agnostic\..* - name: acceptance description: > More coverage adds: - - dummy.* + - dummy.agnostic\..* + - dummy.device\..* diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c index 55c375965aeb9..798fd9756a8c1 100644 --- a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic/group2/src/main.c @@ -9,6 +9,8 @@ ZTEST_SUITE(a2_tests, NULL, NULL, NULL, NULL, NULL); +ZTEST_SUITE(a3_tests, NULL, NULL, NULL, NULL, NULL); + /** * @brief Test Asserts * @@ -34,3 +36,8 @@ ZTEST(a2_tests, test_assert2) zassert_equal(1, 1, "1 was not equal to 1"); zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); } + +ZTEST(a3_tests, test_assert1) +{ + zassert_true(1, "1 was false"); +} diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt new file mode 100644 index 0000000000000..3ffe630f1ad98 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf new file mode 100644 index 0000000000000..9467c2926896d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp new file mode 100644 index 0000000000000..b37d02646cfcb --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + + +// global namespace + +ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a1_1_tests, test_assert) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml new file mode 100644 index 0000000000000..7d8a642054144 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml @@ -0,0 +1,10 @@ +tests: + dummy.agnostic_cpp.group1.subgroup1: + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp + - subgrouped diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt new file mode 100644 index 0000000000000..3ffe630f1ad98 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf new file mode 100644 index 0000000000000..9467c2926896d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp new file mode 100644 index 0000000000000..0ac835a959c04 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +// global namespace + +ZTEST_SUITE(a1_2_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a1_2_tests, test_assert) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml new file mode 100644 index 0000000000000..f0146d3e19344 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml @@ -0,0 +1,11 @@ +tests: + dummy.agnostic_cpp.group1.subgroup2: + build_only: true + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp + - subgrouped diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt new file mode 100644 index 0000000000000..3ffe630f1ad98 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf new file mode 100644 index 0000000000000..9467c2926896d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp new file mode 100644 index 0000000000000..1c86457e76a0d --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + + +namespace +{ + +ZTEST_SUITE(a2_tests, NULL, NULL, NULL, NULL, NULL); + +ZTEST_SUITE(a3_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a2_tests, test_assert1) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +ZTEST(a2_tests, test_assert2) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +ZTEST(a3_tests, test_assert1) +{ + zassert_true(1, "1 was false"); +} + +} // namsespace diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp new file mode 100644 index 0000000000000..bf9c90536b2d2 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +namespace foo_namespace +{ + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a2_tests, test_assert3) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +} // foo_namespace diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml new file mode 100644 index 0000000000000..1a93b591460f6 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml @@ -0,0 +1,9 @@ +tests: + dummy.agnostic_cpp.group2: + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp diff --git a/scripts/tests/twister_blackbox/test_filter.py b/scripts/tests/twister_blackbox/test_filter.py index 90ea95e643075..3558296d40b11 100644 --- a/scripts/tests/twister_blackbox/test_filter.py +++ b/scripts/tests/twister_blackbox/test_filter.py @@ -14,6 +14,7 @@ import json import re +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -81,23 +82,27 @@ def teardown_class(cls): pass @pytest.mark.parametrize( - 'tag, expected_test_count', + 'tags, expected_test_count', [ - ('device', 5), # dummy.agnostic.group1.subgroup1.assert - # dummy.agnostic.group1.subgroup2.assert - # dummy.agnostic.group2.assert1 - # dummy.agnostic.group2.assert2 - # dummy.agnostic.group2.assert3 - ('agnostic', 1) # dummy.device.group.assert + (['device', 'cpp'], 6), + # dummy.agnostic.group1.subgroup1.a1_1_tests.assert + # dummy.agnostic.group1.subgroup2.a2_2_tests.assert + # dummy.agnostic.group2.a2_tests.assert1 + # dummy.agnostic.group2.a2_tests.assert2 + # dummy.agnostic.group2.a2_tests.assert3 + # dummy.agnostic.group2.a3_tests.assert1 + (['agnostic'], 1) # dummy.device.group.assert ], - ids=['no device', 'no agnostic'] + ids=['no device, no cpp', 'no agnostic'] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) - def test_exclude_tag(self, out_path, tag, expected_test_count): + def test_exclude_tag(self, out_path, tags, expected_test_count): test_platforms = ['qemu_x86', 'intel_adl_crb'] path = os.path.join(TEST_DATA, 'tests', 'dummy') args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \ - ['--exclude-tag', tag] + \ + [val for pair in zip( + ['--exclude-tag'] * len(tags), tags + ) for val in pair] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms ) for val in pair] @@ -144,7 +149,7 @@ def test_enable_slow(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 5 + assert len(filtered_j) == 6 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_enable_slow_only(self, out_path): @@ -172,7 +177,7 @@ def test_enable_slow_only(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 3 + assert len(filtered_j) == 4 @pytest.mark.parametrize( 'arch, expected', diff --git a/scripts/tests/twister_blackbox/test_platform.py b/scripts/tests/twister_blackbox/test_platform.py index 43ff3346cf8af..f76dab4738a53 100644 --- a/scripts/tests/twister_blackbox/test_platform.py +++ b/scripts/tests/twister_blackbox/test_platform.py @@ -36,7 +36,7 @@ class TestPlatform: 'built_configurations': 2, 'failed_configurations': 0, 'errored_configurations': 0, - 'executed_test_cases': 8, + 'executed_test_cases': 10, 'skipped_test_cases': 2, 'platform_count': 3, 'executed_on_platform': 4, @@ -64,6 +64,27 @@ class TestPlatform: 'only_built': 0 } ), + ( + os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic_cpp'), + ['native_sim'], + { + 'selected_test_scenarios': 3, + 'selected_test_instances': 3, + 'executed_test_instances': 3, + 'skipped_configurations': 0, + 'skipped_by_static_filter': 0, + 'skipped_at_runtime': 0, + 'passed_configurations': 2, + 'built_configurations': 1, + 'failed_configurations': 0, + 'errored_configurations': 0, + 'executed_test_cases': 5, + 'skipped_test_cases': 0, + 'platform_count': 1, + 'executed_on_platform': 2, + 'only_built': 1 + } + ), ] @classmethod @@ -129,7 +150,7 @@ def test_force_platform(self, out_path): assert str(sys_exit.value) == '0' - assert len(filtered_j) == 12 + assert len(filtered_j) == 26 def test_platform(self, out_path): path = os.path.join(TEST_DATA, 'tests', 'dummy') @@ -250,6 +271,7 @@ def test_exclude_platform(self, capfd, out_path, test_path, test_platforms, expe ids=[ 'emulation_only tests/dummy/agnostic', 'emulation_only tests/dummy/device', + 'native_sim_only tests/dummy/agnostic_cpp', ] ) def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expected): diff --git a/scripts/tests/twister_blackbox/test_printouts.py b/scripts/tests/twister_blackbox/test_printouts.py index 3f65549b8ea9c..853797354f44c 100644 --- a/scripts/tests/twister_blackbox/test_printouts.py +++ b/scripts/tests/twister_blackbox/test_printouts.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 """ @@ -41,17 +41,18 @@ class TestPrintOuts: ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), [ - 'dummy.agnostic.group1.subgroup1.assert', - 'dummy.agnostic.group1.subgroup2.assert', - 'dummy.agnostic.group2.assert1', - 'dummy.agnostic.group2.assert2', - 'dummy.agnostic.group2.assert3' + 'dummy.agnostic.group1.subgroup1.a1_1_tests.assert', + 'dummy.agnostic.group1.subgroup2.a1_2_tests.assert', + 'dummy.agnostic.group2.a2_tests.assert1', + 'dummy.agnostic.group2.a2_tests.assert2', + 'dummy.agnostic.group2.a3_tests.assert1', + 'dummy.agnostic.group2.a2_tests.assert3' ] ), ( os.path.join(TEST_DATA, 'tests', 'dummy', 'device'), [ - 'dummy.device.group.assert' + 'dummy.device.group.d_tests.assert' ] ), ] @@ -64,11 +65,12 @@ class TestPrintOuts: '└── Tests\n' \ ' └── dummy\n' \ ' └── agnostic\n' \ - ' ├── dummy.agnostic.group1.subgroup1.assert\n' \ - ' ├── dummy.agnostic.group1.subgroup2.assert\n' \ - ' ├── dummy.agnostic.group2.assert1\n' \ - ' ├── dummy.agnostic.group2.assert2\n' \ - ' └── dummy.agnostic.group2.assert3\n' + ' ├── dummy.agnostic.group1.subgroup1.a1_1_tests.assert\n' \ + ' ├── dummy.agnostic.group1.subgroup2.a1_2_tests.assert\n' \ + ' ├── dummy.agnostic.group2.a2_tests.assert1\n' \ + ' ├── dummy.agnostic.group2.a2_tests.assert2\n' \ + ' ├── dummy.agnostic.group2.a2_tests.assert3\n' \ + ' └── dummy.agnostic.group2.a3_tests.assert1\n' ), ( os.path.join(TEST_DATA, 'tests', 'dummy', 'device'), @@ -77,7 +79,7 @@ class TestPrintOuts: '└── Tests\n' ' └── dummy\n' ' └── device\n' - ' └── dummy.device.group.assert\n' + ' └── dummy.device.group.d_tests.assert\n' ), ] diff --git a/scripts/tests/twister_blackbox/test_report.py b/scripts/tests/twister_blackbox/test_report.py index 2db1006bc5aa9..bac6ddbbbfa0b 100644 --- a/scripts/tests/twister_blackbox/test_report.py +++ b/scripts/tests/twister_blackbox/test_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 """ @@ -350,12 +350,12 @@ def test_log_file(self, capfd, test_path, test_platforms, out_path, file_name): ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['--detailed-skipped-report'], - {'qemu_x86/atom': 5, 'intel_adl_crb/alder_lake': 1} + {'qemu_x86/atom': 6, 'intel_adl_crb/alder_lake': 1} ), ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['--detailed-skipped-report', '--report-filtered'], - {'qemu_x86/atom': 6, 'intel_adl_crb/alder_lake': 6} + {'qemu_x86/atom': 13, 'intel_adl_crb/alder_lake': 13} ), ], ids=['dummy tests', 'dummy tests with filtered'] @@ -392,7 +392,7 @@ def test_detailed_skipped_report(self, out_path, test_path, flags, expected_test 'test_path, report_filtered, expected_filtered_count', [ (os.path.join(TEST_DATA, 'tests', 'dummy'), False, 0), - (os.path.join(TEST_DATA, 'tests', 'dummy'), True, 4), + (os.path.join(TEST_DATA, 'tests', 'dummy'), True, 10), ], ids=['no filtered', 'with filtered'] ) diff --git a/scripts/tests/twister_blackbox/test_runner.py b/scripts/tests/twister_blackbox/test_runner.py index 1ab50522c5ebf..e85f8abdf0445 100644 --- a/scripts/tests/twister_blackbox/test_runner.py +++ b/scripts/tests/twister_blackbox/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 """ @@ -54,7 +54,7 @@ class TestRunner: 'built_configurations': 0, 'failed_configurations': 0, 'errored_configurations': 0, - 'executed_test_cases': 8, + 'executed_test_cases': 10, 'skipped_test_cases': 0, 'platform_count': 0, 'executed_on_platform': 4, @@ -591,7 +591,7 @@ def test_tag(self, capfd, out_path, test_path, test_platforms, tags, expected): sys.stderr.write(err) for line in expected: - assert re.search(line, err) + assert re.search(line, err), f"no expected:'{line}' in '{err}'" assert str(sys_exit.value) == '0' diff --git a/scripts/tests/twister_blackbox/test_shuffle.py b/scripts/tests/twister_blackbox/test_shuffle.py index ade1267b48227..f45724b7b55e7 100644 --- a/scripts/tests/twister_blackbox/test_shuffle.py +++ b/scripts/tests/twister_blackbox/test_shuffle.py @@ -10,10 +10,10 @@ import mock import os import pytest -import re import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -33,20 +33,21 @@ def teardown_class(cls): @pytest.mark.parametrize( 'seed, ratio, expected_order', [ - ('123', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']), - ('123', '2/2', ['dummy.agnostic.group2', 'dummy.device.group']), - ('321', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']), - ('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), - ('123', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']), + ('123', '1/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), + ('123', '2/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup1']), + ('321', '1/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']), + ('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup1']), + ('123', '1/3', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), ('123', '2/3', ['dummy.agnostic.group2']), - ('123', '3/3', ['dummy.device.group']), - ('321', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']), + ('123', '3/3', ['dummy.agnostic.group1.subgroup1']), + ('321', '1/3', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']), ('321', '2/3', ['dummy.device.group']), - ('321', '3/3', ['dummy.agnostic.group1.subgroup2']) + ('321', '3/3', ['dummy.agnostic.group1.subgroup1']) ], ids=['first half, 123', 'second half, 123', 'first half, 321', 'second half, 321', 'first third, 123', 'middle third, 123', 'last third, 123', - 'first third, 321', 'middle third, 321', 'last third, 321'] + 'first third, 321', 'middle third, 321', 'last third, 321' +] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_shuffle_tests(self, out_path, seed, ratio, expected_order): @@ -65,14 +66,8 @@ def test_shuffle_tests(self, out_path, seed, ratio, expected_order): with open(os.path.join(out_path, 'testplan.json')) as f: j = json.load(f) - filtered_j = [ - (ts['platform'], ts['name'], tc['identifier']) \ - for ts in j['testsuites'] \ - for tc in ts['testcases'] if 'reason' not in tc - ] - testcases = [re.sub(r'\.assert[^\.]*?$', '', j[2]) for j in filtered_j] - testsuites = list(dict.fromkeys(testcases)) + testsuites = [os.path.basename(ts['name']) for ts in j['testsuites']] assert testsuites == expected_order diff --git a/scripts/tests/twister_blackbox/test_testlist.py b/scripts/tests/twister_blackbox/test_testlist.py index 1ef93f072b6b5..ad8eaeddfaf55 100644 --- a/scripts/tests/twister_blackbox/test_testlist.py +++ b/scripts/tests/twister_blackbox/test_testlist.py @@ -13,6 +13,7 @@ import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test from twisterlib.testplan import TestPlan @@ -71,4 +72,4 @@ def test_save_tests(self, out_path): for tc in ts['testcases'] if 'reason' not in tc ] - assert len(filtered_j) == 5 + assert len(filtered_j) == 6 diff --git a/scripts/tests/twister_blackbox/test_testplan.py b/scripts/tests/twister_blackbox/test_testplan.py index 915653a33e15e..8834e03ead039 100644 --- a/scripts/tests/twister_blackbox/test_testplan.py +++ b/scripts/tests/twister_blackbox/test_testplan.py @@ -13,6 +13,7 @@ import sys import json +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan from twisterlib.error import TwisterRuntimeError @@ -20,7 +21,7 @@ class TestTestPlan: TESTDATA_1 = [ - ('dummy.agnostic.group2.assert1', SystemExit, 3), + ('dummy.agnostic.group2.a2_tests.assert1', SystemExit, 4), ( os.path.join('scripts', 'tests', 'twister_blackbox', 'test_data', 'tests', 'dummy', 'agnostic', 'group1', 'subgroup1', @@ -30,12 +31,12 @@ class TestTestPlan: ), ] TESTDATA_2 = [ - ('buildable', 6), - ('runnable', 4), + ('buildable', 7), + ('runnable', 5), ] TESTDATA_3 = [ (True, 1), - (False, 6), + (False, 7), ] @classmethod @@ -52,7 +53,7 @@ def teardown_class(cls): @pytest.mark.parametrize( 'test, expected_exception, expected_subtest_count', TESTDATA_1, - ids=['valid', 'invalid'] + ids=['valid', 'not found'] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_subtest(self, out_path, test, expected_exception, expected_subtest_count):