ztest: Add register functionality

Add new functionality to ztest to improve test modularity. The two
primary new entry points are:
* ztest_register_test_suite
* ztest_run_registered_test_suites

When registering a new test suite, users provide the name as well as
an optional predicate used to filter the tests for each run. Using NULL
as the predicate ensures that the test is run exactly once (after which
it is automatically filtered from future runs).

Calls to ztest_run_registered_test_suites take a state pointer as an
argument. This allows the the pragma functions to decide whether the
test should be run.

The biggest benefit of this system (other than the ability to filter
tests and maintain a larger test state) is the ability to better
modularize the test source code. Instead of all the various tests
having to coordinate and the main function having to know which tests
to run, each source file manages registering its own test
suite and handling the conditions for running the suite.

Signed-off-by: Yuval Peress <[email protected]>

Author: Yuval Peress

include/linker/common-ram.ld

@@ -140,3 +140,7 @@
 #ifdef CONFIG_USERSPACE
 	_static_kernel_objects_end = .;
 #endif
+
+#if defined(CONFIG_ZTEST)
+	ITERABLE_SECTION_RAM(ztest_suite_node, 4)
+#endif /* CONFIG_ZTEST */

scripts/pylib/twister/twisterlib.py

@@ -3,7 +3,6 @@
 #
 # Copyright (c) 2018 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 import contextlib
 import string
@@ -34,6 +33,7 @@
 import yaml
 import json
 from multiprocessing import Lock, Process, Value
+from typing import List
 
 try:
     # Use the C LibYAML parser if available, rather than the Python parser.
@@ -1576,6 +1576,41 @@ class DisablePyTestCollectionMixin(object):
     __test__ = False
 
 
+class ScanPathResult:
+    """Result of the TestCase.scan_path function call.
+
+    Attributes:
+        matches                          A list of test cases
+        warnings                         A string containing one or more
+                                         warnings to display
+        has_registered_test_suites       Whether or not the path contained any
+                                         calls to the ztest_register_test_suite
+                                         macro.
+        has_run_registered_test_suites   Whether or not the path contained at
+                                         least one call to
+                                         ztest_run_registered_test_suites.
+    """
+    def __init__(self,
+                 matches: List[str] = None,
+                 warnings: str = None,
+                 has_registered_test_suites: bool = False,
+                 has_run_registered_test_suites: bool = False):
+        self.matches = matches
+        self.warnings = warnings
+        self.has_registered_test_suites = has_registered_test_suites
+        self.has_run_registered_test_suites = has_run_registered_test_suites
+
+    def __eq__(self, other):
+        if not isinstance(other, ScanPathResult):
+            return False
+        return (sorted(self.matches) == sorted(other.matches) and
+                self.warnings == other.warnings and
+                (self.has_registered_test_suites ==
+                 other.has_registered_test_suites) and
+                (self.has_run_registered_test_suites ==
+                 other.has_run_registered_test_suites))
+
+
 class TestCase(DisablePyTestCollectionMixin):
     """Class representing a test application
     """
@@ -1662,29 +1697,42 @@ def scan_file(inf_name):
             # line--as we only search starting the end of this match
             br"^\s*ztest_test_suite\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
             re.MULTILINE)
+        registered_suite_regex = re.compile(
+            br"^\s*ztest_register_test_suite"
+            br"\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
+            re.MULTILINE)
         stc_regex = re.compile(
-            br"^\s*"  # empy space at the beginning is ok
+            br"""^\s*  # empy space at the beginning is ok
             # catch the case where it is declared in the same sentence, e.g:
             #
             # ztest_test_suite(mutex_complex, ztest_user_unit_test(TESTNAME));
-            br"(?:ztest_test_suite\([a-zA-Z0-9_]+,\s*)?"
+            # ztest_register_test_suite(n, p, ztest_user_unit_test(TESTNAME),
+            (?:ztest_
+              (?:test_suite\(|register_test_suite\([a-zA-Z0-9_]+\s*,\s*)
+              [a-zA-Z0-9_]+\s*,\s*
+            )?
             # Catch ztest[_user]_unit_test-[_setup_teardown](TESTNAME)
-            br"ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)?"
+            ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)?
             # Consume the argument that becomes the extra testcse
-            br"\(\s*"
-            br"(?P<stc_name>[a-zA-Z0-9_]+)"
+            \(\s*(?P<stc_name>[a-zA-Z0-9_]+)
             # _setup_teardown() variant has two extra arguments that we ignore
-            br"(?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)?"
-            br"\s*\)",
+            (?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)?
+            \s*\)""",
             # We don't check how it finishes; we don't care
-            re.MULTILINE)
+            re.MULTILINE | re.VERBOSE)
         suite_run_regex = re.compile(
             br"^\s*ztest_run_test_suite\((?P<suite_name>[a-zA-Z0-9_]+)\)",
             re.MULTILINE)
+        registered_suite_run_regex = re.compile(
+            br"^\s*ztest_run_registered_test_suites\("
+            br"(\*+|&)?(?P<state_identifier>[a-zA-Z0-9_]+)\)",
+            re.MULTILINE)
         achtung_regex = re.compile(
             br"(#ifdef|#endif)",
             re.MULTILINE)
         warnings = None
+        has_registered_test_suites = False
+        has_run_registered_test_suites = False
 
         with open(inf_name) as inf:
             if os.name == 'nt':
@@ -1695,52 +1743,94 @@ def scan_file(inf_name):
 
             with contextlib.closing(mmap.mmap(**mmap_args)) as main_c:
                 suite_regex_match = suite_regex.search(main_c)
-                if not suite_regex_match:
+                registered_suite_regex_match = registered_suite_regex.search(
+                    main_c)
+
+                if registered_suite_regex_match:
+                    has_registered_test_suites = True
+                if registered_suite_run_regex.search(main_c):
+                    has_run_registered_test_suites = True
+
+                if not suite_regex_match and not has_registered_test_suites:
                     # can't find ztest_test_suite, maybe a client, because
                     # it includes ztest.h
-                    return None, None
+                    return ScanPathResult(
+                        matches=None,
+                        warnings=None,
+                        has_registered_test_suites=has_registered_test_suites,
+                        has_run_registered_test_suites=has_run_registered_test_suites)
 
                 suite_run_match = suite_run_regex.search(main_c)
-                if not suite_run_match:
+                if suite_regex_match and not suite_run_match:
                     raise ValueError("can't find ztest_run_test_suite")
 
+                if suite_regex_match:
+                    search_start = suite_regex_match.end()
+                else:
+                    search_start = registered_suite_regex_match.end()
+
+                if suite_run_match:
+                    search_end = suite_run_match.start()
+                else:
+                    search_end = re.compile(br"\);", re.MULTILINE) \
+                        .search(main_c, search_start) \
+                        .end()
                 achtung_matches = re.findall(
                     achtung_regex,
-                    main_c[suite_regex_match.end():suite_run_match.start()])
+                    main_c[search_start:search_end])
                 if achtung_matches:
                     warnings = "found invalid %s in ztest_test_suite()" \
                                % ", ".join(sorted({match.decode() for match in achtung_matches},reverse = True))
                 _matches = re.findall(
                     stc_regex,
-                    main_c[suite_regex_match.end():suite_run_match.start()])
+                    main_c[search_start:search_end])
                 for match in _matches:
                     if not match.decode().startswith("test_"):
                         warnings = "Found a test that does not start with test_"
                 matches = [match.decode().replace("test_", "", 1) for match in _matches]
-                return matches, warnings
+                return ScanPathResult(
+                    matches=matches,
+                    warnings=warnings,
+                    has_registered_test_suites=has_registered_test_suites,
+                    has_run_registered_test_suites=has_run_registered_test_suites)
 
     def scan_path(self, path):
         subcases = []
+        has_registered_test_suites = False
+        has_run_registered_test_suites = False
         for filename in glob.glob(os.path.join(path, "src", "*.c*")):
             try:
-                _subcases, warnings = self.scan_file(filename)
-                if warnings:
-                    logger.error("%s: %s" % (filename, warnings))
-                    raise TwisterRuntimeError("%s: %s" % (filename, warnings))
-                if _subcases:
-                    subcases += _subcases
+                result: ScanPathResult = self.scan_file(filename)
+                if result.warnings:
+                    logger.error("%s: %s" % (filename, result.warnings))
+                    raise TwisterRuntimeError(
+                        "%s: %s" % (filename, result.warnings))
+                if result.matches:
+                    subcases += result.matches
+                if result.has_registered_test_suites:
+                    has_registered_test_suites = True
+                if result.has_run_registered_test_suites:
+                    has_run_registered_test_suites = True
             except ValueError as e:
                 logger.error("%s: can't find: %s" % (filename, e))
 
         for filename in glob.glob(os.path.join(path, "*.c")):
             try:
-                _subcases, warnings = self.scan_file(filename)
-                if warnings:
-                    logger.error("%s: %s" % (filename, warnings))
-                if _subcases:
-                    subcases += _subcases
+                result: ScanPathResult = self.scan_file(filename)
+                if result.warnings:
+                    logger.error("%s: %s" % (filename, result.warnings))
+                if result.matches:
+                    subcases += result.matches
             except ValueError as e:
                 logger.error("%s: can't find: %s" % (filename, e))
+
+        if has_registered_test_suites and not has_run_registered_test_suites:
+            warning = \
+                "Found call to 'ztest_register_test_suite()' but no "\
+                "call to 'ztest_run_registered_test_suites()'"
+            logger.error(warning)
+            raise TwisterRuntimeError(warning)
+
         return subcases
 
     def parse_subcases(self, test_path):

scripts/tests/twister/test_data/testcases/tests/test_d/test_data.yaml

@@ -0,0 +1,4 @@
+tests:
+  test_d.check_1:
+    tags: test_d
+    build_only: True

scripts/tests/twister/test_data/testcases/tests/test_d/test_ztest_error_register_test_suite.c

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+ztest_register_test_suite(feature4, NULL,
+			  ztest_unit_test(test_unit_1a),
+			  ztest_unit_test(test_unit_1b));

scripts/tests/twister/test_testinstance.py

@@ -13,7 +13,8 @@
 
 ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
 sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
-from twisterlib import TestInstance, BuildError, TestCase, TwisterException
+from twisterlib import (TestInstance, BuildError, TestCase, TwisterException,
+                        ScanPathResult)
 
 
 TESTDATA_1 = [
@@ -112,21 +113,45 @@ def test_get_unique_exception(testcase_root, workdir, name, exception):
         unique = TestCase(testcase_root, workdir, name)
         assert unique == exception
 
+
 TESTDATA_5 = [
-    ("testcases/tests/test_ztest.c", None, ['a', 'c', 'unit_a', 'newline', 'test_test_aa', 'user', 'last']),
-    ("testcases/tests/test_a/test_ztest_error.c", "Found a test that does not start with test_", ['1a', '1c', '2a', '2b']),
-    ("testcases/tests/test_a/test_ztest_error_1.c", "found invalid #ifdef, #endif in ztest_test_suite()", ['unit_1a', 'unit_1b', 'Unit_1c']),
+    ("testcases/tests/test_ztest.c",
+     ScanPathResult(
+         warnings=None,
+         matches=['a', 'c', 'unit_a',
+                  'newline',
+                  'test_test_aa',
+                  'user', 'last'],
+         has_registered_test_suites=False,
+         has_run_registered_test_suites=False)),
+    ("testcases/tests/test_a/test_ztest_error.c",
+     ScanPathResult(
+         warnings="Found a test that does not start with test_",
+         matches=['1a', '1c', '2a', '2b'],
+         has_registered_test_suites=False,
+         has_run_registered_test_suites=False)),
+    ("testcases/tests/test_a/test_ztest_error_1.c",
+     ScanPathResult(
+         warnings="found invalid #ifdef, #endif in ztest_test_suite()",
+         matches=['unit_1a', 'unit_1b', 'Unit_1c'],
+         has_registered_test_suites=False,
+         has_run_registered_test_suites=False)),
+    ("testcases/tests/test_d/test_ztest_error_register_test_suite.c",
+     ScanPathResult(
+         warnings=None, matches=['unit_1a', 'unit_1b'],
+         has_registered_test_suites=True,
+         has_run_registered_test_suites=False)),
 ]
 
-@pytest.mark.parametrize("test_file, expected_warnings, expected_subcases", TESTDATA_5)
-def test_scan_file(test_data, test_file, expected_warnings, expected_subcases):
+@pytest.mark.parametrize("test_file, expected", TESTDATA_5)
+def test_scan_file(test_data, test_file, expected: ScanPathResult):
     '''Testing scan_file method with different ztest files for warnings and results'''
 
-    testcase = TestCase("/scripts/tests/twister/test_data/testcases/tests", ".", "test_a.check_1")
+    testcase = TestCase("/scripts/tests/twister/test_data/testcases/tests", ".",
+                        "test_a.check_1")
 
-    results, warnings = testcase.scan_file(os.path.join(test_data, test_file))
-    assert sorted(results) == sorted(expected_subcases)
-    assert warnings == expected_warnings
+    result: ScanPathResult = testcase.scan_file(os.path.join(test_data, test_file))
+    assert result == expected
 
 
 TESTDATA_6 = [

scripts/tests/twister/test_testsuite_class.py

@@ -30,6 +30,7 @@ def test_testsuite_add_testcases(class_testsuite):
                           'test_c.check_2',
                           'test_a.check_1',
                           'test_a.check_2',
+                          'test_d.check_1',
                           'sample_test.app']
     testcase_list = []
     for key in sorted(class_testsuite.testcases.keys()):
@@ -58,9 +59,18 @@ def test_add_configurations(test_data, class_testsuite, board_root_dir):
 def test_get_all_testcases(class_testsuite, all_testcases_dict):
     """ Testing get_all_testcases function of TestSuite class in Twister """
     class_testsuite.testcases = all_testcases_dict
-    expected_tests = ['sample_test.app', 'test_a.check_1.1a', 'test_a.check_1.1c',
-                    'test_a.check_1.2a', 'test_a.check_1.2b', 'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a', 'test_a.check_1.unit_1b', 'test_a.check_2.1a', 'test_a.check_2.1c', 'test_a.check_2.2a', 'test_a.check_2.2b', 'test_a.check_2.Unit_1c', 'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b', 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2']
-    assert len(class_testsuite.get_all_tests()) == 19
+    expected_tests = ['sample_test.app', 'test_a.check_1.1a',
+                      'test_a.check_1.1c',
+                      'test_a.check_1.2a', 'test_a.check_1.2b',
+                      'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a',
+                      'test_a.check_1.unit_1b', 'test_a.check_2.1a',
+                      'test_a.check_2.1c', 'test_a.check_2.2a',
+                      'test_a.check_2.2b', 'test_a.check_2.Unit_1c',
+                      'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
+                      'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
+                      'test_c.check_2', 'test_d.check_1.unit_1a',
+                      'test_d.check_1.unit_1b']
+    assert len(class_testsuite.get_all_tests()) == len(expected_tests)
     assert sorted(class_testsuite.get_all_tests()) == sorted(expected_tests)
 
 def test_get_platforms(class_testsuite, platforms_list):

subsys/testsuite/include/ztest.ld

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+SECTIONS
+{
+	.data.ztest_suite_node_area : ALIGN(4)
+	{
+		_ztest_suite_node_list_start = .;
+		KEEP(*(SORT_BY_NAME(._ztest_suite_node.static.*)))
+		_ztest_suite_node_list_end = .;
+	}
+}
+INSERT AFTER .data;

subsys/testsuite/unittest.cmake

@@ -62,6 +62,10 @@ target_compile_options(testbinary PRIVATE
   $<$<COMPILE_LANGUAGE:ASM>:${EXTRA_AFLAGS_AS_LIST}>
   )
 
+target_link_options(testbinary PRIVATE
+  -T "${ZEPHYR_BASE}/subsys/testsuite/include/ztest.ld"
+  )
+
 target_link_libraries(testbinary PRIVATE
   ${EXTRA_LDFLAGS_AS_LIST}
   )