ztest: add a weak implementation of test_main()

Introduce a weak implementation of test_main() which calls:
* ztest_run_registered_test_suites(NULL);
* ztest_verify_all_registered_test_suites_ran();

This will attempt to run all registered test suites and verify that
they each ran.

Signed-off-by: Yuval Peress <[email protected]>

Author: Yuval Peress

scripts/pylib/twister/twisterlib.py

@@ -1589,16 +1589,20 @@ class ScanPathResult:
         has_run_registered_test_suites   Whether or not the path contained at
                                          least one call to
                                          ztest_run_registered_test_suites.
+        has_test_main                    Whether or not the path contains a
+                                         definition of test_main(void)
     """
     def __init__(self,
                  matches: List[str] = None,
                  warnings: str = None,
                  has_registered_test_suites: bool = False,
-                 has_run_registered_test_suites: bool = False):
+                 has_run_registered_test_suites: bool = False,
+                 has_test_main: bool = False):
         self.matches = matches
         self.warnings = warnings
         self.has_registered_test_suites = has_registered_test_suites
         self.has_run_registered_test_suites = has_run_registered_test_suites
+        self.has_test_main = has_test_main
 
     def __eq__(self, other):
         if not isinstance(other, ScanPathResult):
@@ -1608,7 +1612,8 @@ def __eq__(self, other):
                 (self.has_registered_test_suites ==
                  other.has_registered_test_suites) and
                 (self.has_run_registered_test_suites ==
-                 other.has_run_registered_test_suites))
+                 other.has_run_registered_test_suites) and
+                self.has_test_main == other.has_test_main)
 
 
 class TestCase(DisablePyTestCollectionMixin):
@@ -1701,6 +1706,15 @@ def scan_file(inf_name):
             br"^\s*ztest_register_test_suite"
             br"\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
             re.MULTILINE)
+        # Checks if the file contains a definition of "void test_main(void)"
+        # Since ztest provides a plain test_main implementation it is OK to:
+        # 1. register test suites and not call the run function iff the test
+        #    doesn't have a custom test_main.
+        # 2. register test suites and a custom test_main definition iff the test
+        #    also calls ztest_run_registered_test_suites.
+        test_main_regex = re.compile(
+            br"^\s*void\s+test_main\(void\)",
+            re.MULTILINE)
         stc_regex = re.compile(
             br"""^\s*  # empy space at the beginning is ok
             # catch the case where it is declared in the same sentence, e.g:
@@ -1733,6 +1747,7 @@ def scan_file(inf_name):
         warnings = None
         has_registered_test_suites = False
         has_run_registered_test_suites = False
+        has_test_main = False
 
         with open(inf_name) as inf:
             if os.name == 'nt':
@@ -1750,6 +1765,8 @@ def scan_file(inf_name):
                     has_registered_test_suites = True
                 if registered_suite_run_regex.search(main_c):
                     has_run_registered_test_suites = True
+                if test_main_regex.search(main_c):
+                    has_test_main = True
 
                 if not suite_regex_match and not has_registered_test_suites:
                     # can't find ztest_test_suite, maybe a client, because
@@ -1758,7 +1775,8 @@ def scan_file(inf_name):
                         matches=None,
                         warnings=None,
                         has_registered_test_suites=has_registered_test_suites,
-                        has_run_registered_test_suites=has_run_registered_test_suites)
+                        has_run_registered_test_suites=has_run_registered_test_suites,
+                        has_test_main=has_test_main)
 
                 suite_run_match = suite_run_regex.search(main_c)
                 if suite_regex_match and not suite_run_match:
@@ -1792,12 +1810,14 @@ def scan_file(inf_name):
                     matches=matches,
                     warnings=warnings,
                     has_registered_test_suites=has_registered_test_suites,
-                    has_run_registered_test_suites=has_run_registered_test_suites)
+                    has_run_registered_test_suites=has_run_registered_test_suites,
+                    has_test_main=has_test_main)
 
     def scan_path(self, path):
         subcases = []
         has_registered_test_suites = False
         has_run_registered_test_suites = False
+        has_test_main = False
         for filename in glob.glob(os.path.join(path, "src", "*.c*")):
             try:
                 result: ScanPathResult = self.scan_file(filename)
@@ -1811,6 +1831,8 @@ def scan_path(self, path):
                     has_registered_test_suites = True
                 if result.has_run_registered_test_suites:
                     has_run_registered_test_suites = True
+                if result.has_test_main:
+                    has_test_main = True
             except ValueError as e:
                 logger.error("%s: can't find: %s" % (filename, e))
 
@@ -1824,7 +1846,8 @@ def scan_path(self, path):
             except ValueError as e:
                 logger.error("%s: can't find: %s" % (filename, e))
 
-        if has_registered_test_suites and not has_run_registered_test_suites:
+        if (has_registered_test_suites and has_test_main and
+                not has_run_registered_test_suites):
             warning = \
                 "Found call to 'ztest_register_test_suite()' but no "\
                 "call to 'ztest_run_registered_test_suites()'"

scripts/tests/twister/test_testinstance.py

@@ -123,24 +123,28 @@ def test_get_unique_exception(testcase_root, workdir, name, exception):
                   'test_test_aa',
                   'user', 'last'],
          has_registered_test_suites=False,
-         has_run_registered_test_suites=False)),
+         has_run_registered_test_suites=False,
+         has_test_main=False)),
     ("testcases/tests/test_a/test_ztest_error.c",
      ScanPathResult(
          warnings="Found a test that does not start with test_",
          matches=['1a', '1c', '2a', '2b'],
          has_registered_test_suites=False,
-         has_run_registered_test_suites=False)),
+         has_run_registered_test_suites=False,
+         has_test_main=True)),
     ("testcases/tests/test_a/test_ztest_error_1.c",
      ScanPathResult(
          warnings="found invalid #ifdef, #endif in ztest_test_suite()",
          matches=['unit_1a', 'unit_1b', 'Unit_1c'],
          has_registered_test_suites=False,
-         has_run_registered_test_suites=False)),
+         has_run_registered_test_suites=False,
+         has_test_main=False)),
     ("testcases/tests/test_d/test_ztest_error_register_test_suite.c",
      ScanPathResult(
          warnings=None, matches=['unit_1a', 'unit_1b'],
          has_registered_test_suites=True,
-         has_run_registered_test_suites=False)),
+         has_run_registered_test_suites=False,
+         has_test_main=False)),
 ]
 
 @pytest.mark.parametrize("test_file, expected", TESTDATA_5)

subsys/testsuite/ztest/include/ztest_test.h

@@ -92,6 +92,17 @@ extern struct ztest_suite_node _ztest_suite_node_list_end[];
  */
 int ztest_run_registered_test_suites(const void *state);
 
+/**
+ * @brief Fails the test if any of the registered tests did not run.
+ *
+ * When registering test suites, a pragma function can be provided to determine WHEN the test should
+ * run. It is possible that a test suite could be registered but the pragma always prevents it from
+ * running. In cases where a test should make sure that ALL suites ran at least once, this function
+ * may be called at the end of test_main(). It will cause the test to fail if any suite was
+ * registered but never ran.
+ */
+void ztest_verify_all_registered_test_suites_ran(void);
+
 /**
  * @brief Run a test suite.
  *

subsys/testsuite/ztest/src/ztest.c

@@ -465,6 +465,29 @@ int ztest_run_registered_test_suites(const void *state)
 	return count;
 }
 
+void ztest_verify_all_registered_test_suites_ran(void)
+{
+	bool all_tests_run = true;
+	struct ztest_suite_node *ptr;
+
+	for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) {
+		if (ptr->stats.run_count < 1) {
+			PRINT("ERROR: Test '%s' did not run.\n", ptr->name);
+			all_tests_run = false;
+		}
+	}
+
+	if (!all_tests_run) {
+		test_status = 1;
+	}
+}
+
+void __weak test_main(void)
+{
+	ztest_run_registered_test_suites(NULL);
+	ztest_verify_all_registered_test_suites_ran();
+}
+
 #ifndef KERNEL
 int main(void)
 {

tests/ztest/automain/CMakeLists.txt

@@ -0,0 +1,9 @@
+# Copyright 2021 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(automain)
+
+FILE(GLOB app_sources src/*.c)
+target_sources(app PRIVATE ${app_sources})

tests/ztest/automain/prj.conf

@@ -0,0 +1 @@
+CONFIG_ZTEST=y

tests/ztest/automain/src/test_feature.c

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <ztest.h>
+
+/**
+ * @brief A stub unit test.
+ *
+ * This is a mock unit test that is expected to run. Note that it is not added directly to a test
+ * suite and run via ztest_run_test_suite(). It is instead registered below using
+ * ztest_register_test_suite and will be run by the automatically generated test_main() function.
+ */
+static void test_stub(void)
+{
+}
+
+ztest_register_test_suite(test_suite, NULL,
+			  ztest_unit_test(test_stub));

tests/ztest/automain/testcase.yaml

@@ -0,0 +1,5 @@
+tests:
+  testing.ztest.automain:
+    tags: test_framework
+    integration_platforms:
+      - native_posix