Revert "ztest: Fix test statistics reporting"

This reverts commit 2af5ac8.

Failures are not being captured correctly now:

 - PASS - [test_c_lib.test_strtoul] duration = 0.001 seconds
 - PASS - [test_c_lib.test_strxspn] duration = 0.001 seconds
 - FAIL - [test_c_lib.test_that_fails] duration = 0.002 seconds
 - PASS - [test_c_lib.test_time] duration = 0.001 seconds
 - PASS - [test_c_lib.test_tolower_toupper] duration = 0.001 seconds

------ TESTSUITE SUMMARY END ------

===================================================================
PROJECT EXECUTION SUCCESSFUL

Signed-off-by: Anas Nashif <[email protected]>

Author: nashif

subsys/testsuite/include/zephyr/tc_util.h

@@ -67,7 +67,6 @@
 #define TC_PASS 0
 #define TC_FAIL 1
 #define TC_SKIP 2
-#define TC_FLAKY 3
 
 #ifndef TC_PASS_STR
 #define TC_PASS_STR "PASS"
@@ -78,9 +77,6 @@
 #ifndef TC_SKIP_STR
 #define TC_SKIP_STR "SKIP"
 #endif
-#ifndef TC_FLAKY_STR
-#define TC_FLAKY_STR "FLAKY"
-#endif
 
 static inline const char *TC_RESULT_TO_STR(int result)
 {
@@ -91,8 +87,6 @@ static inline const char *TC_RESULT_TO_STR(int result)
 		return TC_FAIL_STR;
 	case TC_SKIP:
 		return TC_SKIP_STR;
-	case TC_FLAKY:
-		return TC_FLAKY_STR;
 	default:
 		return "?";
 	}

subsys/testsuite/ztest/src/ztest_new.c

@@ -675,7 +675,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
 	int fail = 0;
 	int tc_result = TC_PASS;
 
-	if (test_status != ZTEST_STATUS_OK) {
+	if (test_status < 0) {
 		return test_status;
 	}
 
@@ -766,7 +766,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
 		}
 #endif
 
-		if (test_status == ZTEST_STATUS_OK && fail != 0 && FAIL_FAST) {
+		if (test_status == ZTEST_STATUS_OK && fail != 0) {
 			test_status = ZTEST_STATUS_HAS_FAILURE;
 		}
 	}
@@ -882,28 +882,15 @@ static void __ztest_show_suite_summary_verbose(struct ztest_suite_node *suite)
 			tc_result = TC_SKIP;
 		} else if (test->stats->pass_count == test->stats->run_count) {
 			tc_result = TC_PASS;
-		} else if (test->stats->pass_count == 0) {
-			tc_result = TC_FAIL;
 		} else {
-			tc_result = TC_FLAKY;
+			tc_result = TC_FAIL;
 		}
 
-		if (tc_result == TC_FLAKY) {
-			TC_SUMMARY_PRINT(" - %s - [%s.%s] - (Failed %d of %d attempts)"
-					 " - duration = %u.%03u seconds\n",
-					TC_RESULT_TO_STR(tc_result),
-					test->test_suite_name, test->name,
-					test->stats->run_count - test->stats->pass_count,
-					test->stats->run_count,
-					test->stats->duration_worst_ms / 1000,
-					test->stats->duration_worst_ms % 1000);
-		} else {
-			TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n",
-					TC_RESULT_TO_STR(tc_result),
-					test->test_suite_name, test->name,
-					test->stats->duration_worst_ms / 1000,
-					test->stats->duration_worst_ms % 1000);
-		}
+		TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n",
+				TC_RESULT_TO_STR(tc_result),
+				test->test_suite_name, test->name,
+				test->stats->duration_worst_ms / 1000,
+				test->stats->duration_worst_ms % 1000);
 
 		if (flush_frequency % 3 == 0) {
 			/** Reduce the flush frequencey a bit to speed up the output */
@@ -943,6 +930,7 @@ static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *stat
 
 	for (int i = 0; i < NUM_ITER_PER_SUITE; i++) {
 		if (ztest_api.should_suite_run(state, ptr)) {
+			__ztest_init_unit_test_result_for_suite(ptr);
 			int fail = z_ztest_run_test_suite_ptr(ptr);
 
 			count++;
@@ -970,9 +958,6 @@ int z_impl_ztest_run_test_suites(const void *state)
 	memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *));
 	z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start,
 			ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node));
-	for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
-		__ztest_init_unit_test_result_for_suite(suites_to_run[i]);
-	}
 	for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
 		count += __ztest_run_test_suite(suites_to_run[i], state);
 		/* Stop running tests if we have a critical error or if we have a failure and
@@ -986,7 +971,6 @@ int z_impl_ztest_run_test_suites(const void *state)
 #else
 	for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start;
 	     ptr < _ztest_suite_node_list_end; ++ptr) {
-		__ztest_init_unit_test_result_for_suite(ptr);
 		count += __ztest_run_test_suite(ptr, state);
 		/* Stop running tests if we have a critical error or if we have a failure and
 		 * FAIL_FAST was set