ztest: Add config to shuffle test order

Enable ZTEST_DO_THE_SHUFFLE to shuffle the order tests are ran.
Additional configs ZTEST_DO_THE_SHUFFLE_SUITE_REPEAT_COUNT
ZTEST_DO_THE_SHUFFLE_TEST_REPEAT_COUNT specify the number of times
the test or suite is executed.

Signed-off-by: Al Semjonovs <[email protected]>

Author: asemjonovs

doc/develop/test/twister.rst

@@ -750,3 +750,12 @@ An example of entries in a quarantine yaml::
       platforms:
         - qemu_cortex_m3
         - native_posix
+
+Running in Tests in Random Order
+********************************
+Enable ZTEST framework's :kconfig:option:`CONFIG_ZTEST_SHUFFLE` config option to
+run your tests in random order.  This can be beneficial for identifying
+dependencies between test cases.  For native_posix platforms, you can provide
+the seed to the random number generator by providing ``-seed=value`` as an
+argument to twister. See :ref:`Shuffling Test Sequence <ztest_shuffle>` for more
+details.

doc/develop/test/ztest.rst

@@ -508,3 +508,17 @@ These will be surrounded by blocks such as::
         #ifndef SOMETHING
         #define SOMETHING <default implementation>
         #endif /* SOMETHING */
+
+.. _ztest_shuffle:
+
+Shuffling Test Sequence
+***********************
+By default the tests are sorted and ran in alphanumerical order.  Test cases may
+be dependent on this sequence. Enable `ZTEST_SHUFFLE` to randomize the order. The
+output from the test will display the seed for failed tests.  For native posix
+builds you can provide the seed as an argument to twister with `--seed`
+
+Static configuration of ZTEST_SHUFFLE contains:
+
+ - :c:macro:`ZTEST_SHUFFLE_SUITE_REPEAT_COUNT` - Number of iterations the test suite will run.
+ - :c:macro:`ZTEST_SHUFFLE_TEST_REPEAT_COUNT` - Number of iterations the test will run.

subsys/testsuite/ztest/Kconfig

@@ -97,6 +97,30 @@ config ZTEST_RULE_1CPU
 
 endmenu
 
+config ZTEST_SHUFFLE
+	bool "Shuffle the order of tests and suites"
+	select TEST_RANDOM_GENERATOR if !ENTROPY_HAS_DRIVER
+	help
+	  This rule will shuffle the order of tests and test suites.
+
+if ZTEST_SHUFFLE
+config ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
+	int "Number of iterations the test suite will run"
+	default 3
+	help
+	  This rule will execute a test suite N number of times. The tests
+	  per suite will be shuffled on each iteration.  The test order will likely
+	  be different per iteration.
+
+config ZTEST_SHUFFLE_TEST_REPEAT_COUNT
+	int "Number of iterations the test will run"
+	default 3
+	help
+	  This rule will execute a test N number of times.  The test order will
+	  likely be different per iteration.
+
+endif #ZTEST_SHUFFLE
+
 endif # ZTEST_NEW_API
 
 endif # ZTEST

subsys/testsuite/ztest/src/ztest_new.c

@@ -21,6 +21,17 @@ static struct k_thread ztest_thread;
 #include <unistd.h>
 #endif
 
+#ifdef CONFIG_ZTEST_SHUFFLE
+#include <random/rand32.h>
+#include <stdlib.h>
+#include <time.h>
+#define NUM_ITER_PER_SUITE CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
+#define NUM_ITER_PER_TEST CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT
+#else
+#define NUM_ITER_PER_SUITE 1
+#define NUM_ITER_PER_TEST 1
+#endif
+
 /* ZTEST_DMEM and ZTEST_BMEM are used for the application shared memory test  */
 
 /**
@@ -495,12 +506,34 @@ struct ztest_unit_test *ztest_get_next_test(const char *suite, struct ztest_unit
 	return NULL;
 }
 
+#ifdef CONFIG_ZTEST_SHUFFLE
+static void z_ztest_shuffle(void *array, size_t num_items, void *tmp, size_t elem_size)
+{
+	char *arr = array;
+
+	for (int i = num_items - 1; i > 0; i--) {
+		int j = sys_rand32_get() % (i + 1);
+
+		if (i != j) {
+			memcpy(tmp, arr + (j * elem_size), elem_size);
+			memcpy(arr + (j * elem_size), arr + (i * elem_size), elem_size);
+			memcpy(arr + (i * elem_size), tmp, elem_size);
+		}
+	}
+
+}
+#endif /* CONFIG_ZTEST_SHUFFLE */
+
 static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
 {
 	struct ztest_unit_test *test = NULL;
 	void *data = NULL;
 	int fail = 0;
 
+#ifdef CONFIG_ZTEST_SHUFFLE
+	struct ztest_unit_test tmp;
+#endif
+
 	if (test_status < 0) {
 		return test_status;
 	}
@@ -517,21 +550,33 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
 	if (suite->setup != NULL) {
 		data = suite->setup();
 	}
-	while ((test = ztest_get_next_test(suite->name, test)) != NULL) {
-		fail += run_test(suite, test, data);
 
-		if (fail && FAIL_FAST) {
-			break;
+	for (int i = 0; i < NUM_ITER_PER_TEST; i++) {
+		fail = 0;
+
+#ifdef CONFIG_ZTEST_SHUFFLE
+		z_ztest_shuffle(_ztest_unit_test_list_start,
+				_ztest_unit_test_list_end - _ztest_unit_test_list_start, &tmp,
+				sizeof(struct ztest_unit_test));
+#endif
+
+		while (((test = ztest_get_next_test(suite->name, test)) != NULL)) {
+			fail += run_test(suite, test, data);
+
+			if (fail && FAIL_FAST) {
+				break;
+			}
 		}
+
+		test_status = (test_status || fail) ? 1 : 0;
 	}
+
 	TC_SUITE_END(suite->name, (fail > 0 ? TC_FAIL : TC_PASS));
 	phase = TEST_PHASE_TEARDOWN;
 	if (suite->teardown != NULL) {
 		suite->teardown(data);
 	}
 
-	test_status = (test_status || fail) ? 1 : 0;
-
 	return fail;
 }
 
@@ -558,6 +603,14 @@ int ztest_run_test_suites(const void *state)
 	struct ztest_suite_node *ptr;
 	int count = 0;
 
+#ifdef CONFIG_ZTEST_SHUFFLE
+	struct ztest_suite_node tmp;
+
+	z_ztest_shuffle(_ztest_suite_node_list_start,
+			_ztest_suite_node_list_end - _ztest_suite_node_list_start, &tmp,
+			sizeof(struct ztest_suite_node));
+#endif
+
 	for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) {
 		struct ztest_suite_stats *stats = &ptr->stats;
 		bool should_run = true;
@@ -569,14 +622,16 @@ int ztest_run_test_suites(const void *state)
 			should_run = stats->run_count == 0;
 		}
 
-		if (should_run) {
-			int fail = z_ztest_run_test_suite_ptr(ptr);
+		for (int i = 0; i < NUM_ITER_PER_SUITE; i++) {
+			if (should_run) {
+				int fail = z_ztest_run_test_suite_ptr(ptr);
 
-			count++;
-			stats->run_count++;
-			stats->fail_count += (fail != 0) ? 1 : 0;
-		} else {
-			stats->skip_count++;
+				count++;
+				stats->run_count++;
+				stats->fail_count += (fail != 0) ? 1 : 0;
+			} else {
+				stats->skip_count++;
+			}
 		}
 	}
 

tests/ztest/base/prj_verbose_0.conf

@@ -1,3 +1,11 @@
 CONFIG_ZTEST=y
 CONFIG_ZTEST_NEW_API=y
 CONFIG_ZTEST_ASSERT_VERBOSE=0
+
+CONFIG_ENTROPY_GENERATOR=y
+CONFIG_TEST_RANDOM_GENERATOR=y
+CONFIG_TIMER_RANDOM_GENERATOR=y
+
+CONFIG_ZTEST_SHUFFLE=y
+CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT=2
+CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=2

tests/ztest/base/src/main.c

@@ -76,6 +76,7 @@ enum rule_state {
 
 struct rules_tests_fixture {
 	enum rule_state state;
+	int  run_count;
 };
 
 static struct rules_tests_fixture rule_tests_fixture;
@@ -88,7 +89,11 @@ static void rule_before_each(const struct ztest_unit_test *test, void *data)
 
 		zassert_equal_ptr(&rule_tests_fixture, data,
 				  "Data expected to point to rule_state");
-		zassert_equal(fixture->state, RULE_STATE_SETUP, "Unexpected state");
+		if (fixture->run_count == 0) {
+			zassert_equal(fixture->state, RULE_STATE_SETUP, "Unexpected state");
+		} else {
+			zassert_equal(fixture->state, RULE_STATE_AFTER_EACH, "Unexpected state");
+		}
 		fixture->state = RULE_STATE_BEFORE_EACH;
 	}
 }
@@ -109,6 +114,7 @@ static void rule_after_each(const struct ztest_unit_test *test, void *data)
 static void *rule_test_setup(void)
 {
 	rule_tests_fixture.state = RULE_STATE_SETUP;
+	rule_tests_fixture.run_count = 0;
 	return &rule_tests_fixture;
 }
 
@@ -121,6 +127,9 @@ static void rule_test_teardown(void *data)
 	 * after_each function was called.
 	 */
 	zassert_equal(fixture->state, RULE_STATE_AFTER_EACH, "Unexpected state");
+#ifdef CONFIG_ZTEST_SHUFFLE
+	zassert_equal(fixture->run_count, CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT, NULL);
+#endif
 }
 
 ZTEST_RULE(verify_before_after_rule, rule_before_each, rule_after_each);
@@ -131,4 +140,5 @@ ZTEST_F(rules_tests, test_rules_before_after)
 {
 	zassert_equal(this->state, RULE_STATE_BEFORE_EACH, "Unexpected state");
 	this->state = RULE_STATE_TEST;
+	this->run_count++;
 }