Skip to content

Commit 02205a6

Browse files
author
Christopher Friedt
committed
tests: posix: stress test for pthread_create and pthread_join
Recently, a race condition was discovered in `pthread_create()` and `pthread_join()` that would trigger an assertion in `kernel/sched.c` of the form below. ``` aborted _current back from dead ``` This was mainly observed in Qemu. Unfortunately, Qemu SMP platforms exhibit a real and measurable "scheduler noise" which makes testing rather difficult. Signed-off-by: Christopher Friedt <[email protected]>
1 parent 23d36a7 commit 02205a6

File tree

5 files changed

+308
-0
lines changed

5 files changed

+308
-0
lines changed

‎tests/posix/pthread_pressure/CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Copyright (c) 2023, Meta
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
cmake_minimum_required(VERSION 3.20.0)
6+
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
7+
project(pthread_pressure)
8+
9+
FILE(GLOB app_sources src/*.c)
10+
target_sources(app PRIVATE ${app_sources})

‎tests/posix/pthread_pressure/Kconfig‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Copyright (c) 2023, Meta
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
source "Kconfig.zephyr"
6+
7+
config TEST_NUM_CPUS
8+
int "Number of CPUs to use in parallel"
9+
range 1 MP_NUM_CPUS
10+
default MP_NUM_CPUS
11+
help
12+
The number of parallel threads to run during the test. The test
13+
thread itself yields so that all cores have some probability of
14+
causing racey behaviour.
15+
16+
config TEST_DURATION_S
17+
int "Number of seconds to run the test"
18+
range 1 21600
19+
default 5
20+
help
21+
Duration for the test, in seconds. The range has a reblatively high
22+
upper bound because we should expect that pthread_create() and
23+
pthread_join() are stable enough to run for an arbitrarily long
24+
period of time without encountering any race conditions.
25+
26+
Some exceptions apply, notably Qemu SMP targets.
27+
28+
config TEST_DELAY_US
29+
int "Microseconds to delay between pthread join and create"
30+
default 0
31+
help
32+
If there is a race condition, a value of zero here should
33+
cause a crash.
34+
35+
config TEST_STACK_SIZE
36+
int "Size of each thread stack in this test"
37+
default 512 if !64_BIT
38+
default 1024 if 64_BIT
39+
help
40+
The minimal stack size required to run a no-op thread.
41+
42+
config TEST_KTHREADS
43+
bool "Test k_threads"
44+
default y
45+
help
46+
Run tests for k_threads
47+
48+
config TEST_PTHREADS
49+
bool "Test pthreads"
50+
default y
51+
help
52+
Run tests for pthreads
53+
54+
config TEST_EXTRA_ASSERTIONS
55+
bool "Add extra assertions into the hot path"
56+
default y
57+
help
58+
On Qemu SMP targets, this can potentially lead to "scheduler noise"
59+
leaking in from the host system, which can cause the test to fail.

‎tests/posix/pthread_pressure/prj.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
CONFIG_ZTEST=y
2+
CONFIG_ZTEST_NEW_API=y
3+
CONFIG_POSIX_API=y

‎tests/posix/pthread_pressure/src/main.c‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
/*
2+
* Copyright (c) 2023, Meta
3+
*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
#include <pthread.h>
8+
9+
#include <zephyr/sys/util.h>
10+
#include <zephyr/ztest.h>
11+
12+
#define STACK_SIZE K_THREAD_STACK_LEN(CONFIG_TEST_STACK_SIZE)
13+
14+
/* update interval for printing stats */
15+
#if CONFIG_TEST_DURATION_S >= 60
16+
#define UPDATE_INTERVAL_S 10
17+
#elif CONFIG_TEST_DURATION_S >= 30
18+
#define UPDATE_INTERVAL_S 5
19+
#else
20+
#define UPDATE_INTERVAL_S 1
21+
#endif
22+
23+
/* 32 threads is mainly a limitation of find_lsb_set() */
24+
#define NUM_THREADS MIN(32, MIN(CONFIG_TEST_NUM_CPUS, CONFIG_MAX_PTHREAD_COUNT))
25+
26+
typedef int (*create_fn)(int i);
27+
typedef int (*join_fn)(int i);
28+
29+
static void *setup(void);
30+
static void before(void *fixture);
31+
32+
/* bitmask of available threads */
33+
static bool alive[NUM_THREADS];
34+
35+
/* array of thread stacks */
36+
static K_THREAD_STACK_ARRAY_DEFINE(thread_stacks, NUM_THREADS, STACK_SIZE);
37+
38+
static struct k_thread k_threads[NUM_THREADS];
39+
static size_t counters[NUM_THREADS];
40+
41+
static void print_stats(uint64_t now, uint64_t end)
42+
{
43+
printk("now (ms): %llu end (ms): %llu\n", now, end);
44+
for (int i = 0; i < NUM_THREADS; ++i) {
45+
printk("Thread %d created and joined %zu times\n", i, counters[i]);
46+
}
47+
}
48+
49+
static void test_create_join_common(const char *tag, create_fn create, join_fn join)
50+
{
51+
int i;
52+
int ret;
53+
uint64_t now_ms = k_uptime_get();
54+
const uint64_t end_ms = now_ms + MSEC_PER_SEC * CONFIG_TEST_DURATION_S;
55+
uint64_t update_ms = now_ms + MSEC_PER_SEC * UPDATE_INTERVAL_S;
56+
57+
printk("BOARD: %s\n", CONFIG_BOARD);
58+
printk("NUM_THREADS: %u\n", NUM_THREADS);
59+
printk("TEST_NUM_CPUS: %u\n", CONFIG_TEST_NUM_CPUS);
60+
printk("TEST_DURATION_S: %u\n", CONFIG_TEST_DURATION_S);
61+
printk("TEST_DELAY_US: %u\n", CONFIG_TEST_DELAY_US);
62+
63+
for (i = 0; i < NUM_THREADS; ++i) {
64+
/* spawn thread i */
65+
ret = create(i);
66+
if (IS_ENABLED(CONFIG_TEST_EXTRA_ASSERTIONS)) {
67+
zassert_ok(ret, "%s_create(%d)[%zu] failed: %d", tag, i, counters[i], ret);
68+
}
69+
}
70+
71+
do {
72+
if (!IS_ENABLED(CONFIG_SMP)) {
73+
/* allow the test thread to be swapped-out */
74+
k_yield();
75+
}
76+
77+
for (i = 0; i < NUM_THREADS; ++i) {
78+
if (alive[i]) {
79+
ret = join(i);
80+
if (IS_ENABLED(CONFIG_TEST_EXTRA_ASSERTIONS)) {
81+
zassert_ok(ret, "%s_join(%d)[%zu] failed: %d", tag, i,
82+
counters[i], ret);
83+
}
84+
alive[i] = false;
85+
86+
/* update counter i after each (create,join) pair */
87+
++counters[i];
88+
89+
if (IS_ENABLED(CONFIG_TEST_DELAY_US)) {
90+
/* success with 0 delay means we are ~raceless */
91+
k_busy_wait(CONFIG_TEST_DELAY_US);
92+
}
93+
94+
/* re-spawn thread i */
95+
ret = create(i);
96+
if (IS_ENABLED(CONFIG_TEST_EXTRA_ASSERTIONS)) {
97+
zassert_ok(ret, "%s_create(%d)[%zu] failed: %d", tag, i,
98+
counters[i], ret);
99+
}
100+
}
101+
}
102+
103+
/* are we there yet? */
104+
now_ms = k_uptime_get();
105+
106+
/* dump some stats periodically */
107+
if (now_ms > update_ms) {
108+
update_ms += MSEC_PER_SEC * UPDATE_INTERVAL_S;
109+
110+
/* at this point, we should have seen many context switches */
111+
for (i = 0; i < NUM_THREADS; ++i) {
112+
if (IS_ENABLED(CONFIG_TEST_EXTRA_ASSERTIONS)) {
113+
zassert_true(counters[i] > 0, "%s %d was never scheduled",
114+
tag, i);
115+
}
116+
}
117+
118+
print_stats(now_ms, end_ms);
119+
}
120+
} while (end_ms > now_ms);
121+
122+
print_stats(now_ms, end_ms);
123+
}
124+
125+
/*
126+
* Wrappers for k_threads
127+
*/
128+
129+
static void k_thread_fun(void *arg1, void *arg2, void *arg3)
130+
{
131+
int i = POINTER_TO_INT(arg1);
132+
133+
alive[i] = true;
134+
}
135+
136+
static int k_thread_create_wrapper(int i)
137+
{
138+
k_thread_create(&k_threads[i], thread_stacks[i], STACK_SIZE, k_thread_fun,
139+
INT_TO_POINTER(i), NULL, NULL, K_HIGHEST_APPLICATION_THREAD_PRIO, 0,
140+
K_NO_WAIT);
141+
142+
return 0;
143+
}
144+
145+
static int k_thread_join_wrapper(int i)
146+
{
147+
return k_thread_join(&k_threads[i], K_FOREVER);
148+
}
149+
150+
ZTEST(pthread_pressure, test_k_thread_create_join)
151+
{
152+
if (IS_ENABLED(CONFIG_TEST_KTHREADS)) {
153+
test_create_join_common("k_thread", k_thread_create_wrapper, k_thread_join_wrapper);
154+
} else {
155+
ztest_test_skip();
156+
}
157+
}
158+
159+
/*
160+
* Wrappers for pthreads
161+
*/
162+
163+
static pthread_t pthreads[NUM_THREADS];
164+
static pthread_attr_t pthread_attrs[NUM_THREADS];
165+
166+
static void *pthread_fun(void *arg)
167+
{
168+
k_thread_fun(arg, NULL, NULL);
169+
return NULL;
170+
}
171+
172+
static int pthread_create_wrapper(int i)
173+
{
174+
return pthread_create(&pthreads[i], &pthread_attrs[i], pthread_fun, INT_TO_POINTER(i));
175+
}
176+
177+
static int pthread_join_wrapper(int i)
178+
{
179+
return pthread_join(pthreads[i], NULL);
180+
}
181+
182+
ZTEST(pthread_pressure, test_pthread_create_join)
183+
{
184+
if (IS_ENABLED(CONFIG_TEST_PTHREADS)) {
185+
test_create_join_common("pthread", pthread_create_wrapper, pthread_join_wrapper);
186+
} else {
187+
ztest_test_skip();
188+
}
189+
}
190+
191+
/*
192+
* Test suite / fixture
193+
*/
194+
195+
ZTEST_SUITE(pthread_pressure, NULL, setup, before, NULL, NULL);
196+
197+
static void *setup(void)
198+
{
199+
if (IS_ENABLED(CONFIG_TEST_PTHREADS)) {
200+
const struct sched_param param = {
201+
.sched_priority = sched_get_priority_max(SCHED_FIFO),
202+
};
203+
204+
/* setup pthread stacks */
205+
for (int i = 0; i < NUM_THREADS; ++i) {
206+
zassert_ok(pthread_attr_init(&pthread_attrs[i]));
207+
zassert_ok(pthread_attr_setstack(&pthread_attrs[i], thread_stacks[i],
208+
STACK_SIZE));
209+
zassert_ok(pthread_attr_setschedpolicy(&pthread_attrs[i], SCHED_FIFO));
210+
zassert_ok(pthread_attr_setschedparam(&pthread_attrs[i], &param));
211+
}
212+
}
213+
214+
return NULL;
215+
}
216+
217+
static void before(void *fixture)
218+
{
219+
ARG_UNUSED(before);
220+
221+
for (int i = 0; i < NUM_THREADS; ++i) {
222+
counters[i] = 0;
223+
}
224+
}

‎tests/posix/pthread_pressure/testcase.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
common:
2+
tags: posix
3+
min_ram: 64
4+
arch_exclude:
5+
- posix
6+
integration_platforms:
7+
- qemu_riscv64_smp
8+
platform_exclude:
9+
- qemu_cortex_a53
10+
- qemu_cortex_a53_smp
11+
tests:
12+
portability.posix.pthread_pressure: {}

0 commit comments

Comments
 (0)