| 
3 | 3 | #define _ASM_RQSPINLOCK_H  | 
4 | 4 | 
 
  | 
5 | 5 | #include <asm/barrier.h>  | 
6 |  | - | 
7 |  | -/*  | 
8 |  | - * Hardcode res_smp_cond_load_acquire implementations for arm64 to a custom  | 
9 |  | - * version based on [0]. In rqspinlock code, our conditional expression involves  | 
10 |  | - * checking the value _and_ additionally a timeout. However, on arm64, the  | 
11 |  | - * WFE-based implementation may never spin again if no stores occur to the  | 
12 |  | - * locked byte in the lock word. As such, we may be stuck forever if  | 
13 |  | - * event-stream based unblocking is not available on the platform for WFE spin  | 
14 |  | - * loops (arch_timer_evtstrm_available).  | 
15 |  | - *  | 
16 |  | - * Once support for smp_cond_load_acquire_timewait [0] lands, we can drop this  | 
17 |  | - * copy-paste.  | 
18 |  | - *  | 
19 |  | - * While we rely on the implementation to amortize the cost of sampling  | 
20 |  | - * cond_expr for us, it will not happen when event stream support is  | 
21 |  | - * unavailable, time_expr check is amortized. This is not the common case, and  | 
22 |  | - * it would be difficult to fit our logic in the time_expr_ns >= time_limit_ns  | 
23 |  | - * comparison, hence just let it be. In case of event-stream, the loop is woken  | 
24 |  | - * up at microsecond granularity.  | 
25 |  | - *  | 
26 |  | - * [0]: https://lore.kernel.org/lkml/[email protected]  | 
27 |  | - */  | 
28 |  | - | 
29 |  | -#ifndef smp_cond_load_acquire_timewait  | 
30 |  | - | 
31 |  | -#define smp_cond_time_check_count	200  | 
32 |  | - | 
33 |  | -#define __smp_cond_load_relaxed_spinwait(ptr, cond_expr, time_expr_ns,	\  | 
34 |  | -					 time_limit_ns) ({		\  | 
35 |  | -	typeof(ptr) __PTR = (ptr);					\  | 
36 |  | -	__unqual_scalar_typeof(*ptr) VAL;				\  | 
37 |  | -	unsigned int __count = 0;					\  | 
38 |  | -	for (;;) {							\  | 
39 |  | -		VAL = READ_ONCE(*__PTR);				\  | 
40 |  | -		if (cond_expr)						\  | 
41 |  | -			break;						\  | 
42 |  | -		cpu_relax();						\  | 
43 |  | -		if (__count++ < smp_cond_time_check_count)		\  | 
44 |  | -			continue;					\  | 
45 |  | -		if ((time_expr_ns) >= (time_limit_ns))			\  | 
46 |  | -			break;						\  | 
47 |  | -		__count = 0;						\  | 
48 |  | -	}								\  | 
49 |  | -	(typeof(*ptr))VAL;						\  | 
50 |  | -})  | 
51 |  | - | 
52 |  | -#define __smp_cond_load_acquire_timewait(ptr, cond_expr,		\  | 
53 |  | -					 time_expr_ns, time_limit_ns)	\  | 
54 |  | -({									\  | 
55 |  | -	typeof(ptr) __PTR = (ptr);					\  | 
56 |  | -	__unqual_scalar_typeof(*ptr) VAL;				\  | 
57 |  | -	for (;;) {							\  | 
58 |  | -		VAL = smp_load_acquire(__PTR);				\  | 
59 |  | -		if (cond_expr)						\  | 
60 |  | -			break;						\  | 
61 |  | -		__cmpwait_relaxed(__PTR, VAL);				\  | 
62 |  | -		if ((time_expr_ns) >= (time_limit_ns))			\  | 
63 |  | -			break;						\  | 
64 |  | -	}								\  | 
65 |  | -	(typeof(*ptr))VAL;						\  | 
66 |  | -})  | 
67 |  | - | 
68 |  | -#define smp_cond_load_acquire_timewait(ptr, cond_expr,			\  | 
69 |  | -				      time_expr_ns, time_limit_ns)	\  | 
70 |  | -({									\  | 
71 |  | -	__unqual_scalar_typeof(*ptr) _val;				\  | 
72 |  | -	int __wfe = arch_timer_evtstrm_available();			\  | 
73 |  | -									\  | 
74 |  | -	if (likely(__wfe)) {						\  | 
75 |  | -		_val = __smp_cond_load_acquire_timewait(ptr, cond_expr,	\  | 
76 |  | -							time_expr_ns,	\  | 
77 |  | -							time_limit_ns);	\  | 
78 |  | -	} else {							\  | 
79 |  | -		_val = __smp_cond_load_relaxed_spinwait(ptr, cond_expr,	\  | 
80 |  | -							time_expr_ns,	\  | 
81 |  | -							time_limit_ns);	\  | 
82 |  | -		smp_acquire__after_ctrl_dep();				\  | 
83 |  | -	}								\  | 
84 |  | -	(typeof(*ptr))_val;						\  | 
85 |  | -})  | 
86 |  | - | 
87 |  | -#endif  | 
88 |  | - | 
89 |  | -#define res_smp_cond_load_acquire(v, c) smp_cond_load_acquire_timewait(v, c, 0, 1)  | 
90 |  | - | 
91 | 6 | #include <asm-generic/rqspinlock.h>  | 
92 | 7 | 
 
  | 
93 | 8 | #endif /* _ASM_RQSPINLOCK_H */  | 
0 commit comments