Skip to content

Commit 02219ca

Browse files
paulmckrcupaulmckrcu
committed
Merge branches 'doc.2024.06.06a', 'fixes.2024.07.04a', 'mb.2024.06.28a', 'nocb.2024.06.03a', 'rcu-tasks.2024.06.06a', 'rcutorture.2024.06.06a' and 'srcu.2024.06.18a' into HEAD
doc.2024.06.06a: Documentation updates. fixes.2024.07.04a: Miscellaneous fixes. mb.2024.06.28a: Grace-period memory-barrier redundancy removal. nocb.2024.06.03a: No-CB CPU updates. rcu-tasks.2024.06.06a: RCU-Tasks updates. rcutorture.2024.06.06a: Torture-test updates. srcu.2024.06.18a: SRCU polled-grace-period updates.
7 parents a3fbf86 + 55d4669 + 677ab23 + e4f7805 + 399ced9 + 0ac55d0 + e206f33 commit 02219ca

File tree

22 files changed

+404
-267
lines changed

22 files changed

+404
-267
lines changed

‎Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,9 @@ This case is handled by calls to the strongly ordered
149149
``atomic_add_return()`` read-modify-write atomic operation that
150150
is invoked within ``rcu_dynticks_eqs_enter()`` at idle-entry
151151
time and within ``rcu_dynticks_eqs_exit()`` at idle-exit time.
152-
The grace-period kthread invokes ``rcu_dynticks_snap()`` and
153-
``rcu_dynticks_in_eqs_since()`` (both of which invoke
154-
an ``atomic_add_return()`` of zero) to detect idle CPUs.
152+
The grace-period kthread invokes first ``ct_dynticks_cpu_acquire()``
153+
(preceded by a full memory barrier) and ``rcu_dynticks_in_eqs_since()``
154+
(both of which rely on acquire semantics) to detect idle CPUs.
155155

156156
+-----------------------------------------------------------------------+
157157
| **Quick Quiz**: |

‎Documentation/admin-guide/kernel-parameters.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5018,6 +5018,14 @@
50185018
the ->nocb_bypass queue. The definition of "too
50195019
many" is supplied by this kernel boot parameter.
50205020

5021+
rcutree.nohz_full_patience_delay= [KNL]
5022+
On callback-offloaded (rcu_nocbs) CPUs, avoid
5023+
disturbing RCU unless the grace period has
5024+
reached the specified age in milliseconds.
5025+
Defaults to zero. Large values will be capped
5026+
at five seconds. All values will be rounded down
5027+
to the nearest value representable by jiffies.
5028+
50215029
rcutree.qhimark= [KNL]
50225030
Set threshold of queued RCU callbacks beyond which
50235031
batch limiting is disabled.

‎MAINTAINERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18868,6 +18868,7 @@ M: Neeraj Upadhyay <[email protected]> (kernel/rcu/tasks.h)
1886818868
M: Joel Fernandes <[email protected]>
1886918869
M: Josh Triplett <[email protected]>
1887018870
M: Boqun Feng <[email protected]>
18871+
M: Uladzislau Rezki <[email protected]>
1887118872
R: Steven Rostedt <[email protected]>
1887218873
R: Mathieu Desnoyers <[email protected]>
1887318874
R: Lai Jiangshan <[email protected]>

‎include/linux/rcu_segcblist.h

Lines changed: 38 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -80,36 +80,35 @@ struct rcu_cblist {
8080
* | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED |
8181
* | |
8282
* | Callbacks processed by rcu_core() from softirqs or local |
83-
* | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, |
84-
* | allowing nocb_timer to be armed. |
83+
* | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads. |
8584
* ----------------------------------------------------------------------------
8685
* |
8786
* v
88-
* -----------------------------------
89-
* | |
90-
* v v
91-
* --------------------------------------- ----------------------------------|
92-
* | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | |
93-
* | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | |
94-
* | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | |
95-
* | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP |
96-
* | | | |
97-
* | | | |
98-
* | CB kthread woke up and | | GP kthread woke up and |
99-
* | acknowledged SEGCBLIST_OFFLOADED. | | acknowledged SEGCBLIST_OFFLOADED|
100-
* | Processes callbacks concurrently | | |
101-
* | with rcu_core(), holding | | |
102-
* | nocb_lock. | | |
103-
* --------------------------------------- -----------------------------------
104-
* | |
105-
* -----------------------------------
87+
* ----------------------------------------------------------------------------
88+
* | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED |
89+
* | + unparked CB kthread |
90+
* | |
91+
* | CB kthread got unparked and processes callbacks concurrently with |
92+
* | rcu_core(), holding nocb_lock. |
93+
* ---------------------------------------------------------------------------
94+
* |
95+
* v
96+
* ---------------------------------------------------------------------------|
97+
* | SEGCBLIST_RCU_CORE | |
98+
* | SEGCBLIST_LOCKING | |
99+
* | SEGCBLIST_OFFLOADED | |
100+
* | SEGCBLIST_KTHREAD_GP |
101+
* | + unparked CB kthread |
102+
* | |
103+
* | GP kthread woke up and acknowledged nocb_lock. |
104+
* ---------------------------------------- -----------------------------------
106105
* |
107106
* v
108107
* |--------------------------------------------------------------------------|
109-
* | SEGCBLIST_LOCKING | |
110-
* | SEGCBLIST_OFFLOADED | |
108+
* | SEGCBLIST_LOCKING | |
109+
* | SEGCBLIST_OFFLOADED | |
111110
* | SEGCBLIST_KTHREAD_GP | |
112-
* | SEGCBLIST_KTHREAD_CB |
111+
* | + unparked CB kthread |
113112
* | |
114113
* | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops |
115114
* | handling callbacks. Enable bypass queueing. |
@@ -125,8 +124,8 @@ struct rcu_cblist {
125124
* |--------------------------------------------------------------------------|
126125
* | SEGCBLIST_LOCKING | |
127126
* | SEGCBLIST_OFFLOADED | |
128-
* | SEGCBLIST_KTHREAD_CB | |
129127
* | SEGCBLIST_KTHREAD_GP |
128+
* | + unparked CB kthread |
130129
* | |
131130
* | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() |
132131
* | ignores callbacks. Bypass enqueue is enabled. |
@@ -137,11 +136,11 @@ struct rcu_cblist {
137136
* | SEGCBLIST_RCU_CORE | |
138137
* | SEGCBLIST_LOCKING | |
139138
* | SEGCBLIST_OFFLOADED | |
140-
* | SEGCBLIST_KTHREAD_CB | |
141139
* | SEGCBLIST_KTHREAD_GP |
140+
* | + unparked CB kthread |
142141
* | |
143142
* | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() |
144-
* | handles callbacks concurrently. Bypass enqueue is enabled. |
143+
* | handles callbacks concurrently. Bypass enqueue is disabled. |
145144
* | Invoke RCU core so we make sure not to preempt it in the middle with |
146145
* | leaving some urgent work unattended within a jiffy. |
147146
* ----------------------------------------------------------------------------
@@ -150,42 +149,31 @@ struct rcu_cblist {
150149
* |--------------------------------------------------------------------------|
151150
* | SEGCBLIST_RCU_CORE | |
152151
* | SEGCBLIST_LOCKING | |
153-
* | SEGCBLIST_KTHREAD_CB | |
154152
* | SEGCBLIST_KTHREAD_GP |
153+
* | + unparked CB kthread |
155154
* | |
156155
* | CB/GP kthreads and local rcu_core() handle callbacks concurrently |
157-
* | holding nocb_lock. Wake up CB and GP kthreads if necessary. Disable |
158-
* | bypass enqueue. |
156+
* | holding nocb_lock. Wake up GP kthread if necessary. |
159157
* ----------------------------------------------------------------------------
160158
* |
161159
* v
162-
* -----------------------------------
163-
* | |
164-
* v v
165-
* ---------------------------------------------------------------------------|
166-
* | | |
167-
* | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | |
168-
* | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | |
169-
* | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP |
170-
* | | |
171-
* | GP kthread woke up and | CB kthread woke up and |
172-
* | acknowledged the fact that | acknowledged the fact that |
173-
* | SEGCBLIST_OFFLOADED got cleared. | SEGCBLIST_OFFLOADED got cleared. |
174-
* | | The CB kthread goes to sleep |
175-
* | The callbacks from the target CPU | until it ever gets re-offloaded. |
176-
* | will be ignored from the GP kthread | |
177-
* | loop. | |
160+
* |--------------------------------------------------------------------------|
161+
* | SEGCBLIST_RCU_CORE | |
162+
* | SEGCBLIST_LOCKING | |
163+
* | + unparked CB kthread |
164+
* | |
165+
* | GP kthread woke up and acknowledged the fact that SEGCBLIST_OFFLOADED |
166+
* | got cleared. The callbacks from the target CPU will be ignored from the|
167+
* | GP kthread loop. |
178168
* ----------------------------------------------------------------------------
179-
* | |
180-
* -----------------------------------
181169
* |
182170
* v
183171
* ----------------------------------------------------------------------------
184172
* | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING |
173+
* | + parked CB kthread |
185174
* | |
186-
* | Callbacks processed by rcu_core() from softirqs or local |
187-
* | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. |
188-
* | Flush pending nocb_timer. Flush nocb bypass callbacks. |
175+
* | CB kthread is parked. Callbacks processed by rcu_core() from softirqs or |
176+
* | local rcuc kthread, while holding nocb_lock. |
189177
* ----------------------------------------------------------------------------
190178
* |
191179
* v

‎include/linux/rcupdate.h

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,6 @@ void synchronize_rcu_tasks_rude(void);
209209

210210
#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
211211
void exit_tasks_rcu_start(void);
212-
void exit_tasks_rcu_stop(void);
213212
void exit_tasks_rcu_finish(void);
214213
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
215214
#define rcu_tasks_classic_qs(t, preempt) do { } while (0)
@@ -218,7 +217,6 @@ void exit_tasks_rcu_finish(void);
218217
#define call_rcu_tasks call_rcu
219218
#define synchronize_rcu_tasks synchronize_rcu
220219
static inline void exit_tasks_rcu_start(void) { }
221-
static inline void exit_tasks_rcu_stop(void) { }
222220
static inline void exit_tasks_rcu_finish(void) { }
223221
#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
224222

@@ -421,11 +419,71 @@ static inline void rcu_preempt_sleep_check(void) { }
421419
"Illegal context switch in RCU-sched read-side critical section"); \
422420
} while (0)
423421

422+
// See RCU_LOCKDEP_WARN() for an explanation of the double call to
423+
// debug_lockdep_rcu_enabled().
424+
static inline bool lockdep_assert_rcu_helper(bool c)
425+
{
426+
return debug_lockdep_rcu_enabled() &&
427+
(c || !rcu_is_watching() || !rcu_lockdep_current_cpu_online()) &&
428+
debug_lockdep_rcu_enabled();
429+
}
430+
431+
/**
432+
* lockdep_assert_in_rcu_read_lock - WARN if not protected by rcu_read_lock()
433+
*
434+
* Splats if lockdep is enabled and there is no rcu_read_lock() in effect.
435+
*/
436+
#define lockdep_assert_in_rcu_read_lock() \
437+
WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map)))
438+
439+
/**
440+
* lockdep_assert_in_rcu_read_lock_bh - WARN if not protected by rcu_read_lock_bh()
441+
*
442+
* Splats if lockdep is enabled and there is no rcu_read_lock_bh() in effect.
443+
* Note that local_bh_disable() and friends do not suffice here, instead an
444+
* actual rcu_read_lock_bh() is required.
445+
*/
446+
#define lockdep_assert_in_rcu_read_lock_bh() \
447+
WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_bh_lock_map)))
448+
449+
/**
450+
* lockdep_assert_in_rcu_read_lock_sched - WARN if not protected by rcu_read_lock_sched()
451+
*
452+
* Splats if lockdep is enabled and there is no rcu_read_lock_sched()
453+
* in effect. Note that preempt_disable() and friends do not suffice here,
454+
* instead an actual rcu_read_lock_sched() is required.
455+
*/
456+
#define lockdep_assert_in_rcu_read_lock_sched() \
457+
WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_sched_lock_map)))
458+
459+
/**
460+
* lockdep_assert_in_rcu_reader - WARN if not within some type of RCU reader
461+
*
462+
* Splats if lockdep is enabled and there is no RCU reader of any
463+
* type in effect. Note that regions of code protected by things like
464+
* preempt_disable, local_bh_disable(), and local_irq_disable() all qualify
465+
* as RCU readers.
466+
*
467+
* Note that this will never trigger in PREEMPT_NONE or PREEMPT_VOLUNTARY
468+
* kernels that are not also built with PREEMPT_COUNT. But if you have
469+
* lockdep enabled, you might as well also enable PREEMPT_COUNT.
470+
*/
471+
#define lockdep_assert_in_rcu_reader() \
472+
WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map) && \
473+
!lock_is_held(&rcu_bh_lock_map) && \
474+
!lock_is_held(&rcu_sched_lock_map) && \
475+
preemptible()))
476+
424477
#else /* #ifdef CONFIG_PROVE_RCU */
425478

426479
#define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))
427480
#define rcu_sleep_check() do { } while (0)
428481

482+
#define lockdep_assert_in_rcu_read_lock() do { } while (0)
483+
#define lockdep_assert_in_rcu_read_lock_bh() do { } while (0)
484+
#define lockdep_assert_in_rcu_read_lock_sched() do { } while (0)
485+
#define lockdep_assert_in_rcu_reader() do { } while (0)
486+
429487
#endif /* #else #ifdef CONFIG_PROVE_RCU */
430488

431489
/*

‎include/linux/srcu.h

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,45 @@ void cleanup_srcu_struct(struct srcu_struct *ssp);
5757
int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
5858
void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
5959
void synchronize_srcu(struct srcu_struct *ssp);
60+
61+
#define SRCU_GET_STATE_COMPLETED 0x1
62+
63+
/**
64+
* get_completed_synchronize_srcu - Return a pre-completed polled state cookie
65+
*
66+
* Returns a value that poll_state_synchronize_srcu() will always treat
67+
* as a cookie whose grace period has already completed.
68+
*/
69+
static inline unsigned long get_completed_synchronize_srcu(void)
70+
{
71+
return SRCU_GET_STATE_COMPLETED;
72+
}
73+
6074
unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
6175
unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
6276
bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
6377

78+
// Maximum number of unsigned long values corresponding to
79+
// not-yet-completed SRCU grace periods.
80+
#define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2
81+
82+
/**
83+
* same_state_synchronize_srcu - Are two old-state values identical?
84+
* @oldstate1: First old-state value.
85+
* @oldstate2: Second old-state value.
86+
*
87+
* The two old-state values must have been obtained from either
88+
* get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or
89+
* get_completed_synchronize_srcu(). Returns @true if the two values are
90+
* identical and @false otherwise. This allows structures whose lifetimes
91+
* are tracked by old-state values to push these values to a list header,
92+
* allowing those structures to be slightly smaller.
93+
*/
94+
static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
95+
{
96+
return oldstate1 == oldstate2;
97+
}
98+
6499
#ifdef CONFIG_NEED_SRCU_NMI_SAFE
65100
int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp);
66101
void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp);

‎kernel/pid_namespace.c

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -248,24 +248,7 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns)
248248
set_current_state(TASK_INTERRUPTIBLE);
249249
if (pid_ns->pid_allocated == init_pids)
250250
break;
251-
/*
252-
* Release tasks_rcu_exit_srcu to avoid following deadlock:
253-
*
254-
* 1) TASK A unshare(CLONE_NEWPID)
255-
* 2) TASK A fork() twice -> TASK B (child reaper for new ns)
256-
* and TASK C
257-
* 3) TASK B exits, kills TASK C, waits for TASK A to reap it
258-
* 4) TASK A calls synchronize_rcu_tasks()
259-
* -> synchronize_srcu(tasks_rcu_exit_srcu)
260-
* 5) *DEADLOCK*
261-
*
262-
* It is considered safe to release tasks_rcu_exit_srcu here
263-
* because we assume the current task can not be concurrently
264-
* reaped at this point.
265-
*/
266-
exit_tasks_rcu_stop();
267251
schedule();
268-
exit_tasks_rcu_start();
269252
}
270253
__set_current_state(TASK_RUNNING);
271254

‎kernel/rcu/rcuscale.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242

4343
#include "rcu.h"
4444

45+
MODULE_DESCRIPTION("Read-Copy Update module-based scalability-test facility");
4546
MODULE_LICENSE("GPL");
4647
MODULE_AUTHOR("Paul E. McKenney <[email protected]>");
4748

0 commit comments

Comments
 (0)