Skip to content

Commit 5c0ec49

Browse files
paulmckrcupaulmckrcu
committed
Merge branches 'doc.2022.08.31b', 'fixes.2022.08.31b', 'kvfree.2022.08.31b', 'nocb.2022.09.01a', 'poll.2022.08.31b', 'poll-srcu.2022.08.31b' and 'tasks.2022.08.31b' into HEAD
doc.2022.08.31b: Documentation updates fixes.2022.08.31b: Miscellaneous fixes kvfree.2022.08.31b: kvfree_rcu() updates nocb.2022.09.01a: NOCB CPU updates poll.2022.08.31b: Full-oldstate RCU polling grace-period API poll-srcu.2022.08.31b: Polled SRCU grace-period updates tasks.2022.08.31b: Tasks RCU updates
7 parents ef32aee + 48297a2 + 51824b7 + 638dce2 + 967c298 + 5fe8919 + 528262f commit 5c0ec49

File tree

15 files changed

+761
-162
lines changed

15 files changed

+761
-162
lines changed

‎include/linux/rcupdate.h

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,31 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func);
4242
void rcu_barrier_tasks(void);
4343
void rcu_barrier_tasks_rude(void);
4444
void synchronize_rcu(void);
45+
46+
struct rcu_gp_oldstate;
4547
unsigned long get_completed_synchronize_rcu(void);
48+
void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
49+
50+
// Maximum number of unsigned long values corresponding to
51+
// not-yet-completed RCU grace periods.
52+
#define NUM_ACTIVE_RCU_POLL_OLDSTATE 2
53+
54+
/**
55+
* same_state_synchronize_rcu - Are two old-state values identical?
56+
* @oldstate1: First old-state value.
57+
* @oldstate2: Second old-state value.
58+
*
59+
* The two old-state values must have been obtained from either
60+
* get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or
61+
* get_completed_synchronize_rcu(). Returns @true if the two values are
62+
* identical and @false otherwise. This allows structures whose lifetimes
63+
* are tracked by old-state values to push these values to a list header,
64+
* allowing those structures to be slightly smaller.
65+
*/
66+
static inline bool same_state_synchronize_rcu(unsigned long oldstate1, unsigned long oldstate2)
67+
{
68+
return oldstate1 == oldstate2;
69+
}
4670

4771
#ifdef CONFIG_PREEMPT_RCU
4872

@@ -496,13 +520,21 @@ do { \
496520
* against NULL. Although rcu_access_pointer() may also be used in cases
497521
* where update-side locks prevent the value of the pointer from changing,
498522
* you should instead use rcu_dereference_protected() for this use case.
523+
* Within an RCU read-side critical section, there is little reason to
524+
* use rcu_access_pointer().
525+
*
526+
* It is usually best to test the rcu_access_pointer() return value
527+
* directly in order to avoid accidental dereferences being introduced
528+
* by later inattentive changes. In other words, assigning the
529+
* rcu_access_pointer() return value to a local variable results in an
530+
* accident waiting to happen.
499531
*
500532
* It is also permissible to use rcu_access_pointer() when read-side
501-
* access to the pointer was removed at least one grace period ago, as
502-
* is the case in the context of the RCU callback that is freeing up
503-
* the data, or after a synchronize_rcu() returns. This can be useful
504-
* when tearing down multi-linked structures after a grace period
505-
* has elapsed.
533+
* access to the pointer was removed at least one grace period ago, as is
534+
* the case in the context of the RCU callback that is freeing up the data,
535+
* or after a synchronize_rcu() returns. This can be useful when tearing
536+
* down multi-linked structures after a grace period has elapsed. However,
537+
* rcu_dereference_protected() is normally preferred for this use case.
506538
*/
507539
#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu)
508540

‎include/linux/rcutiny.h

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,75 @@
1414

1515
#include <asm/param.h> /* for HZ */
1616

17+
struct rcu_gp_oldstate {
18+
unsigned long rgos_norm;
19+
};
20+
21+
// Maximum number of rcu_gp_oldstate values corresponding to
22+
// not-yet-completed RCU grace periods.
23+
#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2
24+
25+
/*
26+
* Are the two oldstate values the same? See the Tree RCU version for
27+
* docbook header.
28+
*/
29+
static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1,
30+
struct rcu_gp_oldstate *rgosp2)
31+
{
32+
return rgosp1->rgos_norm == rgosp2->rgos_norm;
33+
}
34+
1735
unsigned long get_state_synchronize_rcu(void);
36+
37+
static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
38+
{
39+
rgosp->rgos_norm = get_state_synchronize_rcu();
40+
}
41+
1842
unsigned long start_poll_synchronize_rcu(void);
43+
44+
static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
45+
{
46+
rgosp->rgos_norm = start_poll_synchronize_rcu();
47+
}
48+
1949
bool poll_state_synchronize_rcu(unsigned long oldstate);
2050

51+
static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
52+
{
53+
return poll_state_synchronize_rcu(rgosp->rgos_norm);
54+
}
55+
2156
static inline void cond_synchronize_rcu(unsigned long oldstate)
2257
{
2358
might_sleep();
2459
}
2560

61+
static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
62+
{
63+
cond_synchronize_rcu(rgosp->rgos_norm);
64+
}
65+
2666
static inline unsigned long start_poll_synchronize_rcu_expedited(void)
2767
{
2868
return start_poll_synchronize_rcu();
2969
}
3070

71+
static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
72+
{
73+
rgosp->rgos_norm = start_poll_synchronize_rcu_expedited();
74+
}
75+
3176
static inline void cond_synchronize_rcu_expedited(unsigned long oldstate)
3277
{
3378
cond_synchronize_rcu(oldstate);
3479
}
3580

81+
static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
82+
{
83+
cond_synchronize_rcu_expedited(rgosp->rgos_norm);
84+
}
85+
3686
extern void rcu_barrier(void);
3787

3888
static inline void synchronize_rcu_expedited(void)

‎include/linux/rcutree.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,52 @@ bool rcu_eqs_special_set(int cpu);
4040
void rcu_momentary_dyntick_idle(void);
4141
void kfree_rcu_scheduler_running(void);
4242
bool rcu_gp_might_be_stalled(void);
43+
44+
struct rcu_gp_oldstate {
45+
unsigned long rgos_norm;
46+
unsigned long rgos_exp;
47+
};
48+
49+
// Maximum number of rcu_gp_oldstate values corresponding to
50+
// not-yet-completed RCU grace periods.
51+
#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4
52+
53+
/**
54+
* same_state_synchronize_rcu_full - Are two old-state values identical?
55+
* @rgosp1: First old-state value.
56+
* @rgosp2: Second old-state value.
57+
*
58+
* The two old-state values must have been obtained from either
59+
* get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),
60+
* or get_completed_synchronize_rcu_full(). Returns @true if the two
61+
* values are identical and @false otherwise. This allows structures
62+
* whose lifetimes are tracked by old-state values to push these values
63+
* to a list header, allowing those structures to be slightly smaller.
64+
*
65+
* Note that equality is judged on a bitwise basis, so that an
66+
* @rcu_gp_oldstate structure with an already-completed state in one field
67+
* will compare not-equal to a structure with an already-completed state
68+
* in the other field. After all, the @rcu_gp_oldstate structure is opaque
69+
* so how did such a situation come to pass in the first place?
70+
*/
71+
static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1,
72+
struct rcu_gp_oldstate *rgosp2)
73+
{
74+
return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp;
75+
}
76+
4377
unsigned long start_poll_synchronize_rcu_expedited(void);
78+
void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
4479
void cond_synchronize_rcu_expedited(unsigned long oldstate);
80+
void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
4581
unsigned long get_state_synchronize_rcu(void);
82+
void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
4683
unsigned long start_poll_synchronize_rcu(void);
84+
void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
4785
bool poll_state_synchronize_rcu(unsigned long oldstate);
86+
bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
4887
void cond_synchronize_rcu(unsigned long oldstate);
88+
void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
4989

5090
bool rcu_is_idle_cpu(int cpu);
5191

‎include/linux/srcutiny.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@
1515

1616
struct srcu_struct {
1717
short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */
18-
unsigned short srcu_idx; /* Current reader array element in bit 0x2. */
19-
unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */
2018
u8 srcu_gp_running; /* GP workqueue running? */
2119
u8 srcu_gp_waiting; /* GP waiting for readers? */
20+
unsigned long srcu_idx; /* Current reader array element in bit 0x2. */
21+
unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */
2222
struct swait_queue_head srcu_wq;
2323
/* Last srcu_read_unlock() wakes GP. */
2424
struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */
@@ -82,10 +82,12 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
8282
int idx;
8383

8484
idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1;
85-
pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",
85+
pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n",
8686
tt, tf, idx,
8787
data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])),
88-
data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])));
88+
data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])),
89+
data_race(READ_ONCE(ssp->srcu_idx)),
90+
data_race(READ_ONCE(ssp->srcu_idx_max)));
8991
}
9092

9193
#endif

0 commit comments

Comments
 (0)