Skip to content

Commit ebd326c

Browse files
torvaldstorvalds
committed
Merge tag 'locking-core-2022-03-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar: "Changes in this cycle were: Bitops & cpumask: - Always inline various generic helpers, to improve code generation, but also for instrumentation, found by noinstr validation. - Add a x86-specific cpumask_clear_cpu() helper to improve code generation Atomics: - Fix atomic64_{read_acquire,set_release} fallbacks Lockdep: - Fix /proc/lockdep output loop iteration for classes - Fix /proc/lockdep potential access to invalid memory - Add Mark Rutland as reviewer for atomic primitives - Minor cleanups Jump labels: - Clean up the code a bit Misc: - Add __sched annotations to percpu rwsem primitives - Enable RT_MUTEXES on PREEMPT_RT by default - Stray v8086_mode() inlining fix, result of noinstr objtool validation" * tag 'locking-core-2022-03-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: jump_label: Refactor #ifdef of struct static_key jump_label: Avoid unneeded casts in STATIC_KEY_INIT_{TRUE,FALSE} locking/lockdep: Iterate lock_classes directly when reading lockdep files x86/ptrace: Always inline v8086_mode() for instrumentation cpumask: Add a x86-specific cpumask_clear_cpu() helper locking: Enable RT_MUTEXES by default on PREEMPT_RT. locking/local_lock: Make the empty local_lock_*() function a macro. atomics: Fix atomic64_{read_acquire,set_release} fallbacks locking: Add missing __sched attributes cpumask: Always inline helpers which use bit manipulation functions asm-generic/bitops: Always inline all bit manipulation helpers locking/lockdep: Avoid potential access of invalid memory in lock_class lockdep: Use memset_startat() helper in reinit_class() MAINTAINERS: add myself as reviewer for atomics
2 parents 95ab0e8 + cd27ccf commit ebd326c

File tree

17 files changed

+168
-74
lines changed

17 files changed

+168
-74
lines changed

‎MAINTAINERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3205,6 +3205,7 @@ ATOMIC INFRASTRUCTURE
32053205
M: Will Deacon <[email protected]>
32063206
M: Peter Zijlstra <[email protected]>
32073207
R: Boqun Feng <[email protected]>
3208+
R: Mark Rutland <[email protected]>
32083209
L: [email protected]
32093210
S: Maintained
32103211
F: arch/*/include/asm/atomic*.h

‎arch/x86/include/asm/cpumask.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,21 @@ static __always_inline bool arch_cpu_online(int cpu)
2020
{
2121
return arch_test_bit(cpu, cpumask_bits(cpu_online_mask));
2222
}
23+
24+
static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
25+
{
26+
arch_clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
27+
}
2328
#else
2429
static __always_inline bool arch_cpu_online(int cpu)
2530
{
2631
return cpu == 0;
2732
}
33+
34+
static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
35+
{
36+
return;
37+
}
2838
#endif
2939

3040
#define arch_cpu_is_offline(cpu) unlikely(!arch_cpu_online(cpu))

‎arch/x86/include/asm/ptrace.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ static __always_inline int user_mode(struct pt_regs *regs)
137137
#endif
138138
}
139139

140-
static inline int v8086_mode(struct pt_regs *regs)
140+
static __always_inline int v8086_mode(struct pt_regs *regs)
141141
{
142142
#ifdef CONFIG_X86_32
143143
return (regs->flags & X86_VM_MASK);

‎include/asm-generic/bitops/instrumented-atomic.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
* Note that @nr may be almost arbitrarily large; this function is not
2424
* restricted to acting on a single-word quantity.
2525
*/
26-
static inline void set_bit(long nr, volatile unsigned long *addr)
26+
static __always_inline void set_bit(long nr, volatile unsigned long *addr)
2727
{
2828
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
2929
arch_set_bit(nr, addr);
@@ -36,7 +36,7 @@ static inline void set_bit(long nr, volatile unsigned long *addr)
3636
*
3737
* This is a relaxed atomic operation (no implied memory barriers).
3838
*/
39-
static inline void clear_bit(long nr, volatile unsigned long *addr)
39+
static __always_inline void clear_bit(long nr, volatile unsigned long *addr)
4040
{
4141
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
4242
arch_clear_bit(nr, addr);
@@ -52,7 +52,7 @@ static inline void clear_bit(long nr, volatile unsigned long *addr)
5252
* Note that @nr may be almost arbitrarily large; this function is not
5353
* restricted to acting on a single-word quantity.
5454
*/
55-
static inline void change_bit(long nr, volatile unsigned long *addr)
55+
static __always_inline void change_bit(long nr, volatile unsigned long *addr)
5656
{
5757
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
5858
arch_change_bit(nr, addr);
@@ -65,7 +65,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
6565
*
6666
* This is an atomic fully-ordered operation (implied full memory barrier).
6767
*/
68-
static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
68+
static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
6969
{
7070
kcsan_mb();
7171
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@@ -79,7 +79,7 @@ static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
7979
*
8080
* This is an atomic fully-ordered operation (implied full memory barrier).
8181
*/
82-
static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
82+
static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
8383
{
8484
kcsan_mb();
8585
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@@ -93,7 +93,7 @@ static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
9393
*
9494
* This is an atomic fully-ordered operation (implied full memory barrier).
9595
*/
96-
static inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
96+
static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
9797
{
9898
kcsan_mb();
9999
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));

‎include/asm-generic/bitops/instrumented-non-atomic.h

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
* region of memory concurrently, the effect may be that only one operation
2323
* succeeds.
2424
*/
25-
static inline void __set_bit(long nr, volatile unsigned long *addr)
25+
static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
2626
{
2727
instrument_write(addr + BIT_WORD(nr), sizeof(long));
2828
arch___set_bit(nr, addr);
@@ -37,7 +37,7 @@ static inline void __set_bit(long nr, volatile unsigned long *addr)
3737
* region of memory concurrently, the effect may be that only one operation
3838
* succeeds.
3939
*/
40-
static inline void __clear_bit(long nr, volatile unsigned long *addr)
40+
static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
4141
{
4242
instrument_write(addr + BIT_WORD(nr), sizeof(long));
4343
arch___clear_bit(nr, addr);
@@ -52,13 +52,13 @@ static inline void __clear_bit(long nr, volatile unsigned long *addr)
5252
* region of memory concurrently, the effect may be that only one operation
5353
* succeeds.
5454
*/
55-
static inline void __change_bit(long nr, volatile unsigned long *addr)
55+
static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
5656
{
5757
instrument_write(addr + BIT_WORD(nr), sizeof(long));
5858
arch___change_bit(nr, addr);
5959
}
6060

61-
static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
61+
static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
6262
{
6363
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
6464
/*
@@ -90,7 +90,7 @@ static inline void __instrument_read_write_bitop(long nr, volatile unsigned long
9090
* This operation is non-atomic. If two instances of this operation race, one
9191
* can appear to succeed but actually fail.
9292
*/
93-
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
93+
static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
9494
{
9595
__instrument_read_write_bitop(nr, addr);
9696
return arch___test_and_set_bit(nr, addr);
@@ -104,7 +104,7 @@ static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
104104
* This operation is non-atomic. If two instances of this operation race, one
105105
* can appear to succeed but actually fail.
106106
*/
107-
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
107+
static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
108108
{
109109
__instrument_read_write_bitop(nr, addr);
110110
return arch___test_and_clear_bit(nr, addr);
@@ -118,7 +118,7 @@ static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
118118
* This operation is non-atomic. If two instances of this operation race, one
119119
* can appear to succeed but actually fail.
120120
*/
121-
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
121+
static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
122122
{
123123
__instrument_read_write_bitop(nr, addr);
124124
return arch___test_and_change_bit(nr, addr);
@@ -129,7 +129,7 @@ static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
129129
* @nr: bit number to test
130130
* @addr: Address to start counting from
131131
*/
132-
static inline bool test_bit(long nr, const volatile unsigned long *addr)
132+
static __always_inline bool test_bit(long nr, const volatile unsigned long *addr)
133133
{
134134
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
135135
return arch_test_bit(nr, addr);

‎include/linux/atomic/atomic-arch-fallback.h

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,16 @@
151151
static __always_inline int
152152
arch_atomic_read_acquire(const atomic_t *v)
153153
{
154-
return smp_load_acquire(&(v)->counter);
154+
int ret;
155+
156+
if (__native_word(atomic_t)) {
157+
ret = smp_load_acquire(&(v)->counter);
158+
} else {
159+
ret = arch_atomic_read(v);
160+
__atomic_acquire_fence();
161+
}
162+
163+
return ret;
155164
}
156165
#define arch_atomic_read_acquire arch_atomic_read_acquire
157166
#endif
@@ -160,7 +169,12 @@ arch_atomic_read_acquire(const atomic_t *v)
160169
static __always_inline void
161170
arch_atomic_set_release(atomic_t *v, int i)
162171
{
163-
smp_store_release(&(v)->counter, i);
172+
if (__native_word(atomic_t)) {
173+
smp_store_release(&(v)->counter, i);
174+
} else {
175+
__atomic_release_fence();
176+
arch_atomic_set(v, i);
177+
}
164178
}
165179
#define arch_atomic_set_release arch_atomic_set_release
166180
#endif
@@ -1258,7 +1272,16 @@ arch_atomic_dec_if_positive(atomic_t *v)
12581272
static __always_inline s64
12591273
arch_atomic64_read_acquire(const atomic64_t *v)
12601274
{
1261-
return smp_load_acquire(&(v)->counter);
1275+
s64 ret;
1276+
1277+
if (__native_word(atomic64_t)) {
1278+
ret = smp_load_acquire(&(v)->counter);
1279+
} else {
1280+
ret = arch_atomic64_read(v);
1281+
__atomic_acquire_fence();
1282+
}
1283+
1284+
return ret;
12621285
}
12631286
#define arch_atomic64_read_acquire arch_atomic64_read_acquire
12641287
#endif
@@ -1267,7 +1290,12 @@ arch_atomic64_read_acquire(const atomic64_t *v)
12671290
static __always_inline void
12681291
arch_atomic64_set_release(atomic64_t *v, s64 i)
12691292
{
1270-
smp_store_release(&(v)->counter, i);
1293+
if (__native_word(atomic64_t)) {
1294+
smp_store_release(&(v)->counter, i);
1295+
} else {
1296+
__atomic_release_fence();
1297+
arch_atomic64_set(v, i);
1298+
}
12711299
}
12721300
#define arch_atomic64_set_release arch_atomic64_set_release
12731301
#endif
@@ -2358,4 +2386,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
23582386
#endif
23592387

23602388
#endif /* _LINUX_ATOMIC_FALLBACK_H */
2361-
// cca554917d7ea73d5e3e7397dd70c484cad9b2c4
2389+
// 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae

‎include/linux/cpumask.h

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -102,15 +102,15 @@ extern atomic_t __num_online_cpus;
102102

103103
extern cpumask_t cpus_booted_once_mask;
104104

105-
static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
105+
static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
106106
{
107107
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
108108
WARN_ON_ONCE(cpu >= bits);
109109
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
110110
}
111111

112112
/* verify cpu argument to cpumask_* operators */
113-
static inline unsigned int cpumask_check(unsigned int cpu)
113+
static __always_inline unsigned int cpumask_check(unsigned int cpu)
114114
{
115115
cpu_max_bits_warn(cpu, nr_cpumask_bits);
116116
return cpu;
@@ -341,12 +341,12 @@ extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool
341341
* @cpu: cpu number (< nr_cpu_ids)
342342
* @dstp: the cpumask pointer
343343
*/
344-
static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
344+
static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
345345
{
346346
set_bit(cpumask_check(cpu), cpumask_bits(dstp));
347347
}
348348

349-
static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
349+
static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
350350
{
351351
__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
352352
}
@@ -357,12 +357,12 @@ static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
357357
* @cpu: cpu number (< nr_cpu_ids)
358358
* @dstp: the cpumask pointer
359359
*/
360-
static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
360+
static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
361361
{
362362
clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
363363
}
364364

365-
static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
365+
static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
366366
{
367367
__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
368368
}
@@ -374,7 +374,7 @@ static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
374374
*
375375
* Returns 1 if @cpu is set in @cpumask, else returns 0
376376
*/
377-
static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
377+
static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
378378
{
379379
return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
380380
}
@@ -388,7 +388,7 @@ static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
388388
*
389389
* test_and_set_bit wrapper for cpumasks.
390390
*/
391-
static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
391+
static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
392392
{
393393
return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
394394
}
@@ -402,7 +402,7 @@ static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
402402
*
403403
* test_and_clear_bit wrapper for cpumasks.
404404
*/
405-
static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
405+
static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
406406
{
407407
return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
408408
}

‎include/linux/jump_label.h

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,9 @@ extern bool static_key_initialized;
8282
"%s(): static key '%pS' used before call to jump_label_init()", \
8383
__func__, (key))
8484

85-
#ifdef CONFIG_JUMP_LABEL
86-
8785
struct static_key {
8886
atomic_t enabled;
87+
#ifdef CONFIG_JUMP_LABEL
8988
/*
9089
* Note:
9190
* To make anonymous unions work with old compilers, the static
@@ -104,13 +103,9 @@ struct static_key {
104103
struct jump_entry *entries;
105104
struct static_key_mod *next;
106105
};
106+
#endif /* CONFIG_JUMP_LABEL */
107107
};
108108

109-
#else
110-
struct static_key {
111-
atomic_t enabled;
112-
};
113-
#endif /* CONFIG_JUMP_LABEL */
114109
#endif /* __ASSEMBLY__ */
115110

116111
#ifdef CONFIG_JUMP_LABEL
@@ -251,10 +246,10 @@ extern void static_key_disable_cpuslocked(struct static_key *key);
251246
*/
252247
#define STATIC_KEY_INIT_TRUE \
253248
{ .enabled = { 1 }, \
254-
{ .entries = (void *)JUMP_TYPE_TRUE } }
249+
{ .type = JUMP_TYPE_TRUE } }
255250
#define STATIC_KEY_INIT_FALSE \
256251
{ .enabled = { 0 }, \
257-
{ .entries = (void *)JUMP_TYPE_FALSE } }
252+
{ .type = JUMP_TYPE_FALSE } }
258253

259254
#else /* !CONFIG_JUMP_LABEL */
260255

‎include/linux/local_lock_internal.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ static inline void local_lock_debug_init(local_lock_t *l)
4444
}
4545
#else /* CONFIG_DEBUG_LOCK_ALLOC */
4646
# define LOCAL_LOCK_DEBUG_INIT(lockname)
47-
static inline void local_lock_acquire(local_lock_t *l) { }
48-
static inline void local_lock_release(local_lock_t *l) { }
49-
static inline void local_lock_debug_init(local_lock_t *l) { }
47+
# define local_lock_acquire(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
48+
# define local_lock_release(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
49+
# define local_lock_debug_init(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
5050
#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
5151

5252
#define INIT_LOCAL_LOCK(lockname) { LOCAL_LOCK_DEBUG_INIT(lockname) }

‎init/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2054,6 +2054,7 @@ source "arch/Kconfig"
20542054

20552055
config RT_MUTEXES
20562056
bool
2057+
default y if PREEMPT_RT
20572058

20582059
config BASE_SMALL
20592060
int

0 commit comments

Comments
 (0)