Skip to content

Commit ef53245

Browse files
mrutland-armctmarinas
mrutland-arm
authored and
ctmarinas
committed
arm64: atomics lse: define SUBs in terms of ADDs
The FEAT_LSE atomic instructions include atomic ADD instructions (`stadd*` and `ldadd*`), but do not include atomic SUB instructions, so we must build all of the SUB operations using the ADD instructions. We open-code these today, with each SUB op implemented as a copy of the corresponding ADD op with a leading `neg` instruction in the inline assembly to negate the `i` argument. As the compiler has no visibility of the `neg`, this leads to less than optimal code generation when generating `i` into a register. For example, __les_atomic_fetch_sub(1, v) can be compiled to: mov w1, #0x1 neg w1, w1 ldaddal w1, w1, [x2] This patch improves this by replacing the `neg` with negation in C before the inline assembly block, e.g. i = -i; This allows the compiler to generate `i` into a register more optimally, e.g. mov w1, #0xffffffff ldaddal w1, w1, [x2] With this change the assembly for each SUB op is identical to the corresponding ADD op (including barriers and clobbers), so I've removed the inline assembly and rewritten each SUB op in terms of the corresponding ADD op, e.g. | static inline void __lse_atomic_sub(int i, atomic_t *v) | { | __lse_atomic_add(-i, v); | } For clarity I've moved the definition of each SUB op immediately after the corresponding ADD op, and used a single macro to create the RETURN forms of both ops. This is intended as an optimization and cleanup. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland <[email protected]> Cc: Boqun Feng <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Will Deacon <[email protected]> Acked-by: Will Deacon <[email protected]> Acked-by: Peter Zijlstra (Intel) <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Catalin Marinas <[email protected]>
1 parent 8e6082e commit ef53245

File tree

1 file changed

+58
-122
lines changed

1 file changed

+58
-122
lines changed

‎arch/arm64/include/asm/atomic_lse.h

Lines changed: 58 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ ATOMIC_OP(or, stset)
2525
ATOMIC_OP(xor, steor)
2626
ATOMIC_OP(add, stadd)
2727

28+
static inline void __lse_atomic_sub(int i, atomic_t *v)
29+
{
30+
__lse_atomic_add(-i, v);
31+
}
32+
2833
#undef ATOMIC_OP
2934

3035
#define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \
@@ -54,7 +59,20 @@ ATOMIC_FETCH_OPS(add, ldadd)
5459
#undef ATOMIC_FETCH_OP
5560
#undef ATOMIC_FETCH_OPS
5661

57-
#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
62+
#define ATOMIC_FETCH_OP_SUB(name) \
63+
static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v) \
64+
{ \
65+
return __lse_atomic_fetch_add##name(-i, v); \
66+
}
67+
68+
ATOMIC_FETCH_OP_SUB(_relaxed)
69+
ATOMIC_FETCH_OP_SUB(_acquire)
70+
ATOMIC_FETCH_OP_SUB(_release)
71+
ATOMIC_FETCH_OP_SUB( )
72+
73+
#undef ATOMIC_FETCH_OP_SUB
74+
75+
#define ATOMIC_OP_ADD_SUB_RETURN(name, mb, cl...) \
5876
static inline int __lse_atomic_add_return##name(int i, atomic_t *v) \
5977
{ \
6078
u32 tmp; \
@@ -68,14 +86,19 @@ static inline int __lse_atomic_add_return##name(int i, atomic_t *v) \
6886
: cl); \
6987
\
7088
return i; \
89+
} \
90+
\
91+
static inline int __lse_atomic_sub_return##name(int i, atomic_t *v) \
92+
{ \
93+
return __lse_atomic_add_return##name(-i, v); \
7194
}
7295

73-
ATOMIC_OP_ADD_RETURN(_relaxed, )
74-
ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
75-
ATOMIC_OP_ADD_RETURN(_release, l, "memory")
76-
ATOMIC_OP_ADD_RETURN( , al, "memory")
96+
ATOMIC_OP_ADD_SUB_RETURN(_relaxed, )
97+
ATOMIC_OP_ADD_SUB_RETURN(_acquire, a, "memory")
98+
ATOMIC_OP_ADD_SUB_RETURN(_release, l, "memory")
99+
ATOMIC_OP_ADD_SUB_RETURN( , al, "memory")
77100

78-
#undef ATOMIC_OP_ADD_RETURN
101+
#undef ATOMIC_OP_ADD_SUB_RETURN
79102

80103
static inline void __lse_atomic_and(int i, atomic_t *v)
81104
{
@@ -108,61 +131,6 @@ ATOMIC_FETCH_OP_AND( , al, "memory")
108131

109132
#undef ATOMIC_FETCH_OP_AND
110133

111-
static inline void __lse_atomic_sub(int i, atomic_t *v)
112-
{
113-
asm volatile(
114-
__LSE_PREAMBLE
115-
" neg %w[i], %w[i]\n"
116-
" stadd %w[i], %[v]"
117-
: [i] "+&r" (i), [v] "+Q" (v->counter)
118-
: "r" (v));
119-
}
120-
121-
#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
122-
static inline int __lse_atomic_sub_return##name(int i, atomic_t *v) \
123-
{ \
124-
u32 tmp; \
125-
\
126-
asm volatile( \
127-
__LSE_PREAMBLE \
128-
" neg %w[i], %w[i]\n" \
129-
" ldadd" #mb " %w[i], %w[tmp], %[v]\n" \
130-
" add %w[i], %w[i], %w[tmp]" \
131-
: [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \
132-
: "r" (v) \
133-
: cl); \
134-
\
135-
return i; \
136-
}
137-
138-
ATOMIC_OP_SUB_RETURN(_relaxed, )
139-
ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
140-
ATOMIC_OP_SUB_RETURN(_release, l, "memory")
141-
ATOMIC_OP_SUB_RETURN( , al, "memory")
142-
143-
#undef ATOMIC_OP_SUB_RETURN
144-
145-
#define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \
146-
static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v) \
147-
{ \
148-
asm volatile( \
149-
__LSE_PREAMBLE \
150-
" neg %w[i], %w[i]\n" \
151-
" ldadd" #mb " %w[i], %w[i], %[v]" \
152-
: [i] "+&r" (i), [v] "+Q" (v->counter) \
153-
: "r" (v) \
154-
: cl); \
155-
\
156-
return i; \
157-
}
158-
159-
ATOMIC_FETCH_OP_SUB(_relaxed, )
160-
ATOMIC_FETCH_OP_SUB(_acquire, a, "memory")
161-
ATOMIC_FETCH_OP_SUB(_release, l, "memory")
162-
ATOMIC_FETCH_OP_SUB( , al, "memory")
163-
164-
#undef ATOMIC_FETCH_OP_SUB
165-
166134
#define ATOMIC64_OP(op, asm_op) \
167135
static inline void __lse_atomic64_##op(s64 i, atomic64_t *v) \
168136
{ \
@@ -178,6 +146,11 @@ ATOMIC64_OP(or, stset)
178146
ATOMIC64_OP(xor, steor)
179147
ATOMIC64_OP(add, stadd)
180148

149+
static inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
150+
{
151+
__lse_atomic64_add(-i, v);
152+
}
153+
181154
#undef ATOMIC64_OP
182155

183156
#define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \
@@ -207,7 +180,20 @@ ATOMIC64_FETCH_OPS(add, ldadd)
207180
#undef ATOMIC64_FETCH_OP
208181
#undef ATOMIC64_FETCH_OPS
209182

210-
#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
183+
#define ATOMIC64_FETCH_OP_SUB(name) \
184+
static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \
185+
{ \
186+
return __lse_atomic64_fetch_add##name(-i, v); \
187+
}
188+
189+
ATOMIC64_FETCH_OP_SUB(_relaxed)
190+
ATOMIC64_FETCH_OP_SUB(_acquire)
191+
ATOMIC64_FETCH_OP_SUB(_release)
192+
ATOMIC64_FETCH_OP_SUB( )
193+
194+
#undef ATOMIC64_FETCH_OP_SUB
195+
196+
#define ATOMIC64_OP_ADD_SUB_RETURN(name, mb, cl...) \
211197
static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\
212198
{ \
213199
unsigned long tmp; \
@@ -221,14 +207,19 @@ static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\
221207
: cl); \
222208
\
223209
return i; \
210+
} \
211+
\
212+
static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)\
213+
{ \
214+
return __lse_atomic64_add_return##name(-i, v); \
224215
}
225216

226-
ATOMIC64_OP_ADD_RETURN(_relaxed, )
227-
ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
228-
ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
229-
ATOMIC64_OP_ADD_RETURN( , al, "memory")
217+
ATOMIC64_OP_ADD_SUB_RETURN(_relaxed, )
218+
ATOMIC64_OP_ADD_SUB_RETURN(_acquire, a, "memory")
219+
ATOMIC64_OP_ADD_SUB_RETURN(_release, l, "memory")
220+
ATOMIC64_OP_ADD_SUB_RETURN( , al, "memory")
230221

231-
#undef ATOMIC64_OP_ADD_RETURN
222+
#undef ATOMIC64_OP_ADD_SUB_RETURN
232223

233224
static inline void __lse_atomic64_and(s64 i, atomic64_t *v)
234225
{
@@ -261,61 +252,6 @@ ATOMIC64_FETCH_OP_AND( , al, "memory")
261252

262253
#undef ATOMIC64_FETCH_OP_AND
263254

264-
static inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
265-
{
266-
asm volatile(
267-
__LSE_PREAMBLE
268-
" neg %[i], %[i]\n"
269-
" stadd %[i], %[v]"
270-
: [i] "+&r" (i), [v] "+Q" (v->counter)
271-
: "r" (v));
272-
}
273-
274-
#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
275-
static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)\
276-
{ \
277-
unsigned long tmp; \
278-
\
279-
asm volatile( \
280-
__LSE_PREAMBLE \
281-
" neg %[i], %[i]\n" \
282-
" ldadd" #mb " %[i], %x[tmp], %[v]\n" \
283-
" add %[i], %[i], %x[tmp]" \
284-
: [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \
285-
: "r" (v) \
286-
: cl); \
287-
\
288-
return i; \
289-
}
290-
291-
ATOMIC64_OP_SUB_RETURN(_relaxed, )
292-
ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
293-
ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
294-
ATOMIC64_OP_SUB_RETURN( , al, "memory")
295-
296-
#undef ATOMIC64_OP_SUB_RETURN
297-
298-
#define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \
299-
static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \
300-
{ \
301-
asm volatile( \
302-
__LSE_PREAMBLE \
303-
" neg %[i], %[i]\n" \
304-
" ldadd" #mb " %[i], %[i], %[v]" \
305-
: [i] "+&r" (i), [v] "+Q" (v->counter) \
306-
: "r" (v) \
307-
: cl); \
308-
\
309-
return i; \
310-
}
311-
312-
ATOMIC64_FETCH_OP_SUB(_relaxed, )
313-
ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory")
314-
ATOMIC64_FETCH_OP_SUB(_release, l, "memory")
315-
ATOMIC64_FETCH_OP_SUB( , al, "memory")
316-
317-
#undef ATOMIC64_FETCH_OP_SUB
318-
319255
static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
320256
{
321257
unsigned long tmp;

0 commit comments

Comments
 (0)