s390/atomic: mark all functions __always_inline

iii-i · Vasily Gorbik · commit 01cac82ae02b · 2024-04-03T15:00:19.000+02:00
Atomic functions are quite ubiquitous and may be called by noinstr ones, introducing unwanted instrumentation. They are very small, so there are no significant downsides to force-inlining them. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Link: https://lore.kernel.org/r/20240320230007.4782-2-iii@linux.ibm.com Signed-off-by: Heiko Carstens <hca@linux.ibm.com> Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h
@@ -15,31 +15,31 @@
 #include <asm/barrier.h>
 #include <asm/cmpxchg.h>
 
-static inline int arch_atomic_read(const atomic_t *v)
+static __always_inline int arch_atomic_read(const atomic_t *v)
 {
 	return __atomic_read(v);
 }
 #define arch_atomic_read arch_atomic_read
 
-static inline void arch_atomic_set(atomic_t *v, int i)
+static __always_inline void arch_atomic_set(atomic_t *v, int i)
 {
 	__atomic_set(v, i);
 }
 #define arch_atomic_set arch_atomic_set
 
-static inline int arch_atomic_add_return(int i, atomic_t *v)
+static __always_inline int arch_atomic_add_return(int i, atomic_t *v)
 {
 	return __atomic_add_barrier(i, &v->counter) + i;
 }
 #define arch_atomic_add_return arch_atomic_add_return
 
-static inline int arch_atomic_fetch_add(int i, atomic_t *v)
+static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)
 {
 	return __atomic_add_barrier(i, &v->counter);
 }
 #define arch_atomic_fetch_add arch_atomic_fetch_add
 
-static inline void arch_atomic_add(int i, atomic_t *v)
+static __always_inline void arch_atomic_add(int i, atomic_t *v)
 {
 	__atomic_add(i, &v->counter);
 }
@@ -50,11 +50,11 @@ static inline void arch_atomic_add(int i, atomic_t *v)
 #define arch_atomic_fetch_sub(_i, _v)	arch_atomic_fetch_add(-(int)(_i), _v)
 
 #define ATOMIC_OPS(op)							\
-static inline void arch_atomic_##op(int i, atomic_t *v)			\
+static __always_inline void arch_atomic_##op(int i, atomic_t *v)	\
 {									\
 	__atomic_##op(i, &v->counter);					\
 }									\
-static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
+static __always_inline int arch_atomic_fetch_##op(int i, atomic_t *v)	\
 {									\
 	return __atomic_##op##_barrier(i, &v->counter);			\
 }
@@ -74,60 +74,60 @@ ATOMIC_OPS(xor)
 
 #define arch_atomic_xchg(v, new)	(arch_xchg(&((v)->counter), new))
 
-static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
+static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
 {
 	return __atomic_cmpxchg(&v->counter, old, new);
 }
 #define arch_atomic_cmpxchg arch_atomic_cmpxchg
 
 #define ATOMIC64_INIT(i)  { (i) }
 
-static inline s64 arch_atomic64_read(const atomic64_t *v)
+static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
 {
 	return __atomic64_read(v);
 }
 #define arch_atomic64_read arch_atomic64_read
 
-static inline void arch_atomic64_set(atomic64_t *v, s64 i)
+static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
 {
 	__atomic64_set(v, i);
 }
 #define arch_atomic64_set arch_atomic64_set
 
-static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
+static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
 {
 	return __atomic64_add_barrier(i, (long *)&v->counter) + i;
 }
 #define arch_atomic64_add_return arch_atomic64_add_return
 
-static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
+static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
 {
 	return __atomic64_add_barrier(i, (long *)&v->counter);
 }
 #define arch_atomic64_fetch_add arch_atomic64_fetch_add
 
-static inline void arch_atomic64_add(s64 i, atomic64_t *v)
+static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
 {
 	__atomic64_add(i, (long *)&v->counter);
 }
 #define arch_atomic64_add arch_atomic64_add
 
 #define arch_atomic64_xchg(v, new)	(arch_xchg(&((v)->counter), new))
 
-static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
+static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
 {
 	return __atomic64_cmpxchg((long *)&v->counter, old, new);
 }
 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
 
-#define ATOMIC64_OPS(op)						\
-static inline void arch_atomic64_##op(s64 i, atomic64_t *v)		\
-{									\
-	__atomic64_##op(i, (long *)&v->counter);			\
-}									\
-static inline long arch_atomic64_fetch_##op(s64 i, atomic64_t *v)	\
-{									\
-	return __atomic64_##op##_barrier(i, (long *)&v->counter);	\
+#define ATOMIC64_OPS(op)							\
+static __always_inline void arch_atomic64_##op(s64 i, atomic64_t *v)		\
+{										\
+	__atomic64_##op(i, (long *)&v->counter);				\
+}										\
+static __always_inline long arch_atomic64_fetch_##op(s64 i, atomic64_t *v)	\
+{										\
+	return __atomic64_##op##_barrier(i, (long *)&v->counter);		\
 }
 
 ATOMIC64_OPS(and)
diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h
@@ -8,7 +8,7 @@
 #ifndef __ARCH_S390_ATOMIC_OPS__
 #define __ARCH_S390_ATOMIC_OPS__
 
-static inline int __atomic_read(const atomic_t *v)
+static __always_inline int __atomic_read(const atomic_t *v)
 {
 	int c;
 
@@ -18,14 +18,14 @@ static inline int __atomic_read(const atomic_t *v)
 	return c;
 }
 
-static inline void __atomic_set(atomic_t *v, int i)
+static __always_inline void __atomic_set(atomic_t *v, int i)
 {
 	asm volatile(
 		"	st	%1,%0\n"
 		: "=R" (v->counter) : "d" (i));
 }
 
-static inline s64 __atomic64_read(const atomic64_t *v)
+static __always_inline s64 __atomic64_read(const atomic64_t *v)
 {
 	s64 c;
 
@@ -35,7 +35,7 @@ static inline s64 __atomic64_read(const atomic64_t *v)
 	return c;
 }
 
-static inline void __atomic64_set(atomic64_t *v, s64 i)
+static __always_inline void __atomic64_set(atomic64_t *v, s64 i)
 {
 	asm volatile(
 		"	stg	%1,%0\n"
@@ -45,7 +45,7 @@ static inline void __atomic64_set(atomic64_t *v, s64 i)
 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 
 #define __ATOMIC_OP(op_name, op_type, op_string, op_barrier)		\
-static inline op_type op_name(op_type val, op_type *ptr)		\
+static __always_inline op_type op_name(op_type val, op_type *ptr)	\
 {									\
 	op_type old;							\
 									\
@@ -96,7 +96,7 @@ __ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi")
 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 
 #define __ATOMIC_OP(op_name, op_string)					\
-static inline int op_name(int val, int *ptr)				\
+static __always_inline int op_name(int val, int *ptr)			\
 {									\
 	int old, new;							\
 									\
@@ -122,7 +122,7 @@ __ATOMIC_OPS(__atomic_xor, "xr")
 #undef __ATOMIC_OPS
 
 #define __ATOMIC64_OP(op_name, op_string)				\
-static inline long op_name(long val, long *ptr)				\
+static __always_inline long op_name(long val, long *ptr)		\
 {									\
 	long old, new;							\
 									\
@@ -154,7 +154,7 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr")
 
 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 
-static inline int __atomic_cmpxchg(int *ptr, int old, int new)
+static __always_inline int __atomic_cmpxchg(int *ptr, int old, int new)
 {
 	asm volatile(
 		"	cs	%[old],%[new],%[ptr]"
@@ -164,7 +164,7 @@ static inline int __atomic_cmpxchg(int *ptr, int old, int new)
 	return old;
 }
 
-static inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
+static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
 {
 	int old_expected = old;
 
@@ -176,7 +176,7 @@ static inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
 	return old == old_expected;
 }
 
-static inline long __atomic64_cmpxchg(long *ptr, long old, long new)
+static __always_inline long __atomic64_cmpxchg(long *ptr, long old, long new)
 {
 	asm volatile(
 		"	csg	%[old],%[new],%[ptr]"
@@ -186,7 +186,7 @@ static inline long __atomic64_cmpxchg(long *ptr, long old, long new)
 	return old;
 }
 
-static inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)
+static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)
 {
 	long old_expected = old;
 

Original file line number	Diff line number	Diff line change
`@@ -15,31 +15,31 @@`
`15`	`15`	`#include <asm/barrier.h>`
`16`	`16`	`#include <asm/cmpxchg.h>`
`17`	`17`
`18`		`-static inline int arch_atomic_read(const atomic_t *v)`
	`18`	`+static __always_inline int arch_atomic_read(const atomic_t *v)`
`19`	`19`	`{`
`20`	`20`	`return __atomic_read(v);`
`21`	`21`	`}`
`22`	`22`	`#define arch_atomic_read arch_atomic_read`
`23`	`23`
`24`		`-static inline void arch_atomic_set(atomic_t *v, int i)`
	`24`	`+static __always_inline void arch_atomic_set(atomic_t *v, int i)`
`25`	`25`	`{`
`26`	`26`	`__atomic_set(v, i);`
`27`	`27`	`}`
`28`	`28`	`#define arch_atomic_set arch_atomic_set`
`29`	`29`
`30`		`-static inline int arch_atomic_add_return(int i, atomic_t *v)`
	`30`	`+static __always_inline int arch_atomic_add_return(int i, atomic_t *v)`
`31`	`31`	`{`
`32`	`32`	`return __atomic_add_barrier(i, &v->counter) + i;`
`33`	`33`	`}`
`34`	`34`	`#define arch_atomic_add_return arch_atomic_add_return`
`35`	`35`
`36`		`-static inline int arch_atomic_fetch_add(int i, atomic_t *v)`
	`36`	`+static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)`
`37`	`37`	`{`
`38`	`38`	`return __atomic_add_barrier(i, &v->counter);`
`39`	`39`	`}`
`40`	`40`	`#define arch_atomic_fetch_add arch_atomic_fetch_add`
`41`	`41`
`42`		`-static inline void arch_atomic_add(int i, atomic_t *v)`
	`42`	`+static __always_inline void arch_atomic_add(int i, atomic_t *v)`
`43`	`43`	`{`
`44`	`44`	`__atomic_add(i, &v->counter);`
`45`	`45`	`}`
`@@ -50,11 +50,11 @@ static inline void arch_atomic_add(int i, atomic_t *v)`
`50`	`50`	`#define arch_atomic_fetch_sub(_i, _v) arch_atomic_fetch_add(-(int)(_i), _v)`
`51`	`51`
`52`	`52`	`#define ATOMIC_OPS(op) \`
`53`		`-static inline void arch_atomic_##op(int i, atomic_t *v) \`
	`53`	`+static __always_inline void arch_atomic_##op(int i, atomic_t *v) \`
`54`	`54`	`{ \`
`55`	`55`	`__atomic_##op(i, &v->counter); \`
`56`	`56`	`} \`
`57`		`-static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \`
	`57`	`+static __always_inline int arch_atomic_fetch_##op(int i, atomic_t *v) \`
`58`	`58`	`{ \`
`59`	`59`	`return __atomic_##op##_barrier(i, &v->counter); \`
`60`	`60`	`}`
`@@ -74,60 +74,60 @@ ATOMIC_OPS(xor)`
`74`	`74`
`75`	`75`	`#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))`
`76`	`76`
`77`		`-static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)`
	`77`	`+static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)`
`78`	`78`	`{`
`79`	`79`	`return __atomic_cmpxchg(&v->counter, old, new);`
`80`	`80`	`}`
`81`	`81`	`#define arch_atomic_cmpxchg arch_atomic_cmpxchg`
`82`	`82`
`83`	`83`	`#define ATOMIC64_INIT(i) { (i) }`
`84`	`84`
`85`		`-static inline s64 arch_atomic64_read(const atomic64_t *v)`
	`85`	`+static __always_inline s64 arch_atomic64_read(const atomic64_t *v)`
`86`	`86`	`{`
`87`	`87`	`return __atomic64_read(v);`
`88`	`88`	`}`
`89`	`89`	`#define arch_atomic64_read arch_atomic64_read`
`90`	`90`
`91`		`-static inline void arch_atomic64_set(atomic64_t *v, s64 i)`
	`91`	`+static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)`
`92`	`92`	`{`
`93`	`93`	`__atomic64_set(v, i);`
`94`	`94`	`}`
`95`	`95`	`#define arch_atomic64_set arch_atomic64_set`
`96`	`96`
`97`		`-static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)`
	`97`	`+static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)`
`98`	`98`	`{`
`99`	`99`	`return __atomic64_add_barrier(i, (long *)&v->counter) + i;`
`100`	`100`	`}`
`101`	`101`	`#define arch_atomic64_add_return arch_atomic64_add_return`
`102`	`102`
`103`		`-static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)`
	`103`	`+static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)`
`104`	`104`	`{`
`105`	`105`	`return __atomic64_add_barrier(i, (long *)&v->counter);`
`106`	`106`	`}`
`107`	`107`	`#define arch_atomic64_fetch_add arch_atomic64_fetch_add`
`108`	`108`
`109`		`-static inline void arch_atomic64_add(s64 i, atomic64_t *v)`
	`109`	`+static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)`
`110`	`110`	`{`
`111`	`111`	`__atomic64_add(i, (long *)&v->counter);`
`112`	`112`	`}`
`113`	`113`	`#define arch_atomic64_add arch_atomic64_add`
`114`	`114`
`115`	`115`	`#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new))`
`116`	`116`
`117`		`-static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)`
	`117`	`+static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)`
`118`	`118`	`{`
`119`	`119`	`return __atomic64_cmpxchg((long *)&v->counter, old, new);`
`120`	`120`	`}`
`121`	`121`	`#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg`
`122`	`122`
`123`		`-#define ATOMIC64_OPS(op) \`
`124`		`-static inline void arch_atomic64_##op(s64 i, atomic64_t *v) \`
`125`		`-{ \`
`126`		`- __atomic64_##op(i, (long *)&v->counter); \`
`127`		`-} \`
`128`		`-static inline long arch_atomic64_fetch_##op(s64 i, atomic64_t *v) \`
`129`		`-{ \`
`130`		`- return __atomic64_##op##_barrier(i, (long *)&v->counter); \`
	`123`	`+#define ATOMIC64_OPS(op) \`
	`124`	`+static __always_inline void arch_atomic64_##op(s64 i, atomic64_t *v) \`
	`125`	`+{ \`
	`126`	`+ __atomic64_##op(i, (long *)&v->counter); \`
	`127`	`+} \`
	`128`	`+static __always_inline long arch_atomic64_fetch_##op(s64 i, atomic64_t *v) \`
	`129`	`+{ \`
	`130`	`+ return __atomic64_##op##_barrier(i, (long *)&v->counter); \`
`131`	`131`	`}`
`132`	`132`
`133`	`133`	`ATOMIC64_OPS(and)`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`	`#ifndef __ARCH_S390_ATOMIC_OPS__`
`9`	`9`	`#define __ARCH_S390_ATOMIC_OPS__`
`10`	`10`
`11`		`-static inline int __atomic_read(const atomic_t *v)`
	`11`	`+static __always_inline int __atomic_read(const atomic_t *v)`
`12`	`12`	`{`
`13`	`13`	`int c;`
`14`	`14`
`@@ -18,14 +18,14 @@ static inline int __atomic_read(const atomic_t *v)`
`18`	`18`	`return c;`
`19`	`19`	`}`
`20`	`20`
`21`		`-static inline void __atomic_set(atomic_t *v, int i)`
	`21`	`+static __always_inline void __atomic_set(atomic_t *v, int i)`
`22`	`22`	`{`
`23`	`23`	`asm volatile(`
`24`	`24`	`" st %1,%0\n"`
`25`	`25`	`: "=R" (v->counter) : "d" (i));`
`26`	`26`	`}`
`27`	`27`
`28`		`-static inline s64 __atomic64_read(const atomic64_t *v)`
	`28`	`+static __always_inline s64 __atomic64_read(const atomic64_t *v)`
`29`	`29`	`{`
`30`	`30`	`s64 c;`
`31`	`31`
`@@ -35,7 +35,7 @@ static inline s64 __atomic64_read(const atomic64_t *v)`
`35`	`35`	`return c;`
`36`	`36`	`}`
`37`	`37`
`38`		`-static inline void __atomic64_set(atomic64_t *v, s64 i)`
	`38`	`+static __always_inline void __atomic64_set(atomic64_t *v, s64 i)`
`39`	`39`	`{`
`40`	`40`	`asm volatile(`
`41`	`41`	`" stg %1,%0\n"`
`@@ -45,7 +45,7 @@ static inline void __atomic64_set(atomic64_t *v, s64 i)`
`45`	`45`	`#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES`
`46`	`46`
`47`	`47`	`#define __ATOMIC_OP(op_name, op_type, op_string, op_barrier) \`
`48`		`-static inline op_type op_name(op_type val, op_type *ptr) \`
	`48`	`+static __always_inline op_type op_name(op_type val, op_type *ptr) \`
`49`	`49`	`{ \`
`50`	`50`	`op_type old; \`
`51`	`51`	`\`
`@@ -96,7 +96,7 @@ __ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi")`
`96`	`96`	`#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */`
`97`	`97`
`98`	`98`	`#define __ATOMIC_OP(op_name, op_string) \`
`99`		`-static inline int op_name(int val, int *ptr) \`
	`99`	`+static __always_inline int op_name(int val, int *ptr) \`
`100`	`100`	`{ \`
`101`	`101`	`int old, new; \`
`102`	`102`	`\`
`@@ -122,7 +122,7 @@ __ATOMIC_OPS(__atomic_xor, "xr")`
`122`	`122`	`#undef __ATOMIC_OPS`
`123`	`123`
`124`	`124`	`#define __ATOMIC64_OP(op_name, op_string) \`
`125`		`-static inline long op_name(long val, long *ptr) \`
	`125`	`+static __always_inline long op_name(long val, long *ptr) \`
`126`	`126`	`{ \`
`127`	`127`	`long old, new; \`
`128`	`128`	`\`
`@@ -154,7 +154,7 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr")`
`154`	`154`
`155`	`155`	`#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */`
`156`	`156`
`157`		`-static inline int __atomic_cmpxchg(int *ptr, int old, int new)`
	`157`	`+static __always_inline int __atomic_cmpxchg(int *ptr, int old, int new)`
`158`	`158`	`{`
`159`	`159`	`asm volatile(`
`160`	`160`	`" cs %[old],%[new],%[ptr]"`
`@@ -164,7 +164,7 @@ static inline int __atomic_cmpxchg(int *ptr, int old, int new)`
`164`	`164`	`return old;`
`165`	`165`	`}`
`166`	`166`
`167`		`-static inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)`
	`167`	`+static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)`
`168`	`168`	`{`
`169`	`169`	`int old_expected = old;`
`170`	`170`
`@@ -176,7 +176,7 @@ static inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)`
`176`	`176`	`return old == old_expected;`
`177`	`177`	`}`
`178`	`178`
`179`		`-static inline long __atomic64_cmpxchg(long *ptr, long old, long new)`
	`179`	`+static __always_inline long __atomic64_cmpxchg(long *ptr, long old, long new)`
`180`	`180`	`{`
`181`	`181`	`asm volatile(`
`182`	`182`	`" csg %[old],%[new],%[ptr]"`
`@@ -186,7 +186,7 @@ static inline long __atomic64_cmpxchg(long *ptr, long old, long new)`
`186`	`186`	`return old;`
`187`	`187`	`}`
`188`	`188`
`189`		`-static inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)`
	`189`	`+static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)`
`190`	`190`	`{`
`191`	`191`	`long old_expected = old;`
`192`	`192`