Skip to content

Commit 99fe09c

Browse files
willdeaconwilldeacon
committed
Merge branch 'for-next/extable' into for-next/core
* for-next/extable: arm64: vmlinux.lds.S: remove `.fixup` section arm64: extable: add load_unaligned_zeropad() handler arm64: extable: add a dedicated uaccess handler arm64: extable: add `type` and `data` fields arm64: extable: use `ex` for `exception_table_entry` arm64: extable: make fixup_exception() return bool arm64: extable: consolidate definitions arm64: gpr-num: support W registers arm64: factor out GPR numbering helpers arm64: kvm: use kvm_exception_table_entry arm64: lib: __arch_copy_to_user(): fold fixups into body arm64: lib: __arch_copy_from_user(): fold fixups into body arm64: lib: __arch_clear_user(): fold fixups into body
2 parents a69483e + bf6e667 commit 99fe09c

File tree

20 files changed

+310
-158
lines changed

20 files changed

+310
-158
lines changed

‎arch/arm64/include/asm/asm-extable.h

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/* SPDX-License-Identifier: GPL-2.0-only */
2+
#ifndef __ASM_ASM_EXTABLE_H
3+
#define __ASM_ASM_EXTABLE_H
4+
5+
#define EX_TYPE_NONE 0
6+
#define EX_TYPE_FIXUP 1
7+
#define EX_TYPE_BPF 2
8+
#define EX_TYPE_UACCESS_ERR_ZERO 3
9+
#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
10+
11+
#ifdef __ASSEMBLY__
12+
13+
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
14+
.pushsection __ex_table, "a"; \
15+
.align 2; \
16+
.long ((insn) - .); \
17+
.long ((fixup) - .); \
18+
.short (type); \
19+
.short (data); \
20+
.popsection;
21+
22+
/*
23+
* Create an exception table entry for `insn`, which will branch to `fixup`
24+
* when an unhandled fault is taken.
25+
*/
26+
.macro _asm_extable, insn, fixup
27+
__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
28+
.endm
29+
30+
/*
31+
* Create an exception table entry for `insn` if `fixup` is provided. Otherwise
32+
* do nothing.
33+
*/
34+
.macro _cond_extable, insn, fixup
35+
.ifnc \fixup,
36+
_asm_extable \insn, \fixup
37+
.endif
38+
.endm
39+
40+
#else /* __ASSEMBLY__ */
41+
42+
#include <linux/bits.h>
43+
#include <linux/stringify.h>
44+
45+
#include <asm/gpr-num.h>
46+
47+
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
48+
".pushsection __ex_table, \"a\"\n" \
49+
".align 2\n" \
50+
".long ((" insn ") - .)\n" \
51+
".long ((" fixup ") - .)\n" \
52+
".short (" type ")\n" \
53+
".short (" data ")\n" \
54+
".popsection\n"
55+
56+
#define _ASM_EXTABLE(insn, fixup) \
57+
__ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")
58+
59+
#define EX_DATA_REG_ERR_SHIFT 0
60+
#define EX_DATA_REG_ERR GENMASK(4, 0)
61+
#define EX_DATA_REG_ZERO_SHIFT 5
62+
#define EX_DATA_REG_ZERO GENMASK(9, 5)
63+
64+
#define EX_DATA_REG(reg, gpr) \
65+
"((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")"
66+
67+
#define _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero) \
68+
__DEFINE_ASM_GPR_NUMS \
69+
__ASM_EXTABLE_RAW(#insn, #fixup, \
70+
__stringify(EX_TYPE_UACCESS_ERR_ZERO), \
71+
"(" \
72+
EX_DATA_REG(ERR, err) " | " \
73+
EX_DATA_REG(ZERO, zero) \
74+
")")
75+
76+
#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) \
77+
_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, wzr)
78+
79+
#define EX_DATA_REG_DATA_SHIFT 0
80+
#define EX_DATA_REG_DATA GENMASK(4, 0)
81+
#define EX_DATA_REG_ADDR_SHIFT 5
82+
#define EX_DATA_REG_ADDR GENMASK(9, 5)
83+
84+
#define _ASM_EXTABLE_LOAD_UNALIGNED_ZEROPAD(insn, fixup, data, addr) \
85+
__DEFINE_ASM_GPR_NUMS \
86+
__ASM_EXTABLE_RAW(#insn, #fixup, \
87+
__stringify(EX_TYPE_LOAD_UNALIGNED_ZEROPAD), \
88+
"(" \
89+
EX_DATA_REG(DATA, data) " | " \
90+
EX_DATA_REG(ADDR, addr) \
91+
")")
92+
93+
#endif /* __ASSEMBLY__ */
94+
95+
#endif /* __ASM_ASM_EXTABLE_H */

‎arch/arm64/include/asm/asm-uaccess.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
#define __ASM_ASM_UACCESS_H
44

55
#include <asm/alternative-macros.h>
6+
#include <asm/asm-extable.h>
7+
#include <asm/assembler.h>
68
#include <asm/kernel-pgtable.h>
79
#include <asm/mmu.h>
810
#include <asm/sysreg.h>
9-
#include <asm/assembler.h>
1011

1112
/*
1213
* User access enabling/disabling macros.
@@ -58,6 +59,10 @@ alternative_else_nop_endif
5859
.endm
5960
#endif
6061

62+
#define USER(l, x...) \
63+
9999: x; \
64+
_asm_extable 9999b, l
65+
6166
/*
6267
* Generate the assembly for LDTR/STTR with exception table entries.
6368
* This is complicated as there is no post-increment or pair versions of the

‎arch/arm64/include/asm/assembler.h

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414

1515
#include <asm-generic/export.h>
1616

17-
#include <asm/asm-offsets.h>
1817
#include <asm/alternative.h>
1918
#include <asm/asm-bug.h>
19+
#include <asm/asm-extable.h>
20+
#include <asm/asm-offsets.h>
2021
#include <asm/cpufeature.h>
2122
#include <asm/cputype.h>
2223
#include <asm/debug-monitors.h>
@@ -129,32 +130,6 @@ alternative_endif
129130
.endr
130131
.endm
131132

132-
/*
133-
* Create an exception table entry for `insn`, which will branch to `fixup`
134-
* when an unhandled fault is taken.
135-
*/
136-
.macro _asm_extable, insn, fixup
137-
.pushsection __ex_table, "a"
138-
.align 3
139-
.long (\insn - .), (\fixup - .)
140-
.popsection
141-
.endm
142-
143-
/*
144-
* Create an exception table entry for `insn` if `fixup` is provided. Otherwise
145-
* do nothing.
146-
*/
147-
.macro _cond_extable, insn, fixup
148-
.ifnc \fixup,
149-
_asm_extable \insn, \fixup
150-
.endif
151-
.endm
152-
153-
154-
#define USER(l, x...) \
155-
9999: x; \
156-
_asm_extable 9999b, l
157-
158133
/*
159134
* Register aliases.
160135
*/

‎arch/arm64/include/asm/extable.h

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,21 @@
1818
struct exception_table_entry
1919
{
2020
int insn, fixup;
21+
short type, data;
2122
};
2223

2324
#define ARCH_HAS_RELATIVE_EXTABLE
2425

26+
#define swap_ex_entry_fixup(a, b, tmp, delta) \
27+
do { \
28+
(a)->fixup = (b)->fixup + (delta); \
29+
(b)->fixup = (tmp).fixup - (delta); \
30+
(a)->type = (b)->type; \
31+
(b)->type = (tmp).type; \
32+
(a)->data = (b)->data; \
33+
(b)->data = (tmp).data; \
34+
} while (0)
35+
2536
static inline bool in_bpf_jit(struct pt_regs *regs)
2637
{
2738
if (!IS_ENABLED(CONFIG_BPF_JIT))
@@ -32,16 +43,16 @@ static inline bool in_bpf_jit(struct pt_regs *regs)
3243
}
3344

3445
#ifdef CONFIG_BPF_JIT
35-
int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
36-
struct pt_regs *regs);
46+
bool ex_handler_bpf(const struct exception_table_entry *ex,
47+
struct pt_regs *regs);
3748
#else /* !CONFIG_BPF_JIT */
3849
static inline
39-
int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
40-
struct pt_regs *regs)
50+
bool ex_handler_bpf(const struct exception_table_entry *ex,
51+
struct pt_regs *regs)
4152
{
42-
return 0;
53+
return false;
4354
}
4455
#endif /* !CONFIG_BPF_JIT */
4556

46-
extern int fixup_exception(struct pt_regs *regs);
57+
bool fixup_exception(struct pt_regs *regs);
4758
#endif

‎arch/arm64/include/asm/futex.h

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,14 @@ do { \
2525
" cbz %w0, 3f\n" \
2626
" sub %w4, %w4, %w0\n" \
2727
" cbnz %w4, 1b\n" \
28-
" mov %w0, %w7\n" \
28+
" mov %w0, %w6\n" \
2929
"3:\n" \
3030
" dmb ish\n" \
31-
" .pushsection .fixup,\"ax\"\n" \
32-
" .align 2\n" \
33-
"4: mov %w0, %w6\n" \
34-
" b 3b\n" \
35-
" .popsection\n" \
36-
_ASM_EXTABLE(1b, 4b) \
37-
_ASM_EXTABLE(2b, 4b) \
31+
_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
32+
_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0) \
3833
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
3934
"+r" (loops) \
40-
: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
35+
: "r" (oparg), "Ir" (-EAGAIN) \
4136
: "memory"); \
4237
uaccess_disable_privileged(); \
4338
} while (0)
@@ -105,18 +100,14 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
105100
" cbz %w3, 3f\n"
106101
" sub %w4, %w4, %w3\n"
107102
" cbnz %w4, 1b\n"
108-
" mov %w0, %w8\n"
103+
" mov %w0, %w7\n"
109104
"3:\n"
110105
" dmb ish\n"
111106
"4:\n"
112-
" .pushsection .fixup,\"ax\"\n"
113-
"5: mov %w0, %w7\n"
114-
" b 4b\n"
115-
" .popsection\n"
116-
_ASM_EXTABLE(1b, 5b)
117-
_ASM_EXTABLE(2b, 5b)
107+
_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)
108+
_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)
118109
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
119-
: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
110+
: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)
120111
: "memory");
121112
uaccess_disable_privileged();
122113

‎arch/arm64/include/asm/gpr-num.h

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/* SPDX-License-Identifier: GPL-2.0-only */
2+
#ifndef __ASM_GPR_NUM_H
3+
#define __ASM_GPR_NUM_H
4+
5+
#ifdef __ASSEMBLY__
6+
7+
.irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
8+
.equ .L__gpr_num_x\num, \num
9+
.equ .L__gpr_num_w\num, \num
10+
.endr
11+
.equ .L__gpr_num_xzr, 31
12+
.equ .L__gpr_num_wzr, 31
13+
14+
#else /* __ASSEMBLY__ */
15+
16+
#define __DEFINE_ASM_GPR_NUMS \
17+
" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n" \
18+
" .equ .L__gpr_num_x\\num, \\num\n" \
19+
" .equ .L__gpr_num_w\\num, \\num\n" \
20+
" .endr\n" \
21+
" .equ .L__gpr_num_xzr, 31\n" \
22+
" .equ .L__gpr_num_wzr, 31\n"
23+
24+
#endif /* __ASSEMBLY__ */
25+
26+
#endif /* __ASM_GPR_NUM_H */

‎arch/arm64/include/asm/kvm_asm.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -263,9 +263,10 @@ extern u64 __kvm_get_mdcr_el2(void);
263263

264264
/*
265265
* KVM extable for unexpected exceptions.
266-
* In the same format _asm_extable, but output to a different section so that
267-
* it can be mapped to EL2. The KVM version is not sorted. The caller must
268-
* ensure:
266+
* Create a struct kvm_exception_table_entry output to a section that can be
267+
* mapped by EL2. The table is not sorted.
268+
*
269+
* The caller must ensure:
269270
* x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented
270271
* code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup.
271272
*/

‎arch/arm64/include/asm/sysreg.h

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
#include <linux/stringify.h>
1414
#include <linux/kasan-tags.h>
1515

16+
#include <asm/gpr-num.h>
17+
1618
/*
1719
* ARMv8 ARM reserves the following encoding for system registers:
1820
* (Ref: ARMv8 ARM, Section: "System instruction class encoding overview",
@@ -1195,17 +1197,12 @@
11951197

11961198
#ifdef __ASSEMBLY__
11971199

1198-
.irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
1199-
.equ .L__reg_num_x\num, \num
1200-
.endr
1201-
.equ .L__reg_num_xzr, 31
1202-
12031200
.macro mrs_s, rt, sreg
1204-
__emit_inst(0xd5200000|(\sreg)|(.L__reg_num_\rt))
1201+
__emit_inst(0xd5200000|(\sreg)|(.L__gpr_num_\rt))
12051202
.endm
12061203

12071204
.macro msr_s, sreg, rt
1208-
__emit_inst(0xd5000000|(\sreg)|(.L__reg_num_\rt))
1205+
__emit_inst(0xd5000000|(\sreg)|(.L__gpr_num_\rt))
12091206
.endm
12101207

12111208
#else
@@ -1214,22 +1211,16 @@
12141211
#include <linux/types.h>
12151212
#include <asm/alternative.h>
12161213

1217-
#define __DEFINE_MRS_MSR_S_REGNUM \
1218-
" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n" \
1219-
" .equ .L__reg_num_x\\num, \\num\n" \
1220-
" .endr\n" \
1221-
" .equ .L__reg_num_xzr, 31\n"
1222-
12231214
#define DEFINE_MRS_S \
1224-
__DEFINE_MRS_MSR_S_REGNUM \
1215+
__DEFINE_ASM_GPR_NUMS \
12251216
" .macro mrs_s, rt, sreg\n" \
1226-
__emit_inst(0xd5200000|(\\sreg)|(.L__reg_num_\\rt)) \
1217+
__emit_inst(0xd5200000|(\\sreg)|(.L__gpr_num_\\rt)) \
12271218
" .endm\n"
12281219

12291220
#define DEFINE_MSR_S \
1230-
__DEFINE_MRS_MSR_S_REGNUM \
1221+
__DEFINE_ASM_GPR_NUMS \
12311222
" .macro msr_s, sreg, rt\n" \
1232-
__emit_inst(0xd5000000|(\\sreg)|(.L__reg_num_\\rt)) \
1223+
__emit_inst(0xd5000000|(\\sreg)|(.L__gpr_num_\\rt)) \
12331224
" .endm\n"
12341225

12351226
#define UNDEFINE_MRS_S \

0 commit comments

Comments
 (0)