Skip to content

Commit 4b5305d

Browse files
author
Peter Zijlstra
committed
x86/extable: Extend extable functionality
In order to remove further .fixup usage, extend the extable infrastructure to take additional information from the extable entry sites. Specifically add _ASM_EXTABLE_TYPE_REG() and EX_TYPE_IMM_REG that extend the existing _ASM_EXTABLE_TYPE() by taking an additional register argument and encoding that and an s16 immediate into the existing s32 type field. This limits the actual types to the first byte, 255 seem plenty. Also add a few flags into the type word, specifically CLEAR_AX and CLEAR_DX which clear the return and extended return register. Notes: - due to the % in our register names it's hard to make it more generally usable as arm64 did. - the s16 is far larger than used in these patches, future extentions can easily shrink this to get more bits. - without the bitfield fix this will not compile, because: 0xFF > -1 and we can't even extract the TYPE field. [nathanchance: Build fix for clang-lto builds: https://lkml.kernel.org/r/[email protected] ] Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Reviewed-by: Josh Poimboeuf <[email protected]> Reviewed-by: Nick Desaulniers <[email protected]> Tested-by: Nick Desaulniers <[email protected]> Link: https://lore.kernel.org/r/[email protected]
1 parent aa93e2a commit 4b5305d

File tree

7 files changed

+145
-32
lines changed

7 files changed

+145
-32
lines changed

‎arch/x86/include/asm/asm.h

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,33 @@
152152

153153
#else /* ! __ASSEMBLY__ */
154154

155+
# define DEFINE_EXTABLE_TYPE_REG \
156+
".macro extable_type_reg type:req reg:req\n" \
157+
".set found, 0\n" \
158+
".set regnr, 0\n" \
159+
".irp rs,rax,rcx,rdx,rbx,rsp,rbp,rsi,rdi,r8,r9,r10,r11,r12,r13,r14,r15\n" \
160+
".ifc \\reg, %%\\rs\n" \
161+
".set found, found+1\n" \
162+
".long \\type + (regnr << 8)\n" \
163+
".endif\n" \
164+
".set regnr, regnr+1\n" \
165+
".endr\n" \
166+
".set regnr, 0\n" \
167+
".irp rs,eax,ecx,edx,ebx,esp,ebp,esi,edi,r8d,r9d,r10d,r11d,r12d,r13d,r14d,r15d\n" \
168+
".ifc \\reg, %%\\rs\n" \
169+
".set found, found+1\n" \
170+
".long \\type + (regnr << 8)\n" \
171+
".endif\n" \
172+
".set regnr, regnr+1\n" \
173+
".endr\n" \
174+
".if (found != 1)\n" \
175+
".error \"extable_type_reg: bad register argument\"\n" \
176+
".endif\n" \
177+
".endm\n"
178+
179+
# define UNDEFINE_EXTABLE_TYPE_REG \
180+
".purgem extable_type_reg\n"
181+
155182
# define _ASM_EXTABLE_TYPE(from, to, type) \
156183
" .pushsection \"__ex_table\",\"a\"\n" \
157184
" .balign 4\n" \
@@ -160,6 +187,16 @@
160187
" .long " __stringify(type) " \n" \
161188
" .popsection\n"
162189

190+
# define _ASM_EXTABLE_TYPE_REG(from, to, type, reg) \
191+
" .pushsection \"__ex_table\",\"a\"\n" \
192+
" .balign 4\n" \
193+
" .long (" #from ") - .\n" \
194+
" .long (" #to ") - .\n" \
195+
DEFINE_EXTABLE_TYPE_REG \
196+
"extable_type_reg reg=" __stringify(reg) ", type=" __stringify(type) " \n"\
197+
UNDEFINE_EXTABLE_TYPE_REG \
198+
" .popsection\n"
199+
163200
/* For C file, we already have NOKPROBE_SYMBOL macro */
164201

165202
/*

‎arch/x86/include/asm/extable.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
*/
2222

2323
struct exception_table_entry {
24-
int insn, fixup, type;
24+
int insn, fixup, data;
2525
};
2626
struct pt_regs;
2727

@@ -31,8 +31,8 @@ struct pt_regs;
3131
do { \
3232
(a)->fixup = (b)->fixup + (delta); \
3333
(b)->fixup = (tmp).fixup - (delta); \
34-
(a)->type = (b)->type; \
35-
(b)->type = (tmp).type; \
34+
(a)->data = (b)->data; \
35+
(b)->data = (tmp).data; \
3636
} while (0)
3737

3838
extern int fixup_exception(struct pt_regs *regs, int trapnr,

‎arch/x86/include/asm/extable_fixup_types.h

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,29 @@
22
#ifndef _ASM_X86_EXTABLE_FIXUP_TYPES_H
33
#define _ASM_X86_EXTABLE_FIXUP_TYPES_H
44

5+
/*
6+
* Our IMM is signed, as such it must live at the top end of the word. Also,
7+
* since C99 hex constants are of ambigious type, force cast the mask to 'int'
8+
* so that FIELD_GET() will DTRT and sign extend the value when it extracts it.
9+
*/
10+
#define EX_DATA_TYPE_MASK ((int)0x000000FF)
11+
#define EX_DATA_REG_MASK ((int)0x00000F00)
12+
#define EX_DATA_FLAG_MASK ((int)0x0000F000)
13+
#define EX_DATA_IMM_MASK ((int)0xFFFF0000)
14+
15+
#define EX_DATA_REG_SHIFT 8
16+
#define EX_DATA_FLAG_SHIFT 12
17+
#define EX_DATA_IMM_SHIFT 16
18+
19+
#define EX_DATA_FLAG(flag) ((flag) << EX_DATA_FLAG_SHIFT)
20+
#define EX_DATA_IMM(imm) ((imm) << EX_DATA_IMM_SHIFT)
21+
22+
/* flags */
23+
#define EX_FLAG_CLEAR_AX EX_DATA_FLAG(1)
24+
#define EX_FLAG_CLEAR_DX EX_DATA_FLAG(2)
25+
#define EX_FLAG_CLEAR_AX_DX EX_DATA_FLAG(3)
26+
27+
/* types */
528
#define EX_TYPE_NONE 0
629
#define EX_TYPE_DEFAULT 1
730
#define EX_TYPE_FAULT 2
@@ -20,5 +43,6 @@
2043
#define EX_TYPE_FAULT_MCE_SAFE 13
2144

2245
#define EX_TYPE_POP_ZERO 14
46+
#define EX_TYPE_IMM_REG 15 /* reg := (long)imm */
2347

2448
#endif

‎arch/x86/include/asm/insn-eval.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
#define INSN_CODE_SEG_OPND_SZ(params) (params & 0xf)
1616
#define INSN_CODE_SEG_PARAMS(oper_sz, addr_sz) (oper_sz | (addr_sz << 4))
1717

18+
int pt_regs_offset(struct pt_regs *regs, int regno);
19+
1820
bool insn_has_rep_prefix(struct insn *insn);
1921
void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs);
2022
int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs);

‎arch/x86/lib/insn-eval.c

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -412,32 +412,39 @@ static short get_segment_selector(struct pt_regs *regs, int seg_reg_idx)
412412
#endif /* CONFIG_X86_64 */
413413
}
414414

415-
static int get_reg_offset(struct insn *insn, struct pt_regs *regs,
416-
enum reg_type type)
415+
static const int pt_regoff[] = {
416+
offsetof(struct pt_regs, ax),
417+
offsetof(struct pt_regs, cx),
418+
offsetof(struct pt_regs, dx),
419+
offsetof(struct pt_regs, bx),
420+
offsetof(struct pt_regs, sp),
421+
offsetof(struct pt_regs, bp),
422+
offsetof(struct pt_regs, si),
423+
offsetof(struct pt_regs, di),
424+
#ifdef CONFIG_X86_64
425+
offsetof(struct pt_regs, r8),
426+
offsetof(struct pt_regs, r9),
427+
offsetof(struct pt_regs, r10),
428+
offsetof(struct pt_regs, r11),
429+
offsetof(struct pt_regs, r12),
430+
offsetof(struct pt_regs, r13),
431+
offsetof(struct pt_regs, r14),
432+
offsetof(struct pt_regs, r15),
433+
#endif
434+
};
435+
436+
int pt_regs_offset(struct pt_regs *regs, int regno)
417437
{
438+
if ((unsigned)regno < ARRAY_SIZE(pt_regoff))
439+
return pt_regoff[regno];
440+
return -EDOM;
441+
}
442+
443+
static int get_regno(struct insn *insn, enum reg_type type)
444+
{
445+
int nr_registers = ARRAY_SIZE(pt_regoff);
418446
int regno = 0;
419447

420-
static const int regoff[] = {
421-
offsetof(struct pt_regs, ax),
422-
offsetof(struct pt_regs, cx),
423-
offsetof(struct pt_regs, dx),
424-
offsetof(struct pt_regs, bx),
425-
offsetof(struct pt_regs, sp),
426-
offsetof(struct pt_regs, bp),
427-
offsetof(struct pt_regs, si),
428-
offsetof(struct pt_regs, di),
429-
#ifdef CONFIG_X86_64
430-
offsetof(struct pt_regs, r8),
431-
offsetof(struct pt_regs, r9),
432-
offsetof(struct pt_regs, r10),
433-
offsetof(struct pt_regs, r11),
434-
offsetof(struct pt_regs, r12),
435-
offsetof(struct pt_regs, r13),
436-
offsetof(struct pt_regs, r14),
437-
offsetof(struct pt_regs, r15),
438-
#endif
439-
};
440-
int nr_registers = ARRAY_SIZE(regoff);
441448
/*
442449
* Don't possibly decode a 32-bit instructions as
443450
* reading a 64-bit-only register.
@@ -505,7 +512,18 @@ static int get_reg_offset(struct insn *insn, struct pt_regs *regs,
505512
WARN_ONCE(1, "decoded an instruction with an invalid register");
506513
return -EINVAL;
507514
}
508-
return regoff[regno];
515+
return regno;
516+
}
517+
518+
static int get_reg_offset(struct insn *insn, struct pt_regs *regs,
519+
enum reg_type type)
520+
{
521+
int regno = get_regno(insn, type);
522+
523+
if (regno < 0)
524+
return regno;
525+
526+
return pt_regs_offset(regs, regno);
509527
}
510528

511529
/**

‎arch/x86/mm/extable.c

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,41 @@
22
#include <linux/extable.h>
33
#include <linux/uaccess.h>
44
#include <linux/sched/debug.h>
5+
#include <linux/bitfield.h>
56
#include <xen/xen.h>
67

78
#include <asm/fpu/api.h>
89
#include <asm/sev.h>
910
#include <asm/traps.h>
1011
#include <asm/kdebug.h>
12+
#include <asm/insn-eval.h>
13+
14+
static inline unsigned long *pt_regs_nr(struct pt_regs *regs, int nr)
15+
{
16+
int reg_offset = pt_regs_offset(regs, nr);
17+
static unsigned long __dummy;
18+
19+
if (WARN_ON_ONCE(reg_offset < 0))
20+
return &__dummy;
21+
22+
return (unsigned long *)((unsigned long)regs + reg_offset);
23+
}
1124

1225
static inline unsigned long
1326
ex_fixup_addr(const struct exception_table_entry *x)
1427
{
1528
return (unsigned long)&x->fixup + x->fixup;
1629
}
1730

18-
static bool ex_handler_default(const struct exception_table_entry *fixup,
31+
static bool ex_handler_default(const struct exception_table_entry *e,
1932
struct pt_regs *regs)
2033
{
21-
regs->ip = ex_fixup_addr(fixup);
34+
if (e->data & EX_FLAG_CLEAR_AX)
35+
regs->ax = 0;
36+
if (e->data & EX_FLAG_CLEAR_DX)
37+
regs->dx = 0;
38+
39+
regs->ip = ex_fixup_addr(e);
2240
return true;
2341
}
2442

@@ -111,17 +129,25 @@ static bool ex_handler_pop_zero(const struct exception_table_entry *fixup,
111129
return ex_handler_default(fixup, regs);
112130
}
113131

132+
static bool ex_handler_imm_reg(const struct exception_table_entry *fixup,
133+
struct pt_regs *regs, int reg, int imm)
134+
{
135+
*pt_regs_nr(regs, reg) = (long)imm;
136+
return ex_handler_default(fixup, regs);
137+
}
138+
114139
int ex_get_fixup_type(unsigned long ip)
115140
{
116141
const struct exception_table_entry *e = search_exception_tables(ip);
117142

118-
return e ? e->type : EX_TYPE_NONE;
143+
return e ? FIELD_GET(EX_DATA_TYPE_MASK, e->data) : EX_TYPE_NONE;
119144
}
120145

121146
int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
122147
unsigned long fault_addr)
123148
{
124149
const struct exception_table_entry *e;
150+
int type, reg, imm;
125151

126152
#ifdef CONFIG_PNPBIOS
127153
if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) {
@@ -141,7 +167,11 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
141167
if (!e)
142168
return 0;
143169

144-
switch (e->type) {
170+
type = FIELD_GET(EX_DATA_TYPE_MASK, e->data);
171+
reg = FIELD_GET(EX_DATA_REG_MASK, e->data);
172+
imm = FIELD_GET(EX_DATA_IMM_MASK, e->data);
173+
174+
switch (type) {
145175
case EX_TYPE_DEFAULT:
146176
case EX_TYPE_DEFAULT_MCE_SAFE:
147177
return ex_handler_default(e, regs);
@@ -170,6 +200,8 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
170200
break;
171201
case EX_TYPE_POP_ZERO:
172202
return ex_handler_pop_zero(e, regs);
203+
case EX_TYPE_IMM_REG:
204+
return ex_handler_imm_reg(e, regs, reg, imm);
173205
}
174206
BUG();
175207
}

‎arch/x86/net/bpf_jit_comp.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1291,7 +1291,7 @@ st: if (is_imm8(insn->off))
12911291
}
12921292
ex->insn = delta;
12931293

1294-
ex->type = EX_TYPE_BPF;
1294+
ex->data = EX_TYPE_BPF;
12951295

12961296
if (dst_reg > BPF_REG_9) {
12971297
pr_err("verifier error\n");

0 commit comments

Comments
 (0)