Skip to content

Commit e63985e

Browse files
puranjaymohananakryiko
puranjaymohan
authored and
anakryiko
committed
bpf, riscv64/cfi: Support kCFI + BPF on riscv64
The riscv BPF JIT doesn't emit proper kCFI prologues for BPF programs and struct_ops trampolines when CONFIG_CFI_CLANG is enabled. This causes CFI failures when calling BPF programs and can even crash the kernel due to invalid memory accesses. Example crash: root@rv-selftester:~/bpf# ./test_progs -a dummy_st_ops Unable to handle kernel paging request at virtual address ffffffff78204ffc Oops [#1] Modules linked in: bpf_testmod(OE) [....] CPU: 3 PID: 356 Comm: test_progs Tainted: P OE 6.8.0-rc1 #1 Hardware name: riscv-virtio,qemu (DT) epc : bpf_struct_ops_test_run+0x28c/0x5fc ra : bpf_struct_ops_test_run+0x26c/0x5fc epc : ffffffff82958010 ra : ffffffff82957ff0 sp : ff200000007abc80 gp : ffffffff868d6218 tp : ff6000008d87b840 t0 : 000000000000000f t1 : 0000000000000000 t2 : 000000002005793e s0 : ff200000007abcf0 s1 : ff6000008a90fee0 a0 : 0000000000000000 a1 : 0000000000000000 a2 : 0000000000000000 a3 : 0000000000000000 a4 : 0000000000000000 a5 : ffffffff868dba26 a6 : 0000000000000001 a7 : 0000000052464e43 s2 : 00007ffffc0a95f0 s3 : ff6000008a90fe80 s4 : ff60000084c24c00 s5 : ffffffff78205000 s6 : ff60000088750648 s7 : ff20000000035008 s8 : fffffffffffffff4 s9 : ffffffff86200610 s10: 0000000000000000 s11: 0000000000000000 t3 : ffffffff8483dc30 t4 : ffffffff8483dc10 t5 : ffffffff8483dbf0 t6 : ffffffff8483dbd0 status: 0000000200000120 badaddr: ffffffff78204ffc cause: 000000000000000d [<ffffffff82958010>] bpf_struct_ops_test_run+0x28c/0x5fc [<ffffffff805083ee>] bpf_prog_test_run+0x170/0x548 [<ffffffff805029c8>] __sys_bpf+0x2d2/0x378 [<ffffffff804ff570>] __riscv_sys_bpf+0x5c/0x120 [<ffffffff8000e8fe>] syscall_handler+0x62/0xe4 [<ffffffff83362df6>] do_trap_ecall_u+0xc6/0x27c [<ffffffff833822c4>] ret_from_exception+0x0/0x64 Code: b603 0109 b683 0189 b703 0209 8493 0609 157d 8d65 (a303) ffca ---[ end trace 0000000000000000 ]--- Kernel panic - not syncing: Fatal exception SMP: stopping secondary CPUs Implement proper kCFI prologues for the BPF programs and callbacks and drop __nocfi for riscv64. Fix the trampoline generation code to emit kCFI prologue when a struct_ops trampoline is being prepared. Signed-off-by: Puranjay Mohan <[email protected]> Signed-off-by: Daniel Borkmann <[email protected]> Signed-off-by: Andrii Nakryiko <[email protected]> Acked-by: Björn Töpel <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent 516fca5 commit e63985e

File tree

6 files changed

+90
-7
lines changed

6 files changed

+90
-7
lines changed

‎arch/riscv/include/asm/cfi.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,28 @@ struct pt_regs;
1313

1414
#ifdef CONFIG_CFI_CLANG
1515
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
16+
#define __bpfcall
17+
static inline int cfi_get_offset(void)
18+
{
19+
return 4;
20+
}
21+
22+
#define cfi_get_offset cfi_get_offset
23+
extern u32 cfi_bpf_hash;
24+
extern u32 cfi_bpf_subprog_hash;
25+
extern u32 cfi_get_func_hash(void *func);
1626
#else
1727
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
1828
{
1929
return BUG_TRAP_TYPE_NONE;
2030
}
31+
32+
#define cfi_bpf_hash 0U
33+
#define cfi_bpf_subprog_hash 0U
34+
static inline u32 cfi_get_func_hash(void *func)
35+
{
36+
return 0;
37+
}
2138
#endif /* CONFIG_CFI_CLANG */
2239

2340
#endif /* _ASM_RISCV_CFI_H */

‎arch/riscv/kernel/cfi.c‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,56 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
7575

7676
return report_cfi_failure(regs, regs->epc, &target, type);
7777
}
78+
79+
#ifdef CONFIG_CFI_CLANG
80+
struct bpf_insn;
81+
82+
/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
83+
extern unsigned int __bpf_prog_runX(const void *ctx,
84+
const struct bpf_insn *insn);
85+
86+
/*
87+
* Force a reference to the external symbol so the compiler generates
88+
* __kcfi_typid.
89+
*/
90+
__ADDRESSABLE(__bpf_prog_runX);
91+
92+
/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
93+
asm (
94+
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
95+
" .type cfi_bpf_hash,@object \n"
96+
" .globl cfi_bpf_hash \n"
97+
" .p2align 2, 0x0 \n"
98+
"cfi_bpf_hash: \n"
99+
" .word __kcfi_typeid___bpf_prog_runX \n"
100+
" .size cfi_bpf_hash, 4 \n"
101+
" .popsection \n"
102+
);
103+
104+
/* Must match bpf_callback_t */
105+
extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
106+
107+
__ADDRESSABLE(__bpf_callback_fn);
108+
109+
/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
110+
asm (
111+
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
112+
" .type cfi_bpf_subprog_hash,@object \n"
113+
" .globl cfi_bpf_subprog_hash \n"
114+
" .p2align 2, 0x0 \n"
115+
"cfi_bpf_subprog_hash: \n"
116+
" .word __kcfi_typeid___bpf_callback_fn \n"
117+
" .size cfi_bpf_subprog_hash, 4 \n"
118+
" .popsection \n"
119+
);
120+
121+
u32 cfi_get_func_hash(void *func)
122+
{
123+
u32 hash;
124+
125+
if (get_kernel_nofault(hash, func - cfi_get_offset()))
126+
return 0;
127+
128+
return hash;
129+
}
130+
#endif

‎arch/riscv/net/bpf_jit.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1223,7 +1223,7 @@ static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx)
12231223

12241224
#endif /* __riscv_xlen == 64 */
12251225

1226-
void bpf_jit_build_prologue(struct rv_jit_context *ctx);
1226+
void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog);
12271227
void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
12281228

12291229
int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,

‎arch/riscv/net/bpf_jit_comp32.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1301,7 +1301,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
13011301
return 0;
13021302
}
13031303

1304-
void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1304+
void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
13051305
{
13061306
const s8 *fp = bpf2rv32[BPF_REG_FP];
13071307
const s8 *r1 = bpf2rv32[BPF_REG_1];

‎arch/riscv/net/bpf_jit_comp64.c‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <linux/memory.h>
1212
#include <linux/stop_machine.h>
1313
#include <asm/patch.h>
14+
#include <asm/cfi.h>
1415
#include "bpf_jit.h"
1516

1617
#define RV_FENTRY_NINSNS 2
@@ -455,6 +456,12 @@ static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx)
455456
return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx);
456457
}
457458

459+
static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx)
460+
{
461+
if (IS_ENABLED(CONFIG_CFI_CLANG))
462+
emit(hash, ctx);
463+
}
464+
458465
static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
459466
struct rv_jit_context *ctx)
460467
{
@@ -869,6 +876,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
869876
emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx);
870877
emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx);
871878
} else {
879+
/* emit kcfi hash */
880+
emit_kcfi(cfi_get_func_hash(func_addr), ctx);
872881
/* For the trampoline called directly, just handle
873882
* the frame of trampoline.
874883
*/
@@ -1711,7 +1720,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
17111720
return 0;
17121721
}
17131722

1714-
void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1723+
void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
17151724
{
17161725
int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
17171726

@@ -1740,6 +1749,9 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx)
17401749

17411750
store_offset = stack_adjust - 8;
17421751

1752+
/* emit kcfi type preamble immediately before the first insn */
1753+
emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx);
1754+
17431755
/* nops reserved for auipc+jalr pair */
17441756
for (i = 0; i < RV_FENTRY_NINSNS; i++)
17451757
emit(rv_nop(), ctx);

‎arch/riscv/net/bpf_jit_core.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <linux/filter.h>
1111
#include <linux/memory.h>
1212
#include <asm/patch.h>
13+
#include <asm/cfi.h>
1314
#include "bpf_jit.h"
1415

1516
/* Number of iterations to try until offsets converge. */
@@ -100,7 +101,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
100101
pass++;
101102
ctx->ninsns = 0;
102103

103-
bpf_jit_build_prologue(ctx);
104+
bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
104105
ctx->prologue_len = ctx->ninsns;
105106

106107
if (build_body(ctx, extra_pass, ctx->offset)) {
@@ -160,7 +161,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
160161
ctx->ninsns = 0;
161162
ctx->nexentries = 0;
162163

163-
bpf_jit_build_prologue(ctx);
164+
bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
164165
if (build_body(ctx, extra_pass, NULL)) {
165166
prog = orig_prog;
166167
goto out_free_hdr;
@@ -170,9 +171,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
170171
if (bpf_jit_enable > 1)
171172
bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
172173

173-
prog->bpf_func = (void *)ctx->ro_insns;
174+
prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset();
174175
prog->jited = 1;
175-
prog->jited_len = prog_size;
176+
prog->jited_len = prog_size - cfi_get_offset();
176177

177178
if (!prog->is_func || extra_pass) {
178179
if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,

0 commit comments

Comments
 (0)