Skip to content

Commit 17f3153

Browse files
Pu Lehuitheihor
Pu Lehui
authored and
theihor
committed
riscv, bpf: Add ex_insn_off and ex_jmp_off for exception table handling
Add ex_insn_off and ex_jmp_off fields to struct rv_jit_context so that add_exception_handler() does not need to be immediately followed by the instruction to add the exception table. ex_insn_off indicates the offset of the instruction to add the exception table, and ex_jmp_off indicates the offset to jump over the faulting instruction. This is to prepare for adding the exception table to atomic instructions later, because some atomic instructions need to perform zext or other operations. Signed-off-by: Pu Lehui <[email protected]> Reviewed-by: Björn Töpel <[email protected]>
1 parent 888fc1d commit 17f3153

File tree

2 files changed

+39
-47
lines changed

2 files changed

+39
-47
lines changed

‎arch/riscv/net/bpf_jit.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ struct rv_jit_context {
7878
int epilogue_offset;
7979
int *offset; /* BPF to RV */
8080
int nexentries;
81+
int ex_insn_off;
82+
int ex_jmp_off;
8183
unsigned long flags;
8284
int stack_size;
8385
u64 arena_vm_start;

‎arch/riscv/net/bpf_jit_comp64.c‎

Lines changed: 37 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -511,57 +511,54 @@ static void emit_stx_insn(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context
511511
}
512512
}
513513

514-
static int emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext,
514+
static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext,
515515
struct rv_jit_context *ctx)
516516
{
517-
int insns_start;
518-
519517
if (is_12b_int(off)) {
520-
insns_start = ctx->ninsns;
518+
ctx->ex_insn_off = ctx->ninsns;
521519
emit_ldx_insn(rd, off, rs, size, sign_ext, ctx);
522-
return ctx->ninsns - insns_start;
520+
ctx->ex_jmp_off = ctx->ninsns;
521+
return;
523522
}
524523

525524
emit_imm(RV_REG_T1, off, ctx);
526525
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
527-
insns_start = ctx->ninsns;
526+
ctx->ex_insn_off = ctx->ninsns;
528527
emit_ldx_insn(rd, 0, RV_REG_T1, size, sign_ext, ctx);
529-
return ctx->ninsns - insns_start;
528+
ctx->ex_jmp_off = ctx->ninsns;
530529
}
531530

532-
static int emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx)
531+
static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx)
533532
{
534-
int insns_start;
535-
536533
emit_imm(RV_REG_T1, imm, ctx);
537534
if (is_12b_int(off)) {
538-
insns_start = ctx->ninsns;
535+
ctx->ex_insn_off = ctx->ninsns;
539536
emit_stx_insn(rd, off, RV_REG_T1, size, ctx);
540-
return ctx->ninsns - insns_start;
537+
ctx->ex_jmp_off = ctx->ninsns;
538+
return;
541539
}
542540

543541
emit_imm(RV_REG_T2, off, ctx);
544542
emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
545-
insns_start = ctx->ninsns;
543+
ctx->ex_insn_off = ctx->ninsns;
546544
emit_stx_insn(RV_REG_T2, 0, RV_REG_T1, size, ctx);
547-
return ctx->ninsns - insns_start;
545+
ctx->ex_jmp_off = ctx->ninsns;
548546
}
549547

550-
static int emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
548+
static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
551549
{
552-
int insns_start;
553-
554550
if (is_12b_int(off)) {
555-
insns_start = ctx->ninsns;
551+
ctx->ex_insn_off = ctx->ninsns;
556552
emit_stx_insn(rd, off, rs, size, ctx);
557-
return ctx->ninsns - insns_start;
553+
ctx->ex_jmp_off = ctx->ninsns;
554+
return;
558555
}
559556

560557
emit_imm(RV_REG_T1, off, ctx);
561558
emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
562-
insns_start = ctx->ninsns;
559+
ctx->ex_insn_off = ctx->ninsns;
563560
emit_stx_insn(RV_REG_T1, 0, rs, size, ctx);
564-
return ctx->ninsns - insns_start;
561+
ctx->ex_jmp_off = ctx->ninsns;
565562
}
566563

567564
static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
@@ -700,31 +697,31 @@ bool ex_handler_bpf(const struct exception_table_entry *ex,
700697
}
701698

702699
/* For accesses to BTF pointers, add an entry to the exception table */
703-
static int add_exception_handler(const struct bpf_insn *insn,
704-
struct rv_jit_context *ctx,
705-
int dst_reg, int insn_len)
700+
static int add_exception_handler(const struct bpf_insn *insn, int dst_reg,
701+
struct rv_jit_context *ctx)
706702
{
707703
struct exception_table_entry *ex;
708704
unsigned long pc;
709705
off_t ins_offset;
710706
off_t fixup_offset;
711707

712708
if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable ||
713-
(BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
714-
BPF_MODE(insn->code) != BPF_PROBE_MEM32))
709+
ctx->ex_insn_off <= 0 || ctx->ex_jmp_off <= 0)
715710
return 0;
716711

717-
if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
718-
return -EINVAL;
712+
if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
713+
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
714+
BPF_MODE(insn->code) != BPF_PROBE_MEM32)
715+
return 0;
719716

720-
if (WARN_ON_ONCE(insn_len > ctx->ninsns))
717+
if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
721718
return -EINVAL;
722719

723-
if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1))
720+
if (WARN_ON_ONCE(ctx->ex_insn_off > ctx->ninsns || ctx->ex_jmp_off > ctx->ninsns))
724721
return -EINVAL;
725722

726723
ex = &ctx->prog->aux->extable[ctx->nexentries];
727-
pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len];
724+
pc = (unsigned long)&ctx->ro_insns[ctx->ex_insn_off];
728725

729726
/*
730727
* This is the relative offset of the instruction that may fault from
@@ -748,7 +745,7 @@ static int add_exception_handler(const struct bpf_insn *insn,
748745
* that may fault. The execution will jump to this after handling the
749746
* fault.
750747
*/
751-
fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
748+
fixup_offset = (long)&ex->fixup - (long)&ctx->ro_insns[ctx->ex_jmp_off];
752749
if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
753750
return -ERANGE;
754751

@@ -765,6 +762,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
765762
FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
766763
ex->type = EX_TYPE_BPF;
767764

765+
ctx->ex_insn_off = 0;
766+
ctx->ex_jmp_off = 0;
768767
ctx->nexentries++;
769768
return 0;
770769
}
@@ -1774,7 +1773,6 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
17741773
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
17751774
{
17761775
bool sign_ext;
1777-
int insn_len;
17781776

17791777
sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||
17801778
BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
@@ -1784,9 +1782,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
17841782
rs = RV_REG_T2;
17851783
}
17861784

1787-
insn_len = emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx);
1785+
emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx);
17881786

1789-
ret = add_exception_handler(insn, ctx, rd, insn_len);
1787+
ret = add_exception_handler(insn, rd, ctx);
17901788
if (ret)
17911789
return ret;
17921790

@@ -1809,21 +1807,17 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
18091807
case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
18101808
case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
18111809
case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
1812-
{
1813-
int insn_len;
1814-
18151810
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
18161811
emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx);
18171812
rd = RV_REG_T3;
18181813
}
18191814

1820-
insn_len = emit_st(rd, off, imm, BPF_SIZE(code), ctx);
1815+
emit_st(rd, off, imm, BPF_SIZE(code), ctx);
18211816

1822-
ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, insn_len);
1817+
ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
18231818
if (ret)
18241819
return ret;
18251820
break;
1826-
}
18271821

18281822
/* STX: *(size *)(dst + off) = src */
18291823
case BPF_STX | BPF_MEM | BPF_B:
@@ -1835,21 +1829,17 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
18351829
case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
18361830
case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
18371831
case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
1838-
{
1839-
int insn_len;
1840-
18411832
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
18421833
emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx);
18431834
rd = RV_REG_T2;
18441835
}
18451836

1846-
insn_len = emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
1837+
emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
18471838

1848-
ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, insn_len);
1839+
ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
18491840
if (ret)
18501841
return ret;
18511842
break;
1852-
}
18531843

18541844
case BPF_STX | BPF_ATOMIC | BPF_B:
18551845
case BPF_STX | BPF_ATOMIC | BPF_H:

0 commit comments

Comments
 (0)