Skip to content

Commit fb7cefa

Browse files
Pu Lehuiborkmann
Pu Lehui
authored and
borkmann
committed
riscv, bpf: Add support arena atomics for RV64
Add arena atomics support for RMW atomics and load-acquire and store-release instructions. Non-Zacas cmpxchg is implemented via loop, which is not currently supported because it requires more complex extable and loop logic. Signed-off-by: Pu Lehui <[email protected]> Signed-off-by: Daniel Borkmann <[email protected]> Tested-by: Björn Töpel <[email protected]> Reviewed-by: Björn Töpel <[email protected]> Acked-by: Björn Töpel <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent b18f4aa commit fb7cefa

File tree

2 files changed

+59
-3
lines changed

2 files changed

+59
-3
lines changed

‎arch/riscv/net/bpf_jit.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1301,8 +1301,10 @@ static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_co
13011301
int jmp_offset;
13021302

13031303
if (rv_ext_enabled(ZACAS)) {
1304+
ctx->ex_insn_off = ctx->ninsns;
13041305
emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) :
13051306
rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx);
1307+
ctx->ex_jmp_off = ctx->ninsns;
13061308
if (!is64)
13071309
emit_zextw(r0, r0, ctx);
13081310
return;

‎arch/riscv/net/bpf_jit_comp64.c

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,11 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
571571
switch (imm) {
572572
/* dst_reg = load_acquire(src_reg + off16) */
573573
case BPF_LOAD_ACQ:
574+
if (BPF_MODE(code) == BPF_PROBE_ATOMIC) {
575+
emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx);
576+
rs = RV_REG_T2;
577+
}
578+
574579
emit_ldx(rd, off, rs, BPF_SIZE(code), false, ctx);
575580
emit_fence_r_rw(ctx);
576581

@@ -582,6 +587,11 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
582587
break;
583588
/* store_release(dst_reg + off16, src_reg) */
584589
case BPF_STORE_REL:
590+
if (BPF_MODE(code) == BPF_PROBE_ATOMIC) {
591+
emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx);
592+
rd = RV_REG_T2;
593+
}
594+
585595
emit_fence_rw_w(ctx);
586596
emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
587597
break;
@@ -599,13 +609,12 @@ static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
599609
u8 code = insn->code;
600610
s16 off = insn->off;
601611
s32 imm = insn->imm;
602-
bool is64;
612+
bool is64 = BPF_SIZE(code) == BPF_DW;
603613

604614
if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {
605615
pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n");
606616
return -EINVAL;
607617
}
608-
is64 = BPF_SIZE(code) == BPF_DW;
609618

610619
if (off) {
611620
if (is_12b_int(off)) {
@@ -617,53 +626,76 @@ static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
617626
rd = RV_REG_T1;
618627
}
619628

629+
if (BPF_MODE(code) == BPF_PROBE_ATOMIC) {
630+
emit_add(RV_REG_T1, rd, RV_REG_ARENA, ctx);
631+
rd = RV_REG_T1;
632+
}
633+
620634
switch (imm) {
621635
/* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */
622636
case BPF_ADD:
637+
ctx->ex_insn_off = ctx->ninsns;
623638
emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) :
624639
rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
640+
ctx->ex_jmp_off = ctx->ninsns;
625641
break;
626642
case BPF_AND:
643+
ctx->ex_insn_off = ctx->ninsns;
627644
emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) :
628645
rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
646+
ctx->ex_jmp_off = ctx->ninsns;
629647
break;
630648
case BPF_OR:
649+
ctx->ex_insn_off = ctx->ninsns;
631650
emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) :
632651
rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
652+
ctx->ex_jmp_off = ctx->ninsns;
633653
break;
634654
case BPF_XOR:
655+
ctx->ex_insn_off = ctx->ninsns;
635656
emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) :
636657
rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
658+
ctx->ex_jmp_off = ctx->ninsns;
637659
break;
638660
/* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */
639661
case BPF_ADD | BPF_FETCH:
662+
ctx->ex_insn_off = ctx->ninsns;
640663
emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) :
641664
rv_amoadd_w(rs, rs, rd, 1, 1), ctx);
665+
ctx->ex_jmp_off = ctx->ninsns;
642666
if (!is64)
643667
emit_zextw(rs, rs, ctx);
644668
break;
645669
case BPF_AND | BPF_FETCH:
670+
ctx->ex_insn_off = ctx->ninsns;
646671
emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) :
647672
rv_amoand_w(rs, rs, rd, 1, 1), ctx);
673+
ctx->ex_jmp_off = ctx->ninsns;
648674
if (!is64)
649675
emit_zextw(rs, rs, ctx);
650676
break;
651677
case BPF_OR | BPF_FETCH:
678+
ctx->ex_insn_off = ctx->ninsns;
652679
emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) :
653680
rv_amoor_w(rs, rs, rd, 1, 1), ctx);
681+
ctx->ex_jmp_off = ctx->ninsns;
654682
if (!is64)
655683
emit_zextw(rs, rs, ctx);
656684
break;
657685
case BPF_XOR | BPF_FETCH:
686+
ctx->ex_insn_off = ctx->ninsns;
658687
emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) :
659688
rv_amoxor_w(rs, rs, rd, 1, 1), ctx);
689+
ctx->ex_jmp_off = ctx->ninsns;
660690
if (!is64)
661691
emit_zextw(rs, rs, ctx);
662692
break;
663693
/* src_reg = atomic_xchg(dst_reg + off16, src_reg); */
664694
case BPF_XCHG:
695+
ctx->ex_insn_off = ctx->ninsns;
665696
emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) :
666697
rv_amoswap_w(rs, rs, rd, 1, 1), ctx);
698+
ctx->ex_jmp_off = ctx->ninsns;
667699
if (!is64)
668700
emit_zextw(rs, rs, ctx);
669701
break;
@@ -711,7 +743,8 @@ static int add_exception_handler(const struct bpf_insn *insn, int dst_reg,
711743

712744
if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
713745
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
714-
BPF_MODE(insn->code) != BPF_PROBE_MEM32)
746+
BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
747+
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
715748
return 0;
716749

717750
if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
@@ -1841,14 +1874,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
18411874
return ret;
18421875
break;
18431876

1877+
/* Atomics */
18441878
case BPF_STX | BPF_ATOMIC | BPF_B:
18451879
case BPF_STX | BPF_ATOMIC | BPF_H:
18461880
case BPF_STX | BPF_ATOMIC | BPF_W:
18471881
case BPF_STX | BPF_ATOMIC | BPF_DW:
1882+
case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
1883+
case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
1884+
case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
1885+
case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
18481886
if (bpf_atomic_is_load_store(insn))
18491887
ret = emit_atomic_ld_st(rd, rs, insn, ctx);
18501888
else
18511889
ret = emit_atomic_rmw(rd, rs, insn, ctx);
1890+
1891+
ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
18521892
if (ret)
18531893
return ret;
18541894
break;
@@ -1979,6 +2019,20 @@ bool bpf_jit_supports_arena(void)
19792019
return true;
19802020
}
19812021

2022+
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
2023+
{
2024+
if (in_arena) {
2025+
switch (insn->code) {
2026+
case BPF_STX | BPF_ATOMIC | BPF_W:
2027+
case BPF_STX | BPF_ATOMIC | BPF_DW:
2028+
if (insn->imm == BPF_CMPXCHG)
2029+
return rv_ext_enabled(ZACAS);
2030+
}
2031+
}
2032+
2033+
return true;
2034+
}
2035+
19822036
bool bpf_jit_supports_percpu_insn(void)
19832037
{
19842038
return true;

0 commit comments

Comments
 (0)