Skip to content

Commit cf2a6de

Browse files
puranjaymohanmaddy-kerneldev
puranjaymohan
authored and
maddy-kerneldev
committed
powerpc64/bpf: Add jit support for load_acquire and store_release
Add JIT support for the load_acquire and store_release instructions. The implementation is similar to the kernel where: load_acquire => plain load -> lwsync store_release => lwsync -> plain store To test the correctness of the implementation, following selftests were run: [fedora@linux-kernel bpf]$ sudo ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #519/1 verifier_load_acquire/load-acquire, 8-bit:OK #519/2 verifier_load_acquire/load-acquire, 8-bit @unpriv:OK #519/3 verifier_load_acquire/load-acquire, 16-bit:OK #519/4 verifier_load_acquire/load-acquire, 16-bit @unpriv:OK #519/5 verifier_load_acquire/load-acquire, 32-bit:OK #519/6 verifier_load_acquire/load-acquire, 32-bit @unpriv:OK #519/7 verifier_load_acquire/load-acquire, 64-bit:OK #519/8 verifier_load_acquire/load-acquire, 64-bit @unpriv:OK #519/9 verifier_load_acquire/load-acquire with uninitialized src_reg:OK #519/10 verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK #519/11 verifier_load_acquire/load-acquire with non-pointer src_reg:OK #519/12 verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK #519/13 verifier_load_acquire/misaligned load-acquire:OK #519/14 verifier_load_acquire/misaligned load-acquire @unpriv:OK #519/15 verifier_load_acquire/load-acquire from ctx pointer:OK #519/16 verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK #519/17 verifier_load_acquire/load-acquire with invalid register R15:OK #519/18 verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK #519/19 verifier_load_acquire/load-acquire from pkt pointer:OK #519/20 verifier_load_acquire/load-acquire from flow_keys pointer:OK #519/21 verifier_load_acquire/load-acquire from sock pointer:OK #519 verifier_load_acquire:OK #556/1 verifier_store_release/store-release, 8-bit:OK #556/2 verifier_store_release/store-release, 8-bit @unpriv:OK #556/3 verifier_store_release/store-release, 16-bit:OK #556/4 verifier_store_release/store-release, 16-bit @unpriv:OK #556/5 verifier_store_release/store-release, 32-bit:OK #556/6 verifier_store_release/store-release, 32-bit @unpriv:OK #556/7 verifier_store_release/store-release, 64-bit:OK #556/8 verifier_store_release/store-release, 64-bit @unpriv:OK #556/9 verifier_store_release/store-release with uninitialized src_reg:OK #556/10 verifier_store_release/store-release with uninitialized src_reg @unpriv:OK #556/11 verifier_store_release/store-release with uninitialized dst_reg:OK #556/12 verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK #556/13 verifier_store_release/store-release with non-pointer dst_reg:OK #556/14 verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK #556/15 verifier_store_release/misaligned store-release:OK #556/16 verifier_store_release/misaligned store-release @unpriv:OK #556/17 verifier_store_release/store-release to ctx pointer:OK #556/18 verifier_store_release/store-release to ctx pointer @unpriv:OK #556/19 verifier_store_release/store-release, leak pointer to stack:OK #556/20 verifier_store_release/store-release, leak pointer to stack @unpriv:OK #556/21 verifier_store_release/store-release, leak pointer to map:OK #556/22 verifier_store_release/store-release, leak pointer to map @unpriv:OK #556/23 verifier_store_release/store-release with invalid register R15:OK #556/24 verifier_store_release/store-release with invalid register R15 @unpriv:OK #556/25 verifier_store_release/store-release to pkt pointer:OK #556/26 verifier_store_release/store-release to flow_keys pointer:OK #556/27 verifier_store_release/store-release to sock pointer:OK #556 verifier_store_release:OK Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED Signed-off-by: Puranjay Mohan <[email protected]> Tested-by: Saket Kumar Bhaskar <[email protected]> Reviewed-by: Hari Bathini <[email protected]> Signed-off-by: Madhavan Srinivasan <[email protected]> Link: https://patch.msgid.link/[email protected]
1 parent 19122a7 commit cf2a6de

File tree

3 files changed

+85
-1
lines changed

3 files changed

+85
-1
lines changed

‎arch/powerpc/include/asm/ppc-opcode.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,7 @@
425425
#define PPC_RAW_SC() (0x44000002)
426426
#define PPC_RAW_SYNC() (0x7c0004ac)
427427
#define PPC_RAW_ISYNC() (0x4c00012c)
428+
#define PPC_RAW_LWSYNC() (0x7c2004ac)
428429

429430
/*
430431
* Define what the VSX XX1 form instructions will look like, then add

‎arch/powerpc/net/bpf_jit_comp64.c

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,71 @@ asm (
392392
" blr ;"
393393
);
394394

395+
static int emit_atomic_ld_st(const struct bpf_insn insn, struct codegen_context *ctx, u32 *image)
396+
{
397+
u32 code = insn.code;
398+
u32 dst_reg = bpf_to_ppc(insn.dst_reg);
399+
u32 src_reg = bpf_to_ppc(insn.src_reg);
400+
u32 size = BPF_SIZE(code);
401+
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
402+
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
403+
s16 off = insn.off;
404+
s32 imm = insn.imm;
405+
406+
switch (imm) {
407+
case BPF_LOAD_ACQ:
408+
switch (size) {
409+
case BPF_B:
410+
EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
411+
break;
412+
case BPF_H:
413+
EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
414+
break;
415+
case BPF_W:
416+
EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
417+
break;
418+
case BPF_DW:
419+
if (off % 4) {
420+
EMIT(PPC_RAW_LI(tmp1_reg, off));
421+
EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg));
422+
} else {
423+
EMIT(PPC_RAW_LD(dst_reg, src_reg, off));
424+
}
425+
break;
426+
}
427+
EMIT(PPC_RAW_LWSYNC());
428+
break;
429+
case BPF_STORE_REL:
430+
EMIT(PPC_RAW_LWSYNC());
431+
switch (size) {
432+
case BPF_B:
433+
EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
434+
break;
435+
case BPF_H:
436+
EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
437+
break;
438+
case BPF_W:
439+
EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
440+
break;
441+
case BPF_DW:
442+
if (off % 4) {
443+
EMIT(PPC_RAW_LI(tmp2_reg, off));
444+
EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg));
445+
} else {
446+
EMIT(PPC_RAW_STD(src_reg, dst_reg, off));
447+
}
448+
break;
449+
}
450+
break;
451+
default:
452+
pr_err_ratelimited("unexpected atomic load/store op code %02x\n",
453+
imm);
454+
return -EINVAL;
455+
}
456+
457+
return 0;
458+
}
459+
395460
/* Assemble the body code between the prologue & epilogue */
396461
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx,
397462
u32 *addrs, int pass, bool extra_pass)
@@ -859,8 +924,25 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
859924
/*
860925
* BPF_STX ATOMIC (atomic ops)
861926
*/
927+
case BPF_STX | BPF_ATOMIC | BPF_B:
928+
case BPF_STX | BPF_ATOMIC | BPF_H:
862929
case BPF_STX | BPF_ATOMIC | BPF_W:
863930
case BPF_STX | BPF_ATOMIC | BPF_DW:
931+
if (bpf_atomic_is_load_store(&insn[i])) {
932+
ret = emit_atomic_ld_st(insn[i], ctx, image);
933+
if (ret)
934+
return ret;
935+
936+
if (size != BPF_DW && insn_is_zext(&insn[i + 1]))
937+
addrs[++i] = ctx->idx * 4;
938+
break;
939+
} else if (size == BPF_B || size == BPF_H) {
940+
pr_err_ratelimited(
941+
"eBPF filter atomic op code %02x (@%d) unsupported\n",
942+
code, i);
943+
return -EOPNOTSUPP;
944+
}
945+
864946
save_reg = tmp2_reg;
865947
ret_reg = src_reg;
866948

‎tools/testing/selftests/bpf/progs/bpf_misc.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,8 @@
227227

228228
#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
229229
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
230-
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
230+
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) || \
231+
(defined(__TARGET_ARCH_powerpc))
231232
#define CAN_USE_LOAD_ACQ_STORE_REL
232233
#endif
233234

0 commit comments

Comments
 (0)