Skip to content

Commit 14c3e44

Browse files
Saket Kumar BhaskarKernel Patches Daemon
Saket Kumar Bhaskar
authored and
Kernel Patches Daemon
committed
powerpc64/bpf: Introduce bpf_jit_emit_atomic_ops() to emit atomic instructions
The existing code for emitting bpf atomic instruction sequences for atomic operations such as XCHG, CMPXCHG, ADD, AND, OR, and XOR has been refactored into a reusable function, bpf_jit_emit_ppc_atomic_op(). It also computes the jump offset and tracks the instruction index for jited LDARX/LWARX to be used in case it causes a fault. Reviewed-by: Hari Bathini <[email protected]> Tested-by: Venkat Rao Bagalkote <[email protected]> Signed-off-by: Saket Kumar Bhaskar <[email protected]>
1 parent 240ae36 commit 14c3e44

File tree

1 file changed

+115
-88
lines changed

1 file changed

+115
-88
lines changed

‎arch/powerpc/net/bpf_jit_comp64.c‎

Lines changed: 115 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,111 @@ asm (
423423
" blr ;"
424424
);
425425

426+
static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx,
427+
const struct bpf_insn *insn, u32 *jmp_off,
428+
u32 *tmp_idx, u32 *addrp)
429+
{
430+
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
431+
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
432+
u32 size = BPF_SIZE(insn->code);
433+
u32 src_reg = bpf_to_ppc(insn->src_reg);
434+
u32 dst_reg = bpf_to_ppc(insn->dst_reg);
435+
s32 imm = insn->imm;
436+
437+
u32 save_reg = tmp2_reg;
438+
u32 ret_reg = src_reg;
439+
u32 fixup_idx;
440+
441+
/* Get offset into TMP_REG_1 */
442+
EMIT(PPC_RAW_LI(tmp1_reg, insn->off));
443+
/*
444+
* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
445+
* before and after the operation.
446+
*
447+
* This is a requirement in the Linux Kernel Memory Model.
448+
* See __cmpxchg_u64() in asm/cmpxchg.h as an example.
449+
*/
450+
if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
451+
EMIT(PPC_RAW_SYNC());
452+
453+
*tmp_idx = ctx->idx;
454+
455+
/* load value from memory into TMP_REG_2 */
456+
if (size == BPF_DW)
457+
EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
458+
else
459+
EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
460+
/* Save old value in _R0 */
461+
if (imm & BPF_FETCH)
462+
EMIT(PPC_RAW_MR(_R0, tmp2_reg));
463+
464+
switch (imm) {
465+
case BPF_ADD:
466+
case BPF_ADD | BPF_FETCH:
467+
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
468+
break;
469+
case BPF_AND:
470+
case BPF_AND | BPF_FETCH:
471+
EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
472+
break;
473+
case BPF_OR:
474+
case BPF_OR | BPF_FETCH:
475+
EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
476+
break;
477+
case BPF_XOR:
478+
case BPF_XOR | BPF_FETCH:
479+
EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
480+
break;
481+
case BPF_CMPXCHG:
482+
/*
483+
* Return old value in BPF_REG_0 for BPF_CMPXCHG &
484+
* in src_reg for other cases.
485+
*/
486+
ret_reg = bpf_to_ppc(BPF_REG_0);
487+
488+
/* Compare with old value in BPF_R0 */
489+
if (size == BPF_DW)
490+
EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
491+
else
492+
EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
493+
/* Don't set if different from old value */
494+
PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
495+
fallthrough;
496+
case BPF_XCHG:
497+
save_reg = src_reg;
498+
break;
499+
default:
500+
return -EOPNOTSUPP;
501+
}
502+
503+
/* store new value */
504+
if (size == BPF_DW)
505+
EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
506+
else
507+
EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
508+
/* we're done if this succeeded */
509+
PPC_BCC_SHORT(COND_NE, *tmp_idx * 4);
510+
fixup_idx = ctx->idx;
511+
512+
if (imm & BPF_FETCH) {
513+
/* Emit 'sync' to enforce full ordering */
514+
if (IS_ENABLED(CONFIG_SMP))
515+
EMIT(PPC_RAW_SYNC());
516+
EMIT(PPC_RAW_MR(ret_reg, _R0));
517+
/*
518+
* Skip unnecessary zero-extension for 32-bit cmpxchg.
519+
* For context, see commit 39491867ace5.
520+
*/
521+
if (size != BPF_DW && imm == BPF_CMPXCHG &&
522+
insn_is_zext(insn + 1))
523+
*addrp = ctx->idx * 4;
524+
}
525+
526+
*jmp_off = (fixup_idx - *tmp_idx) * 4;
527+
528+
return 0;
529+
}
530+
426531
static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off,
427532
u32 code, u32 *image)
428533
{
@@ -538,14 +643,14 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
538643
u32 size = BPF_SIZE(code);
539644
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
540645
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
541-
u32 save_reg, ret_reg;
542646
s16 off = insn[i].off;
543647
s32 imm = insn[i].imm;
544648
bool func_addr_fixed;
545649
u64 func_addr;
546650
u64 imm64;
547651
u32 true_cond;
548652
u32 tmp_idx;
653+
u32 jmp_off;
549654

550655
/*
551656
* addrs[] maps a BPF bytecode address into a real offset from
@@ -1080,93 +1185,15 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
10801185
return -EOPNOTSUPP;
10811186
}
10821187

1083-
save_reg = tmp2_reg;
1084-
ret_reg = src_reg;
1085-
1086-
/* Get offset into TMP_REG_1 */
1087-
EMIT(PPC_RAW_LI(tmp1_reg, off));
1088-
/*
1089-
* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
1090-
* before and after the operation.
1091-
*
1092-
* This is a requirement in the Linux Kernel Memory Model.
1093-
* See __cmpxchg_u64() in asm/cmpxchg.h as an example.
1094-
*/
1095-
if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
1096-
EMIT(PPC_RAW_SYNC());
1097-
tmp_idx = ctx->idx * 4;
1098-
/* load value from memory into TMP_REG_2 */
1099-
if (size == BPF_DW)
1100-
EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
1101-
else
1102-
EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
1103-
1104-
/* Save old value in _R0 */
1105-
if (imm & BPF_FETCH)
1106-
EMIT(PPC_RAW_MR(_R0, tmp2_reg));
1107-
1108-
switch (imm) {
1109-
case BPF_ADD:
1110-
case BPF_ADD | BPF_FETCH:
1111-
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
1112-
break;
1113-
case BPF_AND:
1114-
case BPF_AND | BPF_FETCH:
1115-
EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
1116-
break;
1117-
case BPF_OR:
1118-
case BPF_OR | BPF_FETCH:
1119-
EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
1120-
break;
1121-
case BPF_XOR:
1122-
case BPF_XOR | BPF_FETCH:
1123-
EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
1124-
break;
1125-
case BPF_CMPXCHG:
1126-
/*
1127-
* Return old value in BPF_REG_0 for BPF_CMPXCHG &
1128-
* in src_reg for other cases.
1129-
*/
1130-
ret_reg = bpf_to_ppc(BPF_REG_0);
1131-
1132-
/* Compare with old value in BPF_R0 */
1133-
if (size == BPF_DW)
1134-
EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
1135-
else
1136-
EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
1137-
/* Don't set if different from old value */
1138-
PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
1139-
fallthrough;
1140-
case BPF_XCHG:
1141-
save_reg = src_reg;
1142-
break;
1143-
default:
1144-
pr_err_ratelimited(
1145-
"eBPF filter atomic op code %02x (@%d) unsupported\n",
1146-
code, i);
1147-
return -EOPNOTSUPP;
1148-
}
1149-
1150-
/* store new value */
1151-
if (size == BPF_DW)
1152-
EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
1153-
else
1154-
EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
1155-
/* we're done if this succeeded */
1156-
PPC_BCC_SHORT(COND_NE, tmp_idx);
1157-
1158-
if (imm & BPF_FETCH) {
1159-
/* Emit 'sync' to enforce full ordering */
1160-
if (IS_ENABLED(CONFIG_SMP))
1161-
EMIT(PPC_RAW_SYNC());
1162-
EMIT(PPC_RAW_MR(ret_reg, _R0));
1163-
/*
1164-
* Skip unnecessary zero-extension for 32-bit cmpxchg.
1165-
* For context, see commit 39491867ace5.
1166-
*/
1167-
if (size != BPF_DW && imm == BPF_CMPXCHG &&
1168-
insn_is_zext(&insn[i + 1]))
1169-
addrs[++i] = ctx->idx * 4;
1188+
ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i],
1189+
&jmp_off, &tmp_idx, &addrs[i + 1]);
1190+
if (ret) {
1191+
if (ret == -EOPNOTSUPP) {
1192+
pr_err_ratelimited(
1193+
"eBPF filter atomic op code %02x (@%d) unsupported\n",
1194+
code, i);
1195+
}
1196+
return ret;
11701197
}
11711198
break;
11721199

0 commit comments

Comments
 (0)