Skip to content

Commit 2383e45

Browse files
author
Alexei Starovoitov
committed
Merge branch 'signed-loads-from-arena'
Puranjay Mohan says: ==================== Signed loads from Arena Changelog: v3 -> v4: v3: https://lore.kernel.org/all/[email protected]/ - Update bpf_jit_supports_insn() in riscv jit to reject signed arena loads (Eduard) - Fix coding style related to braces usage in an if statement in x86 jit (Eduard) v2 -> v3: v2: https://lore.kernel.org/bpf/[email protected]/ - Fix encoding for the generated instructions in x86 JIT (Eduard) The patch in v2 was generating instructions like: 42 63 44 20 f8 movslq -0x8(%rax,%r12), %eax This doesn't make sense because movslq outputs a 64-bit result, but the destination register here is set to eax (32-bit). The fix it to set the REX.W bit in the opcode, that means changing EMIT2(add_3mod(0x40, ...)) to EMIT2(add_3mod(0x48, ...)) - Add arm64 support - Add selftests signed laods from arena. v1 -> v2: v1: https://lore.kernel.org/bpf/[email protected] - Use bpf_jit_supports_insn. (Alexei) Currently, signed load instructions into arena memory are unsupported. The compiler is free to generate these, and on GCC-14 we see a corresponding error when it happens. The hurdle in supporting them is deciding which unused opcode to use to mark them for the JIT's own consumption. After much thinking, it appears 0xc0 / BPF_NOSPEC can be combined with load instructions to identify signed arena loads. Use this to recognize and JIT them appropriately, and remove the verifier side limitation on the program if the JIT supports them. ==================== Link: https://patch.msgid.link/[email protected] Signed-off-by: Alexei Starovoitov <[email protected]>
2 parents 348f611 + f616549 commit 2383e45

File tree

7 files changed

+251
-14
lines changed

7 files changed

+251
-14
lines changed

‎arch/arm64/net/bpf_jit_comp.c‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1133,12 +1133,14 @@ static int add_exception_handler(const struct bpf_insn *insn,
11331133
return 0;
11341134

11351135
if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
1136-
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
1137-
BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
1138-
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
1136+
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
1137+
BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
1138+
BPF_MODE(insn->code) != BPF_PROBE_MEM32SX &&
1139+
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
11391140
return 0;
11401141

11411142
is_arena = (BPF_MODE(insn->code) == BPF_PROBE_MEM32) ||
1143+
(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) ||
11421144
(BPF_MODE(insn->code) == BPF_PROBE_ATOMIC);
11431145

11441146
if (!ctx->prog->aux->extable ||
@@ -1659,7 +1661,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
16591661
case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
16601662
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
16611663
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
1662-
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
1664+
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
1665+
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
1666+
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
1667+
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
1668+
BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) {
16631669
emit(A64_ADD(1, tmp2, src, arena_vm_base), ctx);
16641670
src = tmp2;
16651671
}
@@ -1671,7 +1677,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
16711677
off_adj = off;
16721678
}
16731679
sign_extend = (BPF_MODE(insn->code) == BPF_MEMSX ||
1674-
BPF_MODE(insn->code) == BPF_PROBE_MEMSX);
1680+
BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
1681+
BPF_MODE(insn->code) == BPF_PROBE_MEM32SX);
16751682
switch (BPF_SIZE(code)) {
16761683
case BPF_W:
16771684
if (is_lsi_offset(off_adj, 2)) {
@@ -1879,9 +1886,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
18791886
if (ret)
18801887
return ret;
18811888

1882-
ret = add_exception_handler(insn, ctx, dst);
1883-
if (ret)
1884-
return ret;
1889+
if (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
1890+
ret = add_exception_handler(insn, ctx, dst);
1891+
if (ret)
1892+
return ret;
1893+
}
18851894
break;
18861895

18871896
default:

‎arch/riscv/net/bpf_jit_comp64.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2066,6 +2066,11 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
20662066
case BPF_STX | BPF_ATOMIC | BPF_DW:
20672067
if (insn->imm == BPF_CMPXCHG)
20682068
return rv_ext_enabled(ZACAS);
2069+
break;
2070+
case BPF_LDX | BPF_MEMSX | BPF_B:
2071+
case BPF_LDX | BPF_MEMSX | BPF_H:
2072+
case BPF_LDX | BPF_MEMSX | BPF_W:
2073+
return false;
20692074
}
20702075
}
20712076

‎arch/s390/net/bpf_jit_comp.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2967,6 +2967,11 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
29672967
case BPF_STX | BPF_ATOMIC | BPF_DW:
29682968
if (bpf_atomic_is_load_store(insn))
29692969
return false;
2970+
break;
2971+
case BPF_LDX | BPF_MEMSX | BPF_B:
2972+
case BPF_LDX | BPF_MEMSX | BPF_H:
2973+
case BPF_LDX | BPF_MEMSX | BPF_W:
2974+
return false;
29702975
}
29712976
return true;
29722977
}

‎arch/x86/net/bpf_jit_comp.c‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,11 +1152,38 @@ static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i
11521152
*pprog = prog;
11531153
}
11541154

1155+
static void emit_ldsx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)
1156+
{
1157+
u8 *prog = *pprog;
1158+
1159+
switch (size) {
1160+
case BPF_B:
1161+
/* movsx rax, byte ptr [rax + r12 + off] */
1162+
EMIT3(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x0F, 0xBE);
1163+
break;
1164+
case BPF_H:
1165+
/* movsx rax, word ptr [rax + r12 + off] */
1166+
EMIT3(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x0F, 0xBF);
1167+
break;
1168+
case BPF_W:
1169+
/* movsx rax, dword ptr [rax + r12 + off] */
1170+
EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x63);
1171+
break;
1172+
}
1173+
emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off);
1174+
*pprog = prog;
1175+
}
1176+
11551177
static void emit_ldx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
11561178
{
11571179
emit_ldx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);
11581180
}
11591181

1182+
static void emit_ldsx_r12(u8 **prog, u32 size, u32 dst_reg, u32 src_reg, int off)
1183+
{
1184+
emit_ldsx_index(prog, size, dst_reg, src_reg, X86_REG_R12, off);
1185+
}
1186+
11601187
/* STX: *(u8*)(dst_reg + off) = src_reg */
11611188
static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
11621189
{
@@ -2109,15 +2136,22 @@ st: if (is_imm8(insn->off))
21092136
case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
21102137
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
21112138
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
2139+
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
2140+
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
2141+
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
21122142
case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
21132143
case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
21142144
case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
21152145
case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
21162146
start_of_ldx = prog;
2117-
if (BPF_CLASS(insn->code) == BPF_LDX)
2118-
emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
2119-
else
2147+
if (BPF_CLASS(insn->code) == BPF_LDX) {
2148+
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32SX)
2149+
emit_ldsx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
2150+
else
2151+
emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
2152+
} else {
21202153
emit_stx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
2154+
}
21212155
populate_extable:
21222156
{
21232157
struct exception_table_entry *ex;

‎include/linux/filter.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ struct ctl_table_header;
7878
/* unused opcode to mark special atomic instruction */
7979
#define BPF_PROBE_ATOMIC 0xe0
8080

81+
/* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */
82+
#define BPF_PROBE_MEM32SX 0xc0
83+
8184
/* unused opcode to mark call to interpreter with arguments */
8285
#define BPF_CALL_ARGS 0xe0
8386

‎kernel/bpf/verifier.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21379,10 +21379,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
2137921379
continue;
2138021380
case PTR_TO_ARENA:
2138121381
if (BPF_MODE(insn->code) == BPF_MEMSX) {
21382-
verbose(env, "sign extending loads from arena are not supported yet\n");
21383-
return -EOPNOTSUPP;
21382+
if (!bpf_jit_supports_insn(insn, true)) {
21383+
verbose(env, "sign extending loads from arena are not supported yet\n");
21384+
return -EOPNOTSUPP;
21385+
}
21386+
insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code);
21387+
} else {
21388+
insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
2138421389
}
21385-
insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
2138621390
env->prog->aux->num_exentries++;
2138721391
continue;
2138821392
default:
@@ -21588,6 +21592,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
2158821592
if (BPF_CLASS(insn->code) == BPF_LDX &&
2158921593
(BPF_MODE(insn->code) == BPF_PROBE_MEM ||
2159021594
BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
21595+
BPF_MODE(insn->code) == BPF_PROBE_MEM32SX ||
2159121596
BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
2159221597
num_exentries++;
2159321598
if ((BPF_CLASS(insn->code) == BPF_STX ||

‎tools/testing/selftests/bpf/progs/verifier_ldsx.c‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,20 @@
33
#include <linux/bpf.h>
44
#include <bpf/bpf_helpers.h>
55
#include "bpf_misc.h"
6+
#include "bpf_arena_common.h"
67

78
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
89
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
910
defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \
1011
defined(__TARGET_ARCH_loongarch)) && \
1112
__clang_major__ >= 18
1213

14+
struct {
15+
__uint(type, BPF_MAP_TYPE_ARENA);
16+
__uint(map_flags, BPF_F_MMAPABLE);
17+
__uint(max_entries, 1);
18+
} arena SEC(".maps");
19+
1320
SEC("socket")
1421
__description("LDSX, S8")
1522
__success __success_unpriv __retval(-2)
@@ -256,6 +263,175 @@ __naked void ldsx_ctx_8(void)
256263
: __clobber_all);
257264
}
258265

266+
SEC("syscall")
267+
__description("Arena LDSX Disasm")
268+
__success
269+
__arch_x86_64
270+
__jited("movslq 0x10(%rax,%r12), %r14")
271+
__jited("movswq 0x18(%rax,%r12), %r14")
272+
__jited("movsbq 0x20(%rax,%r12), %r14")
273+
__jited("movslq 0x10(%rdi,%r12), %r15")
274+
__jited("movswq 0x18(%rdi,%r12), %r15")
275+
__jited("movsbq 0x20(%rdi,%r12), %r15")
276+
__arch_arm64
277+
__jited("add x11, x7, x28")
278+
__jited("ldrsw x21, [x11, #0x10]")
279+
__jited("add x11, x7, x28")
280+
__jited("ldrsh x21, [x11, #0x18]")
281+
__jited("add x11, x7, x28")
282+
__jited("ldrsb x21, [x11, #0x20]")
283+
__jited("add x11, x0, x28")
284+
__jited("ldrsw x22, [x11, #0x10]")
285+
__jited("add x11, x0, x28")
286+
__jited("ldrsh x22, [x11, #0x18]")
287+
__jited("add x11, x0, x28")
288+
__jited("ldrsb x22, [x11, #0x20]")
289+
__naked void arena_ldsx_disasm(void *ctx)
290+
{
291+
asm volatile (
292+
"r1 = %[arena] ll;"
293+
"r2 = 0;"
294+
"r3 = 1;"
295+
"r4 = %[numa_no_node];"
296+
"r5 = 0;"
297+
"call %[bpf_arena_alloc_pages];"
298+
"r0 = addr_space_cast(r0, 0x0, 0x1);"
299+
"r1 = r0;"
300+
"r8 = *(s32 *)(r0 + 16);"
301+
"r8 = *(s16 *)(r0 + 24);"
302+
"r8 = *(s8 *)(r0 + 32);"
303+
"r9 = *(s32 *)(r1 + 16);"
304+
"r9 = *(s16 *)(r1 + 24);"
305+
"r9 = *(s8 *)(r1 + 32);"
306+
"r0 = 0;"
307+
"exit;"
308+
:: __imm(bpf_arena_alloc_pages),
309+
__imm_addr(arena),
310+
__imm_const(numa_no_node, NUMA_NO_NODE)
311+
: __clobber_all
312+
);
313+
}
314+
315+
SEC("syscall")
316+
__description("Arena LDSX Exception")
317+
__success __retval(0)
318+
__arch_x86_64
319+
__arch_arm64
320+
__naked void arena_ldsx_exception(void *ctx)
321+
{
322+
asm volatile (
323+
"r1 = %[arena] ll;"
324+
"r0 = 0xdeadbeef;"
325+
"r0 = addr_space_cast(r0, 0x0, 0x1);"
326+
"r1 = 0x3fe;"
327+
"*(u64 *)(r0 + 0) = r1;"
328+
"r0 = *(s8 *)(r0 + 0);"
329+
"exit;"
330+
:
331+
: __imm_addr(arena)
332+
: __clobber_all
333+
);
334+
}
335+
336+
SEC("syscall")
337+
__description("Arena LDSX, S8")
338+
__success __retval(-1)
339+
__arch_x86_64
340+
__arch_arm64
341+
__naked void arena_ldsx_s8(void *ctx)
342+
{
343+
asm volatile (
344+
"r1 = %[arena] ll;"
345+
"r2 = 0;"
346+
"r3 = 1;"
347+
"r4 = %[numa_no_node];"
348+
"r5 = 0;"
349+
"call %[bpf_arena_alloc_pages];"
350+
"r0 = addr_space_cast(r0, 0x0, 0x1);"
351+
"r1 = 0x3fe;"
352+
"*(u64 *)(r0 + 0) = r1;"
353+
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
354+
"r0 = *(s8 *)(r0 + 0);"
355+
#else
356+
"r0 = *(s8 *)(r0 + 7);"
357+
#endif
358+
"r0 >>= 1;"
359+
"exit;"
360+
:: __imm(bpf_arena_alloc_pages),
361+
__imm_addr(arena),
362+
__imm_const(numa_no_node, NUMA_NO_NODE)
363+
: __clobber_all
364+
);
365+
}
366+
367+
SEC("syscall")
368+
__description("Arena LDSX, S16")
369+
__success __retval(-1)
370+
__arch_x86_64
371+
__arch_arm64
372+
__naked void arena_ldsx_s16(void *ctx)
373+
{
374+
asm volatile (
375+
"r1 = %[arena] ll;"
376+
"r2 = 0;"
377+
"r3 = 1;"
378+
"r4 = %[numa_no_node];"
379+
"r5 = 0;"
380+
"call %[bpf_arena_alloc_pages];"
381+
"r0 = addr_space_cast(r0, 0x0, 0x1);"
382+
"r1 = 0x3fffe;"
383+
"*(u64 *)(r0 + 0) = r1;"
384+
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
385+
"r0 = *(s16 *)(r0 + 0);"
386+
#else
387+
"r0 = *(s16 *)(r0 + 6);"
388+
#endif
389+
"r0 >>= 1;"
390+
"exit;"
391+
:: __imm(bpf_arena_alloc_pages),
392+
__imm_addr(arena),
393+
__imm_const(numa_no_node, NUMA_NO_NODE)
394+
: __clobber_all
395+
);
396+
}
397+
398+
SEC("syscall")
399+
__description("Arena LDSX, S32")
400+
__success __retval(-1)
401+
__arch_x86_64
402+
__arch_arm64
403+
__naked void arena_ldsx_s32(void *ctx)
404+
{
405+
asm volatile (
406+
"r1 = %[arena] ll;"
407+
"r2 = 0;"
408+
"r3 = 1;"
409+
"r4 = %[numa_no_node];"
410+
"r5 = 0;"
411+
"call %[bpf_arena_alloc_pages];"
412+
"r0 = addr_space_cast(r0, 0x0, 0x1);"
413+
"r1 = 0xfffffffe;"
414+
"*(u64 *)(r0 + 0) = r1;"
415+
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
416+
"r0 = *(s32 *)(r0 + 0);"
417+
#else
418+
"r0 = *(s32 *)(r0 + 4);"
419+
#endif
420+
"r0 >>= 1;"
421+
"exit;"
422+
:: __imm(bpf_arena_alloc_pages),
423+
__imm_addr(arena),
424+
__imm_const(numa_no_node, NUMA_NO_NODE)
425+
: __clobber_all
426+
);
427+
}
428+
429+
/* to retain debug info for BTF generation */
430+
void kfunc_root(void)
431+
{
432+
bpf_arena_alloc_pages(0, 0, 0, 0, 0);
433+
}
434+
259435
#else
260436

261437
SEC("socket")

0 commit comments

Comments
 (0)