Skip to content

Commit 4ccb9f0

Browse files
davem330davem330
committed
Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Daniel Borkmann says: ==================== pull-request: bpf 2021-09-28 The following pull-request contains BPF updates for your *net* tree. We've added 10 non-merge commits during the last 14 day(s) which contain a total of 11 files changed, 139 insertions(+), 53 deletions(-). The main changes are: 1) Fix MIPS JIT jump code emission for too large offsets, from Piotr Krysiuk. 2) Fix x86 JIT atomic/fetch emission when dst reg maps to rax, from Johan Almbladh. 3) Fix cgroup_sk_alloc corner case when called from interrupt, from Daniel Borkmann. 4) Fix segfault in libbpf's linker for objects without BTF, from Kumar Kartikeya Dwivedi. 5) Fix bpf_jit_charge_modmem for applications with CAP_BPF, from Lorenz Bauer. 6) Fix return value handling for struct_ops BPF programs, from Hou Tao. 7) Various fixes to BPF selftests, from Jiri Benc. ==================== Signed-off-by: David S. Miller <[email protected]> ,
2 parents c894b51 + ced1858 commit 4ccb9f0

File tree

11 files changed

+139
-53
lines changed

11 files changed

+139
-53
lines changed

‎MAINTAINERS

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3384,9 +3384,11 @@ F: Documentation/networking/filter.rst
33843384
F: Documentation/userspace-api/ebpf/
33853385
F: arch/*/net/*
33863386
F: include/linux/bpf*
3387+
F: include/linux/btf*
33873388
F: include/linux/filter.h
33883389
F: include/trace/events/xdp.h
33893390
F: include/uapi/linux/bpf*
3391+
F: include/uapi/linux/btf*
33903392
F: include/uapi/linux/filter.h
33913393
F: kernel/bpf/
33923394
F: kernel/trace/bpf_trace.c

‎arch/mips/net/bpf_jit.c

Lines changed: 43 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,11 @@ static void build_epilogue(struct jit_ctx *ctx)
662662
((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative : func) : \
663663
func##_positive)
664664

665+
static bool is_bad_offset(int b_off)
666+
{
667+
return b_off > 0x1ffff || b_off < -0x20000;
668+
}
669+
665670
static int build_body(struct jit_ctx *ctx)
666671
{
667672
const struct bpf_prog *prog = ctx->skf;
@@ -728,7 +733,10 @@ static int build_body(struct jit_ctx *ctx)
728733
/* Load return register on DS for failures */
729734
emit_reg_move(r_ret, r_zero, ctx);
730735
/* Return with error */
731-
emit_b(b_imm(prog->len, ctx), ctx);
736+
b_off = b_imm(prog->len, ctx);
737+
if (is_bad_offset(b_off))
738+
return -E2BIG;
739+
emit_b(b_off, ctx);
732740
emit_nop(ctx);
733741
break;
734742
case BPF_LD | BPF_W | BPF_IND:
@@ -775,8 +783,10 @@ static int build_body(struct jit_ctx *ctx)
775783
emit_jalr(MIPS_R_RA, r_s0, ctx);
776784
emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */
777785
/* Check the error value */
778-
emit_bcond(MIPS_COND_NE, r_ret, 0,
779-
b_imm(prog->len, ctx), ctx);
786+
b_off = b_imm(prog->len, ctx);
787+
if (is_bad_offset(b_off))
788+
return -E2BIG;
789+
emit_bcond(MIPS_COND_NE, r_ret, 0, b_off, ctx);
780790
emit_reg_move(r_ret, r_zero, ctx);
781791
/* We are good */
782792
/* X <- P[1:K] & 0xf */
@@ -855,17 +865,21 @@ static int build_body(struct jit_ctx *ctx)
855865
/* A /= X */
856866
ctx->flags |= SEEN_X | SEEN_A;
857867
/* Check if r_X is zero */
858-
emit_bcond(MIPS_COND_EQ, r_X, r_zero,
859-
b_imm(prog->len, ctx), ctx);
868+
b_off = b_imm(prog->len, ctx);
869+
if (is_bad_offset(b_off))
870+
return -E2BIG;
871+
emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx);
860872
emit_load_imm(r_ret, 0, ctx); /* delay slot */
861873
emit_div(r_A, r_X, ctx);
862874
break;
863875
case BPF_ALU | BPF_MOD | BPF_X:
864876
/* A %= X */
865877
ctx->flags |= SEEN_X | SEEN_A;
866878
/* Check if r_X is zero */
867-
emit_bcond(MIPS_COND_EQ, r_X, r_zero,
868-
b_imm(prog->len, ctx), ctx);
879+
b_off = b_imm(prog->len, ctx);
880+
if (is_bad_offset(b_off))
881+
return -E2BIG;
882+
emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx);
869883
emit_load_imm(r_ret, 0, ctx); /* delay slot */
870884
emit_mod(r_A, r_X, ctx);
871885
break;
@@ -926,7 +940,10 @@ static int build_body(struct jit_ctx *ctx)
926940
break;
927941
case BPF_JMP | BPF_JA:
928942
/* pc += K */
929-
emit_b(b_imm(i + k + 1, ctx), ctx);
943+
b_off = b_imm(i + k + 1, ctx);
944+
if (is_bad_offset(b_off))
945+
return -E2BIG;
946+
emit_b(b_off, ctx);
930947
emit_nop(ctx);
931948
break;
932949
case BPF_JMP | BPF_JEQ | BPF_K:
@@ -1056,12 +1073,16 @@ static int build_body(struct jit_ctx *ctx)
10561073
break;
10571074
case BPF_RET | BPF_A:
10581075
ctx->flags |= SEEN_A;
1059-
if (i != prog->len - 1)
1076+
if (i != prog->len - 1) {
10601077
/*
10611078
* If this is not the last instruction
10621079
* then jump to the epilogue
10631080
*/
1064-
emit_b(b_imm(prog->len, ctx), ctx);
1081+
b_off = b_imm(prog->len, ctx);
1082+
if (is_bad_offset(b_off))
1083+
return -E2BIG;
1084+
emit_b(b_off, ctx);
1085+
}
10651086
emit_reg_move(r_ret, r_A, ctx); /* delay slot */
10661087
break;
10671088
case BPF_RET | BPF_K:
@@ -1075,7 +1096,10 @@ static int build_body(struct jit_ctx *ctx)
10751096
* If this is not the last instruction
10761097
* then jump to the epilogue
10771098
*/
1078-
emit_b(b_imm(prog->len, ctx), ctx);
1099+
b_off = b_imm(prog->len, ctx);
1100+
if (is_bad_offset(b_off))
1101+
return -E2BIG;
1102+
emit_b(b_off, ctx);
10791103
emit_nop(ctx);
10801104
}
10811105
break;
@@ -1133,8 +1157,10 @@ static int build_body(struct jit_ctx *ctx)
11331157
/* Load *dev pointer */
11341158
emit_load_ptr(r_s0, r_skb, off, ctx);
11351159
/* error (0) in the delay slot */
1136-
emit_bcond(MIPS_COND_EQ, r_s0, r_zero,
1137-
b_imm(prog->len, ctx), ctx);
1160+
b_off = b_imm(prog->len, ctx);
1161+
if (is_bad_offset(b_off))
1162+
return -E2BIG;
1163+
emit_bcond(MIPS_COND_EQ, r_s0, r_zero, b_off, ctx);
11381164
emit_reg_move(r_ret, r_zero, ctx);
11391165
if (code == (BPF_ANC | SKF_AD_IFINDEX)) {
11401166
BUILD_BUG_ON(sizeof_field(struct net_device, ifindex) != 4);
@@ -1244,7 +1270,10 @@ void bpf_jit_compile(struct bpf_prog *fp)
12441270

12451271
/* Generate the actual JIT code */
12461272
build_prologue(&ctx);
1247-
build_body(&ctx);
1273+
if (build_body(&ctx)) {
1274+
module_memfree(ctx.target);
1275+
goto out;
1276+
}
12481277
build_epilogue(&ctx);
12491278

12501279
/* Update the icache */

‎arch/x86/net/bpf_jit_comp.c

Lines changed: 48 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1341,9 +1341,10 @@ st: if (is_imm8(insn->off))
13411341
if (insn->imm == (BPF_AND | BPF_FETCH) ||
13421342
insn->imm == (BPF_OR | BPF_FETCH) ||
13431343
insn->imm == (BPF_XOR | BPF_FETCH)) {
1344-
u8 *branch_target;
13451344
bool is64 = BPF_SIZE(insn->code) == BPF_DW;
13461345
u32 real_src_reg = src_reg;
1346+
u32 real_dst_reg = dst_reg;
1347+
u8 *branch_target;
13471348

13481349
/*
13491350
* Can't be implemented with a single x86 insn.
@@ -1354,11 +1355,13 @@ st: if (is_imm8(insn->off))
13541355
emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0);
13551356
if (src_reg == BPF_REG_0)
13561357
real_src_reg = BPF_REG_AX;
1358+
if (dst_reg == BPF_REG_0)
1359+
real_dst_reg = BPF_REG_AX;
13571360

13581361
branch_target = prog;
13591362
/* Load old value */
13601363
emit_ldx(&prog, BPF_SIZE(insn->code),
1361-
BPF_REG_0, dst_reg, insn->off);
1364+
BPF_REG_0, real_dst_reg, insn->off);
13621365
/*
13631366
* Perform the (commutative) operation locally,
13641367
* put the result in the AUX_REG.
@@ -1369,7 +1372,8 @@ st: if (is_imm8(insn->off))
13691372
add_2reg(0xC0, AUX_REG, real_src_reg));
13701373
/* Attempt to swap in new value */
13711374
err = emit_atomic(&prog, BPF_CMPXCHG,
1372-
dst_reg, AUX_REG, insn->off,
1375+
real_dst_reg, AUX_REG,
1376+
insn->off,
13731377
BPF_SIZE(insn->code));
13741378
if (WARN_ON(err))
13751379
return err;
@@ -1383,11 +1387,10 @@ st: if (is_imm8(insn->off))
13831387
/* Restore R0 after clobbering RAX */
13841388
emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX);
13851389
break;
1386-
13871390
}
13881391

13891392
err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,
1390-
insn->off, BPF_SIZE(insn->code));
1393+
insn->off, BPF_SIZE(insn->code));
13911394
if (err)
13921395
return err;
13931396
break;
@@ -1744,7 +1747,7 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
17441747
}
17451748

17461749
static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
1747-
struct bpf_prog *p, int stack_size, bool mod_ret)
1750+
struct bpf_prog *p, int stack_size, bool save_ret)
17481751
{
17491752
u8 *prog = *pprog;
17501753
u8 *jmp_insn;
@@ -1777,11 +1780,15 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
17771780
if (emit_call(&prog, p->bpf_func, prog))
17781781
return -EINVAL;
17791782

1780-
/* BPF_TRAMP_MODIFY_RETURN trampolines can modify the return
1783+
/*
1784+
* BPF_TRAMP_MODIFY_RETURN trampolines can modify the return
17811785
* of the previous call which is then passed on the stack to
17821786
* the next BPF program.
1787+
*
1788+
* BPF_TRAMP_FENTRY trampoline may need to return the return
1789+
* value of BPF_PROG_TYPE_STRUCT_OPS prog.
17831790
*/
1784-
if (mod_ret)
1791+
if (save_ret)
17851792
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
17861793

17871794
/* replace 2 nops with JE insn, since jmp target is known */
@@ -1828,13 +1835,15 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
18281835
}
18291836

18301837
static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
1831-
struct bpf_tramp_progs *tp, int stack_size)
1838+
struct bpf_tramp_progs *tp, int stack_size,
1839+
bool save_ret)
18321840
{
18331841
int i;
18341842
u8 *prog = *pprog;
18351843

18361844
for (i = 0; i < tp->nr_progs; i++) {
1837-
if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, false))
1845+
if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size,
1846+
save_ret))
18381847
return -EINVAL;
18391848
}
18401849
*pprog = prog;
@@ -1877,6 +1886,23 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
18771886
return 0;
18781887
}
18791888

1889+
static bool is_valid_bpf_tramp_flags(unsigned int flags)
1890+
{
1891+
if ((flags & BPF_TRAMP_F_RESTORE_REGS) &&
1892+
(flags & BPF_TRAMP_F_SKIP_FRAME))
1893+
return false;
1894+
1895+
/*
1896+
* BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops,
1897+
* and it must be used alone.
1898+
*/
1899+
if ((flags & BPF_TRAMP_F_RET_FENTRY_RET) &&
1900+
(flags & ~BPF_TRAMP_F_RET_FENTRY_RET))
1901+
return false;
1902+
1903+
return true;
1904+
}
1905+
18801906
/* Example:
18811907
* __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev);
18821908
* its 'struct btf_func_model' will be nr_args=2
@@ -1949,17 +1975,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
19491975
struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
19501976
u8 **branches = NULL;
19511977
u8 *prog;
1978+
bool save_ret;
19521979

19531980
/* x86-64 supports up to 6 arguments. 7+ can be added in the future */
19541981
if (nr_args > 6)
19551982
return -ENOTSUPP;
19561983

1957-
if ((flags & BPF_TRAMP_F_RESTORE_REGS) &&
1958-
(flags & BPF_TRAMP_F_SKIP_FRAME))
1984+
if (!is_valid_bpf_tramp_flags(flags))
19591985
return -EINVAL;
19601986

1961-
if (flags & BPF_TRAMP_F_CALL_ORIG)
1962-
stack_size += 8; /* room for return value of orig_call */
1987+
/* room for return value of orig_call or fentry prog */
1988+
save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
1989+
if (save_ret)
1990+
stack_size += 8;
19631991

19641992
if (flags & BPF_TRAMP_F_IP_ARG)
19651993
stack_size += 8; /* room for IP address argument */
@@ -2005,7 +2033,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
20052033
}
20062034

20072035
if (fentry->nr_progs)
2008-
if (invoke_bpf(m, &prog, fentry, stack_size))
2036+
if (invoke_bpf(m, &prog, fentry, stack_size,
2037+
flags & BPF_TRAMP_F_RET_FENTRY_RET))
20092038
return -EINVAL;
20102039

20112040
if (fmod_ret->nr_progs) {
@@ -2052,7 +2081,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
20522081
}
20532082

20542083
if (fexit->nr_progs)
2055-
if (invoke_bpf(m, &prog, fexit, stack_size)) {
2084+
if (invoke_bpf(m, &prog, fexit, stack_size, false)) {
20562085
ret = -EINVAL;
20572086
goto cleanup;
20582087
}
@@ -2072,9 +2101,10 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
20722101
ret = -EINVAL;
20732102
goto cleanup;
20742103
}
2075-
/* restore original return value back into RAX */
2076-
emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
20772104
}
2105+
/* restore return value of orig_call or fentry prog back into RAX */
2106+
if (save_ret)
2107+
emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
20782108

20792109
EMIT1(0x5B); /* pop rbx */
20802110
EMIT1(0xC9); /* leave */

‎include/linux/bpf.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -578,11 +578,12 @@ struct btf_func_model {
578578
* programs only. Should not be used with normal calls and indirect calls.
579579
*/
580580
#define BPF_TRAMP_F_SKIP_FRAME BIT(2)
581-
582581
/* Store IP address of the caller on the trampoline stack,
583582
* so it's available for trampoline's programs.
584583
*/
585584
#define BPF_TRAMP_F_IP_ARG BIT(3)
585+
/* Return the return value of fentry prog. Only used by bpf_struct_ops. */
586+
#define BPF_TRAMP_F_RET_FENTRY_RET BIT(4)
586587

587588
/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
588589
* bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2

‎kernel/bpf/bpf_struct_ops.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
368368
const struct btf_type *mtype, *ptype;
369369
struct bpf_prog *prog;
370370
u32 moff;
371+
u32 flags;
371372

372373
moff = btf_member_bit_offset(t, member) / 8;
373374
ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL);
@@ -431,10 +432,12 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
431432

432433
tprogs[BPF_TRAMP_FENTRY].progs[0] = prog;
433434
tprogs[BPF_TRAMP_FENTRY].nr_progs = 1;
435+
flags = st_ops->func_models[i].ret_size > 0 ?
436+
BPF_TRAMP_F_RET_FENTRY_RET : 0;
434437
err = arch_prepare_bpf_trampoline(NULL, image,
435438
st_map->image + PAGE_SIZE,
436-
&st_ops->func_models[i], 0,
437-
tprogs, NULL);
439+
&st_ops->func_models[i],
440+
flags, tprogs, NULL);
438441
if (err < 0)
439442
goto reset_unlock;
440443

‎kernel/bpf/core.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -827,7 +827,7 @@ int bpf_jit_charge_modmem(u32 pages)
827827
{
828828
if (atomic_long_add_return(pages, &bpf_jit_current) >
829829
(bpf_jit_limit >> PAGE_SHIFT)) {
830-
if (!capable(CAP_SYS_ADMIN)) {
830+
if (!bpf_capable()) {
831831
atomic_long_sub(pages, &bpf_jit_current);
832832
return -EPERM;
833833
}

‎kernel/cgroup/cgroup.c

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6574,22 +6574,29 @@ int cgroup_parse_float(const char *input, unsigned dec_shift, s64 *v)
65746574

65756575
void cgroup_sk_alloc(struct sock_cgroup_data *skcd)
65766576
{
6577-
/* Don't associate the sock with unrelated interrupted task's cgroup. */
6578-
if (in_interrupt())
6579-
return;
6577+
struct cgroup *cgroup;
65806578

65816579
rcu_read_lock();
6580+
/* Don't associate the sock with unrelated interrupted task's cgroup. */
6581+
if (in_interrupt()) {
6582+
cgroup = &cgrp_dfl_root.cgrp;
6583+
cgroup_get(cgroup);
6584+
goto out;
6585+
}
6586+
65826587
while (true) {
65836588
struct css_set *cset;
65846589

65856590
cset = task_css_set(current);
65866591
if (likely(cgroup_tryget(cset->dfl_cgrp))) {
6587-
skcd->cgroup = cset->dfl_cgrp;
6588-
cgroup_bpf_get(cset->dfl_cgrp);
6592+
cgroup = cset->dfl_cgrp;
65896593
break;
65906594
}
65916595
cpu_relax();
65926596
}
6597+
out:
6598+
skcd->cgroup = cgroup;
6599+
cgroup_bpf_get(cgroup);
65936600
rcu_read_unlock();
65946601
}
65956602

0 commit comments

Comments
 (0)