BPF indirect jumps

[kernel-patches-daemon-bpf]

Pull request for series with
subject: BPF indirect jumps
version: 6
url: https://patchwork.kernel.org/project/netdevbpf/list/?series=1013317

[kernel-patches-daemon-bpf]

Upstream branch: 7361c86
series: https://patchwork.kernel.org/project/netdevbpf/list/?series=1013317
version: 6

[kernel-patches-review-bot]

commit a468fcf91d43099e388a12fff8fa9aaee26e6843
Author: Anton Protopopov <[email protected]>

bpf, x86: allow indirect jumps to r8...r15

This commit refactors emit_indirect_jump() to accept BPF register
constants instead of native x86 register numbers, enabling support for
extended registers (R8-R15) in future indirect jump implementations.

> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
> index c8e628410..7443465ce 100644
> --- a/arch/x86/net/bpf_jit_comp.c
> +++ b/arch/x86/net/bpf_jit_comp.c
> @@ -660,24 +660,38 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
>
>  #define EMIT_LFENCE()	EMIT3(0x0F, 0xAE, 0xE8)
>
> -static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)
> +static void __emit_indirect_jump(u8 **pprog, int reg, bool ereg)
> +{
> +	u8 *prog = *pprog;
> +
> +	if (ereg)
> +		EMIT1(0x41);
> +
> +	EMIT2(0xFF, 0xE0 + reg);
> +
> +	*pprog = prog;
> +}
> +
> +static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)
>  {
>  	u8 *prog = *pprog;
> +	int reg = reg2hex[bpf_reg];
> +	bool ereg = is_ereg(bpf_reg);
>
>  	if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) {
>  		OPTIMIZER_HIDE_VAR(reg);
>  		emit_jump(&prog, its_static_thunk(reg), ip);
                                                ^^^

Can its_static_thunk() select the wrong thunk for extended registers?
The function indexes __x86_indirect_its_thunk_array[] with reg directly,
but reg holds only the lower 3 bits from reg2hex[bpf_reg]. For extended
registers like BPF_REG_5 (which maps to R8), reg would be 0, causing
its_static_thunk() to return the RAX thunk instead of the R8 thunk.

The other code paths (RETPOLINE, RETPOLINE_LFENCE, and fallback) all use
reg + 8*ereg or __emit_indirect_jump(&prog, reg, ereg) which correctly
handle extended registers. Should the ITS path also use reg + 8*ereg?

For example, with BPF_REG_5:
  reg2hex[5] = 0, is_ereg(5) = true
  Expected thunk array index: 8 (for R8)
  Current ITS path: its_static_thunk(0) -> array[0] (RAX thunk)
  RETPOLINE path: array[0 + 8*1] = array[8] (R8 thunk, correct)

>  	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
>  		EMIT_LFENCE();
> -		EMIT2(0xFF, 0xE0 + reg);
> +		__emit_indirect_jump(&prog, reg, ereg);
>  	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
>  		OPTIMIZER_HIDE_VAR(reg);
>  		if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
> -			emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip);
> +			emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg + 8*ereg], ip);
>  		else
> -			emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);
> +			emit_jump(&prog, &__x86_indirect_thunk_array[reg + 8*ereg], ip);
>  	} else {
> -		EMIT2(0xFF, 0xE0 + reg);	/* jmp *%\reg */
> +		__emit_indirect_jump(&prog, reg, ereg);
>  		if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || IS_ENABLED(CONFIG_MITIGATION_SLS))
>  			EMIT1(0xCC);		/* int3 */
>  	}
> @@ -797,7 +811,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog,
>  	 * rdi == ctx (1st arg)
>  	 * rcx == prog->bpf_func + X86_TAIL_CALL_OFFSET
>  	 */
> -	emit_indirect_jump(&prog, 1 /* rcx */, ip + (prog - start));
> +	emit_indirect_jump(&prog, BPF_REG_4 /* R4 -> rcx */, ip + (prog - start));
>
>  	/* out: */
>  	ctx->tail_call_indirect_label = prog - start;
> @@ -3551,7 +3565,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image,
>  		if (err)
>  			return err;
>
> -		emit_indirect_jump(&prog, 2 /* rdx */, image + (prog - buf));
> +		emit_indirect_jump(&prog, BPF_REG_3 /* R3 -> rdx */, image + (prog - buf));
>
>  		*pprog = prog;
>  		return 0;

---

AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

In-Reply-To-Subject: bpf, x86: allow indirect jumps to r8...r15
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/18635636744

[kernel-patches-review-bot]

commit 2fff8febd97938dd95ef0044faa4ddae02706451
Author: Anton Protopopov <[email protected]>

bpf, x86: add support for indirect jumps

This patch adds support for a new BPF instruction that performs indirect
jumps to locations stored in registers with type PTR_TO_INSN. The verifier
tracks jump tables loaded from BPF_MAP_TYPE_INSN_ARRAY maps.

> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
> index 7443465ce..b6e646039 100644
> --- a/arch/x86/net/bpf_jit_comp.c
> +++ b/arch/x86/net/bpf_jit_comp.c
> @@ -2629,6 +2629,9 @@ st:			if (is_imm8(insn->off))
>
>  			break;
>
> +		case BPF_JMP | BPF_JA | BPF_X:
> +			emit_indirect_jump(&prog, insn->dst_reg, image + addrs[i - 1]);
> +			break;
>  		case BPF_JMP | BPF_JA:
>  		case BPF_JMP32 | BPF_JA:
>  			if (BPF_CLASS(insn->code) == BPF_JMP) {

[ ... ]

> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index ae017c032..d2df21fde 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -6003,6 +6003,18 @@ static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
>  	return 0;
>  }
>
> +/*
> + * Return the size of the memory region accessible from a pointer to map value.
> + * For INSN_ARRAY maps whole bpf_insn_array->ips array is accessible.
> + */
> +static u32 map_mem_size(const struct bpf_map *map)
> +{
> +	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
> +		return map->max_entries * sizeof(long);
> +
> +	return map->value_size;
> +}
> +

[ ... ]

> @@ -17823,6 +17851,196 @@ static struct bpf_iarray *iarray_realloc(struct bpf_iarray *old, size_t n_elem)
>  	return new;
>  }
>
> +static int copy_insn_array(struct bpf_map *map, u32 start, u32 end, u32 *items)
> +{
> +	struct bpf_insn_array_value *value;
> +	u32 i;
> +
> +	for (i = start; i <= end; i++) {
> +		value = map->ops->map_lookup_elem(map, &i);
> +		if (!value)
> +			return -EINVAL;
> +		items[i - start] = value->xlated_off;
> +	}
> +	return 0;
> +}
> +
> +static int cmp_ptr_to_u32(const void *a, const void *b)
> +{
> +	return *(u32 *)a - *(u32 *)b;
> +}
> +
> +static int sort_insn_array_uniq(u32 *items, int cnt)
> +{
> +	int unique = 1;
> +	int i;
> +
> +	sort(items, cnt, sizeof(items[0]), cmp_ptr_to_u32, NULL);
> +
> +	for (i = 1; i < cnt; i++)
> +		if (items[i] != items[unique - 1])
> +			items[unique++] = items[i];
> +
> +	return unique;
> +}
> +
> +/*
> + * sort_unique({map[start], ..., map[end]}) into off
> + */
> +static int copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off)
> +{
> +	u32 n = end - start + 1;
> +	int err;
> +
> +	err = copy_insn_array(map, start, end, off);
> +	if (err)
> +		return err;
> +
> +	return sort_insn_array_uniq(off, n);
> +}
> +
> +/*
> + * Copy all unique offsets from the map
> + */
> +static struct bpf_iarray *jt_from_map(struct bpf_map *map)
> +{
> +	struct bpf_iarray *jt;
> +	int n;
> +
> +	jt = iarray_realloc(NULL, map->max_entries);
> +	if (!jt)
> +		return ERR_PTR(-ENOMEM);
> +
> +	n = copy_insn_array_uniq(map, 0, map->max_entries - 1, jt->items);
> +	if (n < 0) {
> +		kvfree(jt);
> +		return ERR_PTR(n);
> +	}
> +
> +	return jt;
> +}
> +
> +/*
> + * Find and collect all maps which fit in the subprog. Return the result as one
> + * combined jump table in jt->items (allocated with kvcalloc)
> + */
> +static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,
> +					  int subprog_start, int subprog_end)
> +{
> +	struct bpf_iarray *jt = NULL;
> +	struct bpf_map *map;
> +	struct bpf_iarray *jt_cur;
> +	int i;
> +
> +	for (i = 0; i < env->insn_array_map_cnt; i++) {
> +		/*
> +		 * TODO (when needed): collect only jump tables, not static keys
> +		 * or maps for indirect calls
> +		 */
> +		map = env->insn_array_maps[i];
> +
> +		jt_cur = jt_from_map(map);
> +		if (IS_ERR(jt_cur)) {
> +			kvfree(jt);
> +			return jt_cur;
> +		}
> +
> +		/*
> +		 * This is enough to check one element. The full table is
> +		 * checked to fit inside the subprog later in create_jt()
> +		 */
> +		if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) {
> +			u32 old_cnt = jt ? jt->cnt : 0;
> +			jt = iarray_realloc(jt, old_cnt + jt_cur->cnt);
> +			if (!jt) {
> +				kvfree(jt_cur);
> +				return ERR_PTR(-ENOMEM);
> +			}
> +			memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2);
> +		}
> +
> +		kvfree(jt_cur);
> +	}
> +
> +	if (!jt) {
> +		verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);
> +		return ERR_PTR(-EINVAL);
> +	}
> +
> +	jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt);
> +	return jt;
> +}
> +
> +static struct bpf_iarray *
> +create_jt(int t, struct bpf_verifier_env *env, int fd)
> +{
> +	static struct bpf_subprog_info *subprog;
> +	int subprog_start, subprog_end;
> +	struct bpf_iarray *jt;
> +	int i;
> +
> +	subprog = bpf_find_containing_subprog(env, t);
> +	subprog_start = subprog->start;
> +	subprog_end = (subprog + 1)->start;
> +	jt = jt_from_subprog(env, subprog_start, subprog_end);
> +	if (IS_ERR(jt))
> +		return jt;
> +
> +	/* Check that the every element of the jump table fits within the given subprogram */
> +	for (i = 0; i < jt->cnt; i++) {
> +		if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) {
> +			verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]",
> +					t, subprog_start, subprog_end);
> +			return ERR_PTR(-EINVAL);
                        ^^^^^^^^^^^^^^^^^^^^^^^^^

Can this leak the jt buffer? The jt_from_subprog() call above successfully
allocated the jump table, but if this validation loop finds an invalid
jump target, the function returns an error without freeing jt. The caller
visit_gotox_insn() won't store the pointer in env->insn_aux_data[t].jt,
so the buffer can never be freed later.

> +		}
> +	}
> +
> +	return jt;
> +}
> +
> +/* "conditional jump with N edges" */
> +static int visit_gotox_insn(int t, struct bpf_verifier_env *env, int fd)
> +{
> +	int *insn_stack = env->cfg.insn_stack;
> +	int *insn_state = env->cfg.insn_state;
> +	bool keep_exploring = false;
> +	struct bpf_iarray *jt;
> +	int i, w;
> +
> +	jt = env->insn_aux_data[t].jt;
> +	if (!jt) {
> +		jt = create_jt(t, env, fd);
> +		if (IS_ERR(jt))
> +			return PTR_ERR(jt);
> +
> +		env->insn_aux_data[t].jt = jt;
> +	}

[ ... ]

> @@ -19854,6 +20075,99 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env,
>  	return PROCESS_BPF_EXIT;
>  }
>
> +static int indirect_jump_min_max_index(struct bpf_verifier_env *env,
> +				       int regno,
> +				       struct bpf_map *map,
> +				       u32 *pmin_index, u32 *pmax_index)
> +{
> +	struct bpf_reg_state *reg = reg_state(env, regno);
> +	u64 min_index, max_index;
> +	const u32 size = 8;
> +
> +	if (check_add_overflow(reg->umin_value, reg->off, &min_index) ||
> +		(min_index > (u64) U32_MAX * size)) {
> +		verbose(env, "the sum of R%u umin_value %llu and off %u is too big\n",
> +			     regno, reg->umin_value, reg->off);
> +		return -ERANGE;
> +	}
> +	if (check_add_overflow(reg->umax_value, reg->off, &max_index) ||
> +		(max_index > (u64) U32_MAX * size)) {
> +		verbose(env, "the sum of R%u umax_value %llu and off %u is too big\n",
> +			     regno, reg->umax_value, reg->off);
> +		return -ERANGE;
> +	}
> +
> +	min_index /= size;
> +	max_index /= size;
> +
> +	if (max_index >= map->max_entries) {
> +		verbose(env, "R%u points to outside of jump table: [%llu,%llu] max_entries %u\n",
> +			     regno, min_index, max_index, map->max_entries);
> +		return -EINVAL;
> +	}
> +
> +	*pmin_index = min_index;
> +	*pmax_index = max_index;
> +	return 0;
> +}

[ ... ]

---

AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

In-Reply-To-Subject: bpf, x86: add support for indirect jumps
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/18635636744

[kernel-patches-daemon-bpf]

Forwarding comment 3419938218 via email
In-Reply-To: [email protected]
Patch: https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

[kernel-patches-daemon-bpf]

Forwarding comment 3419938448 via email
In-Reply-To: [email protected]
Patch: https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/