bpf, x86: add support for indirect jumps

Add support for a new instruction

    BPF_JMP|BPF_X|BPF_JA, SRC=0, DST=Rx, off=0, imm=fd(M)

which does an indirect jump to a location stored in Rx. The map M
is an instruction array map containing all possible targets for this
particular jump.

On the jump the register Rx should have type PTR_TO_INSN. This new
type assures that the Rx register contains a value (or a range of
values) loaded from the map M. Typically, this will be done like this
The code above could have been generated for a switch statement with
(e.g., this could be a switch statement compiled with LLVM):

    0:   r3 = r1                    # "switch (r3)"
    1:   if r3 > 0x13 goto +0x666   # check r3 boundaries
    2:   r3 <<= 0x3                 # r3 is void*, point to an address
    3:   r1 = 0xbeef ll             # r1 is PTR_TO_MAP_VALUE, r1->map_ptr=M
    5:   r1 += r3                   # r1 inherits boundaries from r3
    6:   r1 = *(u64 *)(r1 + 0x0)    # r1 now has type INSN_TO_PTR
    7:   gotox r1[,imm=fd(M)]       # verifier checks that M == r1->map_ptr

On building the jump graph, and the static analysis, a new function
of the INSN_ARRAY is used: bpf_insn_array_iter_xlated_offset(map, n).
It lets to iterate over unique slots in an instruction array (equal
items can be generated, e.g., for a sparse jump table for a switch,
where not all possible branches are taken).

Instruction (3) above loads an address of the first element of the
map. From BPF point of view, the map is a jump table in native
architecture, e.g., an array of jump targets. This patch allows
to grab such an address and then later to adjust an offset, like in
instruction (5). A value of such type can be dereferenced once to
create a PTR_TO_INSN, see instruction (6).

When building the config, the high 16 bytes of the insn_state are
used, so this patch (theoretically) supports jump tables of up to
2^16 slots.

Signed-off-by: Anton Protopopov <[email protected]>

Author: aspsk

arch/x86/net/bpf_jit_comp.c

@@ -671,9 +671,11 @@ static void __emit_indirect_jump(u8 **pprog, int reg, bool ereg)
 	*pprog = prog;
 }
 
-static void emit_indirect_jump(u8 **pprog, int reg, bool ereg, u8 *ip)
+static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)
 {
 	u8 *prog = *pprog;
+	int reg = reg2hex[bpf_reg];
+	bool ereg = is_ereg(bpf_reg);
 
 	if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) {
 		OPTIMIZER_HIDE_VAR(reg);
@@ -808,7 +810,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog,
 	 * rdi == ctx (1st arg)
 	 * rcx == prog->bpf_func + X86_TAIL_CALL_OFFSET
 	 */
-	emit_indirect_jump(&prog, 1 /* rcx */, false, ip + (prog - start));
+	emit_indirect_jump(&prog, BPF_REG_4 /* R4 -> rcx */, ip + (prog - start));
 
 	/* out: */
 	ctx->tail_call_indirect_label = prog - start;
@@ -2518,6 +2520,10 @@ st:			if (is_imm8(insn->off))
 
 			break;
 
+		case BPF_JMP | BPF_JA | BPF_X:
+		case BPF_JMP32 | BPF_JA | BPF_X:
+			emit_indirect_jump(&prog, insn->dst_reg, image + addrs[i - 1]);
+			break;
 		case BPF_JMP | BPF_JA:
 		case BPF_JMP32 | BPF_JA:
 			if (BPF_CLASS(insn->code) == BPF_JMP) {
@@ -3454,7 +3460,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image,
 		if (err)
 			return err;
 
-		emit_indirect_jump(&prog, 2 /* rdx */, false, image + (prog - buf));
+		emit_indirect_jump(&prog, BPF_REG_3 /* R3 -> rdx */, image + (prog - buf));
 
 		*pprog = prog;
 		return 0;

include/linux/bpf.h

@@ -952,6 +952,7 @@ enum bpf_reg_type {
 	PTR_TO_ARENA,
 	PTR_TO_BUF,		 /* reg points to a read/write buffer */
 	PTR_TO_FUNC,		 /* reg points to a bpf program function */
+	PTR_TO_INSN,		 /* reg points to a bpf program instruction */
 	CONST_PTR_TO_DYNPTR,	 /* reg points to a const struct bpf_dynptr */
 	__BPF_REG_TYPE_MAX,
 
@@ -3601,6 +3602,7 @@ int bpf_insn_array_ready(struct bpf_map *map);
 void bpf_insn_array_release(struct bpf_map *map);
 void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len);
 void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len);
+int bpf_insn_array_iter_xlated_offset(struct bpf_map *map, u32 iter_no);
 
 /*
  * The struct bpf_insn_ptr structure describes a pointer to a

include/linux/bpf_verifier.h

@@ -229,6 +229,10 @@ struct bpf_reg_state {
 	enum bpf_reg_liveness live;
 	/* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */
 	bool precise;
+
+	/* Used to track boundaries of a PTR_TO_INSN */
+	u32 min_index;
+	u32 max_index;
 };
 
 enum bpf_stack_slot_type {

kernel/bpf/bpf_insn_array.c

@@ -9,6 +9,8 @@ struct bpf_insn_array {
 	struct bpf_map map;
 	struct mutex state_mutex;
 	int state;
+	u32 **unique_offsets;
+	u32 unique_offsets_cnt;
 	long *ips;
 	DECLARE_FLEX_ARRAY(struct bpf_insn_ptr, ptrs);
 };
@@ -50,6 +52,7 @@ static void insn_array_free(struct bpf_map *map)
 {
 	struct bpf_insn_array *insn_array = cast_insn_array(map);
 
+	kfree(insn_array->unique_offsets);
 	kfree(insn_array->ips);
 	bpf_map_area_free(insn_array);
 }
@@ -69,6 +72,12 @@ static struct bpf_map *insn_array_alloc(union bpf_attr *attr)
 		return ERR_PTR(-ENOMEM);
 	}
 
+	insn_array->unique_offsets = kzalloc(sizeof(long) * attr->max_entries, GFP_KERNEL);
+	if (!insn_array->unique_offsets) {
+		insn_array_free(&insn_array->map);
+		return ERR_PTR(-ENOMEM);
+	}
+
 	bpf_map_init_from_attr(&insn_array->map, attr);
 
 	mutex_init(&insn_array->state_mutex);
@@ -155,10 +164,25 @@ static u64 insn_array_mem_usage(const struct bpf_map *map)
 	u64 extra_size = 0;
 
 	extra_size += sizeof(long) * map->max_entries; /* insn_array->ips */
+	extra_size += 4 * map->max_entries; /* insn_array->unique_offsets */
 
 	return insn_array_alloc_size(map->max_entries) + extra_size;
 }
 
+static int insn_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off)
+{
+	struct bpf_insn_array *insn_array = cast_insn_array(map);
+
+	/* for now, just reject all such loads */
+	if (off > 0)
+		return -EINVAL;
+
+	/* from BPF's point of view, this map is a jump table */
+	*imm = (unsigned long)insn_array->ips;
+
+	return 0;
+}
+
 BTF_ID_LIST_SINGLE(insn_array_btf_ids, struct, bpf_insn_array)
 
 const struct bpf_map_ops insn_array_map_ops = {
@@ -171,6 +195,7 @@ const struct bpf_map_ops insn_array_map_ops = {
 	.map_delete_elem = insn_array_delete_elem,
 	.map_check_btf = insn_array_check_btf,
 	.map_mem_usage = insn_array_mem_usage,
+	.map_direct_value_addr = insn_array_map_direct_value_addr,
 	.map_btf_id = &insn_array_btf_ids[0],
 };
 
@@ -207,6 +232,37 @@ static inline bool valid_offsets(const struct bpf_insn_array *insn_array,
 	return true;
 }
 
+static int cmp_unique_offsets(const void *a, const void *b)
+{
+	return *(u32 *)a - *(u32 *)b;
+}
+
+static int bpf_insn_array_init_unique_offsets(struct bpf_insn_array *insn_array)
+{
+	u32 cnt = insn_array->map.max_entries, ucnt = 1;
+	u32 **off = insn_array->unique_offsets;
+	int i;
+
+	/* [0,3,2,4,6,5,5,5,1,1,0,0] */
+	for (i = 0; i < cnt; i++)
+		off[i] = &insn_array->ptrs[i].user_value.xlated_off;
+
+	/* [0,0,0,1,1,2,3,4,5,5,5,6] */
+	sort(off, cnt, sizeof(off[0]), cmp_unique_offsets, NULL);
+
+	/*
+	 * [0,1,2,3,4,5,6,x,x,x,x,x]
+	 *  \.........../
+	 *    unique_offsets_cnt
+	 */
+	for (i = 1; i < cnt; i++)
+		if (*off[i] != *off[ucnt-1])
+			off[ucnt++] = off[i];
+
+	insn_array->unique_offsets_cnt = ucnt;
+	return 0;
+}
+
 int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog)
 {
 	struct bpf_insn_array *insn_array = cast_insn_array(map);
@@ -237,7 +293,10 @@ int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog)
 	for (i = 0; i < map->max_entries; i++)
 		insn_array->ptrs[i].user_value.xlated_off = insn_array->ptrs[i].orig_xlated_off;
 
-	return 0;
+	/*
+	 * Prepare a set of unique offsets
+	 */
+	return bpf_insn_array_init_unique_offsets(insn_array);
 }
 
 int bpf_insn_array_ready(struct bpf_map *map)
@@ -330,3 +389,13 @@ void bpf_prog_update_insn_ptr(struct bpf_prog *prog,
 		}
 	}
 }
+
+int bpf_insn_array_iter_xlated_offset(struct bpf_map *map, u32 iter_no)
+{
+	struct bpf_insn_array *insn_array = cast_insn_array(map);
+
+	if (iter_no >= insn_array->unique_offsets_cnt)
+		return -ENOENT;
+
+	return *insn_array->unique_offsets[iter_no];
+}

kernel/bpf/core.c

@@ -1725,6 +1725,8 @@ bool bpf_opcode_in_insntable(u8 code)
 		[BPF_LD | BPF_IND | BPF_B] = true,
 		[BPF_LD | BPF_IND | BPF_H] = true,
 		[BPF_LD | BPF_IND | BPF_W] = true,
+		[BPF_JMP | BPF_JA | BPF_X] = true,
+		[BPF_JMP32 | BPF_JA | BPF_X] = true,
 		[BPF_JMP | BPF_JCOND] = true,
 	};
 #undef BPF_INSN_3_TBL