tetragon: split process event tail call

bpf/include/api.h

@@ -323,4 +323,10 @@ static int BPF_FUNC(seq_write, struct seq_file *m, const void *data, uint32_t le
 # define lock_and(ptr, val)	(*(ptr) &= val)
 #endif
 
+enum {
+	__READ_ARG_1,
+	__READ_ARG_2,
+	__READ_ARG_ALL,
+};
+
 #endif /* __BPF_API__ */

bpf/process/bpf_generic_kprobe.c

@@ -18,6 +18,7 @@ char _license[] __attribute__((section("license"), used)) = "Dual BSD/GPL";
 
 int generic_kprobe_setup_event(void *ctx);
 int generic_kprobe_process_event(void *ctx);
+int generic_kprobe_process_event_2(void *ctx);
 int generic_kprobe_process_filter(void *ctx);
 int generic_kprobe_filter_arg(void *ctx);
 int generic_kprobe_actions(void *ctx);
@@ -39,6 +40,9 @@ struct {
 		[TAIL_CALL_SEND] = (void *)&generic_kprobe_output,
 #ifndef __V61_BPF_PROG
 		[TAIL_CALL_PATH] = (void *)&generic_kprobe_path,
+#endif
+#ifndef __LARGE_BPF_PROG
+		[TAIL_CALL_PROCESS_2] = (void *)&generic_kprobe_process_event_2,
 #endif
 	},
 };
@@ -91,12 +95,26 @@ generic_kprobe_setup_event(void *ctx)
 	return generic_process_event_and_setup(ctx, (struct bpf_map_def *)&kprobe_calls);
 }
 
+#ifdef __LARGE_BPF_PROG
+__attribute__((section(COMMON), used)) int
+generic_kprobe_process_event(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls, __READ_ARG_ALL);
+}
+#else
 __attribute__((section(COMMON), used)) int
 generic_kprobe_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls, __READ_ARG_1);
 }
 
+__attribute__((section(COMMON), used)) int
+generic_kprobe_process_event_2(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls, __READ_ARG_2);
+}
+#endif
+
 __attribute__((section(COMMON), used)) int
 generic_kprobe_process_filter(void *ctx)
 {

bpf/process/bpf_generic_lsm_core.c

@@ -63,7 +63,7 @@ generic_lsm_setup_event(void *ctx)
 __attribute__((section("lsm"), used)) int
 generic_lsm_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&lsm_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&lsm_calls, __READ_ARG_ALL);
 }
 
 __attribute__((section("lsm"), used)) int

bpf/process/bpf_generic_rawtp.c

@@ -84,7 +84,7 @@ generic_rawtp_setup_event(void *ctx)
 __attribute__((section("raw_tp"), used)) int
 generic_rawtp_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls, __READ_ARG_ALL);
 }
 
 __attribute__((section("raw_tp"), used)) int

bpf/process/bpf_generic_tracepoint.c

@@ -22,6 +22,7 @@ int generic_tracepoint_filter(void *ctx);
 int generic_tracepoint_arg(void *ctx);
 int generic_tracepoint_actions(void *ctx);
 int generic_tracepoint_output(void *ctx);
+int generic_tracepoint_process_event_2(void *ctx);
 
 struct {
 	__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
@@ -35,6 +36,9 @@ struct {
 		[TAIL_CALL_ARGS] = (void *)&generic_tracepoint_arg,
 		[TAIL_CALL_ACTIONS] = (void *)&generic_tracepoint_actions,
 		[TAIL_CALL_SEND] = (void *)&generic_tracepoint_output,
+#ifndef __LARGE_BPF_PROG
+		[TAIL_CALL_PROCESS_2] = (void *)&generic_tracepoint_process_event_2,
+#endif
 	},
 };
 
@@ -260,12 +264,26 @@ generic_tracepoint_event(struct generic_tracepoint_event_arg *ctx)
 	return 0;
 }
 
+#ifdef __LARGE_BPF_PROG
+__attribute__((section("tracepoint"), used)) int
+generic_tracepoint_process_event(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls, __READ_ARG_ALL);
+}
+#else
 __attribute__((section("tracepoint"), used)) int
 generic_tracepoint_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls, __READ_ARG_1);
 }
 
+__attribute__((section("tracepoint"), used)) int
+generic_tracepoint_process_event_2(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls, __READ_ARG_2);
+}
+#endif
+
 __attribute__((section("tracepoint"), used)) int
 generic_tracepoint_filter(void *ctx)
 {

bpf/process/bpf_generic_uprobe.c

@@ -18,6 +18,7 @@ char _license[] __attribute__((section("license"), used)) = "Dual BSD/GPL";
 
 int generic_uprobe_setup_event(void *ctx);
 int generic_uprobe_process_event(void *ctx);
+int generic_uprobe_process_event_2(void *ctx);
 int generic_uprobe_process_filter(void *ctx);
 int generic_uprobe_filter_arg(void *ctx);
 int generic_uprobe_actions(void *ctx);
@@ -39,6 +40,9 @@ struct {
 		[TAIL_CALL_SEND] = (void *)&generic_uprobe_output,
 #ifndef __V61_BPF_PROG
 		[TAIL_CALL_PATH] = (void *)&generic_uprobe_path,
+#endif
+#ifndef __LARGE_BPF_PROG
+		[TAIL_CALL_PROCESS_2] = (void *)&generic_uprobe_process_event_2,
 #endif
 	},
 };
@@ -68,12 +72,26 @@ generic_uprobe_setup_event(void *ctx)
 	return generic_process_event_and_setup(ctx, (struct bpf_map_def *)&uprobe_calls);
 }
 
+#ifdef __LARGE_BPF_PROG
+__attribute__((section(COMMON), used)) int
+generic_uprobe_process_event(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls, __READ_ARG_ALL);
+}
+#else
 __attribute__((section(COMMON), used)) int
 generic_uprobe_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls, __READ_ARG_1);
 }
 
+__attribute__((section(COMMON), used)) int
+generic_uprobe_process_event_2(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls, __READ_ARG_2);
+}
+#endif
+
 __attribute__((section(COMMON), used)) int
 generic_uprobe_process_filter(void *ctx)
 {

bpf/process/bpf_generic_usdt.c

@@ -72,7 +72,7 @@ generic_usdt_setup_event(void *ctx)
 __attribute__((section(COMMON), used)) int
 generic_usdt_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&usdt_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&usdt_calls, __READ_ARG_ALL);
 }
 
 __attribute__((section(COMMON), used)) int

bpf/process/generic_calls.h

@@ -190,47 +190,45 @@ copy_iov_iter(void *ctx, long off, unsigned long arg, int argm, struct msg_gener
 #define copy_iov_iter(ctx, orig_off, arg, argm, e) 0
 #endif /* __LARGE_BPF_PROG */
 
-/**
- * Read a generic argument
- *
- * @args: destination buffer for the generic argument
- * @type: type of the argument
- * @off: offset of the argument within @args
- * @arg: argument location (generally, address of the argument)
- * @argm: argument metadata. The meaning of this depends on the @type. Some
- *        types use a -1 to designate saving @arg into the retprobe map
- * @filter_map:
- *
- * Returns the size of data appended to @args.
- */
+FUNC_INLINE bool is_read_arg_1(long type)
+{
+	switch (type) {
+	case iov_iter_type:
+	case fd_ty:
+	case filename_ty:
+	case string_type:
+	case net_dev_ty:
+	case data_loc_type:
+	case syscall64_type:
+	case size_type:
+	case s64_ty:
+	case u64_ty:
+	case int_type:
+	case s32_ty:
+	case u32_ty:
+	case s16_ty:
+	case u16_ty:
+	case s8_ty:
+	case u8_ty:
+	case skb_type:
+	case sock_type:
+	case sockaddr_type:
+		return true;
+	}
+	return false;
+}
+
 FUNC_INLINE long
-read_arg(void *ctx, int index, int type, long orig_off, unsigned long arg, int argm)
+__read_arg_1(void *ctx, int type, long orig_off, unsigned long arg, int argm, char *args)
 {
-	size_t min_size = type_to_min_size(type, argm);
 	struct msg_generic_kprobe *e;
-	char *args;
 	long size = -1;
-	const struct path *path_arg = 0;
-	struct path path_buf;
 	int zero = 0;
 
 	e = map_lookup_elem(&process_call_heap, &zero);
 	if (!e)
 		return 0;
 
-	if (orig_off >= 16383 - min_size)
-		return 0;
-
-	orig_off &= 16383;
-	args = args_off(e, orig_off);
-
-	/* Cache args offset for filter use later */
-	e->argsoff[index & MAX_SELECTORS_MASK] = orig_off;
-
-	path_arg = get_path(type, arg, &path_buf);
-	if (path_arg)
-		return copy_path(args, path_arg);
-
 	switch (type) {
 	case iov_iter_type:
 		size = copy_iov_iter(ctx, orig_off, arg, argm, e);
@@ -326,6 +324,25 @@ read_arg(void *ctx, int index, int type, long orig_off, unsigned long arg, int a
 	case sockaddr_type:
 		size = copy_sockaddr(args, arg);
 		break;
+	default:
+		size = 0;
+		break;
+	}
+	return size;
+}
+
+FUNC_INLINE long
+__read_arg_2(void *ctx, int type, long orig_off, unsigned long arg, int argm, char *args)
+{
+	struct msg_generic_kprobe *e;
+	long size = -1;
+	int zero = 0;
+
+	e = map_lookup_elem(&process_call_heap, &zero);
+	if (!e)
+		return 0;
+
+	switch (type) {
 	case socket_type:
 		size = copy_socket(args, arg);
 		// Look up socket in our sock->pid_tgid map
@@ -392,6 +409,62 @@ read_arg(void *ctx, int index, int type, long orig_off, unsigned long arg, int a
 	return size;
 }
 
+/**
+ * Read a generic argument
+ *
+ * @args: destination buffer for the generic argument
+ * @type: type of the argument
+ * @off: offset of the argument within @args
+ * @arg: argument location (generally, address of the argument)
+ * @argm: argument metadata. The meaning of this depends on the @type. Some
+ *        types use a -1 to designate saving @arg into the retprobe map
+ * @filter_map:
+ *
+ * Returns the size of data appended to @args.
+ */
+FUNC_INLINE long
+read_arg(void *ctx, int index, int type, long orig_off, unsigned long arg, int argm, int process)
+{
+	size_t min_size = type_to_min_size(type, argm);
+	struct msg_generic_kprobe *e;
+	char *args;
+	const struct path *path_arg = 0;
+	struct path path_buf;
+	int zero = 0;
+
+	e = map_lookup_elem(&process_call_heap, &zero);
+	if (!e)
+		return 0;
+
+	if (orig_off >= 16383 - min_size)
+		return 0;
+
+	orig_off &= 16383;
+	args = args_off(e, orig_off);
+
+	/* Cache args offset for filter use later */
+	e->argsoff[index & MAX_SELECTORS_MASK] = orig_off;
+
+	path_arg = get_path(type, arg, &path_buf);
+	if (path_arg)
+		return copy_path(args, path_arg);
+
+	/*
+	 * Separate argument processing based on the process const
+	 * for 4.19 kernels..
+	 */
+	if (process == __READ_ARG_1)
+		return __read_arg_1(ctx, type, orig_off, arg, argm, args);
+	if (process == __READ_ARG_2)
+		return __read_arg_2(ctx, type, orig_off, arg, argm, args);
+
+	/* .. and the rest of the world */
+	if (is_read_arg_1(type))
+		return __read_arg_1(ctx, type, orig_off, arg, argm, args);
+	else
+		return __read_arg_2(ctx, type, orig_off, arg, argm, args);
+}
+
 FUNC_INLINE int
 extract_arg_depth(u32 i, struct extract_arg_data *data)
 {
@@ -495,7 +568,8 @@ FUNC_INLINE long get_pt_regs_arg(struct pt_regs *ctx, struct event_config *confi
 }
 #endif /* __TARGET_ARCH_x86 && (GENERIC_KPROBE || GENERIC_UPROBE) */
 
-FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map_def *tailcals)
+FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map_def *tailcals,
+				  int process)
 {
 	struct msg_generic_kprobe *e;
 	struct event_config *config;
@@ -517,6 +591,13 @@ FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map
 	ty = config->arg[index];
 	am = config->arm[index];
 
+#ifndef __LARGE_BPF_PROG
+#if defined(GENERIC_KPROBE) || defined(GENERIC_UPROBE) || defined(GENERIC_TRACEPOINT)
+	if (!is_read_arg_1(ty) && process == __READ_ARG_1)
+		tail_call(ctx, tailcals, TAIL_CALL_PROCESS_2);
+#endif
+#endif
+
 #if defined(GENERIC_TRACEPOINT) || defined(GENERIC_USDT)
 	a = (&e->a0)[index];
 	extract_arg(config, index, &a);
@@ -546,11 +627,11 @@ FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map
 		return generic_path_offload(ctx, ty, a, index, off, tailcals);
 #endif
 
-	return read_arg(ctx, index, ty, off, a, am);
+	return read_arg(ctx, index, ty, off, a, am, process);
 }
 
 FUNC_INLINE int
-generic_process_event(void *ctx, struct bpf_map_def *tailcals)
+generic_process_event(void *ctx, struct bpf_map_def *tailcals, int process)
 {
 	struct msg_generic_kprobe *e;
 	int index, zero = 0;
@@ -567,7 +648,7 @@ generic_process_event(void *ctx, struct bpf_map_def *tailcals)
 	if (total < MAX_TOTAL) {
 		long errv;
 
-		errv = generic_read_arg(ctx, index, total, tailcals);
+		errv = generic_read_arg(ctx, index, total, tailcals, process);
 		if (errv > 0)
 			total += errv;
 		/* Follow filter lookup failed so lets abort the event.
@@ -1205,7 +1286,7 @@ FUNC_INLINE int generic_retprobe(void *ctx, struct bpf_map_def *calls, unsigned
 	ty_arg = config->argreturn;
 	do_copy = config->argreturncopy;
 	if (ty_arg) {
-		size += read_arg(ctx, 0, ty_arg, size, ret, 0);
+		size += read_arg(ctx, 0, ty_arg, size, ret, 0, __READ_ARG_ALL);
 #if defined(__LARGE_BPF_PROG) && defined(GENERIC_KRETPROBE)
 		struct socket_owner owner;