tetragon: Wire extra process event tail call

Split generic_[ku]probe_process_event into two separate tail calls
so we have space (there's 4096 instructions limit on 4.19 kernels)
for new features in processing event code.

Before:
  generic_kprobe_process_event 4029 insns
  generic_uprobe_process_event 4029 insns

Now:
  generic_uprobe_process_event   2795 insns
  generic_uprobe_process_event_2 3147 insns

  generic_kprobe_process_event   2795 insns
  generic_kprobe_process_event_2 3147 insns

Signed-off-by: Jiri Olsa <[email protected]>

Author: olsajiri

bpf/include/api.h

@@ -323,4 +323,10 @@ static int BPF_FUNC(seq_write, struct seq_file *m, const void *data, uint32_t le
 # define lock_and(ptr, val)	(*(ptr) &= val)
 #endif
 
+enum {
+	__READ_ARG_1,
+	__READ_ARG_2,
+	__READ_ARG_ALL,
+};
+
 #endif /* __BPF_API__ */

bpf/process/bpf_generic_kprobe.c

@@ -18,6 +18,7 @@ char _license[] __attribute__((section("license"), used)) = "Dual BSD/GPL";
 
 int generic_kprobe_setup_event(void *ctx);
 int generic_kprobe_process_event(void *ctx);
+int generic_kprobe_process_event_2(void *ctx);
 int generic_kprobe_process_filter(void *ctx);
 int generic_kprobe_filter_arg(void *ctx);
 int generic_kprobe_actions(void *ctx);
@@ -39,6 +40,9 @@ struct {
 		[TAIL_CALL_SEND] = (void *)&generic_kprobe_output,
 #ifndef __V61_BPF_PROG
 		[TAIL_CALL_PATH] = (void *)&generic_kprobe_path,
+#endif
+#ifndef __LARGE_BPF_PROG
+		[TAIL_CALL_PROCESS_2] = (void *)&generic_kprobe_process_event_2,
 #endif
 	},
 };
@@ -91,12 +95,26 @@ generic_kprobe_setup_event(void *ctx)
 	return generic_process_event_and_setup(ctx, (struct bpf_map_def *)&kprobe_calls);
 }
 
+#ifdef __LARGE_BPF_PROG
+__attribute__((section(COMMON), used)) int
+generic_kprobe_process_event(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls, __READ_ARG_ALL);
+}
+#else
 __attribute__((section(COMMON), used)) int
 generic_kprobe_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls, __READ_ARG_1);
 }
 
+__attribute__((section(COMMON), used)) int
+generic_kprobe_process_event_2(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&kprobe_calls, __READ_ARG_2);
+}
+#endif
+
 __attribute__((section(COMMON), used)) int
 generic_kprobe_process_filter(void *ctx)
 {

bpf/process/bpf_generic_lsm_core.c

@@ -63,7 +63,7 @@ generic_lsm_setup_event(void *ctx)
 __attribute__((section("lsm"), used)) int
 generic_lsm_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&lsm_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&lsm_calls, __READ_ARG_ALL);
 }
 
 __attribute__((section("lsm"), used)) int

bpf/process/bpf_generic_rawtp.c

@@ -84,7 +84,7 @@ generic_rawtp_setup_event(void *ctx)
 __attribute__((section("raw_tp"), used)) int
 generic_rawtp_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls, __READ_ARG_ALL);
 }
 
 __attribute__((section("raw_tp"), used)) int

bpf/process/bpf_generic_tracepoint.c

@@ -263,7 +263,7 @@ generic_tracepoint_event(struct generic_tracepoint_event_arg *ctx)
 __attribute__((section("tracepoint"), used)) int
 generic_tracepoint_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&tp_calls, __READ_ARG_ALL);
 }
 
 __attribute__((section("tracepoint"), used)) int

bpf/process/bpf_generic_uprobe.c

@@ -18,6 +18,7 @@ char _license[] __attribute__((section("license"), used)) = "Dual BSD/GPL";
 
 int generic_uprobe_setup_event(void *ctx);
 int generic_uprobe_process_event(void *ctx);
+int generic_uprobe_process_event_2(void *ctx);
 int generic_uprobe_process_filter(void *ctx);
 int generic_uprobe_filter_arg(void *ctx);
 int generic_uprobe_actions(void *ctx);
@@ -39,6 +40,9 @@ struct {
 		[TAIL_CALL_SEND] = (void *)&generic_uprobe_output,
 #ifndef __V61_BPF_PROG
 		[TAIL_CALL_PATH] = (void *)&generic_uprobe_path,
+#endif
+#ifndef __LARGE_BPF_PROG
+		[TAIL_CALL_PROCESS_2] = (void *)&generic_uprobe_process_event_2,
 #endif
 	},
 };
@@ -68,12 +72,26 @@ generic_uprobe_setup_event(void *ctx)
 	return generic_process_event_and_setup(ctx, (struct bpf_map_def *)&uprobe_calls);
 }
 
+#ifdef __LARGE_BPF_PROG
+__attribute__((section(COMMON), used)) int
+generic_uprobe_process_event(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls, __READ_ARG_ALL);
+}
+#else
 __attribute__((section(COMMON), used)) int
 generic_uprobe_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls, __READ_ARG_1);
 }
 
+__attribute__((section(COMMON), used)) int
+generic_uprobe_process_event_2(void *ctx)
+{
+	return generic_process_event(ctx, (struct bpf_map_def *)&uprobe_calls, __READ_ARG_2);
+}
+#endif
+
 __attribute__((section(COMMON), used)) int
 generic_uprobe_process_filter(void *ctx)
 {

bpf/process/bpf_generic_usdt.c

@@ -72,7 +72,7 @@ generic_usdt_setup_event(void *ctx)
 __attribute__((section(COMMON), used)) int
 generic_usdt_process_event(void *ctx)
 {
-	return generic_process_event(ctx, (struct bpf_map_def *)&usdt_calls);
+	return generic_process_event(ctx, (struct bpf_map_def *)&usdt_calls, __READ_ARG_ALL);
 }
 
 __attribute__((section(COMMON), used)) int

bpf/process/generic_calls.h

@@ -423,7 +423,7 @@ __read_arg_2(void *ctx, int type, long orig_off, unsigned long arg, int argm, ch
  * Returns the size of data appended to @args.
  */
 FUNC_INLINE long
-read_arg(void *ctx, int index, int type, long orig_off, unsigned long arg, int argm)
+read_arg(void *ctx, int index, int type, long orig_off, unsigned long arg, int argm, int process)
 {
 	size_t min_size = type_to_min_size(type, argm);
 	struct msg_generic_kprobe *e;
@@ -449,6 +449,16 @@ read_arg(void *ctx, int index, int type, long orig_off, unsigned long arg, int a
 	if (path_arg)
 		return copy_path(args, path_arg);
 
+	/*
+	 * Separate argument processing based on the process const
+	 * for 4.19 kernels..
+	 */
+	if (process == __READ_ARG_1)
+		return __read_arg_1(ctx, type, orig_off, arg, argm, args);
+	if (process == __READ_ARG_2)
+		return __read_arg_2(ctx, type, orig_off, arg, argm, args);
+
+	/* .. and the rest of the world */
 	if (is_read_arg_1(type))
 		return __read_arg_1(ctx, type, orig_off, arg, argm, args);
 	else
@@ -558,7 +568,8 @@ FUNC_INLINE long get_pt_regs_arg(struct pt_regs *ctx, struct event_config *confi
 }
 #endif /* __TARGET_ARCH_x86 && (GENERIC_KPROBE || GENERIC_UPROBE) */
 
-FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map_def *tailcals)
+FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map_def *tailcals,
+				  int process)
 {
 	struct msg_generic_kprobe *e;
 	struct event_config *config;
@@ -580,6 +591,13 @@ FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map
 	ty = config->arg[index];
 	am = config->arm[index];
 
+#ifndef __LARGE_BPF_PROG
+#if defined(GENERIC_KPROBE) || defined(GENERIC_UPROBE)
+	if (!is_read_arg_1(ty) && process == __READ_ARG_1)
+		tail_call(ctx, tailcals, TAIL_CALL_PROCESS_2);
+#endif
+#endif
+
 #if defined(GENERIC_TRACEPOINT) || defined(GENERIC_USDT)
 	a = (&e->a0)[index];
 	extract_arg(config, index, &a);
@@ -609,11 +627,11 @@ FUNC_INLINE long generic_read_arg(void *ctx, int index, long off, struct bpf_map
 		return generic_path_offload(ctx, ty, a, index, off, tailcals);
 #endif
 
-	return read_arg(ctx, index, ty, off, a, am);
+	return read_arg(ctx, index, ty, off, a, am, process);
 }
 
 FUNC_INLINE int
-generic_process_event(void *ctx, struct bpf_map_def *tailcals)
+generic_process_event(void *ctx, struct bpf_map_def *tailcals, int process)
 {
 	struct msg_generic_kprobe *e;
 	int index, zero = 0;
@@ -630,7 +648,7 @@ generic_process_event(void *ctx, struct bpf_map_def *tailcals)
 	if (total < MAX_TOTAL) {
 		long errv;
 
-		errv = generic_read_arg(ctx, index, total, tailcals);
+		errv = generic_read_arg(ctx, index, total, tailcals, process);
 		if (errv > 0)
 			total += errv;
 		/* Follow filter lookup failed so lets abort the event.
@@ -1271,7 +1289,7 @@ FUNC_INLINE int generic_retprobe(void *ctx, struct bpf_map_def *calls, unsigned
 	ty_arg = config->argreturn;
 	do_copy = config->argreturncopy;
 	if (ty_arg) {
-		size += read_arg(ctx, 0, ty_arg, size, ret, 0);
+		size += read_arg(ctx, 0, ty_arg, size, ret, 0, __READ_ARG_ALL);
 #if defined(__LARGE_BPF_PROG) && defined(GENERIC_KRETPROBE)
 		struct socket_owner owner;
 

bpf/process/types/basic.h

@@ -138,6 +138,7 @@ enum {
 	TAIL_CALL_ACTIONS = 4,
 	TAIL_CALL_SEND = 5,
 	TAIL_CALL_PATH = 6,
+	TAIL_CALL_PROCESS_2 = 7,
 };
 
 struct selector_action {

pkg/sensors/tracing/kprobe_test.go

@@ -4424,12 +4424,6 @@ func TestLoadKprobeSensor(t *testing.T) {
 		}
 
 		sensorMaps = []tus.SensorMap{
-			// all kprobe programs
-			{Name: "process_call_heap", Progs: []uint{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}},
-
-			// all but generic_kprobe_output
-			{Name: "kprobe_calls", Progs: []uint{0, 1, 2, 3, 4, 5, 7}},
-
 			// generic_retkprobe_event
 			{Name: "retkprobe_calls", Progs: []uint{8, 9, 10}},
 
@@ -4440,14 +4434,20 @@ func TestLoadKprobeSensor(t *testing.T) {
 			// generic_kprobe_actions
 			{Name: "override_tasks", Progs: []uint{5}},
 
-			// all kprobe but generic_kprobe_process_filter,generic_retkprobe_event
-			{Name: "config_map", Progs: []uint{0, 1, 2}},
-
 			// generic_kprobe_process_event*,generic_kprobe_actions,retkprobe
 			{Name: "fdinstall_map", Progs: []uint{2, 5, 8, 10}},
 		}
 
 		if config.EnableLargeProgs() {
+			// all kprobe programs
+			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "process_call_heap", Progs: []uint{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}})
+
+			// all but generic_kprobe_output
+			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "kprobe_calls", Progs: []uint{0, 1, 2, 3, 4, 5, 7}})
+
+			// all kprobe but generic_kprobe_process_filter,generic_retkprobe_event
+			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "config_map", Progs: []uint{0, 1, 2}})
+
 			// shared with base sensor
 			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "execve_map", Progs: []uint{4, 5, 6, 8, 10}})
 
@@ -4468,6 +4468,17 @@ func TestLoadKprobeSensor(t *testing.T) {
 				sensorMaps = append(sensorMaps, tus.SensorMap{Name: "tg_conf_map", Progs: []uint{0}})
 			}
 		} else {
+			sensorProgs = append(sensorProgs, tus.SensorProg{Name: "generic_kprobe_process_event_2", Type: ebpf.Kprobe})
+
+			// all kprobe programs
+			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "process_call_heap", Progs: []uint{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}})
+
+			// all but generic_kprobe_output
+			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "kprobe_calls", Progs: []uint{0, 1, 2, 3, 4, 5, 7, 12}})
+
+			// all kprobe but generic_kprobe_process_filter,generic_retkprobe_event
+			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "config_map", Progs: []uint{0, 1, 2, 12}})
+
 			// shared with base sensor
 			sensorMaps = append(sensorMaps, tus.SensorMap{Name: "execve_map", Progs: []uint{4, 8}})