libbpf: add support for BTF location attachment

Add support for BTF-based location attachment via multiple kprobes
attaching to each instance of an inline site. Note this is not kprobe
multi attach since that requires fprobe on entry and sites are within
functions. Implementation similar to USDT manager where we use BTF
to create a location manager and populate expected arg values with
metadata based upon BTF_KIND_LOC_PARAM/LOC_PROTOs.

Add new auto-attach SEC("kloc/module:name") where the module is
vmlinux/kernel module and the name is the name of the associated
location; all sites associated with that name will be attached via
kprobes for tracing.

Signed-off-by: Alan Maguire <[email protected]>

Author: alan-maguire

tools/lib/bpf/Build

@@ -1,4 +1,4 @@
 libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_utils.o \
 	    netlink.o bpf_prog_linfo.o libbpf_probes.o hashmap.o \
 	    btf_dump.o ringbuf.o strset.o linker.o gen_loader.o relo_core.o \
-	    usdt.o zip.o elf.o features.o btf_iter.o btf_relocate.o
+	    usdt.o zip.o elf.o features.o btf_iter.o btf_relocate.o loc.o

tools/lib/bpf/Makefile

@@ -234,7 +234,7 @@ install_lib: all_cmd
 
 SRC_HDRS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h	     \
 	    bpf_helpers.h bpf_tracing.h bpf_endian.h bpf_core_read.h	     \
-	    skel_internal.h libbpf_version.h usdt.bpf.h
+	    skel_internal.h libbpf_version.h usdt.bpf.h loc.bpf.h
 GEN_HDRS := $(BPF_GENERATED)
 
 INSTALL_PFX := $(DESTDIR)$(prefix)/include/bpf

tools/lib/bpf/libbpf.c

@@ -407,6 +407,7 @@ enum sec_def_flags {
 	SEC_XDP_FRAGS = 16,
 	/* Setup proper attach type for usdt probes. */
 	SEC_USDT = 32,
+	SEC_LOC = 33,
 };
 
 struct bpf_sec_def {
@@ -671,6 +672,8 @@ struct elf_state {
 
 struct usdt_manager;
 
+struct loc_manager;
+
 enum bpf_object_state {
 	OBJ_OPEN,
 	OBJ_PREPARED,
@@ -733,6 +736,7 @@ struct bpf_object {
 	size_t fd_array_cnt;
 
 	struct usdt_manager *usdt_man;
+	struct loc_manager *loc_man;
 
 	int arena_map_idx;
 	void *arena_data;
@@ -9190,6 +9194,8 @@ void bpf_object__close(struct bpf_object *obj)
 
 	usdt_manager_free(obj->usdt_man);
 	obj->usdt_man = NULL;
+	loc_manager_free(obj->loc_man);
+	obj->loc_man = NULL;
 
 	bpf_gen__free(obj->gen_loader);
 	bpf_object__elf_finish(obj);
@@ -9561,6 +9567,7 @@ static int attach_kprobe_session(const struct bpf_program *prog, long cookie, st
 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
+static int attach_kloc(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 
 static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("socket",		SOCKET_FILTER, 0, SEC_NONE),
@@ -9666,6 +9673,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("struct_ops.s+",	STRUCT_OPS, 0, SEC_SLEEPABLE),
 	SEC_DEF("sk_lookup",		SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
 	SEC_DEF("netfilter",		NETFILTER, BPF_NETFILTER, SEC_NONE),
+	SEC_DEF("kloc+",		KPROBE, 0, SEC_NONE, attach_kloc),
 };
 
 int libbpf_register_prog_handler(const char *sec,
@@ -11155,7 +11163,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
 	attr.size = attr_sz;
 	attr.type = type;
 	attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
-	attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
+	attr.config1 = name ? ptr_to_u64(name) : 0; /* kprobe_func or uprobe_path */
 	attr.config2 = offset;		 /* kprobe_addr or probe_offset */
 
 	/* pid filter is meaningful only for uprobes */
@@ -12601,6 +12609,72 @@ static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_l
 	return err;
 }
 
+struct bpf_link *bpf_program__attach_kloc(const struct bpf_program *prog,
+					 const char *module, const char *name,
+					 const struct bpf_kloc_opts *opts)
+{
+	struct bpf_object *obj = prog->obj;
+	struct bpf_link *link;
+	__u64 loc_cookie;
+	int err;
+
+	if (!OPTS_VALID(opts, bpf_kloc_opts))
+		return libbpf_err_ptr(-EINVAL);
+
+	if (bpf_program__fd(prog) < 0) {
+		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
+			prog->name);
+		return libbpf_err_ptr(-EINVAL);
+	}
+	if (!module || !name)
+		return libbpf_err_ptr(-EINVAL);
+
+	/* loc manager is instantiated lazily on first loc attach. It will
+	 * be destroyed together with BPF object in bpf_object__close().
+	 */
+	if (IS_ERR(obj->loc_man))
+		return libbpf_ptr(obj->loc_man);
+	if (!obj->loc_man) {
+		obj->loc_man = loc_manager_new(obj);
+		if (IS_ERR(obj->loc_man))
+			return libbpf_ptr(obj->loc_man);
+	}
+
+	loc_cookie = OPTS_GET(opts, loc_cookie, 0);
+	link = loc_manager_attach_kloc(obj->loc_man, prog, module, name, loc_cookie);
+	err = libbpf_get_error(link);
+	if (err)
+		return libbpf_err_ptr(err);
+	return link;
+}
+
+static int attach_kloc(const struct bpf_program *prog, long cookie, struct bpf_link **link)
+{
+	char *module = NULL, *name = NULL;
+	const char *sec_name;
+	int n, err;
+
+	sec_name = bpf_program__section_name(prog);
+	if (strcmp(sec_name, "kloc") == 0) {
+		/* no auto-attach for just SEC("kloc") */
+		*link = NULL;
+		return 0;
+	}
+
+	n = sscanf(sec_name, "kloc/%m[^:]:%m[^:]", &module, &name);
+	if (n != 2) {
+		pr_warn("invalid section '%s', expected SEC(\"kloc/<module>:<name>\")\n",
+			sec_name);
+		err = -EINVAL;
+	} else {
+		*link = bpf_program__attach_kloc(prog, module, name, NULL);
+		err = libbpf_get_error(*link);
+	}
+	free(module);
+	free(name);
+	return err;
+}
+
 static int determine_tracepoint_id(const char *tp_category,
 				   const char *tp_name)
 {

tools/lib/bpf/libbpf.h

@@ -799,6 +799,33 @@ bpf_program__attach_usdt(const struct bpf_program *prog,
 			 const char *usdt_provider, const char *usdt_name,
 			 const struct bpf_usdt_opts *opts);
 
+struct bpf_kloc_opts {
+	/* size of this struct, for forward/backward compatibility */
+	size_t sz;
+	/* custom user-provided value fetchable through loc_cookie() */
+	__u64 loc_cookie;
+	size_t:0;
+};
+#define bpf_kloc_opts__last_field loc_cookie
+
+/**
+ * @brief **bpf_program__attach_kloc()** attaches to the location
+ * named *name* in *module* (which can be "vmlinux" or a module name).
+ * Attaches to all locations associated with *name*.
+ *
+ * @param prog BPF program to attach
+ * @param module name
+ * @param name Location name
+ * @param opts Options for altering program attachment
+ * @return Reference to the newly created BPF link: or NULL is returned on error
+ *
+ * error code is stored in errno
+ */
+LIBBPF_API struct bpf_link *
+bpf_program__attach_kloc(const struct bpf_program *prog,
+			 const char *module, const char *name,
+			 const struct bpf_kloc_opts *opts);
+
 struct bpf_tracepoint_opts {
 	/* size of this struct, for forward/backward compatibility */
 	size_t sz;

tools/lib/bpf/libbpf.map

@@ -457,4 +457,5 @@ LIBBPF_1.7.0 {
 		btf__add_locsec;
 		btf__add_locsec_loc;
 		btf__load_btf_extra;
+		bpf_program__attach_kloc;
 } LIBBPF_1.6.0;

tools/lib/bpf/libbpf_internal.h

@@ -719,6 +719,13 @@ struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man,
 					   const char *usdt_provider, const char *usdt_name,
 					   __u64 usdt_cookie);
 
+struct loc_manager *loc_manager_new(struct bpf_object *obj);
+void loc_manager_free(struct loc_manager *man);
+struct bpf_link *loc_manager_attach_kloc(struct loc_manager *man,
+					 const struct bpf_program *prog,
+					 const char *loc_mod, const char *loc_name,
+					 __u64 loc_cookie);
+
 static inline bool is_pow_of_2(size_t x)
 {
 	return x && (x & (x - 1)) == 0;