kernel/bpf: Add BPF_MAP_TYPE_CRED_STORAGE map type and kfuncs

All other bpf local storage is obtained using helpers which benefit from
RET_PTR_TO_MAP_VALUE_OR_NULL, so can return void * pointers directly to
map values. kfuncs don't have that, so return struct
bpf_local_storage_data * and access map values through sdata->data.

Signed-off-by: David Windsor <[email protected]>

Author: dwindsor

include/linux/bpf_lsm.h

@@ -40,10 +40,27 @@ static inline struct bpf_storage_blob *bpf_inode(
 	return inode->i_security + bpf_lsm_blob_sizes.lbs_inode;
 }
 
+static inline struct bpf_storage_blob *bpf_cred(
+	const struct cred *cred)
+{
+	if (unlikely(!cred->security))
+		return NULL;
+
+	return cred->security + bpf_lsm_blob_sizes.lbs_cred;
+}
+
 extern const struct bpf_func_proto bpf_inode_storage_get_proto;
 extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
 void bpf_inode_storage_free(struct inode *inode);
 
+void bpf_cred_storage_free(struct cred *cred);
+struct bpf_local_storage_data *bpf_cred_storage_get(struct bpf_map *map,
+						    struct cred *cred,
+						    void *init,
+						    int init__sz,
+						    u64 flags);
+int bpf_cred_storage_delete(struct bpf_map *map, struct cred *cred);
+
 void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func);
 
 int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
@@ -81,6 +98,24 @@ static inline void bpf_inode_storage_free(struct inode *inode)
 {
 }
 
+static inline void bpf_cred_storage_free(struct cred *cred)
+{
+}
+
+static inline struct bpf_local_storage_data *bpf_cred_storage_get(struct bpf_map *map,
+								  struct cred *cred,
+								  void *init,
+								  int init__sz,
+								  u64 flags)
+{
+	return NULL;
+}
+
+static inline int bpf_cred_storage_delete(struct bpf_map *map, struct cred *cred)
+{
+	return -EOPNOTSUPP;
+}
+
 static inline void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
 					   bpf_func_t *bpf_func)
 {

include/linux/bpf_types.h

@@ -110,6 +110,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_HASH_OF_MAPS, htab_of_maps_map_ops)
 BPF_MAP_TYPE(BPF_MAP_TYPE_INODE_STORAGE, inode_storage_map_ops)
 #endif
 BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)
+BPF_MAP_TYPE(BPF_MAP_TYPE_CRED_STORAGE, cred_storage_map_ops)
 #ifdef CONFIG_NET
 BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops)
 BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP_HASH, dev_map_hash_ops)

include/uapi/linux/bpf.h

@@ -1026,6 +1026,7 @@ enum bpf_map_type {
 	BPF_MAP_TYPE_USER_RINGBUF,
 	BPF_MAP_TYPE_CGRP_STORAGE,
 	BPF_MAP_TYPE_ARENA,
+	BPF_MAP_TYPE_CRED_STORAGE,
 	__MAX_BPF_MAP_TYPE
 };
 

kernel/bpf/Makefile

@@ -12,6 +12,7 @@ obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list
 obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
 obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
 obj-${CONFIG_BPF_LSM}	  += bpf_inode_storage.o
+obj-${CONFIG_BPF_LSM}	  += bpf_cred_storage.o
 obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
 obj-$(CONFIG_BPF_JIT) += trampoline.o
 obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o rqspinlock.o stream.o

kernel/bpf/bpf_cred_storage.c

@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/rculist.h>
+#include <linux/list.h>
+#include <linux/hash.h>
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include <linux/bpf.h>
+#include <linux/bpf_local_storage.h>
+#include <linux/bpf_lsm.h>
+#include <linux/cred.h>
+#include <linux/btf_ids.h>
+#include <linux/rcupdate_trace.h>
+
+DEFINE_BPF_STORAGE_CACHE(cred_cache);
+
+static struct bpf_local_storage __rcu **cred_storage_ptr(void *owner)
+{
+	struct cred *cred = owner;
+	struct bpf_storage_blob *bsb;
+
+	bsb = bpf_cred(cred);
+	if (!bsb)
+		return NULL;
+	return &bsb->storage;
+}
+
+static struct bpf_local_storage_data *cred_storage_lookup(struct cred *cred,
+							  struct bpf_map *map,
+							  bool cacheit_lockit)
+{
+	struct bpf_local_storage *cred_storage;
+	struct bpf_local_storage_map *smap;
+	struct bpf_storage_blob *bsb;
+
+	bsb = bpf_cred(cred);
+	if (!bsb)
+		return NULL;
+
+	cred_storage = rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
+	if (!cred_storage)
+		return NULL;
+
+	smap = (struct bpf_local_storage_map *)map;
+	return bpf_local_storage_lookup(cred_storage, smap, cacheit_lockit);
+}
+
+void bpf_cred_storage_free(struct cred *cred)
+{
+	struct bpf_local_storage *local_storage;
+	struct bpf_storage_blob *bsb;
+
+	bsb = bpf_cred(cred);
+	if (!bsb)
+		return;
+
+	migrate_disable();
+	rcu_read_lock();
+
+	local_storage = rcu_dereference(bsb->storage);
+	if (!local_storage)
+		goto out;
+
+	bpf_local_storage_destroy(local_storage);
+out:
+	rcu_read_unlock();
+	migrate_enable();
+}
+
+static int cred_storage_delete(struct cred *cred, struct bpf_map *map)
+{
+	struct bpf_local_storage_data *sdata;
+
+	sdata = cred_storage_lookup(cred, map, false);
+	if (!sdata)
+		return -ENOENT;
+
+	bpf_selem_unlink(SELEM(sdata), false);
+
+	return 0;
+}
+
+static struct bpf_map *cred_storage_map_alloc(union bpf_attr *attr)
+{
+	return bpf_local_storage_map_alloc(attr, &cred_cache, false);
+}
+
+static void cred_storage_map_free(struct bpf_map *map)
+{
+	bpf_local_storage_map_free(map, &cred_cache, NULL);
+}
+
+static int notsupp_get_next_key(struct bpf_map *map, void *key,
+				void *next_key)
+{
+	return -ENOTSUPP;
+}
+
+const struct bpf_map_ops cred_storage_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
+	.map_alloc_check = bpf_local_storage_map_alloc_check,
+	.map_alloc = cred_storage_map_alloc,
+	.map_free = cred_storage_map_free,
+	.map_get_next_key = notsupp_get_next_key,
+	.map_check_btf = bpf_local_storage_map_check_btf,
+	.map_mem_usage = bpf_local_storage_map_mem_usage,
+	.map_btf_id = &bpf_local_storage_map_btf_id[0],
+	.map_owner_storage_ptr = cred_storage_ptr,
+};
+
+BTF_ID_LIST_SINGLE(bpf_cred_storage_btf_ids, struct, cred)
+
+__bpf_kfunc struct bpf_local_storage_data *bpf_cred_storage_get(struct bpf_map *map,
+								struct cred *cred,
+								void *init,
+								int init__sz,
+								u64 flags)
+{
+	struct bpf_local_storage_data *sdata;
+
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
+	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
+		return NULL;
+
+	if (!cred || !cred_storage_ptr(cred))
+		return NULL;
+
+	sdata = cred_storage_lookup(cred, map, true);
+	if (sdata)
+		return sdata;
+
+	/* This helper must only called from where the cred is guaranteed
+	 * to have a refcount and cannot be freed.
+	 */
+	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
+		sdata = bpf_local_storage_update(
+			cred, (struct bpf_local_storage_map *)map, init,
+			BPF_NOEXIST, false, GFP_ATOMIC);
+		return IS_ERR(sdata) ? NULL : sdata;
+	}
+
+	return NULL;
+}
+
+__bpf_kfunc int bpf_cred_storage_delete(struct bpf_map *map, struct cred *cred)
+{
+	if (!cred)
+		return -EINVAL;
+
+	return cred_storage_delete(cred, map);
+}
+
+BTF_KFUNCS_START(bpf_cred_storage_kfunc_ids)
+BTF_ID_FLAGS(func, bpf_cred_storage_delete, 0)
+BTF_ID_FLAGS(func, bpf_cred_storage_get, KF_RET_NULL)
+BTF_KFUNCS_END(bpf_cred_storage_kfunc_ids)
+
+static const struct btf_kfunc_id_set bpf_cred_storage_kfunc_set = {
+	.owner = THIS_MODULE,
+	.set   = &bpf_cred_storage_kfunc_ids,
+};
+
+static int __init bpf_cred_storage_init(void)
+{
+	int err;
+	err = register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, &bpf_cred_storage_kfunc_set);
+	if (err) {
+		pr_err("bpf_cred_storage: failed to register kfuncs: %d\n", err);
+		return err;
+	}
+
+	pr_info("bpf_cred_storage: kfuncs registered successfully\n");
+	return 0;
+}
+late_initcall(bpf_cred_storage_init);

kernel/bpf/syscall.c

@@ -1262,7 +1262,8 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
 				    map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
 				    map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
 				    map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
-				    map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
+				    map->map_type != BPF_MAP_TYPE_CGRP_STORAGE &&
+				    map->map_type != BPF_MAP_TYPE_CRED_STORAGE) {
 					ret = -EOPNOTSUPP;
 					goto free_map_tab;
 				}
@@ -1289,13 +1290,15 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
 				    map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
 				    map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
 				    map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
-				    map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
+				    map->map_type != BPF_MAP_TYPE_CGRP_STORAGE &&
+				    map->map_type != BPF_MAP_TYPE_CRED_STORAGE) {
 					ret = -EOPNOTSUPP;
 					goto free_map_tab;
 				}
 				break;
 			case BPF_UPTR:
-				if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) {
+				if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
+			    map->map_type != BPF_MAP_TYPE_CRED_STORAGE) {
 					ret = -EOPNOTSUPP;
 					goto free_map_tab;
 				}
@@ -1449,6 +1452,7 @@ static int map_create(union bpf_attr *attr, bool kernel)
 	case BPF_MAP_TYPE_SK_STORAGE:
 	case BPF_MAP_TYPE_INODE_STORAGE:
 	case BPF_MAP_TYPE_TASK_STORAGE:
+	case BPF_MAP_TYPE_CRED_STORAGE:
 	case BPF_MAP_TYPE_CGRP_STORAGE:
 	case BPF_MAP_TYPE_BLOOM_FILTER:
 	case BPF_MAP_TYPE_LPM_TRIE:

kernel/cred.c

@@ -38,6 +38,10 @@ static struct kmem_cache *cred_jar;
 /* init to 2 - one for init_task, one to ensure it is never freed */
 static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) };
 
+#ifdef CONFIG_BPF_LSM
+#include <linux/bpf_lsm.h>
+#endif
+
 /*
  * The initial credentials for the initial task
  */
@@ -76,6 +80,9 @@ static void put_cred_rcu(struct rcu_head *rcu)
 		      cred, atomic_long_read(&cred->usage));
 
 	security_cred_free(cred);
+#ifdef CONFIG_BPF_LSM
+	bpf_cred_storage_free(cred);
+#endif
 	key_put(cred->session_keyring);
 	key_put(cred->process_keyring);
 	key_put(cred->thread_keyring);

security/bpf/hooks.c

@@ -30,6 +30,7 @@ static int __init bpf_lsm_init(void)
 
 struct lsm_blob_sizes bpf_lsm_blob_sizes __ro_after_init = {
 	.lbs_inode = sizeof(struct bpf_storage_blob),
+	.lbs_cred = sizeof(struct bpf_storage_blob),
 };
 
 DEFINE_LSM(bpf) = {