Message-Id: <20250528034712.138701-9-dongml2@chinatelecom.cn>
Date: Wed, 28 May 2025 11:46:55 +0800
From: Menglong Dong <menglong8.dong@...il.com>
To: alexei.starovoitov@...il.com,
	rostedt@...dmis.org,
	jolsa@...nel.org
Cc: bpf@...r.kernel.org,
	Menglong Dong <dongml2@...natelecom.cn>,
	linux-kernel@...r.kernel.org
Subject: [PATCH bpf-next 08/25] bpf: introduce bpf_gtramp_link

Introduce the struct bpf_gtramp_link, which is used to attach
a bpf prog to multi functions. Meanwhile, introduce corresponding
function bpf_gtrampoline_{link,unlink}_prog.

The lock global_tr_lock is held during global trampoline link and unlink.
Why we define the global_tr_lock as rw_semaphore? Well, it should be mutex
here, but we will use the rw_semaphore in the later patch for the
trampoline override case :/

When unlink the global trampoline link, we mark all the function in the
bpf_gtramp_link with KFUNC_MD_FL_BPF_REMOVING and update the global
trampoline with bpf_gtrampoline_update(). If this is the last bpf prog
in the kfunc_md, the function will be remove from the filter_hash of the
ftrace_ops of bpf_global_trampoline. Then, we remove the bpf prog from
the kfunc_md, and free the kfunc_md if necessary.

Signed-off-by: Menglong Dong <dongml2@...natelecom.cn>
---
 include/linux/bpf.h     |  31 +++++++
 kernel/bpf/trampoline.c | 183 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 214 insertions(+)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 8979e397ea06..7527399bab5b 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -58,12 +58,15 @@ struct bpf_token;
 struct user_namespace;
 struct super_block;
 struct inode;
+struct bpf_tramp_link;
+struct bpf_gtramp_link;
 
 extern struct idr btf_idr;
 extern spinlock_t btf_idr_lock;
 extern struct kobject *btf_kobj;
 extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
 extern bool bpf_global_ma_set;
+extern struct bpf_global_trampoline global_tr;
 
 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
@@ -1279,6 +1282,12 @@ struct bpf_trampoline {
 	struct bpf_tramp_image *cur_image;
 };
 
+struct bpf_global_trampoline {
+	struct list_head list;
+	struct ftrace_ops *fops;
+	void *image;
+};
+
 struct bpf_attach_target_info {
 	struct btf_func_model fmodel;
 	long tgt_addr;
@@ -1382,6 +1391,12 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key,
 void bpf_trampoline_put(struct bpf_trampoline *tr);
 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
 
+#ifdef CONFIG_ARCH_HAS_BPF_GLOBAL_CALLER
+void bpf_global_caller(void);
+#endif
+int bpf_gtrampoline_link_prog(struct bpf_gtramp_link *link);
+int bpf_gtrampoline_unlink_prog(struct bpf_gtramp_link *link);
+
 /*
  * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
  * indirection with a direct call to the bpf program. If the architecture does
@@ -1746,6 +1761,22 @@ struct bpf_shim_tramp_link {
 	struct bpf_trampoline *trampoline;
 };
 
+struct bpf_gtramp_link_entry {
+	struct bpf_prog *tgt_prog;
+	struct bpf_trampoline *trampoline;
+	void *addr;
+	struct btf *attach_btf;
+	u64 cookie;
+	u32 btf_id;
+	u32 nr_args;
+};
+
+struct bpf_gtramp_link {
+	struct bpf_link link;
+	struct bpf_gtramp_link_entry *entries;
+	u32 entry_cnt;
+};
+
 struct bpf_tracing_link {
 	struct bpf_tramp_link link;
 	enum bpf_attach_type attach_type;
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index da4be23f03c3..be06dd76505a 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -13,6 +13,7 @@
 #include <linux/bpf_verifier.h>
 #include <linux/bpf_lsm.h>
 #include <linux/delay.h>
+#include <linux/kfunc_md.h>
 
 /* dummy _ops. The verifier will operate on target program's ops. */
 const struct bpf_verifier_ops bpf_extension_verifier_ops = {
@@ -29,6 +30,10 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
 /* serializes access to trampoline_table */
 static DEFINE_MUTEX(trampoline_mutex);
 
+struct bpf_global_trampoline global_tr;
+static DECLARE_RWSEM(global_tr_lock);
+static const struct bpf_link_ops bpf_shim_tramp_link_lops;
+
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
 
@@ -645,6 +650,172 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
 	return err;
 }
 
+#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && defined(CONFIG_ARCH_HAS_BPF_GLOBAL_CALLER)
+static int bpf_gtrampoline_update(struct bpf_global_trampoline *tr)
+{
+	struct ftrace_ops *fops;
+	int ips_count, err = 0;
+	void **ips = NULL;
+
+	ips_count = kfunc_md_bpf_ips(&ips);
+	if (ips_count < 0) {
+		err = ips_count;
+		goto out;
+	}
+
+	fops = tr->fops;
+	if (ips_count == 0) {
+		if (!(fops->flags & FTRACE_OPS_FL_ENABLED))
+			goto out;
+		err = unregister_ftrace_direct(fops, (unsigned long)tr->image,
+					       true);
+		goto out;
+	}
+
+	if (fops->flags & FTRACE_OPS_FL_ENABLED) {
+		err = reset_ftrace_direct_ips(fops, (unsigned long *)ips,
+					      ips_count);
+		goto out;
+	}
+
+	err = ftrace_set_filter_ips(tr->fops, (unsigned long *)ips,
+				    ips_count, 0, 1);
+	if (err)
+		goto out;
+
+	err = register_ftrace_direct(fops, (unsigned long)tr->image);
+out:
+	kfree(ips);
+
+	return err;
+}
+#else
+static int bpf_gtrampoline_update(struct bpf_global_trampoline *tr)
+{
+	return -ENODEV;
+}
+#endif
+
+static int __bpf_gtrampoline_unlink_prog(struct bpf_gtramp_link *link,
+					 u32 cnt)
+{
+	enum bpf_tramp_prog_type kind;
+	struct kfunc_md *md;
+	int err = 0;
+
+	kind = bpf_attach_type_to_tramp(link->link.prog);
+	kfunc_md_lock();
+	for (int i = 0; i < cnt; i++) {
+		md = kfunc_md_get_noref((long)link->entries[i].addr);
+		if (WARN_ON_ONCE(!md)) {
+			err = -EINVAL;
+			break;
+		}
+
+		if (md->tramp)
+			bpf_gtrampoline_remove(md->tramp, link->link.prog, false);
+
+		md->flags &= ~KFUNC_MD_FL_BPF_REMOVING;
+		err = kfunc_md_bpf_unlink(md, link->link.prog, kind);
+		kfunc_md_put_entry(md);
+		if (err)
+			break;
+	}
+	kfunc_md_unlock();
+
+	return err;
+}
+
+int bpf_gtrampoline_unlink_prog(struct bpf_gtramp_link *link)
+{
+	struct kfunc_md *md;
+	int err;
+
+
+	/* hold the global trampoline lock, to make the target functions
+	 * consist during we unlink the prog.
+	 */
+	down_read(&global_tr_lock);
+	/* update the kfunc_md status, meanwhile update corresponding fops */
+	kfunc_md_lock();
+	for (int i = 0; i < link->entry_cnt; i++) {
+		md = kfunc_md_get_noref((long)link->entries[i].addr);
+		if (WARN_ON_ONCE(!md))
+			continue;
+
+		md->flags |= KFUNC_MD_FL_BPF_REMOVING;
+	}
+	kfunc_md_unlock();
+
+	bpf_gtrampoline_update(&global_tr);
+
+	/* update the ftrace filter first, then the corresponding kfunc_md */
+	err = __bpf_gtrampoline_unlink_prog(link, link->entry_cnt);
+	up_read(&global_tr_lock);
+
+	return err;
+}
+
+int bpf_gtrampoline_link_prog(struct bpf_gtramp_link *link)
+{
+	struct bpf_gtramp_link_entry *entry;
+	enum bpf_tramp_prog_type kind;
+	struct bpf_prog *prog;
+	struct kfunc_md *md;
+	bool update = false;
+	int err = 0, i;
+
+	prog = link->link.prog;
+	kind = bpf_attach_type_to_tramp(prog);
+
+	/* hold the global trampoline lock, to make the target functions
+	 * consist during we link the prog.
+	 */
+	down_read(&global_tr_lock);
+
+	/* update the bpf prog to all the corresponding function metadata */
+	for (i = 0; i < link->entry_cnt; i++) {
+		entry = &link->entries[i];
+		/* it seems that we hold this lock too long, we can use rcu
+		 * lock instead.
+		 */
+		kfunc_md_lock();
+		md = kfunc_md_create((long)entry->addr, entry->nr_args);
+		if (md) {
+			/* the function is not in the filter hash of gtr,
+			 * we need update the global trampoline.
+			 */
+			if (!md->bpf_prog_cnt)
+				update = true;
+			err = kfunc_md_bpf_link(md, prog, kind, entry->cookie);
+		} else {
+			err = -ENOMEM;
+		}
+
+		if (err) {
+			kfunc_md_put_entry(md);
+			kfunc_md_unlock();
+			goto on_fallback;
+		}
+		kfunc_md_unlock();
+	}
+
+	if (update) {
+		err = bpf_gtrampoline_update(&global_tr);
+		if (err)
+			goto on_fallback;
+	}
+	up_read(&global_tr_lock);
+
+	return 0;
+
+on_fallback:
+	__bpf_gtrampoline_unlink_prog(link, i);
+	up_read(&global_tr_lock);
+
+	return err;
+}
+
 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
 static void bpf_shim_tramp_link_release(struct bpf_link *link)
 {
@@ -1131,6 +1302,18 @@ static int __init init_trampolines(void)
 {
 	int i;
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+	global_tr.fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
+	if (!global_tr.fops)
+		return -ENOMEM;
+
+	global_tr.fops->private = &global_tr;
+	global_tr.fops->ops_func = bpf_tramp_ftrace_ops_func;
+#endif
+#ifdef CONFIG_ARCH_HAS_BPF_GLOBAL_CALLER
+	global_tr.image = bpf_global_caller;
+#endif
+
 	for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
 		INIT_HLIST_HEAD(&trampoline_table[i]);
 	return 0;
-- 
2.39.5

Powered by blists - more mailing lists