Message-Id: <964eab3190221c0c880ee9a52957865512c8571c.1748594841.git.libo.gcs85@bytedance.com>
Date: Fri, 30 May 2025 17:27:45 +0800
From: Bo Li <libo.gcs85@...edance.com>
To: tglx@...utronix.de,
	mingo@...hat.com,
	bp@...en8.de,
	dave.hansen@...ux.intel.com,
	x86@...nel.org,
	luto@...nel.org,
	kees@...nel.org,
	akpm@...ux-foundation.org,
	david@...hat.com,
	juri.lelli@...hat.com,
	vincent.guittot@...aro.org,
	peterz@...radead.org
Cc: dietmar.eggemann@....com,
	hpa@...or.com,
	acme@...nel.org,
	namhyung@...nel.org,
	mark.rutland@....com,
	alexander.shishkin@...ux.intel.com,
	jolsa@...nel.org,
	irogers@...gle.com,
	adrian.hunter@...el.com,
	kan.liang@...ux.intel.com,
	viro@...iv.linux.org.uk,
	brauner@...nel.org,
	jack@...e.cz,
	lorenzo.stoakes@...cle.com,
	Liam.Howlett@...cle.com,
	vbabka@...e.cz,
	rppt@...nel.org,
	surenb@...gle.com,
	mhocko@...e.com,
	rostedt@...dmis.org,
	bsegall@...gle.com,
	mgorman@...e.de,
	vschneid@...hat.com,
	jannh@...gle.com,
	pfalcato@...e.de,
	riel@...riel.com,
	harry.yoo@...cle.com,
	linux-kernel@...r.kernel.org,
	linux-perf-users@...r.kernel.org,
	linux-fsdevel@...r.kernel.org,
	linux-mm@...ck.org,
	duanxiongchun@...edance.com,
	yinhongbo@...edance.com,
	dengliang.1214@...edance.com,
	xieyongji@...edance.com,
	chaiwen.cc@...edance.com,
	songmuchun@...edance.com,
	yuanzhu@...edance.com,
	chengguozhu@...edance.com,
	sunjiadong.lff@...edance.com,
	Bo Li <libo.gcs85@...edance.com>
Subject: [RFC v2 17/35] RPAL: add a mapping between fsbase and tasks

RPAL relies on the value of the fsbase register to determine whether a lazy
switch is necessary. Therefore, a mapping between fsbase and tasks must be
established.

This patch allows a thread to register its fsbase value when it is
registered as a receiver. The rpal_find_next_task() interface is used to
locate the receiver corresponding to a given fsbase value. Additionally, a
new rpal_misidentify() interface has been added to check if the current
fsbase value matches the current task. If they do not match, the task
corresponding to the fsbase is identified, the RPAL_LAZY_SWITCHED_BIT flag
is set, and the current task is recorded. The kernel can later use this
flag and the recorded task to backtrack to the task before the lazy switch.

Signed-off-by: Bo Li <libo.gcs85@...edance.com>
---
 arch/x86/rpal/core.c   | 85 ++++++++++++++++++++++++++++++++++++++++++
 arch/x86/rpal/thread.c | 57 +++++++++++++++++++++++++++-
 include/linux/rpal.h   | 15 ++++++++
 3 files changed, 156 insertions(+), 1 deletion(-)

diff --git a/arch/x86/rpal/core.c b/arch/x86/rpal/core.c
index c185a453c1b2..19c4ef38bca3 100644
--- a/arch/x86/rpal/core.c
+++ b/arch/x86/rpal/core.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/rpal.h>
+#include <asm/fsgsbase.h>
 
 #include "internal.h"
 
@@ -33,12 +34,96 @@ static inline void rpal_unlock_cpu(struct task_struct *tsk)
 	}
 }
 
+
+static inline struct task_struct *rpal_get_sender_task(void)
+{
+	struct task_struct *next;
+
+	next = current->rpal_rd->sender;
+	current->rpal_rd->sender = NULL;
+
+	return next;
+}
+
+/*
+ * RPAL uses the value of fsbase (which libc uses as the base
+ * address for thread-local storage) to determine whether a
+ * lazy switch should be performed.
+ */
+static inline struct task_struct *rpal_misidentify(void)
+{
+	struct task_struct *next = NULL;
+	struct rpal_service *cur = rpal_current_service();
+	unsigned long fsbase;
+
+	fsbase = rdfsbase();
+	if (unlikely(!rpal_is_correct_address(cur, fsbase))) {
+		if (rpal_test_current_thread_flag(RPAL_LAZY_SWITCHED_BIT)) {
+			/* current is receiver, next is sender */
+			next = rpal_get_sender_task();
+			if (unlikely(next == NULL)) {
+				rpal_err("cannot find sender task\n");
+				goto out;
+			}
+		} else {
+			/* current is sender, next is receiver */
+			next = rpal_find_next_task(fsbase);
+			if (unlikely(next == NULL)) {
+				rpal_err(
+					"cannot find receiver task, fsbase: 0x%016lx\n",
+					fsbase);
+				goto out;
+			}
+			rpal_set_task_thread_flag(next, RPAL_LAZY_SWITCHED_BIT);
+			next->rpal_rd->sender = current;
+		}
+	}
+out:
+	return next;
+}
+
+struct task_struct *rpal_find_next_task(unsigned long fsbase)
+{
+	struct rpal_service *cur = rpal_current_service();
+	struct rpal_service *tgt;
+	struct task_struct *tsk = NULL;
+	int i;
+
+	tgt = rpal_get_mapped_service_by_addr(cur, fsbase);
+	if (unlikely(!tgt)) {
+		pr_debug("rpal debug: cannot find legal rs, fsbase: 0x%016lx\n",
+			 fsbase);
+		return NULL;
+	}
+	for (i = 0; i < RPAL_MAX_RECEIVER_NUM; ++i) {
+		if (tgt->fs_tsk_map[i].fsbase == fsbase) {
+			tsk = tgt->fs_tsk_map[i].tsk;
+			break;
+		}
+	}
+	rpal_put_service(tgt);
+
+	return tsk;
+}
+
+static bool check_hardware_features(void)
+{
+	if (!boot_cpu_has(X86_FEATURE_FSGSBASE)) {
+		rpal_err("no fsgsbase feature\n");
+		return false;
+	}
+	return true;
+}
+
 int __init rpal_init(void)
 {
 	int ret = 0;
 
 	rpal_cap = 0;
 
+	if (!check_hardware_features())
+		goto fail;
+
 	ret = rpal_service_init();
 	if (ret)
 		goto fail;
diff --git a/arch/x86/rpal/thread.c b/arch/x86/rpal/thread.c
index bc203e9c6e5e..db3b13ff82be 100644
--- a/arch/x86/rpal/thread.c
+++ b/arch/x86/rpal/thread.c
@@ -7,9 +7,53 @@
  */
 
 #include <linux/rpal.h>
+#include <asm/fsgsbase.h>
 
 #include "internal.h"
 
+static bool set_fs_tsk_map(void)
+{
+	struct rpal_service *cur = rpal_current_service();
+	struct rpal_fsbase_tsk_map *ftm;
+	unsigned long fsbase = rdfsbase();
+	bool success = false;
+	int i = 0;
+
+	for (i = 0; i < RPAL_MAX_RECEIVER_NUM; ++i) {
+		ftm = &cur->fs_tsk_map[i];
+		if (ftm->fsbase == 0 &&
+		    cmpxchg64(&ftm->fsbase, 0, fsbase) == 0) {
+			ftm->tsk = current;
+			success = true;
+			break;
+		}
+	}
+
+	return success;
+}
+
+static bool clear_fs_tsk_map(void)
+{
+	struct rpal_service *cur = rpal_current_service();
+	struct rpal_fsbase_tsk_map *ftm;
+	unsigned long fsbase = rdfsbase();
+	bool success = false;
+	int i = 0;
+
+	for (i = 0; i < RPAL_MAX_RECEIVER_NUM; ++i) {
+		ftm = &cur->fs_tsk_map[i];
+		if (ftm->fsbase == fsbase) {
+			ftm->tsk = NULL;
+			barrier();
+			ftm->fsbase = 0;
+			success = true;
+			break;
+		}
+	}
+
+	return success;
+}
+
 static void rpal_common_data_init(struct rpal_common_data *rcd)
 {
 	rcd->bp_task = current;
@@ -54,6 +98,7 @@ int rpal_register_sender(unsigned long addr)
 	rsd->rsp = rsp;
 	rsd->scc = (struct rpal_sender_call_context *)(addr - rsp->user_start +
 						       rsp->kernel_start);
+	rsd->receiver = NULL;
 
 	current->rpal_sd = rsd;
 	rpal_set_current_thread_flag(RPAL_SENDER_BIT);
@@ -122,15 +167,21 @@ int rpal_register_receiver(unsigned long addr)
 		goto put_shared_page;
 	}
 
+	if (!set_fs_tsk_map()) {
+		ret = -EAGAIN;
+		goto free_rrd;
+	}
+
 	rpal_common_data_init(&rrd->rcd);
 	if (rpal_init_thread_pending(&rrd->rcd)) {
 		ret = -ENOMEM;
-		goto free_rrd;
+		goto clear_fs;
 	}
 	rrd->rsp = rsp;
 	rrd->rcc =
 		(struct rpal_receiver_call_context *)(addr - rsp->user_start +
 						      rsp->kernel_start);
+	rrd->sender = NULL;
 
 	current->rpal_rd = rrd;
 	rpal_set_current_thread_flag(RPAL_RECEIVER_BIT);
@@ -139,6 +190,8 @@ int rpal_register_receiver(unsigned long addr)
 
 	return 0;
 
+clear_fs:
+	clear_fs_tsk_map();
 free_rrd:
 	kfree(rrd);
 put_shared_page:
@@ -158,6 +211,8 @@ int rpal_unregister_receiver(void)
 		goto out;
 	}
 
+	clear_fs_tsk_map();
+
 	rpal_put_shared_page(rrd->rsp);
 	rpal_clear_current_thread_flag(RPAL_RECEIVER_BIT);
 	rpal_free_thread_pending(&rrd->rcd);
diff --git a/include/linux/rpal.h b/include/linux/rpal.h
index 5b115be14a55..45137770fac6 100644
--- a/include/linux/rpal.h
+++ b/include/linux/rpal.h
@@ -80,6 +80,9 @@
 /* No more than 15 services can be requested due to limitation of MPK. */
 #define MAX_REQUEST_SERVICE 15
 
+/* We allow at most 16 receiver thread in one process */
+#define RPAL_MAX_RECEIVER_NUM  16
+
 enum {
 	RPAL_REQUEST_MAP,
 	RPAL_REVERSE_MAP,
@@ -100,6 +103,7 @@ enum rpal_task_flag_bits {
 	RPAL_SENDER_BIT,
 	RPAL_RECEIVER_BIT,
 	RPAL_CPU_LOCKED_BIT,
+	RPAL_LAZY_SWITCHED_BIT,
 };
 
 enum rpal_receiver_state {
@@ -145,6 +149,11 @@ struct rpal_poll_data {
 	wait_queue_head_t rpal_waitqueue;
 };
 
+struct rpal_fsbase_tsk_map {
+	unsigned long fsbase;
+	struct task_struct *tsk;
+};
+
 /*
  * Each RPAL process (a.k.a RPAL service) should have a pointer to
  * struct rpal_service in all its tasks' task_struct.
@@ -202,6 +211,9 @@ struct rpal_service {
 	/* Notify service is released by others */
 	struct rpal_poll_data rpd;
 
+	/* fsbase / pid map */
+	struct rpal_fsbase_tsk_map fs_tsk_map[RPAL_MAX_RECEIVER_NUM];
+
 	/* delayed service put work */
 	struct delayed_work delayed_put_work;
 
@@ -283,12 +295,14 @@ struct rpal_receiver_data {
 	struct rpal_common_data rcd;
 	struct rpal_shared_page *rsp;
 	struct rpal_receiver_call_context *rcc;
+	struct task_struct *sender;
 };
 
 struct rpal_sender_data {
 	struct rpal_common_data rcd;
 	struct rpal_shared_page *rsp;
 	struct rpal_sender_call_context *scc;
+	struct task_struct *receiver;
 };
 
 enum rpal_command_type {
@@ -465,6 +479,7 @@ struct rpal_service *rpal_get_mapped_service_by_id(struct rpal_service *rs,
 int rpal_rebuild_sender_context_on_fault(struct pt_regs *regs,
 					 unsigned long addr, int error_code);
 struct mm_struct *rpal_pf_get_real_mm(unsigned long address, int *rebuild);
+struct task_struct *rpal_find_next_task(unsigned long fsbase);
 
 extern void rpal_pick_mmap_base(struct mm_struct *mm,
 	struct rlimit *rlim_stack);
-- 
2.20.1

Powered by blists - more mailing lists