| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <8941a17e12edce00c1cc1c78f4dd3e1bf28e47c0.1748594841.git.libo.gcs85@bytedance.com>
Date: Fri, 30 May 2025 17:28:01 +0800
From: Bo Li <libo.gcs85@...edance.com>
To: tglx@...utronix.de,
mingo@...hat.com,
bp@...en8.de,
dave.hansen@...ux.intel.com,
x86@...nel.org,
luto@...nel.org,
kees@...nel.org,
akpm@...ux-foundation.org,
david@...hat.com,
juri.lelli@...hat.com,
vincent.guittot@...aro.org,
peterz@...radead.org
Cc: dietmar.eggemann@....com,
hpa@...or.com,
acme@...nel.org,
namhyung@...nel.org,
mark.rutland@....com,
alexander.shishkin@...ux.intel.com,
jolsa@...nel.org,
irogers@...gle.com,
adrian.hunter@...el.com,
kan.liang@...ux.intel.com,
viro@...iv.linux.org.uk,
brauner@...nel.org,
jack@...e.cz,
lorenzo.stoakes@...cle.com,
Liam.Howlett@...cle.com,
vbabka@...e.cz,
rppt@...nel.org,
surenb@...gle.com,
mhocko@...e.com,
rostedt@...dmis.org,
bsegall@...gle.com,
mgorman@...e.de,
vschneid@...hat.com,
jannh@...gle.com,
pfalcato@...e.de,
riel@...riel.com,
harry.yoo@...cle.com,
linux-kernel@...r.kernel.org,
linux-perf-users@...r.kernel.org,
linux-fsdevel@...r.kernel.org,
linux-mm@...ck.org,
duanxiongchun@...edance.com,
yinhongbo@...edance.com,
dengliang.1214@...edance.com,
xieyongji@...edance.com,
chaiwen.cc@...edance.com,
songmuchun@...edance.com,
yuanzhu@...edance.com,
chengguozhu@...edance.com,
sunjiadong.lff@...edance.com,
Bo Li <libo.gcs85@...edance.com>
Subject: [RFC v2 33/35] RPAL: enable time slice correction
After an RPAL call, the receiver's user mode code executes. However, the
kernel incorrectly attributes this CPU time to the sender due to the
unchanged kernel context. This results in incorrect runtime statistics.
This patch adds a new member total_time to both rpal_sender_call_context
and rpal_receiver_call_context. This member tracks how much runtime (
measured in CPU cycles via rdtsc()) has been incorrectly accounted for.
The kernel measures total_time at the entry of __schedule() and corrects
the delta in the update_rq_clock_task() function.
Additionally, since RPAL calls occur in user space, runtime statistics are
typically calculated by user space. However, when a lazy switch happens,
the kernel takes over. To address this, the patch introduces a start_time
member to record when an RPAL call is initiated, enabling the kernel to
accurately calculate the runtime that needs correction.
Signed-off-by: Bo Li <libo.gcs85@...edance.com>
---
arch/x86/rpal/core.c | 8 ++++++++
arch/x86/rpal/thread.c | 6 ++++++
include/linux/rpal.h | 3 +++
include/linux/sched.h | 1 +
init/init_task.c | 1 +
kernel/fork.c | 1 +
kernel/sched/core.c | 42 ++++++++++++++++++++++++++++++++++++++++++
7 files changed, 62 insertions(+)
diff --git a/arch/x86/rpal/core.c b/arch/x86/rpal/core.c
index 92281b557a6c..2ac5d932f69c 100644
--- a/arch/x86/rpal/core.c
+++ b/arch/x86/rpal/core.c
@@ -144,6 +144,13 @@ rpal_do_kernel_context_switch(struct task_struct *next, struct pt_regs *regs)
struct task_struct *prev = current;
if (rpal_test_task_thread_flag(next, RPAL_LAZY_SWITCHED_BIT)) {
+ struct rpal_receiver_call_context *rcc = next->rpal_rd->rcc;
+ struct rpal_sender_call_context *scc = current->rpal_sd->scc;
+ u64 slice = rdtsc_ordered() - scc->start_time;
+
+ rcc->total_time += slice;
+ scc->total_time += slice;
+
rpal_resume_ep(next);
current->rpal_sd->receiver = next;
rpal_lock_cpu(current);
@@ -169,6 +176,7 @@ rpal_do_kernel_context_switch(struct task_struct *next, struct pt_regs *regs)
rpal_schedule(next);
rpal_clear_task_thread_flag(prev, RPAL_LAZY_SWITCHED_BIT);
prev->rpal_rd->sender = NULL;
+ next->rpal_sd->scc->start_time = rdtsc_ordered();
}
if (unlikely(!irqs_disabled())) {
local_irq_disable();
diff --git a/arch/x86/rpal/thread.c b/arch/x86/rpal/thread.c
index 51c9eec639cb..5cd0be631521 100644
--- a/arch/x86/rpal/thread.c
+++ b/arch/x86/rpal/thread.c
@@ -99,6 +99,8 @@ int rpal_register_sender(unsigned long addr)
rsd->scc = (struct rpal_sender_call_context *)(addr - rsp->user_start +
rsp->kernel_start);
rsd->receiver = NULL;
+ rsd->scc->start_time = 0;
+ rsd->scc->total_time = 0;
current->rpal_sd = rsd;
rpal_set_current_thread_flag(RPAL_SENDER_BIT);
@@ -182,6 +184,7 @@ int rpal_register_receiver(unsigned long addr)
(struct rpal_receiver_call_context *)(addr - rsp->user_start +
rsp->kernel_start);
rrd->sender = NULL;
+ rrd->rcc->total_time = 0;
current->rpal_rd = rrd;
rpal_set_current_thread_flag(RPAL_RECEIVER_BIT);
@@ -289,6 +292,9 @@ int rpal_rebuild_sender_context_on_fault(struct pt_regs *regs,
rpal_pkey_to_pkru(rpal_current_service()->pkey),
RPAL_PKRU_SET);
#endif
+ if (!rpal_is_correct_address(rpal_current_service(), regs->ip))
+ /* receiver has crashed */
+ scc->total_time += rdtsc_ordered() - scc->start_time;
return 0;
}
}
diff --git a/include/linux/rpal.h b/include/linux/rpal.h
index 1d8c1bdc90f2..f5f4da63f28c 100644
--- a/include/linux/rpal.h
+++ b/include/linux/rpal.h
@@ -310,6 +310,7 @@ struct rpal_receiver_call_context {
void __user *events;
int maxevents;
int timeout;
+ int64_t total_time;
};
/* recovery point for sender */
@@ -325,6 +326,8 @@ struct rpal_sender_call_context {
struct rpal_task_context rtc;
struct rpal_error_context ec;
int sender_id;
+ s64 start_time;
+ s64 total_time;
};
/* End */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5f25cc09fb71..a03113fecdc5 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1663,6 +1663,7 @@ struct task_struct {
struct rpal_sender_data *rpal_sd;
struct rpal_receiver_data *rpal_rd;
};
+ s64 rpal_steal_time;
#endif
/* CPU-specific state of this task: */
diff --git a/init/init_task.c b/init/init_task.c
index 2eb08b96e66b..3606cf701dfe 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -224,6 +224,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.rpal_rs = NULL,
.rpal_flag = 0,
.rpal_cd = NULL,
+ .rpal_steal_time = 0,
#endif
};
EXPORT_SYMBOL(init_task);
diff --git a/kernel/fork.c b/kernel/fork.c
index 11cba74d07c8..ff6331a28987 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1222,6 +1222,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
tsk->rpal_rs = NULL;
tsk->rpal_flag = 0;
tsk->rpal_cd = NULL;
+ tsk->rpal_steal_time = 0;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c219ada29d34..d6f8e0d76fc0 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -789,6 +789,14 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
delta -= steal;
}
#endif
+#ifdef CONFIG_RPAL
+ if (unlikely(current->rpal_steal_time != 0)) {
+ delta += current->rpal_steal_time;
+ if (unlikely(delta < 0))
+ delta = 0;
+ current->rpal_steal_time = 0;
+ }
+#endif
rq->clock_task += delta;
@@ -6872,6 +6880,36 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
return true;
}
+#ifdef CONFIG_RPAL
+static void rpal_acct_runtime(void)
+{
+ if (rpal_current_service()) {
+ if (rpal_test_task_thread_flag(current, RPAL_SENDER_BIT) &&
+ current->rpal_sd->scc->total_time != 0) {
+ struct rpal_sender_call_context *scc =
+ current->rpal_sd->scc;
+
+ u64 slice =
+ native_sched_clock_from_tsc(scc->total_time) -
+ native_sched_clock_from_tsc(0);
+ current->rpal_steal_time -= slice;
+ scc->total_time = 0;
+ } else if (rpal_test_task_thread_flag(current,
+ RPAL_RECEIVER_BIT) &&
+ current->rpal_rd->rcc->total_time != 0) {
+ struct rpal_receiver_call_context *rcc =
+ current->rpal_rd->rcc;
+
+ u64 slice =
+ native_sched_clock_from_tsc(rcc->total_time) -
+ native_sched_clock_from_tsc(0);
+ current->rpal_steal_time += slice;
+ rcc->total_time = 0;
+ }
+ }
+}
+#endif
+
/*
* __schedule() is the main scheduler function.
*
@@ -6926,6 +6964,10 @@ static void __sched notrace __schedule(int sched_mode)
struct rq *rq;
int cpu;
+#ifdef CONFIG_RPAL
+ rpal_acct_runtime();
+#endif
+
trace_sched_entry_tp(preempt, CALLER_ADDR0);
cpu = smp_processor_id();
--
2.20.1
Powered by blists - more mailing lists