| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20251015172835.436893677@linutronix.de>
Date: Wed, 15 Oct 2025 19:29:58 +0200 (CEST)
From: Thomas Gleixner <tglx@...utronix.de>
To: LKML <linux-kernel@...r.kernel.org>
Cc: Peter Zijlstra <peterz@...radead.org>,
Gabriele Monaco <gmonaco@...hat.com>,
Mathieu Desnoyers <mathieu.desnoyers@...icios.com>,
Michael Jeanson <mjeanson@...icios.com>,
Jens Axboe <axboe@...nel.dk>,
"Paul E. McKenney" <paulmck@...nel.org>,
"Gautham R. Shenoy" <gautham.shenoy@....com>,
Florian Weimer <fweimer@...hat.com>,
Tim Chen <tim.c.chen@...el.com>,
TCMalloc Team <tcmalloc-eng@...gle.com>
Subject: [patch 18/19] sched/mmcid: Implement deferred mode change
When affinity changes cause an increase of the number of CPUs allowed for
tasks which are related to a MM, that might results in a situation where
the ownership mode can go back from per CPU mode to per task mode.
As affinity changes happen with runqueue lock held there is no way to do
the actual mode change and required fixup right there.
Add the infrastructure to defer it to a workqueue. The scheduled work can
race with a fork() or exit(). Whatever happens first takes care of it.
Signed-off-by: Thomas Gleixner <tglx@...utronix.de>
---
include/linux/rseq_types.h | 8 ++++++
kernel/sched/core.c | 58 +++++++++++++++++++++++++++++++++++++++------
2 files changed, 59 insertions(+), 7 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
+#include <linux/irq_work_types.h>
#include <linux/types.h>
+#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
@@ -127,6 +129,8 @@ struct mm_cid_pcpu {
* do not actually share the MM.
* @pcpu_thrs: Threshold for switching back from per CPU mode
* @update_deferred: A deferred switch back to per task mode is pending.
+ * @irq_work: irq_work to handle the affinity mode change case
+ * @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
*/
@@ -142,6 +146,10 @@ struct mm_mm_cid {
unsigned int pcpu_thrs;
unsigned int update_deferred;
+ /* Rarely used. Moves @lock and @mutex into the second cacheline */
+ struct irq_work irq_work;
+ struct work_struct work;
+
raw_spinlock_t lock;
struct mutex mutex;
}____cacheline_aligned_in_smp;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10566,8 +10566,17 @@ static inline void mm_update_cpus_allowe
/* Adjust the threshold to the wider set */
mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ /* Switch back to per task mode? */
+ if (mc->users >= mc->pcpu_thrs)
+ return;
+
+ /* Don't queue twice */
+ if (mc->update_deferred)
+ return;
- /* Scheduling of deferred mode switch goes here */
+ /* Queue the irq work, which schedules the real work */
+ mc->update_deferred = true;
+ irq_work_queue(&mc->irq_work);
}
static inline void mm_cid_transfer_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
@@ -10578,7 +10587,7 @@ static inline void mm_cid_transfer_to_ta
}
}
-static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
{
unsigned int cpu;
@@ -10722,14 +10731,47 @@ void sched_mm_cid_after_execve(struct ta
mm_cid_select(t);
}
-void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+static void mm_cid_work_fn(struct work_struct *work)
{
- struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
- int cpu;
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
+
+ /* Make it compile, but not functional yet */
+ if (!IS_ENABLED(CONFIG_NEW_MM_CID))
+ return;
- for_each_possible_cpu(cpu)
- per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ guard(mutex)(&mm->mm_cid.mutex);
+ /* Did the last user task exit already? */
+ if (!mm->mm_cid.users)
+ return;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Have fork() or exit() handled it already? */
+ if (!mm->mm_cid.update_deferred)
+ return;
+ /* This clears mm_cid::update_deferred */
+ if (!mm_update_max_cids(mm))
+ return;
+ /* Affinity changes can only switch back to task mode */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return;
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+
+ /*
+ * Needs to be unconditional because mm_cid::lock cannot be held
+ * when scheduling work as mm_update_cpus_allowed() nests inside
+ * rq::lock and schedule_work() might end up in wakeup...
+ */
+ schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
@@ -10738,6 +10780,8 @@ void mm_init_cid(struct mm_struct *mm, s
mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
+ mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+ INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), nr_cpu_ids);
}
Powered by blists - more mailing lists