| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20250620152308.27492-15-frederic@kernel.org>
Date: Fri, 20 Jun 2025 17:22:55 +0200
From: Frederic Weisbecker <frederic@...nel.org>
To: LKML <linux-kernel@...r.kernel.org>
Cc: Frederic Weisbecker <frederic@...nel.org>,
Ingo Molnar <mingo@...hat.com>,
Marco Crivellari <marco.crivellari@...e.com>,
Michal Hocko <mhocko@...e.com>,
Peter Zijlstra <peterz@...radead.org>,
Tejun Heo <tj@...nel.org>,
Thomas Gleixner <tglx@...utronix.de>,
Vlastimil Babka <vbabka@...e.cz>,
Waiman Long <longman@...hat.com>
Subject: [PATCH 14/27] sched/isolation: Convert housekeeping cpumasks to rcu pointers
HK_TYPE_DOMAIN's cpumask will soon be made modifyable by cpuset.
Sleepable users of housekeeping can synchronize against cpumask
modifications using the housekeeping rwsem. Other callsites need an
alternative.
Turn the housekeeping cpumasks into RCU pointers. Once a housekeeping
cpumask will be modified, the update side will wait for an RCU grace
period and propagate the change to interested subsystem when deemed
necessary.
Signed-off-by: Frederic Weisbecker <frederic@...nel.org>
---
kernel/sched/isolation.c | 52 ++++++++++++++++++++++++++--------------
kernel/sched/sched.h | 1 +
2 files changed, 35 insertions(+), 18 deletions(-)
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index 9ecf53c5328b..75505668dcb9 100644
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -15,7 +15,7 @@ enum hk_flags {
HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE),
};
-static cpumask_var_t housekeeping_cpumasks[HK_TYPE_MAX];
+static struct cpumask __rcu *housekeeping_cpumasks[HK_TYPE_MAX];
unsigned long housekeeping_flags;
EXPORT_SYMBOL_GPL(housekeeping_flags);
@@ -37,16 +37,25 @@ void housekeeping_unlock(void)
percpu_up_read(&housekeeping_pcpu_lock);
}
+const struct cpumask *housekeeping_cpumask(enum hk_type type)
+{
+ if (housekeeping_flags & BIT(type)) {
+ return rcu_dereference_check(housekeeping_cpumasks[type], 1);
+ }
+ return cpu_possible_mask;
+}
+EXPORT_SYMBOL_GPL(housekeeping_cpumask);
+
int housekeeping_any_cpu(enum hk_type type)
{
int cpu;
if (housekeeping_flags & BIT(type)) {
- cpu = sched_numa_find_closest(housekeeping_cpumasks[type], smp_processor_id());
+ cpu = sched_numa_find_closest(housekeeping_cpumask(type), smp_processor_id());
if (cpu < nr_cpu_ids)
return cpu;
- cpu = cpumask_any_and_distribute(housekeeping_cpumasks[type], cpu_online_mask);
+ cpu = cpumask_any_and_distribute(housekeeping_cpumask(type), cpu_online_mask);
if (likely(cpu < nr_cpu_ids))
return cpu;
/*
@@ -62,25 +71,17 @@ int housekeeping_any_cpu(enum hk_type type)
}
EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
-const struct cpumask *housekeeping_cpumask(enum hk_type type)
-{
- if (housekeeping_flags & BIT(type))
- return housekeeping_cpumasks[type];
- return cpu_possible_mask;
-}
-EXPORT_SYMBOL_GPL(housekeeping_cpumask);
-
void housekeeping_affine(struct task_struct *t, enum hk_type type)
{
if (housekeeping_flags & BIT(type))
- set_cpus_allowed_ptr(t, housekeeping_cpumasks[type]);
+ set_cpus_allowed_ptr(t, housekeeping_cpumask(type));
}
EXPORT_SYMBOL_GPL(housekeeping_affine);
bool housekeeping_test_cpu(int cpu, enum hk_type type)
{
if (housekeeping_flags & BIT(type))
- return cpumask_test_cpu(cpu, housekeeping_cpumasks[type]);
+ return cpumask_test_cpu(cpu, housekeeping_cpumask(type));
return true;
}
EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
@@ -95,9 +96,23 @@ void __init housekeeping_init(void)
if (housekeeping_flags & HK_FLAG_KERNEL_NOISE)
sched_tick_offload_init();
+ /*
+ * Realloc with a proper allocator so that any cpumask update
+ * can indifferently free the old version with kfree().
+ */
for_each_set_bit(type, &housekeeping_flags, HK_TYPE_MAX) {
+ struct cpumask *omask, *nmask = kmalloc(cpumask_size(), GFP_KERNEL);
+
+ if (WARN_ON_ONCE(!nmask))
+ return;
+
+ omask = rcu_dereference(housekeeping_cpumasks[type]);
+
/* We need at least one CPU to handle housekeeping work */
- WARN_ON_ONCE(cpumask_empty(housekeeping_cpumasks[type]));
+ WARN_ON_ONCE(cpumask_empty(omask));
+ cpumask_copy(nmask, omask);
+ RCU_INIT_POINTER(housekeeping_cpumasks[type], nmask);
+ memblock_free(omask, cpumask_size());
}
}
@@ -105,9 +120,10 @@ static void __init housekeeping_setup_type(enum hk_type type,
cpumask_var_t housekeeping_staging)
{
- alloc_bootmem_cpumask_var(&housekeeping_cpumasks[type]);
- cpumask_copy(housekeeping_cpumasks[type],
- housekeeping_staging);
+ struct cpumask *mask = memblock_alloc_or_panic(cpumask_size(), SMP_CACHE_BYTES);
+
+ cpumask_copy(mask, housekeeping_staging);
+ RCU_INIT_POINTER(housekeeping_cpumasks[type], mask);
}
static int __init housekeeping_setup(char *str, unsigned long flags)
@@ -160,7 +176,7 @@ static int __init housekeeping_setup(char *str, unsigned long flags)
for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
if (!cpumask_equal(housekeeping_staging,
- housekeeping_cpumasks[type])) {
+ housekeeping_cpumask(type))) {
pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
goto free_housekeeping_staging;
}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 0cdb560ef2f3..407e7f5ad929 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -42,6 +42,7 @@
#include <linux/ktime_api.h>
#include <linux/lockdep_api.h>
#include <linux/lockdep.h>
+#include <linux/memblock.h>
#include <linux/minmax.h>
#include <linux/mm.h>
#include <linux/module.h>
--
2.48.1
Powered by blists - more mailing lists