| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <f7026f347934cfe710650d29bb52271a6cabccd9.1764801860.git.tim.c.chen@linux.intel.com>
Date: Wed, 3 Dec 2025 15:07:23 -0800
From: Tim Chen <tim.c.chen@...ux.intel.com>
To: Peter Zijlstra <peterz@...radead.org>,
Ingo Molnar <mingo@...hat.com>,
K Prateek Nayak <kprateek.nayak@....com>,
"Gautham R . Shenoy" <gautham.shenoy@....com>,
Vincent Guittot <vincent.guittot@...aro.org>
Cc: Tim Chen <tim.c.chen@...ux.intel.com>,
Juri Lelli <juri.lelli@...hat.com>,
Dietmar Eggemann <dietmar.eggemann@....com>,
Steven Rostedt <rostedt@...dmis.org>,
Ben Segall <bsegall@...gle.com>,
Mel Gorman <mgorman@...e.de>,
Valentin Schneider <vschneid@...hat.com>,
Madadi Vineeth Reddy <vineethr@...ux.ibm.com>,
Hillf Danton <hdanton@...a.com>,
Shrikanth Hegde <sshegde@...ux.ibm.com>,
Jianyong Wu <jianyong.wu@...look.com>,
Yangyu Chen <cyy@...self.name>,
Tingyin Duan <tingyin.duan@...il.com>,
Vern Hao <vernhao@...cent.com>,
Vern Hao <haoxing990@...il.com>,
Len Brown <len.brown@...el.com>,
Aubrey Li <aubrey.li@...el.com>,
Zhao Liu <zhao1.liu@...el.com>,
Chen Yu <yu.chen.surf@...il.com>,
Chen Yu <yu.c.chen@...el.com>,
Adam Li <adamli@...amperecomputing.com>,
Aaron Lu <ziqianlu@...edance.com>,
Tim Chen <tim.c.chen@...el.com>,
linux-kernel@...r.kernel.org
Subject: [PATCH v2 04/23] sched/cache: Make LLC id continuous
Introduce an index mapping between CPUs and their LLCs. This provides
a continuous per LLC index needed for cache-aware load balancing in
later patches.
The existing per_cpu llc_id usually points to the first CPU of the
LLC domain, which is sparse and unsuitable as an array index. Using
llc_id directly would waste memory.
With the new mapping, CPUs in the same LLC share a continuous id:
per_cpu(llc_id, CPU=0...15) = 0
per_cpu(llc_id, CPU=16...31) = 1
per_cpu(llc_id, CPU=32...47) = 2
...
Co-developed-by: Chen Yu <yu.c.chen@...el.com>
Signed-off-by: Chen Yu <yu.c.chen@...el.com>
Signed-off-by: Tim Chen <tim.c.chen@...ux.intel.com>
---
Notes:
v1->v2:
Convert the static LLC id to be allocated sequentially as LLCs are
discovered, and replace the old sd_llc_id. (Peter Zijlstra)
kernel/sched/fair.c | 9 ++++++-
kernel/sched/sched.h | 1 +
kernel/sched/topology.c | 60 +++++++++++++++++++++++++++++++++++++++--
3 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 710ed9943d27..0a3918269906 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1210,10 +1210,17 @@ __read_mostly unsigned int llc_imb_pct = 20;
static int llc_id(int cpu)
{
+ int llc;
+
if (cpu < 0)
return -1;
- return per_cpu(sd_llc_id, cpu);
+ llc = per_cpu(sd_llc_id, cpu);
+ /* avoid race with cpu hotplug */
+ if (unlikely(llc >= max_llcs))
+ return -1;
+
+ return llc;
}
void mm_init_sched(struct mm_struct *mm, struct mm_sched __percpu *_pcpu_sched)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index bf72c5bab506..728737641847 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2075,6 +2075,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
extern struct static_key_false sched_asym_cpucapacity;
extern struct static_key_false sched_cluster_active;
+extern int max_llcs;
static __always_inline bool sched_asym_cpucap_active(void)
{
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 444bdfdab731..f25d950ab015 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -17,6 +17,8 @@ void sched_domains_mutex_unlock(void)
mutex_unlock(&sched_domains_mutex);
}
+int max_llcs;
+
/* Protected by sched_domains_mutex: */
static cpumask_var_t sched_domains_tmpmask;
static cpumask_var_t sched_domains_tmpmask2;
@@ -668,6 +670,55 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
DEFINE_STATIC_KEY_FALSE(sched_cluster_active);
+/*
+ * Assign continuous llc id for the CPU, and return
+ * the assigned llc id.
+ */
+static int update_llc_id(struct sched_domain *sd,
+ int cpu)
+{
+ int id = per_cpu(sd_llc_id, cpu), i;
+
+ if (id >= 0)
+ return id;
+
+ if (sd) {
+ /* Look for any assigned id and reuse it.*/
+ for_each_cpu(i, sched_domain_span(sd)) {
+ id = per_cpu(sd_llc_id, i);
+
+ if (id >= 0) {
+ per_cpu(sd_llc_id, cpu) = id;
+ return id;
+ }
+ }
+ }
+
+ /*
+ * When 1. there is no id assigned to this LLC domain,
+ * or 2. the sd is NULL, we reach here.
+ * Consider the following scenario,
+ * CPU0~CPU95 are in the node0, CPU96~CPU191 are
+ * in the node1. During bootup, maxcpus=96 is
+ * appended.
+ * case 1: When running cpu_attach_domain(CPU24)
+ * during boot up, CPU24 is the first CPU in its
+ * non-NULL LLC domain. However,
+ * its corresponding llc id has not been assigned yet.
+ *
+ * case 2: After boot up, the CPU100 is brought up
+ * via sysfs manually. As a result, CPU100 has only a
+ * Numa domain attached, because CPU100 is the only CPU
+ * of a sched domain, all its bottom domains are degenerated.
+ * The LLC domain pointer sd is NULL for CPU100.
+ *
+ * For both cases, we want to increase the number of LLCs.
+ */
+ per_cpu(sd_llc_id, cpu) = max_llcs++;
+
+ return per_cpu(sd_llc_id, cpu);
+}
+
static void update_top_cache_domain(int cpu)
{
struct sched_domain_shared *sds = NULL;
@@ -677,14 +728,13 @@ static void update_top_cache_domain(int cpu)
sd = highest_flag_domain(cpu, SD_SHARE_LLC);
if (sd) {
- id = cpumask_first(sched_domain_span(sd));
size = cpumask_weight(sched_domain_span(sd));
sds = sd->shared;
}
rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
per_cpu(sd_llc_size, cpu) = size;
- per_cpu(sd_llc_id, cpu) = id;
+ id = update_llc_id(sd, cpu);
rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
sd = lowest_flag_domain(cpu, SD_CLUSTER);
@@ -2488,6 +2538,12 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
bool has_asym = false;
bool has_cluster = false;
+ /* first scan of LLCs */
+ if (!max_llcs) {
+ for_each_possible_cpu(i)
+ per_cpu(sd_llc_id, i) = -1;
+ }
+
if (WARN_ON(cpumask_empty(cpu_map)))
goto error;
--
2.32.0
Powered by blists - more mailing lists