| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <959d897daadc28b8115c97df04eec2af0fd79c5d.1754712565.git.tim.c.chen@linux.intel.com>
Date: Sat, 9 Aug 2025 13:03:24 +0800
From: Chen Yu <yu.c.chen@...el.com>
To: Peter Zijlstra <peterz@...radead.org>,
Ingo Molnar <mingo@...hat.com>,
K Prateek Nayak <kprateek.nayak@....com>,
"Gautham R . Shenoy" <gautham.shenoy@....com>
Cc: Vincent Guittot <vincent.guittot@...aro.org>,
Juri Lelli <juri.lelli@...hat.com>,
Dietmar Eggemann <dietmar.eggemann@....com>,
Steven Rostedt <rostedt@...dmis.org>,
Ben Segall <bsegall@...gle.com>,
Mel Gorman <mgorman@...e.de>,
Valentin Schneider <vschneid@...hat.com>,
Libo Chen <libo.chen@...cle.com>,
Madadi Vineeth Reddy <vineethr@...ux.ibm.com>,
Hillf Danton <hdanton@...a.com>,
Shrikanth Hegde <sshegde@...ux.ibm.com>,
Jianyong Wu <jianyong.wu@...look.com>,
Yangyu Chen <cyy@...self.name>,
Tingyin Duan <tingyin.duan@...il.com>,
Vern Hao <vernhao@...cent.com>,
Len Brown <len.brown@...el.com>,
Tim Chen <tim.c.chen@...ux.intel.com>,
Aubrey Li <aubrey.li@...el.com>,
Zhao Liu <zhao1.liu@...el.com>,
Chen Yu <yu.chen.surf@...il.com>,
Chen Yu <yu.c.chen@...el.com>,
linux-kernel@...r.kernel.org
Subject: [RFC PATCH v4 08/28] sched: Set up LLC indexing
From: Tim Chen <tim.c.chen@...ux.intel.com>
Prepare for indexing arrays that track in each run queue: the number
of tasks preferring current LLC and each of the other LLC.
The reason to introduce LLC index is because the per LLC-scope data
is needed to do cache aware load balancing. However, the native lld_id
is usually the first CPU of that LLC domain, which is not continuous,
which might waste the space if the per LLC-scope data is stored
in an array (in current implementation).
In the future, this LLC index could be removed after
the native llc_id is used as the key to search into xarray based
array.
Signed-off-by: Tim Chen <tim.c.chen@...ux.intel.com>
---
include/linux/sched.h | 3 +++
kernel/sched/fair.c | 12 ++++++++++++
kernel/sched/sched.h | 2 ++
kernel/sched/topology.c | 29 +++++++++++++++++++++++++++++
4 files changed, 46 insertions(+)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 02ff8b8be25b..81d92e8097f5 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -809,6 +809,9 @@ struct kmap_ctrl {
#endif
};
+/* XXX need fix to not use magic number */
+#define MAX_LLC 64
+
struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
/*
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3128dbcf0a36..f5075d287c51 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1183,6 +1183,18 @@ static int llc_id(int cpu)
return per_cpu(sd_llc_id, cpu);
}
+/*
+ * continuous index.
+ * TBD: replace by xarray with key llc_id()
+ */
+static inline int llc_idx(int cpu)
+{
+ if (cpu < 0)
+ return -1;
+
+ return per_cpu(sd_llc_idx, cpu);
+}
+
void mm_init_sched(struct mm_struct *mm, struct mm_sched __percpu *_pcpu_sched)
{
unsigned long epoch;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 83552aab74fb..c37c74dfce25 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2056,6 +2056,7 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);
DECLARE_PER_CPU(int, sd_llc_size);
DECLARE_PER_CPU(int, sd_llc_id);
+DECLARE_PER_CPU(int, sd_llc_idx);
DECLARE_PER_CPU(int, sd_share_id);
DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
@@ -2064,6 +2065,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
extern struct static_key_false sched_asym_cpucapacity;
extern struct static_key_false sched_cluster_active;
+extern int max_llcs;
static __always_inline bool sched_asym_cpucap_active(void)
{
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index b958fe48e020..91a2b7f65fee 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -657,6 +657,7 @@ static void destroy_sched_domains(struct sched_domain *sd)
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
DEFINE_PER_CPU(int, sd_llc_size);
DEFINE_PER_CPU(int, sd_llc_id);
+DEFINE_PER_CPU(int, sd_llc_idx);
DEFINE_PER_CPU(int, sd_share_id);
DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
@@ -666,6 +667,25 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
DEFINE_STATIC_KEY_FALSE(sched_cluster_active);
+int max_llcs = -1;
+
+static void update_llc_idx(int cpu)
+{
+#ifdef CONFIG_SCHED_CACHE
+ int idx = -1, llc_id = -1;
+
+ llc_id = per_cpu(sd_llc_id, cpu);
+ idx = per_cpu(sd_llc_idx, llc_id);
+
+ if (idx < 0) {
+ idx = max_llcs++;
+ BUG_ON(idx > MAX_LLC);
+ per_cpu(sd_llc_idx, llc_id) = idx;
+ }
+ per_cpu(sd_llc_idx, cpu) = idx;
+#endif
+}
+
static void update_top_cache_domain(int cpu)
{
struct sched_domain_shared *sds = NULL;
@@ -684,6 +704,7 @@ static void update_top_cache_domain(int cpu)
per_cpu(sd_llc_size, cpu) = size;
per_cpu(sd_llc_id, cpu) = id;
rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
+ update_llc_idx(cpu);
sd = lowest_flag_domain(cpu, SD_CLUSTER);
if (sd)
@@ -2456,6 +2477,14 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
bool has_asym = false;
bool has_cluster = false;
+#ifdef CONFIG_SCHED_CACHE
+ if (max_llcs < 0) {
+ for_each_possible_cpu(i)
+ per_cpu(sd_llc_idx, i) = -1;
+ max_llcs = 0;
+ }
+#endif
+
if (WARN_ON(cpumask_empty(cpu_map)))
goto error;
--
2.25.1
Powered by blists - more mailing lists