linux-kernel - [RFC PATCH 14/22] sched/fair: Mark a task if it was picked on a partially throttled hierarchy

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-ID: <20250220093257.9380-15-kprateek.nayak@amd.com>
Date: Thu, 20 Feb 2025 09:32:49 +0000
From: K Prateek Nayak <kprateek.nayak@....com>
To: Peter Zijlstra <peterz@...radead.org>, Ingo Molnar <mingo@...hat.com>,
	Juri Lelli <juri.lelli@...hat.com>, Vincent Guittot
	<vincent.guittot@...aro.org>, Valentin Schneider <vschneid@...hat.com>, "Ben
 Segall" <bsegall@...gle.com>, Thomas Gleixner <tglx@...utronix.de>, "Andy
 Lutomirski" <luto@...nel.org>, <linux-kernel@...r.kernel.org>
CC: Dietmar Eggemann <dietmar.eggemann@....com>, Steven Rostedt
	<rostedt@...dmis.org>, Mel Gorman <mgorman@...e.de>, "Sebastian Andrzej
 Siewior" <bigeasy@...utronix.de>, Clark Williams <clrkwllms@...nel.org>,
	<linux-rt-devel@...ts.linux.dev>, Tejun Heo <tj@...nel.org>, "Frederic
 Weisbecker" <frederic@...nel.org>, Barret Rhoden <brho@...gle.com>, "Petr
 Mladek" <pmladek@...e.com>, Josh Don <joshdon@...gle.com>, Qais Yousef
	<qyousef@...alina.io>, "Paul E. McKenney" <paulmck@...nel.org>, David Vernet
	<dvernet@...a.com>, K Prateek Nayak <kprateek.nayak@....com>, "Gautham R.
 Shenoy" <gautham.shenoy@....com>, Swapnil Sapkal <swapnil.sapkal@....com>
Subject: [RFC PATCH 14/22] sched/fair: Mark a task if it was picked on a partially throttled hierarchy

With partial throttle, the hierarchical throttle indicator, namely
"cfs_rq->throttle_count" will not be set. To detect a task picked on a
partially throttled hierarchy, set a "sched_throttled" indicator in its
sched_entity to trigger a schedule() when "kernel_cs_count" hits 0.

Signed-off-by: K Prateek Nayak <kprateek.nayak@....com>
---
 include/linux/sched.h |  3 +++
 kernel/sched/fair.c   | 30 +++++++++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 48115de839a7..200cc086e121 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -597,6 +597,9 @@ struct sched_entity {
 	 *   is accounted at key decision points.
 	 */
 	int				kernel_cs_count;
+
+	/* Entity picked on a throttled hierarchy */
+	unsigned char			sched_throttled;
 					/* hole */
 
 	/*
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 1d871509b246..68c194169c00 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6856,6 +6856,17 @@ static __always_inline int se_in_kernel(struct sched_entity *se)
 	return se->kernel_cs_count;
 }
 
+/* se picked on a partially throttled hierarchy. */
+static inline void task_mark_throttled(struct task_struct *p)
+{
+	p->se.sched_throttled = 1;
+}
+
+static inline void task_clear_throttled(struct task_struct *p)
+{
+	p->se.sched_throttled = 0;
+}
+
 /*
  * Same as avg_vruntime_add() except avg_kcs_vruntime_add() only adjusts the avg_kcs_vruntime
  * and avg_kcs_load of kernel mode preempted entity when it joins the rbtree.
@@ -7171,6 +7182,8 @@ static __always_inline int se_in_kernel(struct sched_entity *se)
 	return false;
 }
 
+static __always_inline void task_mark_throttled(struct task_struct *p) {}
+static __always_inline void task_clear_throttled(struct task_struct *p) {}
 static __always_inline void avg_kcs_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
 static __always_inline void avg_kcs_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
 static __always_inline void avg_kcs_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) {}
@@ -9281,6 +9294,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int
 
 static struct task_struct *pick_task_fair(struct rq *rq)
 {
+	struct task_struct *next;
 	struct sched_entity *se;
 	struct cfs_rq *cfs_rq;
 	bool h_throttled;
@@ -9307,7 +9321,11 @@ static struct task_struct *pick_task_fair(struct rq *rq)
 		cfs_rq = group_cfs_rq(se);
 	} while (cfs_rq);
 
-	return task_of(se);
+	next = task_of(se);
+	if (h_throttled)
+		task_mark_throttled(next);
+
+	return next;
 }
 
 static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first);
@@ -9349,6 +9367,8 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf
 		bool next_in_kernel = se_in_kernel(se);
 		struct cfs_rq *cfs_rq;
 
+		task_clear_throttled(prev);
+
 		while (!(cfs_rq = is_same_group(se, pse))) {
 			int se_depth = se->depth;
 			int pse_depth = pse->depth;
@@ -9457,6 +9477,14 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t
 	bool task_in_kernel = next && task_on_rq_queued(prev) && se_in_kernel(se);
 	struct cfs_rq *cfs_rq;
 
+	/*
+	 * Clear the pick on throttled indicator only if
+	 * another task was picked and not for a save /
+	 * restore operation for the task.
+	 */
+	if (next)
+		task_clear_throttled(prev);
+
 	for_each_sched_entity(se) {
 		cfs_rq = cfs_rq_of(se);
 		account_kcs_enqueue(cfs_rq, task_in_kernel);
-- 
2.43.0