linux-kernel - [RFC PATCH 10/22] sched/fair: Prepare throttle_cfs

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-ID: <20250220093257.9380-11-kprateek.nayak@amd.com>
Date: Thu, 20 Feb 2025 09:32:45 +0000
From: K Prateek Nayak <kprateek.nayak@....com>
To: Peter Zijlstra <peterz@...radead.org>, Ingo Molnar <mingo@...hat.com>,
	Juri Lelli <juri.lelli@...hat.com>, Vincent Guittot
	<vincent.guittot@...aro.org>, Valentin Schneider <vschneid@...hat.com>, "Ben
 Segall" <bsegall@...gle.com>, Thomas Gleixner <tglx@...utronix.de>, "Andy
 Lutomirski" <luto@...nel.org>, <linux-kernel@...r.kernel.org>
CC: Dietmar Eggemann <dietmar.eggemann@....com>, Steven Rostedt
	<rostedt@...dmis.org>, Mel Gorman <mgorman@...e.de>, "Sebastian Andrzej
 Siewior" <bigeasy@...utronix.de>, Clark Williams <clrkwllms@...nel.org>,
	<linux-rt-devel@...ts.linux.dev>, Tejun Heo <tj@...nel.org>, "Frederic
 Weisbecker" <frederic@...nel.org>, Barret Rhoden <brho@...gle.com>, "Petr
 Mladek" <pmladek@...e.com>, Josh Don <joshdon@...gle.com>, Qais Yousef
	<qyousef@...alina.io>, "Paul E. McKenney" <paulmck@...nel.org>, David Vernet
	<dvernet@...a.com>, K Prateek Nayak <kprateek.nayak@....com>, "Gautham R.
 Shenoy" <gautham.shenoy@....com>, Swapnil Sapkal <swapnil.sapkal@....com>
Subject: [RFC PATCH 10/22] sched/fair: Prepare throttle_cfs_rq() to allow partial throttling

throttle_cfs_rq() will determine the throttle state first time it is
called using the not yet fully implemented determine_throttle_state()
helper to completely or partially throttle a cfs_rq when it has run
out of bandwidth.

For a partial throttle, throttle_cfs_rq() will only set
cfs_rq->throttled to CFS_THROTTLED_PARTIAL, leaving the rest same. When
a partially throttled hierarchy needs to be upgraded once all the kernel
mode preempted entities yield or exit to userspace, throttle_cfs_rq()
will be called again to upgrade the throttle status.

determine_throttle_state() currently only returns CFS_THROTTLED and will
be implemented in the subsequent commits once the foundation is laid.

Signed-off-by: K Prateek Nayak <kprateek.nayak@....com>
---
 kernel/sched/fair.c | 55 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 47 insertions(+), 8 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c84cd2d92343..8e1df614e82f 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5894,6 +5894,16 @@ static inline int throttled_lb_pair(struct task_group *tg,
 	       throttled_hierarchy(dest_cfs_rq);
 }
 
+static enum throttle_state
+determine_throttle_state(struct cfs_rq *gcfs_rq, struct sched_entity *se)
+{
+	/*
+	 * TODO: Implement rest once plumbing for
+	 * CFS_THROTTLED_PARTIAL is done.
+	 */
+	return CFS_THROTTLED;
+}
+
 static int tg_unthrottle_up(struct task_group *tg, void *data)
 {
 	struct rq *rq = data;
@@ -5946,9 +5956,25 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
 {
 	struct rq *rq = rq_of(cfs_rq);
 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
-	struct sched_entity *se;
 	long queued_delta, runnable_delta, idle_delta, dequeue = 1;
+	struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
 	long rq_h_nr_queued = rq->cfs.h_nr_queued;
+	int new_state = determine_throttle_state(cfs_rq, se);
+	int prev_state = cfs_rq->throttled;
+
+	/* Fully throttled cfs_rq should not reach here */
+	SCHED_WARN_ON(cfs_rq_h_throttled(cfs_rq));
+
+	/* Nothing to do */
+	if (new_state == prev_state)
+		return false;
+
+	/*
+	 * We've been upgraded! Just dequeue since we are already on the
+	 * throttled_list. Let distribution unthrottle us.
+	 */
+	if (prev_state)
+		goto throttle_dequeue;
 
 	raw_spin_lock(&cfs_b->lock);
 	/* This will start the period timer if necessary */
@@ -5969,10 +5995,13 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
 	raw_spin_unlock(&cfs_b->lock);
 
 	if (!dequeue)
-		return false;  /* Throttle no longer required. */
+		new_state = CFS_UNTHROTTLED;
 
-	se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
+	/* If the cfs_rq is only partially throttled, we are done. */
+	if (new_state < CFS_THROTTLED)
+		goto done;
 
+throttle_dequeue:
 	/* freeze hierarchy runnable averages while throttled */
 	rcu_read_lock();
 	walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
@@ -6041,11 +6070,16 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
 	 * Note: distribution will already see us throttled via the
 	 * throttled-list.  rq->lock protects completion.
 	 */
-	cfs_rq->throttled = CFS_THROTTLED;
-	SCHED_WARN_ON(cfs_rq->throttled_clock);
-	if (cfs_rq->nr_queued)
-		cfs_rq->throttled_clock = rq_clock(rq);
-	return true;
+	cfs_rq->throttled = new_state;
+	if (new_state == CFS_THROTTLED) {
+		SCHED_WARN_ON(cfs_rq->throttled_clock);
+		if (cfs_rq->nr_queued)
+			cfs_rq->throttled_clock = rq_clock(rq);
+
+		return true;
+	}
+
+	return false;
 }
 
 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
@@ -6536,6 +6570,11 @@ static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
 	if (cfs_rq_h_throttled(cfs_rq))
 		return true;
 
+	/*
+	 * throttle_cfs_rq() will reevaluate the throttle status for
+	 * partially throttled hierarchy and upgrade to a full throttle
+	 * once all the kernel mode entities leave or exit to user mode.
+	 */
 	return throttle_cfs_rq(cfs_rq);
 }
 
-- 
2.43.0