linux-kernel - [PATCH 6/9] sched: Combine the last put_prev_task() and the first set_next

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <20240813224016.158454756@infradead.org>
Date: Wed, 14 Aug 2024 00:25:54 +0200
From: Peter Zijlstra <peterz@...radead.org>
To: mingo@...nel.org,
 tj@...nel.org,
 void@...ifault.com
Cc: peterz@...radead.org,
 juri.lelli@...hat.com,
 vincent.guittot@...aro.org,
 dietmar.eggemann@....com,
 rostedt@...dmis.org,
 bsegall@...gle.com,
 mgorman@...e.de,
 vschneid@...hat.com,
 linux-kernel@...r.kernel.org,
 joelaf@...gle.com
Subject: [PATCH 6/9] sched: Combine the last put_prev_task() and the first set_next_task()

Ensure the last put_prev_task() and the first set_next_task() always
go together.

Signed-off-by: Peter Zijlstra (Intel) <peterz@...radead.org>
---
 kernel/sched/core.c  |   17 ++++-------------
 kernel/sched/fair.c  |    3 +--
 kernel/sched/sched.h |   10 +++++++++-
 3 files changed, 14 insertions(+), 16 deletions(-)

--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5894,8 +5894,7 @@ __pick_next_task(struct rq *rq, struct t
 		/* Assume the next prioritized class is idle_sched_class */
 		if (!p) {
 			p = pick_task_idle(rq);
-			put_prev_task(rq, prev);
-			set_next_task_first(rq, p);
+			put_prev_set_next_task(rq, prev, p);
 		}
 
 		/*
@@ -5926,8 +5925,7 @@ __pick_next_task(struct rq *rq, struct t
 		} else {
 			p = class->pick_task(rq);
 			if (p) {
-				put_prev_task(rq, prev);
-				set_next_task_first(rq, p);
+				put_prev_set_next_task(rq, prev, p);
 				return p;
 			}
 		}
@@ -6016,13 +6014,8 @@ pick_next_task(struct rq *rq, struct tas
 		WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);
 
 		next = rq->core_pick;
-		if (next != prev) {
-			put_prev_task(rq, prev);
-			set_next_task_first(rq, next);
-		}
-
 		rq->core_pick = NULL;
-		goto out;
+		goto out_set_next;
 	}
 
 	prev_balance(rq, prev, rf);
@@ -6192,9 +6185,7 @@ pick_next_task(struct rq *rq, struct tas
 	}
 
 out_set_next:
-	put_prev_task(rq, prev);
-	set_next_task_first(rq, next);
-out:
+	put_prev_set_next_task(rq, prev, next);
 	if (rq->core->core_forceidle_count && next == rq->idle)
 		queue_core_balance(rq);
 
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -8817,8 +8817,7 @@ pick_next_task_fair(struct rq *rq, struc
 
 simple:
 #endif
-	put_prev_task(rq, prev);
-	set_next_task_fair(rq, p, true);
+	put_prev_set_next_task(rq, prev, p);
 	return p;
 
 idle:
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2370,8 +2370,16 @@ static inline void set_next_task(struct
 	next->sched_class->set_next_task(rq, next, false);
 }
 
-static inline void set_next_task_first(struct rq *rq, struct task_struct *next)
+static inline void put_prev_set_next_task(struct rq *rq,
+					  struct task_struct *prev,
+					  struct task_struct *next)
 {
+	WARN_ON_ONCE(rq->curr != prev);
+
+	if (next == prev)
+		return;
+
+	prev->sched_class->put_prev_task(rq, prev);
 	next->sched_class->set_next_task(rq, next, true);
 }