| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20231220001856.3710363-23-jstultz@google.com>
Date: Tue, 19 Dec 2023 16:18:33 -0800
From: John Stultz <jstultz@...gle.com>
To: LKML <linux-kernel@...r.kernel.org>
Cc: John Stultz <jstultz@...gle.com>, Joel Fernandes <joelaf@...gle.com>,
Qais Yousef <qyousef@...gle.com>, Ingo Molnar <mingo@...hat.com>,
Peter Zijlstra <peterz@...radead.org>, Juri Lelli <juri.lelli@...hat.com>,
Vincent Guittot <vincent.guittot@...aro.org>, Dietmar Eggemann <dietmar.eggemann@....com>,
Valentin Schneider <vschneid@...hat.com>, Steven Rostedt <rostedt@...dmis.org>,
Ben Segall <bsegall@...gle.com>, Zimuzo Ezeozue <zezeozue@...gle.com>,
Youssef Esmat <youssefesmat@...gle.com>, Mel Gorman <mgorman@...e.de>,
Daniel Bristot de Oliveira <bristot@...hat.com>, Will Deacon <will@...nel.org>, Waiman Long <longman@...hat.com>,
Boqun Feng <boqun.feng@...il.com>, "Paul E. McKenney" <paulmck@...nel.org>,
Metin Kaya <Metin.Kaya@....com>, Xuewen Yan <xuewen.yan94@...il.com>,
K Prateek Nayak <kprateek.nayak@....com>, Thomas Gleixner <tglx@...utronix.de>, kernel-team@...roid.com
Subject: [PATCH v7 22/23] sched: Refactor dl/rt find_lowest/latest_rq logic
This pulls re-validation logic done in find_lowest_rq
and find_latest_rq after re-acquiring the rq locks out into its
own function.
This allows us to later use a more complicated validation
check for chain-migration when using proxy-exectuion.
TODO: It seems likely we could consolidate this two functions
further and leave the task_is_rt()/task_is_dl() checks externally?
Cc: Joel Fernandes <joelaf@...gle.com>
Cc: Qais Yousef <qyousef@...gle.com>
Cc: Ingo Molnar <mingo@...hat.com>
Cc: Peter Zijlstra <peterz@...radead.org>
Cc: Juri Lelli <juri.lelli@...hat.com>
Cc: Vincent Guittot <vincent.guittot@...aro.org>
Cc: Dietmar Eggemann <dietmar.eggemann@....com>
Cc: Valentin Schneider <vschneid@...hat.com>
Cc: Steven Rostedt <rostedt@...dmis.org>
Cc: Ben Segall <bsegall@...gle.com>
Cc: Zimuzo Ezeozue <zezeozue@...gle.com>
Cc: Youssef Esmat <youssefesmat@...gle.com>
Cc: Mel Gorman <mgorman@...e.de>
Cc: Daniel Bristot de Oliveira <bristot@...hat.com>
Cc: Will Deacon <will@...nel.org>
Cc: Waiman Long <longman@...hat.com>
Cc: Boqun Feng <boqun.feng@...il.com>
Cc: "Paul E. McKenney" <paulmck@...nel.org>
Cc: Metin Kaya <Metin.Kaya@....com>
Cc: Xuewen Yan <xuewen.yan94@...il.com>
Cc: K Prateek Nayak <kprateek.nayak@....com>
Cc: Thomas Gleixner <tglx@...utronix.de>
Cc: kernel-team@...roid.com
Signed-off-by: John Stultz <jstultz@...gle.com>
---
kernel/sched/deadline.c | 31 ++++++++++++++++++++-----
kernel/sched/rt.c | 50 ++++++++++++++++++++++++++++-------------
2 files changed, 59 insertions(+), 22 deletions(-)
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 21e56ac58e32..8b5701727342 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -2172,6 +2172,30 @@ static int find_later_rq(struct task_struct *sched_ctx, struct task_struct *exec
return -1;
}
+static inline bool dl_revalidate_rq_state(struct task_struct *task, struct rq *rq,
+ struct rq *later)
+{
+ if (task_rq(task) != rq)
+ return false;
+
+ if (!cpumask_test_cpu(later->cpu, &task->cpus_mask))
+ return false;
+
+ if (task_on_cpu(rq, task))
+ return false;
+
+ if (!dl_task(task))
+ return false;
+
+ if (is_migration_disabled(task))
+ return false;
+
+ if (!task_on_rq_queued(task))
+ return false;
+
+ return true;
+}
+
/* Locks the rq it finds */
static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)
{
@@ -2204,12 +2228,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)
/* Retry if something changed. */
if (double_lock_balance(rq, later_rq)) {
- if (unlikely(task_rq(task) != rq ||
- !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) ||
- task_on_cpu(rq, task) ||
- !dl_task(task) ||
- is_migration_disabled(task) ||
- !task_on_rq_queued(task))) {
+ if (unlikely(!dl_revalidate_rq_state(task, rq, later_rq))) {
double_unlock_balance(rq, later_rq);
later_rq = NULL;
break;
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index f8134d062fa3..fabb19891e95 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1935,6 +1935,39 @@ static int find_lowest_rq(struct task_struct *sched_ctx, struct task_struct *exe
return -1;
}
+static inline bool rt_revalidate_rq_state(struct task_struct *task, struct rq *rq,
+ struct rq *lowest)
+{
+ /*
+ * We had to unlock the run queue. In
+ * the mean time, task could have
+ * migrated already or had its affinity changed.
+ * Also make sure that it wasn't scheduled on its rq.
+ * It is possible the task was scheduled, set
+ * "migrate_disabled" and then got preempted, so we must
+ * check the task migration disable flag here too.
+ */
+ if (task_rq(task) != rq)
+ return false;
+
+ if (!cpumask_test_cpu(lowest->cpu, &task->cpus_mask))
+ return false;
+
+ if (task_on_cpu(rq, task))
+ return false;
+
+ if (!rt_task(task))
+ return false;
+
+ if (is_migration_disabled(task))
+ return false;
+
+ if (!task_on_rq_queued(task))
+ return false;
+
+ return true;
+}
+
/* Will lock the rq it finds */
static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
{
@@ -1964,22 +1997,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
/* if the prio of this runqueue changed, try again */
if (double_lock_balance(rq, lowest_rq)) {
- /*
- * We had to unlock the run queue. In
- * the mean time, task could have
- * migrated already or had its affinity changed.
- * Also make sure that it wasn't scheduled on its rq.
- * It is possible the task was scheduled, set
- * "migrate_disabled" and then got preempted, so we must
- * check the task migration disable flag here too.
- */
- if (unlikely(task_rq(task) != rq ||
- !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) ||
- task_on_cpu(rq, task) ||
- !rt_task(task) ||
- is_migration_disabled(task) ||
- !task_on_rq_queued(task))) {
-
+ if (unlikely(!rt_revalidate_rq_state(task, rq, lowest_rq))) {
double_unlock_balance(rq, lowest_rq);
lowest_rq = NULL;
break;
--
2.43.0.472.g3155946c3a-goog
Powered by blists - more mailing lists