| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20250920005931.2753828-34-tj@kernel.org>
Date: Fri, 19 Sep 2025 14:58:56 -1000
From: Tejun Heo <tj@...nel.org>
To: void@...ifault.com,
arighi@...dia.com,
multics69@...il.com
Cc: linux-kernel@...r.kernel.org,
sched-ext@...ts.linux.dev,
memxor@...il.com,
bpf@...r.kernel.org,
Tejun Heo <tj@...nel.org>
Subject: [PATCH 33/46] sched_ext: Factor out scx_dispatch_sched()
In preparation of multiple scheduler support, factor out
scx_dispatch_sched() from balance_one(). The function boundary makes
remembering $prev_on_scx and $prev_on_rq less useful. Open code $prev_on_scx
in balance_one() and $prev_on_rq in both balance_one() and
scx_dispatch_sched().
No functional changes.
Signed-off-by: Tejun Heo <tj@...nel.org>
---
kernel/sched/ext.c | 115 ++++++++++++++++++++++++---------------------
1 file changed, 62 insertions(+), 53 deletions(-)
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index 44f9cc7f0915..efe01ba84e2d 100644
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -2178,14 +2178,68 @@ static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
dspc->cursor = 0;
}
-static int balance_one(struct rq *rq, struct task_struct *prev)
+static bool scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
+ struct task_struct *prev)
{
- struct scx_sched *sch = scx_root;
struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
bool prev_on_scx = prev->sched_class == &ext_sched_class;
- bool prev_on_rq = prev->scx.flags & SCX_TASK_QUEUED;
int nr_loops = SCX_DSP_MAX_LOOPS;
+ if (consume_global_dsq(sch, rq))
+ return true;
+
+ if (unlikely(!SCX_HAS_OP(sch, dispatch)) ||
+ scx_bypassing(sch, cpu_of(rq)) || !scx_rq_online(rq))
+ return false;
+
+ dspc->rq = rq;
+
+ /*
+ * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock,
+ * the local DSQ might still end up empty after a successful
+ * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
+ * produced some tasks, retry. The BPF scheduler may depend on this
+ * looping behavior to simplify its implementation.
+ */
+ do {
+ dspc->nr_tasks = 0;
+
+ SCX_CALL_OP(sch, SCX_KF_DISPATCH, dispatch, rq,
+ cpu_of(rq), prev_on_scx ? prev : NULL);
+
+ flush_dispatch_buf(sch, rq);
+
+ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) {
+ rq->scx.flags |= SCX_RQ_BAL_KEEP;
+ return true;
+ }
+ if (rq->scx.local_dsq.nr)
+ return true;
+ if (consume_global_dsq(sch, rq))
+ return true;
+
+ /*
+ * ops.dispatch() can trap us in this loop by repeatedly
+ * dispatching ineligible tasks. Break out once in a while to
+ * allow the watchdog to run. As IRQ can't be enabled in
+ * balance(), we want to complete this scheduling cycle and then
+ * start a new one. IOW, we want to call resched_curr() on the
+ * next, most likely idle, task, not the current one. Use
+ * __scx_bpf_kick_cpu() for deferred kicking.
+ */
+ if (unlikely(!--nr_loops)) {
+ scx_kick_cpu(sch, cpu_of(rq), 0);
+ break;
+ }
+ } while (dspc->nr_tasks);
+
+ return false;
+}
+
+static int balance_one(struct rq *rq, struct task_struct *prev)
+{
+ struct scx_sched *sch = scx_root;
+
lockdep_assert_rq_held(rq);
rq->scx.flags |= SCX_RQ_IN_BALANCE;
rq->scx.flags &= ~(SCX_RQ_BAL_PENDING | SCX_RQ_BAL_KEEP);
@@ -2204,7 +2258,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
rq->scx.cpu_released = false;
}
- if (prev_on_scx) {
+ if (prev->sched_class == &ext_sched_class) {
update_curr_scx(rq);
/*
@@ -2217,7 +2271,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
* See scx_disable_workfn() for the explanation on the bypassing
* test.
*/
- if (prev_on_rq && prev->scx.slice &&
+ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice &&
!scx_bypassing(sch, cpu_of(rq))) {
rq->scx.flags |= SCX_RQ_BAL_KEEP;
goto has_tasks;
@@ -2228,60 +2282,15 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
if (rq->scx.local_dsq.nr)
goto has_tasks;
- if (consume_global_dsq(sch, rq))
+ /* dispatch @sch */
+ if (scx_dispatch_sched(sch, rq, prev))
goto has_tasks;
- if (unlikely(!SCX_HAS_OP(sch, dispatch)) ||
- scx_bypassing(sch, cpu_of(rq)) || !scx_rq_online(rq))
- goto no_tasks;
-
- dspc->rq = rq;
-
- /*
- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock,
- * the local DSQ might still end up empty after a successful
- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
- * produced some tasks, retry. The BPF scheduler may depend on this
- * looping behavior to simplify its implementation.
- */
- do {
- dspc->nr_tasks = 0;
-
- SCX_CALL_OP(sch, SCX_KF_DISPATCH, dispatch, rq,
- cpu_of(rq), prev_on_scx ? prev : NULL);
-
- flush_dispatch_buf(sch, rq);
-
- if (prev_on_rq && prev->scx.slice) {
- rq->scx.flags |= SCX_RQ_BAL_KEEP;
- goto has_tasks;
- }
- if (rq->scx.local_dsq.nr)
- goto has_tasks;
- if (consume_global_dsq(sch, rq))
- goto has_tasks;
-
- /*
- * ops.dispatch() can trap us in this loop by repeatedly
- * dispatching ineligible tasks. Break out once in a while to
- * allow the watchdog to run. As IRQ can't be enabled in
- * balance(), we want to complete this scheduling cycle and then
- * start a new one. IOW, we want to call resched_curr() on the
- * next, most likely idle, task, not the current one. Use
- * scx_kick_cpu() for deferred kicking.
- */
- if (unlikely(!--nr_loops)) {
- scx_kick_cpu(sch, cpu_of(rq), 0);
- break;
- }
- } while (dspc->nr_tasks);
-
-no_tasks:
/*
* Didn't find another task to run. Keep running @prev unless
* %SCX_OPS_ENQ_LAST is in effect.
*/
- if (prev_on_rq &&
+ if ((prev->scx.flags & SCX_TASK_QUEUED) &&
(!(sch->ops.flags & SCX_OPS_ENQ_LAST) ||
scx_bypassing(sch, cpu_of(rq)))) {
rq->scx.flags |= SCX_RQ_BAL_KEEP;
--
2.51.0
Powered by blists - more mailing lists