[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-ID: <aN4mH-t8xISo6uKr@gpd4>
Date: Thu, 2 Oct 2025 09:13:35 +0200
From: Andrea Righi <arighi@...dia.com>
To: Ryan Newton <rrnewton@...il.com>
Cc: linux-kernel@...r.kernel.org, tj@...nel.org, newton@...a.com
Subject: Re: [PATCH 1/3] sched_ext: Add lockless peek operation for DSQs
Hi Ryan,
On Wed, Oct 01, 2025 at 10:57:19PM -0400, Ryan Newton wrote:
> From: Ryan Newton <newton@...a.com>
>
> The builtin DSQ queue data structures are meant to be used by a wide
> range of different sched_ext schedulers with different demands on these
> data structures. They might be per-cpu with low-contention, or
> high-contention shared queues. Unfortunately, DSQs have a coarse-grained
> lock around the whole data structure. Without going all the way to a
> lock-free, more scalable implementation, a small step we can take to
> reduce lock contention is to allow a lockless, small-fixed-cost peek at
> the head of the queue.
>
> This change allows certain custom SCX schedulers to cheaply peek at
> queues, e.g. during load balancing, before locking them. But it
> represents a few extra memory operations to update the pointer each
> time the DSQ is modified, including a memory barrier on ARM so the write
> appears correctly ordered.
>
> This commit adds a first_task pointer field which is updated
> atomically when the DSQ is modified, and allows any thread to peek at
> the head of the queue without holding the lock.
>
> Signed-off-by: Ryan Newton <newton@...a.com>
> ---
> include/linux/sched/ext.h | 1 +
> kernel/sched/ext.c | 37 ++++++++++++++++++++++++
> tools/sched_ext/include/scx/common.bpf.h | 1 +
> tools/sched_ext/include/scx/compat.bpf.h | 19 ++++++++++++
> 4 files changed, 58 insertions(+)
>
> diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
> index d82b7a9b0658..81478d4ae782 100644
> --- a/include/linux/sched/ext.h
> +++ b/include/linux/sched/ext.h
> @@ -58,6 +58,7 @@ enum scx_dsq_id_flags {
> */
> struct scx_dispatch_q {
> raw_spinlock_t lock;
> + struct task_struct __rcu *first_task; /* lockless peek at head */
> struct list_head list; /* tasks in dispatch order */
> struct rb_root priq; /* used to order by p->scx.dsq_vtime */
> u32 nr;
> diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
> index 2b0e88206d07..fd0121c03311 100644
> --- a/kernel/sched/ext.c
> +++ b/kernel/sched/ext.c
> @@ -885,6 +885,15 @@ static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)
> __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
> }
>
> +/* while holding dsq->lock */
> +static void dsq_update_first_task(struct scx_dispatch_q *dsq)
> +{
> + struct task_struct *first_task;
> +
> + first_task = nldsq_next_task(dsq, NULL, false);
This requires holding dsq->lock, but...
> + rcu_assign_pointer(dsq->first_task, first_task);
> +}
> +
> static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
> struct task_struct *p, u64 enq_flags)
> {
> @@ -959,6 +968,9 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
> list_add_tail(&p->scx.dsq_list.node, &dsq->list);
> }
>
> + /* even the add_tail code path may have changed the first element */
> + dsq_update_first_task(dsq);
...we're not holding dsq->lock here when dsq->id == SCX_DSQ_LOCAL.
This seems to be fixed in PATCH 2/3, so I'd suggest squashing that change
into this one to avoid triggering lockdep warnings during bisect.
Moreover, I think splitting them doesn't add much value for the review, so
merging both makes more sense to me.
> +
> /* seq records the order tasks are queued, used by BPF DSQ iterator */
> dsq->seq++;
> p->scx.dsq_seq = dsq->seq;
> @@ -1013,6 +1025,7 @@ static void task_unlink_from_dsq(struct task_struct *p,
>
> list_del_init(&p->scx.dsq_list.node);
> dsq_mod_nr(dsq, -1);
> + dsq_update_first_task(dsq);
> }
>
> static void dispatch_dequeue(struct rq *rq, struct task_struct *p)
> @@ -6084,6 +6097,29 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
> kit->dsq = NULL;
> }
>
> +/**
> + * scx_bpf_dsq_peek - Lockless peek at the first element.
> + * @dsq_id: DSQ to examine.
> + *
> + * Read the first element in the DSQ. This is semantically equivalent to using
> + * the DSQ iterator, but is lockfree.
> + *
> + * Returns the pointer, or uses ERR_PTR() to encode an error as the pointer.
> + */
> +__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id)
> +{
> + struct scx_sched *sch;
> + struct scx_dispatch_q *dsq;
> +
> + /* KF_RCU_PROTECTED means no need to guard(rcu)() */
I think this comment can be dropped, the meaning should already be clear
from the KF_RCU_PROTECTED annotation.
> + sch = rcu_dereference(scx_root);
> +
> + if (unlikely(!sch))
> + return ERR_PTR(-ENODEV);
I'm wondering if we should just return NULL here, to simplify error
handling in the caller. In this way we just need to check for (p != NULL),
instead of (p != NULL) && !IS_ERR(p).
> + dsq = find_user_dsq(sch, dsq_id);
Hm.. let's do something like this:
if (unlikely(!dsq)) {
scx_error(sch, "non-existent DSQ ID 0x%016llx", dsq_id);
return NULL;
}
or bad things can happen. :)
This also implies that scx_bpf_dsq_peek() works only with user DSQs and
will always return NULL for built-in DSQs.
So, what about adding also:
if (unlikely((dsq_id & SCX_DSQ_FLAG_BUILTIN))) {
scx_error(sch, "invalid DSQ ID 0x%016llx (only user DSQs allowed)", dsq_id);
return NULL;
}
I think this would be a reasonable requirement. Typically once a task is
queued to a built-in DSQ (e.g., SCX_DSQ_LOCAL[_ON]), it is effectively
considered dispatched, so peeking at its state is rarely needed.
This would also allows us to get rid of updating the first_task pointer
with built-in DSQs, saving some memory ops in the hot paths.
> + return rcu_dereference(dsq->first_task);
> +}
> +
> __bpf_kfunc_end_defs();
>
> static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,
> @@ -6641,6 +6677,7 @@ BTF_KFUNCS_START(scx_kfunc_ids_any)
> BTF_ID_FLAGS(func, scx_bpf_kick_cpu)
> BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)
> BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)
> +BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_RCU_PROTECTED | KF_RET_NULL)
> BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_ITER_NEW | KF_RCU_PROTECTED)
> BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
> BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
> diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h
> index 06e2551033cb..fbf3e7f9526c 100644
> --- a/tools/sched_ext/include/scx/common.bpf.h
> +++ b/tools/sched_ext/include/scx/common.bpf.h
> @@ -75,6 +75,7 @@ u32 scx_bpf_reenqueue_local(void) __ksym;
> void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
> s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym;
> void scx_bpf_destroy_dsq(u64 dsq_id) __ksym;
> +struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak;
> int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak;
> struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak;
> void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak;
> diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h
> index dd9144624dc9..0af1922d66a8 100644
> --- a/tools/sched_ext/include/scx/compat.bpf.h
> +++ b/tools/sched_ext/include/scx/compat.bpf.h
> @@ -130,6 +130,25 @@ int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym
> false; \
> })
>
> +
> +/*
> + * v6.19: Introduce lockless peek API for user DSQs.
> + *
> + * Preserve the following macro until v6.20.
Usually we keep the __COMPAT_*() helpers for 2 major kernel versions, maybe
let's bump this to v6.21.
> + */
> +static inline struct task_struct *__COMPAT_scx_bpf_dsq_peek(u64 dsq_id)
> +{
> + struct task_struct *p = NULL;
> + struct bpf_iter_scx_dsq it;
> +
> + if (bpf_ksym_exists(scx_bpf_dsq_peek))
> + return scx_bpf_dsq_peek(dsq_id);
> + if (!bpf_iter_scx_dsq_new(&it, dsq_id, 0))
> + p = bpf_iter_scx_dsq_next(&it);
> + bpf_iter_scx_dsq_destroy(&it);
> + return p;
> +}
> +
> /**
> * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on
> * in a compatible way. We will preserve this __COMPAT helper until v6.16.
> --
> 2.51.0
>
Thanks,
-Andrea
Powered by blists - more mailing lists