Message-Id: <DF9J45056GT3.9VDAT91SZW7T@etsalapatis.com>
Date: Sat, 27 Dec 2025 22:28:11 -0500
From: "Emil Tsalapatis" <emil@...alapatis.com>
To: "Andrea Righi" <arighi@...dia.com>, "Tejun Heo" <tj@...nel.org>, "David
 Vernet" <void@...ifault.com>, "Changwoo Min" <changwoo@...lia.com>
Cc: "Daniel Hodges" <hodgesd@...a.com>, <sched-ext@...ts.linux.dev>,
 <linux-kernel@...r.kernel.org>
Subject: Re: [PATCH 2/2] selftests/sched_ext: Add test to validate
 ops.dequeue()

On Fri Dec 19, 2025 at 5:43 PM EST, Andrea Righi wrote:
> Add a kselftest to validate ops.dequeue() semantics with direct
> dispatch, user DSQ dispatch, affinity changes and verify that any
> ops.enqueue() is balanced by a corresponding ops.dequeue().
>
> Cc: Emil Tsalapatis <emil@...alapatis.com>
> Signed-off-by: Andrea Righi <arighi@...dia.com>
> ---

Reviewed-by: Emil Tsalapatis <emil@...alapatis.com>

Hi Andrea,

>  tools/testing/selftests/sched_ext/Makefile    |   1 +
>  .../testing/selftests/sched_ext/dequeue.bpf.c | 139 ++++++++++++++
>  tools/testing/selftests/sched_ext/dequeue.c   | 172 ++++++++++++++++++
>  3 files changed, 312 insertions(+)
>  create mode 100644 tools/testing/selftests/sched_ext/dequeue.bpf.c
>  create mode 100644 tools/testing/selftests/sched_ext/dequeue.c
>
> diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
> index 5fe45f9c5f8fd..764e91edabf93 100644
> --- a/tools/testing/selftests/sched_ext/Makefile
> +++ b/tools/testing/selftests/sched_ext/Makefile
> @@ -161,6 +161,7 @@ all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubs
>  
>  auto-test-targets :=			\
>  	create_dsq			\
> +	dequeue				\
>  	enq_last_no_enq_fails		\
>  	ddsp_bogus_dsq_fail		\
>  	ddsp_vtimelocal_fail		\
> diff --git a/tools/testing/selftests/sched_ext/dequeue.bpf.c b/tools/testing/selftests/sched_ext/dequeue.bpf.c
> new file mode 100644
> index 0000000000000..aae19bfd42f04
> --- /dev/null
> +++ b/tools/testing/selftests/sched_ext/dequeue.bpf.c
> @@ -0,0 +1,139 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * A scheduler that validates ops.dequeue() is called correctly:
> + * - For tasks on BPF data structures (not yet dispatched)
> + * - For tasks already on DSQs (local or shared)
> + * - That every ops.enqueue() is followed by ops.dequeue()
> + *
> + * Copyright (c) 2025 NVIDIA Corporation.
> + */
> +
> +#include <scx/common.bpf.h>
> +
> +#define SHARED_DSQ	0
> +
> +char _license[] SEC("license") = "GPL";
> +
> +UEI_DEFINE(uei);
> +
> +/*
> + * Counters to track the lifecycle of tasks:
> + * - enqueue_cnt: Number of times ops.enqueue() was called
> + * - dequeue_cnt: Number of times ops.dequeue() was called
> + */
> +u64 enqueue_cnt, dequeue_cnt;
> +
> +/*
> + * Test scenarios:
> + * - 0: Dispatch to local DSQ
> + * - 1: Dispatch to shared DSQ
> + */
> +u32 test_scenario;
> +
> +/* Per-task state */
> +struct task_ctx {
> +	u64 enqueued;	/* was this task enqueued? */

Do we use this for anything? If not, can we remove it?
We can also just do an extra sanity check with it during
enqueues/dequeues to ensure we have no double operations.


> +};
> +
> +struct {
> +	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
> +	__uint(map_flags, BPF_F_NO_PREALLOC);
> +	__type(key, int);
> +	__type(value, struct task_ctx);
> +} task_ctx_stor SEC(".maps");
> +
> +static struct task_ctx *try_lookup_task_ctx(struct task_struct *p)
> +{
> +	return bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
> +}
> +
> +s32 BPF_STRUCT_OPS(dequeue_select_cpu, struct task_struct *p,
> +		   s32 prev_cpu, u64 wake_flags)
> +{
> +	/* Always bounce to ops.enqueue() */
> +	return prev_cpu;
> +}
> +
> +void BPF_STRUCT_OPS(dequeue_enqueue, struct task_struct *p, u64 enq_flags)
> +{
> +	struct task_ctx *tctx;
> +
> +	__sync_fetch_and_add(&enqueue_cnt, 1);
> +
> +	tctx = try_lookup_task_ctx(p);
> +	if (!tctx)
> +		return;
> +
> +	tctx->enqueued = 1;
> +
> +	switch (test_scenario) {
> +	case 0:
> +		/* Scenario 0: Direct dispatch to the local DSQ */
> +		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
> +		break;
> +
> +	case 1:
> +		/* Scenario 1: Dispatch to shared DSQ */
> +		scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags);
> +		break;
> +	}
> +}
> +
> +void BPF_STRUCT_OPS(dequeue_dequeue, struct task_struct *p, u64 deq_flags)
> +{
> +	struct task_ctx *tctx;
> +
> +	__sync_fetch_and_add(&dequeue_cnt, 1);
> +
> +	tctx = try_lookup_task_ctx(p);
> +	if (!tctx)
> +		return;
> +
> +	tctx->enqueued = 0;
> +}
> +
> +void BPF_STRUCT_OPS(dequeue_dispatch, s32 cpu, struct task_struct *prev)
> +{
> +	scx_bpf_dsq_move_to_local(SHARED_DSQ);
> +}
> +
> +s32 BPF_STRUCT_OPS(dequeue_init_task, struct task_struct *p,
> +		   struct scx_init_task_args *args)
> +{
> +	struct task_ctx *tctx;
> +
> +	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0,
> +				   BPF_LOCAL_STORAGE_GET_F_CREATE);
> +	if (!tctx)
> +		return -ENOMEM;
> +
> +	return 0;
> +}
> +
> +s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_init)
> +{
> +	s32 ret;
> +
> +	ret = scx_bpf_create_dsq(SHARED_DSQ, -1);
> +	if (ret)
> +		return ret;
> +
> +	return 0;
> +}
> +
> +void BPF_STRUCT_OPS(dequeue_exit, struct scx_exit_info *ei)
> +{
> +	UEI_RECORD(uei, ei);
> +}
> +
> +SEC(".struct_ops.link")
> +struct sched_ext_ops dequeue_ops = {
> +	.select_cpu		= (void *)dequeue_select_cpu,
> +	.enqueue		= (void *)dequeue_enqueue,
> +	.dequeue		= (void *)dequeue_dequeue,
> +	.dispatch		= (void *)dequeue_dispatch,
> +	.init_task		= (void *)dequeue_init_task,
> +	.init			= (void *)dequeue_init,
> +	.exit			= (void *)dequeue_exit,
> +	.name			= "dequeue_test",
> +};
> diff --git a/tools/testing/selftests/sched_ext/dequeue.c b/tools/testing/selftests/sched_ext/dequeue.c
> new file mode 100644
> index 0000000000000..0fd6748786079
> --- /dev/null
> +++ b/tools/testing/selftests/sched_ext/dequeue.c
> @@ -0,0 +1,172 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Copyright (c) 2025 NVIDIA Corporation.
> + */
> +#define _GNU_SOURCE
> +#include <stdio.h>
> +#include <unistd.h>
> +#include <signal.h>
> +#include <bpf/bpf.h>
> +#include <scx/common.h>
> +#include <sys/wait.h>
> +#include <sched.h>
> +#include <pthread.h>
> +#include "scx_test.h"
> +#include "dequeue.bpf.skel.h"
> +
> +#define NUM_WORKERS 8
> +
> +/*
> + * Worker function that creates enqueue/dequeue events. It alternates
> + * between CPU work, sleeping, and affinity changes to trigger dequeues.
> + */
> +static void worker_fn(int id)
> +{
> +	cpu_set_t cpuset;
> +	int i;
> +	volatile int sum = 0;
> +
> +	for (i = 0; i < 1000; i++) {
> +		int j;
> +
> +		/* Do some work to trigger scheduling events */
> +		for (j = 0; j < 10000; j++)
> +			sum += j;
> +
> +		/* Change affinity to trigger dequeue */
> +		if (i % 10 == 0) {
> +			CPU_ZERO(&cpuset);
> +			/* Rotate through the first 4 CPUs */
> +			CPU_SET(i % 4, &cpuset);
> +			sched_setaffinity(0, sizeof(cpuset), &cpuset);
> +		}
> +
> +		/* Do additional work */
> +		for (j = 0; j < 10000; j++)
> +			sum += j;
> +
> +		/* Sleep to trigger dequeue */
> +		usleep(1000 + (id * 100));
> +	}
> +
> +	exit(0);
> +}
> +
> +static enum scx_test_status run_scenario(struct dequeue *skel, u32 scenario,
> +					 const char *scenario_name)
> +{
> +	struct bpf_link *link;
> +	pid_t pids[NUM_WORKERS];
> +	int i, status;
> +	u64 enq_start, deq_start;
> +	u64 enq_delta, deq_delta;
> +
> +	/* Set the test scenario */
> +	skel->bss->test_scenario = scenario;
> +
> +	/* Record starting counts */
> +	enq_start = skel->bss->enqueue_cnt;
> +	deq_start = skel->bss->dequeue_cnt;
> +
> +	link = bpf_map__attach_struct_ops(skel->maps.dequeue_ops);
> +	SCX_FAIL_IF(!link, "Failed to attach struct_ops for scenario %s", scenario_name);
> +
> +	/* Fork worker processes to generate enqueue/dequeue events */
> +	for (i = 0; i < NUM_WORKERS; i++) {
> +		pids[i] = fork();
> +		SCX_FAIL_IF(pids[i] < 0, "Failed to fork worker %d", i);
> +
> +		if (pids[i] == 0) {
> +			worker_fn(i);
> +			/* Should not reach here */
> +			exit(1);
> +		}
> +	}
> +
> +	/* Wait for all workers to complete */
> +	for (i = 0; i < NUM_WORKERS; i++) {
> +		SCX_FAIL_IF(waitpid(pids[i], &status, 0) != pids[i],
> +			    "Failed to wait for worker %d", i);
> +		SCX_FAIL_IF(status != 0, "Worker %d exited with status %d", i, status);
> +	}
> +
> +	bpf_link__destroy(link);
> +
> +	SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG));
> +
> +	/* Calculate deltas */
> +	enq_delta = skel->bss->enqueue_cnt - enq_start;
> +	deq_delta = skel->bss->dequeue_cnt - deq_start;
> +
> +	printf("%s:\n", scenario_name);
> +	printf("  enqueues: %lu\n", (unsigned long)enq_delta);
> +	printf("  dequeues: %lu\n", (unsigned long)deq_delta);
> +
> +	/* Validate that we got enqueue and dequeue events */
> +	SCX_GT(enq_delta, 0);
> +	SCX_GT(deq_delta, 0);
> +
> +	if (enq_delta > deq_delta)
> +		SCX_FAIL("Too many enqueues without dequeues: %lu enqueues, %lu dequeues",
> +			    (unsigned long)enq_delta, (unsigned long)deq_delta);
> +	else if (deq_delta > enq_delta)
> +		SCX_FAIL("More dequeues than enqueues: %lu enqueues, %lu dequeues",
> +			    (unsigned long)enq_delta, (unsigned long)deq_delta);
> +
> +	return SCX_TEST_PASS;
> +}
> +
> +static enum scx_test_status setup(void **ctx)
> +{
> +	struct dequeue *skel;
> +
> +	skel = dequeue__open();
> +	SCX_FAIL_IF(!skel, "Failed to open skel");
> +	SCX_ENUM_INIT(skel);
> +	SCX_FAIL_IF(dequeue__load(skel), "Failed to load skel");
> +
> +	*ctx = skel;
> +
> +	return SCX_TEST_PASS;
> +}
> +
> +static enum scx_test_status run(void *ctx)
> +{
> +	struct dequeue *skel = ctx;
> +	enum scx_test_status status;
> +
> +	status = run_scenario(skel, 0, "Local DSQ");
> +	if (status != SCX_TEST_PASS)
> +		return status;
> +
> +	status = run_scenario(skel, 1, "User DSQ");
> +	if (status != SCX_TEST_PASS)
> +		return status;
> +
> +	printf("\n=== Summary ===\n");
> +	printf("Total enqueues: %lu\n", (unsigned long)skel->bss->enqueue_cnt);
> +	printf("Total dequeues: %lu\n", (unsigned long)skel->bss->dequeue_cnt);
> +	printf("\nAll scenarios passed\n");
> +	printf("-> Validated: ops.dequeue() is called for tasks on local DSQ\n");
> +	printf("-> Validated: ops.dequeue() is called for tasks on user DSQ\n");
> +	printf("-> Validated: Every enqueue is balanced with dequeue or execution\n");
> +
> +	return SCX_TEST_PASS;
> +}
> +
> +static void cleanup(void *ctx)
> +{
> +	struct dequeue *skel = ctx;
> +
> +	dequeue__destroy(skel);
> +}
> +
> +struct scx_test dequeue_test = {
> +	.name = "dequeue",
> +	.description = "Verify that ops.enqueue() is balanced with ops.dequeue()",
> +	.setup = setup,
> +	.run = run,
> +	.cleanup = cleanup,
> +};
> +
> +REGISTER_SCX_TEST(&dequeue_test)

Powered by blists - more mailing lists