[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <20250530080355.1138759-4-yukuai1@huaweicloud.com>
Date: Fri, 30 May 2025 16:03:54 +0800
From: Yu Kuai <yukuai1@...weicloud.com>
To: axboe@...nel.dk
Cc: linux-block@...r.kernel.org,
linux-kernel@...r.kernel.org,
yukuai3@...wei.com,
yukuai1@...weicloud.com,
yi.zhang@...wei.com,
yangerkun@...wei.com,
johnny.chenyi@...wei.com
Subject: [PATCH RFC 3/4] blk-mq-sched: refactor __blk_mq_do_dispatch_sched()
From: Yu Kuai <yukuai3@...wei.com>
Introduce struct sched_dispatch_ctx(), and split it into
elevator_dispatch_one_request() and elevator_finish_dispatch(). Make
code cleaner and prepare to support request batch dispatching.
Signed-off-by: Yu Kuai <yukuai3@...wei.com>
---
block/blk-mq-sched.c | 181 ++++++++++++++++++++++++++-----------------
1 file changed, 109 insertions(+), 72 deletions(-)
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index c1390d3e6381..990d0f19594a 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -74,85 +74,88 @@ static bool blk_mq_dispatch_hctx_list(struct list_head *rq_list)
#define BLK_MQ_BUDGET_DELAY 3 /* ms units */
-/*
- * Only SCSI implements .get_budget and .put_budget, and SCSI restarts
- * its queue by itself in its completion handler, so we don't need to
- * restart queue if .get_budget() fails to get the budget.
- *
- * Returns -EAGAIN if hctx->dispatch was found non-empty and run_work has to
- * be run again. This is necessary to avoid starving flushes.
- */
-static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
-{
- struct request_queue *q = hctx->queue;
- struct elevator_queue *e = q->elevator;
- bool multi_hctxs = false, run_queue = false;
- bool dispatched = false, busy = false;
- unsigned int max_dispatch;
- LIST_HEAD(rq_list);
- int count = 0;
+struct sched_dispatch_ctx {
+ struct blk_mq_hw_ctx *hctx;
+ struct elevator_queue *e;
+ struct request_queue *q;
- if (hctx->dispatch_busy)
- max_dispatch = 1;
- else
- max_dispatch = hctx->queue->nr_requests;
+ struct list_head rq_list;
+ int count;
- do {
- struct request *rq;
- int budget_token;
+ bool multi_hctxs;
+ bool run_queue;
+ bool busy;
+};
- if (e->type->ops.has_work && !e->type->ops.has_work(hctx))
- break;
+static bool elevator_can_dispatch(struct sched_dispatch_ctx *ctx)
+{
+ if (ctx->e->type->ops.has_work &&
+ !ctx->e->type->ops.has_work(ctx->hctx))
+ return false;
- if (!list_empty_careful(&hctx->dispatch)) {
- busy = true;
- break;
- }
+ if (!list_empty_careful(&ctx->hctx->dispatch)) {
+ ctx->busy = true;
+ return false;
+ }
- budget_token = blk_mq_get_dispatch_budget(q);
- if (budget_token < 0)
- break;
+ return true;
+}
- rq = elevator_dispatch_request(hctx);
- if (!rq) {
- blk_mq_put_dispatch_budget(q, budget_token);
- /*
- * We're releasing without dispatching. Holding the
- * budget could have blocked any "hctx"s with the
- * same queue and if we didn't dispatch then there's
- * no guarantee anyone will kick the queue. Kick it
- * ourselves.
- */
- run_queue = true;
- break;
- }
+static bool elevator_dispatch_one_request(struct sched_dispatch_ctx *ctx)
+{
+ struct request *rq;
+ int budget_token;
- blk_mq_set_rq_budget_token(rq, budget_token);
+ if (!elevator_can_dispatch(ctx))
+ return false;
- /*
- * Now this rq owns the budget which has to be released
- * if this rq won't be queued to driver via .queue_rq()
- * in blk_mq_dispatch_rq_list().
- */
- list_add_tail(&rq->queuelist, &rq_list);
- count++;
- if (rq->mq_hctx != hctx)
- multi_hctxs = true;
+ budget_token = blk_mq_get_dispatch_budget(ctx->q);
+ if (budget_token < 0)
+ return false;
+ rq = elevator_dispatch_request(ctx->hctx);
+ if (!rq) {
+ blk_mq_put_dispatch_budget(ctx->q, budget_token);
/*
- * If we cannot get tag for the request, stop dequeueing
- * requests from the IO scheduler. We are unlikely to be able
- * to submit them anyway and it creates false impression for
- * scheduling heuristics that the device can take more IO.
+ * We're releasing without dispatching. Holding the
+ * budget could have blocked any "hctx"s with the
+ * same queue and if we didn't dispatch then there's
+ * no guarantee anyone will kick the queue. Kick it
+ * ourselves.
*/
- if (!blk_mq_get_driver_tag(rq))
- break;
- } while (count < max_dispatch);
+ ctx->run_queue = true;
+ return false;
+ }
- if (!count) {
- if (run_queue)
- blk_mq_delay_run_hw_queues(q, BLK_MQ_BUDGET_DELAY);
- } else if (multi_hctxs) {
+ blk_mq_set_rq_budget_token(rq, budget_token);
+
+ /*
+ * Now this rq owns the budget which has to be released
+ * if this rq won't be queued to driver via .queue_rq()
+ * in blk_mq_dispatch_rq_list().
+ */
+ list_add_tail(&rq->queuelist, &ctx->rq_list);
+ ctx->count++;
+ if (rq->mq_hctx != ctx->hctx)
+ ctx->multi_hctxs = true;
+
+ /*
+ * If we cannot get tag for the request, stop dequeueing
+ * requests from the IO scheduler. We are unlikely to be able
+ * to submit them anyway and it creates false impression for
+ * scheduling heuristics that the device can take more IO.
+ */
+ return blk_mq_get_driver_tag(rq);
+}
+
+static int elevator_finish_dispatch(struct sched_dispatch_ctx *ctx)
+{
+ bool dispatched = false;
+
+ if (!ctx->count) {
+ if (ctx->run_queue)
+ blk_mq_delay_run_hw_queues(ctx->q, BLK_MQ_BUDGET_DELAY);
+ } else if (ctx->multi_hctxs) {
/*
* Requests from different hctx may be dequeued from some
* schedulers, such as bfq and deadline.
@@ -160,19 +163,53 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
* Sort the requests in the list according to their hctx,
* dispatch batching requests from same hctx at a time.
*/
- list_sort(NULL, &rq_list, sched_rq_cmp);
+ list_sort(NULL, &ctx->rq_list, sched_rq_cmp);
do {
- dispatched |= blk_mq_dispatch_hctx_list(&rq_list);
- } while (!list_empty(&rq_list));
+ dispatched |= blk_mq_dispatch_hctx_list(&ctx->rq_list);
+ } while (!list_empty(&ctx->rq_list));
} else {
- dispatched = blk_mq_dispatch_rq_list(hctx, &rq_list, false);
+ dispatched = blk_mq_dispatch_rq_list(ctx->hctx, &ctx->rq_list,
+ false);
}
- if (busy)
+ if (ctx->busy)
return -EAGAIN;
+
return !!dispatched;
}
+/*
+ * Only SCSI implements .get_budget and .put_budget, and SCSI restarts
+ * its queue by itself in its completion handler, so we don't need to
+ * restart queue if .get_budget() fails to get the budget.
+ *
+ * Returns -EAGAIN if hctx->dispatch was found non-empty and run_work has to
+ * be run again. This is necessary to avoid starving flushes.
+ */
+static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
+{
+ unsigned int max_dispatch;
+ struct sched_dispatch_ctx ctx = {
+ .hctx = hctx,
+ .q = hctx->queue,
+ .e = hctx->queue->elevator,
+ };
+
+ INIT_LIST_HEAD(&ctx.rq_list);
+
+ if (hctx->dispatch_busy)
+ max_dispatch = 1;
+ else
+ max_dispatch = hctx->queue->nr_requests;
+
+ do {
+ if (!elevator_dispatch_one_request(&ctx))
+ break;
+ } while (ctx.count < max_dispatch);
+
+ return elevator_finish_dispatch(&ctx);
+}
+
static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
{
unsigned long end = jiffies + HZ;
--
2.39.2
Powered by blists - more mailing lists