[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <20250530080355.1138759-5-yukuai1@huaweicloud.com>
Date: Fri, 30 May 2025 16:03:55 +0800
From: Yu Kuai <yukuai1@...weicloud.com>
To: axboe@...nel.dk
Cc: linux-block@...r.kernel.org,
linux-kernel@...r.kernel.org,
yukuai3@...wei.com,
yukuai1@...weicloud.com,
yi.zhang@...wei.com,
yangerkun@...wei.com,
johnny.chenyi@...wei.com
Subject: [PATCH RFC 4/4] blk-mq-sched: support request batch dispatching for sq elevator
From: Yu Kuai <yukuai3@...wei.com>
Before this patch, each context will hold a global lock to dispatch one
request at a time, which introduce intense lock contention:
lock
ops.dispatch_request
unlock
Hence support dispatch a batch of requests while holding the lock to
reduce lock contention.
Signed-off-by: Yu Kuai <yukuai3@...wei.com>
---
block/blk-mq-sched.c | 53 ++++++++++++++++++++++++++++++++++++++++----
block/blk-mq.h | 21 ++++++++++++++++++
2 files changed, 70 insertions(+), 4 deletions(-)
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 990d0f19594a..d255c3e6c2a8 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -101,6 +101,47 @@ static bool elevator_can_dispatch(struct sched_dispatch_ctx *ctx)
return true;
}
+static void elevator_dispatch_requests(struct sched_dispatch_ctx *ctx)
+{
+ struct request *rq;
+ int budget_token[BUDGET_TOKEN_BATCH];
+ int count;
+ int i;
+
+ while (true) {
+ if (!elevator_can_dispatch(ctx))
+ return;
+
+ count = blk_mq_get_dispatch_budgets(ctx->q, budget_token);
+ if (count <= 0)
+ return;
+
+ elevator_lock(ctx->e);
+ for (i = 0; i < count; ++i) {
+ rq = ctx->e->type->ops.dispatch_request(ctx->hctx);
+ if (!rq) {
+ ctx->run_queue = true;
+ goto err_free_budgets;
+ }
+
+ blk_mq_set_rq_budget_token(rq, budget_token[i]);
+ list_add_tail(&rq->queuelist, &ctx->rq_list);
+ ctx->count++;
+ if (rq->mq_hctx != ctx->hctx)
+ ctx->multi_hctxs = true;
+
+ if (!blk_mq_get_driver_tag(rq))
+ goto err_free_budgets;
+ }
+ elevator_unlock(ctx->e);
+ }
+
+err_free_budgets:
+ elevator_unlock(ctx->e);
+ for (; i < count; ++i)
+ blk_mq_put_dispatch_budget(ctx->q, budget_token[i]);
+}
+
static bool elevator_dispatch_one_request(struct sched_dispatch_ctx *ctx)
{
struct request *rq;
@@ -202,10 +243,14 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
else
max_dispatch = hctx->queue->nr_requests;
- do {
- if (!elevator_dispatch_one_request(&ctx))
- break;
- } while (ctx.count < max_dispatch);
+ if (blk_queue_sq_sched(ctx.q))
+ elevator_dispatch_requests(&ctx);
+ else {
+ do {
+ if (!elevator_dispatch_one_request(&ctx))
+ break;
+ } while (ctx.count < max_dispatch);
+ }
return elevator_finish_dispatch(&ctx);
}
diff --git a/block/blk-mq.h b/block/blk-mq.h
index affb2e14b56e..450c16a07841 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -37,6 +37,7 @@ enum {
};
#define BLK_MQ_CPU_WORK_BATCH (8)
+#define BUDGET_TOKEN_BATCH (8)
typedef unsigned int __bitwise blk_insert_t;
#define BLK_MQ_INSERT_AT_HEAD ((__force blk_insert_t)0x01)
@@ -262,6 +263,26 @@ static inline int blk_mq_get_dispatch_budget(struct request_queue *q)
return 0;
}
+static inline int blk_mq_get_dispatch_budgets(struct request_queue *q,
+ int *budget_token)
+{
+ int count = 0;
+
+ while (count < BUDGET_TOKEN_BATCH) {
+ int token = 0;
+
+ if (q->mq_ops->get_budget)
+ token = q->mq_ops->get_budget(q);
+
+ if (token < 0)
+ return count;
+
+ budget_token[count++] = token;
+ }
+
+ return count;
+}
+
static inline void blk_mq_set_rq_budget_token(struct request *rq, int token)
{
if (token < 0)
--
2.39.2
Powered by blists - more mailing lists