linux-kernel - [PATCH RFC 7/7] blk-mq: allow shared queue to get more driver tags

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <20230618160738.54385-8-yukuai1@huaweicloud.com>
Date:   Mon, 19 Jun 2023 00:07:38 +0800
From:   Yu Kuai <yukuai1@...weicloud.com>
To:     bvanassche@....org, axboe@...nel.dk
Cc:     linux-block@...r.kernel.org, linux-kernel@...r.kernel.org,
        yukuai3@...wei.com, yukuai1@...weicloud.com, yi.zhang@...wei.com,
        yangerkun@...wei.com
Subject: [PATCH RFC 7/7] blk-mq: allow shared queue to get more driver tags

From: Yu Kuai <yukuai3@...wei.com>

If the queue fail to get driver tags frequently, and other queue
doesn't, try to borrow some shared tags from other queue.

Currently, borrowed tags will not be given back untill this queue is
idle.

Signed-off-by: Yu Kuai <yukuai3@...wei.com>
---
 block/blk-mq-tag.c     | 52 ++++++++++++++++++++++++++++++++++++++----
 include/linux/blkdev.h |  1 +
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 5e5742c7277a..aafcc131e3e6 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -45,7 +45,44 @@ static void blk_mq_update_wake_batch(struct blk_mq_tags *tags,
 			users);
 }
 
-static void update_tag_sharing_busy(struct tag_sharing *tag_sharing)
+static void try_to_increase_available_tags(struct blk_mq_tags *tags,
+					   struct tag_sharing *tag_sharing)
+{
+	unsigned int users = tags->ctl.share_queues;
+	unsigned int free_tags = 0;
+	unsigned int borrowed_tags = 0;
+	unsigned int nr_tags;
+	struct tag_sharing *tmp;
+
+	if (users <= 1)
+		return;
+
+	nr_tags = max((tags->nr_tags + tags->ctl.share_queues - 1) /
+		       tags->ctl.share_queues, 4U);
+
+	list_for_each_entry(tmp, &tags->ctl.head, node) {
+		if (tmp == tag_sharing)
+			continue;
+
+		if (tmp->available_tags > nr_tags)
+			borrowed_tags += tmp->available_tags - nr_tags;
+		else if (atomic_read(&tmp->fail_count) <= nr_tags / 2)
+			free_tags += tmp->available_tags -
+				atomic_read(&tmp->active_tags);
+	}
+
+	/* can't borrow more tags */
+	if (free_tags <= borrowed_tags) {
+		WRITE_ONCE(tag_sharing->suspend, jiffies + HZ);
+		return;
+	}
+
+	/* try to borrow half of free tags */
+	tag_sharing->available_tags += (free_tags - borrowed_tags) / 2;
+}
+
+static void update_tag_sharing_busy(struct blk_mq_tags *tags,
+				    struct tag_sharing *tag_sharing)
 {
 	unsigned int count = atomic_inc_return(&tag_sharing->fail_count);
 	unsigned long last_period = READ_ONCE(tag_sharing->period);
@@ -53,7 +90,14 @@ static void update_tag_sharing_busy(struct tag_sharing *tag_sharing)
 	if (time_after(jiffies, last_period + HZ) &&
 	    cmpxchg_relaxed(&tag_sharing->period, last_period, jiffies) ==
 			    last_period)
-		atomic_sub(count / 2, &tag_sharing->fail_count);
+		count = atomic_sub_return(count / 2, &tag_sharing->fail_count);
+
+	if (count >= tags->nr_tags &&
+	    time_after(jiffies, READ_ONCE(tag_sharing->suspend))) {
+		spin_lock_irq(&tags->lock);
+		try_to_increase_available_tags(tags, tag_sharing);
+		spin_unlock_irq(&tags->lock);
+	}
 }
 
 void __blk_mq_driver_tag_busy(struct blk_mq_hw_ctx *hctx)
@@ -69,13 +113,13 @@ void __blk_mq_driver_tag_busy(struct blk_mq_hw_ctx *hctx)
 
 		if (test_bit(QUEUE_FLAG_HCTX_BUSY, &q->queue_flags) ||
 		    test_and_set_bit(QUEUE_FLAG_HCTX_BUSY, &q->queue_flags)) {
-			update_tag_sharing_busy(&q->tag_sharing);
+			update_tag_sharing_busy(tags, &q->tag_sharing);
 			return;
 		}
 	} else {
 		if (test_bit(BLK_MQ_S_DTAG_BUSY, &hctx->state) ||
 		    test_and_set_bit(BLK_MQ_S_DTAG_BUSY, &hctx->state)) {
-			update_tag_sharing_busy(&hctx->tag_sharing);
+			update_tag_sharing_busy(tags, &hctx->tag_sharing);
 			return;
 		}
 	}
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 0d25e7d2a94c..3528bdc96a17 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -381,6 +381,7 @@ struct tag_sharing {
 	atomic_t		active_tags;
 	atomic_t		fail_count;
 	unsigned long		period;
+	unsigned long		suspend;
 };
 
 struct request_queue {
-- 
2.39.2