netdev - [PATCH v2 2/8] net/rds: Introduce a pool of worker threads for connection management

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-ID: <20250411180207.450312-3-allison.henderson@oracle.com>
Date: Fri, 11 Apr 2025 11:02:01 -0700
From: allison.henderson@...cle.com
To: netdev@...r.kernel.org
Subject: [PATCH v2 2/8] net/rds: Introduce a pool of worker threads for connection management

From: Håkon Bugge <haakon.bugge@...cle.com>

RDS uses a single threaded work queue for connection management. This
involves creation and deletion of QPs and CQs. On certain HCAs, such
as CX-3, these operations are para-virtualized and some part of the
work has to be conducted by the Physical Function (PF) driver.

In fail-over and fail-back situations, there might be 1000s of
connections to tear down and re-establish. Hence, expand the number
work queues.

Signed-off-by: Håkon Bugge <haakon.bugge@...cle.com>
Signed-off-by: Allison Henderson <allison.henderson@...cle.com>
---
 net/rds/connection.c |  8 +++++++-
 net/rds/ib.c         |  5 +++++
 net/rds/rds.h        |  4 ++++
 net/rds/threads.c    | 23 +++++++++++++++++++++++
 4 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/net/rds/connection.c b/net/rds/connection.c
index 372542d67fdb..f7f48abf7233 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -171,6 +171,10 @@ static struct rds_connection *__rds_conn_create(struct net *net,
 	unsigned long flags;
 	int ret, i;
 	int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
+	int cp_wqs_inx = jhash_3words(laddr->s6_addr32[3],
+				      faddr->s6_addr32[3],
+				      tos,
+				      0) % RDS_NMBR_CP_WQS;
 
 	rcu_read_lock();
 	conn = rds_conn_lookup(net, head, laddr, faddr, trans, tos, dev_if);
@@ -268,7 +272,9 @@ static struct rds_connection *__rds_conn_create(struct net *net,
 		__rds_conn_path_init(conn, &conn->c_path[i],
 				     is_outgoing);
 		conn->c_path[i].cp_index = i;
-		conn->c_path[i].cp_wq = rds_wq;
+
+		rdsdebug("using rds_cp_wqs index %d\n", cp_wqs_inx);
+		conn->c_path[i].cp_wq = rds_cp_wqs[cp_wqs_inx];
 	}
 	rcu_read_lock();
 	if (rds_destroy_pending(conn))
diff --git a/net/rds/ib.c b/net/rds/ib.c
index 9826fe7f9d00..6694d31e6cfd 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -491,9 +491,14 @@ static int rds_ib_laddr_check(struct net *net, const struct in6_addr *addr,
 
 static void rds_ib_unregister_client(void)
 {
+	int i;
+
 	ib_unregister_client(&rds_ib_client);
 	/* wait for rds_ib_dev_free() to complete */
 	flush_workqueue(rds_wq);
+
+	for (i = 0; i < RDS_NMBR_CP_WQS; ++i)
+		flush_workqueue(rds_cp_wqs[i]);
 }
 
 static void rds_ib_set_unloading(void)
diff --git a/net/rds/rds.h b/net/rds/rds.h
index 1c51f3f6ed15..71a0020fe41d 100644
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -40,6 +40,9 @@
 #ifdef ATOMIC64_INIT
 #define KERNEL_HAS_ATOMIC64
 #endif
+
+#define RDS_NMBR_CP_WQS 16
+
 #ifdef RDS_DEBUG
 #define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
 #else
@@ -994,6 +997,7 @@ extern unsigned int  rds_sysctl_trace_level;
 int rds_threads_init(void);
 void rds_threads_exit(void);
 extern struct workqueue_struct *rds_wq;
+extern struct workqueue_struct *rds_cp_wqs[RDS_NMBR_CP_WQS];
 void rds_queue_reconnect(struct rds_conn_path *cp);
 void rds_connect_worker(struct work_struct *);
 void rds_shutdown_worker(struct work_struct *);
diff --git a/net/rds/threads.c b/net/rds/threads.c
index 639302bab51e..f713c6d9cd32 100644
--- a/net/rds/threads.c
+++ b/net/rds/threads.c
@@ -33,6 +33,7 @@
 #include <linux/kernel.h>
 #include <linux/random.h>
 #include <linux/export.h>
+#include <linux/workqueue.h>
 
 #include "rds.h"
 
@@ -70,6 +71,8 @@
  */
 struct workqueue_struct *rds_wq;
 EXPORT_SYMBOL_GPL(rds_wq);
+struct workqueue_struct *rds_cp_wqs[RDS_NMBR_CP_WQS];
+EXPORT_SYMBOL_GPL(rds_cp_wqs);
 
 void rds_connect_path_complete(struct rds_conn_path *cp, int curr)
 {
@@ -251,16 +254,36 @@ void rds_shutdown_worker(struct work_struct *work)
 
 void rds_threads_exit(void)
 {
+	int i;
+
 	destroy_workqueue(rds_wq);
+	for (i = 0; i < RDS_NMBR_CP_WQS; ++i)
+		destroy_workqueue(rds_cp_wqs[i]);
 }
 
 int rds_threads_init(void)
 {
+	int i, j;
+
 	rds_wq = create_singlethread_workqueue("krdsd");
 	if (!rds_wq)
 		return -ENOMEM;
 
+	for (i = 0; i < RDS_NMBR_CP_WQS; ++i) {
+		rds_cp_wqs[i] = alloc_ordered_workqueue("krds_cp_wq_%d",
+							WQ_MEM_RECLAIM, i);
+		if (!rds_cp_wqs[i])
+			goto err;
+	}
+
 	return 0;
+
+err:
+	destroy_workqueue(rds_wq);
+	for (j = 0; j < i; ++j)
+		destroy_workqueue(rds_cp_wqs[j]);
+
+	return -ENOMEM;
 }
 
 /* Compare two IPv6 addresses.  Return 0 if the two addresses are equal.
-- 
2.43.0