| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20251116-frmr_pools-v1-8-5eb3c8f5c9c4@nvidia.com>
Date: Sun, 16 Nov 2025 21:10:29 +0200
From: Edward Srouji <edwards@...dia.com>
To: Jason Gunthorpe <jgg@...pe.ca>, Leon Romanovsky <leon@...nel.org>, "Saeed
Mahameed" <saeedm@...dia.com>, Tariq Toukan <tariqt@...dia.com>, Mark Bloch
<mbloch@...dia.com>, Andrew Lunn <andrew+netdev@...n.ch>, "David S. Miller"
<davem@...emloft.net>, Eric Dumazet <edumazet@...gle.com>, Jakub Kicinski
<kuba@...nel.org>, Paolo Abeni <pabeni@...hat.com>
CC: <linux-kernel@...r.kernel.org>, <linux-rdma@...r.kernel.org>,
<netdev@...r.kernel.org>, Michael Guralnik <michaelgur@...dia.com>, "Edward
Srouji" <edwards@...dia.com>, Patrisious Haddad <phaddad@...dia.com>
Subject: [PATCH rdma-next 8/9] RDMA/core: Add netlink command to modify
FRMR aging
From: Michael Guralnik <michaelgur@...dia.com>
Allow users to set FRMR pools aging timer through netlink.
This functionality will allow user to control how long handles reside in
the kernel before being destroyed, thus being able to tune the tradeoff
between memory and HW object consumption and memory registration
optimization.
Since FRMR pools is highly beneficial for application restart scenarios,
this command allows users to modify the aging timer to their application
restart time, making sure the FRMR handles deregistered on application
teardown are kept for long enough in the pools for reuse in the
application startup.
Signed-off-by: Michael Guralnik <michaelgur@...dia.com>
Reviewed-by: Patrisious Haddad <phaddad@...dia.com>
Signed-off-by: Edward Srouji <edwards@...dia.com>
---
drivers/infiniband/core/frmr_pools.c | 31 ++++++++++++++++++++++++++++--
drivers/infiniband/core/frmr_pools.h | 2 ++
drivers/infiniband/core/nldev.c | 37 ++++++++++++++++++++++++++++++++++++
include/uapi/rdma/rdma_netlink.h | 3 +++
4 files changed, 71 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c
index 254113d2442d5d6956587a1c444dc74cd48204fb..b150bb78de3c4fd89990f7aed7874e4db94eac0a 100644
--- a/drivers/infiniband/core/frmr_pools.c
+++ b/drivers/infiniband/core/frmr_pools.c
@@ -174,7 +174,7 @@ static void pool_aging_work(struct work_struct *work)
if (has_work)
queue_delayed_work(
pools->aging_wq, &pool->aging_work,
- secs_to_jiffies(FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS));
+ secs_to_jiffies(READ_ONCE(pools->aging_period_sec)));
}
static void destroy_frmr_pool(struct ib_device *device,
@@ -214,6 +214,8 @@ int ib_frmr_pools_init(struct ib_device *device,
return -ENOMEM;
}
+ pools->aging_period_sec = FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS;
+
device->frmr_pools = pools;
return 0;
}
@@ -249,6 +251,31 @@ void ib_frmr_pools_cleanup(struct ib_device *device)
}
EXPORT_SYMBOL(ib_frmr_pools_cleanup);
+int ib_frmr_pools_set_aging_period(struct ib_device *device, u32 period_sec)
+{
+ struct ib_frmr_pools *pools = device->frmr_pools;
+ struct ib_frmr_pool *pool;
+ struct rb_node *node;
+
+ if (!pools)
+ return -EINVAL;
+
+ if (period_sec == 0)
+ return -EINVAL;
+
+ WRITE_ONCE(pools->aging_period_sec, period_sec);
+
+ read_lock(&pools->rb_lock);
+ for (node = rb_first(&pools->rb_root); node; node = rb_next(node)) {
+ pool = rb_entry(node, struct ib_frmr_pool, node);
+ mod_delayed_work(pools->aging_wq, &pool->aging_work,
+ secs_to_jiffies(period_sec));
+ }
+ read_unlock(&pools->rb_lock);
+
+ return 0;
+}
+
static int compare_keys(struct ib_frmr_key *key1, struct ib_frmr_key *key2)
{
int res;
@@ -518,7 +545,7 @@ int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr)
if (ret == 0 && schedule_aging)
queue_delayed_work(pools->aging_wq, &pool->aging_work,
- secs_to_jiffies(FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS));
+ secs_to_jiffies(READ_ONCE(pools->aging_period_sec)));
return ret;
}
diff --git a/drivers/infiniband/core/frmr_pools.h b/drivers/infiniband/core/frmr_pools.h
index b144273ee34785623d2254d19f5af40869e00e83..81149ff15e003358b6d060c98fb68120c9a0e8b9 100644
--- a/drivers/infiniband/core/frmr_pools.h
+++ b/drivers/infiniband/core/frmr_pools.h
@@ -54,8 +54,10 @@ struct ib_frmr_pools {
const struct ib_frmr_pool_ops *pool_ops;
struct workqueue_struct *aging_wq;
+ u32 aging_period_sec;
};
int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key,
u32 pinned_handles);
+int ib_frmr_pools_set_aging_period(struct ib_device *device, u32 period_sec);
#endif /* RDMA_CORE_FRMR_POOLS_H */
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 6cdf6073fdf9c51ee291a63bb86ac690b094aa9f..e22c999d164120ac070b435e92f53c15f976bf5c 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -184,6 +184,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
[RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES] = { .type = NLA_U32 },
[RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE] = { .type = NLA_U64 },
[RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD] = { .type = NLA_U32 },
};
static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -2887,6 +2888,38 @@ static int nldev_frmr_pools_get_dumpit(struct sk_buff *skb,
return ret;
}
+static int nldev_frmr_pools_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+ struct ib_device *device;
+ u32 aging_period;
+ int err;
+
+ err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
+ extack);
+ if (err)
+ return err;
+
+ if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+ return -EINVAL;
+
+ if (!tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD])
+ return -EINVAL;
+
+ device = ib_device_get_by_index(
+ sock_net(skb->sk), nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]));
+ if (!device)
+ return -EINVAL;
+
+ aging_period = nla_get_u32(tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD]);
+
+ err = ib_frmr_pools_set_aging_period(device, aging_period);
+
+ ib_device_put(device);
+ return err;
+}
+
static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
[RDMA_NLDEV_CMD_GET] = {
.doit = nldev_get_doit,
@@ -2997,6 +3030,10 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
.doit = nldev_frmr_pools_get_doit,
.dump = nldev_frmr_pools_get_dumpit,
},
+ [RDMA_NLDEV_CMD_FRMR_POOLS_SET] = {
+ .doit = nldev_frmr_pools_set_doit,
+ .flags = RDMA_NL_ADMIN_PERM,
+ },
};
static int fill_mon_netdev_rename(struct sk_buff *msg,
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index 8f17ffe0190cb86131109209c45caec155ab36da..f9c295caf2b1625e3636d4279a539d481fdeb4ac 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -310,6 +310,8 @@ enum rdma_nldev_command {
RDMA_NLDEV_CMD_FRMR_POOLS_GET, /* can dump */
+ RDMA_NLDEV_CMD_FRMR_POOLS_SET,
+
RDMA_NLDEV_NUM_OPS
};
@@ -598,6 +600,7 @@ enum rdma_nldev_attr {
RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES, /* u32 */
RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE, /* u64 */
RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE, /* u64 */
+ RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD, /* u32 */
/*
* Always the end
--
2.47.1
Powered by blists - more mailing lists