| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20221110213218.28662-2-nnac123@linux.ibm.com>
Date: Thu, 10 Nov 2022 15:32:16 -0600
From: Nick Child <nnac123@...ux.ibm.com>
To: netdev@...r.kernel.org
Cc: bjking1@...ux.ibm.com, haren@...ux.ibm.com, ricklind@...ibm.com,
mmc@...ux.ibm.com, Nick Child <nnac123@...ux.ibm.com>,
Thomas Falcon <tlfalcon@...ux.ibm.com>,
Dany Madden <drt@...ux.ibm.com>,
Rick Lindsley <ricklind@...ux.ibm.com>
Subject: [PATCH net-next 1/3] ibmvnic: Assign IRQ affinity hints to device queues
Assign affinity hints to ibmvnic device queue interrupts.
Affinity hints are assigned and removed during sub-crq init and
teardown, respectively. This update should improve latency if
utilized as interrupt lines and processing are more equally
distributed among CPU's. This implementation is based on the
virtio_net driver.
Signed-off-by: Thomas Falcon <tlfalcon@...ux.ibm.com>
Signed-off-by: Dany Madden <drt@...ux.ibm.com>
Signed-off-by: Nick Child <nnac123@...ux.ibm.com>
Reviewed-by: Rick Lindsley <ricklind@...ux.ibm.com>
Reviewed-by: Haren Myneni <haren@...ux.ibm.com>
---
drivers/net/ethernet/ibm/ibmvnic.c | 140 +++++++++++++++++++++++++++++
drivers/net/ethernet/ibm/ibmvnic.h | 1 +
2 files changed, 141 insertions(+)
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 9282381a438f..0c969bdaf94d 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -68,6 +68,7 @@
#include <linux/workqueue.h>
#include <linux/if_vlan.h>
#include <linux/utsname.h>
+#include <linux/cpu.h>
#include "ibmvnic.h"
@@ -171,6 +172,132 @@ static int send_version_xchg(struct ibmvnic_adapter *adapter)
return ibmvnic_send_crq(adapter, &crq);
}
+static void ibmvnic_clean_queue_affinity(struct ibmvnic_adapter *adapter,
+ struct ibmvnic_sub_crq_queue *queue)
+{
+ if (!(queue && queue->irq))
+ return;
+
+ cpumask_clear(queue->affinity_mask);
+
+ if (irq_set_affinity_and_hint(queue->irq, NULL))
+ netdev_warn(adapter->netdev,
+ "%s: Clear affinity failed, queue addr = %p, IRQ = %d\n",
+ __func__, queue, queue->irq);
+}
+
+static void ibmvnic_clean_affinity(struct ibmvnic_adapter *adapter)
+{
+ struct ibmvnic_sub_crq_queue **rxqs;
+ struct ibmvnic_sub_crq_queue **txqs;
+ int num_rxqs, num_txqs;
+ int rc, i;
+
+ rc = 0;
+ rxqs = adapter->rx_scrq;
+ txqs = adapter->tx_scrq;
+ num_txqs = adapter->num_active_tx_scrqs;
+ num_rxqs = adapter->num_active_rx_scrqs;
+
+ netdev_dbg(adapter->netdev, "%s: Cleaning irq affinity hints", __func__);
+ if (txqs) {
+ for (i = 0; i < num_txqs; i++)
+ ibmvnic_clean_queue_affinity(adapter, txqs[i]);
+ }
+ if (rxqs) {
+ for (i = 0; i < num_rxqs; i++)
+ ibmvnic_clean_queue_affinity(adapter, rxqs[i]);
+ }
+}
+
+static int ibmvnic_set_queue_affinity(struct ibmvnic_sub_crq_queue *queue,
+ unsigned int *cpu, int *stragglers,
+ int stride)
+{
+ cpumask_var_t mask;
+ int i;
+ int rc = 0;
+
+ if (!(queue && queue->irq))
+ return rc;
+
+ /* cpumask_var_t is either a pointer or array, allocation works here */
+ if (!zalloc_cpumask_var(&mask, GFP_KERNEL))
+ return -ENOMEM;
+
+ /* while we have extra cpu give one extra to this irq */
+ if (*stragglers) {
+ stride++;
+ (*stragglers)--;
+ }
+ /* atomic write is safer than writing bit by bit directly */
+ for (i = 0; i < stride; i++) {
+ cpumask_set_cpu(*cpu, mask);
+ *cpu = cpumask_next_wrap(*cpu, cpu_online_mask,
+ nr_cpu_ids, false);
+ }
+ /* set queue affinity mask */
+ cpumask_copy(queue->affinity_mask, mask);
+ rc = irq_set_affinity_and_hint(queue->irq, queue->affinity_mask);
+ free_cpumask_var(mask);
+
+ return rc;
+}
+
+/* assumes cpu read lock is held */
+static void ibmvnic_set_affinity(struct ibmvnic_adapter *adapter)
+{
+ struct ibmvnic_sub_crq_queue **rxqs = adapter->rx_scrq;
+ struct ibmvnic_sub_crq_queue **txqs = adapter->tx_scrq;
+ struct ibmvnic_sub_crq_queue *queue;
+ int num_rxqs = adapter->num_active_rx_scrqs;
+ int num_txqs = adapter->num_active_tx_scrqs;
+ int total_queues, stride, stragglers, i;
+ unsigned int num_cpu, cpu;
+ int rc = 0;
+
+ netdev_dbg(adapter->netdev, "%s: Setting irq affinity hints", __func__);
+ if (!(adapter->rx_scrq && adapter->tx_scrq)) {
+ netdev_warn(adapter->netdev,
+ "%s: Set affinity failed, queues not allocated\n",
+ __func__);
+ return;
+ }
+
+ total_queues = num_rxqs + num_txqs;
+ num_cpu = num_online_cpus();
+ /* number of cpu's assigned per irq */
+ stride = max_t(int, num_cpu / total_queues, 1);
+ /* number of leftover cpu's */
+ stragglers = num_cpu >= total_queues ? num_cpu % total_queues : 0;
+ /* next available cpu to assign irq to */
+ cpu = cpumask_next(-1, cpu_online_mask);
+
+ for (i = 0; i < num_txqs; i++) {
+ queue = txqs[i];
+ rc = ibmvnic_set_queue_affinity(queue, &cpu, &stragglers,
+ stride);
+ if (rc)
+ goto out;
+ }
+
+ for (i = 0; i < num_rxqs; i++) {
+ queue = rxqs[i];
+ rc = ibmvnic_set_queue_affinity(queue, &cpu, &stragglers,
+ stride);
+ if (rc)
+ goto out;
+ }
+
+out:
+ if (rc) {
+ netdev_warn(adapter->netdev,
+ "%s: Set affinity failed, queue addr = %p, IRQ = %d, rc = %d.\n",
+ __func__, queue, queue->irq, rc);
+ ibmvnic_clean_affinity(adapter);
+ }
+}
+
static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,
unsigned long length, unsigned long *number,
unsigned long *irq)
@@ -3626,6 +3753,8 @@ static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)
if (!adapter->tx_scrq || !adapter->rx_scrq)
return -EINVAL;
+ ibmvnic_clean_affinity(adapter);
+
for (i = 0; i < adapter->req_tx_queues; i++) {
netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);
rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);
@@ -3675,6 +3804,7 @@ static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,
dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
DMA_BIDIRECTIONAL);
free_pages((unsigned long)scrq->msgs, 2);
+ free_cpumask_var(scrq->affinity_mask);
kfree(scrq);
}
@@ -3695,6 +3825,8 @@ static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter
dev_warn(dev, "Couldn't allocate crq queue messages page\n");
goto zero_page_failed;
}
+ if (!zalloc_cpumask_var(&scrq->affinity_mask, GFP_KERNEL))
+ goto cpumask_alloc_failed;
scrq->msg_token = dma_map_single(dev, scrq->msgs, 4 * PAGE_SIZE,
DMA_BIDIRECTIONAL);
@@ -3747,6 +3879,8 @@ static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter
dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
DMA_BIDIRECTIONAL);
map_failed:
+ free_cpumask_var(scrq->affinity_mask);
+cpumask_alloc_failed:
free_pages((unsigned long)scrq->msgs, 2);
zero_page_failed:
kfree(scrq);
@@ -3758,6 +3892,7 @@ static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free)
{
int i;
+ ibmvnic_clean_affinity(adapter);
if (adapter->tx_scrq) {
for (i = 0; i < adapter->num_active_tx_scrqs; i++) {
if (!adapter->tx_scrq[i])
@@ -4035,6 +4170,11 @@ static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
goto req_rx_irq_failed;
}
}
+
+ cpus_read_lock();
+ ibmvnic_set_affinity(adapter);
+ cpus_read_unlock();
+
return rc;
req_rx_irq_failed:
diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index e5c6ff3d0c47..6720fec1ae67 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -825,6 +825,7 @@ struct ibmvnic_sub_crq_queue {
atomic_t used;
char name[32];
u64 handle;
+ cpumask_var_t affinity_mask;
} ____cacheline_aligned;
struct ibmvnic_long_term_buff {
--
2.31.1
Powered by blists - more mailing lists