netdev - [PATCH RFC 9/9] veth: Avoid per-packet spinlock of XDP napi ring on enqueueing

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite for Android: free password hash cracker in your pocket

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Date:   Tue, 24 Apr 2018 23:39:23 +0900
From:   Toshiaki Makita <toshiaki.makita1@...il.com>
To:     netdev@...r.kernel.org
Cc:     Toshiaki Makita <makita.toshiaki@....ntt.co.jp>
Subject: [PATCH RFC 9/9] veth: Avoid per-packet spinlock of XDP napi ring on enqueueing

From: Toshiaki Makita <makita.toshiaki@....ntt.co.jp>

Use percpu temporary storage to avoid per-packet spinlock.
This is different from dequeue in that multiple veth devices can be
redirect target in one napi loop so allocate percpu storage in veth
private structure.

Signed-off-by: Toshiaki Makita <makita.toshiaki@....ntt.co.jp>
---
 drivers/net/veth.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 1592119e3873..5978d76f2c00 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -38,12 +38,18 @@ struct pcpu_vstats {
 	struct u64_stats_sync	syncp;
 };
 
+struct xdp_queue {
+	void *q[VETH_XDP_QUEUE_SIZE];
+	unsigned int len;
+};
+
 struct veth_priv {
 	struct napi_struct	xdp_napi;
 	struct net_device	*dev;
 	struct bpf_prog __rcu	*xdp_prog;
 	struct net_device __rcu	*peer;
 	atomic64_t		dropped;
+	struct xdp_queue __percpu *xdp_produce_q;
 	struct xdp_mem_info	xdp_mem;
 	unsigned		requested_headroom;
 	bool			rx_notify_masked;
@@ -147,8 +153,48 @@ static void veth_ptr_free(void *ptr)
 	}
 }
 
+static void veth_xdp_cleanup_queues(struct veth_priv *priv)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct xdp_queue *q = per_cpu_ptr(priv->xdp_produce_q, cpu);
+		int i;
+
+		for (i = 0; i < q->len; i++)
+			veth_ptr_free(q->q[i]);
+
+		q->len = 0;
+	}
+}
+
+static bool veth_xdp_flush_queue(struct veth_priv *priv)
+{
+	struct xdp_queue *q = this_cpu_ptr(priv->xdp_produce_q);
+	int i;
+
+	if (unlikely(!q->len))
+		return false;
+
+	spin_lock(&priv->xdp_ring.producer_lock);
+	for (i = 0; i < q->len; i++) {
+		void *ptr = q->q[i];
+
+		if (unlikely(__ptr_ring_produce(&priv->xdp_ring, ptr)))
+			veth_ptr_free(ptr);
+	}
+	spin_unlock(&priv->xdp_ring.producer_lock);
+
+	q->len = 0;
+
+	return true;
+}
+
 static void __veth_xdp_flush(struct veth_priv *priv)
 {
+	if (unlikely(!veth_xdp_flush_queue(priv)))
+		return;
+
 	/* Write ptr_ring before reading rx_notify_masked */
 	smp_mb();
 	if (!priv->rx_notify_masked) {
@@ -159,9 +205,13 @@ static void __veth_xdp_flush(struct veth_priv *priv)
 
 static int veth_xdp_enqueue(struct veth_priv *priv, void *ptr)
 {
-	if (unlikely(ptr_ring_produce(&priv->xdp_ring, ptr)))
+	struct xdp_queue *q = this_cpu_ptr(priv->xdp_produce_q);
+
+	if (unlikely(q->len >= VETH_XDP_QUEUE_SIZE))
 		return -ENOSPC;
 
+	q->q[q->len++] = ptr;
+
 	return 0;
 }
 
@@ -644,6 +694,7 @@ static void veth_napi_del(struct net_device *dev)
 
 	napi_disable(&priv->xdp_napi);
 	netif_napi_del(&priv->xdp_napi);
+	veth_xdp_cleanup_queues(priv);
 	ptr_ring_cleanup(&priv->xdp_ring, veth_ptr_free);
 }
 
@@ -711,15 +762,28 @@ static int is_valid_veth_mtu(int mtu)
 
 static int veth_dev_init(struct net_device *dev)
 {
+	struct veth_priv *priv = netdev_priv(dev);
+
 	dev->vstats = netdev_alloc_pcpu_stats(struct pcpu_vstats);
 	if (!dev->vstats)
 		return -ENOMEM;
+
+	priv->xdp_produce_q = __alloc_percpu(sizeof(*priv->xdp_produce_q),
+					     sizeof (void *));
+	if (!priv->xdp_produce_q) {
+		free_percpu(dev->vstats);
+		return -ENOMEM;
+	}
+
 	return 0;
 }
 
 static void veth_dev_free(struct net_device *dev)
 {
+	struct veth_priv *priv = netdev_priv(dev);
+
 	free_percpu(dev->vstats);
+	free_percpu(priv->xdp_produce_q);
 }
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
-- 
2.14.3