| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20180610160217.3146-9-toshiaki.makita1@gmail.com>
Date: Mon, 11 Jun 2018 01:02:16 +0900
From: Toshiaki Makita <toshiaki.makita1@...il.com>
To: netdev@...r.kernel.org
Cc: Toshiaki Makita <makita.toshiaki@....ntt.co.jp>,
Jesper Dangaard Brouer <brouer@...hat.com>,
Alexei Starovoitov <ast@...nel.org>,
Daniel Borkmann <daniel@...earbox.net>
Subject: [PATCH RFC v2 8/9] veth: Support per queue XDP ring
From: Toshiaki Makita <makita.toshiaki@....ntt.co.jp>
Move XDP and napi related fields in veth_priv to newly created veth_rq
structure.
When xdp_frames are enqueued from ndo_xdp_xmit and XDP_TX, rxq is
selected by current cpu.
When skbs are enqueued from the peer device, rxq is determined by its
peer's txq. In this way we can implement bulk packet send using
skb->xmit_more later.
Signed-off-by: Toshiaki Makita <makita.toshiaki@....ntt.co.jp>
---
drivers/net/veth.c | 290 +++++++++++++++++++++++++++++++++++------------------
1 file changed, 191 insertions(+), 99 deletions(-)
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index a47e1ba7d7e6..67debd3eafe6 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -38,20 +38,24 @@ struct pcpu_vstats {
struct u64_stats_sync syncp;
};
-struct veth_priv {
+struct veth_rq {
struct napi_struct xdp_napi;
struct net_device *dev;
struct bpf_prog __rcu *xdp_prog;
- struct net_device __rcu *peer;
- atomic64_t dropped;
struct xdp_mem_info xdp_mem;
- unsigned requested_headroom;
bool rx_notify_masked;
struct ptr_ring xdp_ring;
struct ptr_ring xdp_tx_ring;
struct xdp_rxq_info xdp_rxq;
};
+struct veth_priv {
+ struct net_device __rcu *peer;
+ atomic64_t dropped;
+ struct veth_rq *rq;
+ unsigned int requested_headroom;
+};
+
/*
* ethtool interface
*/
@@ -122,19 +126,19 @@ static void veth_xdp_free(void *frame)
xdp_return_frame(frame);
}
-static void __veth_xdp_flush(struct veth_priv *priv)
+static void __veth_xdp_flush(struct veth_rq *rq)
{
/* Write ptr_ring before reading rx_notify_masked */
smp_mb();
- if (!priv->rx_notify_masked) {
- priv->rx_notify_masked = true;
- napi_schedule(&priv->xdp_napi);
+ if (!rq->rx_notify_masked) {
+ rq->rx_notify_masked = true;
+ napi_schedule(&rq->xdp_napi);
}
}
-static int veth_xdp_rx(struct veth_priv *priv, struct sk_buff *skb)
+static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb)
{
- if (unlikely(ptr_ring_produce(&priv->xdp_ring, skb))) {
+ if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) {
dev_kfree_skb_any(skb);
return NET_RX_DROP;
}
@@ -142,12 +146,11 @@ static int veth_xdp_rx(struct veth_priv *priv, struct sk_buff *skb)
return NET_RX_SUCCESS;
}
-static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, bool xdp)
+static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb,
+ struct veth_rq *rq, bool xdp)
{
- struct veth_priv *priv = netdev_priv(dev);
-
return __dev_forward_skb(dev, skb) ?: xdp ?
- veth_xdp_rx(priv, skb) :
+ veth_xdp_rx(rq, skb) :
netif_rx(skb);
}
@@ -157,6 +160,8 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
struct net_device *rcv;
int length = skb->len;
bool rcv_xdp = false;
+ struct veth_rq *rq;
+ int rxq;
rcu_read_lock();
rcv = rcu_dereference(priv->peer);
@@ -166,9 +171,12 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
}
rcv_priv = netdev_priv(rcv);
- rcv_xdp = rcu_access_pointer(rcv_priv->xdp_prog);
+ rxq = skb_get_queue_mapping(skb);
+ skb_record_rx_queue(skb, rxq);
+ rq = &rcv_priv->rq[rxq];
+ rcv_xdp = rcu_access_pointer(rq->xdp_prog);
- if (likely(veth_forward_skb(rcv, skb, rcv_xdp) == NET_RX_SUCCESS)) {
+ if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) {
struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
u64_stats_update_begin(&stats->syncp);
@@ -181,7 +189,7 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
}
if (rcv_xdp)
- __veth_xdp_flush(rcv_priv);
+ __veth_xdp_flush(rq);
rcu_read_unlock();
@@ -256,11 +264,17 @@ static struct sk_buff *veth_build_skb(void *head, int headroom, int len,
return skb;
}
+static int veth_select_rxq(struct net_device *dev)
+{
+ return smp_processor_id() % dev->real_num_rx_queues;
+}
+
static int veth_xdp_xmit(struct net_device *dev, int n,
struct xdp_frame **frames, u32 flags)
{
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
struct net_device *rcv;
+ struct veth_rq *rq;
int i, drops = 0;
if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
@@ -271,24 +285,25 @@ static int veth_xdp_xmit(struct net_device *dev, int n,
return -ENXIO;
rcv_priv = netdev_priv(rcv);
+ rq = &rcv_priv->rq[veth_select_rxq(rcv)];
/* xdp_ring is initialized on receive side? */
- if (!rcu_access_pointer(rcv_priv->xdp_prog))
+ if (!rcu_access_pointer(rq->xdp_prog))
return -ENXIO;
- spin_lock(&rcv_priv->xdp_tx_ring.producer_lock);
+ spin_lock(&rq->xdp_tx_ring.producer_lock);
for (i = 0; i < n; i++) {
struct xdp_frame *frame = frames[i];
if (unlikely(xdp_ok_fwd_dev(rcv, frame->len) ||
- __ptr_ring_produce(&rcv_priv->xdp_tx_ring, frame))) {
+ __ptr_ring_produce(&rq->xdp_tx_ring, frame))) {
xdp_return_frame_rx_napi(frame);
drops++;
}
}
- spin_unlock(&rcv_priv->xdp_tx_ring.producer_lock);
+ spin_unlock(&rq->xdp_tx_ring.producer_lock);
if (flags & XDP_XMIT_FLUSH)
- __veth_xdp_flush(rcv_priv);
+ __veth_xdp_flush(rq);
return n - drops;
}
@@ -297,6 +312,7 @@ static void veth_xdp_flush(struct net_device *dev)
{
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
struct net_device *rcv;
+ struct veth_rq *rq;
rcu_read_lock();
rcv = rcu_dereference(priv->peer);
@@ -304,11 +320,12 @@ static void veth_xdp_flush(struct net_device *dev)
goto out;
rcv_priv = netdev_priv(rcv);
+ rq = &rcv_priv->rq[veth_select_rxq(rcv)];
/* xdp_ring is initialized on receive side? */
- if (unlikely(!rcu_access_pointer(rcv_priv->xdp_prog)))
+ if (unlikely(!rcu_access_pointer(rq->xdp_prog)))
goto out;
- __veth_xdp_flush(rcv_priv);
+ __veth_xdp_flush(rq);
out:
rcu_read_unlock();
}
@@ -323,7 +340,7 @@ static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
return veth_xdp_xmit(dev, 1, &frame, 0);
}
-static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
+static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,
struct xdp_frame *frame, bool *xdp_xmit,
bool *xdp_redir)
{
@@ -334,7 +351,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
struct sk_buff *skb;
rcu_read_lock();
- xdp_prog = rcu_dereference(priv->xdp_prog);
+ xdp_prog = rcu_dereference(rq->xdp_prog);
if (xdp_prog) {
struct xdp_buff xdp;
u32 act;
@@ -343,7 +360,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
xdp.data = frame->data;
xdp.data_end = frame->data + frame->len;
xdp.data_meta = frame->data - frame->metasize;
- xdp.rxq = &priv->xdp_rxq;
+ xdp.rxq = &rq->xdp_rxq;
act = bpf_prog_run_xdp(xdp_prog, &xdp);
@@ -357,8 +374,8 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
xdp.data_hard_start = frame;
xdp.rxq->mem = frame->mem;
xdp.rxq->mem.flags |= XDP_MEM_RF_NO_DIRECT;
- if (unlikely(veth_xdp_tx(priv->dev, &xdp))) {
- trace_xdp_exception(priv->dev, xdp_prog, act);
+ if (unlikely(veth_xdp_tx(rq->dev, &xdp))) {
+ trace_xdp_exception(rq->dev, xdp_prog, act);
frame = &orig_frame;
goto err_xdp;
}
@@ -370,7 +387,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
xdp.data_hard_start = frame;
xdp.rxq->mem = frame->mem;
xdp.rxq->mem.flags |= XDP_MEM_RF_NO_DIRECT;
- if (xdp_do_redirect(priv->dev, &xdp, xdp_prog)) {
+ if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {
frame = &orig_frame;
goto err_xdp;
}
@@ -380,7 +397,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
default:
bpf_warn_invalid_xdp_action(act);
case XDP_ABORTED:
- trace_xdp_exception(priv->dev, xdp_prog, act);
+ trace_xdp_exception(rq->dev, xdp_prog, act);
case XDP_DROP:
goto err_xdp;
}
@@ -395,7 +412,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
}
memset(frame, 0, sizeof(*frame));
- skb->protocol = eth_type_trans(skb, priv->dev);
+ skb->protocol = eth_type_trans(skb, rq->dev);
err:
return skb;
err_xdp:
@@ -405,9 +422,8 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv,
return NULL;
}
-static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
- struct sk_buff *skb, bool *xdp_xmit,
- bool *xdp_redir)
+static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, struct sk_buff *skb,
+ bool *xdp_xmit, bool *xdp_redir)
{
u32 pktlen, headroom, act, metalen;
void *orig_data, *orig_data_end;
@@ -416,7 +432,7 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
struct xdp_buff xdp;
rcu_read_lock();
- xdp_prog = rcu_dereference(priv->xdp_prog);
+ xdp_prog = rcu_dereference(rq->xdp_prog);
if (!xdp_prog) {
rcu_read_unlock();
goto out;
@@ -467,7 +483,7 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
xdp.data = skb_mac_header(skb);
xdp.data_end = xdp.data + pktlen;
xdp.data_meta = xdp.data;
- xdp.rxq = &priv->xdp_rxq;
+ xdp.rxq = &rq->xdp_rxq;
orig_data = xdp.data;
orig_data_end = xdp.data_end;
@@ -479,9 +495,9 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
case XDP_TX:
get_page(virt_to_page(xdp.data));
dev_consume_skb_any(skb);
- xdp.rxq->mem = priv->xdp_mem;
- if (unlikely(veth_xdp_tx(priv->dev, &xdp))) {
- trace_xdp_exception(priv->dev, xdp_prog, act);
+ xdp.rxq->mem = rq->xdp_mem;
+ if (unlikely(veth_xdp_tx(rq->dev, &xdp))) {
+ trace_xdp_exception(rq->dev, xdp_prog, act);
goto err_xdp;
}
*xdp_xmit = true;
@@ -490,8 +506,8 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
case XDP_REDIRECT:
get_page(virt_to_page(xdp.data));
dev_consume_skb_any(skb);
- xdp.rxq->mem = priv->xdp_mem;
- if (xdp_do_redirect(priv->dev, &xdp, xdp_prog))
+ xdp.rxq->mem = rq->xdp_mem;
+ if (xdp_do_redirect(rq->dev, &xdp, xdp_prog))
goto err_xdp;
*xdp_redir = true;
rcu_read_unlock();
@@ -499,7 +515,7 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
default:
bpf_warn_invalid_xdp_action(act);
case XDP_ABORTED:
- trace_xdp_exception(priv->dev, xdp_prog, act);
+ trace_xdp_exception(rq->dev, xdp_prog, act);
case XDP_DROP:
goto drop;
}
@@ -515,7 +531,7 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
off = xdp.data_end - orig_data_end;
if (off != 0)
__skb_put(skb, off);
- skb->protocol = eth_type_trans(skb, priv->dev);
+ skb->protocol = eth_type_trans(skb, rq->dev);
metalen = xdp.data - xdp.data_meta;
if (metalen)
@@ -533,7 +549,7 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
return NULL;
}
-static int veth_xdp_rcv(struct veth_priv *priv, int budget, bool *xdp_xmit,
+static int veth_xdp_rcv(struct veth_rq *rq, int budget, bool *xdp_xmit,
bool *xdp_redir)
{
int done = 0;
@@ -551,15 +567,15 @@ static int veth_xdp_rcv(struct veth_priv *priv, int budget, bool *xdp_xmit,
struct xdp_frame *frame;
struct sk_buff *skb;
- frame = __ptr_ring_consume(&priv->xdp_tx_ring);
+ frame = __ptr_ring_consume(&rq->xdp_tx_ring);
if (!frame) {
curr_more = false;
break;
}
- skb = veth_xdp_rcv_one(priv, frame, xdp_xmit, xdp_redir);
+ skb = veth_xdp_rcv_one(rq, frame, xdp_xmit, xdp_redir);
if (skb)
- napi_gro_receive(&priv->xdp_napi, skb);
+ napi_gro_receive(&rq->xdp_napi, skb);
done++;
}
@@ -568,16 +584,16 @@ static int veth_xdp_rcv(struct veth_priv *priv, int budget, bool *xdp_xmit,
curr_more = true;
curr_budget = min(budget - done, budget >> 1);
for (i = 0; i < curr_budget; i++) {
- struct sk_buff *skb = __ptr_ring_consume(&priv->xdp_ring);
+ struct sk_buff *skb = __ptr_ring_consume(&rq->xdp_ring);
if (!skb) {
curr_more = false;
break;
}
- skb = veth_xdp_rcv_skb(priv, skb, xdp_xmit, xdp_redir);
+ skb = veth_xdp_rcv_skb(rq, skb, xdp_xmit, xdp_redir);
if (skb)
- napi_gro_receive(&priv->xdp_napi, skb);
+ napi_gro_receive(&rq->xdp_napi, skb);
done++;
}
@@ -589,26 +605,26 @@ static int veth_xdp_rcv(struct veth_priv *priv, int budget, bool *xdp_xmit,
static int veth_poll(struct napi_struct *napi, int budget)
{
- struct veth_priv *priv =
- container_of(napi, struct veth_priv, xdp_napi);
+ struct veth_rq *rq =
+ container_of(napi, struct veth_rq, xdp_napi);
bool xdp_xmit = false;
bool xdp_redir = false;
int done;
- done = veth_xdp_rcv(priv, budget, &xdp_xmit, &xdp_redir);
+ done = veth_xdp_rcv(rq, budget, &xdp_xmit, &xdp_redir);
if (done < budget && napi_complete_done(napi, done)) {
/* Write rx_notify_masked before reading ptr_ring */
- smp_store_mb(priv->rx_notify_masked, false);
- if (unlikely(!__ptr_ring_empty(&priv->xdp_tx_ring) ||
- !__ptr_ring_empty(&priv->xdp_ring))) {
- priv->rx_notify_masked = true;
- napi_schedule(&priv->xdp_napi);
+ smp_store_mb(rq->rx_notify_masked, false);
+ if (unlikely(!__ptr_ring_empty(&rq->xdp_tx_ring) ||
+ !__ptr_ring_empty(&rq->xdp_ring))) {
+ rq->rx_notify_masked = true;
+ napi_schedule(&rq->xdp_napi);
}
}
if (xdp_xmit)
- veth_xdp_flush(priv->dev);
+ veth_xdp_flush(rq->dev);
if (xdp_redir)
xdp_do_flush_map();
@@ -618,22 +634,36 @@ static int veth_poll(struct napi_struct *napi, int budget)
static int veth_napi_add(struct net_device *dev)
{
struct veth_priv *priv = netdev_priv(dev);
- int err;
+ int err, i;
- err = ptr_ring_init(&priv->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
- if (err)
- return err;
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ struct veth_rq *rq = &priv->rq[i];
- err = ptr_ring_init(&priv->xdp_tx_ring, VETH_RING_SIZE, GFP_KERNEL);
- if (err)
- goto err_xdp_tx_ring;
+ err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
+ if (err)
+ goto err_xdp_ring;
+
+ err = ptr_ring_init(&rq->xdp_tx_ring, VETH_RING_SIZE,
+ GFP_KERNEL);
+ if (err)
+ goto err_xdp_tx_ring;
+ }
+
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ struct veth_rq *rq = &priv->rq[i];
- netif_napi_add(dev, &priv->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
- napi_enable(&priv->xdp_napi);
+ netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
+ napi_enable(&rq->xdp_napi);
+ }
return 0;
err_xdp_tx_ring:
- ptr_ring_cleanup(&priv->xdp_ring, __skb_array_destroy_skb);
+ ptr_ring_cleanup(&priv->rq[i].xdp_ring, __skb_array_destroy_skb);
+err_xdp_ring:
+ for (i--; i >= 0; i--) {
+ ptr_ring_cleanup(&priv->rq[i].xdp_ring, __skb_array_destroy_skb);
+ ptr_ring_cleanup(&priv->rq[i].xdp_tx_ring, veth_xdp_free);
+ }
return err;
}
@@ -641,37 +671,56 @@ static int veth_napi_add(struct net_device *dev)
static void veth_napi_del(struct net_device *dev)
{
struct veth_priv *priv = netdev_priv(dev);
+ int i;
- napi_disable(&priv->xdp_napi);
- netif_napi_del(&priv->xdp_napi);
- ptr_ring_cleanup(&priv->xdp_ring, __skb_array_destroy_skb);
- ptr_ring_cleanup(&priv->xdp_tx_ring, veth_xdp_free);
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ struct veth_rq *rq = &priv->rq[i];
+
+ napi_disable(&rq->xdp_napi);
+ napi_hash_del(&rq->xdp_napi);
+ }
+ synchronize_net();
+
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ struct veth_rq *rq = &priv->rq[i];
+
+ netif_napi_del(&rq->xdp_napi);
+ ptr_ring_cleanup(&rq->xdp_ring, __skb_array_destroy_skb);
+ ptr_ring_cleanup(&rq->xdp_tx_ring, veth_xdp_free);
+ }
}
static int veth_enable_xdp(struct net_device *dev)
{
struct veth_priv *priv = netdev_priv(dev);
- int err;
+ int err, i;
- err = xdp_rxq_info_reg(&priv->xdp_rxq, dev, 0);
- if (err < 0)
- return err;
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ struct veth_rq *rq = &priv->rq[i];
- err = xdp_rxq_info_reg_mem_model(&priv->xdp_rxq,
- MEM_TYPE_PAGE_SHARED, NULL);
- if (err < 0)
- goto err;
+ err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i);
+ if (err < 0)
+ goto err_rxq_reg;
+
+ err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
+ MEM_TYPE_PAGE_SHARED, NULL);
+ if (err < 0)
+ goto err_reg_mem;
- /* Save original mem info as it can be overwritten */
- priv->xdp_mem = priv->xdp_rxq.mem;
+ /* Save original mem info as it can be overwritten */
+ rq->xdp_mem = rq->xdp_rxq.mem;
+ }
err = veth_napi_add(dev);
if (err)
- goto err;
+ goto err_rxq_reg;
return 0;
-err:
- xdp_rxq_info_unreg(&priv->xdp_rxq);
+err_reg_mem:
+ xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq);
+err_rxq_reg:
+ for (i--; i >= 0; i--)
+ xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq);
return err;
}
@@ -679,10 +728,15 @@ static int veth_enable_xdp(struct net_device *dev)
static void veth_disable_xdp(struct net_device *dev)
{
struct veth_priv *priv = netdev_priv(dev);
+ int i;
veth_napi_del(dev);
- priv->xdp_rxq.mem = priv->xdp_mem;
- xdp_rxq_info_unreg(&priv->xdp_rxq);
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ struct veth_rq *rq = &priv->rq[i];
+
+ rq->xdp_rxq.mem = rq->xdp_mem;
+ xdp_rxq_info_unreg(&rq->xdp_rxq);
+ }
}
static int veth_open(struct net_device *dev)
@@ -694,7 +748,7 @@ static int veth_open(struct net_device *dev)
if (!peer)
return -ENOTCONN;
- if (rtnl_dereference(priv->xdp_prog)) {
+ if (rtnl_dereference(priv->rq[0].xdp_prog)) {
err = veth_enable_xdp(dev);
if (err)
return err;
@@ -717,7 +771,7 @@ static int veth_close(struct net_device *dev)
if (peer)
netif_carrier_off(peer);
- if (rtnl_dereference(priv->xdp_prog))
+ if (rtnl_dereference(priv->rq[0].xdp_prog))
veth_disable_xdp(dev);
return 0;
@@ -780,7 +834,7 @@ static netdev_features_t veth_fix_features(struct net_device *dev,
if (peer) {
struct veth_priv *peer_priv = netdev_priv(peer);
- if (rtnl_dereference(peer_priv->xdp_prog))
+ if (rtnl_dereference(peer_priv->rq[0].xdp_prog))
features &= ~NETIF_F_GSO_SOFTWARE;
}
@@ -816,9 +870,9 @@ static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog,
struct veth_priv *priv = netdev_priv(dev);
struct bpf_prog *old_prog;
struct net_device *peer;
- int err;
+ int err, i;
- old_prog = rtnl_dereference(priv->xdp_prog);
+ old_prog = rtnl_dereference(priv->rq[0].xdp_prog);
peer = rtnl_dereference(priv->peer);
if (prog) {
@@ -826,6 +880,9 @@ static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog,
return -ENOTCONN;
if (!old_prog) {
+ if (dev->real_num_rx_queues < peer->real_num_tx_queues)
+ return -ENOSPC;
+
if (dev->flags & IFF_UP) {
err = veth_enable_xdp(dev);
if (err)
@@ -841,7 +898,8 @@ static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog,
}
}
- rcu_assign_pointer(priv->xdp_prog, prog);
+ for (i = 0; i < dev->real_num_rx_queues; i++)
+ rcu_assign_pointer(priv->rq[i].xdp_prog, prog);
if (old_prog) {
bpf_prog_put(old_prog);
@@ -867,7 +925,7 @@ static u32 veth_xdp_query(struct net_device *dev)
struct veth_priv *priv = netdev_priv(dev);
const struct bpf_prog *xdp_prog;
- xdp_prog = rtnl_dereference(priv->xdp_prog);
+ xdp_prog = rtnl_dereference(priv->rq[0].xdp_prog);
if (xdp_prog)
return xdp_prog->aux->id;
@@ -960,13 +1018,31 @@ static int veth_validate(struct nlattr *tb[], struct nlattr *data[],
return 0;
}
+static int veth_alloc_queues(struct net_device *dev)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+
+ priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL);
+ if (!priv->rq)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void veth_free_queues(struct net_device *dev)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+
+ kfree(priv->rq);
+}
+
static struct rtnl_link_ops veth_link_ops;
static int veth_newlink(struct net *src_net, struct net_device *dev,
struct nlattr *tb[], struct nlattr *data[],
struct netlink_ext_ack *extack)
{
- int err;
+ int err, i;
struct net_device *peer;
struct veth_priv *priv;
char ifname[IFNAMSIZ];
@@ -1019,6 +1095,12 @@ static int veth_newlink(struct net *src_net, struct net_device *dev,
return PTR_ERR(peer);
}
+ err = veth_alloc_queues(peer);
+ if (err) {
+ put_net(net);
+ goto err_peer_alloc_queues;
+ }
+
if (!ifmp || !tbp[IFLA_ADDRESS])
eth_hw_addr_random(peer);
@@ -1047,6 +1129,10 @@ static int veth_newlink(struct net *src_net, struct net_device *dev,
* should be re-allocated
*/
+ err = veth_alloc_queues(dev);
+ if (err)
+ goto err_alloc_queues;
+
if (tb[IFLA_ADDRESS] == NULL)
eth_hw_addr_random(dev);
@@ -1066,22 +1152,28 @@ static int veth_newlink(struct net *src_net, struct net_device *dev,
*/
priv = netdev_priv(dev);
- priv->dev = dev;
+ for (i = 0; i < dev->real_num_rx_queues; i++)
+ priv->rq[i].dev = dev;
rcu_assign_pointer(priv->peer, peer);
priv = netdev_priv(peer);
- priv->dev = peer;
+ for (i = 0; i < peer->real_num_rx_queues; i++)
+ priv->rq[i].dev = peer;
rcu_assign_pointer(priv->peer, dev);
return 0;
err_register_dev:
+ veth_free_queues(dev);
+err_alloc_queues:
/* nothing to do */
err_configure_peer:
unregister_netdevice(peer);
return err;
err_register_peer:
+ veth_free_queues(peer);
+err_peer_alloc_queues:
free_netdev(peer);
return err;
}
--
2.14.3
Powered by blists - more mailing lists