| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <cf484197a50d829f0f6861d8929942fbd38b045b.1570307172.git.lorenzo@kernel.org>
Date: Sat, 5 Oct 2019 22:44:37 +0200
From: Lorenzo Bianconi <lorenzo@...nel.org>
To: netdev@...r.kernel.org
Cc: davem@...emloft.net, thomas.petazzoni@...tlin.com,
ilias.apalodimas@...aro.org, brouer@...hat.com,
lorenzo.bianconi@...hat.com, matteo.croce@...hat.com
Subject: [PATCH 4/7] net: mvneta: add basic XDP support
Add basic XDP support to mvneta driver for devices that rely on software
buffer management. Currently supported verdicts are:
- XDP_DROP
- XDP_PASS
- XDP_REDIRECT
- XDP_ABORTED
Tested-by: Ilias Apalodimas <ilias.apalodimas@...aro.org>
Signed-off-by: Lorenzo Bianconi <lorenzo@...nel.org>
---
drivers/net/ethernet/marvell/mvneta.c | 146 ++++++++++++++++++++++++--
1 file changed, 137 insertions(+), 9 deletions(-)
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index d775fcae9353..7cbcf1b54da6 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -38,6 +38,7 @@
#include <net/ipv6.h>
#include <net/tso.h>
#include <net/page_pool.h>
+#include <linux/bpf_trace.h>
/* Registers */
#define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2))
@@ -323,8 +324,10 @@
ETH_HLEN + ETH_FCS_LEN, \
cache_line_size())
+#define MVNETA_SKB_HEADROOM (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \
+ NET_IP_ALIGN)
#define MVNETA_SKB_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
- NET_SKB_PAD))
+ MVNETA_SKB_HEADROOM))
#define MVNETA_SKB_SIZE(len) (SKB_DATA_ALIGN(len) + MVNETA_SKB_PAD)
#define MVNETA_MAX_RX_BUF_SIZE (PAGE_SIZE - MVNETA_SKB_PAD)
@@ -352,6 +355,11 @@ struct mvneta_statistic {
#define T_REG_64 64
#define T_SW 1
+#define MVNETA_XDP_PASS BIT(0)
+#define MVNETA_XDP_CONSUMED BIT(1)
+#define MVNETA_XDP_TX BIT(2)
+#define MVNETA_XDP_REDIR BIT(3)
+
static const struct mvneta_statistic mvneta_statistics[] = {
{ 0x3000, T_REG_64, "good_octets_received", },
{ 0x3010, T_REG_32, "good_frames_received", },
@@ -431,6 +439,8 @@ struct mvneta_port {
u32 cause_rx_tx;
struct napi_struct napi;
+ struct bpf_prog *xdp_prog;
+
/* Core clock */
struct clk *clk;
/* AXI clock */
@@ -1945,16 +1955,60 @@ int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq)
return i;
}
+static int
+mvneta_run_xdp(struct mvneta_port *pp, struct bpf_prog *prog,
+ struct xdp_buff *xdp)
+{
+ u32 ret, act = bpf_prog_run_xdp(prog, xdp);
+
+ switch (act) {
+ case XDP_PASS:
+ ret = MVNETA_XDP_PASS;
+ break;
+ case XDP_REDIRECT: {
+ int err;
+
+ err = xdp_do_redirect(pp->dev, xdp, prog);
+ if (err) {
+ ret = MVNETA_XDP_CONSUMED;
+ xdp_return_buff(xdp);
+ } else {
+ ret = MVNETA_XDP_REDIR;
+ }
+ break;
+ }
+ default:
+ bpf_warn_invalid_xdp_action(act);
+ /* fall through */
+ case XDP_ABORTED:
+ trace_xdp_exception(pp->dev, prog, act);
+ /* fall through */
+ case XDP_DROP:
+ ret = MVNETA_XDP_CONSUMED;
+ xdp_return_buff(xdp);
+ break;
+ }
+
+ return ret;
+}
+
static int
mvneta_swbm_rx_frame(struct mvneta_port *pp,
struct mvneta_rx_desc *rx_desc,
struct mvneta_rx_queue *rxq,
- struct page *page)
+ struct bpf_prog *xdp_prog,
+ struct page *page, u32 *xdp_ret)
{
unsigned char *data = page_address(page);
int data_len = -MVNETA_MH_SIZE, len;
struct net_device *dev = pp->dev;
enum dma_data_direction dma_dir;
+ struct xdp_buff xdp = {
+ .data_hard_start = data,
+ .data = data + MVNETA_SKB_HEADROOM + MVNETA_MH_SIZE,
+ .rxq = &rxq->xdp_rxq,
+ };
+ xdp_set_data_meta_invalid(&xdp);
if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) {
len = MVNETA_MAX_RX_BUF_SIZE;
@@ -1963,13 +2017,29 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp,
len = rx_desc->data_size;
data_len += len - ETH_FCS_LEN;
}
+ xdp.data_end = xdp.data + data_len;
dma_dir = page_pool_get_dma_dir(rxq->page_pool);
dma_sync_single_range_for_cpu(dev->dev.parent,
rx_desc->buf_phys_addr, 0,
len, dma_dir);
- rxq->skb = build_skb(data, PAGE_SIZE);
+ if (xdp_prog) {
+ u32 ret;
+
+ ret = mvneta_run_xdp(pp, xdp_prog, &xdp);
+ if (ret == MVNETA_XDP_REDIR ||
+ ret == MVNETA_XDP_TX)
+ mvneta_update_stats(pp, 1, xdp.data_end - xdp.data,
+ false);
+ if (ret != MVNETA_XDP_PASS) {
+ rx_desc->buf_phys_addr = 0;
+ *xdp_ret |= ret;
+ return ret;
+ }
+ }
+
+ rxq->skb = build_skb(xdp.data_hard_start, PAGE_SIZE);
if (unlikely(!rxq->skb)) {
netdev_err(dev,
"Can't allocate skb on queue %d\n",
@@ -1980,8 +2050,9 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp,
}
page_pool_release_page(rxq->page_pool, page);
- skb_reserve(rxq->skb, MVNETA_MH_SIZE + NET_SKB_PAD);
- skb_put(rxq->skb, data_len);
+ skb_reserve(rxq->skb,
+ xdp.data - xdp.data_hard_start);
+ skb_put(rxq->skb, xdp.data_end - xdp.data);
mvneta_rx_csum(pp, rx_desc->status, rxq->skb);
rxq->left_size = rx_desc->data_size - len;
@@ -2015,7 +2086,7 @@ mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,
/* refill descriptor with new buffer later */
skb_add_rx_frag(rxq->skb,
skb_shinfo(rxq->skb)->nr_frags,
- page, NET_SKB_PAD, data_len,
+ page, MVNETA_SKB_HEADROOM, data_len,
PAGE_SIZE);
page_pool_release_page(rxq->page_pool, page);
@@ -2031,10 +2102,15 @@ static int mvneta_rx_swbm(struct napi_struct *napi,
{
int rcvd_pkts = 0, rcvd_bytes = 0;
int rx_pending, refill, done = 0;
+ struct bpf_prog *xdp_prog;
+ u32 xdp_ret = 0;
/* Get number of received packets */
rx_pending = mvneta_rxq_busy_desc_num_get(pp, rxq);
+ rcu_read_lock();
+ xdp_prog = READ_ONCE(pp->xdp_prog);
+
/* Fairness NAPI loop */
while (done < budget && done < rx_pending) {
struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
@@ -2062,7 +2138,8 @@ static int mvneta_rx_swbm(struct napi_struct *napi,
continue;
}
- err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, page);
+ err = mvneta_swbm_rx_frame(pp, rx_desc, rxq,
+ xdp_prog, page, &xdp_ret);
if (err)
continue;
} else {
@@ -2097,6 +2174,10 @@ static int mvneta_rx_swbm(struct napi_struct *napi,
/* clean uncomplete skb pointer in queue */
rxq->skb = NULL;
}
+ rcu_read_unlock();
+
+ if (xdp_ret & MVNETA_XDP_REDIR)
+ xdp_do_flush_map();
mvneta_update_stats(pp, rcvd_pkts, rcvd_bytes, false);
@@ -2838,13 +2919,14 @@ static int mvneta_poll(struct napi_struct *napi, int budget)
static int mvneta_create_page_pool(struct mvneta_port *pp,
struct mvneta_rx_queue *rxq, int size)
{
+ struct bpf_prog *xdp_prog = READ_ONCE(pp->xdp_prog);
struct page_pool_params pp_params = {
.order = 0,
.flags = PP_FLAG_DMA_MAP,
.pool_size = size,
.nid = cpu_to_node(0),
.dev = pp->dev->dev.parent,
- .dma_dir = DMA_FROM_DEVICE,
+ .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
};
int err;
@@ -3310,6 +3392,11 @@ static int mvneta_change_mtu(struct net_device *dev, int mtu)
mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8);
}
+ if (pp->xdp_prog && mtu > MVNETA_MAX_RX_BUF_SIZE) {
+ netdev_info(dev, "Illegal MTU value %d for XDP mode\n", mtu);
+ return -EINVAL;
+ }
+
dev->mtu = mtu;
if (!netif_running(dev)) {
@@ -3979,6 +4066,46 @@ static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
return phylink_mii_ioctl(pp->phylink, ifr, cmd);
}
+static int mvneta_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
+ struct netlink_ext_ack *extack)
+{
+ struct mvneta_port *pp = netdev_priv(dev);
+ struct bpf_prog *old_prog;
+
+ if (prog && dev->mtu > MVNETA_MAX_RX_BUF_SIZE) {
+ NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
+ return -EOPNOTSUPP;
+ }
+
+ if (netif_running(dev))
+ mvneta_stop(dev);
+
+ old_prog = xchg(&pp->xdp_prog, prog);
+ if (old_prog)
+ bpf_prog_put(old_prog);
+
+ if (netif_running(dev))
+ mvneta_open(dev);
+
+ return 0;
+}
+
+static int mvneta_xdp(struct net_device *dev, struct netdev_bpf *xdp)
+{
+ struct mvneta_port *pp = netdev_priv(dev);
+
+ switch (xdp->command) {
+ case XDP_SETUP_PROG:
+ return mvneta_xdp_setup(dev, xdp->prog, xdp->extack);
+ case XDP_QUERY_PROG:
+ xdp->prog_id = pp->xdp_prog ? pp->xdp_prog->aux->id : 0;
+ return 0;
+ default:
+ NL_SET_ERR_MSG_MOD(xdp->extack, "unknown XDP command");
+ return -EINVAL;
+ }
+}
+
/* Ethtool methods */
/* Set link ksettings (phy address, speed) for ethtools */
@@ -4375,6 +4502,7 @@ static const struct net_device_ops mvneta_netdev_ops = {
.ndo_fix_features = mvneta_fix_features,
.ndo_get_stats64 = mvneta_get_stats64,
.ndo_do_ioctl = mvneta_ioctl,
+ .ndo_bpf = mvneta_xdp,
};
static const struct ethtool_ops mvneta_eth_tool_ops = {
@@ -4665,7 +4793,7 @@ static int mvneta_probe(struct platform_device *pdev)
SET_NETDEV_DEV(dev, &pdev->dev);
pp->id = global_port_id++;
- pp->rx_offset_correction = NET_SKB_PAD;
+ pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
/* Obtain access to BM resources if enabled and already initialized */
bm_node = of_parse_phandle(dn, "buffer-manager", 0);
--
2.21.0
Powered by blists - more mailing lists