netdev - [PATCH RFC bpf-next 41/52] net, xdp: replace net_device::xdp_prog pointer with &xdp_attachment

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <20220628194812.1453059-42-alexandr.lobakin@intel.com>
Date:   Tue, 28 Jun 2022 21:48:01 +0200
From:   Alexander Lobakin <alexandr.lobakin@...el.com>
To:     Alexei Starovoitov <ast@...nel.org>,
        Daniel Borkmann <daniel@...earbox.net>,
        Andrii Nakryiko <andrii@...nel.org>
Cc:     Alexander Lobakin <alexandr.lobakin@...el.com>,
        Larysa Zaremba <larysa.zaremba@...el.com>,
        Michal Swiatkowski <michal.swiatkowski@...ux.intel.com>,
        Jesper Dangaard Brouer <hawk@...nel.org>,
        Björn Töpel <bjorn@...nel.org>,
        Magnus Karlsson <magnus.karlsson@...el.com>,
        Maciej Fijalkowski <maciej.fijalkowski@...el.com>,
        Jonathan Lemon <jonathan.lemon@...il.com>,
        Toke Hoiland-Jorgensen <toke@...hat.com>,
        Lorenzo Bianconi <lorenzo@...nel.org>,
        "David S. Miller" <davem@...emloft.net>,
        Eric Dumazet <edumazet@...gle.com>,
        Jakub Kicinski <kuba@...nel.org>,
        Paolo Abeni <pabeni@...hat.com>,
        Jesse Brandeburg <jesse.brandeburg@...el.com>,
        John Fastabend <john.fastabend@...il.com>,
        Yajun Deng <yajun.deng@...ux.dev>,
        Willem de Bruijn <willemb@...gle.com>, bpf@...r.kernel.org,
        netdev@...r.kernel.org, linux-kernel@...r.kernel.org,
        xdp-hints@...-project.net
Subject: [PATCH RFC bpf-next 41/52] net, xdp: replace net_device::xdp_prog pointer with &xdp_attachment_info

To have access and store not only BPF prog pointer, but also
auxiliary params on Generic (skb) XDP path, replace it with
an &xdp_attachment_info struct and use xdp_attachment_setup_rcu()
(since Generic XDP code RCU-protects the pointer already).
This slightly changes the struct &net_device cacheline layout, but
nothing performance-critical.

Signed-off-by: Alexander Lobakin <alexandr.lobakin@...el.com>
---
 include/linux/netdevice.h |  7 +++----
 net/bpf/dev.c             | 11 ++++-------
 net/core/dev.c            |  4 +++-
 net/core/rtnetlink.c      |  2 +-
 4 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 60df42b3f116..1c033c164257 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2168,7 +2168,7 @@ struct net_device {
 	unsigned int		num_rx_queues;
 	unsigned int		real_num_rx_queues;
 
-	struct bpf_prog __rcu	*xdp_prog;
+	struct xdp_attachment_info xdp_info;
 	unsigned long		gro_flush_timeout;
 	int			napi_defer_hard_irqs;
 #define GRO_LEGACY_MAX_SIZE	65536u
@@ -2343,9 +2343,8 @@ struct net_device {
 
 static inline bool netif_elide_gro(const struct net_device *dev)
 {
-	if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
-		return true;
-	return false;
+	return !(dev->features & NETIF_F_GRO) ||
+	       rcu_access_pointer(dev->xdp_info.prog_rcu);
 }
 
 #define	NETDEV_ALIGN		32
diff --git a/net/bpf/dev.c b/net/bpf/dev.c
index 82948d0536c8..cc43f73929f3 100644
--- a/net/bpf/dev.c
+++ b/net/bpf/dev.c
@@ -242,19 +242,16 @@ static void dev_disable_gro_hw(struct net_device *dev)
 
 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
 {
-	struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
-	struct bpf_prog *new = xdp->prog;
+	bool old = !!rtnl_dereference(dev->xdp_info.prog_rcu);
 	int ret = 0;
 
 	switch (xdp->command) {
 	case XDP_SETUP_PROG:
-		rcu_assign_pointer(dev->xdp_prog, new);
-		if (old)
-			bpf_prog_put(old);
+		xdp_attachment_setup_rcu(&dev->xdp_info, xdp);
 
-		if (old && !new) {
+		if (old && !xdp->prog) {
 			static_branch_dec(&generic_xdp_needed_key);
-		} else if (new && !old) {
+		} else if (xdp->prog && !old) {
 			static_branch_inc(&generic_xdp_needed_key);
 			dev_disable_lro(dev);
 			dev_disable_gro_hw(dev);
diff --git a/net/core/dev.c b/net/core/dev.c
index 62bf6ee00741..e57ae87d619e 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5055,10 +5055,12 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
 	__this_cpu_inc(softnet_data.processed);
 
 	if (static_branch_unlikely(&generic_xdp_needed_key)) {
+		struct bpf_prog *prog;
 		int ret2;
 
 		migrate_disable();
-		ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
+		prog = rcu_dereference(skb->dev->xdp_info.prog_rcu);
+		ret2 = do_xdp_generic(prog, skb);
 		migrate_enable();
 
 		if (ret2 != XDP_PASS) {
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 500420d5017c..72f696b12df2 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -1451,7 +1451,7 @@ static u32 rtnl_xdp_prog_skb(struct net_device *dev)
 
 	ASSERT_RTNL();
 
-	generic_xdp_prog = rtnl_dereference(dev->xdp_prog);
+	generic_xdp_prog = rtnl_dereference(dev->xdp_info.prog_rcu);
 	if (!generic_xdp_prog)
 		return 0;
 	return generic_xdp_prog->aux->id;
-- 
2.36.1