| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Fri, 14 Sep 2018 17:43:22 +0200
From: Paolo Abeni <pabeni@...hat.com>
To: netdev@...r.kernel.org
Cc: "David S. Miller" <davem@...emloft.net>,
Willem de Bruijn <willemb@...gle.com>,
Steffen Klassert <steffen.klassert@...unet.com>
Subject: [RFC PATCH 2/4] net: enable UDP gro on demand.
Currently, the UDP GRO callback is always invoked, regardless of
the existence of any actual user (e.g. a UDP tunnel). With retpoline
enabled, this causes measurable overhead.
This changeset introduces explicit accounting of the sockets requiring
UDP GRO and updates the UDP offloads at runtime accordingly, so that
the GRO callback is present (and invoked) only when there is at least
one socket requiring it.
Tested with pktgen vs udpgso_bench_rx
Before:
udp rx: 27 MB/s 1613271 calls/s
After:
udp rx: 30 MB/s 1771537 calls/s
Signed-off-by: Paolo Abeni <pabeni@...hat.com>
---
include/linux/udp.h | 18 +++++++++++-
include/net/addrconf.h | 1 +
include/net/udp.h | 12 ++++++++
net/ipv4/udp.c | 2 ++
net/ipv4/udp_offload.c | 63 ++++++++++++++++++++++++++++++++++++++++--
net/ipv4/udp_tunnel.c | 1 +
net/ipv6/af_inet6.c | 1 +
net/ipv6/udp_offload.c | 25 +++++++++++++++--
8 files changed, 117 insertions(+), 6 deletions(-)
diff --git a/include/linux/udp.h b/include/linux/udp.h
index 320d49d85484..56a321a55ba1 100644
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -49,7 +49,8 @@ struct udp_sock {
unsigned int corkflag; /* Cork is required */
__u8 encap_type; /* Is this an Encapsulation socket? */
unsigned char no_check6_tx:1,/* Send zero UDP6 checksums on TX? */
- no_check6_rx:1;/* Allow zero UDP6 checksums on RX? */
+ no_check6_rx:1,/* Allow zero UDP6 checksums on RX? */
+ gro_in_use:1; /* UDP GRO is requested */
/*
* Following member retains the information to create a UDP header
* when the socket is uncorked.
@@ -105,6 +106,11 @@ static inline void udp_set_no_check6_rx(struct sock *sk, bool val)
udp_sk(sk)->no_check6_rx = val;
}
+static inline void udp_set_gro_in_use(struct sock *sk, bool val)
+{
+ udp_sk(sk)->gro_in_use = val;
+}
+
static inline bool udp_get_no_check6_tx(struct sock *sk)
{
return udp_sk(sk)->no_check6_tx;
@@ -115,6 +121,16 @@ static inline bool udp_get_no_check6_rx(struct sock *sk)
return udp_sk(sk)->no_check6_rx;
}
+static inline bool udp_get_gro_in_use(struct sock *sk)
+{
+ return udp_sk(sk)->gro_in_use;
+}
+
+static inline bool udp_want_gro(struct sock *sk)
+{
+ return udp_sk(sk)->gro_receive;
+}
+
#define udp_portaddr_for_each_entry(__sk, list) \
hlist_for_each_entry(__sk, list, __sk_common.skc_portaddr_node)
diff --git a/include/net/addrconf.h b/include/net/addrconf.h
index 6def0351bcc3..fb2ac3ca3417 100644
--- a/include/net/addrconf.h
+++ b/include/net/addrconf.h
@@ -254,6 +254,7 @@ struct ipv6_stub {
struct in6_addr *saddr);
void (*udpv6_encap_enable)(void);
+ void (*udpv6_update_offload)(bool enable_gro);
void (*ndisc_send_na)(struct net_device *dev, const struct in6_addr *daddr,
const struct in6_addr *solicited_addr,
bool router, bool solicited, bool override, bool inc_opt);
diff --git a/include/net/udp.h b/include/net/udp.h
index 8482a990b0bb..eff2dfa0571b 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -444,8 +444,20 @@ int udpv4_offload_init(void);
void udp_init(void);
void udp_encap_enable(void);
+void udp_gro_in_use_changed(struct sock *sk);
+
#if IS_ENABLED(CONFIG_IPV6)
void udpv6_encap_enable(void);
+void udpv6_update_offload(bool);
#endif
+static inline void udp_update_gro_in_use(struct sock *sk, bool want_gro)
+{
+ if (want_gro == udp_get_gro_in_use(sk))
+ return;
+
+ udp_set_gro_in_use(sk, want_gro);
+ udp_gro_in_use_changed(sk);
+}
+
#endif /* _UDP_H */
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index f4e35b2ff8b8..5ac794230013 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1438,6 +1438,8 @@ void udp_destruct_sock(struct sock *sk)
}
udp_rmem_release(sk, total, 0, true);
+ udp_update_gro_in_use(sk, 0);
+
inet_sock_destruct(sk);
}
EXPORT_SYMBOL_GPL(udp_destruct_sock);
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index 0c0522b79b43..08b225adf763 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -14,6 +14,10 @@
#include <net/udp.h>
#include <net/protocol.h>
+#if IS_ENABLED(CONFIG_IPV6)
+#include <net/addrconf.h>
+#endif
+
static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
netdev_features_t features,
struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
@@ -472,7 +476,13 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
}
-static const struct net_offload udpv4_offload = {
+static const struct net_offload udpv4_no_gro_offload = {
+ .callbacks = {
+ .gso_segment = udp4_ufo_fragment,
+ },
+};
+
+static const struct net_offload udpv4_gro_offload = {
.callbacks = {
.gso_segment = udp4_ufo_fragment,
.gro_receive = udp4_gro_receive,
@@ -480,7 +490,56 @@ static const struct net_offload udpv4_offload = {
},
};
+static atomic_t gro_needed;
+
+static void udp_update_offload(struct sock *sk, bool enable_gro)
+{
+ int ret;
+
+#if IS_ENABLED(CONFIG_IPV6)
+ /* to keep the code simple ignore IPV6_V6ONLY */
+ if (sk->sk_family == PF_INET6)
+ ipv6_stub->udpv6_update_offload(enable_gro);
+#endif
+
+ if (enable_gro)
+ ret = inet_update_offload(&udpv4_no_gro_offload,
+ &udpv4_gro_offload, IPPROTO_UDP);
+ else
+ ret = inet_update_offload(&udpv4_gro_offload,
+ &udpv4_no_gro_offload, IPPROTO_UDP);
+ WARN_ON(ret);
+}
+
+static void udp_gro_enable(struct sock *sk)
+{
+ if (atomic_inc_return(&gro_needed) > 1)
+ return;
+
+ udp_update_offload(sk, true);
+}
+
+static void udp_gro_disable(struct sock *sk)
+{
+ int new_gro_needed = atomic_dec_return(&gro_needed);
+
+ WARN_ON(new_gro_needed < 0);
+ if (new_gro_needed != 0)
+ return;
+
+ udp_update_offload(sk, false);
+}
+
+void udp_gro_in_use_changed(struct sock *sk)
+{
+ if (udp_get_gro_in_use(sk))
+ udp_gro_enable(sk);
+ else
+ udp_gro_disable(sk);
+}
+EXPORT_SYMBOL_GPL(udp_gro_in_use_changed);
+
int __init udpv4_offload_init(void)
{
- return inet_add_offload(&udpv4_offload, IPPROTO_UDP);
+ return inet_add_offload(&udpv4_no_gro_offload, IPPROTO_UDP);
}
diff --git a/net/ipv4/udp_tunnel.c b/net/ipv4/udp_tunnel.c
index 6539ff15e9a3..16457d41ff7f 100644
--- a/net/ipv4/udp_tunnel.c
+++ b/net/ipv4/udp_tunnel.c
@@ -73,6 +73,7 @@ void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
udp_sk(sk)->gro_complete = cfg->gro_complete;
udp_tunnel_encap_enable(sock);
+ udp_update_gro_in_use(sk, udp_want_gro(sk));
}
EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index 77ef8478234f..ecb8aba7b535 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -897,6 +897,7 @@ static const struct ipv6_stub ipv6_stub_impl = {
.fib6_multipath_select = fib6_multipath_select,
.ip6_mtu_from_fib6 = ip6_mtu_from_fib6,
.udpv6_encap_enable = udpv6_encap_enable,
+ .udpv6_update_offload = udpv6_update_offload,
.ndisc_send_na = ndisc_send_na,
.nd_tbl = &nd_tbl,
};
diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c
index 95dee9ca8d22..1df968a3e788 100644
--- a/net/ipv6/udp_offload.c
+++ b/net/ipv6/udp_offload.c
@@ -158,7 +158,13 @@ static int udp6_gro_complete(struct sk_buff *skb, int nhoff)
return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
}
-static const struct net_offload udpv6_offload = {
+static const struct net_offload udpv6_no_gro_offload = {
+ .callbacks = {
+ .gso_segment = udp6_ufo_fragment,
+ },
+};
+
+static const struct net_offload udpv6_gro_offload = {
.callbacks = {
.gso_segment = udp6_ufo_fragment,
.gro_receive = udp6_gro_receive,
@@ -166,12 +172,25 @@ static const struct net_offload udpv6_offload = {
},
};
+void udpv6_update_offload(bool enable_gro)
+{
+ int ret;
+
+ if (enable_gro)
+ ret = inet6_update_offload(&udpv6_no_gro_offload,
+ &udpv6_gro_offload, IPPROTO_UDP);
+ else
+ ret = inet6_update_offload(&udpv6_gro_offload,
+ &udpv6_no_gro_offload, IPPROTO_UDP);
+ WARN_ON(ret);
+}
+
int udpv6_offload_init(void)
{
- return inet6_add_offload(&udpv6_offload, IPPROTO_UDP);
+ return inet6_add_offload(&udpv6_no_gro_offload, IPPROTO_UDP);
}
int udpv6_offload_exit(void)
{
- return inet6_del_offload(&udpv6_offload, IPPROTO_UDP);
+ return inet6_del_offload(&udpv6_no_gro_offload, IPPROTO_UDP);
}
--
2.17.1
Powered by blists - more mailing lists