netdev - Re: [PATCH v2 net-next 0/5] Analyze and Reorganize core Networking Structs to optimize cacheline consumption

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-ID: <50ca7bc1-e5c1-cb79-b2af-e5cd83b54dab@iogearbox.net>
Date: Tue, 17 Oct 2023 23:21:57 +0200
From: Daniel Borkmann <daniel@...earbox.net>
To: Eric Dumazet <edumazet@...gle.com>
Cc: Florian Fainelli <f.fainelli@...il.com>, Coco Li <lixiaoyan@...gle.com>,
 Jakub Kicinski <kuba@...nel.org>, Neal Cardwell <ncardwell@...gle.com>,
 Mubashir Adnan Qureshi <mubashirq@...gle.com>,
 Paolo Abeni <pabeni@...hat.com>, netdev@...r.kernel.org,
 Chao Wu <wwchao@...gle.com>, Wei Wang <weiwan@...gle.com>
Subject: Re: [PATCH v2 net-next 0/5] Analyze and Reorganize core Networking
 Structs to optimize cacheline consumption

On 10/17/23 7:07 PM, Daniel Borkmann wrote:
> On 10/17/23 6:50 PM, Eric Dumazet wrote:
[...]
>> Great idea, we only need to generate these automatically from the file
>> describing the fields (currently in Documentation/ )
>>
>> I think the initial intent was to find a way to generate the layout of
>> the structure itself, but this looked a bit tricky.
> 
> Agree, ideally this could be scripted from the Documentation/ file of this
> series, and perhaps the latter may not even be needed then if we have it
> self-documented in code behind some macro magic with BUILD_BUG_ON assertion
> which probes offsetof wrt the field being within markers.

... been playing around a bit, perhaps could be made nicer but this seems
to do it & also pahole will have the markers visible:

  include/linux/cache.h     | 26 ++++++++++++++++++++++++++
  include/linux/netdevice.h |  2 ++
  net/core/dev.c            | 25 +++++++++++++++++++++++++
  3 files changed, 53 insertions(+)

diff --git a/include/linux/cache.h b/include/linux/cache.h
index 9900d20b76c2..f7e166b2897a 100644
--- a/include/linux/cache.h
+++ b/include/linux/cache.h
@@ -85,6 +85,32 @@
  #define cache_line_size()	L1_CACHE_BYTES
  #endif

+#ifndef __cacheline_group_begin
+#define __cacheline_group_begin(GROUP) \
+	__u8 __cacheline_group_begin__##GROUP[0]
+#endif
+
+#ifndef __cacheline_group_end
+#define __cacheline_group_end(GROUP) \
+	__u8 __cacheline_group_end__##GROUP[0]
+#endif
+
+#ifndef CACHELINE_ASSERT_GROUP_MEMBER
+#define CACHELINE_ASSERT_GROUP_MEMBER(TYPE, GROUP, MEMBER) \
+	BUILD_BUG_ON(!(offsetof(TYPE, MEMBER) >= \
+		       offsetofend(TYPE, __cacheline_group_begin__##GROUP) && \
+		       offsetofend(TYPE, MEMBER) <= \
+		       offsetof(TYPE, __cacheline_group_end__##GROUP)))
+#endif
+
+#ifndef CACHELINE_ASSERT_GROUP_MAXSIZ
+#define CACHELINE_ASSERT_MIN_BOUNDARY 64
+#define CACHELINE_ASSERT_GROUP_MAXSIZ(TYPE, GROUP, NUM) \
+	BUILD_BUG_ON(offsetof(TYPE, __cacheline_group_end__##GROUP) - \
+		     offsetofend(TYPE, __cacheline_group_begin__##GROUP) > \
+		     ((NUM) * CACHELINE_ASSERT_MIN_BOUNDARY))
+#endif
+
  /*
   * Helper to add padding within a struct to ensure data fall into separate
   * cachelines.
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index d72b71b76bf8..7a47d43b95de 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2059,6 +2059,7 @@ struct net_device {
  	 */

  	/* TX read-mostly hotpath */
+	__cacheline_group_begin(tx_read_mostly);
  	unsigned long long	priv_flags;
  	const struct net_device_ops *netdev_ops;
  	const struct header_ops *header_ops;
@@ -2082,6 +2083,7 @@ struct net_device {
  #ifdef CONFIG_NETFILTER_EGRESS
  	struct nf_hook_entries __rcu *nf_hooks_egress;
  #endif
+	__cacheline_group_end(tx_read_mostly);

  	/* TXRX read-mostly hotpath */
  	unsigned int		flags;
diff --git a/net/core/dev.c b/net/core/dev.c
index 4420831180c6..5f6b88c2c902 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -11515,6 +11515,29 @@ static struct pernet_operations __net_initdata default_device_ops = {
   *
   */

+static void __init net_dev_struct_check(void)
+{
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, priv_flags);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, netdev_ops);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, header_ops);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, _tx);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, real_num_tx_queues);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, gso_max_size);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, gso_ipv4_max_size);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, gso_max_segs);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, num_tc);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, mtu);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, needed_headroom);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, tc_to_txq);
+#ifdef CONFIG_XPS
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, xps_maps);
+#endif
+#ifdef CONFIG_NETFILTER_EGRESS
+	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, tx_read_mostly, nf_hooks_egress);
+#endif
+	CACHELINE_ASSERT_GROUP_MAXSIZ(struct net_device, tx_read_mostly, 3);
+}
+
  /*
   *       This is called single threaded during boot, so no need
   *       to take the rtnl semaphore.
@@ -11525,6 +11548,8 @@ static int __init net_dev_init(void)

  	BUG_ON(!dev_boot_phase);

+	net_dev_struct_check();
+
  	if (dev_proc_init())
  		goto out;