linux-kernel - [PATCH v5 6/7] AutoFDO: Enable machine function split optimization for AutoFDO

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-ID: <20241023224409.201771-7-xur@google.com>
Date: Wed, 23 Oct 2024 15:44:05 -0700
From: Rong Xu <xur@...gle.com>
To: Alice Ryhl <aliceryhl@...gle.com>, Andrew Morton <akpm@...ux-foundation.org>, 
	Arnd Bergmann <arnd@...db.de>, Bill Wendling <morbo@...gle.com>, Borislav Petkov <bp@...en8.de>, 
	Breno Leitao <leitao@...ian.org>, Brian Gerst <brgerst@...il.com>, 
	Dave Hansen <dave.hansen@...ux.intel.com>, David Li <davidxl@...gle.com>, 
	Han Shen <shenhan@...gle.com>, Heiko Carstens <hca@...ux.ibm.com>, "H. Peter Anvin" <hpa@...or.com>, 
	Ingo Molnar <mingo@...hat.com>, Jann Horn <jannh@...gle.com>, Jonathan Corbet <corbet@....net>, 
	Josh Poimboeuf <jpoimboe@...nel.org>, Juergen Gross <jgross@...e.com>, 
	Justin Stitt <justinstitt@...gle.com>, Kees Cook <kees@...nel.org>, 
	Masahiro Yamada <masahiroy@...nel.org>, "Mike Rapoport (IBM)" <rppt@...nel.org>, 
	Nathan Chancellor <nathan@...nel.org>, Nick Desaulniers <ndesaulniers@...gle.com>, 
	Nicolas Schier <nicolas@...sle.eu>, "Paul E. McKenney" <paulmck@...nel.org>, 
	Peter Zijlstra <peterz@...radead.org>, Rong Xu <xur@...gle.com>, 
	Sami Tolvanen <samitolvanen@...gle.com>, Thomas Gleixner <tglx@...utronix.de>, 
	Wei Yang <richard.weiyang@...il.com>, workflows@...r.kernel.org, 
	Miguel Ojeda <miguel.ojeda.sandonis@...il.com>, Maksim Panchenko <max4bolt@...il.com>, 
	Yonghong Song <yonghong.song@...ux.dev>, Yabin Cui <yabinc@...gle.com>, 
	Krzysztof Pszeniczny <kpszeniczny@...gle.com>, Sriraman Tallam <tmsriram@...gle.com>, 
	Stephane Eranian <eranian@...gle.com>
Cc: x86@...nel.org, linux-arch@...r.kernel.org, linux-doc@...r.kernel.org, 
	linux-kbuild@...r.kernel.org, linux-kernel@...r.kernel.org, 
	llvm@...ts.linux.dev
Subject: [PATCH v5 6/7] AutoFDO: Enable machine function split optimization
 for AutoFDO

Enable the machine function split optimization for AutoFDO in Clang.

Machine function split (MFS) is a pass in the Clang compiler that
splits a function into hot and cold parts. The linker groups all
cold blocks across functions together. This decreases hot code
fragmentation and improves iCache and iTLB utilization.

MFS requires a profile so this is enabled only for the AutoFDO builds.

Co-developed-by: Han Shen <shenhan@...gle.com>
Signed-off-by: Han Shen <shenhan@...gle.com>
Signed-off-by: Rong Xu <xur@...gle.com>
Suggested-by: Sriraman Tallam <tmsriram@...gle.com>
Suggested-by: Krzysztof Pszeniczny <kpszeniczny@...gle.com>
Tested-by: Yonghong Song <yonghong.song@...ux.dev>
---
 include/asm-generic/vmlinux.lds.h | 7 ++++++-
 scripts/Makefile.autofdo          | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index bd64fdedabd2..8a0bb3946cf0 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -556,6 +556,11 @@ defined(CONFIG_AUTOFDO_CLANG)
 		__cpuidle_text_end = .;					\
 		__noinstr_text_end = .;
 
+#define TEXT_SPLIT							\
+		__split_text_start = .;					\
+		*(.text.split .text.split.[0-9a-zA-Z_]*)		\
+		__split_text_end = .;
+
 #define TEXT_UNLIKELY							\
 		__unlikely_text_start = .;				\
 		*(.text.unlikely .text.unlikely.*)			\
@@ -582,6 +587,7 @@ defined(CONFIG_AUTOFDO_CLANG)
 		ALIGN_FUNCTION();					\
 		*(.text.asan.* .text.tsan.*)				\
 		*(.text.unknown .text.unknown.*)			\
+		TEXT_SPLIT						\
 		TEXT_UNLIKELY						\
 		. = ALIGN(PAGE_SIZE);					\
 		TEXT_HOT						\
@@ -589,7 +595,6 @@ defined(CONFIG_AUTOFDO_CLANG)
 		NOINSTR_TEXT						\
 		*(.ref.text)
 
-
 /* sched.text is aling to function alignment to secure we have same
  * address even at second ld pass when generating System.map */
 #define SCHED_TEXT							\
diff --git a/scripts/Makefile.autofdo b/scripts/Makefile.autofdo
index 6155d6fc4ca7..1caf2457e585 100644
--- a/scripts/Makefile.autofdo
+++ b/scripts/Makefile.autofdo
@@ -10,6 +10,7 @@ endif
 
 ifdef CLANG_AUTOFDO_PROFILE
   CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections
+  CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions
 endif
 
 ifdef CONFIG_LTO_CLANG_THIN
@@ -17,6 +18,7 @@ ifdef CONFIG_LTO_CLANG_THIN
     KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE)
   endif
   KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true --mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto
+  KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions
 endif
 
 export CFLAGS_AUTOFDO_CLANG
-- 
2.47.0.105.g07ac214952-goog