Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2024-12-28 16:53:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

AutoFDO: Enable machine function split optimization for AutoFDO

Browse Source 
Enable the machine function split optimization for AutoFDO in Clang.

Machine function split (MFS) is a pass in the Clang compiler that
splits a function into hot and cold parts. The linker groups all
cold blocks across functions together. This decreases hot code
fragmentation and improves iCache and iTLB utilization.

MFS requires a profile so this is enabled only for the AutoFDO builds.

Co-developed-by: Han Shen <shenhan@google.com>
Signed-off-by: Han Shen <shenhan@google.com>
Signed-off-by: Rong Xu <xur@google.com>
Suggested-by: Sriraman Tallam <tmsriram@google.com>
Suggested-by: Krzysztof Pszeniczny <kpszeniczny@google.com>
Tested-by: Yonghong Song <yonghong.song@linux.dev>
Tested-by: Yabin Cui <yabinc@google.com>
Tested-by: Nathan Chancellor <nathan@kernel.org>
Reviewed-by: Kees Cook <kees@kernel.org>
Signed-off-by: Masahiro Yamada <masahiroy@kernel.org>
This commit is contained in:
Rong Xu 2024-11-02 10:51:13 -07:00 committed by Masahiro Yamada
parent 0847420f5e
commit 2fd65f7afd
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/asm-generic/vmlinux.lds.h
View File

@ -556,6 +556,11 @@ defined(CONFIG_AUTOFDO_CLANG)
__cpuidle_text_end = .; \ __cpuidle_text_end = .; \
__noinstr_text_end = .; __noinstr_text_end = .;
#define TEXT_SPLIT \
__split_text_start = .; \
*(.text.split .text.split.[0-9a-zA-Z_]*) \
__split_text_end = .;
#define TEXT_UNLIKELY \ #define TEXT_UNLIKELY \
__unlikely_text_start = .; \ __unlikely_text_start = .; \
*(.text.unlikely .text.unlikely.*) \ *(.text.unlikely .text.unlikely.*) \
@ -582,6 +587,7 @@ defined(CONFIG_AUTOFDO_CLANG)
ALIGN_FUNCTION(); \ ALIGN_FUNCTION(); \
*(.text.asan.* .text.tsan.*) \ *(.text.asan.* .text.tsan.*) \
*(.text.unknown .text.unknown.*) \ *(.text.unknown .text.unknown.*) \
TEXT_SPLIT \
TEXT_UNLIKELY \ TEXT_UNLIKELY \
. = ALIGN(PAGE_SIZE); \ . = ALIGN(PAGE_SIZE); \
TEXT_HOT \ TEXT_HOT \
@ -589,7 +595,6 @@ defined(CONFIG_AUTOFDO_CLANG)
NOINSTR_TEXT \ NOINSTR_TEXT \
*(.ref.text) *(.ref.text)
/* sched.text is aling to function alignment to secure we have same /* sched.text is aling to function alignment to secure we have same
* address even at second ld pass when generating System.map */ * address even at second ld pass when generating System.map */
#define SCHED_TEXT \ #define SCHED_TEXT \

2
scripts/Makefile.autofdo
View File

@ -10,6 +10,7 @@ endif
ifdef CLANG_AUTOFDO_PROFILE ifdef CLANG_AUTOFDO_PROFILE
CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections
CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions
endif endif
ifdef CONFIG_LTO_CLANG_THIN ifdef CONFIG_LTO_CLANG_THIN
@ -17,6 +18,7 @@ ifdef CONFIG_LTO_CLANG_THIN
KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE) KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE)
endif endif
KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true --mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true --mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto
KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions
endif endif
export CFLAGS_AUTOFDO_CLANG export CFLAGS_AUTOFDO_CLANG