Kernel/linux-next
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git synced 2024-12-29 17:22:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ARM: 9384/2: mm: Make tlbflush routines CFI safe

Browse Source 
Instead of avoiding CFI entirely on the TLB flush helpers, reorganize
the code so that the CFI machinery can deal with it. The important
things to take into account are:
- functions in asm called indirectly from C need to be defined using
  SYM_TYPED_FUNC_START()
- a reference to the asm function needs to be visible to the compiler,
  in order to get it to emit the typeid symbol.

The latter means that defining the cpu_tlb_fns structs is best done from
C code, so that the references in the static initializers will be
visible to the compiler.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Tested-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Sami Tolvanen <samitolvanen@google.com>
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
This commit is contained in:
Ard Biesheuvel 2024-04-23 08:25:41 +01:00 committed by Russell King (Oracle)
parent 4cece76496
commit 6b0ef2792c
9 changed files with 119 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/arm/mm/Makefile
View File

@ -62,6 +62,7 @@ obj-$(CONFIG_CPU_TLB_FEROCEON) += tlb-v4wbi.o # reuse v4wbi TLB functions
obj-$(CONFIG_CPU_TLB_V6) += tlb-v6.o
obj-$(CONFIG_CPU_TLB_V7) += tlb-v7.o
obj-$(CONFIG_CPU_TLB_FA) += tlb-fa.o
obj-y += tlb.o
obj-$(CONFIG_CPU_ARM7TDMI) += proc-arm7tdmi.o
obj-$(CONFIG_CPU_ARM720T) += proc-arm720.o

15
arch/arm/mm/proc-macros.S
View File

@ -338,21 +338,6 @@ ENTRY(\name\()_cache_fns)
.size \name\()_cache_fns, . - \name\()_cache_fns
.endm
.macro define_tlb_functions name:req, flags_up:req, flags_smp
.type \name\()_tlb_fns, #object
.align 2
ENTRY(\name\()_tlb_fns)
.long \name\()_flush_user_tlb_range
.long \name\()_flush_kern_tlb_range
.ifnb \flags_smp
ALT_SMP(.long \flags_smp )
ALT_UP(.long \flags_up )
.else
.long \flags_up
.endif
.size \name\()_tlb_fns, . - \name\()_tlb_fns
.endm
.macro globl_equ x, y
.globl \x
.equ \x, \y

12
arch/arm/mm/tlb-fa.S
View File

@ -15,6 +15,7 @@
*/
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/tlbflush.h>
@ -31,7 +32,7 @@
* - mm - mm_struct describing address space
*/
.align 4
ENTRY(fa_flush_user_tlb_range)
SYM_TYPED_FUNC_START(fa_flush_user_tlb_range)
vma_vm_mm ip, r2
act_mm r3 @ get current->active_mm
eors r3, ip, r3 @ == mm ?
@ -46,9 +47,10 @@ ENTRY(fa_flush_user_tlb_range)
blo 1b
mcr p15, 0, r3, c7, c10, 4 @ data write barrier
ret lr
SYM_FUNC_END(fa_flush_user_tlb_range)
ENTRY(fa_flush_kern_tlb_range)
SYM_TYPED_FUNC_START(fa_flush_kern_tlb_range)
mov r3, #0
mcr p15, 0, r3, c7, c10, 4 @ drain WB
bic r0, r0, #0x0ff
@ -60,8 +62,4 @@ ENTRY(fa_flush_kern_tlb_range)
mcr p15, 0, r3, c7, c10, 4 @ data write barrier
mcr p15, 0, r3, c7, c5, 4 @ prefetch flush (isb)
ret lr
__INITDATA
/* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
define_tlb_functions fa, fa_tlb_flags
SYM_FUNC_END(fa_flush_kern_tlb_range)

15
arch/arm/mm/tlb-v4.S
View File

@ -11,6 +11,7 @@
*/
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/tlbflush.h>
@ -27,7 +28,7 @@
* - mm - mm_struct describing address space
*/
.align 5
ENTRY(v4_flush_user_tlb_range)
SYM_TYPED_FUNC_START(v4_flush_user_tlb_range)
vma_vm_mm ip, r2
act_mm r3 @ get current->active_mm
eors r3, ip, r3 @ == mm ?
@ -40,6 +41,7 @@ ENTRY(v4_flush_user_tlb_range)
cmp r0, r1
blo 1b
ret lr
SYM_FUNC_END(v4_flush_user_tlb_range)
/*
* v4_flush_kern_tlb_range(start, end)
@ -50,10 +52,11 @@ ENTRY(v4_flush_user_tlb_range)
* - start - virtual address (may not be aligned)
* - end - virtual address (may not be aligned)
*/
#ifdef CONFIG_CFI_CLANG
SYM_TYPED_FUNC_START(v4_flush_kern_tlb_range)
b .v4_flush_kern_tlb_range
SYM_FUNC_END(v4_flush_kern_tlb_range)
#else
.globl v4_flush_kern_tlb_range
.equ v4_flush_kern_tlb_range, .v4_flush_kern_tlb_range
__INITDATA
/* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
define_tlb_functions v4, v4_tlb_flags
#endif

12
arch/arm/mm/tlb-v4wb.S
View File

@ -11,6 +11,7 @@
*/
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/tlbflush.h>
@ -27,7 +28,7 @@
* - mm - mm_struct describing address space
*/
.align 5
ENTRY(v4wb_flush_user_tlb_range)
SYM_TYPED_FUNC_START(v4wb_flush_user_tlb_range)
vma_vm_mm ip, r2
act_mm r3 @ get current->active_mm
eors r3, ip, r3 @ == mm ?
@ -43,6 +44,7 @@ ENTRY(v4wb_flush_user_tlb_range)
cmp r0, r1
blo 1b
ret lr
SYM_FUNC_END(v4wb_flush_user_tlb_range)
/*
* v4_flush_kern_tlb_range(start, end)
@ -53,7 +55,7 @@ ENTRY(v4wb_flush_user_tlb_range)
* - start - virtual address (may not be aligned)
* - end - virtual address (may not be aligned)
*/
ENTRY(v4wb_flush_kern_tlb_range)
SYM_TYPED_FUNC_START(v4wb_flush_kern_tlb_range)
mov r3, #0
mcr p15, 0, r3, c7, c10, 4 @ drain WB
bic r0, r0, #0x0ff
@ -64,8 +66,4 @@ ENTRY(v4wb_flush_kern_tlb_range)
cmp r0, r1
blo 1b
ret lr
__INITDATA
/* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
define_tlb_functions v4wb, v4wb_tlb_flags
SYM_FUNC_END(v4wb_flush_kern_tlb_range)

12
arch/arm/mm/tlb-v4wbi.S
View File

@ -11,6 +11,7 @@
*/
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/tlbflush.h>
@ -26,7 +27,7 @@
* - mm - mm_struct describing address space
*/
.align 5
ENTRY(v4wbi_flush_user_tlb_range)
SYM_TYPED_FUNC_START(v4wbi_flush_user_tlb_range)
vma_vm_mm ip, r2
act_mm r3 @ get current->active_mm
eors r3, ip, r3 @ == mm ?
@ -43,8 +44,9 @@ ENTRY(v4wbi_flush_user_tlb_range)
cmp r0, r1
blo 1b
ret lr
SYM_FUNC_END(v4wbi_flush_user_tlb_range)
ENTRY(v4wbi_flush_kern_tlb_range)
SYM_TYPED_FUNC_START(v4wbi_flush_kern_tlb_range)
mov r3, #0
mcr p15, 0, r3, c7, c10, 4 @ drain WB
bic r0, r0, #0x0ff
@ -55,8 +57,4 @@ ENTRY(v4wbi_flush_kern_tlb_range)
cmp r0, r1
blo 1b
ret lr
__INITDATA
/* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
define_tlb_functions v4wbi, v4wbi_tlb_flags
SYM_FUNC_END(v4wbi_flush_kern_tlb_range)

12
arch/arm/mm/tlb-v6.S
View File

@ -9,6 +9,7 @@
*/
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
#include <asm/page.h>
@ -32,7 +33,7 @@
* - the "Invalidate single entry" instruction will invalidate
* both the I and the D TLBs on Harvard-style TLBs
*/
ENTRY(v6wbi_flush_user_tlb_range)
SYM_TYPED_FUNC_START(v6wbi_flush_user_tlb_range)
vma_vm_mm r3, r2 @ get vma->vm_mm
mov ip, #0
mmid r3, r3 @ get vm_mm->context.id
@ -56,6 +57,7 @@ ENTRY(v6wbi_flush_user_tlb_range)
blo 1b
mcr p15, 0, ip, c7, c10, 4 @ data synchronization barrier
ret lr
SYM_FUNC_END(v6wbi_flush_user_tlb_range)
/*
* v6wbi_flush_kern_tlb_range(start,end)
@ -65,7 +67,7 @@ ENTRY(v6wbi_flush_user_tlb_range)
* - start - start address (may not be aligned)
* - end - end address (exclusive, may not be aligned)
*/
ENTRY(v6wbi_flush_kern_tlb_range)
SYM_TYPED_FUNC_START(v6wbi_flush_kern_tlb_range)
mov r2, #0
mcr p15, 0, r2, c7, c10, 4 @ drain write buffer
mov r0, r0, lsr #PAGE_SHIFT @ align address
@ -85,8 +87,4 @@ ENTRY(v6wbi_flush_kern_tlb_range)
mcr p15, 0, r2, c7, c10, 4 @ data synchronization barrier
mcr p15, 0, r2, c7, c5, 4 @ prefetch flush (isb)
ret lr
__INIT
/* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
define_tlb_functions v6wbi, v6wbi_tlb_flags
SYM_FUNC_END(v6wbi_flush_kern_tlb_range)

14
arch/arm/mm/tlb-v7.S
View File

@ -10,6 +10,7 @@
*/
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/page.h>
@ -31,7 +32,7 @@
* - the "Invalidate single entry" instruction will invalidate
* both the I and the D TLBs on Harvard-style TLBs
*/
ENTRY(v7wbi_flush_user_tlb_range)
SYM_TYPED_FUNC_START(v7wbi_flush_user_tlb_range)
vma_vm_mm r3, r2 @ get vma->vm_mm
mmid r3, r3 @ get vm_mm->context.id
dsb ish
@ -57,7 +58,7 @@ ENTRY(v7wbi_flush_user_tlb_range)
blo 1b
dsb ish
ret lr
ENDPROC(v7wbi_flush_user_tlb_range)
SYM_FUNC_END(v7wbi_flush_user_tlb_range)
/*
* v7wbi_flush_kern_tlb_range(start,end)
@ -67,7 +68,7 @@ ENDPROC(v7wbi_flush_user_tlb_range)
* - start - start address (may not be aligned)
* - end - end address (exclusive, may not be aligned)
*/
ENTRY(v7wbi_flush_kern_tlb_range)
SYM_TYPED_FUNC_START(v7wbi_flush_kern_tlb_range)
dsb ish
mov r0, r0, lsr #PAGE_SHIFT @ align address
mov r1, r1, lsr #PAGE_SHIFT
@ -86,9 +87,4 @@ ENTRY(v7wbi_flush_kern_tlb_range)
dsb ish
isb
ret lr
ENDPROC(v7wbi_flush_kern_tlb_range)
__INIT
/* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */
define_tlb_functions v7wbi, v7wbi_tlb_flags_up, flags_smp=v7wbi_tlb_flags_smp
SYM_FUNC_END(v7wbi_flush_kern_tlb_range)

84
arch/arm/mm/tlb.c Normal file
View File

@ -0,0 +1,84 @@
// SPDX-License-Identifier: GPL-2.0-only
// Copyright 2024 Google LLC
// Author: Ard Biesheuvel <ardb@google.com>
#include <linux/types.h>
#include <asm/tlbflush.h>
#ifdef CONFIG_CPU_TLB_V4WT
void v4_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
void v4_flush_kern_tlb_range(unsigned long, unsigned long);
struct cpu_tlb_fns v4_tlb_fns __initconst = {
.flush_user_range = v4_flush_user_tlb_range,
.flush_kern_range = v4_flush_kern_tlb_range,
.tlb_flags = v4_tlb_flags,
};
#endif
#ifdef CONFIG_CPU_TLB_V4WB
void v4wb_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
void v4wb_flush_kern_tlb_range(unsigned long, unsigned long);
struct cpu_tlb_fns v4wb_tlb_fns __initconst = {
.flush_user_range = v4wb_flush_user_tlb_range,
.flush_kern_range = v4wb_flush_kern_tlb_range,
.tlb_flags = v4wb_tlb_flags,
};
#endif
#if defined(CONFIG_CPU_TLB_V4WBI) || defined(CONFIG_CPU_TLB_FEROCEON)
void v4wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
void v4wbi_flush_kern_tlb_range(unsigned long, unsigned long);
struct cpu_tlb_fns v4wbi_tlb_fns __initconst = {
.flush_user_range = v4wbi_flush_user_tlb_range,
.flush_kern_range = v4wbi_flush_kern_tlb_range,
.tlb_flags = v4wbi_tlb_flags,
};
#endif
#ifdef CONFIG_CPU_TLB_V6
void v6wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
void v6wbi_flush_kern_tlb_range(unsigned long, unsigned long);
struct cpu_tlb_fns v6wbi_tlb_fns __initconst = {
.flush_user_range = v6wbi_flush_user_tlb_range,
.flush_kern_range = v6wbi_flush_kern_tlb_range,
.tlb_flags = v6wbi_tlb_flags,
};
#endif
#ifdef CONFIG_CPU_TLB_V7
void v7wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
void v7wbi_flush_kern_tlb_range(unsigned long, unsigned long);
struct cpu_tlb_fns v7wbi_tlb_fns __initconst = {
.flush_user_range = v7wbi_flush_user_tlb_range,
.flush_kern_range = v7wbi_flush_kern_tlb_range,
.tlb_flags = IS_ENABLED(CONFIG_SMP) ? v7wbi_tlb_flags_smp
: v7wbi_tlb_flags_up,
};
#ifdef CONFIG_SMP_ON_UP
/* This will be run-time patched so the offset better be right */
static_assert(offsetof(struct cpu_tlb_fns, tlb_flags) == 8);
asm(" .pushsection \".alt.smp.init\", \"a\" \n" \
" .align 2 \n" \
" .long v7wbi_tlb_fns + 8 - . \n" \
" .long " __stringify(v7wbi_tlb_flags_up) " \n" \
" .popsection \n");
#endif
#endif
#ifdef CONFIG_CPU_TLB_FA
void fa_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
void fa_flush_kern_tlb_range(unsigned long, unsigned long);
struct cpu_tlb_fns fa_tlb_fns __initconst = {
.flush_user_range = fa_flush_user_tlb_range,
.flush_kern_range = fa_flush_kern_tlb_range,
.tlb_flags = fa_tlb_flags,
};
#endif