From ad8d1e323dd37f91d0c973e2a74c7b9054219adc Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 2 Sep 2024 07:39:20 +0100 Subject: [PATCH 1/8] ARM: 9415/1: amba: Add dev_is_amba() function and export it for modules Add dev_is_amba() function to determine whether the device is a AMBA device. Suggested-by: Andy Shevchenko Signed-off-by: Kunwu Chan Signed-off-by: Russell King (Oracle) --- drivers/amba/bus.c | 6 ++++++ include/linux/amba/bus.h | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 0230c43377c1..8ef259b4d037 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -449,6 +449,12 @@ const struct bus_type amba_bustype = { }; EXPORT_SYMBOL_GPL(amba_bustype); +bool dev_is_amba(const struct device *dev) +{ + return dev->bus == &amba_bustype; +} +EXPORT_SYMBOL_GPL(dev_is_amba); + static int __init amba_init(void) { return bus_register(&amba_bustype); diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index dda2f3ea89cb..9946276aff73 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -121,6 +121,7 @@ extern const struct bus_type amba_bustype; #ifdef CONFIG_ARM_AMBA int __amba_driver_register(struct amba_driver *, struct module *); void amba_driver_unregister(struct amba_driver *); +bool dev_is_amba(const struct device *dev); #else static inline int __amba_driver_register(struct amba_driver *drv, struct module *owner) @@ -130,6 +131,10 @@ static inline int __amba_driver_register(struct amba_driver *drv, static inline void amba_driver_unregister(struct amba_driver *drv) { } +static inline bool dev_is_amba(const struct device *dev) +{ + return false; +} #endif struct amba_device *amba_device_alloc(const char *, resource_size_t, size_t); From f26bdbe1fa9f79fa8cb0d0bf39303c3573c60552 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 2 Oct 2024 17:15:32 +0100 Subject: [PATCH 2/8] ARM: 9423/1: vfp: Provide vfp_state_hold() for VFP locking. kernel_neon_begin() uses local_bh_disable() to ensure exclusive access to the VFP unit. This is broken on PREEMPT_RT because a BH disabled section remains preemptible on PREEMPT_RT. Introduce vfp_state_hold() which uses local_bh_disable() and preempt_disable() on PREEMPT_RT. Since softirqs are processed always in thread context, disabling preemption is enough to ensure that the current context won't get interrupted by something that is using the VFP. Use it in kernel_neon_begin(). Reviewed-by: Ard Biesheuvel Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Russell King (Oracle) --- arch/arm/vfp/vfpmodule.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index b68efe643a12..63de164c7fc7 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -55,6 +55,34 @@ extern unsigned int VFP_arch_feroceon __alias(VFP_arch); */ union vfp_state *vfp_current_hw_state[NR_CPUS]; +/* + * Claim ownership of the VFP unit. + * + * The caller may change VFP registers until vfp_state_release() is called. + * + * local_bh_disable() is used to disable preemption and to disable VFP + * processing in softirq context. On PREEMPT_RT kernels local_bh_disable() is + * not sufficient because it only serializes soft interrupt related sections + * via a local lock, but stays preemptible. Disabling preemption is the right + * choice here as bottom half processing is always in thread context on RT + * kernels so it implicitly prevents bottom half processing as well. + */ +static void vfp_state_hold(void) +{ + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + local_bh_disable(); + else + preempt_disable(); +} + +static void vfp_state_release(void) +{ + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + local_bh_enable(); + else + preempt_enable(); +} + /* * Is 'thread's most up to date state stored in this CPUs hardware? * Must be called from non-preemptible context. @@ -837,7 +865,7 @@ void kernel_neon_begin(void) unsigned int cpu; u32 fpexc; - local_bh_disable(); + vfp_state_hold(); /* * Kernel mode NEON is only allowed outside of hardirq context with @@ -868,7 +896,7 @@ void kernel_neon_end(void) { /* Disable the NEON/VFP unit. */ fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); - local_bh_enable(); + vfp_state_release(); } EXPORT_SYMBOL(kernel_neon_end); From b54cdbad4a39bb3abc85ac151f4882b3d92c5d79 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 2 Oct 2024 17:16:33 +0100 Subject: [PATCH 3/8] ARM: 9424/1: vfp: Use vfp_state_hold() in vfp_sync_hwstate(). vfp_sync_hwstate() uses preempt_disable() followed by local_bh_disable() to ensure that it won't get interrupted while checking the VFP state. This harms PREEMPT_RT because softirq handling can get preempted and local_bh_disable() synchronizes the related section with a sleeping lock which does not work with disabled preemption. Use the vfp_state_hold() to synchronize the access. Reviewed-by: Ard Biesheuvel Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Russell King (Oracle) --- arch/arm/vfp/vfpmodule.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 63de164c7fc7..1726db050076 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -540,11 +540,9 @@ static inline void vfp_pm_init(void) { } */ void vfp_sync_hwstate(struct thread_info *thread) { - unsigned int cpu = get_cpu(); + vfp_state_hold(); - local_bh_disable(); - - if (vfp_state_in_hw(cpu, thread)) { + if (vfp_state_in_hw(raw_smp_processor_id(), thread)) { u32 fpexc = fmrx(FPEXC); /* @@ -555,8 +553,7 @@ void vfp_sync_hwstate(struct thread_info *thread) fmxr(FPEXC, fpexc); } - local_bh_enable(); - put_cpu(); + vfp_state_release(); } /* Ensure that the thread reloads the hardware VFP state on the next use. */ From 27035c23bad54ed552c6741210dd1c4fa50cb386 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 2 Oct 2024 17:18:11 +0100 Subject: [PATCH 4/8] ARM: 9425/1: vfp: Use vfp_state_hold() in vfp_support_entry(). vfp_entry() is invoked from exception handler and is fully preemptible. It uses local_bh_disable() to remain uninterrupted while checking the VFP state. This is not working on PREEMPT_RT because local_bh_disable() synchronizes the relevant section but the context remains fully preemptible. Use vfp_state_hold() for uninterrupted access. Reviewed-by: Ard Biesheuvel Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Russell King (Oracle) --- arch/arm/vfp/vfpmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 1726db050076..28dafae7758b 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -708,7 +708,7 @@ static int vfp_support_entry(struct pt_regs *regs, u32 trigger) if (!user_mode(regs)) return vfp_kmode_exception(regs, trigger); - local_bh_disable(); + vfp_state_hold(); fpexc = fmrx(FPEXC); /* @@ -787,7 +787,7 @@ static int vfp_support_entry(struct pt_regs *regs, u32 trigger) if (!(fpscr & FPSCR_IXE)) { if (!(fpscr & FPSCR_LENGTH_MASK)) { pr_debug("not VFP\n"); - local_bh_enable(); + vfp_state_release(); return -ENOEXEC; } fpexc |= FPEXC_DEX; @@ -797,7 +797,7 @@ bounce: regs->ARM_pc += 4; VFP_bounce(trigger, fpexc, regs); } - local_bh_enable(); + vfp_state_release(); return 0; } From c0b5195bad63b18022f2776372fccd3ae9177705 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 2 Oct 2024 17:18:54 +0100 Subject: [PATCH 5/8] ARM: 9426/1: vfp: Move sending signals outside of vfp_state_hold()ed section. VFP_bounce() is invoked from within vfp_support_entry() and may send a signal. Sending a signal uses spinlock_t which becomes a sleeping lock on PREEMPT_RT and must not be acquired within a preempt-disabled section. Move the vfp_raise_sigfpe() block outside of the vfp_state_hold() section. Reviewed-by: Ard Biesheuvel Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Russell King (Oracle) --- arch/arm/vfp/vfpmodule.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 28dafae7758b..d44867fc0c5e 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -268,7 +268,7 @@ static void vfp_panic(char *reason, u32 inst) /* * Process bitmask of exception conditions. */ -static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs) +static int vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr) { int si_code = 0; @@ -276,8 +276,7 @@ static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_ if (exceptions == VFP_EXCEPTION_ERROR) { vfp_panic("unhandled bounce", inst); - vfp_raise_sigfpe(FPE_FLTINV, regs); - return; + return FPE_FLTINV; } /* @@ -305,8 +304,7 @@ static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_ RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF); RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV); - if (si_code) - vfp_raise_sigfpe(si_code, regs); + return si_code; } /* @@ -352,6 +350,8 @@ static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) { u32 fpscr, orig_fpscr, fpsid, exceptions; + int si_code2 = 0; + int si_code = 0; pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc); @@ -397,8 +397,8 @@ static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) * unallocated VFP instruction but with FPSCR.IXE set and not * on VFP subarch 1. */ - vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs); - return; + si_code = vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr); + goto exit; } /* @@ -422,14 +422,14 @@ static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) */ exceptions = vfp_emulate_instruction(trigger, fpscr, regs); if (exceptions) - vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs); + si_code2 = vfp_raise_exceptions(exceptions, trigger, orig_fpscr); /* * If there isn't a second FP instruction, exit now. Note that * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1. */ if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V)) - return; + goto exit; /* * The barrier() here prevents fpinst2 being read @@ -441,7 +441,13 @@ static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) emulate: exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs); if (exceptions) - vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs); + si_code = vfp_raise_exceptions(exceptions, trigger, orig_fpscr); +exit: + vfp_state_release(); + if (si_code2) + vfp_raise_sigfpe(si_code2, regs); + if (si_code) + vfp_raise_sigfpe(si_code, regs); } static void vfp_enable(void *unused) @@ -773,6 +779,7 @@ static int vfp_support_entry(struct pt_regs *regs, u32 trigger) * replay the instruction that trapped. */ fmxr(FPEXC, fpexc); + vfp_state_release(); } else { /* Check for synchronous or asynchronous exceptions */ if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) { @@ -794,10 +801,10 @@ static int vfp_support_entry(struct pt_regs *regs, u32 trigger) } } bounce: regs->ARM_pc += 4; + /* VFP_bounce() will invoke vfp_state_release() */ VFP_bounce(trigger, fpexc, regs); } - vfp_state_release(); return 0; } From d6e6a74d4cea853b5321eeabb69c611148eedefe Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 23 Oct 2024 13:03:14 +0100 Subject: [PATCH 6/8] ARM: 9429/1: ioremap: Sync PGDs for VMALLOC shadow When sync:ing the VMALLOC area to other CPUs, make sure to also sync the KASAN shadow memory for the VMALLOC area, so that we don't get stale entries for the shadow memory in the top level PGD. Since we are now copying PGDs in two instances, create a helper function named memcpy_pgd() to do the actual copying, and create a helper to map the addresses of VMALLOC_START and VMALLOC_END into the corresponding shadow memory. Co-developed-by: Melon Liu Cc: stable@vger.kernel.org Fixes: 565cbaad83d8 ("ARM: 9202/1: kasan: support CONFIG_KASAN_VMALLOC") Link: https://lore.kernel.org/linux-arm-kernel/a1a1d062-f3a2-4d05-9836-3b098de9db6d@foss.st.com/ Reported-by: Clement LE GOFFIC Suggested-by: Mark Rutland Suggested-by: Russell King (Oracle) Acked-by: Mark Rutland Signed-off-by: Linus Walleij Signed-off-by: Russell King (Oracle) --- arch/arm/mm/ioremap.c | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 794cfea9f9d4..ff555823cceb 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -23,6 +23,7 @@ */ #include #include +#include #include #include #include @@ -115,16 +116,40 @@ int ioremap_page(unsigned long virt, unsigned long phys, } EXPORT_SYMBOL(ioremap_page); +#ifdef CONFIG_KASAN +static unsigned long arm_kasan_mem_to_shadow(unsigned long addr) +{ + return (unsigned long)kasan_mem_to_shadow((void *)addr); +} +#else +static unsigned long arm_kasan_mem_to_shadow(unsigned long addr) +{ + return 0; +} +#endif + +static void memcpy_pgd(struct mm_struct *mm, unsigned long start, + unsigned long end) +{ + end = ALIGN(end, PGDIR_SIZE); + memcpy(pgd_offset(mm, start), pgd_offset_k(start), + sizeof(pgd_t) * (pgd_index(end) - pgd_index(start))); +} + void __check_vmalloc_seq(struct mm_struct *mm) { int seq; do { seq = atomic_read(&init_mm.context.vmalloc_seq); - memcpy(pgd_offset(mm, VMALLOC_START), - pgd_offset_k(VMALLOC_START), - sizeof(pgd_t) * (pgd_index(VMALLOC_END) - - pgd_index(VMALLOC_START))); + memcpy_pgd(mm, VMALLOC_START, VMALLOC_END); + if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { + unsigned long start = + arm_kasan_mem_to_shadow(VMALLOC_START); + unsigned long end = + arm_kasan_mem_to_shadow(VMALLOC_END); + memcpy_pgd(mm, start, end); + } /* * Use a store-release so that other CPUs that observe the * counter's new value are guaranteed to see the results of the From 44e9a3bb76e5f2eecd374c8176b2c5163c8bb2e2 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 23 Oct 2024 13:04:44 +0100 Subject: [PATCH 7/8] ARM: 9430/1: entry: Do a dummy read from VMAP shadow When switching task, in addition to a dummy read from the new VMAP stack, also do a dummy read from the VMAP stack's corresponding KASAN shadow memory to sync things up in the new MM context. Cc: stable@vger.kernel.org Fixes: a1c510d0adc6 ("ARM: implement support for vmap'ed stacks") Link: https://lore.kernel.org/linux-arm-kernel/a1a1d062-f3a2-4d05-9836-3b098de9db6d@foss.st.com/ Reported-by: Clement LE GOFFIC Suggested-by: Ard Biesheuvel Signed-off-by: Linus Walleij Signed-off-by: Russell King (Oracle) --- arch/arm/kernel/entry-armv.S | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 1dfae1af8e31..ef6a657c8d13 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -25,6 +25,7 @@ #include #include #include +#include #include "entry-header.S" #include @@ -561,6 +562,13 @@ ENTRY(__switch_to) @ entries covering the vmalloc region. @ ldr r2, [ip] +#ifdef CONFIG_KASAN_VMALLOC + @ Also dummy read from the KASAN shadow memory for the new stack if we + @ are using KASAN + mov_l r2, KASAN_SHADOW_OFFSET + add r2, r2, ip, lsr #KASAN_SHADOW_SCALE_SHIFT + ldr r2, [r2] +#endif #endif @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what From 93ee385254d53849c01dd8ab9bc9d02790ee7f0e Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 23 Oct 2024 13:05:34 +0100 Subject: [PATCH 8/8] ARM: 9431/1: mm: Pair atomic_set_release() with _read_acquire() The code for syncing vmalloc memory PGD pointers is using atomic_read() in pair with atomic_set_release() but the proper pairing is atomic_read_acquire() paired with atomic_set_release(). This is done to clearly instruct the compiler to not reorder the memcpy() or similar calls inside the section so that we do not observe changes to init_mm. memcpy() calls should be identified by the compiler as having unpredictable side effects, but let's try to be on the safe side. Cc: stable@vger.kernel.org Fixes: d31e23aff011 ("ARM: mm: make vmalloc_seq handling SMP safe") Suggested-by: Mark Rutland Signed-off-by: Linus Walleij Signed-off-by: Russell King (Oracle) --- arch/arm/mm/ioremap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index ff555823cceb..89f1c97f3079 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -141,7 +141,7 @@ void __check_vmalloc_seq(struct mm_struct *mm) int seq; do { - seq = atomic_read(&init_mm.context.vmalloc_seq); + seq = atomic_read_acquire(&init_mm.context.vmalloc_seq); memcpy_pgd(mm, VMALLOC_START, VMALLOC_END); if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { unsigned long start =