mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
synced 2025-01-08 15:04:45 +00:00
LoongArch: Add Loongson Binary Translation (LBT) extension support
Loongson Binary Translation (LBT) is used to accelerate binary translation, which contains 4 scratch registers (scr0 to scr3), x86/ARM eflags (eflags) and x87 fpu stack pointer (ftop). This patch support kernel to save/restore these registers, handle the LBT exception and maintain sigcontext. Signed-off-by: Qi Hu <huqi@loongson.cn> Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
This commit is contained in:
Qi Hu
Huacai Chen
parent
f209132104
commit
bd3c579848
@ -254,6 +254,9 @@ config AS_HAS_LSX_EXTENSION
|
||||
config AS_HAS_LASX_EXTENSION
|
||||
def_bool $(as-instr,xvld \$xr0$(comma)\$a0$(comma)0)
|
||||
|
||||
config AS_HAS_LBT_EXTENSION
|
||||
def_bool $(as-instr,movscr2gr \$a0$(comma)\$scr0)
|
||||
|
||||
menu "Kernel type and options"
|
||||
|
||||
source "kernel/Kconfig.hz"
|
||||
@ -534,6 +537,18 @@ config CPU_HAS_LASX
|
||||
|
||||
If unsure, say Y.
|
||||
|
||||
config CPU_HAS_LBT
|
||||
bool "Support for the Loongson Binary Translation Extension"
|
||||
depends on AS_HAS_LBT_EXTENSION
|
||||
help
|
||||
Loongson Binary Translation (LBT) introduces 4 scratch registers (SCR0
|
||||
to SCR3), x86/ARM eflags (eflags) and x87 fpu stack pointer (ftop).
|
||||
Enabling this option allows the kernel to allocate and switch registers
|
||||
specific to LBT.
|
||||
|
||||
If you want to use this feature, such as the Loongson Architecture
|
||||
Translator (LAT), say Y.
|
||||
|
||||
config CPU_HAS_PREFETCH
|
||||
bool
|
||||
default y
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#include <linux/uaccess.h>
|
||||
#include <asm/fpu.h>
|
||||
#include <asm/lbt.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/page.h>
|
||||
#include <asm/ftrace.h>
|
||||
|
||||
@ -42,11 +42,50 @@
|
||||
.macro fpu_save_csr thread tmp
|
||||
movfcsr2gr \tmp, fcsr0
|
||||
stptr.w \tmp, \thread, THREAD_FCSR
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
/* TM bit is always 0 if LBT not supported */
|
||||
andi \tmp, \tmp, FPU_CSR_TM
|
||||
beqz \tmp, 1f
|
||||
/* Save FTOP */
|
||||
x86mftop \tmp
|
||||
stptr.w \tmp, \thread, THREAD_FTOP
|
||||
/* Turn off TM to ensure the order of FPR in memory independent of TM */
|
||||
x86clrtm
|
||||
1:
|
||||
#endif
|
||||
.endm
|
||||
|
||||
.macro fpu_restore_csr thread tmp
|
||||
ldptr.w \tmp, \thread, THREAD_FCSR
|
||||
movgr2fcsr fcsr0, \tmp
|
||||
.macro fpu_restore_csr thread tmp0 tmp1
|
||||
ldptr.w \tmp0, \thread, THREAD_FCSR
|
||||
movgr2fcsr fcsr0, \tmp0
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
/* TM bit is always 0 if LBT not supported */
|
||||
andi \tmp0, \tmp0, FPU_CSR_TM
|
||||
beqz \tmp0, 2f
|
||||
/* Restore FTOP */
|
||||
ldptr.w \tmp0, \thread, THREAD_FTOP
|
||||
andi \tmp0, \tmp0, 0x7
|
||||
la.pcrel \tmp1, 1f
|
||||
alsl.d \tmp1, \tmp0, \tmp1, 3
|
||||
jr \tmp1
|
||||
1:
|
||||
x86mttop 0
|
||||
b 2f
|
||||
x86mttop 1
|
||||
b 2f
|
||||
x86mttop 2
|
||||
b 2f
|
||||
x86mttop 3
|
||||
b 2f
|
||||
x86mttop 4
|
||||
b 2f
|
||||
x86mttop 5
|
||||
b 2f
|
||||
x86mttop 6
|
||||
b 2f
|
||||
x86mttop 7
|
||||
2:
|
||||
#endif
|
||||
.endm
|
||||
|
||||
.macro fpu_save_cc thread tmp0 tmp1
|
||||
@ -246,7 +285,7 @@
|
||||
.macro lsx_restore_all thread tmp0 tmp1
|
||||
lsx_restore_data \thread, \tmp0
|
||||
fpu_restore_cc \thread, \tmp0, \tmp1
|
||||
fpu_restore_csr \thread, \tmp0
|
||||
fpu_restore_csr \thread, \tmp0, \tmp1
|
||||
.endm
|
||||
|
||||
.macro lsx_save_upper vd base tmp off
|
||||
@ -456,7 +495,7 @@
|
||||
.macro lasx_restore_all thread tmp0 tmp1
|
||||
lasx_restore_data \thread, \tmp0
|
||||
fpu_restore_cc \thread, \tmp0, \tmp1
|
||||
fpu_restore_csr \thread, \tmp0
|
||||
fpu_restore_csr \thread, \tmp0, \tmp1
|
||||
.endm
|
||||
|
||||
.macro lasx_save_upper xd base tmp off
|
||||
|
||||
109
arch/loongarch/include/asm/lbt.h
Normal file
109
arch/loongarch/include/asm/lbt.h
Normal file
@ -0,0 +1,109 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* Author: Qi Hu <huqi@loongson.cn>
|
||||
* Huacai Chen <chenhuacai@loongson.cn>
|
||||
* Copyright (C) 2020-2023 Loongson Technology Corporation Limited
|
||||
*/
|
||||
#ifndef _ASM_LBT_H
|
||||
#define _ASM_LBT_H
|
||||
|
||||
#include <asm/cpu.h>
|
||||
#include <asm/current.h>
|
||||
#include <asm/loongarch.h>
|
||||
#include <asm/processor.h>
|
||||
|
||||
extern void _init_lbt(void);
|
||||
extern void _save_lbt(struct loongarch_lbt *);
|
||||
extern void _restore_lbt(struct loongarch_lbt *);
|
||||
|
||||
static inline int is_lbt_enabled(void)
|
||||
{
|
||||
if (!cpu_has_lbt)
|
||||
return 0;
|
||||
|
||||
return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LBTEN) ?
|
||||
1 : 0;
|
||||
}
|
||||
|
||||
static inline int is_lbt_owner(void)
|
||||
{
|
||||
return test_thread_flag(TIF_USEDLBT);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
|
||||
static inline void enable_lbt(void)
|
||||
{
|
||||
if (cpu_has_lbt)
|
||||
csr_xchg32(CSR_EUEN_LBTEN, CSR_EUEN_LBTEN, LOONGARCH_CSR_EUEN);
|
||||
}
|
||||
|
||||
static inline void disable_lbt(void)
|
||||
{
|
||||
if (cpu_has_lbt)
|
||||
csr_xchg32(0, CSR_EUEN_LBTEN, LOONGARCH_CSR_EUEN);
|
||||
}
|
||||
|
||||
static inline void __own_lbt(void)
|
||||
{
|
||||
enable_lbt();
|
||||
set_thread_flag(TIF_USEDLBT);
|
||||
KSTK_EUEN(current) |= CSR_EUEN_LBTEN;
|
||||
}
|
||||
|
||||
static inline void own_lbt_inatomic(int restore)
|
||||
{
|
||||
if (cpu_has_lbt && !is_lbt_owner()) {
|
||||
__own_lbt();
|
||||
if (restore)
|
||||
_restore_lbt(¤t->thread.lbt);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void own_lbt(int restore)
|
||||
{
|
||||
preempt_disable();
|
||||
own_lbt_inatomic(restore);
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
static inline void lose_lbt_inatomic(int save, struct task_struct *tsk)
|
||||
{
|
||||
if (cpu_has_lbt && is_lbt_owner()) {
|
||||
if (save)
|
||||
_save_lbt(&tsk->thread.lbt);
|
||||
|
||||
disable_lbt();
|
||||
clear_tsk_thread_flag(tsk, TIF_USEDLBT);
|
||||
}
|
||||
KSTK_EUEN(tsk) &= ~(CSR_EUEN_LBTEN);
|
||||
}
|
||||
|
||||
static inline void lose_lbt(int save)
|
||||
{
|
||||
preempt_disable();
|
||||
lose_lbt_inatomic(save, current);
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
static inline void init_lbt(void)
|
||||
{
|
||||
__own_lbt();
|
||||
_init_lbt();
|
||||
}
|
||||
#else
|
||||
static inline void own_lbt_inatomic(int restore) {}
|
||||
static inline void lose_lbt_inatomic(int save, struct task_struct *tsk) {}
|
||||
static inline void init_lbt(void) {}
|
||||
static inline void lose_lbt(int save) {}
|
||||
#endif
|
||||
|
||||
static inline int thread_lbt_context_live(void)
|
||||
{
|
||||
if (!cpu_has_lbt)
|
||||
return 0;
|
||||
|
||||
return test_thread_flag(TIF_LBT_CTX_LIVE);
|
||||
}
|
||||
|
||||
#endif /* _ASM_LBT_H */
|
||||
@ -1410,6 +1410,10 @@ __BUILD_CSR_OP(tlbidx)
|
||||
#define FPU_CSR_RU 0x200 /* towards +Infinity */
|
||||
#define FPU_CSR_RD 0x300 /* towards -Infinity */
|
||||
|
||||
/* Bit 6 of FPU Status Register specify the LBT TOP simulation mode */
|
||||
#define FPU_CSR_TM_SHIFT 0x6
|
||||
#define FPU_CSR_TM (_ULCAST_(1) << FPU_CSR_TM_SHIFT)
|
||||
|
||||
#define read_fcsr(source) \
|
||||
({ \
|
||||
unsigned int __res; \
|
||||
|
||||
@ -80,11 +80,22 @@ BUILD_FPR_ACCESS(32)
|
||||
BUILD_FPR_ACCESS(64)
|
||||
|
||||
struct loongarch_fpu {
|
||||
unsigned int fcsr;
|
||||
uint64_t fcc; /* 8x8 */
|
||||
uint32_t fcsr;
|
||||
uint32_t ftop;
|
||||
union fpureg fpr[NUM_FPU_REGS];
|
||||
};
|
||||
|
||||
struct loongarch_lbt {
|
||||
/* Scratch registers */
|
||||
unsigned long scr0;
|
||||
unsigned long scr1;
|
||||
unsigned long scr2;
|
||||
unsigned long scr3;
|
||||
/* Eflags register */
|
||||
unsigned long eflags;
|
||||
};
|
||||
|
||||
#define INIT_CPUMASK { \
|
||||
{0,} \
|
||||
}
|
||||
@ -113,15 +124,6 @@ struct thread_struct {
|
||||
unsigned long csr_ecfg;
|
||||
unsigned long csr_badvaddr; /* Last user fault */
|
||||
|
||||
/* Scratch registers */
|
||||
unsigned long scr0;
|
||||
unsigned long scr1;
|
||||
unsigned long scr2;
|
||||
unsigned long scr3;
|
||||
|
||||
/* Eflags register */
|
||||
unsigned long eflags;
|
||||
|
||||
/* Other stuff associated with the thread. */
|
||||
unsigned long trap_nr;
|
||||
unsigned long error_code;
|
||||
@ -133,6 +135,7 @@ struct thread_struct {
|
||||
* context because they are conditionally copied at fork().
|
||||
*/
|
||||
struct loongarch_fpu fpu FPU_ALIGN;
|
||||
struct loongarch_lbt lbt; /* Also conditionally copied */
|
||||
|
||||
/* Hardware breakpoints pinned to this task. */
|
||||
struct perf_event *hbp_break[LOONGARCH_MAX_BRP];
|
||||
@ -174,8 +177,9 @@ struct thread_struct {
|
||||
* FPU & vector registers \
|
||||
*/ \
|
||||
.fpu = { \
|
||||
.fcsr = 0, \
|
||||
.fcc = 0, \
|
||||
.fcsr = 0, \
|
||||
.ftop = 0, \
|
||||
.fpr = {{{0,},},}, \
|
||||
}, \
|
||||
.hbp_break = {0}, \
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
|
||||
#include <asm/cpu-features.h>
|
||||
#include <asm/fpu.h>
|
||||
#include <asm/lbt.h>
|
||||
|
||||
struct task_struct;
|
||||
|
||||
@ -34,6 +35,7 @@ extern asmlinkage struct task_struct *__switch_to(struct task_struct *prev,
|
||||
#define switch_to(prev, next, last) \
|
||||
do { \
|
||||
lose_fpu_inatomic(1, prev); \
|
||||
lose_lbt_inatomic(1, prev); \
|
||||
hw_breakpoint_thread_switch(next); \
|
||||
(last) = __switch_to(prev, next, task_thread_info(next), \
|
||||
__builtin_return_address(0), __builtin_frame_address(0)); \
|
||||
|
||||
@ -84,6 +84,8 @@ register unsigned long current_stack_pointer __asm__("$sp");
|
||||
#define TIF_SINGLESTEP 16 /* Single Step */
|
||||
#define TIF_LSX_CTX_LIVE 17 /* LSX context must be preserved */
|
||||
#define TIF_LASX_CTX_LIVE 18 /* LASX context must be preserved */
|
||||
#define TIF_USEDLBT 19 /* LBT was used by this task this quantum (SMP) */
|
||||
#define TIF_LBT_CTX_LIVE 20 /* LBT context must be preserved */
|
||||
|
||||
#define _TIF_SIGPENDING (1<<TIF_SIGPENDING)
|
||||
#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED)
|
||||
@ -101,6 +103,8 @@ register unsigned long current_stack_pointer __asm__("$sp");
|
||||
#define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP)
|
||||
#define _TIF_LSX_CTX_LIVE (1<<TIF_LSX_CTX_LIVE)
|
||||
#define _TIF_LASX_CTX_LIVE (1<<TIF_LASX_CTX_LIVE)
|
||||
#define _TIF_USEDLBT (1<<TIF_USEDLBT)
|
||||
#define _TIF_LBT_CTX_LIVE (1<<TIF_LBT_CTX_LIVE)
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
#endif /* _ASM_THREAD_INFO_H */
|
||||
|
||||
@ -56,6 +56,12 @@ struct user_lasx_state {
|
||||
uint64_t vregs[32*4];
|
||||
};
|
||||
|
||||
struct user_lbt_state {
|
||||
uint64_t scr[4];
|
||||
uint32_t eflags;
|
||||
uint32_t ftop;
|
||||
};
|
||||
|
||||
struct user_watch_state {
|
||||
uint64_t dbg_info;
|
||||
struct {
|
||||
|
||||
@ -59,4 +59,14 @@ struct lasx_context {
|
||||
__u32 fcsr;
|
||||
};
|
||||
|
||||
/* LBT context */
|
||||
#define LBT_CTX_MAGIC 0x42540001
|
||||
#define LBT_CTX_ALIGN 8
|
||||
struct lbt_context {
|
||||
__u64 regs[4];
|
||||
__u32 eflags;
|
||||
__u32 ftop;
|
||||
};
|
||||
|
||||
|
||||
#endif /* _UAPI_ASM_SIGCONTEXT_H */
|
||||
|
||||
@ -15,6 +15,8 @@ obj-$(CONFIG_EFI) += efi.o
|
||||
|
||||
obj-$(CONFIG_CPU_HAS_FPU) += fpu.o kfpu.o
|
||||
|
||||
obj-$(CONFIG_CPU_HAS_LBT) += lbt.o
|
||||
|
||||
obj-$(CONFIG_ARCH_STRICT_ALIGN) += unaligned.o
|
||||
|
||||
ifdef CONFIG_FUNCTION_TRACER
|
||||
|
||||
@ -118,13 +118,6 @@ void output_thread_defines(void)
|
||||
OFFSET(THREAD_CSRECFG, task_struct,
|
||||
thread.csr_ecfg);
|
||||
|
||||
OFFSET(THREAD_SCR0, task_struct, thread.scr0);
|
||||
OFFSET(THREAD_SCR1, task_struct, thread.scr1);
|
||||
OFFSET(THREAD_SCR2, task_struct, thread.scr2);
|
||||
OFFSET(THREAD_SCR3, task_struct, thread.scr3);
|
||||
|
||||
OFFSET(THREAD_EFLAGS, task_struct, thread.eflags);
|
||||
|
||||
OFFSET(THREAD_FPU, task_struct, thread.fpu);
|
||||
|
||||
OFFSET(THREAD_BVADDR, task_struct, \
|
||||
@ -172,6 +165,17 @@ void output_thread_fpu_defines(void)
|
||||
|
||||
OFFSET(THREAD_FCSR, loongarch_fpu, fcsr);
|
||||
OFFSET(THREAD_FCC, loongarch_fpu, fcc);
|
||||
OFFSET(THREAD_FTOP, loongarch_fpu, ftop);
|
||||
BLANK();
|
||||
}
|
||||
|
||||
void output_thread_lbt_defines(void)
|
||||
{
|
||||
OFFSET(THREAD_SCR0, loongarch_lbt, scr0);
|
||||
OFFSET(THREAD_SCR1, loongarch_lbt, scr1);
|
||||
OFFSET(THREAD_SCR2, loongarch_lbt, scr2);
|
||||
OFFSET(THREAD_SCR3, loongarch_lbt, scr3);
|
||||
OFFSET(THREAD_EFLAGS, loongarch_lbt, eflags);
|
||||
BLANK();
|
||||
}
|
||||
|
||||
|
||||
@ -144,6 +144,20 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c)
|
||||
c->options |= LOONGARCH_CPU_LVZ;
|
||||
elf_hwcap |= HWCAP_LOONGARCH_LVZ;
|
||||
}
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
if (config & CPUCFG2_X86BT) {
|
||||
c->options |= LOONGARCH_CPU_LBT_X86;
|
||||
elf_hwcap |= HWCAP_LOONGARCH_LBT_X86;
|
||||
}
|
||||
if (config & CPUCFG2_ARMBT) {
|
||||
c->options |= LOONGARCH_CPU_LBT_ARM;
|
||||
elf_hwcap |= HWCAP_LOONGARCH_LBT_ARM;
|
||||
}
|
||||
if (config & CPUCFG2_MIPSBT) {
|
||||
c->options |= LOONGARCH_CPU_LBT_MIPS;
|
||||
elf_hwcap |= HWCAP_LOONGARCH_LBT_MIPS;
|
||||
}
|
||||
#endif
|
||||
|
||||
config = read_cpucfg(LOONGARCH_CPUCFG6);
|
||||
if (config & CPUCFG6_PMP)
|
||||
|
||||
@ -138,6 +138,13 @@
|
||||
.macro sc_save_fcsr base, tmp0
|
||||
movfcsr2gr \tmp0, fcsr0
|
||||
EX st.w \tmp0, \base, 0
|
||||
#if defined(CONFIG_CPU_HAS_LBT)
|
||||
/* TM bit is always 0 if LBT not supported */
|
||||
andi \tmp0, \tmp0, FPU_CSR_TM
|
||||
beqz \tmp0, 1f
|
||||
x86clrtm
|
||||
1:
|
||||
#endif
|
||||
.endm
|
||||
|
||||
.macro sc_restore_fcsr base, tmp0
|
||||
@ -309,7 +316,7 @@ EXPORT_SYMBOL(_save_fp)
|
||||
*/
|
||||
SYM_FUNC_START(_restore_fp)
|
||||
fpu_restore_double a0 t1 # clobbers t1
|
||||
fpu_restore_csr a0 t1
|
||||
fpu_restore_csr a0 t1 t2
|
||||
fpu_restore_cc a0 t1 t2 # clobbers t1, t2
|
||||
jr ra
|
||||
SYM_FUNC_END(_restore_fp)
|
||||
|
||||
155
arch/loongarch/kernel/lbt.S
Normal file
155
arch/loongarch/kernel/lbt.S
Normal file
@ -0,0 +1,155 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* Author: Qi Hu <huqi@loongson.cn>
|
||||
* Huacai Chen <chenhuacai@loongson.cn>
|
||||
*
|
||||
* Copyright (C) 2020-2023 Loongson Technology Corporation Limited
|
||||
*/
|
||||
#include <asm/asm.h>
|
||||
#include <asm/asmmacro.h>
|
||||
#include <asm/asm-extable.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/errno.h>
|
||||
#include <asm/regdef.h>
|
||||
|
||||
#define SCR_REG_WIDTH 8
|
||||
|
||||
.macro EX insn, reg, src, offs
|
||||
.ex\@: \insn \reg, \src, \offs
|
||||
_asm_extable .ex\@, .L_lbt_fault
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Save a thread's lbt context.
|
||||
*/
|
||||
SYM_FUNC_START(_save_lbt)
|
||||
movscr2gr t1, $scr0 # save scr
|
||||
stptr.d t1, a0, THREAD_SCR0
|
||||
movscr2gr t1, $scr1
|
||||
stptr.d t1, a0, THREAD_SCR1
|
||||
movscr2gr t1, $scr2
|
||||
stptr.d t1, a0, THREAD_SCR2
|
||||
movscr2gr t1, $scr3
|
||||
stptr.d t1, a0, THREAD_SCR3
|
||||
|
||||
x86mfflag t1, 0x3f # save eflags
|
||||
stptr.d t1, a0, THREAD_EFLAGS
|
||||
jr ra
|
||||
SYM_FUNC_END(_save_lbt)
|
||||
EXPORT_SYMBOL(_save_lbt)
|
||||
|
||||
/*
|
||||
* Restore a thread's lbt context.
|
||||
*/
|
||||
SYM_FUNC_START(_restore_lbt)
|
||||
ldptr.d t1, a0, THREAD_SCR0 # restore scr
|
||||
movgr2scr $scr0, t1
|
||||
ldptr.d t1, a0, THREAD_SCR1
|
||||
movgr2scr $scr1, t1
|
||||
ldptr.d t1, a0, THREAD_SCR2
|
||||
movgr2scr $scr2, t1
|
||||
ldptr.d t1, a0, THREAD_SCR3
|
||||
movgr2scr $scr3, t1
|
||||
|
||||
ldptr.d t1, a0, THREAD_EFLAGS # restore eflags
|
||||
x86mtflag t1, 0x3f
|
||||
jr ra
|
||||
SYM_FUNC_END(_restore_lbt)
|
||||
EXPORT_SYMBOL(_restore_lbt)
|
||||
|
||||
/*
|
||||
* Load scr/eflag with zero.
|
||||
*/
|
||||
SYM_FUNC_START(_init_lbt)
|
||||
movgr2scr $scr0, zero
|
||||
movgr2scr $scr1, zero
|
||||
movgr2scr $scr2, zero
|
||||
movgr2scr $scr3, zero
|
||||
|
||||
x86mtflag zero, 0x3f
|
||||
jr ra
|
||||
SYM_FUNC_END(_init_lbt)
|
||||
|
||||
/*
|
||||
* a0: scr
|
||||
* a1: eflag
|
||||
*/
|
||||
SYM_FUNC_START(_save_lbt_context)
|
||||
movscr2gr t1, $scr0 # save scr
|
||||
EX st.d t1, a0, (0 * SCR_REG_WIDTH)
|
||||
movscr2gr t1, $scr1
|
||||
EX st.d t1, a0, (1 * SCR_REG_WIDTH)
|
||||
movscr2gr t1, $scr2
|
||||
EX st.d t1, a0, (2 * SCR_REG_WIDTH)
|
||||
movscr2gr t1, $scr3
|
||||
EX st.d t1, a0, (3 * SCR_REG_WIDTH)
|
||||
|
||||
x86mfflag t1, 0x3f # save eflags
|
||||
EX st.w t1, a1, 0
|
||||
li.w a0, 0 # success
|
||||
jr ra
|
||||
SYM_FUNC_END(_save_lbt_context)
|
||||
|
||||
/*
|
||||
* a0: scr
|
||||
* a1: eflag
|
||||
*/
|
||||
SYM_FUNC_START(_restore_lbt_context)
|
||||
EX ld.d t1, a0, (0 * SCR_REG_WIDTH) # restore scr
|
||||
movgr2scr $scr0, t1
|
||||
EX ld.d t1, a0, (1 * SCR_REG_WIDTH)
|
||||
movgr2scr $scr1, t1
|
||||
EX ld.d t1, a0, (2 * SCR_REG_WIDTH)
|
||||
movgr2scr $scr2, t1
|
||||
EX ld.d t1, a0, (3 * SCR_REG_WIDTH)
|
||||
movgr2scr $scr3, t1
|
||||
|
||||
EX ld.w t1, a1, 0 # restore eflags
|
||||
x86mtflag t1, 0x3f
|
||||
li.w a0, 0 # success
|
||||
jr ra
|
||||
SYM_FUNC_END(_restore_lbt_context)
|
||||
|
||||
/*
|
||||
* a0: ftop
|
||||
*/
|
||||
SYM_FUNC_START(_save_ftop_context)
|
||||
x86mftop t1
|
||||
st.w t1, a0, 0
|
||||
li.w a0, 0 # success
|
||||
jr ra
|
||||
SYM_FUNC_END(_save_ftop_context)
|
||||
|
||||
/*
|
||||
* a0: ftop
|
||||
*/
|
||||
SYM_FUNC_START(_restore_ftop_context)
|
||||
ld.w t1, a0, 0
|
||||
andi t1, t1, 0x7
|
||||
la.pcrel a0, 1f
|
||||
alsl.d a0, t1, a0, 3
|
||||
jr a0
|
||||
1:
|
||||
x86mttop 0
|
||||
b 2f
|
||||
x86mttop 1
|
||||
b 2f
|
||||
x86mttop 2
|
||||
b 2f
|
||||
x86mttop 3
|
||||
b 2f
|
||||
x86mttop 4
|
||||
b 2f
|
||||
x86mttop 5
|
||||
b 2f
|
||||
x86mttop 6
|
||||
b 2f
|
||||
x86mttop 7
|
||||
2:
|
||||
li.w a0, 0 # success
|
||||
jr ra
|
||||
SYM_FUNC_END(_restore_ftop_context)
|
||||
|
||||
.L_lbt_fault:
|
||||
li.w a0, -EFAULT # failure
|
||||
jr ra
|
||||
@ -38,6 +38,7 @@
|
||||
#include <asm/cpu.h>
|
||||
#include <asm/elf.h>
|
||||
#include <asm/fpu.h>
|
||||
#include <asm/lbt.h>
|
||||
#include <asm/io.h>
|
||||
#include <asm/irq.h>
|
||||
#include <asm/irq_regs.h>
|
||||
@ -82,9 +83,11 @@ void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp)
|
||||
euen = regs->csr_euen & ~(CSR_EUEN_FPEN);
|
||||
regs->csr_euen = euen;
|
||||
lose_fpu(0);
|
||||
lose_lbt(0);
|
||||
|
||||
clear_thread_flag(TIF_LSX_CTX_LIVE);
|
||||
clear_thread_flag(TIF_LASX_CTX_LIVE);
|
||||
clear_thread_flag(TIF_LBT_CTX_LIVE);
|
||||
clear_used_math();
|
||||
regs->csr_era = pc;
|
||||
regs->regs[3] = sp;
|
||||
@ -121,10 +124,14 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
|
||||
|
||||
preempt_enable();
|
||||
|
||||
if (used_math())
|
||||
memcpy(dst, src, sizeof(struct task_struct));
|
||||
else
|
||||
if (!used_math())
|
||||
memcpy(dst, src, offsetof(struct task_struct, thread.fpu.fpr));
|
||||
else
|
||||
memcpy(dst, src, offsetof(struct task_struct, thread.lbt.scr0));
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
memcpy(&dst->thread.lbt, &src->thread.lbt, sizeof(struct loongarch_lbt));
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -189,8 +196,10 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
|
||||
ptrace_hw_copy_thread(p);
|
||||
clear_tsk_thread_flag(p, TIF_USEDFPU);
|
||||
clear_tsk_thread_flag(p, TIF_USEDSIMD);
|
||||
clear_tsk_thread_flag(p, TIF_USEDLBT);
|
||||
clear_tsk_thread_flag(p, TIF_LSX_CTX_LIVE);
|
||||
clear_tsk_thread_flag(p, TIF_LASX_CTX_LIVE);
|
||||
clear_tsk_thread_flag(p, TIF_LBT_CTX_LIVE);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -38,6 +38,7 @@
|
||||
#include <asm/cpu.h>
|
||||
#include <asm/cpu-info.h>
|
||||
#include <asm/fpu.h>
|
||||
#include <asm/lbt.h>
|
||||
#include <asm/loongarch.h>
|
||||
#include <asm/page.h>
|
||||
#include <asm/pgtable.h>
|
||||
@ -338,6 +339,46 @@ static int simd_set(struct task_struct *target,
|
||||
|
||||
#endif /* CONFIG_CPU_HAS_LSX */
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
static int lbt_get(struct task_struct *target,
|
||||
const struct user_regset *regset,
|
||||
struct membuf to)
|
||||
{
|
||||
int r;
|
||||
|
||||
r = membuf_write(&to, &target->thread.lbt.scr0, sizeof(target->thread.lbt.scr0));
|
||||
r = membuf_write(&to, &target->thread.lbt.scr1, sizeof(target->thread.lbt.scr1));
|
||||
r = membuf_write(&to, &target->thread.lbt.scr2, sizeof(target->thread.lbt.scr2));
|
||||
r = membuf_write(&to, &target->thread.lbt.scr3, sizeof(target->thread.lbt.scr3));
|
||||
r = membuf_write(&to, &target->thread.lbt.eflags, sizeof(u32));
|
||||
r = membuf_write(&to, &target->thread.fpu.ftop, sizeof(u32));
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static int lbt_set(struct task_struct *target,
|
||||
const struct user_regset *regset,
|
||||
unsigned int pos, unsigned int count,
|
||||
const void *kbuf, const void __user *ubuf)
|
||||
{
|
||||
int err = 0;
|
||||
const int eflags_start = 4 * sizeof(target->thread.lbt.scr0);
|
||||
const int ftop_start = eflags_start + sizeof(u32);
|
||||
|
||||
err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf,
|
||||
&target->thread.lbt.scr0,
|
||||
0, 4 * sizeof(target->thread.lbt.scr0));
|
||||
err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf,
|
||||
&target->thread.lbt.eflags,
|
||||
eflags_start, ftop_start);
|
||||
err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf,
|
||||
&target->thread.fpu.ftop,
|
||||
ftop_start, ftop_start + sizeof(u32));
|
||||
|
||||
return err;
|
||||
}
|
||||
#endif /* CONFIG_CPU_HAS_LBT */
|
||||
|
||||
#ifdef CONFIG_HAVE_HW_BREAKPOINT
|
||||
|
||||
/*
|
||||
@ -802,6 +843,9 @@ enum loongarch_regset {
|
||||
#ifdef CONFIG_CPU_HAS_LASX
|
||||
REGSET_LASX,
|
||||
#endif
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
REGSET_LBT,
|
||||
#endif
|
||||
#ifdef CONFIG_HAVE_HW_BREAKPOINT
|
||||
REGSET_HW_BREAK,
|
||||
REGSET_HW_WATCH,
|
||||
@ -853,6 +897,16 @@ static const struct user_regset loongarch64_regsets[] = {
|
||||
.set = simd_set,
|
||||
},
|
||||
#endif
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
[REGSET_LBT] = {
|
||||
.core_note_type = NT_LOONGARCH_LBT,
|
||||
.n = 5,
|
||||
.size = sizeof(u64),
|
||||
.align = sizeof(u64),
|
||||
.regset_get = lbt_get,
|
||||
.set = lbt_set,
|
||||
},
|
||||
#endif
|
||||
#ifdef CONFIG_HAVE_HW_BREAKPOINT
|
||||
[REGSET_HW_BREAK] = {
|
||||
.core_note_type = NT_LOONGARCH_HW_BREAK,
|
||||
|
||||
@ -32,6 +32,7 @@
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/cpu-features.h>
|
||||
#include <asm/fpu.h>
|
||||
#include <asm/lbt.h>
|
||||
#include <asm/ucontext.h>
|
||||
#include <asm/vdso.h>
|
||||
|
||||
@ -44,6 +45,9 @@
|
||||
/* Make sure we will not lose FPU ownership */
|
||||
#define lock_fpu_owner() ({ preempt_disable(); pagefault_disable(); })
|
||||
#define unlock_fpu_owner() ({ pagefault_enable(); preempt_enable(); })
|
||||
/* Make sure we will not lose LBT ownership */
|
||||
#define lock_lbt_owner() ({ preempt_disable(); pagefault_disable(); })
|
||||
#define unlock_lbt_owner() ({ pagefault_enable(); preempt_enable(); })
|
||||
|
||||
/* Assembly functions to move context to/from the FPU */
|
||||
extern asmlinkage int
|
||||
@ -59,6 +63,13 @@ _save_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr);
|
||||
extern asmlinkage int
|
||||
_restore_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr);
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
extern asmlinkage int _save_lbt_context(void __user *regs, void __user *eflags);
|
||||
extern asmlinkage int _restore_lbt_context(void __user *regs, void __user *eflags);
|
||||
extern asmlinkage int _save_ftop_context(void __user *ftop);
|
||||
extern asmlinkage int _restore_ftop_context(void __user *ftop);
|
||||
#endif
|
||||
|
||||
struct rt_sigframe {
|
||||
struct siginfo rs_info;
|
||||
struct ucontext rs_uctx;
|
||||
@ -75,6 +86,7 @@ struct extctx_layout {
|
||||
struct _ctx_layout fpu;
|
||||
struct _ctx_layout lsx;
|
||||
struct _ctx_layout lasx;
|
||||
struct _ctx_layout lbt;
|
||||
struct _ctx_layout end;
|
||||
};
|
||||
|
||||
@ -215,6 +227,52 @@ static int copy_lasx_from_sigcontext(struct lasx_context __user *ctx)
|
||||
return err;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
static int copy_lbt_to_sigcontext(struct lbt_context __user *ctx)
|
||||
{
|
||||
int err = 0;
|
||||
uint64_t __user *regs = (uint64_t *)&ctx->regs;
|
||||
uint32_t __user *eflags = (uint32_t *)&ctx->eflags;
|
||||
|
||||
err |= __put_user(current->thread.lbt.scr0, ®s[0]);
|
||||
err |= __put_user(current->thread.lbt.scr1, ®s[1]);
|
||||
err |= __put_user(current->thread.lbt.scr2, ®s[2]);
|
||||
err |= __put_user(current->thread.lbt.scr3, ®s[3]);
|
||||
err |= __put_user(current->thread.lbt.eflags, eflags);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
static int copy_lbt_from_sigcontext(struct lbt_context __user *ctx)
|
||||
{
|
||||
int err = 0;
|
||||
uint64_t __user *regs = (uint64_t *)&ctx->regs;
|
||||
uint32_t __user *eflags = (uint32_t *)&ctx->eflags;
|
||||
|
||||
err |= __get_user(current->thread.lbt.scr0, ®s[0]);
|
||||
err |= __get_user(current->thread.lbt.scr1, ®s[1]);
|
||||
err |= __get_user(current->thread.lbt.scr2, ®s[2]);
|
||||
err |= __get_user(current->thread.lbt.scr3, ®s[3]);
|
||||
err |= __get_user(current->thread.lbt.eflags, eflags);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
static int copy_ftop_to_sigcontext(struct lbt_context __user *ctx)
|
||||
{
|
||||
uint32_t __user *ftop = &ctx->ftop;
|
||||
|
||||
return __put_user(current->thread.fpu.ftop, ftop);
|
||||
}
|
||||
|
||||
static int copy_ftop_from_sigcontext(struct lbt_context __user *ctx)
|
||||
{
|
||||
uint32_t __user *ftop = &ctx->ftop;
|
||||
|
||||
return __get_user(current->thread.fpu.ftop, ftop);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Wrappers for the assembly _{save,restore}_fp_context functions.
|
||||
*/
|
||||
@ -272,6 +330,41 @@ static int restore_hw_lasx_context(struct lasx_context __user *ctx)
|
||||
return _restore_lasx_context(regs, fcc, fcsr);
|
||||
}
|
||||
|
||||
/*
|
||||
* Wrappers for the assembly _{save,restore}_lbt_context functions.
|
||||
*/
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
static int save_hw_lbt_context(struct lbt_context __user *ctx)
|
||||
{
|
||||
uint64_t __user *regs = (uint64_t *)&ctx->regs;
|
||||
uint32_t __user *eflags = (uint32_t *)&ctx->eflags;
|
||||
|
||||
return _save_lbt_context(regs, eflags);
|
||||
}
|
||||
|
||||
static int restore_hw_lbt_context(struct lbt_context __user *ctx)
|
||||
{
|
||||
uint64_t __user *regs = (uint64_t *)&ctx->regs;
|
||||
uint32_t __user *eflags = (uint32_t *)&ctx->eflags;
|
||||
|
||||
return _restore_lbt_context(regs, eflags);
|
||||
}
|
||||
|
||||
static int save_hw_ftop_context(struct lbt_context __user *ctx)
|
||||
{
|
||||
uint32_t __user *ftop = &ctx->ftop;
|
||||
|
||||
return _save_ftop_context(ftop);
|
||||
}
|
||||
|
||||
static int restore_hw_ftop_context(struct lbt_context __user *ctx)
|
||||
{
|
||||
uint32_t __user *ftop = &ctx->ftop;
|
||||
|
||||
return _restore_ftop_context(ftop);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int fcsr_pending(unsigned int __user *fcsr)
|
||||
{
|
||||
int err, sig = 0;
|
||||
@ -519,6 +612,77 @@ static int protected_restore_lasx_context(struct extctx_layout *extctx)
|
||||
return err ?: sig;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
static int protected_save_lbt_context(struct extctx_layout *extctx)
|
||||
{
|
||||
int err = 0;
|
||||
struct sctx_info __user *info = extctx->lbt.addr;
|
||||
struct lbt_context __user *lbt_ctx =
|
||||
(struct lbt_context *)get_ctx_through_ctxinfo(info);
|
||||
uint64_t __user *regs = (uint64_t *)&lbt_ctx->regs;
|
||||
uint32_t __user *eflags = (uint32_t *)&lbt_ctx->eflags;
|
||||
|
||||
while (1) {
|
||||
lock_lbt_owner();
|
||||
if (is_lbt_owner())
|
||||
err |= save_hw_lbt_context(lbt_ctx);
|
||||
else
|
||||
err |= copy_lbt_to_sigcontext(lbt_ctx);
|
||||
if (is_fpu_owner())
|
||||
err |= save_hw_ftop_context(lbt_ctx);
|
||||
else
|
||||
err |= copy_ftop_to_sigcontext(lbt_ctx);
|
||||
unlock_lbt_owner();
|
||||
|
||||
err |= __put_user(LBT_CTX_MAGIC, &info->magic);
|
||||
err |= __put_user(extctx->lbt.size, &info->size);
|
||||
|
||||
if (likely(!err))
|
||||
break;
|
||||
/* Touch the LBT context and try again */
|
||||
err = __put_user(0, ®s[0]) | __put_user(0, eflags);
|
||||
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
static int protected_restore_lbt_context(struct extctx_layout *extctx)
|
||||
{
|
||||
int err = 0, tmp __maybe_unused;
|
||||
struct sctx_info __user *info = extctx->lbt.addr;
|
||||
struct lbt_context __user *lbt_ctx =
|
||||
(struct lbt_context *)get_ctx_through_ctxinfo(info);
|
||||
uint64_t __user *regs = (uint64_t *)&lbt_ctx->regs;
|
||||
uint32_t __user *eflags = (uint32_t *)&lbt_ctx->eflags;
|
||||
|
||||
while (1) {
|
||||
lock_lbt_owner();
|
||||
if (is_lbt_owner())
|
||||
err |= restore_hw_lbt_context(lbt_ctx);
|
||||
else
|
||||
err |= copy_lbt_from_sigcontext(lbt_ctx);
|
||||
if (is_fpu_owner())
|
||||
err |= restore_hw_ftop_context(lbt_ctx);
|
||||
else
|
||||
err |= copy_ftop_from_sigcontext(lbt_ctx);
|
||||
unlock_lbt_owner();
|
||||
|
||||
if (likely(!err))
|
||||
break;
|
||||
/* Touch the LBT context and try again */
|
||||
err = __get_user(tmp, ®s[0]) | __get_user(tmp, eflags);
|
||||
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
|
||||
struct extctx_layout *extctx)
|
||||
{
|
||||
@ -539,6 +703,11 @@ static int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
|
||||
else if (extctx->fpu.addr)
|
||||
err |= protected_save_fpu_context(extctx);
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
if (extctx->lbt.addr)
|
||||
err |= protected_save_lbt_context(extctx);
|
||||
#endif
|
||||
|
||||
/* Set the "end" magic */
|
||||
info = (struct sctx_info *)extctx->end.addr;
|
||||
err |= __put_user(0, &info->magic);
|
||||
@ -584,6 +753,13 @@ static int parse_extcontext(struct sigcontext __user *sc, struct extctx_layout *
|
||||
extctx->lasx.addr = info;
|
||||
break;
|
||||
|
||||
case LBT_CTX_MAGIC:
|
||||
if (size < (sizeof(struct sctx_info) +
|
||||
sizeof(struct lbt_context)))
|
||||
goto invalid;
|
||||
extctx->lbt.addr = info;
|
||||
break;
|
||||
|
||||
default:
|
||||
goto invalid;
|
||||
}
|
||||
@ -636,6 +812,11 @@ static int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc
|
||||
else if (extctx.fpu.addr)
|
||||
err |= protected_restore_fpu_context(&extctx);
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
if (extctx.lbt.addr)
|
||||
err |= protected_restore_lbt_context(&extctx);
|
||||
#endif
|
||||
|
||||
bad:
|
||||
return err;
|
||||
}
|
||||
@ -700,6 +881,13 @@ static unsigned long setup_extcontext(struct extctx_layout *extctx, unsigned lon
|
||||
sizeof(struct fpu_context), FPU_CTX_ALIGN, new_sp);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CPU_HAS_LBT
|
||||
if (cpu_has_lbt && thread_lbt_context_live()) {
|
||||
new_sp = extframe_alloc(extctx, &extctx->lbt,
|
||||
sizeof(struct lbt_context), LBT_CTX_ALIGN, new_sp);
|
||||
}
|
||||
#endif
|
||||
|
||||
return new_sp;
|
||||
}
|
||||
|
||||
|
||||
@ -36,6 +36,7 @@
|
||||
#include <asm/break.h>
|
||||
#include <asm/cpu.h>
|
||||
#include <asm/fpu.h>
|
||||
#include <asm/lbt.h>
|
||||
#include <asm/inst.h>
|
||||
#include <asm/loongarch.h>
|
||||
#include <asm/mmu_context.h>
|
||||
@ -966,12 +967,46 @@ asmlinkage void noinstr do_lasx(struct pt_regs *regs)
|
||||
irqentry_exit(regs, state);
|
||||
}
|
||||
|
||||
static void init_restore_lbt(void)
|
||||
{
|
||||
if (!thread_lbt_context_live()) {
|
||||
/* First time LBT context user */
|
||||
init_lbt();
|
||||
set_thread_flag(TIF_LBT_CTX_LIVE);
|
||||
} else {
|
||||
if (!is_lbt_owner())
|
||||
own_lbt_inatomic(1);
|
||||
}
|
||||
|
||||
BUG_ON(!is_lbt_enabled());
|
||||
}
|
||||
|
||||
asmlinkage void noinstr do_lbt(struct pt_regs *regs)
|
||||
{
|
||||
irqentry_state_t state = irqentry_enter(regs);
|
||||
|
||||
/*
|
||||
* BTD (Binary Translation Disable exception) can be triggered
|
||||
* during FP save/restore if TM (Top Mode) is on, which may
|
||||
* cause irq_enable during 'switch_to'. To avoid this situation
|
||||
* (including the user using 'MOVGR2GCSR' to turn on TM, which
|
||||
* will not trigger the BTE), we need to check PRMD first.
|
||||
*/
|
||||
if (regs->csr_prmd & CSR_PRMD_PIE)
|
||||
local_irq_enable();
|
||||
|
||||
if (!cpu_has_lbt) {
|
||||
force_sig(SIGILL);
|
||||
goto out;
|
||||
}
|
||||
BUG_ON(is_lbt_enabled());
|
||||
|
||||
preempt_disable();
|
||||
init_restore_lbt();
|
||||
preempt_enable();
|
||||
|
||||
out:
|
||||
if (regs->csr_prmd & CSR_PRMD_PIE)
|
||||
local_irq_disable();
|
||||
|
||||
irqentry_exit(regs, state);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user