Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-09 23:00:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'for-next/undef-traps' into for-next/core

Browse Source 
* for-next/undef-traps:
  arm64: armv8_deprecated: fix unused-function error
  arm64: armv8_deprecated: rework deprected instruction handling
  arm64: armv8_deprecated: move aarch32 helper earlier
  arm64: armv8_deprecated move emulation functions
  arm64: armv8_deprecated: fold ops into insn_emulation
  arm64: rework EL0 MRS emulation
  arm64: factor insn read out of call_undef_hook()
  arm64: factor out EL1 SSBS emulation hook
  arm64: split EL0/EL1 UNDEF handlers
  arm64: allow kprobes on EL0 handlers
This commit is contained in:
Will Deacon 2022-12-06 11:34:25 +00:00
commit 5f4c374760
9 changed files with 346 additions and 404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
arch/arm64/include/asm/cpufeature.h
View File

@ -832,7 +832,8 @@ static inline bool system_supports_tlb_range(void)
cpus_have_const_cap(ARM64_HAS_TLB_RANGE);
}
extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
bool try_emulate_mrs(struct pt_regs *regs, u32 isn);
static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
{

7
arch/arm64/include/asm/exception.h
View File

@ -58,7 +58,8 @@ asmlinkage void call_on_irq_stack(struct pt_regs *regs,
asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs);
void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs);
void do_undefinstr(struct pt_regs *regs, unsigned long esr);
void do_el0_undef(struct pt_regs *regs, unsigned long esr);
void do_el1_undef(struct pt_regs *regs, unsigned long esr);
void do_el0_bti(struct pt_regs *regs);
void do_el1_bti(struct pt_regs *regs, unsigned long esr);
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
@ -67,10 +68,10 @@ void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs);
void do_sve_acc(unsigned long esr, struct pt_regs *regs);
void do_sme_acc(unsigned long esr, struct pt_regs *regs);
void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs);
void do_sysinstr(unsigned long esr, struct pt_regs *regs);
void do_el0_sys(unsigned long esr, struct pt_regs *regs);
void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs);
void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr);
void do_cp15instr(unsigned long esr, struct pt_regs *regs);
void do_el0_cp15(unsigned long esr, struct pt_regs *regs);
int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs);
void do_el0_svc(struct pt_regs *regs);
void do_el0_svc_compat(struct pt_regs *regs);

2
arch/arm64/include/asm/spectre.h
View File

@ -26,6 +26,7 @@ enum mitigation_state {
SPECTRE_VULNERABLE,
};
struct pt_regs;
struct task_struct;
/*
@ -98,5 +99,6 @@ enum mitigation_state arm64_get_spectre_bhb_state(void);
bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
u8 spectre_bhb_loop_affected(int scope);
void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr);
#endif /* __ASSEMBLY__ */
#endif /* __ASM_SPECTRE_H */

19
arch/arm64/include/asm/traps.h
View File

@ -13,17 +13,16 @@
struct pt_regs;
struct undef_hook {
struct list_head node;
u32 instr_mask;
u32 instr_val;
u64 pstate_mask;
u64 pstate_val;
int (*fn)(struct pt_regs *regs, u32 instr);
};
#ifdef CONFIG_ARMV8_DEPRECATED
bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn);
#else
static inline bool
try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
{
return false;
}
#endif /* CONFIG_ARMV8_DEPRECATED */
void register_undef_hook(struct undef_hook *hook);
void unregister_undef_hook(struct undef_hook *hook);
void force_signal_inject(int signal, int code, unsigned long address, unsigned long err);
void arm64_notify_segfault(unsigned long addr);
void arm64_force_sig_fault(int signo, int code, unsigned long far, const char *str);

569
arch/arm64/kernel/armv8_deprecated.c
View File

@ -17,7 +17,6 @@
#include <asm/sysreg.h>
#include <asm/system_misc.h>
#include <asm/traps.h>
#include <asm/kprobes.h>
#define CREATE_TRACE_POINTS
#include "trace-events-emulation.h"
@ -39,226 +38,46 @@ enum insn_emulation_mode {
enum legacy_insn_status {
INSN_DEPRECATED,
INSN_OBSOLETE,
};
struct insn_emulation_ops {
const char *name;
enum legacy_insn_status status;
struct undef_hook *hooks;
int (*set_hw_mode)(bool enable);
INSN_UNAVAILABLE,
};
struct insn_emulation {
struct list_head node;
struct insn_emulation_ops *ops;
const char *name;
enum legacy_insn_status status;
bool (*try_emulate)(struct pt_regs *regs,
u32 insn);
int (*set_hw_mode)(bool enable);
int current_mode;
int min;
int max;
/*
* sysctl for this emulation + a sentinal entry.
*/
struct ctl_table sysctl[2];
};
static LIST_HEAD(insn_emulation);
static int nr_insn_emulated __initdata;
static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
static DEFINE_MUTEX(insn_emulation_mutex);
#define ARM_OPCODE_CONDTEST_FAIL 0
#define ARM_OPCODE_CONDTEST_PASS 1
#define ARM_OPCODE_CONDTEST_UNCOND 2
static void register_emulation_hooks(struct insn_emulation_ops *ops)
#define ARM_OPCODE_CONDITION_UNCOND 0xf
static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
{
struct undef_hook *hook;
u32 cc_bits = opcode >> 28;
BUG_ON(!ops->hooks);
for (hook = ops->hooks; hook->instr_mask; hook++)
register_undef_hook(hook);
pr_notice("Registered %s emulation handler\n", ops->name);
}
static void remove_emulation_hooks(struct insn_emulation_ops *ops)
{
struct undef_hook *hook;
BUG_ON(!ops->hooks);
for (hook = ops->hooks; hook->instr_mask; hook++)
unregister_undef_hook(hook);
pr_notice("Removed %s emulation handler\n", ops->name);
}
static void enable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->ops->set_hw_mode)
insn->ops->set_hw_mode(true);
}
static void disable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->ops->set_hw_mode)
insn->ops->set_hw_mode(false);
}
/* Run set_hw_mode(mode) on all active CPUs */
static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
{
if (!insn->ops->set_hw_mode)
return -EINVAL;
if (enable)
on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
else
on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
return 0;
}
/*
* Run set_hw_mode for all insns on a starting CPU.
* Returns:
* 0 - If all the hooks ran successfully.
* -EINVAL - At least one hook is not supported by the CPU.
*/
static int run_all_insn_set_hw_mode(unsigned int cpu)
{
int rc = 0;
unsigned long flags;
struct insn_emulation *insn;
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
list_for_each_entry(insn, &insn_emulation, node) {
bool enable = (insn->current_mode == INSN_HW);
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
pr_warn("CPU[%u] cannot support the emulation of %s",
cpu, insn->ops->name);
rc = -EINVAL;
}
if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
return ARM_OPCODE_CONDTEST_PASS;
else
return ARM_OPCODE_CONDTEST_FAIL;
}
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
return rc;
}
static int update_insn_emulation_mode(struct insn_emulation *insn,
enum insn_emulation_mode prev)
{
int ret = 0;
switch (prev) {
case INSN_UNDEF: /* Nothing to be done */
break;
case INSN_EMULATE:
remove_emulation_hooks(insn->ops);
break;
case INSN_HW:
if (!run_all_cpu_set_hw_mode(insn, false))
pr_notice("Disabled %s support\n", insn->ops->name);
break;
}
switch (insn->current_mode) {
case INSN_UNDEF:
break;
case INSN_EMULATE:
register_emulation_hooks(insn->ops);
break;
case INSN_HW:
ret = run_all_cpu_set_hw_mode(insn, true);
if (!ret)
pr_notice("Enabled %s support\n", insn->ops->name);
break;
}
return ret;
}
static void __init register_insn_emulation(struct insn_emulation_ops *ops)
{
unsigned long flags;
struct insn_emulation *insn;
insn = kzalloc(sizeof(*insn), GFP_KERNEL);
if (!insn)
return;
insn->ops = ops;
insn->min = INSN_UNDEF;
switch (ops->status) {
case INSN_DEPRECATED:
insn->current_mode = INSN_EMULATE;
/* Disable the HW mode if it was turned on at early boot time */
run_all_cpu_set_hw_mode(insn, false);
insn->max = INSN_HW;
break;
case INSN_OBSOLETE:
insn->current_mode = INSN_UNDEF;
insn->max = INSN_EMULATE;
break;
}
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
list_add(&insn->node, &insn_emulation);
nr_insn_emulated++;
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
/* Register any handlers if required */
update_insn_emulation_mode(insn, INSN_UNDEF);
}
static int emulation_proc_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp,
loff_t *ppos)
{
int ret = 0;
struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
enum insn_emulation_mode prev_mode = insn->current_mode;
mutex_lock(&insn_emulation_mutex);
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (ret || !write || prev_mode == insn->current_mode)
goto ret;
ret = update_insn_emulation_mode(insn, prev_mode);
if (ret) {
/* Mode change failed, revert to previous mode. */
insn->current_mode = prev_mode;
update_insn_emulation_mode(insn, INSN_UNDEF);
}
ret:
mutex_unlock(&insn_emulation_mutex);
return ret;
}
static void __init register_insn_emulation_sysctl(void)
{
unsigned long flags;
int i = 0;
struct insn_emulation *insn;
struct ctl_table *insns_sysctl, *sysctl;
insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
GFP_KERNEL);
if (!insns_sysctl)
return;
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
list_for_each_entry(insn, &insn_emulation, node) {
sysctl = &insns_sysctl[i];
sysctl->mode = 0644;
sysctl->maxlen = sizeof(int);
sysctl->procname = insn->ops->name;
sysctl->data = &insn->current_mode;
sysctl->extra1 = &insn->min;
sysctl->extra2 = &insn->max;
sysctl->proc_handler = emulation_proc_handler;
i++;
}
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
register_sysctl("abi", insns_sysctl);
return ARM_OPCODE_CONDTEST_UNCOND;
}
#ifdef CONFIG_SWP_EMULATION
/*
* Implement emulation of the SWP/SWPB instructions using load-exclusive and
* store-exclusive.
@ -339,25 +158,6 @@ static int emulate_swpX(unsigned int address, unsigned int *data,
return res;
}
#define ARM_OPCODE_CONDTEST_FAIL 0
#define ARM_OPCODE_CONDTEST_PASS 1
#define ARM_OPCODE_CONDTEST_UNCOND 2
#define ARM_OPCODE_CONDITION_UNCOND 0xf
static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
{
u32 cc_bits = opcode >> 28;
if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
return ARM_OPCODE_CONDTEST_PASS;
else
return ARM_OPCODE_CONDTEST_FAIL;
}
return ARM_OPCODE_CONDTEST_UNCOND;
}
/*
* swp_handler logs the id of calling process, dissects the instruction, sanity
* checks the memory location, calls emulate_swpX for the actual operation and
@ -430,28 +230,27 @@ fault:
return 0;
}
/*
* Only emulate SWP/SWPB executed in ARM state/User mode.
* The kernel must be SWP free and SWP{B} does not exist in Thumb.
*/
static struct undef_hook swp_hooks[] = {
{
.instr_mask = 0x0fb00ff0,
.instr_val = 0x01000090,
.pstate_mask = PSR_AA32_MODE_MASK,
.pstate_val = PSR_AA32_MODE_USR,
.fn = swp_handler
},
{ }
};
static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
{
/* SWP{B} only exists in ARM state and does not exist in Thumb */
if (!compat_user_mode(regs) || compat_thumb_mode(regs))
return false;
static struct insn_emulation_ops swp_ops = {
if ((insn & 0x0fb00ff0) != 0x01000090)
return false;
return swp_handler(regs, insn) == 0;
}
static struct insn_emulation insn_swp = {
.name = "swp",
.status = INSN_OBSOLETE,
.hooks = swp_hooks,
.try_emulate = try_emulate_swp,
.set_hw_mode = NULL,
};
#endif /* CONFIG_SWP_EMULATION */
#ifdef CONFIG_CP15_BARRIER_EMULATION
static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
{
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
@ -514,31 +313,29 @@ static int cp15_barrier_set_hw_mode(bool enable)
return 0;
}
static struct undef_hook cp15_barrier_hooks[] = {
{
.instr_mask = 0x0fff0fdf,
.instr_val = 0x0e070f9a,
.pstate_mask = PSR_AA32_MODE_MASK,
.pstate_val = PSR_AA32_MODE_USR,
.fn = cp15barrier_handler,
},
{
.instr_mask = 0x0fff0fff,
.instr_val = 0x0e070f95,
.pstate_mask = PSR_AA32_MODE_MASK,
.pstate_val = PSR_AA32_MODE_USR,
.fn = cp15barrier_handler,
},
{ }
};
static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
{
if (!compat_user_mode(regs) || compat_thumb_mode(regs))
return false;
static struct insn_emulation_ops cp15_barrier_ops = {
if ((insn & 0x0fff0fdf) == 0x0e070f9a)
return cp15barrier_handler(regs, insn) == 0;
if ((insn & 0x0fff0fff) == 0x0e070f95)
return cp15barrier_handler(regs, insn) == 0;
return false;
}
static struct insn_emulation insn_cp15_barrier = {
.name = "cp15_barrier",
.status = INSN_DEPRECATED,
.hooks = cp15_barrier_hooks,
.try_emulate = try_emulate_cp15_barrier,
.set_hw_mode = cp15_barrier_set_hw_mode,
};
#endif /* CONFIG_CP15_BARRIER_EMULATION */
#ifdef CONFIG_SETEND_EMULATION
static int setend_set_hw_mode(bool enable)
{
if (!cpu_supports_mixed_endian_el0())
@ -586,31 +383,218 @@ static int t16_setend_handler(struct pt_regs *regs, u32 instr)
return rc;
}
static struct undef_hook setend_hooks[] = {
{
.instr_mask = 0xfffffdff,
.instr_val = 0xf1010000,
.pstate_mask = PSR_AA32_MODE_MASK,
.pstate_val = PSR_AA32_MODE_USR,
.fn = a32_setend_handler,
},
{
/* Thumb mode */
.instr_mask = 0xfffffff7,
.instr_val = 0x0000b650,
.pstate_mask = (PSR_AA32_T_BIT | PSR_AA32_MODE_MASK),
.pstate_val = (PSR_AA32_T_BIT | PSR_AA32_MODE_USR),
.fn = t16_setend_handler,
},
{}
};
static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
{
if (compat_thumb_mode(regs) &&
(insn & 0xfffffff7) == 0x0000b650)
return t16_setend_handler(regs, insn) == 0;
static struct insn_emulation_ops setend_ops = {
if (compat_user_mode(regs) &&
(insn & 0xfffffdff) == 0xf1010000)
return a32_setend_handler(regs, insn) == 0;
return false;
}
static struct insn_emulation insn_setend = {
.name = "setend",
.status = INSN_DEPRECATED,
.hooks = setend_hooks,
.try_emulate = try_emulate_setend,
.set_hw_mode = setend_set_hw_mode,
};
#endif /* CONFIG_SETEND_EMULATION */
static struct insn_emulation *insn_emulations[] = {
#ifdef CONFIG_SWP_EMULATION
&insn_swp,
#endif
#ifdef CONFIG_CP15_BARRIER_EMULATION
&insn_cp15_barrier,
#endif
#ifdef CONFIG_SETEND_EMULATION
&insn_setend,
#endif
};
static DEFINE_MUTEX(insn_emulation_mutex);
static void enable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->set_hw_mode)
insn->set_hw_mode(true);
}
static void disable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->set_hw_mode)
insn->set_hw_mode(false);
}
/* Run set_hw_mode(mode) on all active CPUs */
static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
{
if (!insn->set_hw_mode)
return -EINVAL;
if (enable)
on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
else
on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
return 0;
}
/*
* Run set_hw_mode for all insns on a starting CPU.
* Returns:
* 0 - If all the hooks ran successfully.
* -EINVAL - At least one hook is not supported by the CPU.
*/
static int run_all_insn_set_hw_mode(unsigned int cpu)
{
int rc = 0;
unsigned long flags;
/*
* Disable IRQs to serialize against an IPI from
* run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
* recent enablement state if the two race with one another.
*/
local_irq_save(flags);
for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
struct insn_emulation *insn = insn_emulations[i];
bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
pr_warn("CPU[%u] cannot support the emulation of %s",
cpu, insn->name);
rc = -EINVAL;
}
}
local_irq_restore(flags);
return rc;
}
static int update_insn_emulation_mode(struct insn_emulation *insn,
enum insn_emulation_mode prev)
{
int ret = 0;
switch (prev) {
case INSN_UNDEF: /* Nothing to be done */
break;
case INSN_EMULATE:
break;
case INSN_HW:
if (!run_all_cpu_set_hw_mode(insn, false))
pr_notice("Disabled %s support\n", insn->name);
break;
}
switch (insn->current_mode) {
case INSN_UNDEF:
break;
case INSN_EMULATE:
break;
case INSN_HW:
ret = run_all_cpu_set_hw_mode(insn, true);
if (!ret)
pr_notice("Enabled %s support\n", insn->name);
break;
}
return ret;
}
static int emulation_proc_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp,
loff_t *ppos)
{
int ret = 0;
struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
enum insn_emulation_mode prev_mode = insn->current_mode;
mutex_lock(&insn_emulation_mutex);
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (ret || !write || prev_mode == insn->current_mode)
goto ret;
ret = update_insn_emulation_mode(insn, prev_mode);
if (ret) {
/* Mode change failed, revert to previous mode. */
WRITE_ONCE(insn->current_mode, prev_mode);
update_insn_emulation_mode(insn, INSN_UNDEF);
}
ret:
mutex_unlock(&insn_emulation_mutex);
return ret;
}
static void __init register_insn_emulation(struct insn_emulation *insn)
{
struct ctl_table *sysctl;
insn->min = INSN_UNDEF;
switch (insn->status) {
case INSN_DEPRECATED:
insn->current_mode = INSN_EMULATE;
/* Disable the HW mode if it was turned on at early boot time */
run_all_cpu_set_hw_mode(insn, false);
insn->max = INSN_HW;
break;
case INSN_OBSOLETE:
insn->current_mode = INSN_UNDEF;
insn->max = INSN_EMULATE;
break;
case INSN_UNAVAILABLE:
insn->current_mode = INSN_UNDEF;
insn->max = INSN_UNDEF;
break;
}
/* Program the HW if required */
update_insn_emulation_mode(insn, INSN_UNDEF);
if (insn->status != INSN_UNAVAILABLE) {
sysctl = &insn->sysctl[0];
sysctl->mode = 0644;
sysctl->maxlen = sizeof(int);
sysctl->procname = insn->name;
sysctl->data = &insn->current_mode;
sysctl->extra1 = &insn->min;
sysctl->extra2 = &insn->max;
sysctl->proc_handler = emulation_proc_handler;
register_sysctl("abi", sysctl);
}
}
bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
{
for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
struct insn_emulation *ie = insn_emulations[i];
if (ie->status == INSN_UNAVAILABLE)
continue;
/*
* A trap may race with the mode being changed
* INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
* avoid a spurious UNDEF.
*/
if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
continue;
if (ie->try_emulate(regs, insn))
return true;
}
return false;
}
/*
* Invoked as core_initcall, which guarantees that the instruction
@ -618,24 +602,25 @@ static struct insn_emulation_ops setend_ops = {
*/
static int __init armv8_deprecated_init(void)
{
if (IS_ENABLED(CONFIG_SWP_EMULATION))
register_insn_emulation(&swp_ops);
#ifdef CONFIG_SETEND_EMULATION
if (!system_supports_mixed_endian_el0()) {
insn_setend.status = INSN_UNAVAILABLE;
pr_info("setend instruction emulation is not supported on this system\n");
}
if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
register_insn_emulation(&cp15_barrier_ops);
#endif
for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
struct insn_emulation *ie = insn_emulations[i];
if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
if (system_supports_mixed_endian_el0())
register_insn_emulation(&setend_ops);
else
pr_info("setend instruction emulation is not supported on this system\n");
if (ie->status == INSN_UNAVAILABLE)
continue;
register_insn_emulation(ie);
}
cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
"arm64/isndep:starting",
run_all_insn_set_hw_mode, NULL);
register_insn_emulation_sysctl();
return 0;
}

23
arch/arm64/kernel/cpufeature.c
View File

@ -3457,35 +3457,22 @@ int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt)
return rc;
}
static int emulate_mrs(struct pt_regs *regs, u32 insn)
bool try_emulate_mrs(struct pt_regs *regs, u32 insn)
{
u32 sys_reg, rt;
if (compat_user_mode(regs) || !aarch64_insn_is_mrs(insn))
return false;
/*
* sys_reg values are defined as used in mrs/msr instruction.
* shift the imm value to get the encoding.
*/
sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
return do_emulate_mrs(regs, sys_reg, rt);
return do_emulate_mrs(regs, sys_reg, rt) == 0;
}
static struct undef_hook mrs_hook = {
.instr_mask = 0xffff0000,
.instr_val = 0xd5380000,
.pstate_mask = PSR_AA32_MODE_MASK,
.pstate_val = PSR_MODE_EL0t,
.fn = emulate_mrs,
};
static int __init enable_mrs_emulation(void)
{
register_undef_hook(&mrs_hook);
return 0;
}
core_initcall(enable_mrs_emulation);
enum mitigation_state arm64_get_meltdown_state(void)
{
if (__meltdown_safe)

8
arch/arm64/kernel/entry-common.c
View File

@ -384,7 +384,7 @@ static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
{
enter_from_kernel_mode(regs);
local_daif_inherit(regs);
do_undefinstr(regs, esr);
do_el1_undef(regs, esr);
local_daif_mask();
exit_to_kernel_mode(regs);
}
@ -570,7 +570,7 @@ static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_sysinstr(esr, regs);
do_el0_sys(esr, regs);
exit_to_user_mode(regs);
}
@ -599,7 +599,7 @@ static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_undefinstr(regs, esr);
do_el0_undef(regs, esr);
exit_to_user_mode(regs);
}
@ -762,7 +762,7 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_cp15instr(esr, regs);
do_el0_cp15(esr, regs);
exit_to_user_mode(regs);
}

26
arch/arm64/kernel/proton-pack.c
View File

@ -521,10 +521,13 @@ bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
return state != SPECTRE_UNAFFECTED;
}
static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
{
if (user_mode(regs))
return 1;
const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
const u32 instr_val = 0xd500401f | PSTATE_SSBS;
if ((instr & instr_mask) != instr_val)
return false;
if (instr & BIT(PSTATE_Imm_shift))
regs->pstate |= PSR_SSBS_BIT;
@ -532,19 +535,11 @@ static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
regs->pstate &= ~PSR_SSBS_BIT;
arm64_skip_faulting_instruction(regs, 4);
return 0;
return true;
}
static struct undef_hook ssbs_emulation_hook = {
.instr_mask = ~(1U << PSTATE_Imm_shift),
.instr_val = 0xd500401f | PSTATE_SSBS,
.fn = ssbs_emulation_handler,
};
static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
{
static bool undef_hook_registered = false;
static DEFINE_RAW_SPINLOCK(hook_lock);
enum mitigation_state state;
/*
@ -555,13 +550,6 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
return state;
raw_spin_lock(&hook_lock);
if (!undef_hook_registered) {
register_undef_hook(&ssbs_emulation_hook);
undef_hook_registered = true;
}
raw_spin_unlock(&hook_lock);
if (spectre_v4_mitigations_off()) {
sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
set_pstate_ssbs(1);

93
arch/arm64/kernel/traps.c
View File

@ -373,51 +373,22 @@ void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
regs->pstate &= ~PSR_BTYPE_MASK;
}
static LIST_HEAD(undef_hook);
static DEFINE_RAW_SPINLOCK(undef_lock);
void register_undef_hook(struct undef_hook *hook)
static int user_insn_read(struct pt_regs *regs, u32 *insnp)
{
unsigned long flags;
raw_spin_lock_irqsave(&undef_lock, flags);
list_add(&hook->node, &undef_hook);
raw_spin_unlock_irqrestore(&undef_lock, flags);
}
void unregister_undef_hook(struct undef_hook *hook)
{
unsigned long flags;
raw_spin_lock_irqsave(&undef_lock, flags);
list_del(&hook->node);
raw_spin_unlock_irqrestore(&undef_lock, flags);
}
static int call_undef_hook(struct pt_regs *regs)
{
struct undef_hook *hook;
unsigned long flags;
u32 instr;
int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
unsigned long pc = instruction_pointer(regs);
if (!user_mode(regs)) {
__le32 instr_le;
if (get_kernel_nofault(instr_le, (__le32 *)pc))
goto exit;
instr = le32_to_cpu(instr_le);
} else if (compat_thumb_mode(regs)) {
if (compat_thumb_mode(regs)) {
/* 16-bit Thumb instruction */
__le16 instr_le;
if (get_user(instr_le, (__le16 __user *)pc))
goto exit;
return -EFAULT;
instr = le16_to_cpu(instr_le);
if (aarch32_insn_is_wide(instr)) {
u32 instr2;
if (get_user(instr_le, (__le16 __user *)(pc + 2)))
goto exit;
return -EFAULT;
instr2 = le16_to_cpu(instr_le);
instr = (instr << 16) | instr2;
}
@ -425,19 +396,12 @@ static int call_undef_hook(struct pt_regs *regs)
/* 32-bit ARM instruction */
__le32 instr_le;
if (get_user(instr_le, (__le32 __user *)pc))
goto exit;
return -EFAULT;
instr = le32_to_cpu(instr_le);
}
raw_spin_lock_irqsave(&undef_lock, flags);
list_for_each_entry(hook, &undef_hook, node)
if ((instr & hook->instr_mask) == hook->instr_val &&
(regs->pstate & hook->pstate_mask) == hook->pstate_val)
fn = hook->fn;
raw_spin_unlock_irqrestore(&undef_lock, flags);
exit:
return fn ? fn(regs, instr) : 1;
*insnp = instr;
return 0;
}
void force_signal_inject(int signal, int code, unsigned long address, unsigned long err)
@ -486,21 +450,40 @@ void arm64_notify_segfault(unsigned long addr)
force_signal_inject(SIGSEGV, code, addr, 0);
}
void do_undefinstr(struct pt_regs *regs, unsigned long esr)
void do_el0_undef(struct pt_regs *regs, unsigned long esr)
{
u32 insn;
/* check for AArch32 breakpoint instructions */
if (!aarch32_break_handler(regs))
return;
if (call_undef_hook(regs) == 0)
if (user_insn_read(regs, &insn))
goto out_err;
if (try_emulate_mrs(regs, insn))
return;
if (!user_mode(regs))
die("Oops - Undefined instruction", regs, esr);
if (try_emulate_armv8_deprecated(regs, insn))
return;
out_err:
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
NOKPROBE_SYMBOL(do_undefinstr);
void do_el1_undef(struct pt_regs *regs, unsigned long esr)
{
u32 insn;
if (aarch64_insn_read((void *)regs->pc, &insn))
goto out_err;
if (try_emulate_el1_ssbs(regs, insn))
return;
out_err:
die("Oops - Undefined instruction", regs, esr);
}
void do_el0_bti(struct pt_regs *regs)
{
@ -511,7 +494,6 @@ void do_el1_bti(struct pt_regs *regs, unsigned long esr)
{
die("Oops - BTI", regs, esr);
}
NOKPROBE_SYMBOL(do_el1_bti);
void do_el0_fpac(struct pt_regs *regs, unsigned long esr)
{
@ -526,7 +508,6 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
*/
die("Oops - FPAC", regs, esr);
}
NOKPROBE_SYMBOL(do_el1_fpac)
#define __user_cache_maint(insn, address, res) \
if (address >= TASK_SIZE_MAX) { \
@ -748,7 +729,7 @@ static const struct sys64_hook cp15_64_hooks[] = {
{},
};
void do_cp15instr(unsigned long esr, struct pt_regs *regs)
void do_el0_cp15(unsigned long esr, struct pt_regs *regs)
{
const struct sys64_hook *hook, *hook_base;
@ -769,7 +750,7 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs)
hook_base = cp15_64_hooks;
break;
default:
do_undefinstr(regs, esr);
do_el0_undef(regs, esr);
return;
}
@ -784,12 +765,11 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs)
* EL0. Fall back to our usual undefined instruction handler
* so that we handle these consistently.
*/
do_undefinstr(regs, esr);
do_el0_undef(regs, esr);
}
NOKPROBE_SYMBOL(do_cp15instr);
#endif
void do_sysinstr(unsigned long esr, struct pt_regs *regs)
void do_el0_sys(unsigned long esr, struct pt_regs *regs)
{
const struct sys64_hook *hook;
@ -804,9 +784,8 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs)
* back to our usual undefined instruction handler so that we handle
* these consistently.
*/
do_undefinstr(regs, esr);
do_el0_undef(regs, esr);
}
NOKPROBE_SYMBOL(do_sysinstr);
static const char *esr_class_str[] = {
[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",