Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2025-01-07 13:43:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
d97e66fbcb
linux-stable/arch/arm64/kvm/emulate-nested.c
Oliver Upton d97e66fbcb KVM: arm64: nv: Reinject traps that take effect in Host EL0 
Wire up the other end of traps that affect host EL0 by actually
injecting them into the guest hypervisor. Skip over FGT entirely, as a
cursory glance suggests no FGT is effective in host EL0.

Note that kvm_inject_nested() is already equipped for handling
exceptions while the VM is already in a host context.

Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20241025182354.3364124-9-oliver.upton@linux.dev
Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
2024-10-31 19:00:39 +00:00

2528 lines
89 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 - Linaro and Columbia University
* Author: Jintack Lim <jintack.lim@linaro.org>
*/
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_nested.h>
#include "hyp/include/hyp/adjust_pc.h"
#include "trace.h"
enum trap_behaviour {
BEHAVE_HANDLE_LOCALLY = 0,
BEHAVE_FORWARD_READ = BIT(0),
BEHAVE_FORWARD_WRITE = BIT(1),
BEHAVE_FORWARD_RW = BEHAVE_FORWARD_READ | BEHAVE_FORWARD_WRITE,
/* Traps that take effect in Host EL0, this is rare! */
BEHAVE_FORWARD_IN_HOST_EL0 = BIT(2),
};
struct trap_bits {
const enum vcpu_sysreg index;
const enum trap_behaviour behaviour;
const u64 value;
const u64 mask;
};
/* Coarse Grained Trap definitions */
enum cgt_group_id {
/* Indicates no coarse trap control */
__RESERVED__,
/*
* The first batch of IDs denote coarse trapping that are used
* on their own instead of being part of a combination of
* trap controls.
*/
CGT_HCR_TID1,
CGT_HCR_TID2,
CGT_HCR_TID3,
CGT_HCR_IMO,
CGT_HCR_FMO,
CGT_HCR_TIDCP,
CGT_HCR_TACR,
CGT_HCR_TSW,
CGT_HCR_TPC,
CGT_HCR_TPU,
CGT_HCR_TTLB,
CGT_HCR_TVM,
CGT_HCR_TDZ,
CGT_HCR_TRVM,
CGT_HCR_TLOR,
CGT_HCR_TERR,
CGT_HCR_APK,
CGT_HCR_NV,
CGT_HCR_NV_nNV2,
CGT_HCR_NV1_nNV2,
CGT_HCR_AT,
CGT_HCR_nFIEN,
CGT_HCR_TID4,
CGT_HCR_TICAB,
CGT_HCR_TOCU,
CGT_HCR_ENSCXT,
CGT_HCR_TTLBIS,
CGT_HCR_TTLBOS,
CGT_MDCR_TPMCR,
CGT_MDCR_TPM,
CGT_MDCR_TDE,
CGT_MDCR_TDA,
CGT_MDCR_TDOSA,
CGT_MDCR_TDRA,
CGT_MDCR_E2PB,
CGT_MDCR_TPMS,
CGT_MDCR_TTRF,
CGT_MDCR_E2TB,
CGT_MDCR_TDCC,
CGT_CPACR_E0POE,
CGT_CPTR_TAM,
CGT_CPTR_TCPAC,
CGT_HCRX_EnFPM,
CGT_HCRX_TCR2En,
CGT_ICH_HCR_TC,
CGT_ICH_HCR_TALL0,
CGT_ICH_HCR_TALL1,
CGT_ICH_HCR_TDIR,
/*
* Anything after this point is a combination of coarse trap
* controls, which must all be evaluated to decide what to do.
*/
__MULTIPLE_CONTROL_BITS__,
CGT_HCR_IMO_FMO_ICH_HCR_TC = __MULTIPLE_CONTROL_BITS__,
CGT_HCR_TID2_TID4,
CGT_HCR_TTLB_TTLBIS,
CGT_HCR_TTLB_TTLBOS,
CGT_HCR_TVM_TRVM,
CGT_HCR_TVM_TRVM_HCRX_TCR2En,
CGT_HCR_TPU_TICAB,
CGT_HCR_TPU_TOCU,
CGT_HCR_NV1_nNV2_ENSCXT,
CGT_MDCR_TPM_TPMCR,
CGT_MDCR_TDE_TDA,
CGT_MDCR_TDE_TDOSA,
CGT_MDCR_TDE_TDRA,
CGT_MDCR_TDCC_TDE_TDA,
CGT_ICH_HCR_TC_TDIR,
/*
* Anything after this point requires a callback evaluating a
* complex trap condition. Ugly stuff.
*/
__COMPLEX_CONDITIONS__,
CGT_CNTHCTL_EL1PCTEN = __COMPLEX_CONDITIONS__,
CGT_CNTHCTL_EL1PTEN,
CGT_CPTR_TTA,
/* Must be last */
__NR_CGT_GROUP_IDS__
};
static const struct trap_bits coarse_trap_bits[] = {
[CGT_HCR_TID1] = {
.index = HCR_EL2,
.value = HCR_TID1,
.mask = HCR_TID1,
.behaviour = BEHAVE_FORWARD_READ,
},
[CGT_HCR_TID2] = {
.index = HCR_EL2,
.value = HCR_TID2,
.mask = HCR_TID2,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TID3] = {
.index = HCR_EL2,
.value = HCR_TID3,
.mask = HCR_TID3,
.behaviour = BEHAVE_FORWARD_READ,
},
[CGT_HCR_IMO] = {
.index = HCR_EL2,
.value = HCR_IMO,
.mask = HCR_IMO,
.behaviour = BEHAVE_FORWARD_WRITE,
},
[CGT_HCR_FMO] = {
.index = HCR_EL2,
.value = HCR_FMO,
.mask = HCR_FMO,
.behaviour = BEHAVE_FORWARD_WRITE,
},
[CGT_HCR_TIDCP] = {
.index = HCR_EL2,
.value = HCR_TIDCP,
.mask = HCR_TIDCP,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TACR] = {
.index = HCR_EL2,
.value = HCR_TACR,
.mask = HCR_TACR,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TSW] = {
.index = HCR_EL2,
.value = HCR_TSW,
.mask = HCR_TSW,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TPC] = { /* Also called TCPC when FEAT_DPB is implemented */
.index = HCR_EL2,
.value = HCR_TPC,
.mask = HCR_TPC,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TPU] = {
.index = HCR_EL2,
.value = HCR_TPU,
.mask = HCR_TPU,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TTLB] = {
.index = HCR_EL2,
.value = HCR_TTLB,
.mask = HCR_TTLB,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TVM] = {
.index = HCR_EL2,
.value = HCR_TVM,
.mask = HCR_TVM,
.behaviour = BEHAVE_FORWARD_WRITE,
},
[CGT_HCR_TDZ] = {
.index = HCR_EL2,
.value = HCR_TDZ,
.mask = HCR_TDZ,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TRVM] = {
.index = HCR_EL2,
.value = HCR_TRVM,
.mask = HCR_TRVM,
.behaviour = BEHAVE_FORWARD_READ,
},
[CGT_HCR_TLOR] = {
.index = HCR_EL2,
.value = HCR_TLOR,
.mask = HCR_TLOR,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TERR] = {
.index = HCR_EL2,
.value = HCR_TERR,
.mask = HCR_TERR,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_APK] = {
.index = HCR_EL2,
.value = 0,
.mask = HCR_APK,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_NV] = {
.index = HCR_EL2,
.value = HCR_NV,
.mask = HCR_NV,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_NV_nNV2] = {
.index = HCR_EL2,
.value = HCR_NV,
.mask = HCR_NV | HCR_NV2,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_NV1_nNV2] = {
.index = HCR_EL2,
.value = HCR_NV | HCR_NV1,
.mask = HCR_NV | HCR_NV1 | HCR_NV2,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_AT] = {
.index = HCR_EL2,
.value = HCR_AT,
.mask = HCR_AT,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_nFIEN] = {
.index = HCR_EL2,
.value = 0,
.mask = HCR_FIEN,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TID4] = {
.index = HCR_EL2,
.value = HCR_TID4,
.mask = HCR_TID4,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TICAB] = {
.index = HCR_EL2,
.value = HCR_TICAB,
.mask = HCR_TICAB,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TOCU] = {
.index = HCR_EL2,
.value = HCR_TOCU,
.mask = HCR_TOCU,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_ENSCXT] = {
.index = HCR_EL2,
.value = 0,
.mask = HCR_ENSCXT,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TTLBIS] = {
.index = HCR_EL2,
.value = HCR_TTLBIS,
.mask = HCR_TTLBIS,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCR_TTLBOS] = {
.index = HCR_EL2,
.value = HCR_TTLBOS,
.mask = HCR_TTLBOS,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TPMCR] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TPMCR,
.mask = MDCR_EL2_TPMCR,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TPM] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TPM,
.mask = MDCR_EL2_TPM,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TDE] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDE,
.mask = MDCR_EL2_TDE,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TDA] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDA,
.mask = MDCR_EL2_TDA,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TDOSA] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDOSA,
.mask = MDCR_EL2_TDOSA,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TDRA] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDRA,
.mask = MDCR_EL2_TDRA,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_E2PB] = {
.index = MDCR_EL2,
.value = 0,
.mask = BIT(MDCR_EL2_E2PB_SHIFT),
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TPMS] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TPMS,
.mask = MDCR_EL2_TPMS,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TTRF] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TTRF,
.mask = MDCR_EL2_TTRF,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_E2TB] = {
.index = MDCR_EL2,
.value = 0,
.mask = BIT(MDCR_EL2_E2TB_SHIFT),
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_MDCR_TDCC] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDCC,
.mask = MDCR_EL2_TDCC,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_CPACR_E0POE] = {
.index = CPTR_EL2,
.value = CPACR_ELx_E0POE,
.mask = CPACR_ELx_E0POE,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_CPTR_TAM] = {
.index = CPTR_EL2,
.value = CPTR_EL2_TAM,
.mask = CPTR_EL2_TAM,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_CPTR_TCPAC] = {
.index = CPTR_EL2,
.value = CPTR_EL2_TCPAC,
.mask = CPTR_EL2_TCPAC,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCRX_EnFPM] = {
.index = HCRX_EL2,
.value = 0,
.mask = HCRX_EL2_EnFPM,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCRX_TCR2En] = {
.index = HCRX_EL2,
.value = 0,
.mask = HCRX_EL2_TCR2En,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_ICH_HCR_TC] = {
.index = ICH_HCR_EL2,
.value = ICH_HCR_TC,
.mask = ICH_HCR_TC,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_ICH_HCR_TALL0] = {
.index = ICH_HCR_EL2,
.value = ICH_HCR_TALL0,
.mask = ICH_HCR_TALL0,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_ICH_HCR_TALL1] = {
.index = ICH_HCR_EL2,
.value = ICH_HCR_TALL1,
.mask = ICH_HCR_TALL1,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_ICH_HCR_TDIR] = {
.index = ICH_HCR_EL2,
.value = ICH_HCR_TDIR,
.mask = ICH_HCR_TDIR,
.behaviour = BEHAVE_FORWARD_RW,
},
};
#define MCB(id, ...) \
[id - __MULTIPLE_CONTROL_BITS__] = \
(const enum cgt_group_id[]){ \
__VA_ARGS__, __RESERVED__ \
}
static const enum cgt_group_id *coarse_control_combo[] = {
MCB(CGT_HCR_TID2_TID4, CGT_HCR_TID2, CGT_HCR_TID4),
MCB(CGT_HCR_TTLB_TTLBIS, CGT_HCR_TTLB, CGT_HCR_TTLBIS),
MCB(CGT_HCR_TTLB_TTLBOS, CGT_HCR_TTLB, CGT_HCR_TTLBOS),
MCB(CGT_HCR_TVM_TRVM, CGT_HCR_TVM, CGT_HCR_TRVM),
MCB(CGT_HCR_TVM_TRVM_HCRX_TCR2En,
CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_TCR2En),
MCB(CGT_HCR_TPU_TICAB, CGT_HCR_TPU, CGT_HCR_TICAB),
MCB(CGT_HCR_TPU_TOCU, CGT_HCR_TPU, CGT_HCR_TOCU),
MCB(CGT_HCR_NV1_nNV2_ENSCXT, CGT_HCR_NV1_nNV2, CGT_HCR_ENSCXT),
MCB(CGT_MDCR_TPM_TPMCR, CGT_MDCR_TPM, CGT_MDCR_TPMCR),
MCB(CGT_MDCR_TDE_TDA, CGT_MDCR_TDE, CGT_MDCR_TDA),
MCB(CGT_MDCR_TDE_TDOSA, CGT_MDCR_TDE, CGT_MDCR_TDOSA),
MCB(CGT_MDCR_TDE_TDRA, CGT_MDCR_TDE, CGT_MDCR_TDRA),
MCB(CGT_MDCR_TDCC_TDE_TDA, CGT_MDCR_TDCC, CGT_MDCR_TDE, CGT_MDCR_TDA),
MCB(CGT_HCR_IMO_FMO_ICH_HCR_TC, CGT_HCR_IMO, CGT_HCR_FMO, CGT_ICH_HCR_TC),
MCB(CGT_ICH_HCR_TC_TDIR, CGT_ICH_HCR_TC, CGT_ICH_HCR_TDIR),
};
typedef enum trap_behaviour (*complex_condition_check)(struct kvm_vcpu *);
/*
* Warning, maximum confusion ahead.
*
* When E2H=0, CNTHCTL_EL2[1:0] are defined as EL1PCEN:EL1PCTEN
* When E2H=1, CNTHCTL_EL2[11:10] are defined as EL1PTEN:EL1PCTEN
*
* Note the single letter difference? Yet, the bits have the same
* function despite a different layout and a different name.
*
* We don't try to reconcile this mess. We just use the E2H=0 bits
* to generate something that is in the E2H=1 format, and live with
* it. You're welcome.
*/
static u64 get_sanitized_cnthctl(struct kvm_vcpu *vcpu)
{
u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
if (!vcpu_el2_e2h_is_set(vcpu))
val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10;
return val & ((CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN) << 10);
}
static enum trap_behaviour check_cnthctl_el1pcten(struct kvm_vcpu *vcpu)
{
if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCTEN << 10))
return BEHAVE_HANDLE_LOCALLY;
return BEHAVE_FORWARD_RW;
}
static enum trap_behaviour check_cnthctl_el1pten(struct kvm_vcpu *vcpu)
{
if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCEN << 10))
return BEHAVE_HANDLE_LOCALLY;
return BEHAVE_FORWARD_RW;
}
static enum trap_behaviour check_cptr_tta(struct kvm_vcpu *vcpu)
{
u64 val = __vcpu_sys_reg(vcpu, CPTR_EL2);
if (!vcpu_el2_e2h_is_set(vcpu))
val = translate_cptr_el2_to_cpacr_el1(val);
if (val & CPACR_ELx_TTA)
return BEHAVE_FORWARD_RW;
return BEHAVE_HANDLE_LOCALLY;
}
#define CCC(id, fn) \
[id - __COMPLEX_CONDITIONS__] = fn
static const complex_condition_check ccc[] = {
CCC(CGT_CNTHCTL_EL1PCTEN, check_cnthctl_el1pcten),
CCC(CGT_CNTHCTL_EL1PTEN, check_cnthctl_el1pten),
CCC(CGT_CPTR_TTA, check_cptr_tta),
};
/*
* Bit assignment for the trap controls. We use a 64bit word with the
* following layout for each trapped sysreg:
*
* [9:0] enum cgt_group_id (10 bits)
* [13:10] enum fgt_group_id (4 bits)
* [19:14] bit number in the FGT register (6 bits)
* [20] trap polarity (1 bit)
* [25:21] FG filter (5 bits)
* [35:26] Main SysReg table index (10 bits)
* [62:36] Unused (27 bits)
* [63] RES0 - Must be zero, as lost on insertion in the xarray
*/
#define TC_CGT_BITS 10
#define TC_FGT_BITS 4
#define TC_FGF_BITS 5
#define TC_SRI_BITS 10
union trap_config {
u64 val;
struct {
unsigned long cgt:TC_CGT_BITS; /* Coarse Grained Trap id */
unsigned long fgt:TC_FGT_BITS; /* Fine Grained Trap id */
unsigned long bit:6; /* Bit number */
unsigned long pol:1; /* Polarity */
unsigned long fgf:TC_FGF_BITS; /* Fine Grained Filter */
unsigned long sri:TC_SRI_BITS; /* SysReg Index */
unsigned long unused:27; /* Unused, should be zero */
unsigned long mbz:1; /* Must Be Zero */
};
};
struct encoding_to_trap_config {
const u32 encoding;
const u32 end;
const union trap_config tc;
const unsigned int line;
};
#define SR_RANGE_TRAP(sr_start, sr_end, trap_id) \
{ \
.encoding = sr_start, \
.end = sr_end, \
.tc = { \
.cgt = trap_id, \
}, \
.line = __LINE__, \
}
#define SR_TRAP(sr, trap_id) SR_RANGE_TRAP(sr, sr, trap_id)
/*
* Map encoding to trap bits for exception reported with EC=0x18.
* These must only be evaluated when running a nested hypervisor, but
* that the current context is not a hypervisor context. When the
* trapped access matches one of the trap controls, the exception is
* re-injected in the nested hypervisor.
*/
static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = {
SR_TRAP(SYS_REVIDR_EL1, CGT_HCR_TID1),
SR_TRAP(SYS_AIDR_EL1, CGT_HCR_TID1),
SR_TRAP(SYS_SMIDR_EL1, CGT_HCR_TID1),
SR_TRAP(SYS_CTR_EL0, CGT_HCR_TID2),
SR_TRAP(SYS_CCSIDR_EL1, CGT_HCR_TID2_TID4),
SR_TRAP(SYS_CCSIDR2_EL1, CGT_HCR_TID2_TID4),
SR_TRAP(SYS_CLIDR_EL1, CGT_HCR_TID2_TID4),
SR_TRAP(SYS_CSSELR_EL1, CGT_HCR_TID2_TID4),
SR_RANGE_TRAP(SYS_ID_PFR0_EL1,
sys_reg(3, 0, 0, 7, 7), CGT_HCR_TID3),
SR_TRAP(SYS_ICC_SGI0R_EL1, CGT_HCR_IMO_FMO_ICH_HCR_TC),
SR_TRAP(SYS_ICC_ASGI1R_EL1, CGT_HCR_IMO_FMO_ICH_HCR_TC),
SR_TRAP(SYS_ICC_SGI1R_EL1, CGT_HCR_IMO_FMO_ICH_HCR_TC),
SR_RANGE_TRAP(sys_reg(3, 0, 11, 0, 0),
sys_reg(3, 0, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 1, 11, 0, 0),
sys_reg(3, 1, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 2, 11, 0, 0),
sys_reg(3, 2, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 3, 11, 0, 0),
sys_reg(3, 3, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 4, 11, 0, 0),
sys_reg(3, 4, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 5, 11, 0, 0),
sys_reg(3, 5, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 6, 11, 0, 0),
sys_reg(3, 6, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 7, 11, 0, 0),
sys_reg(3, 7, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 0, 15, 0, 0),
sys_reg(3, 0, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 1, 15, 0, 0),
sys_reg(3, 1, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 2, 15, 0, 0),
sys_reg(3, 2, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 3, 15, 0, 0),
sys_reg(3, 3, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 4, 15, 0, 0),
sys_reg(3, 4, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 5, 15, 0, 0),
sys_reg(3, 5, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 6, 15, 0, 0),
sys_reg(3, 6, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 7, 15, 0, 0),
sys_reg(3, 7, 15, 15, 7), CGT_HCR_TIDCP),
SR_TRAP(SYS_ACTLR_EL1, CGT_HCR_TACR),
SR_TRAP(SYS_DC_ISW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CISW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_IGSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_IGDSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CGSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CGDSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CIGSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CIGDSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CIVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CVAP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CVADP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_IVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CIGVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CIGDVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_IGVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_IGDVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGDVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGVAP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGDVAP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGVADP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGDVADP, CGT_HCR_TPC),
SR_TRAP(SYS_IC_IVAU, CGT_HCR_TPU_TOCU),
SR_TRAP(SYS_IC_IALLU, CGT_HCR_TPU_TOCU),
SR_TRAP(SYS_IC_IALLUIS, CGT_HCR_TPU_TICAB),
SR_TRAP(SYS_DC_CVAU, CGT_HCR_TPU_TOCU),
SR_TRAP(OP_TLBI_RVAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VMALLE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_ASIDE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VMALLE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_ASIDE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VMALLE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_ASIDE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VMALLE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_ASIDE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VMALLE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_ASIDE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VMALLE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_ASIDE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(SYS_SCTLR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TTBR0_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TTBR1_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TCR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_ESR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_FAR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_AFSR0_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_AFSR1_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_MAIR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_AMAIR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_CONTEXTIDR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TCR2_EL1, CGT_HCR_TVM_TRVM_HCRX_TCR2En),
SR_TRAP(SYS_DC_ZVA, CGT_HCR_TDZ),
SR_TRAP(SYS_DC_GVA, CGT_HCR_TDZ),
SR_TRAP(SYS_DC_GZVA, CGT_HCR_TDZ),
SR_TRAP(SYS_LORSA_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LOREA_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LORN_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LORC_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LORID_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_ERRIDR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERRSELR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXADDR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXCTLR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXFR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC0_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC1_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC2_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC3_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXSTATUS_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_APIAKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APIAKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APIBKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APIBKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDAKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDAKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDBKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDBKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APGAKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APGAKEYHI_EL1, CGT_HCR_APK),
/* All _EL2 registers */
SR_TRAP(SYS_BRBCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VPIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VMPIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SCTLR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ACTLR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SCTLR2_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_HCR_EL2,
SYS_HCRX_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SMPRIMAP_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SMCR_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_TTBR0_EL2,
SYS_TCR2_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VTTBR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VTCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VNCR_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_HDFGRTR_EL2,
SYS_HAFGRTR_EL2, CGT_HCR_NV),
/* Skip the SP_EL1 encoding... */
SR_TRAP(SYS_SPSR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ELR_EL2, CGT_HCR_NV),
/* Skip SPSR_irq, SPSR_abt, SPSR_und, SPSR_fiq */
SR_TRAP(SYS_AFSR0_EL2, CGT_HCR_NV),
SR_TRAP(SYS_AFSR1_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ESR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VSESR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_TFSR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_FAR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_HPFAR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_PMSCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MAIR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_AMAIR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MPAMHCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MPAMVPMV_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MPAM2_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_MPAMVPM0_EL2,
SYS_MPAMVPM7_EL2, CGT_HCR_NV),
/*
* Note that the spec. describes a group of MEC registers
* whose access should not trap, therefore skip the following:
* MECID_A0_EL2, MECID_A1_EL2, MECID_P0_EL2,
* MECID_P1_EL2, MECIDR_EL2, VMECID_A_EL2,
* VMECID_P_EL2.
*/
SR_RANGE_TRAP(SYS_VBAR_EL2,
SYS_RMR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VDISR_EL2, CGT_HCR_NV),
/* ICH_AP0R<m>_EL2 */
SR_RANGE_TRAP(SYS_ICH_AP0R0_EL2,
SYS_ICH_AP0R3_EL2, CGT_HCR_NV),
/* ICH_AP1R<m>_EL2 */
SR_RANGE_TRAP(SYS_ICH_AP1R0_EL2,
SYS_ICH_AP1R3_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ICC_SRE_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_ICH_HCR_EL2,
SYS_ICH_EISR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ICH_ELRSR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ICH_VMCR_EL2, CGT_HCR_NV),
/* ICH_LR<m>_EL2 */
SR_RANGE_TRAP(SYS_ICH_LR0_EL2,
SYS_ICH_LR15_EL2, CGT_HCR_NV),
SR_TRAP(SYS_CONTEXTIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_TPIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SCXTNUM_EL2, CGT_HCR_NV),
/* AMEVCNTVOFF0<n>_EL2, AMEVCNTVOFF1<n>_EL2 */
SR_RANGE_TRAP(SYS_AMEVCNTVOFF0n_EL2(0),
SYS_AMEVCNTVOFF1n_EL2(15), CGT_HCR_NV),
/* CNT*_EL2 */
SR_TRAP(SYS_CNTVOFF_EL2, CGT_HCR_NV),
SR_TRAP(SYS_CNTPOFF_EL2, CGT_HCR_NV),
SR_TRAP(SYS_CNTHCTL_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_CNTHP_TVAL_EL2,
SYS_CNTHP_CVAL_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_CNTHV_TVAL_EL2,
SYS_CNTHV_CVAL_EL2, CGT_HCR_NV),
/* All _EL02, _EL12 registers */
SR_RANGE_TRAP(sys_reg(3, 5, 0, 0, 0),
sys_reg(3, 5, 10, 15, 7), CGT_HCR_NV),
SR_RANGE_TRAP(sys_reg(3, 5, 12, 0, 0),
sys_reg(3, 5, 14, 15, 7), CGT_HCR_NV),
SR_TRAP(OP_AT_S1E2R, CGT_HCR_NV),
SR_TRAP(OP_AT_S1E2W, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E1R, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E1W, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E0R, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E0W, CGT_HCR_NV),
SR_TRAP(OP_AT_S1E2A, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1ISNXS,CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1OSNXS,CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_CPP_RCTX, CGT_HCR_NV),
SR_TRAP(OP_DVP_RCTX, CGT_HCR_NV),
SR_TRAP(OP_CFP_RCTX, CGT_HCR_NV),
SR_TRAP(SYS_SP_EL1, CGT_HCR_NV_nNV2),
SR_TRAP(SYS_VBAR_EL1, CGT_HCR_NV1_nNV2),
SR_TRAP(SYS_ELR_EL1, CGT_HCR_NV1_nNV2),
SR_TRAP(SYS_SPSR_EL1, CGT_HCR_NV1_nNV2),
SR_TRAP(SYS_SCXTNUM_EL1, CGT_HCR_NV1_nNV2_ENSCXT),
SR_TRAP(SYS_SCXTNUM_EL0, CGT_HCR_ENSCXT),
SR_TRAP(OP_AT_S1E1R, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E1W, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E0R, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E0W, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E1RP, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E1WP, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E1A, CGT_HCR_AT),
SR_TRAP(SYS_ERXPFGF_EL1, CGT_HCR_nFIEN),
SR_TRAP(SYS_ERXPFGCTL_EL1, CGT_HCR_nFIEN),
SR_TRAP(SYS_ERXPFGCDN_EL1, CGT_HCR_nFIEN),
SR_TRAP(SYS_PMCR_EL0, CGT_MDCR_TPM_TPMCR),
SR_TRAP(SYS_PMCNTENSET_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCNTENCLR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMOVSSET_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMOVSCLR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCEID0_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCEID1_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMXEVTYPER_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMSWINC_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMSELR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMXEVCNTR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCCNTR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMUSERENR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMINTENSET_EL1, CGT_MDCR_TPM),
SR_TRAP(SYS_PMINTENCLR_EL1, CGT_MDCR_TPM),
SR_TRAP(SYS_PMMIR_EL1, CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(0), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(1), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(2), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(3), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(4), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(5), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(6), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(7), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(8), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(9), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(10), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(11), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(12), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(13), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(14), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(15), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(16), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(17), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(18), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(19), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(20), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(21), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(22), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(23), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(24), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(25), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(26), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(27), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(28), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(29), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(30), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(0), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(1), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(2), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(3), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(4), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(5), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(6), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(7), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(8), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(9), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(10), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(11), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(12), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(13), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(14), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(15), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(16), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(17), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(18), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(19), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(20), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(21), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(22), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(23), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(24), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(25), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(26), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(27), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(28), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(29), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(30), CGT_MDCR_TPM),
SR_TRAP(SYS_PMCCFILTR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_MDCCSR_EL0, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_MDCCINT_EL1, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_OSDTRRX_EL1, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_OSDTRTX_EL1, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_DBGDTR_EL0, CGT_MDCR_TDCC_TDE_TDA),
/*
* Also covers DBGDTRRX_EL0, which has the same encoding as
* SYS_DBGDTRTX_EL0...
*/
SR_TRAP(SYS_DBGDTRTX_EL0, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_MDSCR_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_OSECCR_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(15), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(15), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(15), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGCLAIMSET_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGCLAIMCLR_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGAUTHSTATUS_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_OSLAR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_OSLSR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_OSDLR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_DBGPRCR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_MDRAR_EL1, CGT_MDCR_TDE_TDRA),
SR_TRAP(SYS_PMBLIMITR_EL1, CGT_MDCR_E2PB),
SR_TRAP(SYS_PMBPTR_EL1, CGT_MDCR_E2PB),
SR_TRAP(SYS_PMBSR_EL1, CGT_MDCR_E2PB),
SR_TRAP(SYS_PMSCR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSEVFR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSFCR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSICR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSIDR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSIRR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSLATFR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSNEVFR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_TRFCR_EL1, CGT_MDCR_TTRF),
SR_TRAP(SYS_TRBBASER_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBLIMITR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBMAR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBPTR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBSR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBTRG_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_CPACR_EL1, CGT_CPTR_TCPAC),
SR_TRAP(SYS_AMUSERENR_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMCFGR_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMCGCR_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMCNTENCLR0_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMCNTENCLR1_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMCNTENSET0_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMCNTENSET1_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMCR_EL0, CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR0_EL0(0), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR0_EL0(1), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR0_EL0(2), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR0_EL0(3), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(0), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(1), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(2), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(3), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(4), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(5), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(6), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(7), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(8), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(9), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(10), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(11), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(12), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(13), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(14), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVCNTR1_EL0(15), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER0_EL0(0), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER0_EL0(1), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER0_EL0(2), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER0_EL0(3), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(0), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(1), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(2), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(3), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(4), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(5), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(6), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(7), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(8), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(9), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(10), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(11), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(12), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(13), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(14), CGT_CPTR_TAM),
SR_TRAP(SYS_AMEVTYPER1_EL0(15), CGT_CPTR_TAM),
SR_TRAP(SYS_POR_EL0, CGT_CPACR_E0POE),
/* op0=2, op1=1, and CRn<0b1000 */
SR_RANGE_TRAP(sys_reg(2, 1, 0, 0, 0),
sys_reg(2, 1, 7, 15, 7), CGT_CPTR_TTA),
SR_TRAP(SYS_CNTP_TVAL_EL0, CGT_CNTHCTL_EL1PTEN),
SR_TRAP(SYS_CNTP_CVAL_EL0, CGT_CNTHCTL_EL1PTEN),
SR_TRAP(SYS_CNTP_CTL_EL0, CGT_CNTHCTL_EL1PTEN),
SR_TRAP(SYS_CNTPCT_EL0, CGT_CNTHCTL_EL1PCTEN),
SR_TRAP(SYS_CNTPCTSS_EL0, CGT_CNTHCTL_EL1PCTEN),
SR_TRAP(SYS_FPMR, CGT_HCRX_EnFPM),
/*
* IMPDEF choice:
* We treat ICC_SRE_EL2.{SRE,Enable) and ICV_SRE_EL1.SRE as
* RAO/WI. We therefore never consider ICC_SRE_EL2.Enable for
* ICC_SRE_EL1 access, and always handle it locally.
*/
SR_TRAP(SYS_ICC_AP0R0_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_AP0R1_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_AP0R2_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_AP0R3_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_AP1R0_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_AP1R1_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_AP1R2_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_AP1R3_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_BPR0_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_BPR1_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_CTLR_EL1, CGT_ICH_HCR_TC),
SR_TRAP(SYS_ICC_DIR_EL1, CGT_ICH_HCR_TC_TDIR),
SR_TRAP(SYS_ICC_EOIR0_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_EOIR1_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_HPPIR0_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_HPPIR1_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_IAR0_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_IAR1_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_IGRPEN0_EL1, CGT_ICH_HCR_TALL0),
SR_TRAP(SYS_ICC_IGRPEN1_EL1, CGT_ICH_HCR_TALL1),
SR_TRAP(SYS_ICC_PMR_EL1, CGT_ICH_HCR_TC),
SR_TRAP(SYS_ICC_RPR_EL1, CGT_ICH_HCR_TC),
};
static DEFINE_XARRAY(sr_forward_xa);
enum fg_filter_id {
__NO_FGF__,
HCRX_FGTnXS,
/* Must be last */
__NR_FG_FILTER_IDS__
};
#define SR_FGF(sr, g, b, p, f) \
{ \
.encoding = sr, \
.end = sr, \
.tc = { \
.fgt = g ## _GROUP, \
.bit = g ## _EL2_ ## b ## _SHIFT, \
.pol = p, \
.fgf = f, \
}, \
.line = __LINE__, \
}
#define SR_FGT(sr, g, b, p) SR_FGF(sr, g, b, p, __NO_FGF__)
static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = {
/* HFGRTR_EL2, HFGWTR_EL2 */
SR_FGT(SYS_AMAIR2_EL1, HFGxTR, nAMAIR2_EL1, 0),
SR_FGT(SYS_MAIR2_EL1, HFGxTR, nMAIR2_EL1, 0),
SR_FGT(SYS_S2POR_EL1, HFGxTR, nS2POR_EL1, 0),
SR_FGT(SYS_POR_EL1, HFGxTR, nPOR_EL1, 0),
SR_FGT(SYS_POR_EL0, HFGxTR, nPOR_EL0, 0),
SR_FGT(SYS_PIR_EL1, HFGxTR, nPIR_EL1, 0),
SR_FGT(SYS_PIRE0_EL1, HFGxTR, nPIRE0_EL1, 0),
SR_FGT(SYS_RCWMASK_EL1, HFGxTR, nRCWMASK_EL1, 0),
SR_FGT(SYS_TPIDR2_EL0, HFGxTR, nTPIDR2_EL0, 0),
SR_FGT(SYS_SMPRI_EL1, HFGxTR, nSMPRI_EL1, 0),
SR_FGT(SYS_GCSCR_EL1, HFGxTR, nGCS_EL1, 0),
SR_FGT(SYS_GCSPR_EL1, HFGxTR, nGCS_EL1, 0),
SR_FGT(SYS_GCSCRE0_EL1, HFGxTR, nGCS_EL0, 0),
SR_FGT(SYS_GCSPR_EL0, HFGxTR, nGCS_EL0, 0),
SR_FGT(SYS_ACCDATA_EL1, HFGxTR, nACCDATA_EL1, 0),
SR_FGT(SYS_ERXADDR_EL1, HFGxTR, ERXADDR_EL1, 1),
SR_FGT(SYS_ERXPFGCDN_EL1, HFGxTR, ERXPFGCDN_EL1, 1),
SR_FGT(SYS_ERXPFGCTL_EL1, HFGxTR, ERXPFGCTL_EL1, 1),
SR_FGT(SYS_ERXPFGF_EL1, HFGxTR, ERXPFGF_EL1, 1),
SR_FGT(SYS_ERXMISC0_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXMISC1_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXMISC2_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXMISC3_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXSTATUS_EL1, HFGxTR, ERXSTATUS_EL1, 1),
SR_FGT(SYS_ERXCTLR_EL1, HFGxTR, ERXCTLR_EL1, 1),
SR_FGT(SYS_ERXFR_EL1, HFGxTR, ERXFR_EL1, 1),
SR_FGT(SYS_ERRSELR_EL1, HFGxTR, ERRSELR_EL1, 1),
SR_FGT(SYS_ERRIDR_EL1, HFGxTR, ERRIDR_EL1, 1),
SR_FGT(SYS_ICC_IGRPEN0_EL1, HFGxTR, ICC_IGRPENn_EL1, 1),
SR_FGT(SYS_ICC_IGRPEN1_EL1, HFGxTR, ICC_IGRPENn_EL1, 1),
SR_FGT(SYS_VBAR_EL1, HFGxTR, VBAR_EL1, 1),
SR_FGT(SYS_TTBR1_EL1, HFGxTR, TTBR1_EL1, 1),
SR_FGT(SYS_TTBR0_EL1, HFGxTR, TTBR0_EL1, 1),
SR_FGT(SYS_TPIDR_EL0, HFGxTR, TPIDR_EL0, 1),
SR_FGT(SYS_TPIDRRO_EL0, HFGxTR, TPIDRRO_EL0, 1),
SR_FGT(SYS_TPIDR_EL1, HFGxTR, TPIDR_EL1, 1),
SR_FGT(SYS_TCR_EL1, HFGxTR, TCR_EL1, 1),
SR_FGT(SYS_TCR2_EL1, HFGxTR, TCR_EL1, 1),
SR_FGT(SYS_SCXTNUM_EL0, HFGxTR, SCXTNUM_EL0, 1),
SR_FGT(SYS_SCXTNUM_EL1, HFGxTR, SCXTNUM_EL1, 1),
SR_FGT(SYS_SCTLR_EL1, HFGxTR, SCTLR_EL1, 1),
SR_FGT(SYS_REVIDR_EL1, HFGxTR, REVIDR_EL1, 1),
SR_FGT(SYS_PAR_EL1, HFGxTR, PAR_EL1, 1),
SR_FGT(SYS_MPIDR_EL1, HFGxTR, MPIDR_EL1, 1),
SR_FGT(SYS_MIDR_EL1, HFGxTR, MIDR_EL1, 1),
SR_FGT(SYS_MAIR_EL1, HFGxTR, MAIR_EL1, 1),
SR_FGT(SYS_LORSA_EL1, HFGxTR, LORSA_EL1, 1),
SR_FGT(SYS_LORN_EL1, HFGxTR, LORN_EL1, 1),
SR_FGT(SYS_LORID_EL1, HFGxTR, LORID_EL1, 1),
SR_FGT(SYS_LOREA_EL1, HFGxTR, LOREA_EL1, 1),
SR_FGT(SYS_LORC_EL1, HFGxTR, LORC_EL1, 1),
SR_FGT(SYS_ISR_EL1, HFGxTR, ISR_EL1, 1),
SR_FGT(SYS_FAR_EL1, HFGxTR, FAR_EL1, 1),
SR_FGT(SYS_ESR_EL1, HFGxTR, ESR_EL1, 1),
SR_FGT(SYS_DCZID_EL0, HFGxTR, DCZID_EL0, 1),
SR_FGT(SYS_CTR_EL0, HFGxTR, CTR_EL0, 1),
SR_FGT(SYS_CSSELR_EL1, HFGxTR, CSSELR_EL1, 1),
SR_FGT(SYS_CPACR_EL1, HFGxTR, CPACR_EL1, 1),
SR_FGT(SYS_CONTEXTIDR_EL1, HFGxTR, CONTEXTIDR_EL1, 1),
SR_FGT(SYS_CLIDR_EL1, HFGxTR, CLIDR_EL1, 1),
SR_FGT(SYS_CCSIDR_EL1, HFGxTR, CCSIDR_EL1, 1),
SR_FGT(SYS_APIBKEYLO_EL1, HFGxTR, APIBKey, 1),
SR_FGT(SYS_APIBKEYHI_EL1, HFGxTR, APIBKey, 1),
SR_FGT(SYS_APIAKEYLO_EL1, HFGxTR, APIAKey, 1),
SR_FGT(SYS_APIAKEYHI_EL1, HFGxTR, APIAKey, 1),
SR_FGT(SYS_APGAKEYLO_EL1, HFGxTR, APGAKey, 1),
SR_FGT(SYS_APGAKEYHI_EL1, HFGxTR, APGAKey, 1),
SR_FGT(SYS_APDBKEYLO_EL1, HFGxTR, APDBKey, 1),
SR_FGT(SYS_APDBKEYHI_EL1, HFGxTR, APDBKey, 1),
SR_FGT(SYS_APDAKEYLO_EL1, HFGxTR, APDAKey, 1),
SR_FGT(SYS_APDAKEYHI_EL1, HFGxTR, APDAKey, 1),
SR_FGT(SYS_AMAIR_EL1, HFGxTR, AMAIR_EL1, 1),
SR_FGT(SYS_AIDR_EL1, HFGxTR, AIDR_EL1, 1),
SR_FGT(SYS_AFSR1_EL1, HFGxTR, AFSR1_EL1, 1),
SR_FGT(SYS_AFSR0_EL1, HFGxTR, AFSR0_EL1, 1),
/* HFGITR_EL2 */
SR_FGT(OP_AT_S1E1A, HFGITR, ATS1E1A, 1),
SR_FGT(OP_COSP_RCTX, HFGITR, COSPRCTX, 1),
SR_FGT(OP_GCSPUSHX, HFGITR, nGCSEPP, 0),
SR_FGT(OP_GCSPOPX, HFGITR, nGCSEPP, 0),
SR_FGT(OP_GCSPUSHM, HFGITR, nGCSPUSHM_EL1, 0),
SR_FGT(OP_BRB_IALL, HFGITR, nBRBIALL, 0),
SR_FGT(OP_BRB_INJ, HFGITR, nBRBINJ, 0),
SR_FGT(SYS_DC_CVAC, HFGITR, DCCVAC, 1),
SR_FGT(SYS_DC_CGVAC, HFGITR, DCCVAC, 1),
SR_FGT(SYS_DC_CGDVAC, HFGITR, DCCVAC, 1),
SR_FGT(OP_CPP_RCTX, HFGITR, CPPRCTX, 1),
SR_FGT(OP_DVP_RCTX, HFGITR, DVPRCTX, 1),
SR_FGT(OP_CFP_RCTX, HFGITR, CFPRCTX, 1),
SR_FGT(OP_TLBI_VAALE1, HFGITR, TLBIVAALE1, 1),
SR_FGT(OP_TLBI_VALE1, HFGITR, TLBIVALE1, 1),
SR_FGT(OP_TLBI_VAAE1, HFGITR, TLBIVAAE1, 1),
SR_FGT(OP_TLBI_ASIDE1, HFGITR, TLBIASIDE1, 1),
SR_FGT(OP_TLBI_VAE1, HFGITR, TLBIVAE1, 1),
SR_FGT(OP_TLBI_VMALLE1, HFGITR, TLBIVMALLE1, 1),
SR_FGT(OP_TLBI_RVAALE1, HFGITR, TLBIRVAALE1, 1),
SR_FGT(OP_TLBI_RVALE1, HFGITR, TLBIRVALE1, 1),
SR_FGT(OP_TLBI_RVAAE1, HFGITR, TLBIRVAAE1, 1),
SR_FGT(OP_TLBI_RVAE1, HFGITR, TLBIRVAE1, 1),
SR_FGT(OP_TLBI_RVAALE1IS, HFGITR, TLBIRVAALE1IS, 1),
SR_FGT(OP_TLBI_RVALE1IS, HFGITR, TLBIRVALE1IS, 1),
SR_FGT(OP_TLBI_RVAAE1IS, HFGITR, TLBIRVAAE1IS, 1),
SR_FGT(OP_TLBI_RVAE1IS, HFGITR, TLBIRVAE1IS, 1),
SR_FGT(OP_TLBI_VAALE1IS, HFGITR, TLBIVAALE1IS, 1),
SR_FGT(OP_TLBI_VALE1IS, HFGITR, TLBIVALE1IS, 1),
SR_FGT(OP_TLBI_VAAE1IS, HFGITR, TLBIVAAE1IS, 1),
SR_FGT(OP_TLBI_ASIDE1IS, HFGITR, TLBIASIDE1IS, 1),
SR_FGT(OP_TLBI_VAE1IS, HFGITR, TLBIVAE1IS, 1),
SR_FGT(OP_TLBI_VMALLE1IS, HFGITR, TLBIVMALLE1IS, 1),
SR_FGT(OP_TLBI_RVAALE1OS, HFGITR, TLBIRVAALE1OS, 1),
SR_FGT(OP_TLBI_RVALE1OS, HFGITR, TLBIRVALE1OS, 1),
SR_FGT(OP_TLBI_RVAAE1OS, HFGITR, TLBIRVAAE1OS, 1),
SR_FGT(OP_TLBI_RVAE1OS, HFGITR, TLBIRVAE1OS, 1),
SR_FGT(OP_TLBI_VAALE1OS, HFGITR, TLBIVAALE1OS, 1),
SR_FGT(OP_TLBI_VALE1OS, HFGITR, TLBIVALE1OS, 1),
SR_FGT(OP_TLBI_VAAE1OS, HFGITR, TLBIVAAE1OS, 1),
SR_FGT(OP_TLBI_ASIDE1OS, HFGITR, TLBIASIDE1OS, 1),
SR_FGT(OP_TLBI_VAE1OS, HFGITR, TLBIVAE1OS, 1),
SR_FGT(OP_TLBI_VMALLE1OS, HFGITR, TLBIVMALLE1OS, 1),
/* nXS variants must be checked against HCRX_EL2.FGTnXS */
SR_FGF(OP_TLBI_VAALE1NXS, HFGITR, TLBIVAALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VALE1NXS, HFGITR, TLBIVALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAAE1NXS, HFGITR, TLBIVAAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_ASIDE1NXS, HFGITR, TLBIASIDE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAE1NXS, HFGITR, TLBIVAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VMALLE1NXS, HFGITR, TLBIVMALLE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAALE1NXS, HFGITR, TLBIRVAALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVALE1NXS, HFGITR, TLBIRVALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAAE1NXS, HFGITR, TLBIRVAAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAE1NXS, HFGITR, TLBIRVAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAALE1ISNXS, HFGITR, TLBIRVAALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVALE1ISNXS, HFGITR, TLBIRVALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAAE1ISNXS, HFGITR, TLBIRVAAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAE1ISNXS, HFGITR, TLBIRVAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAALE1ISNXS, HFGITR, TLBIVAALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VALE1ISNXS, HFGITR, TLBIVALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAAE1ISNXS, HFGITR, TLBIVAAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_ASIDE1ISNXS, HFGITR, TLBIASIDE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAE1ISNXS, HFGITR, TLBIVAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VMALLE1ISNXS, HFGITR, TLBIVMALLE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAALE1OSNXS, HFGITR, TLBIRVAALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVALE1OSNXS, HFGITR, TLBIRVALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAAE1OSNXS, HFGITR, TLBIRVAAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAE1OSNXS, HFGITR, TLBIRVAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAALE1OSNXS, HFGITR, TLBIVAALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VALE1OSNXS, HFGITR, TLBIVALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAAE1OSNXS, HFGITR, TLBIVAAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_ASIDE1OSNXS, HFGITR, TLBIASIDE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAE1OSNXS, HFGITR, TLBIVAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VMALLE1OSNXS, HFGITR, TLBIVMALLE1OS, 1, HCRX_FGTnXS),
SR_FGT(OP_AT_S1E1WP, HFGITR, ATS1E1WP, 1),
SR_FGT(OP_AT_S1E1RP, HFGITR, ATS1E1RP, 1),
SR_FGT(OP_AT_S1E0W, HFGITR, ATS1E0W, 1),
SR_FGT(OP_AT_S1E0R, HFGITR, ATS1E0R, 1),
SR_FGT(OP_AT_S1E1W, HFGITR, ATS1E1W, 1),
SR_FGT(OP_AT_S1E1R, HFGITR, ATS1E1R, 1),
SR_FGT(SYS_DC_ZVA, HFGITR, DCZVA, 1),
SR_FGT(SYS_DC_GVA, HFGITR, DCZVA, 1),
SR_FGT(SYS_DC_GZVA, HFGITR, DCZVA, 1),
SR_FGT(SYS_DC_CIVAC, HFGITR, DCCIVAC, 1),
SR_FGT(SYS_DC_CIGVAC, HFGITR, DCCIVAC, 1),
SR_FGT(SYS_DC_CIGDVAC, HFGITR, DCCIVAC, 1),
SR_FGT(SYS_DC_CVADP, HFGITR, DCCVADP, 1),
SR_FGT(SYS_DC_CGVADP, HFGITR, DCCVADP, 1),
SR_FGT(SYS_DC_CGDVADP, HFGITR, DCCVADP, 1),
SR_FGT(SYS_DC_CVAP, HFGITR, DCCVAP, 1),
SR_FGT(SYS_DC_CGVAP, HFGITR, DCCVAP, 1),
SR_FGT(SYS_DC_CGDVAP, HFGITR, DCCVAP, 1),
SR_FGT(SYS_DC_CVAU, HFGITR, DCCVAU, 1),
SR_FGT(SYS_DC_CISW, HFGITR, DCCISW, 1),
SR_FGT(SYS_DC_CIGSW, HFGITR, DCCISW, 1),
SR_FGT(SYS_DC_CIGDSW, HFGITR, DCCISW, 1),
SR_FGT(SYS_DC_CSW, HFGITR, DCCSW, 1),
SR_FGT(SYS_DC_CGSW, HFGITR, DCCSW, 1),
SR_FGT(SYS_DC_CGDSW, HFGITR, DCCSW, 1),
SR_FGT(SYS_DC_ISW, HFGITR, DCISW, 1),
SR_FGT(SYS_DC_IGSW, HFGITR, DCISW, 1),
SR_FGT(SYS_DC_IGDSW, HFGITR, DCISW, 1),
SR_FGT(SYS_DC_IVAC, HFGITR, DCIVAC, 1),
SR_FGT(SYS_DC_IGVAC, HFGITR, DCIVAC, 1),
SR_FGT(SYS_DC_IGDVAC, HFGITR, DCIVAC, 1),
SR_FGT(SYS_IC_IVAU, HFGITR, ICIVAU, 1),
SR_FGT(SYS_IC_IALLU, HFGITR, ICIALLU, 1),
SR_FGT(SYS_IC_IALLUIS, HFGITR, ICIALLUIS, 1),
/* HDFGRTR_EL2 */
SR_FGT(SYS_PMBIDR_EL1, HDFGRTR, PMBIDR_EL1, 1),
SR_FGT(SYS_PMSNEVFR_EL1, HDFGRTR, nPMSNEVFR_EL1, 0),
SR_FGT(SYS_BRBINF_EL1(0), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(1), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(2), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(3), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(4), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(5), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(6), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(7), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(8), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(9), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(10), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(11), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(12), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(13), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(14), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(15), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(16), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(17), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(18), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(19), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(20), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(21), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(22), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(23), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(24), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(25), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(26), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(27), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(28), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(29), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(30), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(31), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINFINJ_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(0), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(1), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(2), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(3), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(4), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(5), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(6), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(7), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(8), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(9), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(10), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(11), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(12), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(13), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(14), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(15), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(16), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(17), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(18), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(19), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(20), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(21), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(22), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(23), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(24), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(25), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(26), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(27), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(28), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(29), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(30), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(31), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRCINJ_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(0), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(1), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(2), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(3), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(4), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(5), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(6), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(7), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(8), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(9), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(10), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(11), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(12), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(13), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(14), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(15), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(16), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(17), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(18), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(19), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(20), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(21), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(22), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(23), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(24), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(25), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(26), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(27), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(28), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(29), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(30), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(31), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGTINJ_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTS_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBCR_EL1, HDFGRTR, nBRBCTL, 0),
SR_FGT(SYS_BRBFCR_EL1, HDFGRTR, nBRBCTL, 0),
SR_FGT(SYS_BRBIDR0_EL1, HDFGRTR, nBRBIDR, 0),
SR_FGT(SYS_PMCEID0_EL0, HDFGRTR, PMCEIDn_EL0, 1),
SR_FGT(SYS_PMCEID1_EL0, HDFGRTR, PMCEIDn_EL0, 1),
SR_FGT(SYS_PMUSERENR_EL0, HDFGRTR, PMUSERENR_EL0, 1),
SR_FGT(SYS_TRBTRG_EL1, HDFGRTR, TRBTRG_EL1, 1),
SR_FGT(SYS_TRBSR_EL1, HDFGRTR, TRBSR_EL1, 1),
SR_FGT(SYS_TRBPTR_EL1, HDFGRTR, TRBPTR_EL1, 1),
SR_FGT(SYS_TRBMAR_EL1, HDFGRTR, TRBMAR_EL1, 1),
SR_FGT(SYS_TRBLIMITR_EL1, HDFGRTR, TRBLIMITR_EL1, 1),
SR_FGT(SYS_TRBIDR_EL1, HDFGRTR, TRBIDR_EL1, 1),
SR_FGT(SYS_TRBBASER_EL1, HDFGRTR, TRBBASER_EL1, 1),
SR_FGT(SYS_TRCVICTLR, HDFGRTR, TRCVICTLR, 1),
SR_FGT(SYS_TRCSTATR, HDFGRTR, TRCSTATR, 1),
SR_FGT(SYS_TRCSSCSR(0), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(1), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(2), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(3), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(4), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(5), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(6), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(7), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSEQSTR, HDFGRTR, TRCSEQSTR, 1),
SR_FGT(SYS_TRCPRGCTLR, HDFGRTR, TRCPRGCTLR, 1),
SR_FGT(SYS_TRCOSLSR, HDFGRTR, TRCOSLSR, 1),
SR_FGT(SYS_TRCIMSPEC(0), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(1), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(2), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(3), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(4), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(5), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(6), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(7), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCDEVARCH, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCDEVID, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR0, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR1, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR2, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR3, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR4, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR5, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR6, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR7, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR8, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR9, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR10, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR11, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR12, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR13, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCCNTVR(0), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCNTVR(1), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCNTVR(2), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCNTVR(3), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCLAIMCLR, HDFGRTR, TRCCLAIM, 1),
SR_FGT(SYS_TRCCLAIMSET, HDFGRTR, TRCCLAIM, 1),
SR_FGT(SYS_TRCAUXCTLR, HDFGRTR, TRCAUXCTLR, 1),
SR_FGT(SYS_TRCAUTHSTATUS, HDFGRTR, TRCAUTHSTATUS, 1),
SR_FGT(SYS_TRCACATR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(8), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(9), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(10), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(11), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(12), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(13), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(14), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(15), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(8), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(9), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(10), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(11), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(12), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(13), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(14), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(15), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCBBCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCCCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCCTLR0, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCCTLR1, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCONFIGR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEVENTCTL0R, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEVENTCTL1R, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCQCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(8), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(9), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(10), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(11), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(12), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(13), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(14), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(15), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(16), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(17), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(18), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(19), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(20), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(21), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(22), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(23), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(24), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(25), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(26), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(27), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(28), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(29), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(30), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(31), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQEVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQEVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQEVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQRSTEVR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSTALLCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSYNCPR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCTRACEIDR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCTSCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVIIECTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVIPCSSCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVISSCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCCTLR0, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCCTLR1, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_PMSLATFR_EL1, HDFGRTR, PMSLATFR_EL1, 1),
SR_FGT(SYS_PMSIRR_EL1, HDFGRTR, PMSIRR_EL1, 1),
SR_FGT(SYS_PMSIDR_EL1, HDFGRTR, PMSIDR_EL1, 1),
SR_FGT(SYS_PMSICR_EL1, HDFGRTR, PMSICR_EL1, 1),
SR_FGT(SYS_PMSFCR_EL1, HDFGRTR, PMSFCR_EL1, 1),
SR_FGT(SYS_PMSEVFR_EL1, HDFGRTR, PMSEVFR_EL1, 1),
SR_FGT(SYS_PMSCR_EL1, HDFGRTR, PMSCR_EL1, 1),
SR_FGT(SYS_PMBSR_EL1, HDFGRTR, PMBSR_EL1, 1),
SR_FGT(SYS_PMBPTR_EL1, HDFGRTR, PMBPTR_EL1, 1),
SR_FGT(SYS_PMBLIMITR_EL1, HDFGRTR, PMBLIMITR_EL1, 1),
SR_FGT(SYS_PMMIR_EL1, HDFGRTR, PMMIR_EL1, 1),
SR_FGT(SYS_PMSELR_EL0, HDFGRTR, PMSELR_EL0, 1),
SR_FGT(SYS_PMOVSCLR_EL0, HDFGRTR, PMOVS, 1),
SR_FGT(SYS_PMOVSSET_EL0, HDFGRTR, PMOVS, 1),
SR_FGT(SYS_PMINTENCLR_EL1, HDFGRTR, PMINTEN, 1),
SR_FGT(SYS_PMINTENSET_EL1, HDFGRTR, PMINTEN, 1),
SR_FGT(SYS_PMCNTENCLR_EL0, HDFGRTR, PMCNTEN, 1),
SR_FGT(SYS_PMCNTENSET_EL0, HDFGRTR, PMCNTEN, 1),
SR_FGT(SYS_PMCCNTR_EL0, HDFGRTR, PMCCNTR_EL0, 1),
SR_FGT(SYS_PMCCFILTR_EL0, HDFGRTR, PMCCFILTR_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(0), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(1), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(2), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(3), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(4), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(5), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(6), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(7), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(8), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(9), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(10), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(11), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(12), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(13), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(14), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(15), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(16), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(17), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(18), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(19), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(20), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(21), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(22), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(23), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(24), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(25), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(26), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(27), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(28), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(29), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(30), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(0), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(1), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(2), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(3), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(4), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(5), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(6), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(7), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(8), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(9), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(10), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(11), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(12), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(13), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(14), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(15), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(16), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(17), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(18), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(19), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(20), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(21), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(22), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(23), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(24), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(25), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(26), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(27), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(28), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(29), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(30), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_OSDLR_EL1, HDFGRTR, OSDLR_EL1, 1),
SR_FGT(SYS_OSECCR_EL1, HDFGRTR, OSECCR_EL1, 1),
SR_FGT(SYS_OSLSR_EL1, HDFGRTR, OSLSR_EL1, 1),
SR_FGT(SYS_DBGPRCR_EL1, HDFGRTR, DBGPRCR_EL1, 1),
SR_FGT(SYS_DBGAUTHSTATUS_EL1, HDFGRTR, DBGAUTHSTATUS_EL1, 1),
SR_FGT(SYS_DBGCLAIMSET_EL1, HDFGRTR, DBGCLAIM, 1),
SR_FGT(SYS_DBGCLAIMCLR_EL1, HDFGRTR, DBGCLAIM, 1),
SR_FGT(SYS_MDSCR_EL1, HDFGRTR, MDSCR_EL1, 1),
/*
* The trap bits capture *64* debug registers per bit, but the
* ARM ARM only describes the encoding for the first 16, and
* we don't really support more than that anyway.
*/
SR_FGT(SYS_DBGWVRn_EL1(0), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(1), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(2), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(3), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(4), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(5), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(6), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(7), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(8), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(9), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(10), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(11), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(12), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(13), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(14), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(15), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(0), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(1), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(2), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(3), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(4), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(5), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(6), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(7), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(8), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(9), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(10), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(11), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(12), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(13), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(14), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(15), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(0), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(1), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(2), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(3), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(4), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(5), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(6), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(7), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(8), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(9), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(10), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(11), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(12), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(13), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(14), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(15), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(0), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(1), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(2), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(3), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(4), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(5), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(6), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(7), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(8), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(9), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(10), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(11), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(12), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(13), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(14), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(15), HDFGRTR, DBGBCRn_EL1, 1),
/*
* HDFGWTR_EL2
*
* Although HDFGRTR_EL2 and HDFGWTR_EL2 registers largely
* overlap in their bit assignment, there are a number of bits
* that are RES0 on one side, and an actual trap bit on the
* other. The policy chosen here is to describe all the
* read-side mappings, and only the write-side mappings that
* differ from the read side, and the trap handler will pick
* the correct shadow register based on the access type.
*/
SR_FGT(SYS_TRFCR_EL1, HDFGWTR, TRFCR_EL1, 1),
SR_FGT(SYS_TRCOSLAR, HDFGWTR, TRCOSLAR, 1),
SR_FGT(SYS_PMCR_EL0, HDFGWTR, PMCR_EL0, 1),
SR_FGT(SYS_PMSWINC_EL0, HDFGWTR, PMSWINC_EL0, 1),
SR_FGT(SYS_OSLAR_EL1, HDFGWTR, OSLAR_EL1, 1),
/*
* HAFGRTR_EL2
*/
SR_FGT(SYS_AMEVTYPER1_EL0(15), HAFGRTR, AMEVTYPER115_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(14), HAFGRTR, AMEVTYPER114_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(13), HAFGRTR, AMEVTYPER113_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(12), HAFGRTR, AMEVTYPER112_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(11), HAFGRTR, AMEVTYPER111_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(10), HAFGRTR, AMEVTYPER110_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(9), HAFGRTR, AMEVTYPER19_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(8), HAFGRTR, AMEVTYPER18_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(7), HAFGRTR, AMEVTYPER17_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(6), HAFGRTR, AMEVTYPER16_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(5), HAFGRTR, AMEVTYPER15_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(4), HAFGRTR, AMEVTYPER14_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(3), HAFGRTR, AMEVTYPER13_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(2), HAFGRTR, AMEVTYPER12_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(1), HAFGRTR, AMEVTYPER11_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(0), HAFGRTR, AMEVTYPER10_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(15), HAFGRTR, AMEVCNTR115_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(14), HAFGRTR, AMEVCNTR114_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(13), HAFGRTR, AMEVCNTR113_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(12), HAFGRTR, AMEVCNTR112_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(11), HAFGRTR, AMEVCNTR111_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(10), HAFGRTR, AMEVCNTR110_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(9), HAFGRTR, AMEVCNTR19_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(8), HAFGRTR, AMEVCNTR18_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(7), HAFGRTR, AMEVCNTR17_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(6), HAFGRTR, AMEVCNTR16_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(5), HAFGRTR, AMEVCNTR15_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(4), HAFGRTR, AMEVCNTR14_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(3), HAFGRTR, AMEVCNTR13_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(2), HAFGRTR, AMEVCNTR12_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(1), HAFGRTR, AMEVCNTR11_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(0), HAFGRTR, AMEVCNTR10_EL0, 1),
SR_FGT(SYS_AMCNTENCLR1_EL0, HAFGRTR, AMCNTEN1, 1),
SR_FGT(SYS_AMCNTENSET1_EL0, HAFGRTR, AMCNTEN1, 1),
SR_FGT(SYS_AMCNTENCLR0_EL0, HAFGRTR, AMCNTEN0, 1),
SR_FGT(SYS_AMCNTENSET0_EL0, HAFGRTR, AMCNTEN0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(3), HAFGRTR, AMEVCNTR03_EL0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(2), HAFGRTR, AMEVCNTR02_EL0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(1), HAFGRTR, AMEVCNTR01_EL0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(0), HAFGRTR, AMEVCNTR00_EL0, 1),
};
static union trap_config get_trap_config(u32 sysreg)
{
return (union trap_config) {
.val = xa_to_value(xa_load(&sr_forward_xa, sysreg)),
};
}
static __init void print_nv_trap_error(const struct encoding_to_trap_config *tc,
const char *type, int err)
{
kvm_err("%s line %d encoding range "
"(%d, %d, %d, %d, %d) - (%d, %d, %d, %d, %d) (err=%d)\n",
type, tc->line,
sys_reg_Op0(tc->encoding), sys_reg_Op1(tc->encoding),
sys_reg_CRn(tc->encoding), sys_reg_CRm(tc->encoding),
sys_reg_Op2(tc->encoding),
sys_reg_Op0(tc->end), sys_reg_Op1(tc->end),
sys_reg_CRn(tc->end), sys_reg_CRm(tc->end),
sys_reg_Op2(tc->end),
err);
}
static u32 encoding_next(u32 encoding)
{
u8 op0, op1, crn, crm, op2;
op0 = sys_reg_Op0(encoding);
op1 = sys_reg_Op1(encoding);
crn = sys_reg_CRn(encoding);
crm = sys_reg_CRm(encoding);
op2 = sys_reg_Op2(encoding);
if (op2 < Op2_mask)
return sys_reg(op0, op1, crn, crm, op2 + 1);
if (crm < CRm_mask)
return sys_reg(op0, op1, crn, crm + 1, 0);
if (crn < CRn_mask)
return sys_reg(op0, op1, crn + 1, 0, 0);
if (op1 < Op1_mask)
return sys_reg(op0, op1 + 1, 0, 0, 0);
return sys_reg(op0 + 1, 0, 0, 0, 0);
}
int __init populate_nv_trap_config(void)
{
int ret = 0;
BUILD_BUG_ON(sizeof(union trap_config) != sizeof(void *));
BUILD_BUG_ON(__NR_CGT_GROUP_IDS__ > BIT(TC_CGT_BITS));
BUILD_BUG_ON(__NR_FGT_GROUP_IDS__ > BIT(TC_FGT_BITS));
BUILD_BUG_ON(__NR_FG_FILTER_IDS__ > BIT(TC_FGF_BITS));
for (int i = 0; i < ARRAY_SIZE(encoding_to_cgt); i++) {
const struct encoding_to_trap_config *cgt = &encoding_to_cgt[i];
void *prev;
if (cgt->tc.val & BIT(63)) {
kvm_err("CGT[%d] has MBZ bit set\n", i);
ret = -EINVAL;
}
for (u32 enc = cgt->encoding; enc <= cgt->end; enc = encoding_next(enc)) {
prev = xa_store(&sr_forward_xa, enc,
xa_mk_value(cgt->tc.val), GFP_KERNEL);
if (prev && !xa_is_err(prev)) {
ret = -EINVAL;
print_nv_trap_error(cgt, "Duplicate CGT", ret);
}
if (xa_is_err(prev)) {
ret = xa_err(prev);
print_nv_trap_error(cgt, "Failed CGT insertion", ret);
}
}
}
kvm_info("nv: %ld coarse grained trap handlers\n",
ARRAY_SIZE(encoding_to_cgt));
if (!cpus_have_final_cap(ARM64_HAS_FGT))
goto check_mcb;
for (int i = 0; i < ARRAY_SIZE(encoding_to_fgt); i++) {
const struct encoding_to_trap_config *fgt = &encoding_to_fgt[i];
union trap_config tc;
void *prev;
if (fgt->tc.fgt >= __NR_FGT_GROUP_IDS__) {
ret = -EINVAL;
print_nv_trap_error(fgt, "Invalid FGT", ret);
}
tc = get_trap_config(fgt->encoding);
if (tc.fgt) {
ret = -EINVAL;
print_nv_trap_error(fgt, "Duplicate FGT", ret);
}
tc.val |= fgt->tc.val;
prev = xa_store(&sr_forward_xa, fgt->encoding,
xa_mk_value(tc.val), GFP_KERNEL);
if (xa_is_err(prev)) {
ret = xa_err(prev);
print_nv_trap_error(fgt, "Failed FGT insertion", ret);
}
}
kvm_info("nv: %ld fine grained trap handlers\n",
ARRAY_SIZE(encoding_to_fgt));
check_mcb:
for (int id = __MULTIPLE_CONTROL_BITS__; id < __COMPLEX_CONDITIONS__; id++) {
const enum cgt_group_id *cgids;
cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
for (int i = 0; cgids[i] != __RESERVED__; i++) {
if (cgids[i] >= __MULTIPLE_CONTROL_BITS__ &&
cgids[i] < __COMPLEX_CONDITIONS__) {
kvm_err("Recursive MCB %d/%d\n", id, cgids[i]);
ret = -EINVAL;
}
}
}
if (ret)
xa_destroy(&sr_forward_xa);
return ret;
}
int __init populate_sysreg_config(const struct sys_reg_desc *sr,
unsigned int idx)
{
union trap_config tc;
u32 encoding;
void *ret;
/*
* 0 is a valid value for the index, but not for the storage.
* We'll store (idx+1), so check against an offset'd limit.
*/
if (idx >= (BIT(TC_SRI_BITS) - 1)) {
kvm_err("sysreg %s (%d) out of range\n", sr->name, idx);
return -EINVAL;
}
encoding = sys_reg(sr->Op0, sr->Op1, sr->CRn, sr->CRm, sr->Op2);
tc = get_trap_config(encoding);
if (tc.sri) {
kvm_err("sysreg %s (%d) duplicate entry (%d)\n",
sr->name, idx - 1, tc.sri);
return -EINVAL;
}
tc.sri = idx + 1;
ret = xa_store(&sr_forward_xa, encoding,
xa_mk_value(tc.val), GFP_KERNEL);
return xa_err(ret);
}
static enum trap_behaviour get_behaviour(struct kvm_vcpu *vcpu,
const struct trap_bits *tb)
{
enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
u64 val;
val = __vcpu_sys_reg(vcpu, tb->index);
if ((val & tb->mask) == tb->value)
b |= tb->behaviour;
return b;
}
static enum trap_behaviour __compute_trap_behaviour(struct kvm_vcpu *vcpu,
const enum cgt_group_id id,
enum trap_behaviour b)
{
switch (id) {
const enum cgt_group_id *cgids;
case __RESERVED__ ... __MULTIPLE_CONTROL_BITS__ - 1:
if (likely(id != __RESERVED__))
b |= get_behaviour(vcpu, &coarse_trap_bits[id]);
break;
case __MULTIPLE_CONTROL_BITS__ ... __COMPLEX_CONDITIONS__ - 1:
/* Yes, this is recursive. Don't do anything stupid. */
cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
for (int i = 0; cgids[i] != __RESERVED__; i++)
b |= __compute_trap_behaviour(vcpu, cgids[i], b);
break;
default:
if (ARRAY_SIZE(ccc))
b |= ccc[id - __COMPLEX_CONDITIONS__](vcpu);
break;
}
return b;
}
static enum trap_behaviour compute_trap_behaviour(struct kvm_vcpu *vcpu,
const union trap_config tc)
{
enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
return __compute_trap_behaviour(vcpu, tc.cgt, b);
}
static u64 kvm_get_sysreg_res0(struct kvm *kvm, enum vcpu_sysreg sr)
{
struct kvm_sysreg_masks *masks;
/* Only handle the VNCR-backed regs for now */
if (sr < __VNCR_START__)
return 0;
masks = kvm->arch.sysreg_masks;
return masks->mask[sr - __VNCR_START__].res0;
}
static bool check_fgt_bit(struct kvm_vcpu *vcpu, bool is_read,
u64 val, const union trap_config tc)
{
struct kvm *kvm = vcpu->kvm;
enum vcpu_sysreg sr;
/*
* KVM doesn't know about any FGTs that apply to the host, and hopefully
* that'll remain the case.
*/
if (is_hyp_ctxt(vcpu))
return false;
if (tc.pol)
return (val & BIT(tc.bit));
/*
* FGTs with negative polarities are an absolute nightmare, as
* we need to evaluate the bit in the light of the feature
* that defines it. WTF were they thinking?
*
* So let's check if the bit has been earmarked as RES0, as
* this indicates an unimplemented feature.
*/
if (val & BIT(tc.bit))
return false;
switch ((enum fgt_group_id)tc.fgt) {
case HFGxTR_GROUP:
sr = is_read ? HFGRTR_EL2 : HFGWTR_EL2;
break;
case HDFGRTR_GROUP:
sr = is_read ? HDFGRTR_EL2 : HDFGWTR_EL2;
break;
case HAFGRTR_GROUP:
sr = HAFGRTR_EL2;
break;
case HFGITR_GROUP:
sr = HFGITR_EL2;
break;
default:
WARN_ONCE(1, "Unhandled FGT group");
return false;
}
return !(kvm_get_sysreg_res0(kvm, sr) & BIT(tc.bit));
}
bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
{
union trap_config tc;
enum trap_behaviour b;
bool is_read;
u32 sysreg;
u64 esr, val;
esr = kvm_vcpu_get_esr(vcpu);
sysreg = esr_sys64_to_sysreg(esr);
is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
tc = get_trap_config(sysreg);
/*
* A value of 0 for the whole entry means that we know nothing
* for this sysreg, and that it cannot be re-injected into the
* nested hypervisor. In this situation, let's cut it short.
*/
if (!tc.val)
goto local;
/*
* If a sysreg can be trapped using a FGT, first check whether we
* trap for the purpose of forbidding the feature. In that case,
* inject an UNDEF.
*/
if (tc.fgt != __NO_FGT_GROUP__ &&
(vcpu->kvm->arch.fgu[tc.fgt] & BIT(tc.bit))) {
kvm_inject_undefined(vcpu);
return true;
}
/*
* If we're not nesting, immediately return to the caller, with the
* sysreg index, should we have it.
*/
if (!vcpu_has_nv(vcpu))
goto local;
/*
* There are a few traps that take effect InHost, but are constrained
* to EL0. Don't bother with computing the trap behaviour if the vCPU
* isn't in EL0.
*/
if (is_hyp_ctxt(vcpu) && !vcpu_is_host_el0(vcpu))
goto local;
switch ((enum fgt_group_id)tc.fgt) {
case __NO_FGT_GROUP__:
break;
case HFGxTR_GROUP:
if (is_read)
val = __vcpu_sys_reg(vcpu, HFGRTR_EL2);
else
val = __vcpu_sys_reg(vcpu, HFGWTR_EL2);
break;
case HDFGRTR_GROUP:
if (is_read)
val = __vcpu_sys_reg(vcpu, HDFGRTR_EL2);
else
val = __vcpu_sys_reg(vcpu, HDFGWTR_EL2);
break;
case HAFGRTR_GROUP:
val = __vcpu_sys_reg(vcpu, HAFGRTR_EL2);
break;
case HFGITR_GROUP:
val = __vcpu_sys_reg(vcpu, HFGITR_EL2);
switch (tc.fgf) {
u64 tmp;
case __NO_FGF__:
break;
case HCRX_FGTnXS:
tmp = __vcpu_sys_reg(vcpu, HCRX_EL2);
if (tmp & HCRX_EL2_FGTnXS)
tc.fgt = __NO_FGT_GROUP__;
}
break;
case __NR_FGT_GROUP_IDS__:
/* Something is really wrong, bail out */
WARN_ONCE(1, "__NR_FGT_GROUP_IDS__");
goto local;
}
if (tc.fgt != __NO_FGT_GROUP__ && check_fgt_bit(vcpu, is_read, val, tc))
goto inject;
b = compute_trap_behaviour(vcpu, tc);
if (!(b & BEHAVE_FORWARD_IN_HOST_EL0) && vcpu_is_host_el0(vcpu))
goto local;
if (((b & BEHAVE_FORWARD_READ) && is_read) ||
((b & BEHAVE_FORWARD_WRITE) && !is_read))
goto inject;
local:
if (!tc.sri) {
struct sys_reg_params params;
params = esr_sys64_to_params(esr);
/*
* Check for the IMPDEF range, as per DDI0487 J.a,
* D18.3.2 Reserved encodings for IMPLEMENTATION
* DEFINED registers.
*/
if (!(params.Op0 == 3 && (params.CRn & 0b1011) == 0b1011))
print_sys_reg_msg(&params,
"Unsupported guest access at: %lx\n",
*vcpu_pc(vcpu));
kvm_inject_undefined(vcpu);
return true;
}
*sr_index = tc.sri - 1;
return false;
inject:
trace_kvm_forward_sysreg_trap(vcpu, sysreg, is_read);
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
return true;
}
static bool forward_traps(struct kvm_vcpu *vcpu, u64 control_bit)
{
bool control_bit_set;
if (!vcpu_has_nv(vcpu))
return false;
control_bit_set = __vcpu_sys_reg(vcpu, HCR_EL2) & control_bit;
if (!is_hyp_ctxt(vcpu) && control_bit_set) {
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
return true;
}
return false;
}
bool forward_smc_trap(struct kvm_vcpu *vcpu)
{
return forward_traps(vcpu, HCR_TSC);
}
static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr)
{
u64 mode = spsr & PSR_MODE_MASK;
/*
* Possible causes for an Illegal Exception Return from EL2:
* - trying to return to EL3
* - trying to return to an illegal M value
* - trying to return to a 32bit EL
* - trying to return to EL1 with HCR_EL2.TGE set
*/
if (mode == PSR_MODE_EL3t || mode == PSR_MODE_EL3h ||
mode == 0b00001 || (mode & BIT(1)) ||
(spsr & PSR_MODE32_BIT) ||
(vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t ||
mode == PSR_MODE_EL1h))) {
/*
* The guest is playing with our nerves. Preserve EL, SP,
* masks, flags from the existing PSTATE, and set IL.
* The HW will then generate an Illegal State Exception
* immediately after ERET.
*/
spsr = *vcpu_cpsr(vcpu);
spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT |
PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT |
PSR_MODE_MASK | PSR_MODE32_BIT);
spsr |= PSR_IL_BIT;
}
return spsr;
}
void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu)
{
u64 spsr, elr, esr;
/*
* Forward this trap to the virtual EL2 if the virtual
* HCR_EL2.NV bit is set and this is coming from !EL2.
*/
if (forward_traps(vcpu, HCR_NV))
return;
spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
spsr = kvm_check_illegal_exception_return(vcpu, spsr);
/* Check for an ERETAx */
esr = kvm_vcpu_get_esr(vcpu);
if (esr_iss_is_eretax(esr) && !kvm_auth_eretax(vcpu, &elr)) {
/*
* Oh no, ERETAx failed to authenticate.
*
* If we have FPACCOMBINE and we don't have a pending
* Illegal Execution State exception (which has priority
* over FPAC), deliver an exception right away.
*
* Otherwise, let the mangled ELR value trickle down the
* ERET handling, and the guest will have a little surprise.
*/
if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) {
esr &= ESR_ELx_ERET_ISS_ERETA;
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_FPAC);
kvm_inject_nested_sync(vcpu, esr);
return;
}
}
preempt_disable();
kvm_arch_vcpu_put(vcpu);
if (!esr_iss_is_eretax(esr))
elr = __vcpu_sys_reg(vcpu, ELR_EL2);
trace_kvm_nested_eret(vcpu, elr, spsr);
*vcpu_pc(vcpu) = elr;
*vcpu_cpsr(vcpu) = spsr;
kvm_arch_vcpu_load(vcpu, smp_processor_id());
preempt_enable();
}
static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2,
enum exception_type type)
{
trace_kvm_inject_nested_exception(vcpu, esr_el2, type);
switch (type) {
case except_type_sync:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2);
break;
case except_type_irq:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
break;
default:
WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
}
}
/*
* Emulate taking an exception to EL2.
* See ARM ARM J8.1.2 AArch64.TakeException()
*/
static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2,
enum exception_type type)
{
u64 pstate, mode;
bool direct_inject;
if (!vcpu_has_nv(vcpu)) {
kvm_err("Unexpected call to %s for the non-nesting configuration\n",
__func__);
return -EINVAL;
}
/*
* As for ERET, we can avoid doing too much on the injection path by
* checking that we either took the exception from a VHE host
* userspace or from vEL2. In these cases, there is no change in
* translation regime (or anything else), so let's do as little as
* possible.
*/
pstate = *vcpu_cpsr(vcpu);
mode = pstate & (PSR_MODE_MASK | PSR_MODE32_BIT);
direct_inject = (mode == PSR_MODE_EL0t &&
vcpu_el2_e2h_is_set(vcpu) &&
vcpu_el2_tge_is_set(vcpu));
direct_inject |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
if (direct_inject) {
kvm_inject_el2_exception(vcpu, esr_el2, type);
return 1;
}
preempt_disable();
/*
* We may have an exception or PC update in the EL0/EL1 context.
* Commit it before entering EL2.
*/
__kvm_adjust_pc(vcpu);
kvm_arch_vcpu_put(vcpu);
kvm_inject_el2_exception(vcpu, esr_el2, type);
/*
* A hard requirement is that a switch between EL1 and EL2
* contexts has to happen between a put/load, so that we can
* pick the correct timer and interrupt configuration, among
* other things.
*
* Make sure the exception actually took place before we load
* the new context.
*/
__kvm_adjust_pc(vcpu);
kvm_arch_vcpu_load(vcpu, smp_processor_id());
preempt_enable();
return 1;
}
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2)
{
return kvm_inject_nested(vcpu, esr_el2, except_type_sync);
}
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu)
{
/*
* Do not inject an irq if the:
* - Current exception level is EL2, and
* - virtual HCR_EL2.TGE == 0
* - virtual HCR_EL2.IMO == 0
*
* See Table D1-17 "Physical interrupt target and masking when EL3 is
* not implemented and EL2 is implemented" in ARM DDI 0487C.a.
*/
if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) &&
!(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO))
return 1;
/* esr_el2 value doesn't matter for exits due to irqs. */
return kvm_inject_nested(vcpu, 0, except_type_irq);
}
Reference in New Issue View Git Blame Copy Permalink