Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2025-01-07 13:43:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

- A serious scrubbing of the MTRR code including adding a new map

Browse Source 
mechanism in order to look up the memory type of a region easily. Also
   address memory range lookup issues like returning an invalid memory
   type. Furthermore, this handles the decoupling of PAT from MTRR more
   naturally. All work by Juergen Gross
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEzv7L6UO9uDPlPSfHEsHwGGHeVUoFAmSazOIACgkQEsHwGGHe
 VUqltQ/8D1oA4LrgnbFO25J/27U/MwKo7ZI3hN6/OkH2FfdBgqeOlOV4TnndDL88
 l/UrzOfWJQxpVLTO3SLMtDla0VrT24B4HZ4hvzDEdJZ8f1DLZ+gLN7sKOMjoIcO9
 fvBZ5+/gFtVxSquwZmWvM0qKiCkKxmznJfpOx1/lt9UtKyKmpPSMVdrpqOeufL7k
 xxWqGRh2s104ZKwfMOj4dgvCVK9ZUlsPqqiARzkqc0bCg7SeIyPea/S2eljhTl15
 BTOA/wW/lcVQ9yWmDD8inzxrZI4EHEohEaNMfof3AqFyYCOU4RzvE9tpAFEK3GXp
 NilxYkZ+JbEljq2QiEt0Ll8XEVKedi7YC1oN3ciiy9RS6+rWSPIvuMFV9tgPRjr1
 AbWYmDoiLz+5ePI+0fckStRRntWKiao+hOaXb5RbEcg+85hkDHZZC7b0tCAUvnh7
 OwuQfbzAqipn2G1hg+LThHDSjI4qHfHJlpeuPcsAxWef1diJbe15StdVWm+ttRE0
 MTXSn3J9qT9MoY5y6m4KSybp0c1nSFlCK/ZkNvzwWHmkAG6M7wuFmBn3pVzEaCew
 fneGZcX9Ija4MY8Ygajp8GI1aQ4mBNif+uVE7UUY17hH9qAf8vI8Joqs+4L35u8h
 SZl/IqJO9ziEmVLdy9ajgm1xW04AFE1RYRfa6aH6K6tRaIoh8bE=
 =Dmx5
 -----END PGP SIGNATURE-----

Merge tag 'x86_mtrr_for_v6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 mtrr updates from Borislav Petkov:
 "A serious scrubbing of the MTRR code including adding a new map
  mechanism in order to look up the memory type of a region easily.

  Also address memory range lookup issues like returning an invalid
  memory type. Furthermore, this handles the decoupling of PAT from MTRR
  more naturally.

  All work by Juergen Gross"

* tag 'x86_mtrr_for_v6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/xen: Set default memory type for PV guests to WB
  x86/mtrr: Unify debugging printing
  x86/mtrr: Remove unused code
  x86/mm: Only check uniform after calling mtrr_type_lookup()
  x86/mtrr: Don't let mtrr_type_lookup() return MTRR_TYPE_INVALID
  x86/mtrr: Use new cache_map in mtrr_type_lookup()
  x86/mtrr: Add mtrr=debug command line option
  x86/mtrr: Construct a memory map with cache modes
  x86/mtrr: Add get_effective_type() service function
  x86/mtrr: Allocate mtrr_value array dynamically
  x86/mtrr: Move 32-bit code from mtrr.c to legacy.c
  x86/mtrr: Have only one set_mtrr() variant
  x86/mtrr: Replace vendor tests in MTRR code
  x86/xen: Set MTRR state when running as Xen PV initial domain
  x86/hyperv: Set MTRR state when running as SEV-SNP Hyper-V guest
  x86/mtrr: Support setting MTRR state for software defined MTRRs
  x86/mtrr: Replace size_or_mask and size_and_mask with a much easier concept
  x86/mtrr: Remove physical address size calculation
This commit is contained in:
Linus Torvalds 2023-06-27 13:11:32 -07:00
commit dc43fc753b
16 changed files with 785 additions and 490 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Documentation/admin-guide/kernel-parameters.txt
View File

@ -3433,6 +3433,10 @@
[HW] Make the MicroTouch USB driver use raw coordinates
('y', default) or cooked coordinates ('n')
mtrr=debug [X86]
Enable printing debug information related to MTRR
registers at boot time.
mtrr_chunk_size=nn[KMG] [X86]
used for mtrr cleanup. It is largest continuous chunk
that could hold holes aka. UC entries.

4
arch/x86/hyperv/ivm.c
View File

@ -17,6 +17,7 @@
#include <asm/mem_encrypt.h>
#include <asm/mshyperv.h>
#include <asm/hypervisor.h>
#include <asm/mtrr.h>
#ifdef CONFIG_AMD_MEM_ENCRYPT
@ -372,6 +373,9 @@ void __init hv_vtom_init(void)
x86_platform.guest.enc_cache_flush_required = hv_vtom_cache_flush_required;
x86_platform.guest.enc_tlb_flush_required = hv_vtom_tlb_flush_required;
x86_platform.guest.enc_status_change_finish = hv_vtom_set_host_visibility;
/* Set WB as the default cache mode. */
mtrr_overwrite_state(NULL, 0, MTRR_TYPE_WRBACK);
}
#endif /* CONFIG_AMD_MEM_ENCRYPT */

51
arch/x86/include/asm/mtrr.h
View File

@ -23,14 +23,43 @@
#ifndef _ASM_X86_MTRR_H
#define _ASM_X86_MTRR_H
#include <linux/bits.h>
#include <uapi/asm/mtrr.h>
/* Defines for hardware MTRR registers. */
#define MTRR_CAP_VCNT GENMASK(7, 0)
#define MTRR_CAP_FIX BIT_MASK(8)
#define MTRR_CAP_WC BIT_MASK(10)
#define MTRR_DEF_TYPE_TYPE GENMASK(7, 0)
#define MTRR_DEF_TYPE_FE BIT_MASK(10)
#define MTRR_DEF_TYPE_E BIT_MASK(11)
#define MTRR_DEF_TYPE_ENABLE (MTRR_DEF_TYPE_FE | MTRR_DEF_TYPE_E)
#define MTRR_DEF_TYPE_DISABLE ~(MTRR_DEF_TYPE_TYPE | MTRR_DEF_TYPE_ENABLE)
#define MTRR_PHYSBASE_TYPE GENMASK(7, 0)
#define MTRR_PHYSBASE_RSVD GENMASK(11, 8)
#define MTRR_PHYSMASK_RSVD GENMASK(10, 0)
#define MTRR_PHYSMASK_V BIT_MASK(11)
struct mtrr_state_type {
struct mtrr_var_range var_ranges[MTRR_MAX_VAR_RANGES];
mtrr_type fixed_ranges[MTRR_NUM_FIXED_RANGES];
unsigned char enabled;
bool have_fixed;
mtrr_type def_type;
};
/*
* The following functions are for use by other drivers that cannot use
* arch_phys_wc_add and arch_phys_wc_del.
*/
# ifdef CONFIG_MTRR
void mtrr_bp_init(void);
void mtrr_overwrite_state(struct mtrr_var_range *var, unsigned int num_var,
mtrr_type def_type);
extern u8 mtrr_type_lookup(u64 addr, u64 end, u8 *uniform);
extern void mtrr_save_fixed_ranges(void *);
extern void mtrr_save_state(void);
@ -40,7 +69,6 @@ extern int mtrr_add_page(unsigned long base, unsigned long size,
unsigned int type, bool increment);
extern int mtrr_del(int reg, unsigned long base, unsigned long size);
extern int mtrr_del_page(int reg, unsigned long base, unsigned long size);
extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi);
extern void mtrr_bp_restore(void);
extern int mtrr_trim_uncached_memory(unsigned long end_pfn);
extern int amd_special_default_mtrr(void);
@ -48,12 +76,21 @@ void mtrr_disable(void);
void mtrr_enable(void);
void mtrr_generic_set_state(void);
# else
static inline void mtrr_overwrite_state(struct mtrr_var_range *var,
unsigned int num_var,
mtrr_type def_type)
{
}
static inline u8 mtrr_type_lookup(u64 addr, u64 end, u8 *uniform)
{
/*
* Return no-MTRRs:
* Return the default MTRR type, without any known other types in
* that range.
*/
return MTRR_TYPE_INVALID;
*uniform = 1;
return MTRR_TYPE_UNCACHABLE;
}
#define mtrr_save_fixed_ranges(arg) do {} while (0)
#define mtrr_save_state() do {} while (0)
@ -79,9 +116,6 @@ static inline int mtrr_trim_uncached_memory(unsigned long end_pfn)
{
return 0;
}
static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
{
}
#define mtrr_bp_init() do {} while (0)
#define mtrr_bp_restore() do {} while (0)
#define mtrr_disable() do {} while (0)
@ -121,7 +155,8 @@ struct mtrr_gentry32 {
#endif /* CONFIG_COMPAT */
/* Bit fields for enabled in struct mtrr_state_type */
#define MTRR_STATE_MTRR_FIXED_ENABLED 0x01
#define MTRR_STATE_MTRR_ENABLED 0x02
#define MTRR_STATE_SHIFT 10
#define MTRR_STATE_MTRR_FIXED_ENABLED (MTRR_DEF_TYPE_FE >> MTRR_STATE_SHIFT)
#define MTRR_STATE_MTRR_ENABLED (MTRR_DEF_TYPE_E >> MTRR_STATE_SHIFT)
#endif /* _ASM_X86_MTRR_H */

14
arch/x86/include/uapi/asm/mtrr.h
View File

@ -81,14 +81,6 @@ typedef __u8 mtrr_type;
#define MTRR_NUM_FIXED_RANGES 88
#define MTRR_MAX_VAR_RANGES 256
struct mtrr_state_type {
struct mtrr_var_range var_ranges[MTRR_MAX_VAR_RANGES];
mtrr_type fixed_ranges[MTRR_NUM_FIXED_RANGES];
unsigned char enabled;
unsigned char have_fixed;
mtrr_type def_type;
};
#define MTRRphysBase_MSR(reg) (0x200 + 2 * (reg))
#define MTRRphysMask_MSR(reg) (0x200 + 2 * (reg) + 1)
@ -115,9 +107,9 @@ struct mtrr_state_type {
#define MTRR_NUM_TYPES 7
/*
* Invalid MTRR memory type. mtrr_type_lookup() returns this value when
* MTRRs are disabled. Note, this value is allocated from the reserved
* values (0x7-0xff) of the MTRR memory types.
* Invalid MTRR memory type. No longer used outside of MTRR code.
* Note, this value is allocated from the reserved values (0x7-0xff) of
* the MTRR memory types.
*/
#define MTRR_TYPE_INVALID 0xff

2
arch/x86/kernel/cpu/mtrr/Makefile
View File

@ -1,4 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-y := mtrr.o if.o generic.o cleanup.o
obj-$(CONFIG_X86_32) += amd.o cyrix.o centaur.o
obj-$(CONFIG_X86_32) += amd.o cyrix.o centaur.o legacy.o

2
arch/x86/kernel/cpu/mtrr/amd.c
View File

@ -110,7 +110,7 @@ amd_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
}
const struct mtrr_ops amd_mtrr_ops = {
.vendor = X86_VENDOR_AMD,
.var_regs = 2,
.set = amd_set_mtrr,
.get = amd_get_mtrr,
.get_free_region = generic_get_free_region,

11
arch/x86/kernel/cpu/mtrr/centaur.c
View File

@ -45,15 +45,6 @@ centaur_get_free_region(unsigned long base, unsigned long size, int replace_reg)
return -ENOSPC;
}
/*
* Report boot time MCR setups
*/
void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
{
centaur_mcr[mcr].low = lo;
centaur_mcr[mcr].high = hi;
}
static void
centaur_get_mcr(unsigned int reg, unsigned long *base,
unsigned long *size, mtrr_type * type)
@ -112,7 +103,7 @@ centaur_validate_add_page(unsigned long base, unsigned long size, unsigned int t
}
const struct mtrr_ops centaur_mtrr_ops = {
.vendor = X86_VENDOR_CENTAUR,
.var_regs = 8,
.set = centaur_set_mcr,
.get = centaur_get_mcr,
.get_free_region = centaur_get_free_region,

87
arch/x86/kernel/cpu/mtrr/cleanup.c
View File

@ -55,9 +55,6 @@ static int __initdata nr_range;
static struct var_mtrr_range_state __initdata range_state[RANGE_NUM];
static int __initdata debug_print;
#define Dprintk(x...) do { if (debug_print) pr_debug(x); } while (0)
#define BIOS_BUG_MSG \
"WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n"
@ -79,12 +76,11 @@ x86_get_mtrr_mem_range(struct range *range, int nr_range,
nr_range = add_range_with_merge(range, RANGE_NUM, nr_range,
base, base + size);
}
if (debug_print) {
pr_debug("After WB checking\n");
for (i = 0; i < nr_range; i++)
pr_debug("MTRR MAP PFN: %016llx - %016llx\n",
range[i].start, range[i].end);
}
Dprintk("After WB checking\n");
for (i = 0; i < nr_range; i++)
Dprintk("MTRR MAP PFN: %016llx - %016llx\n",
range[i].start, range[i].end);
/* Take out UC ranges: */
for (i = 0; i < num_var_ranges; i++) {
@ -112,24 +108,22 @@ x86_get_mtrr_mem_range(struct range *range, int nr_range,
subtract_range(range, RANGE_NUM, extra_remove_base,
extra_remove_base + extra_remove_size);
if (debug_print) {
pr_debug("After UC checking\n");
for (i = 0; i < RANGE_NUM; i++) {
if (!range[i].end)
continue;
pr_debug("MTRR MAP PFN: %016llx - %016llx\n",
range[i].start, range[i].end);
}
Dprintk("After UC checking\n");
for (i = 0; i < RANGE_NUM; i++) {
if (!range[i].end)
continue;
Dprintk("MTRR MAP PFN: %016llx - %016llx\n",
range[i].start, range[i].end);
}
/* sort the ranges */
nr_range = clean_sort_range(range, RANGE_NUM);
if (debug_print) {
pr_debug("After sorting\n");
for (i = 0; i < nr_range; i++)
pr_debug("MTRR MAP PFN: %016llx - %016llx\n",
range[i].start, range[i].end);
}
Dprintk("After sorting\n");
for (i = 0; i < nr_range; i++)
Dprintk("MTRR MAP PFN: %016llx - %016llx\n",
range[i].start, range[i].end);
return nr_range;
}
@ -164,16 +158,9 @@ static int __init enable_mtrr_cleanup_setup(char *str)
}
early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup);
static int __init mtrr_cleanup_debug_setup(char *str)
{
debug_print = 1;
return 0;
}
early_param("mtrr_cleanup_debug", mtrr_cleanup_debug_setup);
static void __init
set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
unsigned char type, unsigned int address_bits)
unsigned char type)
{
u32 base_lo, base_hi, mask_lo, mask_hi;
u64 base, mask;
@ -183,7 +170,7 @@ set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
return;
}
mask = (1ULL << address_bits) - 1;
mask = (1ULL << boot_cpu_data.x86_phys_bits) - 1;
mask &= ~((((u64)sizek) << 10) - 1);
base = ((u64)basek) << 10;
@ -209,7 +196,7 @@ save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
range_state[reg].type = type;
}
static void __init set_var_mtrr_all(unsigned int address_bits)
static void __init set_var_mtrr_all(void)
{
unsigned long basek, sizek;
unsigned char type;
@ -220,7 +207,7 @@ static void __init set_var_mtrr_all(unsigned int address_bits)
sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10);
type = range_state[reg].type;
set_var_mtrr(reg, basek, sizek, type, address_bits);
set_var_mtrr(reg, basek, sizek, type);
}
}
@ -267,7 +254,7 @@ range_to_mtrr(unsigned int reg, unsigned long range_startk,
align = max_align;
sizek = 1UL << align;
if (debug_print) {
if (mtrr_debug) {
char start_factor = 'K', size_factor = 'K';
unsigned long start_base, size_base;
@ -542,7 +529,7 @@ static void __init print_out_mtrr_range_state(void)
start_base = to_size_factor(start_base, &start_factor);
type = range_state[i].type;
pr_debug("reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
Dprintk("reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
i, start_base, start_factor,
size_base, size_factor,
(type == MTRR_TYPE_UNCACHABLE) ? "UC" :
@ -680,7 +667,7 @@ static int __init mtrr_search_optimal_index(void)
return index_good;
}
int __init mtrr_cleanup(unsigned address_bits)
int __init mtrr_cleanup(void)
{
unsigned long x_remove_base, x_remove_size;
unsigned long base, size, def, dummy;
@ -689,7 +676,10 @@ int __init mtrr_cleanup(unsigned address_bits)
int index_good;
int i;
if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
if (!mtrr_enabled())
return 0;
if (!cpu_feature_enabled(X86_FEATURE_MTRR) || enable_mtrr_cleanup < 1)
return 0;
rdmsr(MSR_MTRRdefType, def, dummy);
@ -711,7 +701,7 @@ int __init mtrr_cleanup(unsigned address_bits)
return 0;
/* Print original var MTRRs at first, for debugging: */
pr_debug("original variable MTRRs\n");
Dprintk("original variable MTRRs\n");
print_out_mtrr_range_state();
memset(range, 0, sizeof(range));
@ -742,8 +732,8 @@ int __init mtrr_cleanup(unsigned address_bits)
mtrr_print_out_one_result(i);
if (!result[i].bad) {
set_var_mtrr_all(address_bits);
pr_debug("New variable MTRRs\n");
set_var_mtrr_all();
Dprintk("New variable MTRRs\n");
print_out_mtrr_range_state();
return 1;
}
@ -763,7 +753,7 @@ int __init mtrr_cleanup(unsigned address_bits)
mtrr_calc_range_state(chunk_size, gran_size,
x_remove_base, x_remove_size, i);
if (debug_print) {
if (mtrr_debug) {
mtrr_print_out_one_result(i);
pr_info("\n");
}
@ -786,8 +776,8 @@ int __init mtrr_cleanup(unsigned address_bits)
gran_size = result[i].gran_sizek;
gran_size <<= 10;
x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
set_var_mtrr_all(address_bits);
pr_debug("New variable MTRRs\n");
set_var_mtrr_all();
Dprintk("New variable MTRRs\n");
print_out_mtrr_range_state();
return 1;
} else {
@ -802,7 +792,7 @@ int __init mtrr_cleanup(unsigned address_bits)
return 0;
}
#else
int __init mtrr_cleanup(unsigned address_bits)
int __init mtrr_cleanup(void)
{
return 0;
}
@ -882,15 +872,18 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
/* extra one for all 0 */
int num[MTRR_NUM_TYPES + 1];
if (!mtrr_enabled())
return 0;
/*
* Make sure we only trim uncachable memory on machines that
* support the Intel MTRR architecture:
*/
if (!is_cpu(INTEL) || disable_mtrr_trim)
if (!cpu_feature_enabled(X86_FEATURE_MTRR) || disable_mtrr_trim)
return 0;
rdmsr(MSR_MTRRdefType, def, dummy);
def &= 0xff;
def &= MTRR_DEF_TYPE_TYPE;
if (def != MTRR_TYPE_UNCACHABLE)
return 0;

2
arch/x86/kernel/cpu/mtrr/cyrix.c
View File

@ -235,7 +235,7 @@ static void cyrix_set_arr(unsigned int reg, unsigned long base,
}
const struct mtrr_ops cyrix_mtrr_ops = {
.vendor = X86_VENDOR_CYRIX,
.var_regs = 8,
.set = cyrix_set_arr,
.get = cyrix_get_arr,
.get_free_region = cyrix_get_free_region,

693
arch/x86/kernel/cpu/mtrr/generic.c
View File

@ -8,10 +8,12 @@
#include <linux/init.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/cc_platform.h>
#include <asm/processor-flags.h>
#include <asm/cacheinfo.h>
#include <asm/cpufeature.h>
#include <asm/hypervisor.h>
#include <asm/mshyperv.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
#include <asm/msr.h>
@ -31,6 +33,55 @@ static struct fixed_range_block fixed_range_blocks[] = {
{}
};
struct cache_map {
u64 start;
u64 end;
u64 flags;
u64 type:8;
u64 fixed:1;
};
bool mtrr_debug;
static int __init mtrr_param_setup(char *str)
{
int rc = 0;
if (!str)
return -EINVAL;
if (!strcmp(str, "debug"))
mtrr_debug = true;
else
rc = -EINVAL;
return rc;
}
early_param("mtrr", mtrr_param_setup);
/*
* CACHE_MAP_MAX is the maximum number of memory ranges in cache_map, where
* no 2 adjacent ranges have the same cache mode (those would be merged).
* The number is based on the worst case:
* - no two adjacent fixed MTRRs share the same cache mode
* - one variable MTRR is spanning a huge area with mode WB
* - 255 variable MTRRs with mode UC all overlap with the WB MTRR, creating 2
* additional ranges each (result like "ababababa...aba" with a = WB, b = UC),
* accounting for MTRR_MAX_VAR_RANGES * 2 - 1 range entries
* - a TOP_MEM2 area (even with overlapping an UC MTRR can't add 2 range entries
* to the possible maximum, as it always starts at 4GB, thus it can't be in
* the middle of that MTRR, unless that MTRR starts at 0, which would remove
* the initial "a" from the "abababa" pattern above)
* The map won't contain ranges with no matching MTRR (those fall back to the
* default cache mode).
*/
#define CACHE_MAP_MAX (MTRR_NUM_FIXED_RANGES + MTRR_MAX_VAR_RANGES * 2)
static struct cache_map init_cache_map[CACHE_MAP_MAX] __initdata;
static struct cache_map *cache_map __refdata = init_cache_map;
static unsigned int cache_map_size = CACHE_MAP_MAX;
static unsigned int cache_map_n;
static unsigned int cache_map_fixed;
static unsigned long smp_changes_mask;
static int mtrr_state_set;
u64 mtrr_tom2;
@ -38,6 +89,9 @@ u64 mtrr_tom2;
struct mtrr_state_type mtrr_state;
EXPORT_SYMBOL_GPL(mtrr_state);
/* Reserved bits in the high portion of the MTRRphysBaseN MSR. */
u32 phys_hi_rsvd;
/*
* BIOS is expected to clear MtrrFixDramModEn bit, see for example
* "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
@ -69,175 +123,370 @@ static u64 get_mtrr_size(u64 mask)
{
u64 size;
mask >>= PAGE_SHIFT;
mask |= size_or_mask;
mask |= (u64)phys_hi_rsvd << 32;
size = -mask;
size <<= PAGE_SHIFT;
return size;
}
/*
* Check and return the effective type for MTRR-MTRR type overlap.
* Returns 1 if the effective type is UNCACHEABLE, else returns 0
*/
static int check_type_overlap(u8 *prev, u8 *curr)
static u8 get_var_mtrr_state(unsigned int reg, u64 *start, u64 *size)
{
if (*prev == MTRR_TYPE_UNCACHABLE || *curr == MTRR_TYPE_UNCACHABLE) {
*prev = MTRR_TYPE_UNCACHABLE;
*curr = MTRR_TYPE_UNCACHABLE;
struct mtrr_var_range *mtrr = mtrr_state.var_ranges + reg;
if (!(mtrr->mask_lo & MTRR_PHYSMASK_V))
return MTRR_TYPE_INVALID;
*start = (((u64)mtrr->base_hi) << 32) + (mtrr->base_lo & PAGE_MASK);
*size = get_mtrr_size((((u64)mtrr->mask_hi) << 32) +
(mtrr->mask_lo & PAGE_MASK));
return mtrr->base_lo & MTRR_PHYSBASE_TYPE;
}
static u8 get_effective_type(u8 type1, u8 type2)
{
if (type1 == MTRR_TYPE_UNCACHABLE || type2 == MTRR_TYPE_UNCACHABLE)
return MTRR_TYPE_UNCACHABLE;
if ((type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH) ||
(type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK))
return MTRR_TYPE_WRTHROUGH;
if (type1 != type2)
return MTRR_TYPE_UNCACHABLE;
return type1;
}
static void rm_map_entry_at(int idx)
{
cache_map_n--;
if (cache_map_n > idx) {
memmove(cache_map + idx, cache_map + idx + 1,
sizeof(*cache_map) * (cache_map_n - idx));
}
}
/*
* Add an entry into cache_map at a specific index. Merges adjacent entries if
* appropriate. Return the number of merges for correcting the scan index
* (this is needed as merging will reduce the number of entries, which will
* result in skipping entries in future iterations if the scan index isn't
* corrected).
* Note that the corrected index can never go below -1 (resulting in being 0 in
* the next scan iteration), as "2" is returned only if the current index is
* larger than zero.
*/
static int add_map_entry_at(u64 start, u64 end, u8 type, int idx)
{
bool merge_prev = false, merge_next = false;
if (start >= end)
return 0;
if (idx > 0) {
struct cache_map *prev = cache_map + idx - 1;
if (!prev->fixed && start == prev->end && type == prev->type)
merge_prev = true;
}
if (idx < cache_map_n) {
struct cache_map *next = cache_map + idx;
if (!next->fixed && end == next->start && type == next->type)
merge_next = true;
}
if (merge_prev && merge_next) {
cache_map[idx - 1].end = cache_map[idx].end;
rm_map_entry_at(idx);
return 2;
}
if (merge_prev) {
cache_map[idx - 1].end = end;
return 1;
}
if (merge_next) {
cache_map[idx].start = start;
return 1;
}
if ((*prev == MTRR_TYPE_WRBACK && *curr == MTRR_TYPE_WRTHROUGH) ||
(*prev == MTRR_TYPE_WRTHROUGH && *curr == MTRR_TYPE_WRBACK)) {
*prev = MTRR_TYPE_WRTHROUGH;
*curr = MTRR_TYPE_WRTHROUGH;
/* Sanity check: the array should NEVER be too small! */
if (cache_map_n == cache_map_size) {
WARN(1, "MTRR cache mode memory map exhausted!\n");
cache_map_n = cache_map_fixed;
return 0;
}
if (*prev != *curr) {
*prev = MTRR_TYPE_UNCACHABLE;
*curr = MTRR_TYPE_UNCACHABLE;
return 1;
if (cache_map_n > idx) {
memmove(cache_map + idx + 1, cache_map + idx,
sizeof(*cache_map) * (cache_map_n - idx));
}
cache_map[idx].start = start;
cache_map[idx].end = end;
cache_map[idx].type = type;
cache_map[idx].fixed = 0;
cache_map_n++;
return 0;
}
/**
* mtrr_type_lookup_fixed - look up memory type in MTRR fixed entries
*
* Return the MTRR fixed memory type of 'start'.
*
* MTRR fixed entries are divided into the following ways:
* 0x00000 - 0x7FFFF : This range is divided into eight 64KB sub-ranges
* 0x80000 - 0xBFFFF : This range is divided into sixteen 16KB sub-ranges
* 0xC0000 - 0xFFFFF : This range is divided into sixty-four 4KB sub-ranges
*
* Return Values:
* MTRR_TYPE_(type) - Matched memory type
* MTRR_TYPE_INVALID - Unmatched
*/
static u8 mtrr_type_lookup_fixed(u64 start, u64 end)
/* Clear a part of an entry. Return 1 if start of entry is still valid. */
static int clr_map_range_at(u64 start, u64 end, int idx)
{
int idx;
int ret = start != cache_map[idx].start;
u64 tmp;
if (start >= 0x100000)
return MTRR_TYPE_INVALID;
/* 0x0 - 0x7FFFF */
if (start < 0x80000) {
idx = 0;
idx += (start >> 16);
return mtrr_state.fixed_ranges[idx];
/* 0x80000 - 0xBFFFF */
} else if (start < 0xC0000) {
idx = 1 * 8;
idx += ((start - 0x80000) >> 14);
return mtrr_state.fixed_ranges[idx];
if (start == cache_map[idx].start && end == cache_map[idx].end) {
rm_map_entry_at(idx);
} else if (start == cache_map[idx].start) {
cache_map[idx].start = end;
} else if (end == cache_map[idx].end) {
cache_map[idx].end = start;
} else {
tmp = cache_map[idx].end;
cache_map[idx].end = start;
add_map_entry_at(end, tmp, cache_map[idx].type, idx + 1);
}
/* 0xC0000 - 0xFFFFF */
idx = 3 * 8;
idx += ((start - 0xC0000) >> 12);
return mtrr_state.fixed_ranges[idx];
return ret;
}
/*
* Add MTRR to the map. The current map is scanned and each part of the MTRR
* either overlapping with an existing entry or with a hole in the map is
* handled separately.
*/
static void add_map_entry(u64 start, u64 end, u8 type)
{
u8 new_type, old_type;
u64 tmp;
int i;
for (i = 0; i < cache_map_n && start < end; i++) {
if (start >= cache_map[i].end)
continue;
if (start < cache_map[i].start) {
/* Region start has no overlap. */
tmp = min(end, cache_map[i].start);
i -= add_map_entry_at(start, tmp, type, i);
start = tmp;
continue;
}
new_type = get_effective_type(type, cache_map[i].type);
old_type = cache_map[i].type;
if (cache_map[i].fixed || new_type == old_type) {
/* Cut off start of new entry. */
start = cache_map[i].end;
continue;
}
/* Handle only overlapping part of region. */
tmp = min(end, cache_map[i].end);
i += clr_map_range_at(start, tmp, i);
i -= add_map_entry_at(start, tmp, new_type, i);
start = tmp;
}
/* Add rest of region after last map entry (rest might be empty). */
add_map_entry_at(start, end, type, i);
}
/* Add variable MTRRs to cache map. */
static void map_add_var(void)
{
u64 start, size;
unsigned int i;
u8 type;
/*
* Add AMD TOP_MEM2 area. Can't be added in mtrr_build_map(), as it
* needs to be added again when rebuilding the map due to potentially
* having moved as a result of variable MTRRs for memory below 4GB.
*/
if (mtrr_tom2) {
add_map_entry(BIT_ULL(32), mtrr_tom2, MTRR_TYPE_WRBACK);
cache_map[cache_map_n - 1].fixed = 1;
}
for (i = 0; i < num_var_ranges; i++) {
type = get_var_mtrr_state(i, &start, &size);
if (type != MTRR_TYPE_INVALID)
add_map_entry(start, start + size, type);
}
}
/*
* Rebuild map by replacing variable entries. Needs to be called when MTRR
* registers are being changed after boot, as such changes could include
* removals of registers, which are complicated to handle without rebuild of
* the map.
*/
void generic_rebuild_map(void)
{
if (mtrr_if != &generic_mtrr_ops)
return;
cache_map_n = cache_map_fixed;
map_add_var();
}
static unsigned int __init get_cache_map_size(void)
{
return cache_map_fixed + 2 * num_var_ranges + (mtrr_tom2 != 0);
}
/* Build the cache_map containing the cache modes per memory range. */
void __init mtrr_build_map(void)
{
u64 start, end, size;
unsigned int i;
u8 type;
/* Add fixed MTRRs, optimize for adjacent entries with same type. */
if (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED) {
/*
* Start with 64k size fixed entries, preset 1st one (hence the
* loop below is starting with index 1).
*/
start = 0;
end = size = 0x10000;
type = mtrr_state.fixed_ranges[0];
for (i = 1; i < MTRR_NUM_FIXED_RANGES; i++) {
/* 8 64k entries, then 16 16k ones, rest 4k. */
if (i == 8 || i == 24)
size >>= 2;
if (mtrr_state.fixed_ranges[i] != type) {
add_map_entry(start, end, type);
start = end;
type = mtrr_state.fixed_ranges[i];
}
end += size;
}
add_map_entry(start, end, type);
}
/* Mark fixed, they take precedence. */
for (i = 0; i < cache_map_n; i++)
cache_map[i].fixed = 1;
cache_map_fixed = cache_map_n;
map_add_var();
pr_info("MTRR map: %u entries (%u fixed + %u variable; max %u), built from %u variable MTRRs\n",
cache_map_n, cache_map_fixed, cache_map_n - cache_map_fixed,
get_cache_map_size(), num_var_ranges + (mtrr_tom2 != 0));
if (mtrr_debug) {
for (i = 0; i < cache_map_n; i++) {
pr_info("%3u: %016llx-%016llx %s\n", i,
cache_map[i].start, cache_map[i].end - 1,
mtrr_attrib_to_str(cache_map[i].type));
}
}
}
/* Copy the cache_map from __initdata memory to dynamically allocated one. */
void __init mtrr_copy_map(void)
{
unsigned int new_size = get_cache_map_size();
if (!mtrr_state.enabled || !new_size) {
cache_map = NULL;
return;
}
mutex_lock(&mtrr_mutex);
cache_map = kcalloc(new_size, sizeof(*cache_map), GFP_KERNEL);
if (cache_map) {
memmove(cache_map, init_cache_map,
cache_map_n * sizeof(*cache_map));
cache_map_size = new_size;
} else {
mtrr_state.enabled = 0;
pr_err("MTRRs disabled due to allocation failure for lookup map.\n");
}
mutex_unlock(&mtrr_mutex);
}
/**
* mtrr_type_lookup_variable - look up memory type in MTRR variable entries
* mtrr_overwrite_state - set static MTRR state
*
* Return Value:
* MTRR_TYPE_(type) - Matched memory type or default memory type (unmatched)
*
* Output Arguments:
* repeat - Set to 1 when [start:end] spanned across MTRR range and type
* returned corresponds only to [start:*partial_end]. Caller has
* to lookup again for [*partial_end:end].
*
* uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the
* region is fully covered by a single MTRR entry or the default
* type.
* Used to set MTRR state via different means (e.g. with data obtained from
* a hypervisor).
* Is allowed only for special cases when running virtualized. Must be called
* from the x86_init.hyper.init_platform() hook. It can be called only once.
* The MTRR state can't be changed afterwards. To ensure that, X86_FEATURE_MTRR
* is cleared.
*/
static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
int *repeat, u8 *uniform)
void mtrr_overwrite_state(struct mtrr_var_range *var, unsigned int num_var,
mtrr_type def_type)
{
int i;
u64 base, mask;
u8 prev_match, curr_match;
unsigned int i;
*repeat = 0;
*uniform = 1;
/* Only allowed to be called once before mtrr_bp_init(). */
if (WARN_ON_ONCE(mtrr_state_set))
return;
prev_match = MTRR_TYPE_INVALID;
for (i = 0; i < num_var_ranges; ++i) {
unsigned short start_state, end_state, inclusive;
/* Only allowed when running virtualized. */
if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
return;
if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
continue;
/*
* Only allowed for special virtualization cases:
* - when running as Hyper-V, SEV-SNP guest using vTOM
* - when running as Xen PV guest
* - when running as SEV-SNP or TDX guest to avoid unnecessary
* VMM communication/Virtualization exceptions (#VC, #VE)
*/
if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP) &&
!hv_is_isolation_supported() &&
!cpu_feature_enabled(X86_FEATURE_XENPV) &&
!cpu_feature_enabled(X86_FEATURE_TDX_GUEST))
return;
base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
(mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
(mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);
/* Disable MTRR in order to disable MTRR modifications. */
setup_clear_cpu_cap(X86_FEATURE_MTRR);
start_state = ((start & mask) == (base & mask));
end_state = ((end & mask) == (base & mask));
inclusive = ((start < base) && (end > base));
if ((start_state != end_state) || inclusive) {
/*
* We have start:end spanning across an MTRR.
* We split the region into either
*
* - start_state:1
* (start:mtrr_end)(mtrr_end:end)
* - end_state:1
* (start:mtrr_start)(mtrr_start:end)
* - inclusive:1
* (start:mtrr_start)(mtrr_start:mtrr_end)(mtrr_end:end)
*
* depending on kind of overlap.
*
* Return the type of the first region and a pointer
* to the start of next region so that caller will be
* advised to lookup again after having adjusted start
* and end.
*
* Note: This way we handle overlaps with multiple
* entries and the default type properly.
*/
if (start_state)
*partial_end = base + get_mtrr_size(mask);
else
*partial_end = base;
if (unlikely(*partial_end <= start)) {
WARN_ON(1);
*partial_end = start + PAGE_SIZE;
}
end = *partial_end - 1; /* end is inclusive */
*repeat = 1;
*uniform = 0;
if (var) {
if (num_var > MTRR_MAX_VAR_RANGES) {
pr_warn("Trying to overwrite MTRR state with %u variable entries\n",
num_var);
num_var = MTRR_MAX_VAR_RANGES;
}
if ((start & mask) != (base & mask))
continue;
curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
if (prev_match == MTRR_TYPE_INVALID) {
prev_match = curr_match;
continue;
}
*uniform = 0;
if (check_type_overlap(&prev_match, &curr_match))
return curr_match;
for (i = 0; i < num_var; i++)
mtrr_state.var_ranges[i] = var[i];
num_var_ranges = num_var;
}
if (prev_match != MTRR_TYPE_INVALID)
return prev_match;
mtrr_state.def_type = def_type;
mtrr_state.enabled |= MTRR_STATE_MTRR_ENABLED;
return mtrr_state.def_type;
mtrr_state_set = 1;
}
static u8 type_merge(u8 type, u8 new_type, u8 *uniform)
{
u8 effective_type;
if (type == MTRR_TYPE_INVALID)
return new_type;
effective_type = get_effective_type(type, new_type);
if (type != effective_type)
*uniform = 0;
return effective_type;
}
/**
@ -248,66 +497,49 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
* MTRR_TYPE_INVALID - MTRR is disabled
*
* Output Argument:
* uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the
* region is fully covered by a single MTRR entry or the default
* type.
* uniform - Set to 1 when the returned MTRR type is valid for the whole
* region, set to 0 else.
*/
u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform)
{
u8 type, prev_type, is_uniform = 1, dummy;
int repeat;
u64 partial_end;
u8 type = MTRR_TYPE_INVALID;
unsigned int i;
/* Make end inclusive instead of exclusive */
end--;
if (!mtrr_state_set) {
/* Uniformity is unknown. */
*uniform = 0;
return MTRR_TYPE_UNCACHABLE;
}
if (!mtrr_state_set)
return MTRR_TYPE_INVALID;
*uniform = 1;
if (!(mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED))
return MTRR_TYPE_INVALID;
return MTRR_TYPE_UNCACHABLE;
/*
* Look up the fixed ranges first, which take priority over
* the variable ranges.
*/
if ((start < 0x100000) &&
(mtrr_state.have_fixed) &&
(mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) {
is_uniform = 0;
type = mtrr_type_lookup_fixed(start, end);
goto out;
for (i = 0; i < cache_map_n && start < end; i++) {
/* Region after current map entry? -> continue with next one. */
if (start >= cache_map[i].end)
continue;
/* Start of region not covered by current map entry? */
if (start < cache_map[i].start) {
/* At least some part of region has default type. */
type = type_merge(type, mtrr_state.def_type, uniform);
/* End of region not covered, too? -> lookup done. */
if (end <= cache_map[i].start)
return type;
}
/* At least part of region covered by map entry. */
type = type_merge(type, cache_map[i].type, uniform);
start = cache_map[i].end;
}
/*
* Look up the variable ranges. Look of multiple ranges matching
* this address and pick type as per MTRR precedence.
*/
type = mtrr_type_lookup_variable(start, end, &partial_end,
&repeat, &is_uniform);
/* End of region past last entry in map? -> use default type. */
if (start < end)
type = type_merge(type, mtrr_state.def_type, uniform);
/*
* Common path is with repeat = 0.
* However, we can have cases where [start:end] spans across some
* MTRR ranges and/or the default type. Do repeated lookups for
* that case here.
*/
while (repeat) {
prev_type = type;
start = partial_end;
is_uniform = 0;
type = mtrr_type_lookup_variable(start, end, &partial_end,
&repeat, &dummy);
if (check_type_overlap(&prev_type, &type))
goto out;
}
if (mtrr_tom2 && (start >= (1ULL<<32)) && (end < mtrr_tom2))
type = MTRR_TYPE_WRBACK;
out:
*uniform = is_uniform;
return type;
}
@ -363,8 +595,8 @@ static void __init print_fixed_last(void)
if (!last_fixed_end)
return;
pr_debug(" %05X-%05X %s\n", last_fixed_start,
last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
pr_info(" %05X-%05X %s\n", last_fixed_start,
last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
last_fixed_end = 0;
}
@ -402,10 +634,10 @@ static void __init print_mtrr_state(void)
unsigned int i;
int high_width;
pr_debug("MTRR default type: %s\n",
mtrr_attrib_to_str(mtrr_state.def_type));
pr_info("MTRR default type: %s\n",
mtrr_attrib_to_str(mtrr_state.def_type));
if (mtrr_state.have_fixed) {
pr_debug("MTRR fixed ranges %sabled:\n",
pr_info("MTRR fixed ranges %sabled:\n",
((mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) &&
(mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) ?
"en" : "dis");
@ -420,26 +652,27 @@ static void __init print_mtrr_state(void)
/* tail */
print_fixed_last();
}
pr_debug("MTRR variable ranges %sabled:\n",
mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED ? "en" : "dis");
high_width = (__ffs64(size_or_mask) - (32 - PAGE_SHIFT) + 3) / 4;
pr_info("MTRR variable ranges %sabled:\n",
mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED ? "en" : "dis");
high_width = (boot_cpu_data.x86_phys_bits - (32 - PAGE_SHIFT) + 3) / 4;
for (i = 0; i < num_var_ranges; ++i) {
if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
pr_debug(" %u base %0*X%05X000 mask %0*X%05X000 %s\n",
i,
high_width,
mtrr_state.var_ranges[i].base_hi,
mtrr_state.var_ranges[i].base_lo >> 12,
high_width,
mtrr_state.var_ranges[i].mask_hi,
mtrr_state.var_ranges[i].mask_lo >> 12,
mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
if (mtrr_state.var_ranges[i].mask_lo & MTRR_PHYSMASK_V)
pr_info(" %u base %0*X%05X000 mask %0*X%05X000 %s\n",
i,
high_width,
mtrr_state.var_ranges[i].base_hi,
mtrr_state.var_ranges[i].base_lo >> 12,
high_width,
mtrr_state.var_ranges[i].mask_hi,
mtrr_state.var_ranges[i].mask_lo >> 12,
mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo &
MTRR_PHYSBASE_TYPE));
else
pr_debug(" %u disabled\n", i);
pr_info(" %u disabled\n", i);
}
if (mtrr_tom2)
pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
pr_info("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
}
/* Grab all of the MTRR state for this CPU into *state */
@ -452,7 +685,7 @@ bool __init get_mtrr_state(void)
vrs = mtrr_state.var_ranges;
rdmsr(MSR_MTRRcap, lo, dummy);
mtrr_state.have_fixed = (lo >> 8) & 1;
mtrr_state.have_fixed = lo & MTRR_CAP_FIX;
for (i = 0; i < num_var_ranges; i++)
get_mtrr_var_range(i, &vrs[i]);
@ -460,8 +693,8 @@ bool __init get_mtrr_state(void)
get_fixed_ranges(mtrr_state.fixed_ranges);
rdmsr(MSR_MTRRdefType, lo, dummy);
mtrr_state.def_type = (lo & 0xff);
mtrr_state.enabled = (lo & 0xc00) >> 10;
mtrr_state.def_type = lo & MTRR_DEF_TYPE_TYPE;
mtrr_state.enabled = (lo & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT;
if (amd_special_default_mtrr()) {
unsigned low, high;
@ -474,7 +707,8 @@ bool __init get_mtrr_state(void)
mtrr_tom2 &= 0xffffff800000ULL;
}
print_mtrr_state();
if (mtrr_debug)
print_mtrr_state();
mtrr_state_set = 1;
@ -574,7 +808,7 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
if ((mask_lo & 0x800) == 0) {
if (!(mask_lo & MTRR_PHYSMASK_V)) {
/* Invalid (i.e. free) range */
*base = 0;
*size = 0;
@ -585,8 +819,8 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
/* Work out the shifted address mask: */
tmp = (u64)mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
mask = size_or_mask | tmp;
tmp = (u64)mask_hi << 32 | (mask_lo & PAGE_MASK);
mask = (u64)phys_hi_rsvd << 32 | tmp;
/* Expand tmp with high bits to all 1s: */
hi = fls64(tmp);
@ -604,9 +838,9 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
* This works correctly if size is a power of two, i.e. a
* contiguous range:
*/
*size = -mask;
*size = -mask >> PAGE_SHIFT;
*base = (u64)base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
*type = base_lo & 0xff;
*type = base_lo & MTRR_PHYSBASE_TYPE;
out_put_cpu:
put_cpu();
@ -644,9 +878,8 @@ static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
bool changed = false;
rdmsr(MTRRphysBase_MSR(index), lo, hi);
if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
|| (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (lo & ~MTRR_PHYSBASE_RSVD)
|| (vr->base_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) {
mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
changed = true;
@ -654,9 +887,8 @@ static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
rdmsr(MTRRphysMask_MSR(index), lo, hi);
if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
|| (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (lo & ~MTRR_PHYSMASK_RSVD)
|| (vr->mask_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) {
mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
changed = true;
}
@ -691,11 +923,12 @@ static unsigned long set_mtrr_state(void)
* Set_mtrr_restore restores the old value of MTRRdefType,
* so to set it we fiddle with the saved value:
*/
if ((deftype_lo & 0xff) != mtrr_state.def_type
|| ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
if ((deftype_lo & MTRR_DEF_TYPE_TYPE) != mtrr_state.def_type ||
((deftype_lo & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT) != mtrr_state.enabled) {
deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
(mtrr_state.enabled << 10);
deftype_lo = (deftype_lo & MTRR_DEF_TYPE_DISABLE) |
mtrr_state.def_type |
(mtrr_state.enabled << MTRR_STATE_SHIFT);
change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
}
@ -708,7 +941,7 @@ void mtrr_disable(void)
rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
/* Disable MTRRs, and set the default type to uncached */
mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & MTRR_DEF_TYPE_DISABLE, deftype_hi);
}
void mtrr_enable(void)
@ -762,9 +995,9 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
memset(vr, 0, sizeof(struct mtrr_var_range));
} else {
vr->base_lo = base << PAGE_SHIFT | type;
vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
vr->mask_lo = -size << PAGE_SHIFT | 0x800;
vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
vr->base_hi = (base >> (32 - PAGE_SHIFT)) & ~phys_hi_rsvd;
vr->mask_lo = -size << PAGE_SHIFT | MTRR_PHYSMASK_V;
vr->mask_hi = (-size >> (32 - PAGE_SHIFT)) & ~phys_hi_rsvd;
mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
@ -783,7 +1016,7 @@ int generic_validate_add_page(unsigned long base, unsigned long size,
* For Intel PPro stepping <= 7
* must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
*/
if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
if (mtrr_if == &generic_mtrr_ops && boot_cpu_data.x86 == 6 &&
boot_cpu_data.x86_model == 1 &&
boot_cpu_data.x86_stepping <= 7) {
if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
@ -817,7 +1050,7 @@ static int generic_have_wrcomb(void)
{
unsigned long config, dummy;
rdmsr(MSR_MTRRcap, config, dummy);
return config & (1 << 10);
return config & MTRR_CAP_WC;
}
int positive_have_wrcomb(void)

90
arch/x86/kernel/cpu/mtrr/legacy.c Normal file
View File

@ -0,0 +1,90 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/syscore_ops.h>
#include <asm/cpufeature.h>
#include <asm/mtrr.h>
#include <asm/processor.h>
#include "mtrr.h"
void mtrr_set_if(void)
{
switch (boot_cpu_data.x86_vendor) {
case X86_VENDOR_AMD:
/* Pre-Athlon (K6) AMD CPU MTRRs */
if (cpu_feature_enabled(X86_FEATURE_K6_MTRR))
mtrr_if = &amd_mtrr_ops;
break;
case X86_VENDOR_CENTAUR:
if (cpu_feature_enabled(X86_FEATURE_CENTAUR_MCR))
mtrr_if = &centaur_mtrr_ops;
break;
case X86_VENDOR_CYRIX:
if (cpu_feature_enabled(X86_FEATURE_CYRIX_ARR))
mtrr_if = &cyrix_mtrr_ops;
break;
default:
break;
}
}
/*
* The suspend/resume methods are only for CPUs without MTRR. CPUs using generic
* MTRR driver don't require this.
*/
struct mtrr_value {
mtrr_type ltype;
unsigned long lbase;
unsigned long lsize;
};
static struct mtrr_value *mtrr_value;
static int mtrr_save(void)
{
int i;
if (!mtrr_value)
return -ENOMEM;
for (i = 0; i < num_var_ranges; i++) {
mtrr_if->get(i, &mtrr_value[i].lbase,
&mtrr_value[i].lsize,
&mtrr_value[i].ltype);
}
return 0;
}
static void mtrr_restore(void)
{
int i;
for (i = 0; i < num_var_ranges; i++) {
if (mtrr_value[i].lsize) {
mtrr_if->set(i, mtrr_value[i].lbase,
mtrr_value[i].lsize,
mtrr_value[i].ltype);
}
}
}
static struct syscore_ops mtrr_syscore_ops = {
.suspend = mtrr_save,
.resume = mtrr_restore,
};
void mtrr_register_syscore(void)
{
mtrr_value = kcalloc(num_var_ranges, sizeof(*mtrr_value), GFP_KERNEL);
/*
* The CPU has no MTRR and seems to not support SMP. They have
* specific drivers, we use a tricky method to support
* suspend/resume for them.
*
* TBD: is there any system with such CPU which supports
* suspend/resume? If no, we should remove the code.
*/
register_syscore_ops(&mtrr_syscore_ops);
}

204
arch/x86/kernel/cpu/mtrr/mtrr.c
View File

@ -59,15 +59,9 @@
#define MTRR_TO_PHYS_WC_OFFSET 1000
u32 num_var_ranges;
static bool mtrr_enabled(void)
{
return !!mtrr_if;
}
unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
static DEFINE_MUTEX(mtrr_mutex);
u64 size_or_mask, size_and_mask;
DEFINE_MUTEX(mtrr_mutex);
const struct mtrr_ops *mtrr_if;
@ -105,21 +99,6 @@ static int have_wrcomb(void)
return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
}
/* This function returns the number of variable MTRRs */
static void __init set_num_var_ranges(bool use_generic)
{
unsigned long config = 0, dummy;
if (use_generic)
rdmsr(MSR_MTRRcap, config, dummy);
else if (is_cpu(AMD) || is_cpu(HYGON))
config = 2;
else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
config = 8;
num_var_ranges = config & 0xff;
}
static void __init init_table(void)
{
int i, max;
@ -194,20 +173,8 @@ static inline int types_compatible(mtrr_type type1, mtrr_type type2)
* Note that the mechanism is the same for UP systems, too; all the SMP stuff
* becomes nops.
*/
static void
set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
{
struct set_mtrr_data data = { .smp_reg = reg,
.smp_base = base,
.smp_size = size,
.smp_type = type
};
stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);
}
static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base,
unsigned long size, mtrr_type type)
static void set_mtrr(unsigned int reg, unsigned long base, unsigned long size,
mtrr_type type)
{
struct set_mtrr_data data = { .smp_reg = reg,
.smp_base = base,
@ -216,6 +183,8 @@ static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base,
};
stop_machine_cpuslocked(mtrr_rendezvous_handler, &data, cpu_online_mask);
generic_rebuild_map();
}
/**
@ -337,7 +306,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
/* Search for an empty MTRR */
i = mtrr_if->get_free_region(base, size, replace);
if (i >= 0) {
set_mtrr_cpuslocked(i, base, size, type);
set_mtrr(i, base, size, type);
if (likely(replace < 0)) {
mtrr_usage_table[i] = 1;
} else {
@ -345,7 +314,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
if (increment)
mtrr_usage_table[i]++;
if (unlikely(replace != i)) {
set_mtrr_cpuslocked(replace, 0, 0, 0);
set_mtrr(replace, 0, 0, 0);
mtrr_usage_table[replace] = 0;
}
}
@ -363,7 +332,7 @@ static int mtrr_check(unsigned long base, unsigned long size)
{
if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
pr_warn("size and base must be multiples of 4 kiB\n");
pr_debug("size: 0x%lx base: 0x%lx\n", size, base);
Dprintk("size: 0x%lx base: 0x%lx\n", size, base);
dump_stack();
return -1;
}
@ -454,8 +423,7 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size)
}
}
if (reg < 0) {
pr_debug("no MTRR for %lx000,%lx000 found\n",
base, size);
Dprintk("no MTRR for %lx000,%lx000 found\n", base, size);
goto out;
}
}
@ -473,7 +441,7 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size)
goto out;
}
if (--mtrr_usage_table[reg] < 1)
set_mtrr_cpuslocked(reg, 0, 0, 0);
set_mtrr(reg, 0, 0, 0);
error = reg;
out:
mutex_unlock(&mtrr_mutex);
@ -574,136 +542,54 @@ int arch_phys_wc_index(int handle)
}
EXPORT_SYMBOL_GPL(arch_phys_wc_index);
/* The suspend/resume methods are only for CPU without MTRR. CPU using generic
* MTRR driver doesn't require this
*/
struct mtrr_value {
mtrr_type ltype;
unsigned long lbase;
unsigned long lsize;
};
static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
static int mtrr_save(void)
{
int i;
for (i = 0; i < num_var_ranges; i++) {
mtrr_if->get(i, &mtrr_value[i].lbase,
&mtrr_value[i].lsize,
&mtrr_value[i].ltype);
}
return 0;
}
static void mtrr_restore(void)
{
int i;
for (i = 0; i < num_var_ranges; i++) {
if (mtrr_value[i].lsize) {
set_mtrr(i, mtrr_value[i].lbase,
mtrr_value[i].lsize,
mtrr_value[i].ltype);
}
}
}
static struct syscore_ops mtrr_syscore_ops = {
.suspend = mtrr_save,
.resume = mtrr_restore,
};
int __initdata changed_by_mtrr_cleanup;
#define SIZE_OR_MASK_BITS(n) (~((1ULL << ((n) - PAGE_SHIFT)) - 1))
/**
* mtrr_bp_init - initialize mtrrs on the boot CPU
* mtrr_bp_init - initialize MTRRs on the boot CPU
*
* This needs to be called early; before any of the other CPUs are
* initialized (i.e. before smp_init()).
*
*/
void __init mtrr_bp_init(void)
{
bool generic_mtrrs = cpu_feature_enabled(X86_FEATURE_MTRR);
const char *why = "(not available)";
u32 phys_addr;
unsigned long config, dummy;
phys_addr = 32;
if (boot_cpu_has(X86_FEATURE_MTRR)) {
mtrr_if = &generic_mtrr_ops;
size_or_mask = SIZE_OR_MASK_BITS(36);
size_and_mask = 0x00f00000;
phys_addr = 36;
phys_hi_rsvd = GENMASK(31, boot_cpu_data.x86_phys_bits - 32);
if (!generic_mtrrs && mtrr_state.enabled) {
/*
* This is an AMD specific MSR, but we assume(hope?) that
* Intel will implement it too when they extend the address
* bus of the Xeon.
* Software overwrite of MTRR state, only for generic case.
* Note that X86_FEATURE_MTRR has been reset in this case.
*/
if (cpuid_eax(0x80000000) >= 0x80000008) {
phys_addr = cpuid_eax(0x80000008) & 0xff;
/* CPUID workaround for Intel 0F33/0F34 CPU */
if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
boot_cpu_data.x86 == 0xF &&
boot_cpu_data.x86_model == 0x3 &&
(boot_cpu_data.x86_stepping == 0x3 ||
boot_cpu_data.x86_stepping == 0x4))
phys_addr = 36;
init_table();
mtrr_build_map();
pr_info("MTRRs set to read-only\n");
size_or_mask = SIZE_OR_MASK_BITS(phys_addr);
size_and_mask = ~size_or_mask & 0xfffff00000ULL;
} else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
boot_cpu_data.x86 == 6) {
/*
* VIA C* family have Intel style MTRRs,
* but don't support PAE
*/
size_or_mask = SIZE_OR_MASK_BITS(32);
size_and_mask = 0;
phys_addr = 32;
}
} else {
switch (boot_cpu_data.x86_vendor) {
case X86_VENDOR_AMD:
if (cpu_feature_enabled(X86_FEATURE_K6_MTRR)) {
/* Pre-Athlon (K6) AMD CPU MTRRs */
mtrr_if = &amd_mtrr_ops;
size_or_mask = SIZE_OR_MASK_BITS(32);
size_and_mask = 0;
}
break;
case X86_VENDOR_CENTAUR:
if (cpu_feature_enabled(X86_FEATURE_CENTAUR_MCR)) {
mtrr_if = &centaur_mtrr_ops;
size_or_mask = SIZE_OR_MASK_BITS(32);
size_and_mask = 0;
}
break;
case X86_VENDOR_CYRIX:
if (cpu_feature_enabled(X86_FEATURE_CYRIX_ARR)) {
mtrr_if = &cyrix_mtrr_ops;
size_or_mask = SIZE_OR_MASK_BITS(32);
size_and_mask = 0;
}
break;
default:
break;
}
return;
}
if (generic_mtrrs)
mtrr_if = &generic_mtrr_ops;
else
mtrr_set_if();
if (mtrr_enabled()) {
set_num_var_ranges(mtrr_if == &generic_mtrr_ops);
/* Get the number of variable MTRR ranges. */
if (mtrr_if == &generic_mtrr_ops)
rdmsr(MSR_MTRRcap, config, dummy);
else
config = mtrr_if->var_regs;
num_var_ranges = config & MTRR_CAP_VCNT;
init_table();
if (mtrr_if == &generic_mtrr_ops) {
/* BIOS may override */
if (get_mtrr_state()) {
memory_caching_control |= CACHE_MTRR;
changed_by_mtrr_cleanup = mtrr_cleanup(phys_addr);
changed_by_mtrr_cleanup = mtrr_cleanup();
mtrr_build_map();
} else {
mtrr_if = NULL;
why = "by BIOS";
@ -730,8 +616,14 @@ void mtrr_save_state(void)
smp_call_function_single(first_cpu, mtrr_save_fixed_ranges, NULL, 1);
}
static int __init mtrr_init_finialize(void)
static int __init mtrr_init_finalize(void)
{
/*
* Map might exist if mtrr_overwrite_state() has been called or if
* mtrr_enabled() returns true.
*/
mtrr_copy_map();
if (!mtrr_enabled())
return 0;
@ -741,16 +633,8 @@ static int __init mtrr_init_finialize(void)
return 0;
}
/*
* The CPU has no MTRR and seems to not support SMP. They have
* specific drivers, we use a tricky method to support
* suspend/resume for them.
*
* TBD: is there any system with such CPU which supports
* suspend/resume? If no, we should remove the code.
*/
register_syscore_ops(&mtrr_syscore_ops);
mtrr_register_syscore();
return 0;
}
subsys_initcall(mtrr_init_finialize);
subsys_initcall(mtrr_init_finalize);

31
arch/x86/kernel/cpu/mtrr/mtrr.h
View File

@ -10,10 +10,13 @@
#define MTRR_CHANGE_MASK_VARIABLE 0x02
#define MTRR_CHANGE_MASK_DEFTYPE 0x04
extern bool mtrr_debug;
#define Dprintk(x...) do { if (mtrr_debug) pr_info(x); } while (0)
extern unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
struct mtrr_ops {
u32 vendor;
u32 var_regs;
void (*set)(unsigned int reg, unsigned long base,
unsigned long size, mtrr_type type);
void (*get)(unsigned int reg, unsigned long *base,
@ -51,18 +54,26 @@ void fill_mtrr_var_range(unsigned int index,
u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi);
bool get_mtrr_state(void);
extern u64 size_or_mask, size_and_mask;
extern const struct mtrr_ops *mtrr_if;
#define is_cpu(vnd) (mtrr_if && mtrr_if->vendor == X86_VENDOR_##vnd)
extern struct mutex mtrr_mutex;
extern unsigned int num_var_ranges;
extern u64 mtrr_tom2;
extern struct mtrr_state_type mtrr_state;
extern u32 phys_hi_rsvd;
void mtrr_state_warn(void);
const char *mtrr_attrib_to_str(int x);
void mtrr_wrmsr(unsigned, unsigned, unsigned);
#ifdef CONFIG_X86_32
void mtrr_set_if(void);
void mtrr_register_syscore(void);
#else
static inline void mtrr_set_if(void) { }
static inline void mtrr_register_syscore(void) { }
#endif
void mtrr_build_map(void);
void mtrr_copy_map(void);
/* CPU specific mtrr_ops vectors. */
extern const struct mtrr_ops amd_mtrr_ops;
@ -70,4 +81,14 @@ extern const struct mtrr_ops cyrix_mtrr_ops;
extern const struct mtrr_ops centaur_mtrr_ops;
extern int changed_by_mtrr_cleanup;
extern int mtrr_cleanup(unsigned address_bits);
extern int mtrr_cleanup(void);
/*
* Must be used by code which uses mtrr_if to call platform-specific
* MTRR manipulation functions.
*/
static inline bool mtrr_enabled(void)
{
return !!mtrr_if;
}
void generic_rebuild_map(void);

2
arch/x86/kernel/setup.c
View File

@ -1037,6 +1037,8 @@ void __init setup_arch(char **cmdline_p)
/*
* VMware detection requires dmi to be available, so this
* needs to be done after dmi_setup(), for the boot CPU.
* For some guest types (Xen PV, SEV-SNP, TDX) it is required to be
* called before cache_bp_init() for setting up MTRR state.
*/
init_hypervisor_platform();

24
arch/x86/mm/pgtable.c
View File

@ -702,14 +702,8 @@ void p4d_clear_huge(p4d_t *p4d)
* pud_set_huge - setup kernel PUD mapping
*
* MTRRs can override PAT memory types with 4KiB granularity. Therefore, this
* function sets up a huge page only if any of the following conditions are met:
*
* - MTRRs are disabled, or
*
* - MTRRs are enabled and the range is completely covered by a single MTRR, or
*
* - MTRRs are enabled and the corresponding MTRR memory type is WB, which
* has no effect on the requested PAT memory type.
* function sets up a huge page only if the complete range has the same MTRR
* caching mode.
*
* Callers should try to decrease page size (1GB -> 2MB -> 4K) if the bigger
* page mapping attempt fails.
@ -718,11 +712,10 @@ void p4d_clear_huge(p4d_t *p4d)
*/
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
{
u8 mtrr, uniform;
u8 uniform;
mtrr = mtrr_type_lookup(addr, addr + PUD_SIZE, &uniform);
if ((mtrr != MTRR_TYPE_INVALID) && (!uniform) &&
(mtrr != MTRR_TYPE_WRBACK))
mtrr_type_lookup(addr, addr + PUD_SIZE, &uniform);
if (!uniform)
return 0;
/* Bail out if we are we on a populated non-leaf entry: */
@ -745,11 +738,10 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
*/
int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
{
u8 mtrr, uniform;
u8 uniform;
mtrr = mtrr_type_lookup(addr, addr + PMD_SIZE, &uniform);
if ((mtrr != MTRR_TYPE_INVALID) && (!uniform) &&
(mtrr != MTRR_TYPE_WRBACK)) {
mtrr_type_lookup(addr, addr + PMD_SIZE, &uniform);
if (!uniform) {
pr_warn_once("%s: Cannot satisfy [mem %#010llx-%#010llx] with a huge-page mapping due to MTRR override.\n",
__func__, addr, addr + PMD_SIZE);
return 0;

54
arch/x86/xen/enlighten_pv.c
View File

@ -68,6 +68,7 @@
#include <asm/reboot.h>
#include <asm/hypervisor.h>
#include <asm/mach_traps.h>
#include <asm/mtrr.h>
#include <asm/mwait.h>
#include <asm/pci_x86.h>
#include <asm/cpu.h>
@ -119,6 +120,54 @@ static int __init parse_xen_msr_safe(char *str)
}
early_param("xen_msr_safe", parse_xen_msr_safe);
/* Get MTRR settings from Xen and put them into mtrr_state. */
static void __init xen_set_mtrr_data(void)
{
#ifdef CONFIG_MTRR
struct xen_platform_op op = {
.cmd = XENPF_read_memtype,
.interface_version = XENPF_INTERFACE_VERSION,
};
unsigned int reg;
unsigned long mask;
uint32_t eax, width;
static struct mtrr_var_range var[MTRR_MAX_VAR_RANGES] __initdata;
/* Get physical address width (only 64-bit cpus supported). */
width = 36;
eax = cpuid_eax(0x80000000);
if ((eax >> 16) == 0x8000 && eax >= 0x80000008) {
eax = cpuid_eax(0x80000008);
width = eax & 0xff;
}
for (reg = 0; reg < MTRR_MAX_VAR_RANGES; reg++) {
op.u.read_memtype.reg = reg;
if (HYPERVISOR_platform_op(&op))
break;
/*
* Only called in dom0, which has all RAM PFNs mapped at
* RAM MFNs, and all PCI space etc. is identity mapped.
* This means we can treat MFN == PFN regarding MTRR settings.
*/
var[reg].base_lo = op.u.read_memtype.type;
var[reg].base_lo |= op.u.read_memtype.mfn << PAGE_SHIFT;
var[reg].base_hi = op.u.read_memtype.mfn >> (32 - PAGE_SHIFT);
mask = ~((op.u.read_memtype.nr_mfns << PAGE_SHIFT) - 1);
mask &= (1UL << width) - 1;
if (mask)
mask |= MTRR_PHYSMASK_V;
var[reg].mask_lo = mask;
var[reg].mask_hi = mask >> 32;
}
/* Only overwrite MTRR state if any MTRR could be got from Xen. */
if (reg)
mtrr_overwrite_state(var, reg, MTRR_TYPE_UNCACHABLE);
#endif
}
static void __init xen_pv_init_platform(void)
{
/* PV guests can't operate virtio devices without grants. */
@ -135,6 +184,11 @@ static void __init xen_pv_init_platform(void)
/* pvclock is in shared info area */
xen_init_time_ops();
if (xen_initial_domain())
xen_set_mtrr_data();
else
mtrr_overwrite_state(NULL, 0, MTRR_TYPE_WRBACK);
}
static void __init xen_pv_guest_late_init(void)