Skip to content

Commit 6cf78d4

Browse files
torvaldstorvalds
committed
Merge branch 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 mm changes from Ingo Molnar: "The main changes in this cycle were: - reduce the x86/32 PAE per task PGD allocation overhead from 4K to 0.032k (Fenghua Yu) - early_ioremap/memunmap() usage cleanups (Juergen Gross) - gbpages support cleanups (Luis R Rodriguez) - improve AMD Bulldozer (family 0x15) ASLR I$ aliasing workaround to increase randomization by 3 bits (per bootup) (Hector Marco-Gisbert) - misc fixlets" * 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/mm: Improve AMD Bulldozer ASLR workaround x86/mm/pat: Initialize __cachemode2pte_tbl[] and __pte2cachemode_tbl[] in a bit more readable fashion init.h: Clean up the __setup()/early_param() macros x86/mm: Simplify probe_page_size_mask() x86/mm: Further simplify 1 GB kernel linear mappings handling x86/mm: Use early_param_on_off() for direct_gbpages init.h: Add early_param_on_off() x86/mm: Simplify enabling direct_gbpages x86/mm: Use IS_ENABLED() for direct_gbpages x86/mm: Unexport set_memory_ro() and set_memory_rw() x86/mm, efi: Use early_ioremap() in arch/x86/platform/efi/efi-bgrt.c x86/mm: Use early_memunmap() instead of early_iounmap() x86/mm/pat: Ensure different messages in STRICT_DEVMEM and PAT cases x86/mm: Reduce PAE-mode per task pgd allocation overhead from 4K to 32 bytes
2 parents 0ad5c6b + 4e26d11 commit 6cf78d4

File tree

14 files changed

+189
-89
lines changed

14 files changed

+189
-89
lines changed

‎arch/x86/Kconfig

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1295,14 +1295,14 @@ config ARCH_DMA_ADDR_T_64BIT
12951295
def_bool y
12961296
depends on X86_64 || HIGHMEM64G
12971297

1298-
config DIRECT_GBPAGES
1299-
bool "Enable 1GB pages for kernel pagetables" if EXPERT
1300-
default y
1301-
depends on X86_64
1298+
config X86_DIRECT_GBPAGES
1299+
def_bool y
1300+
depends on X86_64 && !DEBUG_PAGEALLOC && !KMEMCHECK
13021301
---help---
1303-
Allow the kernel linear mapping to use 1GB pages on CPUs that
1304-
support it. This can improve the kernel's performance a tiny bit by
1305-
reducing TLB pressure. If in doubt, say "Y".
1302+
Certain kernel features effectively disable kernel
1303+
linear 1 GB mappings (even if the CPU otherwise
1304+
supports them), so don't confuse the user by printing
1305+
that we have them enabled.
13061306

13071307
# Common NUMA Features
13081308
config NUMA

‎arch/x86/include/asm/elf.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ enum align_flags {
366366
struct va_alignment {
367367
int flags;
368368
unsigned long mask;
369+
unsigned long bits;
369370
} ____cacheline_aligned;
370371

371372
extern struct va_alignment va_align;

‎arch/x86/kernel/cpu/amd.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
#include <linux/io.h>
77
#include <linux/sched.h>
8+
#include <linux/random.h>
89
#include <asm/processor.h>
910
#include <asm/apic.h>
1011
#include <asm/cpu.h>
@@ -488,6 +489,9 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
488489

489490
va_align.mask = (upperbit - 1) & PAGE_MASK;
490491
va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
492+
493+
/* A random value per boot for bit slice [12:upper_bit) */
494+
va_align.bits = get_random_int() & va_align.mask;
491495
}
492496
}
493497

‎arch/x86/kernel/devicetree.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,13 +286,13 @@ static void __init x86_flattree_get_config(void)
286286
initial_boot_params = dt = early_memremap(initial_dtb, map_len);
287287
size = of_get_flat_dt_size();
288288
if (map_len < size) {
289-
early_iounmap(dt, map_len);
289+
early_memunmap(dt, map_len);
290290
initial_boot_params = dt = early_memremap(initial_dtb, size);
291291
map_len = size;
292292
}
293293

294294
unflatten_and_copy_device_tree();
295-
early_iounmap(dt, map_len);
295+
early_memunmap(dt, map_len);
296296
}
297297
#else
298298
static inline void x86_flattree_get_config(void) { }

‎arch/x86/kernel/e820.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -661,7 +661,7 @@ void __init parse_e820_ext(u64 phys_addr, u32 data_len)
661661
extmap = (struct e820entry *)(sdata->data);
662662
__append_e820_map(extmap, entries);
663663
sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
664-
early_iounmap(sdata, data_len);
664+
early_memunmap(sdata, data_len);
665665
printk(KERN_INFO "e820: extended physical RAM map:\n");
666666
e820_print_map("extended");
667667
}

‎arch/x86/kernel/setup.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ static void __init relocate_initrd(void)
354354
mapaddr = ramdisk_image & PAGE_MASK;
355355
p = early_memremap(mapaddr, clen+slop);
356356
memcpy(q, p+slop, clen);
357-
early_iounmap(p, clen+slop);
357+
early_memunmap(p, clen+slop);
358358
q += clen;
359359
ramdisk_image += clen;
360360
ramdisk_size -= clen;
@@ -438,7 +438,7 @@ static void __init parse_setup_data(void)
438438
data_len = data->len + sizeof(struct setup_data);
439439
data_type = data->type;
440440
pa_next = data->next;
441-
early_iounmap(data, sizeof(*data));
441+
early_memunmap(data, sizeof(*data));
442442

443443
switch (data_type) {
444444
case SETUP_E820_EXT:
@@ -470,7 +470,7 @@ static void __init e820_reserve_setup_data(void)
470470
E820_RAM, E820_RESERVED_KERN);
471471
found = 1;
472472
pa_data = data->next;
473-
early_iounmap(data, sizeof(*data));
473+
early_memunmap(data, sizeof(*data));
474474
}
475475
if (!found)
476476
return;
@@ -491,7 +491,7 @@ static void __init memblock_x86_reserve_range_setup_data(void)
491491
data = early_memremap(pa_data, sizeof(*data));
492492
memblock_reserve(pa_data, sizeof(*data) + data->len);
493493
pa_data = data->next;
494-
early_iounmap(data, sizeof(*data));
494+
early_memunmap(data, sizeof(*data));
495495
}
496496
}
497497

‎arch/x86/kernel/sys_x86_64.c

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,26 @@ static unsigned long get_align_mask(void)
3434
return va_align.mask;
3535
}
3636

37+
/*
38+
* To avoid aliasing in the I$ on AMD F15h, the bits defined by the
39+
* va_align.bits, [12:upper_bit), are set to a random value instead of
40+
* zeroing them. This random value is computed once per boot. This form
41+
* of ASLR is known as "per-boot ASLR".
42+
*
43+
* To achieve this, the random value is added to the info.align_offset
44+
* value before calling vm_unmapped_area() or ORed directly to the
45+
* address.
46+
*/
47+
static unsigned long get_align_bits(void)
48+
{
49+
return va_align.bits & get_align_mask();
50+
}
51+
3752
unsigned long align_vdso_addr(unsigned long addr)
3853
{
3954
unsigned long align_mask = get_align_mask();
40-
return (addr + align_mask) & ~align_mask;
55+
addr = (addr + align_mask) & ~align_mask;
56+
return addr | get_align_bits();
4157
}
4258

4359
static int __init control_va_addr_alignment(char *str)
@@ -135,8 +151,12 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
135151
info.length = len;
136152
info.low_limit = begin;
137153
info.high_limit = end;
138-
info.align_mask = filp ? get_align_mask() : 0;
154+
info.align_mask = 0;
139155
info.align_offset = pgoff << PAGE_SHIFT;
156+
if (filp) {
157+
info.align_mask = get_align_mask();
158+
info.align_offset += get_align_bits();
159+
}
140160
return vm_unmapped_area(&info);
141161
}
142162

@@ -174,8 +194,12 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
174194
info.length = len;
175195
info.low_limit = PAGE_SIZE;
176196
info.high_limit = mm->mmap_base;
177-
info.align_mask = filp ? get_align_mask() : 0;
197+
info.align_mask = 0;
178198
info.align_offset = pgoff << PAGE_SHIFT;
199+
if (filp) {
200+
info.align_mask = get_align_mask();
201+
info.align_offset += get_align_bits();
202+
}
179203
addr = vm_unmapped_area(&info);
180204
if (!(addr & ~PAGE_MASK))
181205
return addr;

‎arch/x86/mm/init.c

Lines changed: 31 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -29,29 +29,33 @@
2929

3030
/*
3131
* Tables translating between page_cache_type_t and pte encoding.
32-
* Minimal supported modes are defined statically, modified if more supported
33-
* cache modes are available.
34-
* Index into __cachemode2pte_tbl is the cachemode.
35-
* Index into __pte2cachemode_tbl are the caching attribute bits of the pte
36-
* (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2.
32+
*
33+
* Minimal supported modes are defined statically, they are modified
34+
* during bootup if more supported cache modes are available.
35+
*
36+
* Index into __cachemode2pte_tbl[] is the cachemode.
37+
*
38+
* Index into __pte2cachemode_tbl[] are the caching attribute bits of the pte
39+
* (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2.
3740
*/
3841
uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM] = {
39-
[_PAGE_CACHE_MODE_WB] = 0,
40-
[_PAGE_CACHE_MODE_WC] = _PAGE_PWT,
41-
[_PAGE_CACHE_MODE_UC_MINUS] = _PAGE_PCD,
42-
[_PAGE_CACHE_MODE_UC] = _PAGE_PCD | _PAGE_PWT,
43-
[_PAGE_CACHE_MODE_WT] = _PAGE_PCD,
44-
[_PAGE_CACHE_MODE_WP] = _PAGE_PCD,
42+
[_PAGE_CACHE_MODE_WB ] = 0 | 0 ,
43+
[_PAGE_CACHE_MODE_WC ] = _PAGE_PWT | 0 ,
44+
[_PAGE_CACHE_MODE_UC_MINUS] = 0 | _PAGE_PCD,
45+
[_PAGE_CACHE_MODE_UC ] = _PAGE_PWT | _PAGE_PCD,
46+
[_PAGE_CACHE_MODE_WT ] = 0 | _PAGE_PCD,
47+
[_PAGE_CACHE_MODE_WP ] = 0 | _PAGE_PCD,
4548
};
4649
EXPORT_SYMBOL(__cachemode2pte_tbl);
50+
4751
uint8_t __pte2cachemode_tbl[8] = {
48-
[__pte2cm_idx(0)] = _PAGE_CACHE_MODE_WB,
49-
[__pte2cm_idx(_PAGE_PWT)] = _PAGE_CACHE_MODE_WC,
50-
[__pte2cm_idx(_PAGE_PCD)] = _PAGE_CACHE_MODE_UC_MINUS,
51-
[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD)] = _PAGE_CACHE_MODE_UC,
52-
[__pte2cm_idx(_PAGE_PAT)] = _PAGE_CACHE_MODE_WB,
53-
[__pte2cm_idx(_PAGE_PWT | _PAGE_PAT)] = _PAGE_CACHE_MODE_WC,
54-
[__pte2cm_idx(_PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
52+
[__pte2cm_idx( 0 | 0 | 0 )] = _PAGE_CACHE_MODE_WB,
53+
[__pte2cm_idx(_PAGE_PWT | 0 | 0 )] = _PAGE_CACHE_MODE_WC,
54+
[__pte2cm_idx( 0 | _PAGE_PCD | 0 )] = _PAGE_CACHE_MODE_UC_MINUS,
55+
[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | 0 )] = _PAGE_CACHE_MODE_UC,
56+
[__pte2cm_idx( 0 | 0 | _PAGE_PAT)] = _PAGE_CACHE_MODE_WB,
57+
[__pte2cm_idx(_PAGE_PWT | 0 | _PAGE_PAT)] = _PAGE_CACHE_MODE_WC,
58+
[__pte2cm_idx(0 | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
5559
[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC,
5660
};
5761
EXPORT_SYMBOL(__pte2cachemode_tbl);
@@ -131,21 +135,7 @@ void __init early_alloc_pgt_buf(void)
131135

132136
int after_bootmem;
133137

134-
int direct_gbpages
135-
#ifdef CONFIG_DIRECT_GBPAGES
136-
= 1
137-
#endif
138-
;
139-
140-
static void __init init_gbpages(void)
141-
{
142-
#ifdef CONFIG_X86_64
143-
if (direct_gbpages && cpu_has_gbpages)
144-
printk(KERN_INFO "Using GB pages for direct mapping\n");
145-
else
146-
direct_gbpages = 0;
147-
#endif
148-
}
138+
early_param_on_off("gbpages", "nogbpages", direct_gbpages, CONFIG_X86_DIRECT_GBPAGES);
149139

150140
struct map_range {
151141
unsigned long start;
@@ -157,16 +147,12 @@ static int page_size_mask;
157147

158148
static void __init probe_page_size_mask(void)
159149
{
160-
init_gbpages();
161-
162150
#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK)
163151
/*
164152
* For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
165153
* This will simplify cpa(), which otherwise needs to support splitting
166154
* large pages into small in interrupt context, etc.
167155
*/
168-
if (direct_gbpages)
169-
page_size_mask |= 1 << PG_LEVEL_1G;
170156
if (cpu_has_pse)
171157
page_size_mask |= 1 << PG_LEVEL_2M;
172158
#endif
@@ -181,6 +167,14 @@ static void __init probe_page_size_mask(void)
181167
__supported_pte_mask |= _PAGE_GLOBAL;
182168
} else
183169
__supported_pte_mask &= ~_PAGE_GLOBAL;
170+
171+
/* Enable 1 GB linear kernel mappings if available: */
172+
if (direct_gbpages && cpu_has_gbpages) {
173+
printk(KERN_INFO "Using GB pages for direct mapping\n");
174+
page_size_mask |= 1 << PG_LEVEL_1G;
175+
} else {
176+
direct_gbpages = 0;
177+
}
184178
}
185179

186180
#ifdef CONFIG_X86_32

‎arch/x86/mm/init_64.c

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -130,20 +130,6 @@ int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
130130
return 0;
131131
}
132132

133-
static int __init parse_direct_gbpages_off(char *arg)
134-
{
135-
direct_gbpages = 0;
136-
return 0;
137-
}
138-
early_param("nogbpages", parse_direct_gbpages_off);
139-
140-
static int __init parse_direct_gbpages_on(char *arg)
141-
{
142-
direct_gbpages = 1;
143-
return 0;
144-
}
145-
early_param("gbpages", parse_direct_gbpages_on);
146-
147133
/*
148134
* NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
149135
* physical space so we can cache the place of the first one and move

‎arch/x86/mm/pageattr.c

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,9 @@ void arch_report_meminfo(struct seq_file *m)
8181
seq_printf(m, "DirectMap4M: %8lu kB\n",
8282
direct_pages_count[PG_LEVEL_2M] << 12);
8383
#endif
84-
#ifdef CONFIG_X86_64
8584
if (direct_gbpages)
8685
seq_printf(m, "DirectMap1G: %8lu kB\n",
8786
direct_pages_count[PG_LEVEL_1G] << 20);
88-
#endif
8987
}
9088
#else
9189
static inline void split_page_count(int level) { }
@@ -1654,13 +1652,11 @@ int set_memory_ro(unsigned long addr, int numpages)
16541652
{
16551653
return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
16561654
}
1657-
EXPORT_SYMBOL_GPL(set_memory_ro);
16581655

16591656
int set_memory_rw(unsigned long addr, int numpages)
16601657
{
16611658
return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
16621659
}
1663-
EXPORT_SYMBOL_GPL(set_memory_rw);
16641660

16651661
int set_memory_np(unsigned long addr, int numpages)
16661662
{

‎arch/x86/mm/pat.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
610610
}
611611

612612
#ifdef CONFIG_STRICT_DEVMEM
613-
/* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM*/
613+
/* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM */
614614
static inline int range_is_allowed(unsigned long pfn, unsigned long size)
615615
{
616616
return 1;
@@ -628,8 +628,8 @@ static inline int range_is_allowed(unsigned long pfn, unsigned long size)
628628

629629
while (cursor < to) {
630630
if (!devmem_is_allowed(pfn)) {
631-
printk(KERN_INFO "Program %s tried to access /dev/mem between [mem %#010Lx-%#010Lx]\n",
632-
current->comm, from, to - 1);
631+
printk(KERN_INFO "Program %s tried to access /dev/mem between [mem %#010Lx-%#010Lx], PAT prevents it\n",
632+
current->comm, from, to - 1);
633633
return 0;
634634
}
635635
cursor += PAGE_SIZE;

0 commit comments

Comments
 (0)