Skip to content

Commit dfb09f9

Browse files
Borislav PetkovH. Peter Anvin
Borislav Petkov
authored and
H. Peter Anvin
committed
x86, amd: Avoid cache aliasing penalties on AMD family 15h
This patch provides performance tuning for the "Bulldozer" CPU. With its shared instruction cache there is a chance of generating an excessive number of cache cross-invalidates when running specific workloads on the cores of a compute module. This excessive amount of cross-invalidations can be observed if cache lines backed by shared physical memory alias in bits [14:12] of their virtual addresses, as those bits are used for the index generation. This patch addresses the issue by clearing all the bits in the [14:12] slice of the file mapping's virtual address at generation time, thus forcing those bits the same for all mappings of a single shared library across processes and, in doing so, avoids instruction cache aliases. It also adds the command line option "align_va_addr=(32|64|on|off)" with which virtual address alignment can be enabled for 32-bit or 64-bit x86 individually, or both, or be completely disabled. This change leaves virtual region address allocation on other families and/or vendors unaffected. Signed-off-by: Borislav Petkov <borislav.petkov@amd.com> Link: http://lkml.kernel.org/r/1312550110-24160-2-git-send-email-bp@amd64.org Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
1 parent 13f9a37 commit dfb09f9

File tree

6 files changed

+144
-18
lines changed

6 files changed

+144
-18
lines changed

‎Documentation/kernel-parameters.txt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,19 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
299299
behaviour to be specified. Bit 0 enables warnings,
300300
bit 1 enables fixups, and bit 2 sends a segfault.
301301

302+
align_va_addr= [X86-64]
303+
Align virtual addresses by clearing slice [14:12] when
304+
allocating a VMA at process creation time. This option
305+
gives you up to 3% performance improvement on AMD F15h
306+
machines (where it is enabled by default) for a
307+
CPU-intensive style benchmark, and it can vary highly in
308+
a microbenchmark depending on workload and compiler.
309+
310+
1: only for 32-bit processes
311+
2: only for 64-bit processes
312+
on: enable for both 32- and 64-bit processes
313+
off: disable for both 32- and 64-bit processes
314+
302315
amd_iommu= [HW,X86-84]
303316
Pass parameters to the AMD IOMMU driver in the system.
304317
Possible values are:

‎arch/x86/include/asm/elf.h

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
/*
55
* ELF register definitions..
66
*/
7+
#include <linux/thread_info.h>
78

89
#include <asm/ptrace.h>
910
#include <asm/user.h>
@@ -320,4 +321,34 @@ extern int syscall32_setup_pages(struct linux_binprm *, int exstack);
320321
extern unsigned long arch_randomize_brk(struct mm_struct *mm);
321322
#define arch_randomize_brk arch_randomize_brk
322323

324+
/*
325+
* True on X86_32 or when emulating IA32 on X86_64
326+
*/
327+
static inline int mmap_is_ia32(void)
328+
{
329+
#ifdef CONFIG_X86_32
330+
return 1;
331+
#endif
332+
#ifdef CONFIG_IA32_EMULATION
333+
if (test_thread_flag(TIF_IA32))
334+
return 1;
335+
#endif
336+
return 0;
337+
}
338+
339+
/* The first two values are special, do not change. See align_addr() */
340+
enum align_flags {
341+
ALIGN_VA_32 = BIT(0),
342+
ALIGN_VA_64 = BIT(1),
343+
ALIGN_VDSO = BIT(2),
344+
ALIGN_TOPDOWN = BIT(3),
345+
};
346+
347+
struct va_alignment {
348+
int flags;
349+
unsigned long mask;
350+
} ____cacheline_aligned;
351+
352+
extern struct va_alignment va_align;
353+
extern unsigned long align_addr(unsigned long, struct file *, enum align_flags);
323354
#endif /* _ASM_X86_ELF_H */

‎arch/x86/kernel/cpu/amd.c

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,19 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
458458
"with P0 frequency!\n");
459459
}
460460
}
461+
462+
if (c->x86 == 0x15) {
463+
unsigned long upperbit;
464+
u32 cpuid, assoc;
465+
466+
cpuid = cpuid_edx(0x80000005);
467+
assoc = cpuid >> 16 & 0xff;
468+
upperbit = ((cpuid >> 24) << 10) / assoc;
469+
470+
va_align.mask = (upperbit - 1) & PAGE_MASK;
471+
va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
472+
473+
}
461474
}
462475

463476
static void __cpuinit init_amd(struct cpuinfo_x86 *c)

‎arch/x86/kernel/sys_x86_64.c

Lines changed: 78 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,72 @@
1818
#include <asm/ia32.h>
1919
#include <asm/syscalls.h>
2020

21+
struct __read_mostly va_alignment va_align = {
22+
.flags = -1,
23+
};
24+
25+
/*
26+
* Align a virtual address to avoid aliasing in the I$ on AMD F15h.
27+
*
28+
* @flags denotes the allocation direction - bottomup or topdown -
29+
* or vDSO; see call sites below.
30+
*/
31+
unsigned long align_addr(unsigned long addr, struct file *filp,
32+
enum align_flags flags)
33+
{
34+
unsigned long tmp_addr;
35+
36+
/* handle 32- and 64-bit case with a single conditional */
37+
if (va_align.flags < 0 || !(va_align.flags & (2 - mmap_is_ia32())))
38+
return addr;
39+
40+
if (!(current->flags & PF_RANDOMIZE))
41+
return addr;
42+
43+
if (!((flags & ALIGN_VDSO) || filp))
44+
return addr;
45+
46+
tmp_addr = addr;
47+
48+
/*
49+
* We need an address which is <= than the original
50+
* one only when in topdown direction.
51+
*/
52+
if (!(flags & ALIGN_TOPDOWN))
53+
tmp_addr += va_align.mask;
54+
55+
tmp_addr &= ~va_align.mask;
56+
57+
return tmp_addr;
58+
}
59+
60+
static int __init control_va_addr_alignment(char *str)
61+
{
62+
/* guard against enabling this on other CPU families */
63+
if (va_align.flags < 0)
64+
return 1;
65+
66+
if (*str == 0)
67+
return 1;
68+
69+
if (*str == '=')
70+
str++;
71+
72+
if (!strcmp(str, "32"))
73+
va_align.flags = ALIGN_VA_32;
74+
else if (!strcmp(str, "64"))
75+
va_align.flags = ALIGN_VA_64;
76+
else if (!strcmp(str, "off"))
77+
va_align.flags = 0;
78+
else if (!strcmp(str, "on"))
79+
va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
80+
else
81+
return 0;
82+
83+
return 1;
84+
}
85+
__setup("align_va_addr", control_va_addr_alignment);
86+
2187
SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
2288
unsigned long, prot, unsigned long, flags,
2389
unsigned long, fd, unsigned long, off)
@@ -92,6 +158,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
92158
start_addr = addr;
93159

94160
full_search:
161+
162+
addr = align_addr(addr, filp, 0);
163+
95164
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
96165
/* At this point: (!vma || addr < vma->vm_end). */
97166
if (end - len < addr) {
@@ -117,6 +186,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
117186
mm->cached_hole_size = vma->vm_start - addr;
118187

119188
addr = vma->vm_end;
189+
addr = align_addr(addr, filp, 0);
120190
}
121191
}
122192

@@ -161,10 +231,13 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
161231

162232
/* make sure it can fit in the remaining address space */
163233
if (addr > len) {
164-
vma = find_vma(mm, addr-len);
165-
if (!vma || addr <= vma->vm_start)
234+
unsigned long tmp_addr = align_addr(addr - len, filp,
235+
ALIGN_TOPDOWN);
236+
237+
vma = find_vma(mm, tmp_addr);
238+
if (!vma || tmp_addr + len <= vma->vm_start)
166239
/* remember the address as a hint for next time */
167-
return mm->free_area_cache = addr-len;
240+
return mm->free_area_cache = tmp_addr;
168241
}
169242

170243
if (mm->mmap_base < len)
@@ -173,6 +246,8 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
173246
addr = mm->mmap_base-len;
174247

175248
do {
249+
addr = align_addr(addr, filp, ALIGN_TOPDOWN);
250+
176251
/*
177252
* Lookup failure means no vma is above this address,
178253
* else if new region fits below vma->vm_start,

‎arch/x86/mm/mmap.c

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -51,21 +51,6 @@ static unsigned int stack_maxrandom_size(void)
5151
#define MIN_GAP (128*1024*1024UL + stack_maxrandom_size())
5252
#define MAX_GAP (TASK_SIZE/6*5)
5353

54-
/*
55-
* True on X86_32 or when emulating IA32 on X86_64
56-
*/
57-
static int mmap_is_ia32(void)
58-
{
59-
#ifdef CONFIG_X86_32
60-
return 1;
61-
#endif
62-
#ifdef CONFIG_IA32_EMULATION
63-
if (test_thread_flag(TIF_IA32))
64-
return 1;
65-
#endif
66-
return 0;
67-
}
68-
6954
static int mmap_is_legacy(void)
7055
{
7156
if (current->personality & ADDR_COMPAT_LAYOUT)

‎arch/x86/vdso/vma.c

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,15 @@ static unsigned long vdso_addr(unsigned long start, unsigned len)
6969
addr = start + (offset << PAGE_SHIFT);
7070
if (addr >= end)
7171
addr = end;
72+
73+
/*
74+
* page-align it here so that get_unmapped_area doesn't
75+
* align it wrongfully again to the next page. addr can come in 4K
76+
* unaligned here as a result of stack start randomization.
77+
*/
78+
addr = PAGE_ALIGN(addr);
79+
addr = align_addr(addr, NULL, ALIGN_VDSO);
80+
7281
return addr;
7382
}
7483

0 commit comments

Comments
 (0)