Skip to content

Commit c8790d6

Browse files
paulburtonpaulburton
committed
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to Address Space IDs (ASIDs). The major difference between the two is that MMIDs are global - ie. an MMID uniquely identifies an address space across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs, wherein each address space is allocated a separate ASID for each CPU upon which it is used. This global namespace allows a new GINVT instruction be used to globally invalidate TLB entries associated with a particular MMID across all coherent CPUs in the system, removing the need for IPIs to invalidate entries with separate ASIDs on each CPU. The allocation scheme used here is largely borrowed from arm64 (see arch/arm64/mm/context.c). In essence we maintain a bitmap to track available MMIDs, and MMIDs in active use at the time of a rollover to a new MMID version are preserved in the new version. The allocation scheme requires efficient 64 bit atomics in order to perform reasonably, so this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it will only be included in MIPS64 kernels). The first, and currently only, available CPU with support for MMIDs is the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap our MMIDs to 16 bits wide in order to prevent the bitmap growing to absurd sizes if any future CPU does implement 32 bit MMIDs as the architecture manuals suggest is recommended. When MMIDs are in use we also make use of GINVT instruction which is available due to the global nature of MMIDs. By executing a sequence of GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to each remote CPU in many cases. One complication is that GINVT will invalidate wired entries (in all cases apart from type 0, which targets the entire TLB). In order to avoid GINVT invalidating any wired TLB entries we set up, we make sure to create those entries using a reserved MMID (0) that we never associate with any address space. Also of note is that KVM will require further work in order to support MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in configuring the MMU. That work is not part of this patch, so for now when MMIDs are in use KVM is disabled. Signed-off-by: Paul Burton <paul.burton@mips.com> Cc: linux-mips@vger.kernel.org
1 parent 5351138 commit c8790d6

File tree

15 files changed

+509
-31
lines changed

15 files changed

+509
-31
lines changed

‎arch/mips/include/asm/cpu-features.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,19 @@
590590
# define cpu_has_mipsmt_pertccounters 0
591591
#endif /* CONFIG_MIPS_MT_SMP */
592592

593+
/*
594+
* We only enable MMID support for configurations which natively support 64 bit
595+
* atomics because getting good performance from the allocator relies upon
596+
* efficient atomic64_*() functions.
597+
*/
598+
#ifndef cpu_has_mmid
599+
# ifdef CONFIG_GENERIC_ATOMIC64
600+
# define cpu_has_mmid 0
601+
# else
602+
# define cpu_has_mmid __isa_ge_and_opt(6, MIPS_CPU_MMID)
603+
# endif
604+
#endif
605+
593606
/*
594607
* Guest capabilities
595608
*/

‎arch/mips/include/asm/cpu.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,7 @@ enum cpu_type_enum {
422422
MBIT_ULL(55) /* CPU shares FTLB entries with another */
423423
#define MIPS_CPU_MT_PER_TC_PERF_COUNTERS \
424424
MBIT_ULL(56) /* CPU has perf counters implemented per TC (MIPSMT ASE) */
425+
#define MIPS_CPU_MMID MBIT_ULL(57) /* CPU supports MemoryMapIDs */
425426

426427
/*
427428
* CPU ASE encodings

‎arch/mips/include/asm/mipsregs.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,7 @@
667667
#define MIPS_CONF5_FRE (_ULCAST_(1) << 8)
668668
#define MIPS_CONF5_UFE (_ULCAST_(1) << 9)
669669
#define MIPS_CONF5_CA2 (_ULCAST_(1) << 14)
670+
#define MIPS_CONF5_MI (_ULCAST_(1) << 17)
670671
#define MIPS_CONF5_CRCP (_ULCAST_(1) << 18)
671672
#define MIPS_CONF5_MSAEN (_ULCAST_(1) << 27)
672673
#define MIPS_CONF5_EVA (_ULCAST_(1) << 28)
@@ -1610,6 +1611,9 @@ do { \
16101611
#define read_c0_xcontextconfig() __read_ulong_c0_register($4, 3)
16111612
#define write_c0_xcontextconfig(val) __write_ulong_c0_register($4, 3, val)
16121613

1614+
#define read_c0_memorymapid() __read_32bit_c0_register($4, 5)
1615+
#define write_c0_memorymapid(val) __write_32bit_c0_register($4, 5, val)
1616+
16131617
#define read_c0_pagemask() __read_32bit_c0_register($5, 0)
16141618
#define write_c0_pagemask(val) __write_32bit_c0_register($5, 0, val)
16151619

‎arch/mips/include/asm/mmu.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77
#include <linux/wait.h>
88

99
typedef struct {
10-
u64 asid[NR_CPUS];
10+
union {
11+
u64 asid[NR_CPUS];
12+
atomic64_t mmid;
13+
};
14+
1115
void *vdso;
1216

1317
/* lock to be held whilst modifying fp_bd_emupage_allocmap */

‎arch/mips/include/asm/mmu_context.h

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717
#include <linux/smp.h>
1818
#include <linux/slab.h>
1919

20+
#include <asm/barrier.h>
2021
#include <asm/cacheflush.h>
2122
#include <asm/dsemul.h>
23+
#include <asm/ginvt.h>
2224
#include <asm/hazards.h>
2325
#include <asm/tlbflush.h>
2426
#include <asm-generic/mm_hooks.h>
@@ -72,6 +74,19 @@ extern unsigned long pgd_current[];
7274
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
7375
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT*/
7476

77+
/*
78+
* The ginvt instruction will invalidate wired entries when its type field
79+
* targets anything other than the entire TLB. That means that if we were to
80+
* allow the kernel to create wired entries with the MMID of current->active_mm
81+
* then those wired entries could be invalidated when we later use ginvt to
82+
* invalidate TLB entries with that MMID.
83+
*
84+
* In order to prevent ginvt from trashing wired entries, we reserve one MMID
85+
* for use by the kernel when creating wired entries. This MMID will never be
86+
* assigned to a struct mm, and we'll never target it with a ginvt instruction.
87+
*/
88+
#define MMID_KERNEL_WIRED 0
89+
7590
/*
7691
* All unused by hardware upper bits will be considered
7792
* as a software asid extension.
@@ -90,13 +105,19 @@ static inline u64 asid_first_version(unsigned int cpu)
90105

91106
static inline u64 cpu_context(unsigned int cpu, const struct mm_struct *mm)
92107
{
108+
if (cpu_has_mmid)
109+
return atomic64_read(&mm->context.mmid);
110+
93111
return mm->context.asid[cpu];
94112
}
95113

96114
static inline void set_cpu_context(unsigned int cpu,
97115
struct mm_struct *mm, u64 ctx)
98116
{
99-
mm->context.asid[cpu] = ctx;
117+
if (cpu_has_mmid)
118+
atomic64_set(&mm->context.mmid, ctx);
119+
else
120+
mm->context.asid[cpu] = ctx;
100121
}
101122

102123
#define asid_cache(cpu) (cpu_data[cpu].asid_cache)
@@ -120,8 +141,12 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm)
120141
{
121142
int i;
122143

123-
for_each_possible_cpu(i)
124-
set_cpu_context(i, mm, 0);
144+
if (cpu_has_mmid) {
145+
set_cpu_context(0, mm, 0);
146+
} else {
147+
for_each_possible_cpu(i)
148+
set_cpu_context(i, mm, 0);
149+
}
125150

126151
mm->context.bd_emupage_allocmap = NULL;
127152
spin_lock_init(&mm->context.bd_emupage_lock);
@@ -168,12 +193,33 @@ drop_mmu_context(struct mm_struct *mm)
168193
{
169194
unsigned long flags;
170195
unsigned int cpu;
196+
u32 old_mmid;
197+
u64 ctx;
171198

172199
local_irq_save(flags);
173200

174201
cpu = smp_processor_id();
175-
if (!cpu_context(cpu, mm)) {
202+
ctx = cpu_context(cpu, mm);
203+
204+
if (!ctx) {
176205
/* no-op */
206+
} else if (cpu_has_mmid) {
207+
/*
208+
* Globally invalidating TLB entries associated with the MMID
209+
* is pretty cheap using the GINVT instruction, so we'll do
210+
* that rather than incur the overhead of allocating a new
211+
* MMID. The latter would be especially difficult since MMIDs
212+
* are global & other CPUs may be actively using ctx.
213+
*/
214+
htw_stop();
215+
old_mmid = read_c0_memorymapid();
216+
write_c0_memorymapid(ctx & cpu_asid_mask(&cpu_data[cpu]));
217+
mtc0_tlbw_hazard();
218+
ginvt_mmid();
219+
sync_ginv();
220+
write_c0_memorymapid(old_mmid);
221+
instruction_hazard();
222+
htw_start();
177223
} else if (cpumask_test_cpu(cpu, mm_cpumask(mm))) {
178224
/*
179225
* mm is currently active, so we can't really drop it.

‎arch/mips/kernel/cpu-probe.c

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -872,10 +872,19 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c)
872872

873873
static inline unsigned int decode_config5(struct cpuinfo_mips *c)
874874
{
875-
unsigned int config5;
875+
unsigned int config5, max_mmid_width;
876+
unsigned long asid_mask;
876877

877878
config5 = read_c0_config5();
878879
config5 &= ~(MIPS_CONF5_UFR | MIPS_CONF5_UFE);
880+
881+
if (cpu_has_mips_r6) {
882+
if (!__builtin_constant_p(cpu_has_mmid) || cpu_has_mmid)
883+
config5 |= MIPS_CONF5_MI;
884+
else
885+
config5 &= ~MIPS_CONF5_MI;
886+
}
887+
879888
write_c0_config5(config5);
880889

881890
if (config5 & MIPS_CONF5_EVA)
@@ -894,6 +903,50 @@ static inline unsigned int decode_config5(struct cpuinfo_mips *c)
894903
if (config5 & MIPS_CONF5_CRCP)
895904
elf_hwcap |= HWCAP_MIPS_CRC32;
896905

906+
if (cpu_has_mips_r6) {
907+
/* Ensure the write to config5 above takes effect */
908+
back_to_back_c0_hazard();
909+
910+
/* Check whether we successfully enabled MMID support */
911+
config5 = read_c0_config5();
912+
if (config5 & MIPS_CONF5_MI)
913+
c->options |= MIPS_CPU_MMID;
914+
915+
/*
916+
* Warn if we've hardcoded cpu_has_mmid to a value unsuitable
917+
* for the CPU we're running on, or if CPUs in an SMP system
918+
* have inconsistent MMID support.
919+
*/
920+
WARN_ON(!!cpu_has_mmid != !!(config5 & MIPS_CONF5_MI));
921+
922+
if (cpu_has_mmid) {
923+
write_c0_memorymapid(~0ul);
924+
back_to_back_c0_hazard();
925+
asid_mask = read_c0_memorymapid();
926+
927+
/*
928+
* We maintain a bitmap to track MMID allocation, and
929+
* need a sensible upper bound on the size of that
930+
* bitmap. The initial CPU with MMID support (I6500)
931+
* supports 16 bit MMIDs, which gives us an 8KiB
932+
* bitmap. The architecture recommends that hardware
933+
* support 32 bit MMIDs, which would give us a 512MiB
934+
* bitmap - that's too big in most cases.
935+
*
936+
* Cap MMID width at 16 bits for now & we can revisit
937+
* this if & when hardware supports anything wider.
938+
*/
939+
max_mmid_width = 16;
940+
if (asid_mask > GENMASK(max_mmid_width - 1, 0)) {
941+
pr_info("Capping MMID width at %d bits",
942+
max_mmid_width);
943+
asid_mask = GENMASK(max_mmid_width - 1, 0);
944+
}
945+
946+
set_cpu_asid_mask(c, asid_mask);
947+
}
948+
}
949+
897950
return config5 & MIPS_CONF_M;
898951
}
899952

‎arch/mips/kernel/smp.c

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939

4040
#include <linux/atomic.h>
4141
#include <asm/cpu.h>
42+
#include <asm/ginvt.h>
4243
#include <asm/processor.h>
4344
#include <asm/idle.h>
4445
#include <asm/r4k-timer.h>
@@ -482,6 +483,15 @@ static void flush_tlb_all_ipi(void *info)
482483

483484
void flush_tlb_all(void)
484485
{
486+
if (cpu_has_mmid) {
487+
htw_stop();
488+
ginvt_full();
489+
sync_ginv();
490+
instruction_hazard();
491+
htw_start();
492+
return;
493+
}
494+
485495
on_each_cpu(flush_tlb_all_ipi, NULL, 1);
486496
}
487497

@@ -530,7 +540,12 @@ void flush_tlb_mm(struct mm_struct *mm)
530540
{
531541
preempt_disable();
532542

533-
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
543+
if (cpu_has_mmid) {
544+
/*
545+
* No need to worry about other CPUs - the ginvt in
546+
* drop_mmu_context() will be globalized.
547+
*/
548+
} else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
534549
smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
535550
} else {
536551
unsigned int cpu;
@@ -561,16 +576,34 @@ static void flush_tlb_range_ipi(void *info)
561576
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
562577
{
563578
struct mm_struct *mm = vma->vm_mm;
579+
unsigned long addr;
580+
u32 old_mmid;
564581

565582
preempt_disable();
566-
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
583+
if (cpu_has_mmid) {
584+
htw_stop();
585+
old_mmid = read_c0_memorymapid();
586+
write_c0_memorymapid(cpu_asid(0, mm));
587+
mtc0_tlbw_hazard();
588+
addr = round_down(start, PAGE_SIZE * 2);
589+
end = round_up(end, PAGE_SIZE * 2);
590+
do {
591+
ginvt_va_mmid(addr);
592+
sync_ginv();
593+
addr += PAGE_SIZE * 2;
594+
} while (addr < end);
595+
write_c0_memorymapid(old_mmid);
596+
instruction_hazard();
597+
htw_start();
598+
} else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
567599
struct flush_tlb_data fd = {
568600
.vma = vma,
569601
.addr1 = start,
570602
.addr2 = end,
571603
};
572604

573605
smp_on_other_tlbs(flush_tlb_range_ipi, &fd);
606+
local_flush_tlb_range(vma, start, end);
574607
} else {
575608
unsigned int cpu;
576609
int exec = vma->vm_flags & VM_EXEC;
@@ -585,8 +618,8 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
585618
if (cpu != smp_processor_id() && cpu_context(cpu, mm))
586619
set_cpu_context(cpu, mm, !exec);
587620
}
621+
local_flush_tlb_range(vma, start, end);
588622
}
589-
local_flush_tlb_range(vma, start, end);
590623
preempt_enable();
591624
}
592625

@@ -616,14 +649,28 @@ static void flush_tlb_page_ipi(void *info)
616649

617650
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
618651
{
652+
u32 old_mmid;
653+
619654
preempt_disable();
620-
if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
655+
if (cpu_has_mmid) {
656+
htw_stop();
657+
old_mmid = read_c0_memorymapid();
658+
write_c0_memorymapid(cpu_asid(0, vma->vm_mm));
659+
mtc0_tlbw_hazard();
660+
ginvt_va_mmid(page);
661+
sync_ginv();
662+
write_c0_memorymapid(old_mmid);
663+
instruction_hazard();
664+
htw_start();
665+
} else if ((atomic_read(&vma->vm_mm->mm_users) != 1) ||
666+
(current->mm != vma->vm_mm)) {
621667
struct flush_tlb_data fd = {
622668
.vma = vma,
623669
.addr1 = page,
624670
};
625671

626672
smp_on_other_tlbs(flush_tlb_page_ipi, &fd);
673+
local_flush_tlb_page(vma, page);
627674
} else {
628675
unsigned int cpu;
629676

@@ -637,8 +684,8 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
637684
if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm))
638685
set_cpu_context(cpu, vma->vm_mm, 1);
639686
}
687+
local_flush_tlb_page(vma, page);
640688
}
641-
local_flush_tlb_page(vma, page);
642689
preempt_enable();
643690
}
644691

‎arch/mips/kernel/traps.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2223,7 +2223,9 @@ void per_cpu_trap_init(bool is_boot_cpu)
22232223
cp0_fdc_irq = -1;
22242224
}
22252225

2226-
if (!cpu_data[cpu].asid_cache)
2226+
if (cpu_has_mmid)
2227+
cpu_data[cpu].asid_cache = 0;
2228+
else if (!cpu_data[cpu].asid_cache)
22272229
cpu_data[cpu].asid_cache = asid_first_version(cpu);
22282230

22292231
mmgrab(&init_mm);

‎arch/mips/kernel/unaligned.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
#include <asm/fpu.h>
9090
#include <asm/fpu_emulator.h>
9191
#include <asm/inst.h>
92+
#include <asm/mmu_context.h>
9293
#include <linux/uaccess.h>
9394

9495
#define STR(x) __STR(x)

‎arch/mips/kvm/mips.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1723,6 +1723,11 @@ static int __init kvm_mips_init(void)
17231723
{
17241724
int ret;
17251725

1726+
if (cpu_has_mmid) {
1727+
pr_warn("KVM does not yet support MMIDs. KVM Disabled\n");
1728+
return -EOPNOTSUPP;
1729+
}
1730+
17261731
ret = kvm_mips_entry_setup();
17271732
if (ret)
17281733
return ret;

0 commit comments

Comments
 (0)