Skip to content

Commit a2055ab

Browse files
amlutoIngo Molnar
amluto
authored and
Ingo Molnar
committed
x86/mm: Pass flush_tlb_info to flush_tlb_others() etc
Rather than passing all the contents of flush_tlb_info to flush_tlb_others(), pass a pointer to the structure directly. For consistency, this also removes the unnecessary cpu parameter from uv_flush_tlb_others() to make its signature match the other *flush_tlb_others() functions. This serves two purposes: - It will dramatically simplify future patches that change struct flush_tlb_info, which I'm planning to do. - struct flush_tlb_info is an adequate description of what to do for a local flush, too, so by reusing it we can remove duplicated code between local and remove flushes in a future patch. Signed-off-by: Andy Lutomirski <luto@kernel.org> Acked-by: Rik van Riel <riel@redhat.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Borislav Petkov <bpetkov@suse.de> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Michal Hocko <mhocko@suse.com> Cc: Nadav Amit <nadav.amit@gmail.com> Cc: Nadav Amit <namit@vmware.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Rik van Riel <riel@redhat.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-mm@kvack.org [ Fix build warning. ] Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent 4241119 commit a2055ab

File tree

7 files changed

+61
-64
lines changed

7 files changed

+61
-64
lines changed

‎arch/x86/include/asm/paravirt.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -312,11 +312,9 @@ static inline void __flush_tlb_single(unsigned long addr)
312312
}
313313

314314
static inline void flush_tlb_others(const struct cpumask *cpumask,
315-
struct mm_struct *mm,
316-
unsigned long start,
317-
unsigned long end)
315+
const struct flush_tlb_info *info)
318316
{
319-
PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
317+
PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
320318
}
321319

322320
static inline int paravirt_pgd_alloc(struct mm_struct *mm)

‎arch/x86/include/asm/paravirt_types.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ struct mm_struct;
5151
struct desc_struct;
5252
struct task_struct;
5353
struct cpumask;
54+
struct flush_tlb_info;
5455

5556
/*
5657
* Wrapper type for pointers to code which uses the non-standard
@@ -223,9 +224,7 @@ struct pv_mmu_ops {
223224
void (*flush_tlb_kernel)(void);
224225
void (*flush_tlb_single)(unsigned long addr);
225226
void (*flush_tlb_others)(const struct cpumask *cpus,
226-
struct mm_struct *mm,
227-
unsigned long start,
228-
unsigned long end);
227+
const struct flush_tlb_info *info);
229228

230229
/* Hooks for allocating and freeing a pagetable top-level */
231230
int (*pgd_alloc)(struct mm_struct *mm);

‎arch/x86/include/asm/tlbflush.h

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -220,12 +220,18 @@ static inline void __flush_tlb_one(unsigned long addr)
220220
* - flush_tlb_page(vma, vmaddr) flushes one page
221221
* - flush_tlb_range(vma, start, end) flushes a range of pages
222222
* - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
223-
* - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
223+
* - flush_tlb_others(cpumask, info) flushes TLBs on other cpus
224224
*
225225
* ..but the i386 has somewhat limited tlb flushing capabilities,
226226
* and page-granular flushes are available only on i486 and up.
227227
*/
228228

229+
struct flush_tlb_info {
230+
struct mm_struct *mm;
231+
unsigned long start;
232+
unsigned long end;
233+
};
234+
229235
#ifndef CONFIG_SMP
230236

231237
/* "_up" is for UniProcessor.
@@ -279,9 +285,7 @@ static inline void flush_tlb_mm_range(struct mm_struct *mm,
279285
}
280286

281287
static inline void native_flush_tlb_others(const struct cpumask *cpumask,
282-
struct mm_struct *mm,
283-
unsigned long start,
284-
unsigned long end)
288+
const struct flush_tlb_info *info)
285289
{
286290
}
287291

@@ -317,8 +321,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a)
317321
}
318322

319323
void native_flush_tlb_others(const struct cpumask *cpumask,
320-
struct mm_struct *mm,
321-
unsigned long start, unsigned long end);
324+
const struct flush_tlb_info *info);
322325

323326
#define TLBSTATE_OK 1
324327
#define TLBSTATE_LAZY 2
@@ -340,8 +343,8 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
340343
#endif /* SMP */
341344

342345
#ifndef CONFIG_PARAVIRT
343-
#define flush_tlb_others(mask, mm, start, end) \
344-
native_flush_tlb_others(mask, mm, start, end)
346+
#define flush_tlb_others(mask, info) \
347+
native_flush_tlb_others(mask, info)
345348
#endif
346349

347350
#endif /* _ASM_X86_TLBFLUSH_H */

‎arch/x86/include/asm/uv/uv.h

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#ifndef _ASM_X86_UV_UV_H
22
#define _ASM_X86_UV_UV_H
33

4+
#include <asm/tlbflush.h>
5+
46
enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC};
57

68
struct cpumask;
@@ -15,10 +17,7 @@ extern void uv_cpu_init(void);
1517
extern void uv_nmi_init(void);
1618
extern void uv_system_init(void);
1719
extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
18-
struct mm_struct *mm,
19-
unsigned long start,
20-
unsigned long end,
21-
unsigned int cpu);
20+
const struct flush_tlb_info *info);
2221

2322
#else /* X86_UV */
2423

@@ -28,8 +27,8 @@ static inline int is_uv_hubless(void) { return 0; }
2827
static inline void uv_cpu_init(void) { }
2928
static inline void uv_system_init(void) { }
3029
static inline const struct cpumask *
31-
uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
32-
unsigned long start, unsigned long end, unsigned int cpu)
30+
uv_flush_tlb_others(const struct cpumask *cpumask,
31+
const struct flush_tlb_info *info)
3332
{ return cpumask; }
3433

3534
#endif /* X86_UV */

‎arch/x86/mm/tlb.c

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,6 @@
3030

3131
#ifdef CONFIG_SMP
3232

33-
struct flush_tlb_info {
34-
struct mm_struct *flush_mm;
35-
unsigned long flush_start;
36-
unsigned long flush_end;
37-
};
38-
3933
/*
4034
* We cannot call mmdrop() because we are in interrupt context,
4135
* instead update mm->cpu_vm_mask.
@@ -229,11 +223,11 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
229223
*/
230224
static void flush_tlb_func(void *info)
231225
{
232-
struct flush_tlb_info *f = info;
226+
const struct flush_tlb_info *f = info;
233227

234228
inc_irq_stat(irq_tlb_count);
235229

236-
if (f->flush_mm && f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
230+
if (f->mm && f->mm != this_cpu_read(cpu_tlbstate.active_mm))
237231
return;
238232

239233
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
@@ -243,15 +237,15 @@ static void flush_tlb_func(void *info)
243237
return;
244238
}
245239

246-
if (f->flush_end == TLB_FLUSH_ALL) {
240+
if (f->end == TLB_FLUSH_ALL) {
247241
local_flush_tlb();
248242
trace_tlb_flush(TLB_REMOTE_SHOOTDOWN, TLB_FLUSH_ALL);
249243
} else {
250244
unsigned long addr;
251245
unsigned long nr_pages =
252-
(f->flush_end - f->flush_start) / PAGE_SIZE;
253-
addr = f->flush_start;
254-
while (addr < f->flush_end) {
246+
(f->end - f->start) / PAGE_SIZE;
247+
addr = f->start;
248+
while (addr < f->end) {
255249
__flush_tlb_single(addr);
256250
addr += PAGE_SIZE;
257251
}
@@ -260,33 +254,27 @@ static void flush_tlb_func(void *info)
260254
}
261255

262256
void native_flush_tlb_others(const struct cpumask *cpumask,
263-
struct mm_struct *mm, unsigned long start,
264-
unsigned long end)
257+
const struct flush_tlb_info *info)
265258
{
266-
struct flush_tlb_info info;
267-
268-
info.flush_mm = mm;
269-
info.flush_start = start;
270-
info.flush_end = end;
271-
272259
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
273-
if (end == TLB_FLUSH_ALL)
260+
if (info->end == TLB_FLUSH_ALL)
274261
trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL);
275262
else
276263
trace_tlb_flush(TLB_REMOTE_SEND_IPI,
277-
(end - start) >> PAGE_SHIFT);
264+
(info->end - info->start) >> PAGE_SHIFT);
278265

279266
if (is_uv_system()) {
280267
unsigned int cpu;
281268

282269
cpu = smp_processor_id();
283-
cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
270+
cpumask = uv_flush_tlb_others(cpumask, info);
284271
if (cpumask)
285272
smp_call_function_many(cpumask, flush_tlb_func,
286-
&info, 1);
273+
(void *)info, 1);
287274
return;
288275
}
289-
smp_call_function_many(cpumask, flush_tlb_func, &info, 1);
276+
smp_call_function_many(cpumask, flush_tlb_func,
277+
(void *)info, 1);
290278
}
291279

292280
/*
@@ -305,6 +293,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
305293
unsigned long end, unsigned long vmflag)
306294
{
307295
unsigned long addr;
296+
struct flush_tlb_info info;
308297
/* do a global flush by default */
309298
unsigned long base_pages_to_flush = TLB_FLUSH_ALL;
310299

@@ -347,15 +336,20 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
347336
}
348337
trace_tlb_flush(TLB_LOCAL_MM_SHOOTDOWN, base_pages_to_flush);
349338
out:
339+
info.mm = mm;
350340
if (base_pages_to_flush == TLB_FLUSH_ALL) {
351-
start = 0UL;
352-
end = TLB_FLUSH_ALL;
341+
info.start = 0UL;
342+
info.end = TLB_FLUSH_ALL;
343+
} else {
344+
info.start = start;
345+
info.end = end;
353346
}
354347
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
355-
flush_tlb_others(mm_cpumask(mm), mm, start, end);
348+
flush_tlb_others(mm_cpumask(mm), &info);
356349
preempt_enable();
357350
}
358351

352+
359353
static void do_flush_tlb_all(void *info)
360354
{
361355
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
@@ -376,7 +370,7 @@ static void do_kernel_range_flush(void *info)
376370
unsigned long addr;
377371

378372
/* flush range by one by one 'invlpg' */
379-
for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE)
373+
for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
380374
__flush_tlb_single(addr);
381375
}
382376

@@ -389,14 +383,20 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
389383
on_each_cpu(do_flush_tlb_all, NULL, 1);
390384
} else {
391385
struct flush_tlb_info info;
392-
info.flush_start = start;
393-
info.flush_end = end;
386+
info.start = start;
387+
info.end = end;
394388
on_each_cpu(do_kernel_range_flush, &info, 1);
395389
}
396390
}
397391

398392
void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
399393
{
394+
struct flush_tlb_info info = {
395+
.mm = NULL,
396+
.start = 0UL,
397+
.end = TLB_FLUSH_ALL,
398+
};
399+
400400
int cpu = get_cpu();
401401

402402
if (cpumask_test_cpu(cpu, &batch->cpumask)) {
@@ -406,7 +406,7 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
406406
}
407407

408408
if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
409-
flush_tlb_others(&batch->cpumask, NULL, 0, TLB_FLUSH_ALL);
409+
flush_tlb_others(&batch->cpumask, &info);
410410
cpumask_clear(&batch->cpumask);
411411

412412
put_cpu();

‎arch/x86/platform/uv/tlb_uv.c

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1121,11 +1121,9 @@ static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp,
11211121
* done. The returned pointer is valid till preemption is re-enabled.
11221122
*/
11231123
const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
1124-
struct mm_struct *mm,
1125-
unsigned long start,
1126-
unsigned long end,
1127-
unsigned int cpu)
1124+
const struct flush_tlb_info *info)
11281125
{
1126+
unsigned int cpu = smp_processor_id();
11291127
int locals = 0, remotes = 0, hubs = 0;
11301128
struct bau_desc *bau_desc;
11311129
struct cpumask *flush_mask;
@@ -1179,8 +1177,8 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
11791177

11801178
record_send_statistics(stat, locals, hubs, remotes, bau_desc);
11811179

1182-
if (!end || (end - start) <= PAGE_SIZE)
1183-
address = start;
1180+
if (!info->end || (info->end - info->start) <= PAGE_SIZE)
1181+
address = info->start;
11841182
else
11851183
address = TLB_FLUSH_ALL;
11861184

‎arch/x86/xen/mmu_pv.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1366,8 +1366,7 @@ static void xen_flush_tlb_single(unsigned long addr)
13661366
}
13671367

13681368
static void xen_flush_tlb_others(const struct cpumask *cpus,
1369-
struct mm_struct *mm, unsigned long start,
1370-
unsigned long end)
1369+
const struct flush_tlb_info *info)
13711370
{
13721371
struct {
13731372
struct mmuext_op op;
@@ -1379,7 +1378,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
13791378
} *args;
13801379
struct multicall_space mcs;
13811380

1382-
trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
1381+
trace_xen_mmu_flush_tlb_others(cpus, info->mm, info->start, info->end);
13831382

13841383
if (cpumask_empty(cpus))
13851384
return; /* nothing to do */
@@ -1393,9 +1392,10 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
13931392
cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
13941393

13951394
args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1396-
if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
1395+
if (info->end != TLB_FLUSH_ALL &&
1396+
(info->end - info->start) <= PAGE_SIZE) {
13971397
args->op.cmd = MMUEXT_INVLPG_MULTI;
1398-
args->op.arg1.linear_addr = start;
1398+
args->op.arg1.linear_addr = info->start;
13991399
}
14001400

14011401
MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);

0 commit comments

Comments
 (0)