Skip to content

Commit 2e2dcc7

Browse files
author
Ingo Molnar
committed
Merge branch 'x86/paravirt-spinlocks' into x86/for-linus
2 parents acee709 + 1c29dd9 commit 2e2dcc7

File tree

9 files changed

+379
-17
lines changed

9 files changed

+379
-17
lines changed

‎arch/x86/kernel/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ extra-y := head_$(BITS).o head$(BITS).o head.o init_task.o vmlinu
77
CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE)
88

99
ifdef CONFIG_FTRACE
10-
# Do not profile debug utilities
10+
# Do not profile debug and lowlevel utilities
1111
CFLAGS_REMOVE_tsc.o = -pg
1212
CFLAGS_REMOVE_rtc.o = -pg
13+
CFLAGS_REMOVE_paravirt.o = -pg
1314
endif
1415

1516
#

‎arch/x86/kernel/paravirt.c

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ static void *get_call_destination(u8 type)
124124
.pv_irq_ops = pv_irq_ops,
125125
.pv_apic_ops = pv_apic_ops,
126126
.pv_mmu_ops = pv_mmu_ops,
127+
.pv_lock_ops = pv_lock_ops,
127128
};
128129
return *((void **)&tmpl + type);
129130
}
@@ -267,6 +268,17 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
267268
return __get_cpu_var(paravirt_lazy_mode);
268269
}
269270

271+
void __init paravirt_use_bytelocks(void)
272+
{
273+
#ifdef CONFIG_SMP
274+
pv_lock_ops.spin_is_locked = __byte_spin_is_locked;
275+
pv_lock_ops.spin_is_contended = __byte_spin_is_contended;
276+
pv_lock_ops.spin_lock = __byte_spin_lock;
277+
pv_lock_ops.spin_trylock = __byte_spin_trylock;
278+
pv_lock_ops.spin_unlock = __byte_spin_unlock;
279+
#endif
280+
}
281+
270282
struct pv_info pv_info = {
271283
.name = "bare hardware",
272284
.paravirt_enabled = 0,
@@ -449,6 +461,18 @@ struct pv_mmu_ops pv_mmu_ops = {
449461
.set_fixmap = native_set_fixmap,
450462
};
451463

464+
struct pv_lock_ops pv_lock_ops = {
465+
#ifdef CONFIG_SMP
466+
.spin_is_locked = __ticket_spin_is_locked,
467+
.spin_is_contended = __ticket_spin_is_contended,
468+
469+
.spin_lock = __ticket_spin_lock,
470+
.spin_trylock = __ticket_spin_trylock,
471+
.spin_unlock = __ticket_spin_unlock,
472+
#endif
473+
};
474+
EXPORT_SYMBOL_GPL(pv_lock_ops);
475+
452476
EXPORT_SYMBOL_GPL(pv_time_ops);
453477
EXPORT_SYMBOL (pv_cpu_ops);
454478
EXPORT_SYMBOL (pv_mmu_ops);

‎arch/x86/xen/smp.c

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
* This does not handle HOTPLUG_CPU yet.
1616
*/
1717
#include <linux/sched.h>
18+
#include <linux/kernel_stat.h>
1819
#include <linux/err.h>
1920
#include <linux/smp.h>
2021

@@ -35,6 +36,8 @@
3536
#include "xen-ops.h"
3637
#include "mmu.h"
3738

39+
static void __cpuinit xen_init_lock_cpu(int cpu);
40+
3841
cpumask_t xen_cpu_initialized_map;
3942

4043
static DEFINE_PER_CPU(int, resched_irq);
@@ -179,6 +182,8 @@ static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
179182
{
180183
unsigned cpu;
181184

185+
xen_init_lock_cpu(0);
186+
182187
smp_store_cpu_info(0);
183188
cpu_data(0).x86_max_cores = 1;
184189
set_cpu_sibling_map(0);
@@ -301,6 +306,7 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
301306
clear_tsk_thread_flag(idle, TIF_FORK);
302307
#endif
303308
xen_setup_timer(cpu);
309+
xen_init_lock_cpu(cpu);
304310

305311
per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
306312

@@ -413,6 +419,170 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
413419
return IRQ_HANDLED;
414420
}
415421

422+
struct xen_spinlock {
423+
unsigned char lock; /* 0 -> free; 1 -> locked */
424+
unsigned short spinners; /* count of waiting cpus */
425+
};
426+
427+
static int xen_spin_is_locked(struct raw_spinlock *lock)
428+
{
429+
struct xen_spinlock *xl = (struct xen_spinlock *)lock;
430+
431+
return xl->lock != 0;
432+
}
433+
434+
static int xen_spin_is_contended(struct raw_spinlock *lock)
435+
{
436+
struct xen_spinlock *xl = (struct xen_spinlock *)lock;
437+
438+
/* Not strictly true; this is only the count of contended
439+
lock-takers entering the slow path. */
440+
return xl->spinners != 0;
441+
}
442+
443+
static int xen_spin_trylock(struct raw_spinlock *lock)
444+
{
445+
struct xen_spinlock *xl = (struct xen_spinlock *)lock;
446+
u8 old = 1;
447+
448+
asm("xchgb %b0,%1"
449+
: "+q" (old), "+m" (xl->lock) : : "memory");
450+
451+
return old == 0;
452+
}
453+
454+
static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
455+
static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners);
456+
457+
static inline void spinning_lock(struct xen_spinlock *xl)
458+
{
459+
__get_cpu_var(lock_spinners) = xl;
460+
wmb(); /* set lock of interest before count */
461+
asm(LOCK_PREFIX " incw %0"
462+
: "+m" (xl->spinners) : : "memory");
463+
}
464+
465+
static inline void unspinning_lock(struct xen_spinlock *xl)
466+
{
467+
asm(LOCK_PREFIX " decw %0"
468+
: "+m" (xl->spinners) : : "memory");
469+
wmb(); /* decrement count before clearing lock */
470+
__get_cpu_var(lock_spinners) = NULL;
471+
}
472+
473+
static noinline int xen_spin_lock_slow(struct raw_spinlock *lock)
474+
{
475+
struct xen_spinlock *xl = (struct xen_spinlock *)lock;
476+
int irq = __get_cpu_var(lock_kicker_irq);
477+
int ret;
478+
479+
/* If kicker interrupts not initialized yet, just spin */
480+
if (irq == -1)
481+
return 0;
482+
483+
/* announce we're spinning */
484+
spinning_lock(xl);
485+
486+
/* clear pending */
487+
xen_clear_irq_pending(irq);
488+
489+
/* check again make sure it didn't become free while
490+
we weren't looking */
491+
ret = xen_spin_trylock(lock);
492+
if (ret)
493+
goto out;
494+
495+
/* block until irq becomes pending */
496+
xen_poll_irq(irq);
497+
kstat_this_cpu.irqs[irq]++;
498+
499+
out:
500+
unspinning_lock(xl);
501+
return ret;
502+
}
503+
504+
static void xen_spin_lock(struct raw_spinlock *lock)
505+
{
506+
struct xen_spinlock *xl = (struct xen_spinlock *)lock;
507+
int timeout;
508+
u8 oldval;
509+
510+
do {
511+
timeout = 1 << 10;
512+
513+
asm("1: xchgb %1,%0\n"
514+
" testb %1,%1\n"
515+
" jz 3f\n"
516+
"2: rep;nop\n"
517+
" cmpb $0,%0\n"
518+
" je 1b\n"
519+
" dec %2\n"
520+
" jnz 2b\n"
521+
"3:\n"
522+
: "+m" (xl->lock), "=q" (oldval), "+r" (timeout)
523+
: "1" (1)
524+
: "memory");
525+
526+
} while (unlikely(oldval != 0 && !xen_spin_lock_slow(lock)));
527+
}
528+
529+
static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl)
530+
{
531+
int cpu;
532+
533+
for_each_online_cpu(cpu) {
534+
/* XXX should mix up next cpu selection */
535+
if (per_cpu(lock_spinners, cpu) == xl) {
536+
xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
537+
break;
538+
}
539+
}
540+
}
541+
542+
static void xen_spin_unlock(struct raw_spinlock *lock)
543+
{
544+
struct xen_spinlock *xl = (struct xen_spinlock *)lock;
545+
546+
smp_wmb(); /* make sure no writes get moved after unlock */
547+
xl->lock = 0; /* release lock */
548+
549+
/* make sure unlock happens before kick */
550+
barrier();
551+
552+
if (unlikely(xl->spinners))
553+
xen_spin_unlock_slow(xl);
554+
}
555+
556+
static __cpuinit void xen_init_lock_cpu(int cpu)
557+
{
558+
int irq;
559+
const char *name;
560+
561+
name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
562+
irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
563+
cpu,
564+
xen_reschedule_interrupt,
565+
IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
566+
name,
567+
NULL);
568+
569+
if (irq >= 0) {
570+
disable_irq(irq); /* make sure it's never delivered */
571+
per_cpu(lock_kicker_irq, cpu) = irq;
572+
}
573+
574+
printk("cpu %d spinlock event irq %d\n", cpu, irq);
575+
}
576+
577+
static void __init xen_init_spinlocks(void)
578+
{
579+
pv_lock_ops.spin_is_locked = xen_spin_is_locked;
580+
pv_lock_ops.spin_is_contended = xen_spin_is_contended;
581+
pv_lock_ops.spin_lock = xen_spin_lock;
582+
pv_lock_ops.spin_trylock = xen_spin_trylock;
583+
pv_lock_ops.spin_unlock = xen_spin_unlock;
584+
}
585+
416586
static const struct smp_ops xen_smp_ops __initdata = {
417587
.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
418588
.smp_prepare_cpus = xen_smp_prepare_cpus,
@@ -430,4 +600,5 @@ void __init xen_smp_init(void)
430600
{
431601
smp_ops = xen_smp_ops;
432602
xen_fill_possible_map();
603+
xen_init_spinlocks();
433604
}

‎drivers/xen/events.c

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -734,6 +734,33 @@ static void restore_cpu_ipis(unsigned int cpu)
734734
}
735735
}
736736

737+
/* Clear an irq's pending state, in preparation for polling on it */
738+
void xen_clear_irq_pending(int irq)
739+
{
740+
int evtchn = evtchn_from_irq(irq);
741+
742+
if (VALID_EVTCHN(evtchn))
743+
clear_evtchn(evtchn);
744+
}
745+
746+
/* Poll waiting for an irq to become pending. In the usual case, the
747+
irq will be disabled so it won't deliver an interrupt. */
748+
void xen_poll_irq(int irq)
749+
{
750+
evtchn_port_t evtchn = evtchn_from_irq(irq);
751+
752+
if (VALID_EVTCHN(evtchn)) {
753+
struct sched_poll poll;
754+
755+
poll.nr_ports = 1;
756+
poll.timeout = 0;
757+
poll.ports = &evtchn;
758+
759+
if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
760+
BUG();
761+
}
762+
}
763+
737764
void xen_irq_resume(void)
738765
{
739766
unsigned int cpu, irq, evtchn;

‎include/asm-x86/paravirt.h

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,15 @@ struct pv_mmu_ops {
325325
unsigned long phys, pgprot_t flags);
326326
};
327327

328+
struct raw_spinlock;
329+
struct pv_lock_ops {
330+
int (*spin_is_locked)(struct raw_spinlock *lock);
331+
int (*spin_is_contended)(struct raw_spinlock *lock);
332+
void (*spin_lock)(struct raw_spinlock *lock);
333+
int (*spin_trylock)(struct raw_spinlock *lock);
334+
void (*spin_unlock)(struct raw_spinlock *lock);
335+
};
336+
328337
/* This contains all the paravirt structures: we get a convenient
329338
* number for each function using the offset which we use to indicate
330339
* what to patch. */
@@ -335,6 +344,7 @@ struct paravirt_patch_template {
335344
struct pv_irq_ops pv_irq_ops;
336345
struct pv_apic_ops pv_apic_ops;
337346
struct pv_mmu_ops pv_mmu_ops;
347+
struct pv_lock_ops pv_lock_ops;
338348
};
339349

340350
extern struct pv_info pv_info;
@@ -344,6 +354,7 @@ extern struct pv_cpu_ops pv_cpu_ops;
344354
extern struct pv_irq_ops pv_irq_ops;
345355
extern struct pv_apic_ops pv_apic_ops;
346356
extern struct pv_mmu_ops pv_mmu_ops;
357+
extern struct pv_lock_ops pv_lock_ops;
347358

348359
#define PARAVIRT_PATCH(x) \
349360
(offsetof(struct paravirt_patch_template, x) / sizeof(void *))
@@ -1368,6 +1379,37 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
13681379
void _paravirt_nop(void);
13691380
#define paravirt_nop ((void *)_paravirt_nop)
13701381

1382+
void paravirt_use_bytelocks(void);
1383+
1384+
#ifdef CONFIG_SMP
1385+
1386+
static inline int __raw_spin_is_locked(struct raw_spinlock *lock)
1387+
{
1388+
return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);
1389+
}
1390+
1391+
static inline int __raw_spin_is_contended(struct raw_spinlock *lock)
1392+
{
1393+
return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);
1394+
}
1395+
1396+
static __always_inline void __raw_spin_lock(struct raw_spinlock *lock)
1397+
{
1398+
PVOP_VCALL1(pv_lock_ops.spin_lock, lock);
1399+
}
1400+
1401+
static __always_inline int __raw_spin_trylock(struct raw_spinlock *lock)
1402+
{
1403+
return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);
1404+
}
1405+
1406+
static __always_inline void __raw_spin_unlock(struct raw_spinlock *lock)
1407+
{
1408+
PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);
1409+
}
1410+
1411+
#endif
1412+
13711413
/* These all sit in the .parainstructions section to tell us what to patch. */
13721414
struct paravirt_patch_site {
13731415
u8 *instr; /* original instructions */
@@ -1452,6 +1494,7 @@ static inline unsigned long __raw_local_irq_save(void)
14521494
return f;
14531495
}
14541496

1497+
14551498
/* Make sure as little as possible of this mess escapes. */
14561499
#undef PARAVIRT_CALL
14571500
#undef __PVOP_CALL

0 commit comments

Comments
 (0)