Skip to content

Commit a200dcb

Browse files
torvaldstorvalds
committed
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
Pull virtio barrier rework+fixes from Michael Tsirkin: "This adds a new kind of barrier, and reworks virtio and xen to use it. Plus some fixes here and there" * tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (44 commits) checkpatch: add virt barriers checkpatch: check for __smp outside barrier.h checkpatch.pl: add missing memory barriers virtio: make find_vqs() checkpatch.pl-friendly virtio_balloon: fix race between migration and ballooning virtio_balloon: fix race by fill and leak s390: more efficient smp barriers s390: use generic memory barriers xen/events: use virt_xxx barriers xen/io: use virt_xxx barriers xenbus: use virt_xxx barriers virtio_ring: use virt_store_mb sh: move xchg_cmpxchg to a header by itself sh: support 1 and 2 byte xchg virtio_ring: update weak barriers to use virt_xxx Revert "virtio_ring: Update weak barriers to use dma_wmb/rmb" asm-generic: implement virt_xxx memory barriers x86: define __smp_xxx xtensa: define __smp_xxx tile: define __smp_xxx ...
2 parents d05d82f + 43e361f commit a200dcb

File tree

44 files changed

+401
-319
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+401
-319
lines changed

‎Documentation/memory-barriers.txt

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1655,17 +1655,18 @@ macro is a good place to start looking.
16551655
SMP memory barriers are reduced to compiler barriers on uniprocessor compiled
16561656
systems because it is assumed that a CPU will appear to be self-consistent,
16571657
and will order overlapping accesses correctly with respect to itself.
1658+
However, see the subsection on "Virtual Machine Guests" below.
16581659

16591660
[!] Note that SMP memory barriers _must_ be used to control the ordering of
16601661
references to shared memory on SMP systems, though the use of locking instead
16611662
is sufficient.
16621663

16631664
Mandatory barriers should not be used to control SMP effects, since mandatory
1664-
barriers unnecessarily impose overhead on UP systems. They may, however, be
1665-
used to control MMIO effects on accesses through relaxed memory I/O windows.
1666-
These are required even on non-SMP systems as they affect the order in which
1667-
memory operations appear to a device by prohibiting both the compiler and the
1668-
CPU from reordering them.
1665+
barriers impose unnecessary overhead on both SMP and UP systems. They may,
1666+
however, be used to control MMIO effects on accesses through relaxed memory I/O
1667+
windows. These barriers are required even on non-SMP systems as they affect
1668+
the order in which memory operations appear to a device by prohibiting both the
1669+
compiler and the CPU from reordering them.
16691670

16701671

16711672
There are some more advanced barrier functions:
@@ -2948,6 +2949,23 @@ The Alpha defines the Linux kernel's memory barrier model.
29482949

29492950
See the subsection on "Cache Coherency" above.
29502951

2952+
VIRTUAL MACHINE GUESTS
2953+
-------------------
2954+
2955+
Guests running within virtual machines might be affected by SMP effects even if
2956+
the guest itself is compiled without SMP support. This is an artifact of
2957+
interfacing with an SMP host while running an UP kernel. Using mandatory
2958+
barriers for this use-case would be possible but is often suboptimal.
2959+
2960+
To handle this case optimally, low-level virt_mb() etc macros are available.
2961+
These have the same effect as smp_mb() etc when SMP is enabled, but generate
2962+
identical code for SMP and non-SMP systems. For example, virtual machine guests
2963+
should use virt_mb() rather than smp_mb() when synchronizing against a
2964+
(possibly SMP) host.
2965+
2966+
These are equivalent to smp_mb() etc counterparts in all other respects,
2967+
in particular, they do not control MMIO effects: to control
2968+
MMIO effects, use mandatory barriers.
29512969

29522970
============
29532971
EXAMPLE USES

‎arch/arm/include/asm/barrier.h

Lines changed: 4 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -60,38 +60,11 @@ extern void arm_heavy_mb(void);
6060
#define dma_wmb() barrier()
6161
#endif
6262

63-
#ifndef CONFIG_SMP
64-
#define smp_mb() barrier()
65-
#define smp_rmb() barrier()
66-
#define smp_wmb() barrier()
67-
#else
68-
#define smp_mb() dmb(ish)
69-
#define smp_rmb() smp_mb()
70-
#define smp_wmb() dmb(ishst)
71-
#endif
72-
73-
#define smp_store_release(p, v) \
74-
do { \
75-
compiletime_assert_atomic_type(*p); \
76-
smp_mb(); \
77-
WRITE_ONCE(*p, v); \
78-
} while (0)
79-
80-
#define smp_load_acquire(p) \
81-
({ \
82-
typeof(*p) ___p1 = READ_ONCE(*p); \
83-
compiletime_assert_atomic_type(*p); \
84-
smp_mb(); \
85-
___p1; \
86-
})
87-
88-
#define read_barrier_depends() do { } while(0)
89-
#define smp_read_barrier_depends() do { } while(0)
90-
91-
#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
63+
#define __smp_mb() dmb(ish)
64+
#define __smp_rmb() __smp_mb()
65+
#define __smp_wmb() dmb(ishst)
9266

93-
#define smp_mb__before_atomic() smp_mb()
94-
#define smp_mb__after_atomic() smp_mb()
67+
#include <asm-generic/barrier.h>
9568

9669
#endif /* !__ASSEMBLY__ */
9770
#endif /* __ASM_BARRIER_H */

‎arch/arm64/include/asm/barrier.h

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,11 @@
3535
#define dma_rmb() dmb(oshld)
3636
#define dma_wmb() dmb(oshst)
3737

38-
#define smp_mb() dmb(ish)
39-
#define smp_rmb() dmb(ishld)
40-
#define smp_wmb() dmb(ishst)
38+
#define __smp_mb() dmb(ish)
39+
#define __smp_rmb() dmb(ishld)
40+
#define __smp_wmb() dmb(ishst)
4141

42-
#define smp_store_release(p, v) \
42+
#define __smp_store_release(p, v) \
4343
do { \
4444
compiletime_assert_atomic_type(*p); \
4545
switch (sizeof(*p)) { \
@@ -62,7 +62,7 @@ do { \
6262
} \
6363
} while (0)
6464

65-
#define smp_load_acquire(p) \
65+
#define __smp_load_acquire(p) \
6666
({ \
6767
union { typeof(*p) __val; char __c[1]; } __u; \
6868
compiletime_assert_atomic_type(*p); \
@@ -91,14 +91,7 @@ do { \
9191
__u.__val; \
9292
})
9393

94-
#define read_barrier_depends() do { } while(0)
95-
#define smp_read_barrier_depends() do { } while(0)
96-
97-
#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
98-
#define nop() asm volatile("nop");
99-
100-
#define smp_mb__before_atomic() smp_mb()
101-
#define smp_mb__after_atomic() smp_mb()
94+
#include <asm-generic/barrier.h>
10295

10396
#endif /* __ASSEMBLY__ */
10497

‎arch/blackfin/include/asm/barrier.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,8 @@
7878

7979
#endif /* !CONFIG_SMP */
8080

81-
#define smp_mb__before_atomic() barrier()
82-
#define smp_mb__after_atomic() barrier()
81+
#define __smp_mb__before_atomic() barrier()
82+
#define __smp_mb__after_atomic() barrier()
8383

8484
#include <asm-generic/barrier.h>
8585

‎arch/ia64/include/asm/barrier.h

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -42,47 +42,37 @@
4242
#define dma_rmb() mb()
4343
#define dma_wmb() mb()
4444

45-
#ifdef CONFIG_SMP
46-
# define smp_mb() mb()
47-
#else
48-
# define smp_mb() barrier()
49-
#endif
45+
# define __smp_mb() mb()
5046

51-
#define smp_rmb() smp_mb()
52-
#define smp_wmb() smp_mb()
53-
54-
#define read_barrier_depends() do { } while (0)
55-
#define smp_read_barrier_depends() do { } while (0)
56-
57-
#define smp_mb__before_atomic() barrier()
58-
#define smp_mb__after_atomic() barrier()
47+
#define __smp_mb__before_atomic() barrier()
48+
#define __smp_mb__after_atomic() barrier()
5949

6050
/*
6151
* IA64 GCC turns volatile stores into st.rel and volatile loads into ld.acq no
6252
* need for asm trickery!
6353
*/
6454

65-
#define smp_store_release(p, v) \
55+
#define __smp_store_release(p, v) \
6656
do { \
6757
compiletime_assert_atomic_type(*p); \
6858
barrier(); \
6959
WRITE_ONCE(*p, v); \
7060
} while (0)
7161

72-
#define smp_load_acquire(p) \
62+
#define __smp_load_acquire(p) \
7363
({ \
7464
typeof(*p) ___p1 = READ_ONCE(*p); \
7565
compiletime_assert_atomic_type(*p); \
7666
barrier(); \
7767
___p1; \
7868
})
7969

80-
#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
81-
8270
/*
8371
* The group barrier in front of the rsm & ssm are necessary to ensure
8472
* that none of the previous instructions in the same group are
8573
* affected by the rsm/ssm.
8674
*/
8775

76+
#include <asm-generic/barrier.h>
77+
8878
#endif /* _ASM_IA64_BARRIER_H */

‎arch/ia64/kernel/iosapic.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ set_rte (unsigned int gsi, unsigned int irq, unsigned int dest, int mask)
256256
}
257257

258258
static void
259-
nop (struct irq_data *data)
259+
iosapic_nop (struct irq_data *data)
260260
{
261261
/* do nothing... */
262262
}
@@ -415,7 +415,7 @@ iosapic_unmask_level_irq (struct irq_data *data)
415415
#define iosapic_shutdown_level_irq mask_irq
416416
#define iosapic_enable_level_irq unmask_irq
417417
#define iosapic_disable_level_irq mask_irq
418-
#define iosapic_ack_level_irq nop
418+
#define iosapic_ack_level_irq iosapic_nop
419419

420420
static struct irq_chip irq_type_iosapic_level = {
421421
.name = "IO-SAPIC-level",
@@ -453,7 +453,7 @@ iosapic_ack_edge_irq (struct irq_data *data)
453453
}
454454

455455
#define iosapic_enable_edge_irq unmask_irq
456-
#define iosapic_disable_edge_irq nop
456+
#define iosapic_disable_edge_irq iosapic_nop
457457

458458
static struct irq_chip irq_type_iosapic_edge = {
459459
.name = "IO-SAPIC-edge",

‎arch/metag/include/asm/barrier.h

Lines changed: 16 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,6 @@ static inline void wr_fence(void)
4444
#define rmb() barrier()
4545
#define wmb() mb()
4646

47-
#define dma_rmb() rmb()
48-
#define dma_wmb() wmb()
49-
50-
#ifndef CONFIG_SMP
51-
#define fence() do { } while (0)
52-
#define smp_mb() barrier()
53-
#define smp_rmb() barrier()
54-
#define smp_wmb() barrier()
55-
#else
56-
5747
#ifdef CONFIG_METAG_SMP_WRITE_REORDERING
5848
/*
5949
* Write to the atomic memory unlock system event register (command 0). This is
@@ -63,45 +53,32 @@ static inline void wr_fence(void)
6353
* incoherence). It is therefore ineffective if used after and on the same
6454
* thread as a write.
6555
*/
66-
static inline void fence(void)
56+
static inline void metag_fence(void)
6757
{
6858
volatile int *flushptr = (volatile int *) LINSYSEVENT_WR_ATOMIC_UNLOCK;
6959
barrier();
7060
*flushptr = 0;
7161
barrier();
7262
}
73-
#define smp_mb() fence()
74-
#define smp_rmb() fence()
75-
#define smp_wmb() barrier()
63+
#define __smp_mb() metag_fence()
64+
#define __smp_rmb() metag_fence()
65+
#define __smp_wmb() barrier()
7666
#else
77-
#define fence() do { } while (0)
78-
#define smp_mb() barrier()
79-
#define smp_rmb() barrier()
80-
#define smp_wmb() barrier()
81-
#endif
67+
#define metag_fence() do { } while (0)
68+
#define __smp_mb() barrier()
69+
#define __smp_rmb() barrier()
70+
#define __smp_wmb() barrier()
8271
#endif
8372

84-
#define read_barrier_depends() do { } while (0)
85-
#define smp_read_barrier_depends() do { } while (0)
86-
87-
#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
88-
89-
#define smp_store_release(p, v) \
90-
do { \
91-
compiletime_assert_atomic_type(*p); \
92-
smp_mb(); \
93-
WRITE_ONCE(*p, v); \
94-
} while (0)
73+
#ifdef CONFIG_SMP
74+
#define fence() metag_fence()
75+
#else
76+
#define fence() do { } while (0)
77+
#endif
9578

96-
#define smp_load_acquire(p) \
97-
({ \
98-
typeof(*p) ___p1 = READ_ONCE(*p); \
99-
compiletime_assert_atomic_type(*p); \
100-
smp_mb(); \
101-
___p1; \
102-
})
79+
#define __smp_mb__before_atomic() barrier()
80+
#define __smp_mb__after_atomic() barrier()
10381

104-
#define smp_mb__before_atomic() barrier()
105-
#define smp_mb__after_atomic() barrier()
82+
#include <asm-generic/barrier.h>
10683

10784
#endif /* _ASM_METAG_BARRIER_H */

‎arch/mips/include/asm/barrier.h

Lines changed: 16 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,6 @@
1010

1111
#include <asm/addrspace.h>
1212

13-
#define read_barrier_depends() do { } while(0)
14-
#define smp_read_barrier_depends() do { } while(0)
15-
1613
#ifdef CONFIG_CPU_HAS_SYNC
1714
#define __sync() \
1815
__asm__ __volatile__( \
@@ -87,23 +84,21 @@
8784

8885
#define wmb() fast_wmb()
8986
#define rmb() fast_rmb()
90-
#define dma_wmb() fast_wmb()
91-
#define dma_rmb() fast_rmb()
9287

93-
#if defined(CONFIG_WEAK_ORDERING) && defined(CONFIG_SMP)
88+
#if defined(CONFIG_WEAK_ORDERING)
9489
# ifdef CONFIG_CPU_CAVIUM_OCTEON
95-
# define smp_mb() __sync()
96-
# define smp_rmb() barrier()
97-
# define smp_wmb() __syncw()
90+
# define __smp_mb() __sync()
91+
# define __smp_rmb() barrier()
92+
# define __smp_wmb() __syncw()
9893
# else
99-
# define smp_mb() __asm__ __volatile__("sync" : : :"memory")
100-
# define smp_rmb() __asm__ __volatile__("sync" : : :"memory")
101-
# define smp_wmb() __asm__ __volatile__("sync" : : :"memory")
94+
# define __smp_mb() __asm__ __volatile__("sync" : : :"memory")
95+
# define __smp_rmb() __asm__ __volatile__("sync" : : :"memory")
96+
# define __smp_wmb() __asm__ __volatile__("sync" : : :"memory")
10297
# endif
10398
#else
104-
#define smp_mb() barrier()
105-
#define smp_rmb() barrier()
106-
#define smp_wmb() barrier()
99+
#define __smp_mb() barrier()
100+
#define __smp_rmb() barrier()
101+
#define __smp_wmb() barrier()
107102
#endif
108103

109104
#if defined(CONFIG_WEAK_REORDERING_BEYOND_LLSC) && defined(CONFIG_SMP)
@@ -112,39 +107,25 @@
112107
#define __WEAK_LLSC_MB " \n"
113108
#endif
114109

115-
#define smp_store_mb(var, value) \
116-
do { WRITE_ONCE(var, value); smp_mb(); } while (0)
117-
118110
#define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
119111

120112
#ifdef CONFIG_CPU_CAVIUM_OCTEON
121113
#define smp_mb__before_llsc() smp_wmb()
114+
#define __smp_mb__before_llsc() __smp_wmb()
122115
/* Cause previous writes to become visible on all CPUs as soon as possible */
123116
#define nudge_writes() __asm__ __volatile__(".set push\n\t" \
124117
".set arch=octeon\n\t" \
125118
"syncw\n\t" \
126119
".set pop" : : : "memory")
127120
#else
128121
#define smp_mb__before_llsc() smp_llsc_mb()
122+
#define __smp_mb__before_llsc() smp_llsc_mb()
129123
#define nudge_writes() mb()
130124
#endif
131125

132-
#define smp_store_release(p, v) \
133-
do { \
134-
compiletime_assert_atomic_type(*p); \
135-
smp_mb(); \
136-
WRITE_ONCE(*p, v); \
137-
} while (0)
138-
139-
#define smp_load_acquire(p) \
140-
({ \
141-
typeof(*p) ___p1 = READ_ONCE(*p); \
142-
compiletime_assert_atomic_type(*p); \
143-
smp_mb(); \
144-
___p1; \
145-
})
146-
147-
#define smp_mb__before_atomic() smp_mb__before_llsc()
148-
#define smp_mb__after_atomic() smp_llsc_mb()
126+
#define __smp_mb__before_atomic() __smp_mb__before_llsc()
127+
#define __smp_mb__after_atomic() smp_llsc_mb()
128+
129+
#include <asm-generic/barrier.h>
149130

150131
#endif /* __ASM_BARRIER_H */

0 commit comments

Comments
 (0)