Skip to content

Commit a94d342

Browse files
Peter ZijlstraIngo Molnar
Peter Zijlstra
authored and
Ingo Molnar
committed
tools/perf: Add required memory barriers
To match patch bf378d3 ("perf: Fix perf ring buffer memory ordering") change userspace to also adhere to the ordering outlined. Signed-off-by: Peter Zijlstra <peterz@infradead.org> Cc: Michael Neuling <mikey@neuling.org> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: james.hogan@imgtec.com Cc: Vince Weaver <vince@deater.net> Cc: Victor Kaplansky <VICTORK@il.ibm.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Anton Blanchard <anton@samba.org> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> Cc: Michael Ellerman <michael@ellerman.id.au> Link: http://lkml.kernel.org/r/20131030104246.GH16117@laptop.programming.kicks-ass.net Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent 0a19684 commit a94d342

File tree

3 files changed

+49
-16
lines changed

3 files changed

+49
-16
lines changed

‎tools/perf/perf.h

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#include <asm/unistd.h>
55

66
#if defined(__i386__)
7+
#define mb() asm volatile("lock; addl $0,0(%%esp)" ::: "memory")
8+
#define wmb() asm volatile("lock; addl $0,0(%%esp)" ::: "memory")
79
#define rmb() asm volatile("lock; addl $0,0(%%esp)" ::: "memory")
810
#define cpu_relax() asm volatile("rep; nop" ::: "memory");
911
#define CPUINFO_PROC "model name"
@@ -13,6 +15,8 @@
1315
#endif
1416

1517
#if defined(__x86_64__)
18+
#define mb() asm volatile("mfence" ::: "memory")
19+
#define wmb() asm volatile("sfence" ::: "memory")
1620
#define rmb() asm volatile("lfence" ::: "memory")
1721
#define cpu_relax() asm volatile("rep; nop" ::: "memory");
1822
#define CPUINFO_PROC "model name"
@@ -23,45 +27,61 @@
2327

2428
#ifdef __powerpc__
2529
#include "../../arch/powerpc/include/uapi/asm/unistd.h"
30+
#define mb() asm volatile ("sync" ::: "memory")
31+
#define wmb() asm volatile ("sync" ::: "memory")
2632
#define rmb() asm volatile ("sync" ::: "memory")
27-
#define cpu_relax() asm volatile ("" ::: "memory");
2833
#define CPUINFO_PROC "cpu"
2934
#endif
3035

3136
#ifdef __s390__
37+
#define mb() asm volatile("bcr 15,0" ::: "memory")
38+
#define wmb() asm volatile("bcr 15,0" ::: "memory")
3239
#define rmb() asm volatile("bcr 15,0" ::: "memory")
33-
#define cpu_relax() asm volatile("" ::: "memory");
3440
#endif
3541

3642
#ifdef __sh__
3743
#if defined(__SH4A__) || defined(__SH5__)
44+
# define mb() asm volatile("synco" ::: "memory")
45+
# define wmb() asm volatile("synco" ::: "memory")
3846
# define rmb() asm volatile("synco" ::: "memory")
3947
#else
48+
# define mb() asm volatile("" ::: "memory")
49+
# define wmb() asm volatile("" ::: "memory")
4050
# define rmb() asm volatile("" ::: "memory")
4151
#endif
42-
#define cpu_relax() asm volatile("" ::: "memory")
4352
#define CPUINFO_PROC "cpu type"
4453
#endif
4554

4655
#ifdef __hppa__
56+
#define mb() asm volatile("" ::: "memory")
57+
#define wmb() asm volatile("" ::: "memory")
4758
#define rmb() asm volatile("" ::: "memory")
48-
#define cpu_relax() asm volatile("" ::: "memory");
4959
#define CPUINFO_PROC "cpu"
5060
#endif
5161

5262
#ifdef __sparc__
63+
#ifdef __LP64__
64+
#define mb() asm volatile("ba,pt %%xcc, 1f\n" \
65+
"membar #StoreLoad\n" \
66+
"1:\n":::"memory")
67+
#else
68+
#define mb() asm volatile("":::"memory")
69+
#endif
70+
#define wmb() asm volatile("":::"memory")
5371
#define rmb() asm volatile("":::"memory")
54-
#define cpu_relax() asm volatile("":::"memory")
5572
#define CPUINFO_PROC "cpu"
5673
#endif
5774

5875
#ifdef __alpha__
76+
#define mb() asm volatile("mb" ::: "memory")
77+
#define wmb() asm volatile("wmb" ::: "memory")
5978
#define rmb() asm volatile("mb" ::: "memory")
60-
#define cpu_relax() asm volatile("" ::: "memory")
6179
#define CPUINFO_PROC "cpu model"
6280
#endif
6381

6482
#ifdef __ia64__
83+
#define mb() asm volatile ("mf" ::: "memory")
84+
#define wmb() asm volatile ("mf" ::: "memory")
6585
#define rmb() asm volatile ("mf" ::: "memory")
6686
#define cpu_relax() asm volatile ("hint @pause" ::: "memory")
6787
#define CPUINFO_PROC "model name"
@@ -72,40 +92,55 @@
7292
* Use the __kuser_memory_barrier helper in the CPU helper page. See
7393
* arch/arm/kernel/entry-armv.S in the kernel source for details.
7494
*/
95+
#define mb() ((void(*)(void))0xffff0fa0)()
96+
#define wmb() ((void(*)(void))0xffff0fa0)()
7597
#define rmb() ((void(*)(void))0xffff0fa0)()
76-
#define cpu_relax() asm volatile("":::"memory")
7798
#define CPUINFO_PROC "Processor"
7899
#endif
79100

80101
#ifdef __aarch64__
81-
#define rmb() asm volatile("dmb ld" ::: "memory")
102+
#define mb() asm volatile("dmb ish" ::: "memory")
103+
#define wmb() asm volatile("dmb ishld" ::: "memory")
104+
#define rmb() asm volatile("dmb ishst" ::: "memory")
82105
#define cpu_relax() asm volatile("yield" ::: "memory")
83106
#endif
84107

85108
#ifdef __mips__
86-
#define rmb() asm volatile( \
109+
#define mb() asm volatile( \
87110
".set mips2\n\t" \
88111
"sync\n\t" \
89112
".set mips0" \
90113
: /* no output */ \
91114
: /* no input */ \
92115
: "memory")
93-
#define cpu_relax() asm volatile("" ::: "memory")
116+
#define wmb() mb()
117+
#define rmb() mb()
94118
#define CPUINFO_PROC "cpu model"
95119
#endif
96120

97121
#ifdef __arc__
122+
#define mb() asm volatile("" ::: "memory")
123+
#define wmb() asm volatile("" ::: "memory")
98124
#define rmb() asm volatile("" ::: "memory")
99-
#define cpu_relax() rmb()
100125
#define CPUINFO_PROC "Processor"
101126
#endif
102127

103128
#ifdef __metag__
129+
#define mb() asm volatile("" ::: "memory")
130+
#define wmb() asm volatile("" ::: "memory")
104131
#define rmb() asm volatile("" ::: "memory")
105-
#define cpu_relax() asm volatile("" ::: "memory")
106132
#define CPUINFO_PROC "CPU"
107133
#endif
108134

135+
#define barrier() asm volatile ("" ::: "memory")
136+
137+
#ifndef cpu_relax
138+
#define cpu_relax() barrier()
139+
#endif
140+
141+
#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
142+
143+
109144
#include <time.h>
110145
#include <unistd.h>
111146
#include <sys/types.h>

‎tools/perf/tests/rdpmc.c

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@
99

1010
#if defined(__x86_64__) || defined(__i386__)
1111

12-
#define barrier() asm volatile("" ::: "memory")
13-
1412
static u64 rdpmc(unsigned int counter)
1513
{
1614
unsigned int low, high;

‎tools/perf/util/evlist.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ int perf_evlist__strerror_open(struct perf_evlist *evlist, int err, char *buf, s
177177
static inline unsigned int perf_mmap__read_head(struct perf_mmap *mm)
178178
{
179179
struct perf_event_mmap_page *pc = mm->base;
180-
int head = pc->data_head;
180+
int head = ACCESS_ONCE(pc->data_head);
181181
rmb();
182182
return head;
183183
}
@@ -190,7 +190,7 @@ static inline void perf_mmap__write_tail(struct perf_mmap *md,
190190
/*
191191
* ensure all reads are done before we write the tail out.
192192
*/
193-
/* mb(); */
193+
mb();
194194
pc->data_tail = tail;
195195
}
196196

0 commit comments

Comments
 (0)