@@ -44,6 +44,221 @@ static inline unsigned long __my_cpu_offset(void)
44
44
45
45
#endif /* CONFIG_SMP */
46
46
47
+ #define PERCPU_OP (op , asm_op ) \
48
+ static inline unsigned long __percpu_##op(void *ptr, \
49
+ unsigned long val, int size) \
50
+ { \
51
+ unsigned long loop, ret; \
52
+ \
53
+ switch (size) { \
54
+ case 1: \
55
+ do { \
56
+ asm ("//__per_cpu_" #op "_1\n" \
57
+ "ldxrb %w[ret], %[ptr]\n" \
58
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
59
+ "stxrb %w[loop], %w[ret], %[ptr]\n" \
60
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
61
+ [ptr] "+Q"(*(u8 *)ptr) \
62
+ : [val] "Ir" (val)); \
63
+ } while (loop); \
64
+ break; \
65
+ case 2: \
66
+ do { \
67
+ asm ("//__per_cpu_" #op "_2\n" \
68
+ "ldxrh %w[ret], %[ptr]\n" \
69
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
70
+ "stxrh %w[loop], %w[ret], %[ptr]\n" \
71
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
72
+ [ptr] "+Q"(*(u16 *)ptr) \
73
+ : [val] "Ir" (val)); \
74
+ } while (loop); \
75
+ break; \
76
+ case 4: \
77
+ do { \
78
+ asm ("//__per_cpu_" #op "_4\n" \
79
+ "ldxr %w[ret], %[ptr]\n" \
80
+ #asm_op " %w[ret], %w[ret], %w[val]\n" \
81
+ "stxr %w[loop], %w[ret], %[ptr]\n" \
82
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
83
+ [ptr] "+Q"(*(u32 *)ptr) \
84
+ : [val] "Ir" (val)); \
85
+ } while (loop); \
86
+ break; \
87
+ case 8: \
88
+ do { \
89
+ asm ("//__per_cpu_" #op "_8\n" \
90
+ "ldxr %[ret], %[ptr]\n" \
91
+ #asm_op " %[ret], %[ret], %[val]\n" \
92
+ "stxr %w[loop], %[ret], %[ptr]\n" \
93
+ : [loop] "=&r" (loop), [ret] "=&r" (ret), \
94
+ [ptr] "+Q"(*(u64 *)ptr) \
95
+ : [val] "Ir" (val)); \
96
+ } while (loop); \
97
+ break; \
98
+ default: \
99
+ BUILD_BUG(); \
100
+ } \
101
+ \
102
+ return ret; \
103
+ }
104
+
105
+ PERCPU_OP (add , add )
106
+ PERCPU_OP (and , and )
107
+ PERCPU_OP (or , orr )
108
+ #undef PERCPU_OP
109
+
110
+ static inline unsigned long __percpu_read (void * ptr , int size )
111
+ {
112
+ unsigned long ret ;
113
+
114
+ switch (size ) {
115
+ case 1 :
116
+ ret = ACCESS_ONCE (* (u8 * )ptr );
117
+ break ;
118
+ case 2 :
119
+ ret = ACCESS_ONCE (* (u16 * )ptr );
120
+ break ;
121
+ case 4 :
122
+ ret = ACCESS_ONCE (* (u32 * )ptr );
123
+ break ;
124
+ case 8 :
125
+ ret = ACCESS_ONCE (* (u64 * )ptr );
126
+ break ;
127
+ default :
128
+ BUILD_BUG ();
129
+ }
130
+
131
+ return ret ;
132
+ }
133
+
134
+ static inline void __percpu_write (void * ptr , unsigned long val , int size )
135
+ {
136
+ switch (size ) {
137
+ case 1 :
138
+ ACCESS_ONCE (* (u8 * )ptr ) = (u8 )val ;
139
+ break ;
140
+ case 2 :
141
+ ACCESS_ONCE (* (u16 * )ptr ) = (u16 )val ;
142
+ break ;
143
+ case 4 :
144
+ ACCESS_ONCE (* (u32 * )ptr ) = (u32 )val ;
145
+ break ;
146
+ case 8 :
147
+ ACCESS_ONCE (* (u64 * )ptr ) = (u64 )val ;
148
+ break ;
149
+ default :
150
+ BUILD_BUG ();
151
+ }
152
+ }
153
+
154
+ static inline unsigned long __percpu_xchg (void * ptr , unsigned long val ,
155
+ int size )
156
+ {
157
+ unsigned long ret , loop ;
158
+
159
+ switch (size ) {
160
+ case 1 :
161
+ do {
162
+ asm ("//__percpu_xchg_1\n"
163
+ "ldxrb %w[ret], %[ptr]\n"
164
+ "stxrb %w[loop], %w[val], %[ptr]\n"
165
+ : [loop ] "=&r" (loop ), [ret ] "=&r" (ret ),
166
+ [ptr ] "+Q" (* (u8 * )ptr )
167
+ : [val ] "r" (val ));
168
+ } while (loop );
169
+ break ;
170
+ case 2 :
171
+ do {
172
+ asm ("//__percpu_xchg_2\n"
173
+ "ldxrh %w[ret], %[ptr]\n"
174
+ "stxrh %w[loop], %w[val], %[ptr]\n"
175
+ : [loop ] "=&r" (loop ), [ret ] "=&r" (ret ),
176
+ [ptr ] "+Q" (* (u16 * )ptr )
177
+ : [val ] "r" (val ));
178
+ } while (loop );
179
+ break ;
180
+ case 4 :
181
+ do {
182
+ asm ("//__percpu_xchg_4\n"
183
+ "ldxr %w[ret], %[ptr]\n"
184
+ "stxr %w[loop], %w[val], %[ptr]\n"
185
+ : [loop ] "=&r" (loop ), [ret ] "=&r" (ret ),
186
+ [ptr ] "+Q" (* (u32 * )ptr )
187
+ : [val ] "r" (val ));
188
+ } while (loop );
189
+ break ;
190
+ case 8 :
191
+ do {
192
+ asm ("//__percpu_xchg_8\n"
193
+ "ldxr %[ret], %[ptr]\n"
194
+ "stxr %w[loop], %[val], %[ptr]\n"
195
+ : [loop ] "=&r" (loop ), [ret ] "=&r" (ret ),
196
+ [ptr ] "+Q" (* (u64 * )ptr )
197
+ : [val ] "r" (val ));
198
+ } while (loop );
199
+ break ;
200
+ default :
201
+ BUILD_BUG ();
202
+ }
203
+
204
+ return ret ;
205
+ }
206
+
207
+ #define _percpu_add (pcp , val ) \
208
+ __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
209
+
210
+ #define _percpu_add_return (pcp , val ) (typeof(pcp)) (_percpu_add(pcp, val))
211
+
212
+ #define _percpu_and (pcp , val ) \
213
+ __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
214
+
215
+ #define _percpu_or (pcp , val ) \
216
+ __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
217
+
218
+ #define _percpu_read (pcp ) (typeof(pcp)) \
219
+ (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
220
+
221
+ #define _percpu_write (pcp , val ) \
222
+ __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
223
+
224
+ #define _percpu_xchg (pcp , val ) (typeof(pcp)) \
225
+ (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
226
+
227
+ #define this_cpu_add_1 (pcp , val ) _percpu_add(pcp, val)
228
+ #define this_cpu_add_2 (pcp , val ) _percpu_add(pcp, val)
229
+ #define this_cpu_add_4 (pcp , val ) _percpu_add(pcp, val)
230
+ #define this_cpu_add_8 (pcp , val ) _percpu_add(pcp, val)
231
+
232
+ #define this_cpu_add_return_1 (pcp , val ) _percpu_add_return(pcp, val)
233
+ #define this_cpu_add_return_2 (pcp , val ) _percpu_add_return(pcp, val)
234
+ #define this_cpu_add_return_4 (pcp , val ) _percpu_add_return(pcp, val)
235
+ #define this_cpu_add_return_8 (pcp , val ) _percpu_add_return(pcp, val)
236
+
237
+ #define this_cpu_and_1 (pcp , val ) _percpu_and(pcp, val)
238
+ #define this_cpu_and_2 (pcp , val ) _percpu_and(pcp, val)
239
+ #define this_cpu_and_4 (pcp , val ) _percpu_and(pcp, val)
240
+ #define this_cpu_and_8 (pcp , val ) _percpu_and(pcp, val)
241
+
242
+ #define this_cpu_or_1 (pcp , val ) _percpu_or(pcp, val)
243
+ #define this_cpu_or_2 (pcp , val ) _percpu_or(pcp, val)
244
+ #define this_cpu_or_4 (pcp , val ) _percpu_or(pcp, val)
245
+ #define this_cpu_or_8 (pcp , val ) _percpu_or(pcp, val)
246
+
247
+ #define this_cpu_read_1 (pcp ) _percpu_read(pcp)
248
+ #define this_cpu_read_2 (pcp ) _percpu_read(pcp)
249
+ #define this_cpu_read_4 (pcp ) _percpu_read(pcp)
250
+ #define this_cpu_read_8 (pcp ) _percpu_read(pcp)
251
+
252
+ #define this_cpu_write_1 (pcp , val ) _percpu_write(pcp, val)
253
+ #define this_cpu_write_2 (pcp , val ) _percpu_write(pcp, val)
254
+ #define this_cpu_write_4 (pcp , val ) _percpu_write(pcp, val)
255
+ #define this_cpu_write_8 (pcp , val ) _percpu_write(pcp, val)
256
+
257
+ #define this_cpu_xchg_1 (pcp , val ) _percpu_xchg(pcp, val)
258
+ #define this_cpu_xchg_2 (pcp , val ) _percpu_xchg(pcp, val)
259
+ #define this_cpu_xchg_4 (pcp , val ) _percpu_xchg(pcp, val)
260
+ #define this_cpu_xchg_8 (pcp , val ) _percpu_xchg(pcp, val)
261
+
47
262
#include <asm-generic/percpu.h>
48
263
49
264
#endif /* __ASM_PERCPU_H */
0 commit comments