Skip to content

Commit bae8f56

Browse files
author
Martin Schwidefsky
committed
s390/spinlock,rwlock: always to a load-and-test first
In case a lock is contended it is better to do a load-and-test first before trying to get the lock with compare-and-swap. This helps to avoid unnecessary cache invalidations of the cacheline for the lock if the CPU has to wait for the lock. For an uncontended lock doing the compare-and-swap directly is a bit better, if the CPU does not have the cacheline in its cache yet the compare-and-swap will get it read-write immediately while a load-and-test would get it read-only first. Always to the load-and-test first to avoid the cacheline invalidations for the contended case outweight the potential read-only to read-write cacheline upgrade for the uncontended case. Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
1 parent 2bf29df commit bae8f56

File tree

2 files changed

+46
-33
lines changed

2 files changed

+46
-33
lines changed

‎arch/s390/include/asm/spinlock.h

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ static inline int arch_spin_is_locked(arch_spinlock_t *lp)
5959

6060
static inline int arch_spin_trylock_once(arch_spinlock_t *lp)
6161
{
62-
return _raw_compare_and_swap(&lp->lock, 0, SPINLOCK_LOCKVAL);
62+
barrier();
63+
return likely(arch_spin_value_unlocked(*lp) &&
64+
_raw_compare_and_swap(&lp->lock, 0, SPINLOCK_LOCKVAL));
6365
}
6466

6567
static inline int arch_spin_tryrelease_once(arch_spinlock_t *lp)
@@ -69,20 +71,20 @@ static inline int arch_spin_tryrelease_once(arch_spinlock_t *lp)
6971

7072
static inline void arch_spin_lock(arch_spinlock_t *lp)
7173
{
72-
if (unlikely(!arch_spin_trylock_once(lp)))
74+
if (!arch_spin_trylock_once(lp))
7375
arch_spin_lock_wait(lp);
7476
}
7577

7678
static inline void arch_spin_lock_flags(arch_spinlock_t *lp,
7779
unsigned long flags)
7880
{
79-
if (unlikely(!arch_spin_trylock_once(lp)))
81+
if (!arch_spin_trylock_once(lp))
8082
arch_spin_lock_wait_flags(lp, flags);
8183
}
8284

8385
static inline int arch_spin_trylock(arch_spinlock_t *lp)
8486
{
85-
if (unlikely(!arch_spin_trylock_once(lp)))
87+
if (!arch_spin_trylock_once(lp))
8688
return arch_spin_trylock_retry(lp);
8789
return 1;
8890
}
@@ -128,19 +130,29 @@ extern void _raw_write_lock_wait(arch_rwlock_t *lp);
128130
extern void _raw_write_lock_wait_flags(arch_rwlock_t *lp, unsigned long flags);
129131
extern int _raw_write_trylock_retry(arch_rwlock_t *lp);
130132

133+
static inline int arch_read_trylock_once(arch_rwlock_t *rw)
134+
{
135+
unsigned int old = ACCESS_ONCE(rw->lock);
136+
return likely((int) old >= 0 &&
137+
_raw_compare_and_swap(&rw->lock, old, old + 1));
138+
}
139+
140+
static inline int arch_write_trylock_once(arch_rwlock_t *rw)
141+
{
142+
unsigned int old = ACCESS_ONCE(rw->lock);
143+
return likely(old == 0 &&
144+
_raw_compare_and_swap(&rw->lock, 0, 0x80000000));
145+
}
146+
131147
static inline void arch_read_lock(arch_rwlock_t *rw)
132148
{
133-
unsigned int old;
134-
old = rw->lock & 0x7fffffffU;
135-
if (!_raw_compare_and_swap(&rw->lock, old, old + 1))
149+
if (!arch_read_trylock_once(rw))
136150
_raw_read_lock_wait(rw);
137151
}
138152

139153
static inline void arch_read_lock_flags(arch_rwlock_t *rw, unsigned long flags)
140154
{
141-
unsigned int old;
142-
old = rw->lock & 0x7fffffffU;
143-
if (!_raw_compare_and_swap(&rw->lock, old, old + 1))
155+
if (!arch_read_trylock_once(rw))
144156
_raw_read_lock_wait_flags(rw, flags);
145157
}
146158

@@ -155,13 +167,13 @@ static inline void arch_read_unlock(arch_rwlock_t *rw)
155167

156168
static inline void arch_write_lock(arch_rwlock_t *rw)
157169
{
158-
if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
170+
if (!arch_write_trylock_once(rw))
159171
_raw_write_lock_wait(rw);
160172
}
161173

162174
static inline void arch_write_lock_flags(arch_rwlock_t *rw, unsigned long flags)
163175
{
164-
if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
176+
if (!arch_write_trylock_once(rw))
165177
_raw_write_lock_wait_flags(rw, flags);
166178
}
167179

@@ -172,18 +184,16 @@ static inline void arch_write_unlock(arch_rwlock_t *rw)
172184

173185
static inline int arch_read_trylock(arch_rwlock_t *rw)
174186
{
175-
unsigned int old;
176-
old = rw->lock & 0x7fffffffU;
177-
if (likely(_raw_compare_and_swap(&rw->lock, old, old + 1)))
178-
return 1;
179-
return _raw_read_trylock_retry(rw);
187+
if (!arch_read_trylock_once(rw))
188+
return _raw_read_trylock_retry(rw);
189+
return 1;
180190
}
181191

182192
static inline int arch_write_trylock(arch_rwlock_t *rw)
183193
{
184-
if (likely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
185-
return 1;
186-
return _raw_write_trylock_retry(rw);
194+
if (!arch_write_trylock_once(rw))
195+
return _raw_write_trylock_retry(rw);
196+
return 1;
187197
}
188198

189199
#define arch_read_relax(lock) cpu_relax()

‎arch/s390/lib/spinlock.c

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,9 @@ int arch_spin_trylock_retry(arch_spinlock_t *lp)
100100
{
101101
int count;
102102

103-
for (count = spin_retry; count > 0; count--) {
104-
if (arch_spin_is_locked(lp))
105-
continue;
103+
for (count = spin_retry; count > 0; count--)
106104
if (arch_spin_trylock_once(lp))
107105
return 1;
108-
}
109106
return 0;
110107
}
111108
EXPORT_SYMBOL(arch_spin_trylock_retry);
@@ -120,9 +117,9 @@ void _raw_read_lock_wait(arch_rwlock_t *rw)
120117
smp_yield();
121118
count = spin_retry;
122119
}
123-
if (!arch_read_can_lock(rw))
120+
old = ACCESS_ONCE(rw->lock);
121+
if ((int) old < 0)
124122
continue;
125-
old = rw->lock & 0x7fffffffU;
126123
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
127124
return;
128125
}
@@ -140,9 +137,9 @@ void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
140137
smp_yield();
141138
count = spin_retry;
142139
}
143-
if (!arch_read_can_lock(rw))
140+
old = ACCESS_ONCE(rw->lock);
141+
if ((int) old < 0)
144142
continue;
145-
old = rw->lock & 0x7fffffffU;
146143
local_irq_disable();
147144
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
148145
return;
@@ -156,9 +153,9 @@ int _raw_read_trylock_retry(arch_rwlock_t *rw)
156153
int count = spin_retry;
157154

158155
while (count-- > 0) {
159-
if (!arch_read_can_lock(rw))
156+
old = ACCESS_ONCE(rw->lock);
157+
if ((int) old < 0)
160158
continue;
161-
old = rw->lock & 0x7fffffffU;
162159
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
163160
return 1;
164161
}
@@ -168,14 +165,16 @@ EXPORT_SYMBOL(_raw_read_trylock_retry);
168165

169166
void _raw_write_lock_wait(arch_rwlock_t *rw)
170167
{
168+
unsigned int old;
171169
int count = spin_retry;
172170

173171
while (1) {
174172
if (count-- <= 0) {
175173
smp_yield();
176174
count = spin_retry;
177175
}
178-
if (!arch_write_can_lock(rw))
176+
old = ACCESS_ONCE(rw->lock);
177+
if (old)
179178
continue;
180179
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
181180
return;
@@ -185,6 +184,7 @@ EXPORT_SYMBOL(_raw_write_lock_wait);
185184

186185
void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
187186
{
187+
unsigned int old;
188188
int count = spin_retry;
189189

190190
local_irq_restore(flags);
@@ -193,7 +193,8 @@ void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
193193
smp_yield();
194194
count = spin_retry;
195195
}
196-
if (!arch_write_can_lock(rw))
196+
old = ACCESS_ONCE(rw->lock);
197+
if (old)
197198
continue;
198199
local_irq_disable();
199200
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
@@ -204,10 +205,12 @@ EXPORT_SYMBOL(_raw_write_lock_wait_flags);
204205

205206
int _raw_write_trylock_retry(arch_rwlock_t *rw)
206207
{
208+
unsigned int old;
207209
int count = spin_retry;
208210

209211
while (count-- > 0) {
210-
if (!arch_write_can_lock(rw))
212+
old = ACCESS_ONCE(rw->lock);
213+
if (old)
211214
continue;
212215
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
213216
return 1;

0 commit comments

Comments
 (0)