Skip to content

Commit 3e71f01

Browse files
torvaldstorvalds
committed
Merge tag 'locking-core-2022-10-07' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar: - Disable preemption in rwsem_write_trylock()'s attempt to take the rwsem, to avoid RT tasks hogging the CPU, which managed to preempt this function after the owner has been cleared but before a new owner is set. Also add debug checks to enforce this. - Add __lockfunc to more slow path functions and add __sched to semaphore functions. - Mark spinlock APIs noinline when the respective CONFIG_INLINE_SPIN_* toggles are disabled, to reduce LTO text size. - Print more debug information when lockdep gets confused in look_up_lock_class(). - Improve header file abuse checks. - Misc cleanups * tag 'locking-core-2022-10-07' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/lockdep: Print more debug information - report name and key when look_up_lock_class() got confused locking: Add __sched to semaphore functions locking/rwsem: Disable preemption while trying for rwsem lock locking: Detect includes rwlock.h outside of spinlock.h locking: Add __lockfunc to slow path functions locking/spinlocks: Mark spinlocks noinline when inline spinlocks are disabled selftests: futex: Fix 'the the' typo in comment
2 parents 3871d93 + 76e64c7 commit 3e71f01

File tree

15 files changed

+70
-55
lines changed

15 files changed

+70
-55
lines changed

‎arch/x86/include/asm/qspinlock_paravirt.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,18 @@
1212
*/
1313
#ifdef CONFIG_64BIT
1414

15-
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
15+
__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
1616
#define __pv_queued_spin_unlock __pv_queued_spin_unlock
1717
#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
1818
#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
1919

2020
/*
2121
* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
2222
* which combines the registers saving trunk and the body of the following
23-
* C code:
23+
* C code. Note that it puts the code in the .spinlock.text section which
24+
* is equivalent to adding __lockfunc in the C code:
2425
*
25-
* void __pv_queued_spin_unlock(struct qspinlock *lock)
26+
* void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
2627
* {
2728
* u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
2829
*
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
3637
* rsi = lockval (second argument)
3738
* rdx = internal variable (set to 0)
3839
*/
39-
asm (".pushsection .text;"
40+
asm (".pushsection .spinlock.text;"
4041
".globl " PV_UNLOCK ";"
4142
".type " PV_UNLOCK ", @function;"
4243
".align 4,0x90;"
@@ -65,8 +66,8 @@ asm (".pushsection .text;"
6566

6667
#else /* CONFIG_64BIT */
6768

68-
extern void __pv_queued_spin_unlock(struct qspinlock *lock);
69-
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
69+
extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
70+
__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
7071

7172
#endif /* CONFIG_64BIT */
7273
#endif

‎include/linux/rwlock.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#ifndef __LINUX_RWLOCK_H
22
#define __LINUX_RWLOCK_H
33

4-
#ifndef __LINUX_SPINLOCK_H
4+
#ifndef __LINUX_INSIDE_SPINLOCK_H
55
# error "please don't include this file directly"
66
#endif
77

‎include/linux/spinlock.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
/* SPDX-License-Identifier: GPL-2.0 */
22
#ifndef __LINUX_SPINLOCK_H
33
#define __LINUX_SPINLOCK_H
4+
#define __LINUX_INSIDE_SPINLOCK_H
45

56
/*
67
* include/linux/spinlock.h - generic spinlock/rwlock declarations
@@ -492,4 +493,5 @@ int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask,
492493

493494
void free_bucket_spinlocks(spinlock_t *locks);
494495

496+
#undef __LINUX_INSIDE_SPINLOCK_H
495497
#endif /* __LINUX_SPINLOCK_H */

‎include/linux/spinlock_api_smp.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#ifndef __LINUX_SPINLOCK_API_SMP_H
22
#define __LINUX_SPINLOCK_API_SMP_H
33

4-
#ifndef __LINUX_SPINLOCK_H
4+
#ifndef __LINUX_INSIDE_SPINLOCK_H
55
# error "please don't include this file directly"
66
#endif
77

‎include/linux/spinlock_api_up.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#ifndef __LINUX_SPINLOCK_API_UP_H
22
#define __LINUX_SPINLOCK_API_UP_H
33

4-
#ifndef __LINUX_SPINLOCK_H
4+
#ifndef __LINUX_INSIDE_SPINLOCK_H
55
# error "please don't include this file directly"
66
#endif
77

‎include/linux/spinlock_rt.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
#ifndef __LINUX_SPINLOCK_RT_H
33
#define __LINUX_SPINLOCK_RT_H
44

5-
#ifndef __LINUX_SPINLOCK_H
5+
#ifndef __LINUX_INSIDE_SPINLOCK_H
66
#error Do not include directly. Use spinlock.h
77
#endif
88

‎include/linux/spinlock_up.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#ifndef __LINUX_SPINLOCK_UP_H
22
#define __LINUX_SPINLOCK_UP_H
33

4-
#ifndef __LINUX_SPINLOCK_H
4+
#ifndef __LINUX_INSIDE_SPINLOCK_H
55
# error "please don't include this file directly"
66
#endif
77

‎kernel/locking/lockdep.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -934,8 +934,10 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
934934
* Huh! same key, different name? Did someone trample
935935
* on some memory? We're most confused.
936936
*/
937-
WARN_ON_ONCE(class->name != lock->name &&
938-
lock->key != &__lockdep_no_validate__);
937+
WARN_ONCE(class->name != lock->name &&
938+
lock->key != &__lockdep_no_validate__,
939+
"Looking for class \"%s\" with key %ps, but found a different class \"%s\" with the same key\n",
940+
lock->name, lock->key, class->name);
939941
return class;
940942
}
941943
}

‎kernel/locking/qrwlock.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
* queued_read_lock_slowpath - acquire read lock of a queued rwlock
1919
* @lock: Pointer to queued rwlock structure
2020
*/
21-
void queued_read_lock_slowpath(struct qrwlock *lock)
21+
void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
2222
{
2323
/*
2424
* Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
6363
* queued_write_lock_slowpath - acquire write lock of a queued rwlock
6464
* @lock : Pointer to queued rwlock structure
6565
*/
66-
void queued_write_lock_slowpath(struct qrwlock *lock)
66+
void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
6767
{
6868
int cnts;
6969

‎kernel/locking/qspinlock.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
313313
* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
314314
* queue : ^--' :
315315
*/
316-
void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
316+
void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
317317
{
318318
struct mcs_spinlock *prev, *next, *node;
319319
u32 old, tail;

‎kernel/locking/qspinlock_paravirt.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node)
489489
* PV versions of the unlock fastpath and slowpath functions to be used
490490
* instead of queued_spin_unlock().
491491
*/
492-
__visible void
492+
__visible __lockfunc void
493493
__pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
494494
{
495495
struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
544544
#include <asm/qspinlock_paravirt.h>
545545

546546
#ifndef __pv_queued_spin_unlock
547-
__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
547+
__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
548548
{
549549
u8 locked;
550550

‎kernel/locking/rwsem.c

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,14 +133,19 @@
133133
* the owner value concurrently without lock. Read from owner, however,
134134
* may not need READ_ONCE() as long as the pointer value is only used
135135
* for comparison and isn't being dereferenced.
136+
*
137+
* Both rwsem_{set,clear}_owner() functions should be in the same
138+
* preempt disable section as the atomic op that changes sem->count.
136139
*/
137140
static inline void rwsem_set_owner(struct rw_semaphore *sem)
138141
{
142+
lockdep_assert_preemption_disabled();
139143
atomic_long_set(&sem->owner, (long)current);
140144
}
141145

142146
static inline void rwsem_clear_owner(struct rw_semaphore *sem)
143147
{
148+
lockdep_assert_preemption_disabled();
144149
atomic_long_set(&sem->owner, 0);
145150
}
146151

@@ -251,13 +256,16 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp)
251256
static inline bool rwsem_write_trylock(struct rw_semaphore *sem)
252257
{
253258
long tmp = RWSEM_UNLOCKED_VALUE;
259+
bool ret = false;
254260

261+
preempt_disable();
255262
if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) {
256263
rwsem_set_owner(sem);
257-
return true;
264+
ret = true;
258265
}
259266

260-
return false;
267+
preempt_enable();
268+
return ret;
261269
}
262270

263271
/*
@@ -1352,8 +1360,10 @@ static inline void __up_write(struct rw_semaphore *sem)
13521360
DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) &&
13531361
!rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);
13541362

1363+
preempt_disable();
13551364
rwsem_clear_owner(sem);
13561365
tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
1366+
preempt_enable();
13571367
if (unlikely(tmp & RWSEM_FLAG_WAITERS))
13581368
rwsem_wake(sem);
13591369
}

‎kernel/locking/semaphore.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ static noinline void __up(struct semaphore *sem);
5151
* Use of this function is deprecated, please use down_interruptible() or
5252
* down_killable() instead.
5353
*/
54-
void down(struct semaphore *sem)
54+
void __sched down(struct semaphore *sem)
5555
{
5656
unsigned long flags;
5757

@@ -74,7 +74,7 @@ EXPORT_SYMBOL(down);
7474
* If the sleep is interrupted by a signal, this function will return -EINTR.
7575
* If the semaphore is successfully acquired, this function returns 0.
7676
*/
77-
int down_interruptible(struct semaphore *sem)
77+
int __sched down_interruptible(struct semaphore *sem)
7878
{
7979
unsigned long flags;
8080
int result = 0;
@@ -101,7 +101,7 @@ EXPORT_SYMBOL(down_interruptible);
101101
* -EINTR. If the semaphore is successfully acquired, this function returns
102102
* 0.
103103
*/
104-
int down_killable(struct semaphore *sem)
104+
int __sched down_killable(struct semaphore *sem)
105105
{
106106
unsigned long flags;
107107
int result = 0;
@@ -131,7 +131,7 @@ EXPORT_SYMBOL(down_killable);
131131
* Unlike mutex_trylock, this function can be used from interrupt context,
132132
* and the semaphore can be released by any task or interrupt.
133133
*/
134-
int down_trylock(struct semaphore *sem)
134+
int __sched down_trylock(struct semaphore *sem)
135135
{
136136
unsigned long flags;
137137
int count;
@@ -156,7 +156,7 @@ EXPORT_SYMBOL(down_trylock);
156156
* If the semaphore is not released within the specified number of jiffies,
157157
* this function returns -ETIME. It returns 0 if the semaphore was acquired.
158158
*/
159-
int down_timeout(struct semaphore *sem, long timeout)
159+
int __sched down_timeout(struct semaphore *sem, long timeout)
160160
{
161161
unsigned long flags;
162162
int result = 0;
@@ -180,7 +180,7 @@ EXPORT_SYMBOL(down_timeout);
180180
* Release the semaphore. Unlike mutexes, up() may be called from any
181181
* context and even by tasks which have never called down().
182182
*/
183-
void up(struct semaphore *sem)
183+
void __sched up(struct semaphore *sem)
184184
{
185185
unsigned long flags;
186186

0 commit comments

Comments
 (0)