summaryrefslogtreecommitdiff
path: root/kernel/locking
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/locking')
-rw-r--r--kernel/locking/Makefile5
-rw-r--r--kernel/locking/lockdep.c144
-rw-r--r--kernel/locking/locktorture.c27
-rw-r--r--kernel/locking/mutex-debug.c9
-rw-r--r--kernel/locking/mutex.c203
-rw-r--r--kernel/locking/mutex.h7
-rw-r--r--kernel/locking/percpu-rwsem.c31
-rw-r--r--kernel/locking/qspinlock.c22
-rw-r--r--kernel/locking/rtmutex.c39
-rw-r--r--kernel/locking/rtmutex_api.c37
-rw-r--r--kernel/locking/rtmutex_common.h27
-rw-r--r--kernel/locking/rwbase_rt.c7
-rw-r--r--kernel/locking/rwsem.c123
-rw-r--r--kernel/locking/semaphore.c45
-rw-r--r--kernel/locking/spinlock.c43
-rw-r--r--kernel/locking/spinlock_rt.c27
-rw-r--r--kernel/locking/test-ww_mutex.c176
-rw-r--r--kernel/locking/ww_mutex.h99
-rw-r--r--kernel/locking/ww_rt_mutex.c1
19 files changed, 830 insertions, 242 deletions
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index a114949eeed5..cee1901d4cff 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -3,6 +3,11 @@
# and is generally not a function of system call inputs.
KCOV_INSTRUMENT := n
+CONTEXT_ANALYSIS_mutex.o := y
+CONTEXT_ANALYSIS_rtmutex_api.o := y
+CONTEXT_ANALYSIS_ww_rt_mutex.o := y
+CONTEXT_ANALYSIS_rwsem.o := y
+
obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
# Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 2d4c5bab5af8..c3dc84a7cef2 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -787,17 +787,33 @@ static void lockdep_print_held_locks(struct task_struct *p)
{
int i, depth = READ_ONCE(p->lockdep_depth);
- if (!depth)
- printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p));
- else
- printk("%d lock%s held by %s/%d:\n", depth,
- str_plural(depth), p->comm, task_pid_nr(p));
/*
- * It's not reliable to print a task's held locks if it's not sleeping
- * and it's not the current task.
+ * Note that it's always somewhat unreliable to print held locks
+ * of a task that is running on another CPU, but we cannot guarantee
+ * the stability of ->held_locks without actually stopping all active
+ * remote CPUs, which we absolutely do not want to do because it's
+ * very intrusive and thus slow.
+ *
+ * So we do the next best thing here: we print out the held lock
+ * array on a best-effort basis, without crashing even if the
+ * fields are being modified on another CPU. Note the careful
+ * construction of print_lock() so that it never crashes.
+ *
+ * We also print out the CPU the task is or was last running on, with
+ * the message saying 'on CPU...' if the task is running, and
+ * 'last CPU' if it's not.
+ *
+ * Also note that the task_is_running(p) information is fundamentally
+ * racy: even if the message says the task is 'on CPU', the task may
+ * have scheduled out already, or if it says 'last CPU', it may just
+ * have scheduled in on another CPU. But even with these limitations
+ * it's still useful debuggining information.
*/
- if (p != current && task_is_running(p))
- return;
+ printk("locks held by %s/%d: %d, %s CPU#%d%s\n",
+ p->comm, task_pid_nr(p), depth,
+ task_is_running(p) ? "last" : "on", task_cpu(p),
+ depth > 0 ? ":" : "");
+
for (i = 0; i < depth; i++) {
printk(" #%d: ", i);
print_lock(p->held_locks + i);
@@ -947,6 +963,34 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
return NULL;
}
+static __always_inline bool lock_class_cache_is_valid(const struct lockdep_map *lock,
+ const struct lock_class *class,
+ unsigned int subclass)
+{
+ unsigned int class_subclass;
+
+ if (!class)
+ return false;
+
+ if (unlikely(class < lock_classes || class >= lock_classes + MAX_LOCKDEP_KEYS))
+ return false;
+
+ if (unlikely(!arch_test_bit(class - lock_classes, lock_classes_in_use)))
+ return false;
+
+ if (unlikely(!lock->key))
+ return false;
+
+ class_subclass = subclass ? subclass : class->subclass;
+ if (unlikely(class_subclass >= MAX_LOCKDEP_SUBCLASSES))
+ return false;
+
+ if (unlikely(READ_ONCE(class->key) != lock->key->subkeys + class_subclass))
+ return false;
+
+ return true;
+}
+
/*
* Static locks do not have their class-keys yet - for them the key is
* the lock object itself. If the lock is in the per cpu area, the
@@ -1379,9 +1423,9 @@ out_unlock_set:
out_set_class_cache:
if (!subclass || force)
- lock->class_cache[0] = class;
+ WRITE_ONCE(lock->class_cache[0], class);
else if (subclass < NR_LOCKDEP_CACHING_CLASSES)
- lock->class_cache[subclass] = class;
+ WRITE_ONCE(lock->class_cache[subclass], class);
/*
* Hash collision, did we smoke some? We found a class with a matching
@@ -4941,7 +4985,7 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name,
int i;
for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++)
- lock->class_cache[i] = NULL;
+ WRITE_ONCE(lock->class_cache[i], NULL);
#ifdef CONFIG_LOCK_STAT
lock->cpu = raw_smp_processor_id();
@@ -5006,12 +5050,15 @@ EXPORT_SYMBOL_GPL(__lockdep_no_track__);
void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn,
lock_print_fn print_fn)
{
- struct lock_class *class = lock->class_cache[0];
+ struct lock_class *class = READ_ONCE(lock->class_cache[0]);
unsigned long flags;
raw_local_irq_save(flags);
lockdep_recursion_inc();
+ if (!lock_class_cache_is_valid(lock, class, 0))
+ class = NULL;
+
if (!class)
class = register_lock_class(lock, 0, 0);
@@ -5077,7 +5124,7 @@ static int __lock_is_held(const struct lockdep_map *lock, int read);
static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
int trylock, int read, int check, int hardirqs_off,
struct lockdep_map *nest_lock, unsigned long ip,
- int references, int pin_count, int sync)
+ int references, int pin_count, int sync, int seq)
{
struct task_struct *curr = current;
struct lock_class *class = NULL;
@@ -5103,8 +5150,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
if (DEBUG_LOCKS_WARN_ON(subclass >= MAX_LOCKDEP_SUBCLASSES))
return 0;
- if (subclass < NR_LOCKDEP_CACHING_CLASSES)
- class = lock->class_cache[subclass];
+ if (subclass < NR_LOCKDEP_CACHING_CLASSES) {
+ class = READ_ONCE(lock->class_cache[subclass]);
+ if (!lock_class_cache_is_valid(lock, class, subclass))
+ class = NULL;
+ }
/*
* Not cached?
*/
@@ -5183,6 +5233,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
hlock->holdtime_stamp = lockstat_clock();
#endif
hlock->pin_count = pin_count;
+ hlock->seq_count = seq;
if (check_wait_context(curr, hlock))
return 0;
@@ -5307,9 +5358,9 @@ static noinstr int match_held_lock(const struct held_lock *hlock,
return 1;
if (hlock->references) {
- const struct lock_class *class = lock->class_cache[0];
+ const struct lock_class *class = READ_ONCE(lock->class_cache[0]);
- if (!class)
+ if (!lock_class_cache_is_valid(lock, class, 0))
class = look_up_lock_class(lock, 0);
/*
@@ -5388,7 +5439,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
hlock->read, hlock->check,
hlock->hardirqs_off,
hlock->nest_lock, hlock->acquire_ip,
- hlock->references, hlock->pin_count, 0)) {
+ hlock->references, hlock->pin_count, 0, hlock->seq_count)) {
case 0:
return 1;
case 1:
@@ -5437,6 +5488,8 @@ __lock_set_class(struct lockdep_map *lock, const char *name,
lock->wait_type_outer,
lock->lock_type);
class = register_lock_class(lock, subclass, 0);
+ if (!class)
+ return 0;
hlock->class_idx = class - lock_classes;
curr->lockdep_depth = i;
@@ -5669,14 +5722,17 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie
struct held_lock *hlock = curr->held_locks + i;
if (match_held_lock(hlock, lock)) {
+ int pin_count;
+
if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))
return;
- hlock->pin_count -= cookie.val;
+ pin_count = hlock->pin_count - cookie.val;
- if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))
- hlock->pin_count = 0;
+ if (WARN(pin_count < 0, "pin count corrupted\n"))
+ pin_count = 0;
+ hlock->pin_count = pin_count;
return;
}
}
@@ -5684,6 +5740,24 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie
WARN(1, "unpinning an unheld lock\n");
}
+static u32 __lock_sequence(struct lockdep_map *lock)
+{
+ struct task_struct *curr = current;
+ int i;
+
+ if (unlikely(!debug_locks))
+ return ~0;
+
+ for (i = 0; i < curr->lockdep_depth; i++) {
+ struct held_lock *hlock = curr->held_locks + i;
+
+ if (match_held_lock(hlock, lock))
+ return hlock->seq_count;
+ }
+
+ return ~0;
+}
+
/*
* Check whether we follow the irq-flags state precisely:
*/
@@ -5866,7 +5940,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
lockdep_recursion_inc();
__lock_acquire(lock, subclass, trylock, read, check,
- irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0);
+ irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0,
+ ++current->lockdep_seq);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
}
@@ -5914,7 +5989,8 @@ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read,
lockdep_recursion_inc();
__lock_acquire(lock, subclass, 0, read, check,
- irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1);
+ irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1,
+ ++current->lockdep_seq);
check_chain_key(current);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
@@ -6000,6 +6076,26 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
}
EXPORT_SYMBOL_GPL(lock_unpin_lock);
+u32 lock_sequence(struct lockdep_map *lock)
+{
+ unsigned long flags;
+ u32 seq = ~0;
+
+ if (unlikely(!lockdep_enabled()))
+ return seq;
+
+ raw_local_irq_save(flags);
+ check_flags(flags);
+
+ lockdep_recursion_inc();
+ seq = __lock_sequence(lock);
+ lockdep_recursion_finish();
+ raw_local_irq_restore(flags);
+
+ return seq;
+}
+EXPORT_SYMBOL_GPL(lock_sequence);
+
#ifdef CONFIG_LOCK_STAT
static void print_lock_contention_bug(struct task_struct *curr,
struct lockdep_map *lock,
diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c
index 6567e5eeacc0..e618bcf75e2d 100644
--- a/kernel/locking/locktorture.c
+++ b/kernel/locking/locktorture.c
@@ -610,9 +610,8 @@ static void torture_ww_mutex_init(void)
ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class);
ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class);
- ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress,
- sizeof(*ww_acquire_ctxs),
- GFP_KERNEL);
+ ww_acquire_ctxs = kmalloc_objs(*ww_acquire_ctxs,
+ cxt.nrealwriters_stress);
if (!ww_acquire_ctxs)
VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory");
}
@@ -1129,7 +1128,8 @@ static int call_rcu_chain_init(void)
if (call_rcu_chains <= 0)
return 0;
- call_rcu_chain_list = kcalloc(call_rcu_chains, sizeof(*call_rcu_chain_list), GFP_KERNEL);
+ call_rcu_chain_list = kzalloc_objs(*call_rcu_chain_list,
+ call_rcu_chains);
if (!call_rcu_chain_list)
return -ENOMEM;
for (i = 0; i < call_rcu_chains; i++) {
@@ -1293,9 +1293,7 @@ static int __init lock_torture_init(void)
/* Initialize the statistics so that each run gets its own numbers. */
if (nwriters_stress) {
lock_is_write_held = false;
- cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
- sizeof(*cxt.lwsa),
- GFP_KERNEL);
+ cxt.lwsa = kmalloc_objs(*cxt.lwsa, cxt.nrealwriters_stress);
if (cxt.lwsa == NULL) {
VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
firsterr = -ENOMEM;
@@ -1323,9 +1321,8 @@ static int __init lock_torture_init(void)
}
if (nreaders_stress) {
- cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
- sizeof(*cxt.lrsa),
- GFP_KERNEL);
+ cxt.lrsa = kmalloc_objs(*cxt.lrsa,
+ cxt.nrealreaders_stress);
if (cxt.lrsa == NULL) {
VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
firsterr = -ENOMEM;
@@ -1372,9 +1369,8 @@ static int __init lock_torture_init(void)
}
if (nwriters_stress) {
- writer_tasks = kcalloc(cxt.nrealwriters_stress,
- sizeof(writer_tasks[0]),
- GFP_KERNEL);
+ writer_tasks = kzalloc_objs(writer_tasks[0],
+ cxt.nrealwriters_stress);
if (writer_tasks == NULL) {
TOROUT_ERRSTRING("writer_tasks: Out of memory");
firsterr = -ENOMEM;
@@ -1387,9 +1383,8 @@ static int __init lock_torture_init(void)
nested_locks = MAX_NESTED_LOCKS;
if (cxt.cur_ops->readlock) {
- reader_tasks = kcalloc(cxt.nrealreaders_stress,
- sizeof(reader_tasks[0]),
- GFP_KERNEL);
+ reader_tasks = kzalloc_objs(reader_tasks[0],
+ cxt.nrealreaders_stress);
if (reader_tasks == NULL) {
TOROUT_ERRSTRING("reader_tasks: Out of memory");
kfree(writer_tasks);
diff --git a/kernel/locking/mutex-debug.c b/kernel/locking/mutex-debug.c
index 2c6b02d4699b..785decd9d0c0 100644
--- a/kernel/locking/mutex-debug.c
+++ b/kernel/locking/mutex-debug.c
@@ -37,9 +37,8 @@ void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)
void debug_mutex_wake_waiter(struct mutex *lock, struct mutex_waiter *waiter)
{
lockdep_assert_held(&lock->wait_lock);
- DEBUG_LOCKS_WARN_ON(list_empty(&lock->wait_list));
+ DEBUG_LOCKS_WARN_ON(!lock->first_waiter);
DEBUG_LOCKS_WARN_ON(waiter->magic != waiter);
- DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
}
void debug_mutex_free_waiter(struct mutex_waiter *waiter)
@@ -54,15 +53,14 @@ void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
lockdep_assert_held(&lock->wait_lock);
/* Current thread can't be already blocked (since it's executing!) */
- DEBUG_LOCKS_WARN_ON(__get_task_blocked_on(task));
+ DEBUG_LOCKS_WARN_ON(get_task_blocked_on(task));
}
void debug_mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
struct task_struct *task)
{
- struct mutex *blocked_on = __get_task_blocked_on(task);
+ struct mutex *blocked_on = get_task_blocked_on(task);
- DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
DEBUG_LOCKS_WARN_ON(waiter->task != task);
DEBUG_LOCKS_WARN_ON(blocked_on && blocked_on != lock);
@@ -74,7 +72,6 @@ void debug_mutex_unlock(struct mutex *lock)
{
if (likely(debug_locks)) {
DEBUG_LOCKS_WARN_ON(lock->magic != lock);
- DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
}
}
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 2a1d165b3167..942a939cee95 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -46,8 +46,9 @@
static void __mutex_init_generic(struct mutex *lock)
{
atomic_long_set(&lock->owner, 0);
- raw_spin_lock_init(&lock->wait_lock);
- INIT_LIST_HEAD(&lock->wait_list);
+ scoped_guard (raw_spinlock_init, &lock->wait_lock) {
+ lock->first_waiter = NULL;
+ }
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
osq_lock_init(&lock->osq);
#endif
@@ -150,6 +151,7 @@ EXPORT_SYMBOL(mutex_init_generic);
* follow with a __mutex_trylock() before failing.
*/
static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
+ __cond_acquires(true, lock)
{
unsigned long curr = (unsigned long)current;
unsigned long zero = 0UL;
@@ -163,6 +165,7 @@ static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
}
static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
+ __cond_releases(true, lock)
{
unsigned long curr = (unsigned long)current;
@@ -171,7 +174,7 @@ static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
#else /* !CONFIG_DEBUG_LOCK_ALLOC */
-void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key)
+void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key)
{
__mutex_init_generic(lock);
@@ -181,7 +184,7 @@ void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_k
debug_check_no_locks_freed((void *)lock, sizeof(*lock));
lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP);
}
-EXPORT_SYMBOL(mutex_init_lockep);
+EXPORT_SYMBOL(mutex_init_lockdep);
#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
@@ -194,33 +197,58 @@ static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
atomic_long_andnot(flag, &lock->owner);
}
-static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
-{
- return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
-}
-
/*
- * Add @waiter to a given location in the lock wait_list and set the
- * FLAG_WAITERS flag if it's the first waiter.
+ * Add @waiter to the @lock wait_list and set the FLAG_WAITERS flag if it's
+ * the first waiter.
+ *
+ * When @pos, @waiter is added before the waiter indicated by @pos. Otherwise
+ * @waiter will be added to the tail of the list.
*/
static void
__mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
- struct list_head *list)
+ struct mutex_waiter *pos)
+ __must_hold(&lock->wait_lock)
{
+ struct mutex_waiter *first = lock->first_waiter;
+
hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX);
debug_mutex_add_waiter(lock, waiter, current);
- list_add_tail(&waiter->list, list);
- if (__mutex_waiter_is_first(lock, waiter))
- __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
+ if (pos) {
+ /*
+ * Insert @waiter before @pos.
+ */
+ list_add_tail(&waiter->list, &pos->list);
+ /*
+ * If @pos == @first, then @waiter will be the new first.
+ */
+ if (pos == first)
+ lock->first_waiter = waiter;
+ return;
+ }
+
+ if (first) {
+ list_add_tail(&waiter->list, &first->list);
+ return;
+ }
+
+ INIT_LIST_HEAD(&waiter->list);
+ lock->first_waiter = waiter;
+ __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
}
static void
__mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
- list_del(&waiter->list);
- if (likely(list_empty(&lock->wait_list)))
+ if (list_empty(&waiter->list)) {
__mutex_clear_flag(lock, MUTEX_FLAGS);
+ lock->first_waiter = NULL;
+ } else {
+ if (lock->first_waiter == waiter)
+ lock->first_waiter = list_next_entry(waiter, list);
+ list_del(&waiter->list);
+ }
debug_mutex_remove_waiter(lock, waiter, current);
hung_task_clear_blocker();
@@ -259,7 +287,8 @@ static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
* We also put the fastpath first in the kernel image, to make sure the
* branch is predicted by the CPU as default-untaken.
*/
-static void __sched __mutex_lock_slowpath(struct mutex *lock);
+static void __sched __mutex_lock_slowpath(struct mutex *lock)
+ __acquires(lock);
/**
* mutex_lock - acquire the mutex
@@ -340,7 +369,7 @@ bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
* Similarly, stop spinning if we are no longer the
* first waiter.
*/
- if (waiter && !__mutex_waiter_is_first(lock, waiter))
+ if (waiter && data_race(lock->first_waiter != waiter))
return false;
return true;
@@ -525,7 +554,8 @@ mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
}
#endif
-static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
+static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
+ __releases(lock);
/**
* mutex_unlock - release the mutex
@@ -565,6 +595,7 @@ EXPORT_SYMBOL(mutex_unlock);
* of a unlocked mutex is not allowed.
*/
void __sched ww_mutex_unlock(struct ww_mutex *lock)
+ __no_context_analysis
{
__ww_mutex_unlock(lock);
mutex_unlock(&lock->base);
@@ -578,6 +609,7 @@ static __always_inline int __sched
__mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,
struct lockdep_map *nest_lock, unsigned long ip,
struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
+ __cond_acquires(0, lock)
{
DEFINE_WAKE_Q(wake_q);
struct mutex_waiter waiter;
@@ -645,7 +677,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
if (!use_ww_ctx) {
/* add waiting tasks to the end of the waitqueue (FIFO): */
- __mutex_add_waiter(lock, &waiter, &lock->wait_list);
+ __mutex_add_waiter(lock, &waiter, NULL);
} else {
/*
* Add in stamp order, waking up waiters that must kill
@@ -656,6 +688,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
goto err_early_kill;
}
+ raw_spin_lock(&current->blocked_lock);
__set_task_blocked_on(current, lock);
set_current_state(state);
trace_contention_begin(lock, LCB_F_MUTEX);
@@ -669,8 +702,9 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
* the handoff.
*/
if (__mutex_trylock(lock))
- goto acquired;
+ break;
+ raw_spin_unlock(&current->blocked_lock);
/*
* Check for signals and kill conditions while holding
* wait_lock. This ensures the lock cancellation is ordered
@@ -691,14 +725,16 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
schedule_preempt_disabled();
- first = __mutex_waiter_is_first(lock, &waiter);
+ first = lock->first_waiter == &waiter;
+ raw_spin_lock_irqsave(&lock->wait_lock, flags);
+ raw_spin_lock(&current->blocked_lock);
/*
* As we likely have been woken up by task
* that has cleared our blocked_on state, re-set
* it to the lock we are trying to acquire.
*/
- set_task_blocked_on(current, lock);
+ __set_task_blocked_on(current, lock);
set_current_state(state);
/*
* Here we order against unlock; we must either see it change
@@ -709,33 +745,41 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
break;
if (first) {
- trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
+ bool opt_acquired;
+
/*
* mutex_optimistic_spin() can call schedule(), so
- * clear blocked on so we don't become unselectable
+ * we need to release these locks before calling it,
+ * and clear blocked on so we don't become unselectable
* to run.
*/
- clear_task_blocked_on(current, lock);
- if (mutex_optimistic_spin(lock, ww_ctx, &waiter))
+ __clear_task_blocked_on(current, lock);
+ raw_spin_unlock(&current->blocked_lock);
+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
+
+ trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
+ opt_acquired = mutex_optimistic_spin(lock, ww_ctx, &waiter);
+
+ raw_spin_lock_irqsave(&lock->wait_lock, flags);
+ raw_spin_lock(&current->blocked_lock);
+ __set_task_blocked_on(current, lock);
+ set_current_state(state);
+
+ if (opt_acquired)
break;
- set_task_blocked_on(current, lock);
trace_contention_begin(lock, LCB_F_MUTEX);
}
-
- raw_spin_lock_irqsave(&lock->wait_lock, flags);
}
- raw_spin_lock_irqsave(&lock->wait_lock, flags);
-acquired:
__clear_task_blocked_on(current, lock);
__set_current_state(TASK_RUNNING);
+ raw_spin_unlock(&current->blocked_lock);
if (ww_ctx) {
/*
* Wound-Wait; we stole the lock (!first_waiter), check the
* waiters as anyone might want to wound us.
*/
- if (!ww_ctx->is_wait_die &&
- !__mutex_waiter_is_first(lock, &waiter))
+ if (!ww_ctx->is_wait_die && lock->first_waiter != &waiter)
__ww_mutex_check_waiters(lock, ww_ctx, &wake_q);
}
@@ -756,11 +800,11 @@ skip_wait:
return 0;
err:
- __clear_task_blocked_on(current, lock);
+ clear_task_blocked_on(current, lock);
__set_current_state(TASK_RUNNING);
__mutex_remove_waiter(lock, &waiter);
err_early_kill:
- WARN_ON(__get_task_blocked_on(current));
+ WARN_ON(get_task_blocked_on(current));
trace_contention_end(lock, ret);
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
debug_mutex_free_waiter(&waiter);
@@ -772,6 +816,7 @@ err_early_kill:
static int __sched
__mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
struct lockdep_map *nest_lock, unsigned long ip)
+ __cond_acquires(0, lock)
{
return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);
}
@@ -779,6 +824,7 @@ __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
static int __sched
__ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
unsigned long ip, struct ww_acquire_ctx *ww_ctx)
+ __cond_acquires(0, lock)
{
return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true);
}
@@ -826,6 +872,7 @@ void __sched
mutex_lock_nested(struct mutex *lock, unsigned int subclass)
{
__mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(mutex_lock_nested);
@@ -834,6 +881,7 @@ void __sched
_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
{
__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
@@ -862,12 +910,14 @@ mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
token = io_schedule_prepare();
__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
subclass, NULL, _RET_IP_, NULL, 0);
+ __acquire(lock);
io_schedule_finish(token);
}
EXPORT_SYMBOL_GPL(mutex_lock_io_nested);
static inline int
ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+ __cond_releases(nonzero, lock)
{
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
unsigned tmp;
@@ -929,15 +979,25 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
* Release the lock, slowpath:
*/
static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
+ __releases(lock)
{
- struct task_struct *next = NULL;
- DEFINE_WAKE_Q(wake_q);
+ struct task_struct *donor, *next = NULL;
+ struct mutex_waiter *waiter;
unsigned long owner;
unsigned long flags;
mutex_release(&lock->dep_map, ip);
+ __release(lock);
/*
+ * Ensures the proxy donor stack is stable across unlock and handoff.
+ * Specifically, it avoids the case where current->blocked_donor is
+ * NULL when it is inspected while doing the unlock, but a preemption
+ * before taking the wake_lock would make it set and a hand-off is
+ * missed.
+ */
+ guard(preempt)();
+ /*
* Release the lock before (potentially) taking the spinlock such that
* other contenders can get on with things ASAP.
*
@@ -949,6 +1009,12 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
MUTEX_WARN_ON(__owner_task(owner) != current);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
+ if (sched_proxy_exec() && current->blocked_donor) {
+ /* force handoff if we have a blocked_donor */
+ owner = MUTEX_FLAG_HANDOFF;
+ break;
+ }
+
if (owner & MUTEX_FLAG_HANDOFF)
break;
@@ -961,24 +1027,56 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
}
raw_spin_lock_irqsave(&lock->wait_lock, flags);
+ raw_spin_lock(&current->blocked_lock);
debug_mutex_unlock(lock);
- if (!list_empty(&lock->wait_list)) {
- /* get the first entry from the wait-list: */
- struct mutex_waiter *waiter =
- list_first_entry(&lock->wait_list,
- struct mutex_waiter, list);
- next = waiter->task;
+ if (sched_proxy_exec()) {
+ /*
+ * If we have a task boosting current, and that task was boosting
+ * current through this lock, hand the lock to that task, as that
+ * is the highest waiter, as selected by the scheduling function.
+ */
+ donor = current->blocked_donor;
+ if (donor) {
+ struct mutex *next_lock;
+
+ raw_spin_lock_nested(&donor->blocked_lock, SINGLE_DEPTH_NESTING);
+ next_lock = __get_task_blocked_on(donor);
+ if (next_lock == lock) {
+ next = get_task_struct(donor);
+ __clear_task_blocked_on(next, lock);
+ current->blocked_donor = NULL;
+ }
+ raw_spin_unlock(&donor->blocked_lock);
+ }
+ }
+
+ /*
+ * Failing that, pick first on the wait list.
+ */
+ waiter = lock->first_waiter;
+ if (!next && waiter) {
+ next = get_task_struct(waiter->task);
+ raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING);
debug_mutex_wake_waiter(lock, waiter);
__clear_task_blocked_on(next, lock);
- wake_q_add(&wake_q, next);
+ raw_spin_unlock(&next->blocked_lock);
+
}
+ if (trace_contended_release_enabled() && waiter)
+ trace_call__contended_release(lock);
+
if (owner & MUTEX_FLAG_HANDOFF)
__mutex_handoff(lock, next);
- raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
+ raw_spin_unlock(&current->blocked_lock);
+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
+ if (next) {
+ wake_up_process(next);
+ put_task_struct(next);
+ }
}
#ifndef CONFIG_DEBUG_LOCK_ALLOC
@@ -1061,24 +1159,29 @@ EXPORT_SYMBOL_GPL(mutex_lock_io);
static noinline void __sched
__mutex_lock_slowpath(struct mutex *lock)
+ __acquires(lock)
{
__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
+ __acquire(lock);
}
static noinline int __sched
__mutex_lock_killable_slowpath(struct mutex *lock)
+ __cond_acquires(0, lock)
{
return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
}
static noinline int __sched
__mutex_lock_interruptible_slowpath(struct mutex *lock)
+ __cond_acquires(0, lock)
{
return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
}
static noinline int __sched
__ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+ __cond_acquires(0, lock)
{
return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0,
_RET_IP_, ctx);
@@ -1087,6 +1190,7 @@ __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
static noinline int __sched
__ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
struct ww_acquire_ctx *ctx)
+ __cond_acquires(0, lock)
{
return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0,
_RET_IP_, ctx);
@@ -1166,6 +1270,11 @@ EXPORT_SYMBOL(ww_mutex_lock_interruptible);
EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin);
EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end);
+EXPORT_TRACEPOINT_SYMBOL_GPL(contended_release);
+
+__weak int arch_contended_release_trace_reg(void) { return 0; }
+
+__weak void arch_contended_release_trace_unreg(void) { }
/**
* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
diff --git a/kernel/locking/mutex.h b/kernel/locking/mutex.h
index 9ad4da8cea00..3e263e98e5fc 100644
--- a/kernel/locking/mutex.h
+++ b/kernel/locking/mutex.h
@@ -7,6 +7,7 @@
* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
*/
#ifndef CONFIG_PREEMPT_RT
+#include <linux/mutex.h>
/*
* This is the control structure for tasks blocked on mutex, which resides
* on the blocked task's kernel stack:
@@ -47,6 +48,12 @@ static inline struct task_struct *__mutex_owner(struct mutex *lock)
return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);
}
+static inline struct mutex *get_task_blocked_on(struct task_struct *p)
+{
+ guard(raw_spinlock_irqsave)(&p->blocked_lock);
+ return __get_task_blocked_on(p);
+}
+
#ifdef CONFIG_DEBUG_MUTEXES
extern void debug_mutex_lock_common(struct mutex *lock,
struct mutex_waiter *waiter);
diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
index ef234469baac..6c78961fe753 100644
--- a/kernel/locking/percpu-rwsem.c
+++ b/kernel/locking/percpu-rwsem.c
@@ -211,7 +211,7 @@ EXPORT_SYMBOL_GPL(percpu_is_read_locked);
*/
static bool readers_active_check(struct percpu_rw_semaphore *sem)
{
- if (per_cpu_sum(*sem->read_count) != 0)
+ if (data_race(per_cpu_sum(*sem->read_count)) != 0)
return false;
/*
@@ -263,6 +263,9 @@ void percpu_up_write(struct percpu_rw_semaphore *sem)
{
rwsem_release(&sem->dep_map, _RET_IP_);
+ if (trace_contended_release_enabled() && wq_has_sleeper(&sem->waiters))
+ trace_call__contended_release(sem);
+
/*
* Signal the writer is done, no fast path yet.
*
@@ -288,3 +291,29 @@ void percpu_up_write(struct percpu_rw_semaphore *sem)
rcu_sync_exit(&sem->rss);
}
EXPORT_SYMBOL_GPL(percpu_up_write);
+
+void __percpu_up_read(struct percpu_rw_semaphore *sem)
+{
+ lockdep_assert_preemption_disabled();
+ /*
+ * After percpu_up_write() completes, rcu_sync_is_idle() can still
+ * return false during the grace period, forcing readers into this
+ * slowpath. Only trace when a writer is actually waiting for
+ * readers to drain.
+ */
+ if (trace_contended_release_enabled() && rcuwait_active(&sem->writer))
+ trace_call__contended_release(sem);
+ /*
+ * slowpath; reader will only ever wake a single blocked
+ * writer.
+ */
+ smp_mb(); /* B matches C */
+ /*
+ * In other words, if they see our decrement (presumably to
+ * aggregate zero, as that is the only time it matters) they
+ * will also see our critical section.
+ */
+ this_cpu_dec(*sem->read_count);
+ rcuwait_wake_up(&sem->writer);
+}
+EXPORT_SYMBOL_GPL(__percpu_up_read);
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index af8d122bb649..33fe6d437c8f 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -104,6 +104,28 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
#define queued_spin_lock_slowpath native_queued_spin_lock_slowpath
#endif
+#if !defined(queued_spin_unlock) && \
+ IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE)
+/*
+ * Out-of-line trace-and-release path for queued_spin_unlock(), used when
+ * the contended_release tracepoint is enabled.
+ *
+ * queued_spin_release() is duplicated here on purpose: doing the release
+ * in this function (rather than tracing here and releasing in the caller)
+ * lets queued_spin_unlock() return right after the call, so the
+ * tracepoint-disabled hot path never has to keep lock live across a call
+ * in a callee-saved register. Keep this release in sync with the one in
+ * queued_spin_unlock().
+ */
+void __lockfunc queued_spin_release_traced(struct qspinlock *lock)
+{
+ if (queued_spin_is_contended(lock))
+ trace_call__contended_release(lock);
+ queued_spin_release(lock);
+}
+EXPORT_SYMBOL(queued_spin_release_traced);
+#endif
+
#endif /* _GEN_PV_LOCK_SLOWPATH */
/**
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index c80902eacd79..4728631ae719 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -94,6 +94,7 @@ static inline int __ww_mutex_check_kill(struct rt_mutex *lock,
static __always_inline struct task_struct *
rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)
+ __must_hold(&lock->wait_lock)
{
unsigned long val = (unsigned long)owner;
@@ -105,6 +106,7 @@ rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)
static __always_inline void
rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
+ __must_hold(&lock->wait_lock)
{
/*
* lock->wait_lock is held but explicit acquire semantics are needed
@@ -114,12 +116,14 @@ rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
}
static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
/* lock->wait_lock is held so the unlock provides release semantics. */
WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));
}
static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
lock->owner = (struct task_struct *)
((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
@@ -127,6 +131,7 @@ static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
static __always_inline void
fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock)
+ __must_hold(&lock->wait_lock)
{
unsigned long owner, *p = (unsigned long *) &lock->owner;
@@ -328,6 +333,7 @@ static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
}
static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
lock->owner = (struct task_struct *)
((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
@@ -478,6 +484,7 @@ static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_nod
static __always_inline void
rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -486,6 +493,7 @@ rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
static __always_inline void
rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -1086,6 +1094,7 @@ static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
static int __sched
try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,
struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -1206,6 +1215,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
struct ww_acquire_ctx *ww_ctx,
enum rtmutex_chainwalk chwalk,
struct wake_q_head *wake_q)
+ __must_hold(&lock->wait_lock)
{
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex_waiter *top_waiter = waiter;
@@ -1249,6 +1259,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
/* Check whether the waiter should back out immediately */
rtm = container_of(lock, struct rt_mutex, rtmutex);
+ __assume_ctx_lock(&rtm->rtmutex.wait_lock);
res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx, wake_q);
if (res) {
raw_spin_lock(&task->pi_lock);
@@ -1311,6 +1322,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
*/
static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh,
struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
struct rt_mutex_waiter *waiter;
@@ -1356,6 +1368,7 @@ static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh,
}
static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
int ret = try_to_take_rt_mutex(lock, current, NULL);
@@ -1457,6 +1470,7 @@ static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)
raw_spin_lock_irqsave(&lock->wait_lock, flags);
}
+ trace_contended_release(lock);
/*
* The wakeup next waiter path does not suffer from the above
* race. See the comments there.
@@ -1505,7 +1519,7 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
* - the VCPU on which owner runs is preempted
*/
if (!owner_on_cpu(owner) || need_resched() ||
- !rt_mutex_waiter_is_top_waiter(lock, waiter)) {
+ !data_race(rt_mutex_waiter_is_top_waiter(lock, waiter))) {
res = false;
break;
}
@@ -1535,20 +1549,27 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
*
* Must be called with lock->wait_lock held and interrupts disabled. It must
* have just failed to try_to_take_rt_mutex().
+ *
+ * When invoked from rt_mutex_start_proxy_lock() waiter::task != current !
*/
static void __sched remove_waiter(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
struct task_struct *owner = rt_mutex_owner(lock);
+ struct task_struct *waiter_task = waiter->task;
struct rt_mutex_base *next_lock;
lockdep_assert_held(&lock->wait_lock);
- raw_spin_lock(&current->pi_lock);
- rt_mutex_dequeue(lock, waiter);
- current->pi_blocked_on = NULL;
- raw_spin_unlock(&current->pi_lock);
+ if (!waiter_task) /* never enqueued */
+ return;
+
+ scoped_guard(raw_spinlock, &waiter_task->pi_lock) {
+ rt_mutex_dequeue(lock, waiter);
+ waiter_task->pi_blocked_on = NULL;
+ }
/*
* Only update priority if the waiter was the highest priority
@@ -1584,7 +1605,7 @@ static void __sched remove_waiter(struct rt_mutex_base *lock,
raw_spin_unlock_irq(&lock->wait_lock);
rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
- next_lock, NULL, current);
+ next_lock, NULL, waiter_task);
raw_spin_lock_irq(&lock->wait_lock);
}
@@ -1613,6 +1634,8 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
struct task_struct *owner;
int ret = 0;
+ __assume_ctx_lock(&rtm->rtmutex.wait_lock);
+
lockevent_inc(rtmutex_slow_block);
for (;;) {
/* Try to acquire the lock: */
@@ -1658,6 +1681,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,
struct rt_mutex_base *lock,
struct rt_mutex_waiter *w)
+ __must_hold(&lock->wait_lock)
{
/*
* If the result is not -EDEADLOCK or the caller requested
@@ -1694,11 +1718,13 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
enum rtmutex_chainwalk chwalk,
struct rt_mutex_waiter *waiter,
struct wake_q_head *wake_q)
+ __must_hold(&lock->wait_lock)
{
struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);
struct ww_mutex *ww = ww_container_of(rtm);
int ret;
+ __assume_ctx_lock(&rtm->rtmutex.wait_lock);
lockdep_assert_held(&lock->wait_lock);
lockevent_inc(rtmutex_slowlock);
@@ -1750,6 +1776,7 @@ static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,
struct ww_acquire_ctx *ww_ctx,
unsigned int state,
struct wake_q_head *wake_q)
+ __must_hold(&lock->wait_lock)
{
struct rt_mutex_waiter waiter;
int ret;
diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c
index 59dbd29cb219..eb18b094473c 100644
--- a/kernel/locking/rtmutex_api.c
+++ b/kernel/locking/rtmutex_api.c
@@ -41,6 +41,7 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,
unsigned int state,
struct lockdep_map *nest_lock,
unsigned int subclass)
+ __cond_acquires(0, lock)
{
int ret;
@@ -67,13 +68,27 @@ EXPORT_SYMBOL(rt_mutex_base_init);
*/
void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
{
- __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass);
+ if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass) == 0)
+ return;
+ /*
+ * The code below is never reached because __rt_mutex_lock_common() only
+ * returns an error code if interrupted by a signal or upon a timeout.
+ */
+ WARN_ON_ONCE(true);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)
{
- __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0);
+ if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0) == 0)
+ return;
+ /*
+ * The code below is never reached because __rt_mutex_lock_common() only
+ * returns an error code if interrupted by a signal or upon a timeout.
+ */
+ WARN_ON_ONCE(true);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
@@ -86,7 +101,14 @@ EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
*/
void __sched rt_mutex_lock(struct rt_mutex *lock)
{
- __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0);
+ if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0) == 0)
+ return;
+ /*
+ * The code below is never reached because __rt_mutex_lock_common() only
+ * returns an error code if interrupted by a signal or upon a timeout.
+ */
+ WARN_ON_ONCE(true);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(rt_mutex_lock);
#endif
@@ -157,6 +179,7 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock)
{
mutex_release(&lock->dep_map, _RET_IP_);
__rt_mutex_unlock(&lock->rtmutex);
+ __release(lock);
}
EXPORT_SYMBOL_GPL(rt_mutex_unlock);
@@ -182,6 +205,7 @@ int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock)
*/
bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock,
struct rt_wake_q_head *wqh)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -312,6 +336,7 @@ int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
struct wake_q_head *wake_q)
+ __must_hold(&lock->wait_lock)
{
int ret;
@@ -365,7 +390,7 @@ int __sched rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
raw_spin_lock_irq(&lock->wait_lock);
ret = __rt_mutex_start_proxy_lock(lock, waiter, task, &wake_q);
- if (unlikely(ret))
+ if (unlikely(ret < 0))
remove_waiter(lock, waiter);
preempt_disable();
raw_spin_unlock_irq(&lock->wait_lock);
@@ -398,6 +423,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock,
{
int ret;
+ rt_mutex_futex_pre_schedule();
raw_spin_lock_irq(&lock->wait_lock);
/* sleep on the mutex */
set_current_state(TASK_INTERRUPTIBLE);
@@ -408,6 +434,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock,
*/
fixup_rt_mutex_waiters(lock, true);
raw_spin_unlock_irq(&lock->wait_lock);
+ rt_mutex_futex_post_schedule();
return ret;
}
@@ -526,6 +553,7 @@ static __always_inline int __mutex_lock_common(struct mutex *lock,
unsigned int subclass,
struct lockdep_map *nest_lock,
unsigned long ip)
+ __acquires(lock) __no_context_analysis
{
int ret;
@@ -647,6 +675,7 @@ EXPORT_SYMBOL(mutex_trylock);
#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
void __sched mutex_unlock(struct mutex *lock)
+ __releases(lock) __no_context_analysis
{
mutex_release(&lock->dep_map, _RET_IP_);
__rt_mutex_unlock(&lock->rtmutex);
diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h
index cf6ddd1b23a2..c38b7bdea7b3 100644
--- a/kernel/locking/rtmutex_common.h
+++ b/kernel/locking/rtmutex_common.h
@@ -79,12 +79,18 @@ struct rt_wake_q_head {
* PI-futex support (proxy locking functions, etc.):
*/
extern void rt_mutex_init_proxy_locked(struct rt_mutex_base *lock,
- struct task_struct *proxy_owner);
-extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock);
+ struct task_struct *proxy_owner)
+ __must_hold(&lock->wait_lock);
+
+extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock);
+
extern int __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
- struct wake_q_head *);
+ struct wake_q_head *)
+ __must_hold(&lock->wait_lock);
+
extern int rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task);
@@ -94,8 +100,9 @@ extern int rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock,
extern bool rt_mutex_cleanup_proxy_lock(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter);
-extern int rt_mutex_futex_trylock(struct rt_mutex_base *l);
-extern int __rt_mutex_futex_trylock(struct rt_mutex_base *l);
+extern int rt_mutex_futex_trylock(struct rt_mutex_base *lock);
+extern int __rt_mutex_futex_trylock(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock);
extern void rt_mutex_futex_unlock(struct rt_mutex_base *lock);
extern bool __rt_mutex_futex_unlock(struct rt_mutex_base *lock,
@@ -109,6 +116,7 @@ extern void rt_mutex_postunlock(struct rt_wake_q_head *wqh);
*/
#ifdef CONFIG_RT_MUTEXES
static inline int rt_mutex_has_waiters(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
return !RB_EMPTY_ROOT(&lock->waiters.rb_root);
}
@@ -120,6 +128,7 @@ static inline int rt_mutex_has_waiters(struct rt_mutex_base *lock)
*/
static inline bool rt_mutex_waiter_is_top_waiter(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
struct rb_node *leftmost = rb_first_cached(&lock->waiters);
@@ -127,6 +136,7 @@ static inline bool rt_mutex_waiter_is_top_waiter(struct rt_mutex_base *lock,
}
static inline struct rt_mutex_waiter *rt_mutex_top_waiter(struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
struct rb_node *leftmost = rb_first_cached(&lock->waiters);
struct rt_mutex_waiter *w = NULL;
@@ -170,9 +180,10 @@ enum rtmutex_chainwalk {
static inline void __rt_mutex_base_init(struct rt_mutex_base *lock)
{
- raw_spin_lock_init(&lock->wait_lock);
- lock->waiters = RB_ROOT_CACHED;
- lock->owner = NULL;
+ scoped_guard (raw_spinlock_init, &lock->wait_lock) {
+ lock->waiters = RB_ROOT_CACHED;
+ lock->owner = NULL;
+ }
}
/* Debug functions */
diff --git a/kernel/locking/rwbase_rt.c b/kernel/locking/rwbase_rt.c
index 9f4322c07486..2835c9ef9b3f 100644
--- a/kernel/locking/rwbase_rt.c
+++ b/kernel/locking/rwbase_rt.c
@@ -174,6 +174,8 @@ static void __sched __rwbase_read_unlock(struct rwbase_rt *rwb,
static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb,
unsigned int state)
{
+ if (trace_contended_release_enabled() && rt_mutex_owner(&rwb->rtmutex))
+ trace_call__contended_release(rwb);
/*
* rwb->readers can only hit 0 when a writer is waiting for the
* active readers to leave the critical section.
@@ -186,6 +188,7 @@ static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb,
static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias,
unsigned long flags)
+ __releases(&rwb->rtmutex.wait_lock)
{
struct rt_mutex_base *rtm = &rwb->rtmutex;
@@ -204,6 +207,8 @@ static inline void rwbase_write_unlock(struct rwbase_rt *rwb)
unsigned long flags;
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
+ if (trace_contended_release_enabled() && rt_mutex_has_waiters(rtm))
+ trace_call__contended_release(rwb);
__rwbase_write_unlock(rwb, WRITER_BIAS, flags);
}
@@ -213,6 +218,8 @@ static inline void rwbase_write_downgrade(struct rwbase_rt *rwb)
unsigned long flags;
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
+ if (trace_contended_release_enabled() && rt_mutex_has_waiters(rtm))
+ trace_call__contended_release(rwb);
/* Release it and account current as reader */
__rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags);
}
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 24df4d98f7d2..b9c180ac1eee 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -72,7 +72,7 @@
#c, atomic_long_read(&(sem)->count), \
(unsigned long) sem->magic, \
atomic_long_read(&(sem)->owner), (long)current, \
- list_empty(&(sem)->wait_list) ? "" : "not ")) \
+ rwsem_is_contended(sem) ? "" : "not ")) \
debug_locks_off(); \
} while (0)
#else
@@ -320,9 +320,10 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
sem->magic = sem;
#endif
atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
- raw_spin_lock_init(&sem->wait_lock);
- INIT_LIST_HEAD(&sem->wait_list);
atomic_long_set(&sem->owner, 0L);
+ scoped_guard (raw_spinlock_init, &sem->wait_lock) {
+ sem->first_waiter = NULL;
+ }
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
osq_lock_init(&sem->osq);
#endif
@@ -341,8 +342,6 @@ struct rwsem_waiter {
unsigned long timeout;
bool handoff_set;
};
-#define rwsem_first_waiter(sem) \
- list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
enum rwsem_wake_type {
RWSEM_WAKE_ANY, /* Wake whatever's at head of wait list */
@@ -365,12 +364,22 @@ enum rwsem_wake_type {
*/
#define MAX_READERS_WAKEUP 0x100
-static inline void
-rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
+static inline
+bool __rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
- lockdep_assert_held(&sem->wait_lock);
- list_add_tail(&waiter->list, &sem->wait_list);
- /* caller will set RWSEM_FLAG_WAITERS */
+ if (list_empty(&waiter->list)) {
+ sem->first_waiter = NULL;
+ return false;
+ }
+
+ if (sem->first_waiter == waiter) {
+ sem->first_waiter = list_first_entry(&waiter->list,
+ struct rwsem_waiter, list);
+ }
+ list_del(&waiter->list);
+
+ return true;
}
/*
@@ -385,14 +394,24 @@ static inline bool
rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
{
lockdep_assert_held(&sem->wait_lock);
- list_del(&waiter->list);
- if (likely(!list_empty(&sem->wait_list)))
+ if (__rwsem_del_waiter(sem, waiter))
return true;
-
atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count);
return false;
}
+static inline
+struct rwsem_waiter *next_waiter(const struct rw_semaphore *sem,
+ const struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
+{
+ struct rwsem_waiter *next = list_first_entry(&waiter->list,
+ struct rwsem_waiter, list);
+ if (next == sem->first_waiter)
+ return NULL;
+ return next;
+}
+
/*
* handle the lock release when processes blocked on it that can now run
* - if we come here from up_xxxx(), then the RWSEM_FLAG_WAITERS bit must
@@ -411,7 +430,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
enum rwsem_wake_type wake_type,
struct wake_q_head *wake_q)
{
- struct rwsem_waiter *waiter, *tmp;
+ struct rwsem_waiter *waiter, *next;
long oldcount, woken = 0, adjustment = 0;
struct list_head wlist;
@@ -421,7 +440,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
* Take a peek at the queue head waiter such that we can determine
* the wakeup(s) to perform.
*/
- waiter = rwsem_first_waiter(sem);
+ waiter = sem->first_waiter;
if (waiter->type == RWSEM_WAITING_FOR_WRITE) {
if (wake_type == RWSEM_WAKE_ANY) {
@@ -506,25 +525,28 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
* put them into wake_q to be woken up later.
*/
INIT_LIST_HEAD(&wlist);
- list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {
+ do {
+ next = next_waiter(sem, waiter);
if (waiter->type == RWSEM_WAITING_FOR_WRITE)
continue;
woken++;
list_move_tail(&waiter->list, &wlist);
+ if (sem->first_waiter == waiter)
+ sem->first_waiter = next;
/*
* Limit # of readers that can be woken up per wakeup call.
*/
if (unlikely(woken >= MAX_READERS_WAKEUP))
break;
- }
+ } while ((waiter = next) != NULL);
adjustment = woken * RWSEM_READER_BIAS - adjustment;
lockevent_cond_inc(rwsem_wake_reader, woken);
oldcount = atomic_long_read(&sem->count);
- if (list_empty(&sem->wait_list)) {
+ if (!sem->first_waiter) {
/*
* Combined with list_move_tail() above, this implies
* rwsem_del_waiter().
@@ -545,7 +567,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
atomic_long_add(adjustment, &sem->count);
/* 2nd pass */
- list_for_each_entry_safe(waiter, tmp, &wlist, list) {
+ list_for_each_entry_safe(waiter, next, &wlist, list) {
struct task_struct *tsk;
tsk = waiter->task;
@@ -577,7 +599,7 @@ rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,
struct wake_q_head *wake_q)
__releases(&sem->wait_lock)
{
- bool first = rwsem_first_waiter(sem) == waiter;
+ bool first = sem->first_waiter == waiter;
wake_q_init(wake_q);
@@ -602,8 +624,9 @@ rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,
*/
static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
- struct rwsem_waiter *first = rwsem_first_waiter(sem);
+ struct rwsem_waiter *first = sem->first_waiter;
long count, new;
lockdep_assert_held(&sem->wait_lock);
@@ -639,7 +662,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
new |= RWSEM_WRITER_LOCKED;
new &= ~RWSEM_FLAG_HANDOFF;
- if (list_is_singular(&sem->wait_list))
+ if (list_empty(&first->list))
new &= ~RWSEM_FLAG_WAITERS;
}
} while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new));
@@ -659,7 +682,8 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
* Have rwsem_try_write_lock() fully imply rwsem_del_waiter() on
* success.
*/
- list_del(&waiter->list);
+ __rwsem_del_waiter(sem, waiter);
+
rwsem_set_owner(sem);
return true;
}
@@ -994,7 +1018,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
{
long adjustment = -RWSEM_READER_BIAS;
long rcnt = (count >> RWSEM_READER_SHIFT);
- struct rwsem_waiter waiter;
+ struct rwsem_waiter waiter, *first;
DEFINE_WAKE_Q(wake_q);
/*
@@ -1019,7 +1043,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
*/
if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) {
raw_spin_lock_irq(&sem->wait_lock);
- if (!list_empty(&sem->wait_list))
+ if (sem->first_waiter)
rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED,
&wake_q);
raw_spin_unlock_irq(&sem->wait_lock);
@@ -1035,7 +1059,8 @@ queue:
waiter.handoff_set = false;
raw_spin_lock_irq(&sem->wait_lock);
- if (list_empty(&sem->wait_list)) {
+ first = sem->first_waiter;
+ if (!first) {
/*
* In case the wait queue is empty and the lock isn't owned
* by a writer, this reader can exit the slowpath and return
@@ -1051,8 +1076,11 @@ queue:
return sem;
}
adjustment += RWSEM_FLAG_WAITERS;
+ INIT_LIST_HEAD(&waiter.list);
+ sem->first_waiter = &waiter;
+ } else {
+ list_add_tail(&waiter.list, &first->list);
}
- rwsem_add_waiter(sem, &waiter);
/* we're now waiting on the lock, but no longer actively locking */
count = atomic_long_add_return(adjustment, &sem->count);
@@ -1110,7 +1138,7 @@ out_nolock:
static struct rw_semaphore __sched *
rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
{
- struct rwsem_waiter waiter;
+ struct rwsem_waiter waiter, *first;
DEFINE_WAKE_Q(wake_q);
/* do optimistic spinning and steal lock if possible */
@@ -1129,10 +1157,10 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
waiter.handoff_set = false;
raw_spin_lock_irq(&sem->wait_lock);
- rwsem_add_waiter(sem, &waiter);
- /* we're now waiting on the lock */
- if (rwsem_first_waiter(sem) != &waiter) {
+ first = sem->first_waiter;
+ if (first) {
+ list_add_tail(&waiter.list, &first->list);
rwsem_cond_wake_waiter(sem, atomic_long_read(&sem->count),
&wake_q);
if (!wake_q_empty(&wake_q)) {
@@ -1145,6 +1173,8 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
raw_spin_lock_irq(&sem->wait_lock);
}
} else {
+ INIT_LIST_HEAD(&waiter.list);
+ sem->first_waiter = &waiter;
atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count);
}
@@ -1218,7 +1248,7 @@ static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)
raw_spin_lock_irqsave(&sem->wait_lock, flags);
- if (!list_empty(&sem->wait_list))
+ if (sem->first_waiter)
rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
@@ -1239,7 +1269,7 @@ static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)
raw_spin_lock_irqsave(&sem->wait_lock, flags);
- if (!list_empty(&sem->wait_list))
+ if (sem->first_waiter)
rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q);
raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
@@ -1357,6 +1387,8 @@ static inline void __up_read(struct rw_semaphore *sem)
rwsem_clear_reader_owned(sem);
tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);
DEBUG_RWSEMS_WARN_ON(tmp < 0, sem);
+ if (trace_contended_release_enabled() && (tmp & RWSEM_FLAG_WAITERS))
+ trace_call__contended_release(sem);
if (unlikely((tmp & (RWSEM_LOCK_MASK|RWSEM_FLAG_WAITERS)) ==
RWSEM_FLAG_WAITERS)) {
clear_nonspinnable(sem);
@@ -1383,8 +1415,10 @@ static inline void __up_write(struct rw_semaphore *sem)
preempt_disable();
rwsem_clear_owner(sem);
tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
- if (unlikely(tmp & RWSEM_FLAG_WAITERS))
+ if (unlikely(tmp & RWSEM_FLAG_WAITERS)) {
+ trace_contended_release(sem);
rwsem_wake(sem);
+ }
preempt_enable();
}
@@ -1407,8 +1441,10 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
tmp = atomic_long_fetch_add_release(
-RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count);
rwsem_set_reader_owned(sem);
- if (tmp & RWSEM_FLAG_WAITERS)
+ if (tmp & RWSEM_FLAG_WAITERS) {
+ trace_contended_release(sem);
rwsem_downgrade_wake(sem);
+ }
preempt_enable();
}
@@ -1532,6 +1568,7 @@ static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem)
* lock for reading
*/
void __sched down_read(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1541,6 +1578,7 @@ void __sched down_read(struct rw_semaphore *sem)
EXPORT_SYMBOL(down_read);
int __sched down_read_interruptible(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1555,6 +1593,7 @@ int __sched down_read_interruptible(struct rw_semaphore *sem)
EXPORT_SYMBOL(down_read_interruptible);
int __sched down_read_killable(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1572,6 +1611,7 @@ EXPORT_SYMBOL(down_read_killable);
* trylock for reading -- returns 1 if successful, 0 if contention
*/
int down_read_trylock(struct rw_semaphore *sem)
+ __no_context_analysis
{
int ret = __down_read_trylock(sem);
@@ -1585,6 +1625,7 @@ EXPORT_SYMBOL(down_read_trylock);
* lock for writing
*/
void __sched down_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1596,6 +1637,7 @@ EXPORT_SYMBOL(down_write);
* lock for writing
*/
int __sched down_write_killable(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1614,6 +1656,7 @@ EXPORT_SYMBOL(down_write_killable);
* trylock for writing -- returns 1 if successful, 0 if contention
*/
int down_write_trylock(struct rw_semaphore *sem)
+ __no_context_analysis
{
int ret = __down_write_trylock(sem);
@@ -1628,6 +1671,7 @@ EXPORT_SYMBOL(down_write_trylock);
* release a read lock
*/
void up_read(struct rw_semaphore *sem)
+ __no_context_analysis
{
rwsem_release(&sem->dep_map, _RET_IP_);
__up_read(sem);
@@ -1638,6 +1682,7 @@ EXPORT_SYMBOL(up_read);
* release a write lock
*/
void up_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
rwsem_release(&sem->dep_map, _RET_IP_);
__up_write(sem);
@@ -1648,6 +1693,7 @@ EXPORT_SYMBOL(up_write);
* downgrade write lock to read lock
*/
void downgrade_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
lock_downgrade(&sem->dep_map, _RET_IP_);
__downgrade_write(sem);
@@ -1657,6 +1703,7 @@ EXPORT_SYMBOL(downgrade_write);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
void down_read_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1665,6 +1712,7 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_read_nested);
int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1679,6 +1727,7 @@ int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_read_killable_nested);
void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
@@ -1687,6 +1736,7 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
EXPORT_SYMBOL(_down_write_nest_lock);
void down_read_non_owner(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
__down_read(sem);
@@ -1701,6 +1751,7 @@ void down_read_non_owner(struct rw_semaphore *sem)
EXPORT_SYMBOL(down_read_non_owner);
void down_write_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1709,6 +1760,7 @@ void down_write_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_write_nested);
int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1724,6 +1776,7 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_write_killable_nested);
void up_read_non_owner(struct rw_semaphore *sem)
+ __no_context_analysis
{
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
__up_read(sem);
diff --git a/kernel/locking/semaphore.c b/kernel/locking/semaphore.c
index 3ef032e22f7e..233730c25933 100644
--- a/kernel/locking/semaphore.c
+++ b/kernel/locking/semaphore.c
@@ -21,7 +21,7 @@
* too.
*
* The ->count variable represents how many more tasks can acquire this
- * semaphore. If it's zero, there may be tasks waiting on the wait_list.
+ * semaphore. If it's zero, there may be waiters.
*/
#include <linux/compiler.h>
@@ -226,10 +226,14 @@ void __sched up(struct semaphore *sem)
hung_task_sem_clear_if_holder(sem);
- if (likely(list_empty(&sem->wait_list)))
+ if (likely(!sem->first_waiter))
sem->count++;
else
__up(sem, &wake_q);
+
+ if (trace_contended_release_enabled() && !wake_q_empty(&wake_q))
+ trace_call__contended_release(sem);
+
raw_spin_unlock_irqrestore(&sem->lock, flags);
if (!wake_q_empty(&wake_q))
wake_up_q(&wake_q);
@@ -244,6 +248,21 @@ struct semaphore_waiter {
bool up;
};
+static inline
+void sem_del_waiter(struct semaphore *sem, struct semaphore_waiter *waiter)
+{
+ if (list_empty(&waiter->list)) {
+ sem->first_waiter = NULL;
+ return;
+ }
+
+ if (sem->first_waiter == waiter) {
+ sem->first_waiter = list_first_entry(&waiter->list,
+ struct semaphore_waiter, list);
+ }
+ list_del(&waiter->list);
+}
+
/*
* Because this function is inlined, the 'state' parameter will be
* constant, and thus optimised away by the compiler. Likewise the
@@ -252,9 +271,15 @@ struct semaphore_waiter {
static inline int __sched ___down_common(struct semaphore *sem, long state,
long timeout)
{
- struct semaphore_waiter waiter;
-
- list_add_tail(&waiter.list, &sem->wait_list);
+ struct semaphore_waiter waiter, *first;
+
+ first = sem->first_waiter;
+ if (first) {
+ list_add_tail(&waiter.list, &first->list);
+ } else {
+ INIT_LIST_HEAD(&waiter.list);
+ sem->first_waiter = &waiter;
+ }
waiter.task = current;
waiter.up = false;
@@ -274,11 +299,11 @@ static inline int __sched ___down_common(struct semaphore *sem, long state,
}
timed_out:
- list_del(&waiter.list);
+ sem_del_waiter(sem, &waiter);
return -ETIME;
interrupted:
- list_del(&waiter.list);
+ sem_del_waiter(sem, &waiter);
return -EINTR;
}
@@ -321,9 +346,9 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long timeout)
static noinline void __sched __up(struct semaphore *sem,
struct wake_q_head *wake_q)
{
- struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,
- struct semaphore_waiter, list);
- list_del(&waiter->list);
+ struct semaphore_waiter *waiter = sem->first_waiter;
+
+ sem_del_waiter(sem, waiter);
waiter->up = true;
wake_q_add(wake_q, waiter->task);
}
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index 7685defd7c52..83a17eaf5717 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -64,8 +64,9 @@ EXPORT_PER_CPU_SYMBOL(__mmiowb_state);
* time (making _this_ CPU preemptible if possible), and we also signal
* towards that other CPU that it should break the lock ASAP.
*/
-#define BUILD_LOCK_OPS(op, locktype) \
+#define BUILD_LOCK_OPS(op, locktype, lock_ctx_op) \
static void __lockfunc __raw_##op##_lock(locktype##_t *lock) \
+ lock_ctx_op(lock) \
{ \
for (;;) { \
preempt_disable(); \
@@ -78,6 +79,7 @@ static void __lockfunc __raw_##op##_lock(locktype##_t *lock) \
} \
\
static unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \
+ lock_ctx_op(lock) \
{ \
unsigned long flags; \
\
@@ -96,11 +98,13 @@ static unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \
} \
\
static void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \
+ lock_ctx_op(lock) \
{ \
_raw_##op##_lock_irqsave(lock); \
} \
\
static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
+ lock_ctx_op(lock) \
{ \
unsigned long flags; \
\
@@ -123,11 +127,26 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
* __[spin|read|write]_lock_irqsave()
* __[spin|read|write]_lock_bh()
*/
-BUILD_LOCK_OPS(spin, raw_spinlock);
+BUILD_LOCK_OPS(spin, raw_spinlock, __acquires);
+
+/* No rwlock_t variants for now, so just build this function by hand */
+static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ for (;;) {
+ preempt_disable();
+ local_interrupt_disable();
+ if (likely(do_raw_spin_trylock(lock)))
+ break;
+ local_interrupt_enable();
+ preempt_enable();
+
+ arch_spin_relax(&lock->raw_lock);
+ }
+}
#ifndef CONFIG_PREEMPT_RT
-BUILD_LOCK_OPS(read, rwlock);
-BUILD_LOCK_OPS(write, rwlock);
+BUILD_LOCK_OPS(read, rwlock, __acquires_shared);
+BUILD_LOCK_OPS(write, rwlock, __acquires);
#endif
#endif
@@ -172,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_lock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
+noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ __raw_spin_lock_irq_disable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_LOCK_BH
noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
{
@@ -204,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_unlock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+{
+ __raw_spin_unlock_irq_enable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
{
diff --git a/kernel/locking/spinlock_rt.c b/kernel/locking/spinlock_rt.c
index db1e11b45de6..1d5e1b3c60bf 100644
--- a/kernel/locking/spinlock_rt.c
+++ b/kernel/locking/spinlock_rt.c
@@ -79,10 +79,27 @@ void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU)
{
spin_release(&lock->dep_map, _RET_IP_);
migrate_enable();
- rcu_read_unlock();
if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
rt_mutex_slowunlock(&lock->lock);
+
+ /*
+ * This must be last to prevent the following UAF:
+ *
+ * T1 T2
+ * spin_lock(&p->lock); rcu_read_lock();
+ * invalidate(p); p = rcu_dereference(ptr);
+ * rcu_assign_pointer(ptr, NULL); if (!p) return;
+ * spin_unlock(&p->lock); spin_lock(&p->lock);
+ * kfree_rcu(p); rcu_read_unlock();
+ * ....
+ * spin_unlock(&p->lock)
+ * rcu_read_unlock(); // Ends grace period
+ * rcu_do_batch()
+ * kfree(p);
+ * UAF -> rt_mutex_cmpxchg_release(&p->lock.lock...)
+ */
+ rcu_read_unlock();
}
EXPORT_SYMBOL(rt_spin_unlock);
@@ -262,17 +279,21 @@ void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU)
{
rwlock_release(&rwlock->dep_map, _RET_IP_);
migrate_enable();
- rcu_read_unlock();
rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
+
+ /* This must be last. See comment in rt_spin_unlock() */
+ rcu_read_unlock();
}
EXPORT_SYMBOL(rt_read_unlock);
void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU)
{
rwlock_release(&rwlock->dep_map, _RET_IP_);
- rcu_read_unlock();
migrate_enable();
rwbase_write_unlock(&rwlock->rwbase);
+
+ /* This must be last. See comment in rt_spin_unlock() */
+ rcu_read_unlock();
}
EXPORT_SYMBOL(rt_write_unlock);
diff --git a/kernel/locking/test-ww_mutex.c b/kernel/locking/test-ww_mutex.c
index bcb1b9fea588..838d631544ed 100644
--- a/kernel/locking/test-ww_mutex.c
+++ b/kernel/locking/test-ww_mutex.c
@@ -13,7 +13,8 @@
#include <linux/slab.h>
#include <linux/ww_mutex.h>
-static DEFINE_WD_CLASS(ww_class);
+static DEFINE_WD_CLASS(wd_class);
+static DEFINE_WW_CLASS(ww_class);
struct workqueue_struct *wq;
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
@@ -54,16 +55,16 @@ static void test_mutex_work(struct work_struct *work)
ww_mutex_unlock(&mtx->mutex);
}
-static int __test_mutex(unsigned int flags)
+static int __test_mutex(struct ww_class *class, unsigned int flags)
{
#define TIMEOUT (HZ / 16)
struct test_mutex mtx;
struct ww_acquire_ctx ctx;
int ret;
- ww_mutex_init(&mtx.mutex, &ww_class);
+ ww_mutex_init(&mtx.mutex, class);
if (flags & TEST_MTX_CTX)
- ww_acquire_init(&ctx, &ww_class);
+ ww_acquire_init(&ctx, class);
INIT_WORK_ONSTACK(&mtx.work, test_mutex_work);
init_completion(&mtx.ready);
@@ -71,7 +72,7 @@ static int __test_mutex(unsigned int flags)
init_completion(&mtx.done);
mtx.flags = flags;
- schedule_work(&mtx.work);
+ queue_work(wq, &mtx.work);
wait_for_completion(&mtx.ready);
ww_mutex_lock(&mtx.mutex, (flags & TEST_MTX_CTX) ? &ctx : NULL);
@@ -106,13 +107,13 @@ static int __test_mutex(unsigned int flags)
#undef TIMEOUT
}
-static int test_mutex(void)
+static int test_mutex(struct ww_class *class)
{
int ret;
int i;
for (i = 0; i < __TEST_MTX_LAST; i++) {
- ret = __test_mutex(i);
+ ret = __test_mutex(class, i);
if (ret)
return ret;
}
@@ -120,15 +121,15 @@ static int test_mutex(void)
return 0;
}
-static int test_aa(bool trylock)
+static int test_aa(struct ww_class *class, bool trylock)
{
struct ww_mutex mutex;
struct ww_acquire_ctx ctx;
int ret;
const char *from = trylock ? "trylock" : "lock";
- ww_mutex_init(&mutex, &ww_class);
- ww_acquire_init(&ctx, &ww_class);
+ ww_mutex_init(&mutex, class);
+ ww_acquire_init(&ctx, class);
if (!trylock) {
ret = ww_mutex_lock(&mutex, &ctx);
@@ -177,6 +178,7 @@ out:
struct test_abba {
struct work_struct work;
+ struct ww_class *class;
struct ww_mutex a_mutex;
struct ww_mutex b_mutex;
struct completion a_ready;
@@ -191,7 +193,7 @@ static void test_abba_work(struct work_struct *work)
struct ww_acquire_ctx ctx;
int err;
- ww_acquire_init_noinject(&ctx, &ww_class);
+ ww_acquire_init_noinject(&ctx, abba->class);
if (!abba->trylock)
ww_mutex_lock(&abba->b_mutex, &ctx);
else
@@ -217,23 +219,24 @@ static void test_abba_work(struct work_struct *work)
abba->result = err;
}
-static int test_abba(bool trylock, bool resolve)
+static int test_abba(struct ww_class *class, bool trylock, bool resolve)
{
struct test_abba abba;
struct ww_acquire_ctx ctx;
int err, ret;
- ww_mutex_init(&abba.a_mutex, &ww_class);
- ww_mutex_init(&abba.b_mutex, &ww_class);
+ ww_mutex_init(&abba.a_mutex, class);
+ ww_mutex_init(&abba.b_mutex, class);
INIT_WORK_ONSTACK(&abba.work, test_abba_work);
init_completion(&abba.a_ready);
init_completion(&abba.b_ready);
+ abba.class = class;
abba.trylock = trylock;
abba.resolve = resolve;
- schedule_work(&abba.work);
+ queue_work(wq, &abba.work);
- ww_acquire_init_noinject(&ctx, &ww_class);
+ ww_acquire_init_noinject(&ctx, class);
if (!trylock)
ww_mutex_lock(&abba.a_mutex, &ctx);
else
@@ -278,6 +281,7 @@ static int test_abba(bool trylock, bool resolve)
struct test_cycle {
struct work_struct work;
+ struct ww_class *class;
struct ww_mutex a_mutex;
struct ww_mutex *b_mutex;
struct completion *a_signal;
@@ -291,7 +295,7 @@ static void test_cycle_work(struct work_struct *work)
struct ww_acquire_ctx ctx;
int err, erra = 0;
- ww_acquire_init_noinject(&ctx, &ww_class);
+ ww_acquire_init_noinject(&ctx, cycle->class);
ww_mutex_lock(&cycle->a_mutex, &ctx);
complete(cycle->a_signal);
@@ -314,20 +318,21 @@ static void test_cycle_work(struct work_struct *work)
cycle->result = err ?: erra;
}
-static int __test_cycle(unsigned int nthreads)
+static int __test_cycle(struct ww_class *class, unsigned int nthreads)
{
struct test_cycle *cycles;
unsigned int n, last = nthreads - 1;
int ret;
- cycles = kmalloc_array(nthreads, sizeof(*cycles), GFP_KERNEL);
+ cycles = kmalloc_objs(*cycles, nthreads);
if (!cycles)
return -ENOMEM;
for (n = 0; n < nthreads; n++) {
struct test_cycle *cycle = &cycles[n];
- ww_mutex_init(&cycle->a_mutex, &ww_class);
+ cycle->class = class;
+ ww_mutex_init(&cycle->a_mutex, class);
if (n == last)
cycle->b_mutex = &cycles[0].a_mutex;
else
@@ -367,13 +372,13 @@ static int __test_cycle(unsigned int nthreads)
return ret;
}
-static int test_cycle(unsigned int ncpus)
+static int test_cycle(struct ww_class *class, unsigned int ncpus)
{
unsigned int n;
int ret;
for (n = 2; n <= ncpus + 1; n++) {
- ret = __test_cycle(n);
+ ret = __test_cycle(class, n);
if (ret)
return ret;
}
@@ -384,6 +389,7 @@ static int test_cycle(unsigned int ncpus)
struct stress {
struct work_struct work;
struct ww_mutex *locks;
+ struct ww_class *class;
unsigned long timeout;
int nlocks;
};
@@ -406,7 +412,7 @@ static int *get_random_order(int count)
int *order;
int n, r;
- order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
+ order = kmalloc_objs(*order, count);
if (!order)
return order;
@@ -443,7 +449,7 @@ static void stress_inorder_work(struct work_struct *work)
int contended = -1;
int n, err;
- ww_acquire_init(&ctx, &ww_class);
+ ww_acquire_init(&ctx, stress->class);
retry:
err = 0;
for (n = 0; n < nlocks; n++) {
@@ -500,7 +506,7 @@ static void stress_reorder_work(struct work_struct *work)
return;
for (n = 0; n < stress->nlocks; n++) {
- ll = kmalloc(sizeof(*ll), GFP_KERNEL);
+ ll = kmalloc_obj(*ll);
if (!ll)
goto out;
@@ -511,7 +517,7 @@ static void stress_reorder_work(struct work_struct *work)
order = NULL;
do {
- ww_acquire_init(&ctx, &ww_class);
+ ww_acquire_init(&ctx, stress->class);
list_for_each_entry(ll, &locks, link) {
err = ww_mutex_lock(ll->lock, &ctx);
@@ -570,25 +576,24 @@ static void stress_one_work(struct work_struct *work)
#define STRESS_ONE BIT(2)
#define STRESS_ALL (STRESS_INORDER | STRESS_REORDER | STRESS_ONE)
-static int stress(int nlocks, int nthreads, unsigned int flags)
+static int stress(struct ww_class *class, int nlocks, int nthreads, unsigned int flags)
{
struct ww_mutex *locks;
struct stress *stress_array;
int n, count;
- locks = kmalloc_array(nlocks, sizeof(*locks), GFP_KERNEL);
+ locks = kmalloc_objs(*locks, nlocks);
if (!locks)
return -ENOMEM;
- stress_array = kmalloc_array(nthreads, sizeof(*stress_array),
- GFP_KERNEL);
+ stress_array = kmalloc_objs(*stress_array, nthreads);
if (!stress_array) {
kfree(locks);
return -ENOMEM;
}
for (n = 0; n < nlocks; n++)
- ww_mutex_init(&locks[n], &ww_class);
+ ww_mutex_init(&locks[n], class);
count = 0;
for (n = 0; nthreads; n++) {
@@ -617,6 +622,7 @@ static int stress(int nlocks, int nthreads, unsigned int flags)
stress = &stress_array[count++];
INIT_WORK(&stress->work, fn);
+ stress->class = class;
stress->locks = locks;
stress->nlocks = nlocks;
stress->timeout = jiffies + 2*HZ;
@@ -635,59 +641,131 @@ static int stress(int nlocks, int nthreads, unsigned int flags)
return 0;
}
-static int __init test_ww_mutex_init(void)
+static int run_tests(struct ww_class *class)
{
int ncpus = num_online_cpus();
int ret, i;
- printk(KERN_INFO "Beginning ww mutex selftests\n");
-
- prandom_seed_state(&rng, get_random_u64());
-
- wq = alloc_workqueue("test-ww_mutex", WQ_UNBOUND, 0);
- if (!wq)
- return -ENOMEM;
-
- ret = test_mutex();
+ ret = test_mutex(class);
if (ret)
return ret;
- ret = test_aa(false);
+ ret = test_aa(class, false);
if (ret)
return ret;
- ret = test_aa(true);
+ ret = test_aa(class, true);
if (ret)
return ret;
for (i = 0; i < 4; i++) {
- ret = test_abba(i & 1, i & 2);
+ ret = test_abba(class, i & 1, i & 2);
if (ret)
return ret;
}
- ret = test_cycle(ncpus);
+ ret = test_cycle(class, ncpus);
+ if (ret)
+ return ret;
+
+ ret = stress(class, 16, 2 * ncpus, STRESS_INORDER);
+ if (ret)
+ return ret;
+
+ ret = stress(class, 16, 2 * ncpus, STRESS_REORDER);
if (ret)
return ret;
- ret = stress(16, 2*ncpus, STRESS_INORDER);
+ ret = stress(class, 2046, hweight32(STRESS_ALL) * ncpus, STRESS_ALL);
if (ret)
return ret;
- ret = stress(16, 2*ncpus, STRESS_REORDER);
+ return 0;
+}
+
+static int run_test_classes(void)
+{
+ int ret;
+
+ pr_info("Beginning ww (wound) mutex selftests\n");
+
+ ret = run_tests(&ww_class);
if (ret)
return ret;
- ret = stress(2046, hweight32(STRESS_ALL)*ncpus, STRESS_ALL);
+ pr_info("Beginning ww (die) mutex selftests\n");
+ ret = run_tests(&wd_class);
if (ret)
return ret;
- printk(KERN_INFO "All ww mutex selftests passed\n");
+ pr_info("All ww mutex selftests passed\n");
return 0;
}
+static DEFINE_MUTEX(run_lock);
+
+static ssize_t run_tests_store(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ if (!mutex_trylock(&run_lock)) {
+ pr_err("Test already running\n");
+ return count;
+ }
+
+ run_test_classes();
+ mutex_unlock(&run_lock);
+
+ return count;
+}
+
+static struct kobj_attribute run_tests_attribute =
+ __ATTR(run_tests, 0664, NULL, run_tests_store);
+
+static struct attribute *attrs[] = {
+ &run_tests_attribute.attr,
+ NULL, /* need to NULL terminate the list of attributes */
+};
+
+static struct attribute_group attr_group = {
+ .attrs = attrs,
+};
+
+static struct kobject *test_ww_mutex_kobj;
+
+static int __init test_ww_mutex_init(void)
+{
+ int ret;
+
+ prandom_seed_state(&rng, get_random_u64());
+
+ wq = alloc_workqueue("test-ww_mutex", WQ_UNBOUND, 0);
+ if (!wq)
+ return -ENOMEM;
+
+ test_ww_mutex_kobj = kobject_create_and_add("test_ww_mutex", kernel_kobj);
+ if (!test_ww_mutex_kobj) {
+ destroy_workqueue(wq);
+ return -ENOMEM;
+ }
+
+ /* Create the files associated with this kobject */
+ ret = sysfs_create_group(test_ww_mutex_kobj, &attr_group);
+ if (ret) {
+ kobject_put(test_ww_mutex_kobj);
+ destroy_workqueue(wq);
+ return ret;
+ }
+
+ mutex_lock(&run_lock);
+ ret = run_test_classes();
+ mutex_unlock(&run_lock);
+
+ return ret;
+}
+
static void __exit test_ww_mutex_exit(void)
{
+ kobject_put(test_ww_mutex_kobj);
destroy_workqueue(wq);
}
diff --git a/kernel/locking/ww_mutex.h b/kernel/locking/ww_mutex.h
index 31a785afee6c..d62b49b53ec3 100644
--- a/kernel/locking/ww_mutex.h
+++ b/kernel/locking/ww_mutex.h
@@ -4,58 +4,83 @@
#define MUTEX mutex
#define MUTEX_WAITER mutex_waiter
+#define WAIT_LOCK wait_lock
+
+/*
+ * +--------+
+ * | first |
+ * +--------+
+ * |
+ * v
+ * +----+ +----+ +----+
+ * | W3 | <-> | W1 | <-> | W2 |
+ * +----+ +----+ +----+
+ * ^ ^
+ * +---------------------+
+ */
static inline struct mutex_waiter *
__ww_waiter_first(struct mutex *lock)
+ __must_hold(&lock->wait_lock)
{
- struct mutex_waiter *w;
-
- w = list_first_entry(&lock->wait_list, struct mutex_waiter, list);
- if (list_entry_is_head(w, &lock->wait_list, list))
- return NULL;
-
- return w;
+ return lock->first_waiter;
}
+/*
+ * for (cur = __ww_waiter_first(); cur; cur = __ww_waiter_next())
+ *
+ * Should iterate like: W1, W2, W3
+ */
static inline struct mutex_waiter *
__ww_waiter_next(struct mutex *lock, struct mutex_waiter *w)
+ __must_hold(&lock->wait_lock)
{
w = list_next_entry(w, list);
- if (list_entry_is_head(w, &lock->wait_list, list))
+ /*
+ * Terminate if the next entry is the first again, that has already
+ * been observed.
+ */
+ if (lock->first_waiter == w)
return NULL;
return w;
}
+/*
+ * for (cur = __ww_waiter_last(); cur; cur = __ww_waiter_prev())
+ *
+ * Should iterate like: W3, W2, W1
+ */
static inline struct mutex_waiter *
__ww_waiter_prev(struct mutex *lock, struct mutex_waiter *w)
+ __must_hold(&lock->wait_lock)
{
- w = list_prev_entry(w, list);
- if (list_entry_is_head(w, &lock->wait_list, list))
+ /*
+ * Terminate at the first entry, the previous entry of first is the
+ * last and that has already been observed.
+ */
+ if (lock->first_waiter == w)
return NULL;
- return w;
+ return list_prev_entry(w, list);
}
static inline struct mutex_waiter *
__ww_waiter_last(struct mutex *lock)
+ __must_hold(&lock->wait_lock)
{
- struct mutex_waiter *w;
-
- w = list_last_entry(&lock->wait_list, struct mutex_waiter, list);
- if (list_entry_is_head(w, &lock->wait_list, list))
- return NULL;
+ struct mutex_waiter *w = lock->first_waiter;
+ if (w)
+ w = list_prev_entry(w, list);
return w;
}
static inline void
__ww_waiter_add(struct mutex *lock, struct mutex_waiter *waiter, struct mutex_waiter *pos)
+ __must_hold(&lock->wait_lock)
{
- struct list_head *p = &lock->wait_list;
- if (pos)
- p = &pos->list;
- __mutex_add_waiter(lock, waiter, p);
+ __mutex_add_waiter(lock, waiter, pos);
}
static inline struct task_struct *
@@ -71,16 +96,19 @@ __ww_mutex_has_waiters(struct mutex *lock)
}
static inline void lock_wait_lock(struct mutex *lock, unsigned long *flags)
+ __acquires(&lock->wait_lock)
{
raw_spin_lock_irqsave(&lock->wait_lock, *flags);
}
static inline void unlock_wait_lock(struct mutex *lock, unsigned long *flags)
+ __releases(&lock->wait_lock)
{
raw_spin_unlock_irqrestore(&lock->wait_lock, *flags);
}
static inline void lockdep_assert_wait_lock_held(struct mutex *lock)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
}
@@ -89,9 +117,11 @@ static inline void lockdep_assert_wait_lock_held(struct mutex *lock)
#define MUTEX rt_mutex
#define MUTEX_WAITER rt_mutex_waiter
+#define WAIT_LOCK rtmutex.wait_lock
static inline struct rt_mutex_waiter *
__ww_waiter_first(struct rt_mutex *lock)
+ __must_hold(&lock->rtmutex.wait_lock)
{
struct rb_node *n = rb_first(&lock->rtmutex.waiters.rb_root);
if (!n)
@@ -119,6 +149,7 @@ __ww_waiter_prev(struct rt_mutex *lock, struct rt_mutex_waiter *w)
static inline struct rt_mutex_waiter *
__ww_waiter_last(struct rt_mutex *lock)
+ __must_hold(&lock->rtmutex.wait_lock)
{
struct rb_node *n = rb_last(&lock->rtmutex.waiters.rb_root);
if (!n)
@@ -140,21 +171,25 @@ __ww_mutex_owner(struct rt_mutex *lock)
static inline bool
__ww_mutex_has_waiters(struct rt_mutex *lock)
+ __must_hold(&lock->rtmutex.wait_lock)
{
return rt_mutex_has_waiters(&lock->rtmutex);
}
static inline void lock_wait_lock(struct rt_mutex *lock, unsigned long *flags)
+ __acquires(&lock->rtmutex.wait_lock)
{
raw_spin_lock_irqsave(&lock->rtmutex.wait_lock, *flags);
}
static inline void unlock_wait_lock(struct rt_mutex *lock, unsigned long *flags)
+ __releases(&lock->rtmutex.wait_lock)
{
raw_spin_unlock_irqrestore(&lock->rtmutex.wait_lock, *flags);
}
static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock)
+ __must_hold(&lock->rtmutex.wait_lock)
{
lockdep_assert_held(&lock->rtmutex.wait_lock);
}
@@ -285,11 +320,11 @@ __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter,
debug_mutex_wake_waiter(lock, waiter);
#endif
/*
- * When waking up the task to die, be sure to clear the
- * blocked_on pointer. Otherwise we can see circular
- * blocked_on relationships that can't resolve.
+ * When waking up the task to die, be sure to set the
+ * blocked_on to PROXY_WAKING. Otherwise we can see
+ * circular blocked_on relationships that can't resolve.
*/
- __clear_task_blocked_on(waiter->task, lock);
+ clear_task_blocked_on(waiter->task, lock);
wake_q_add(wake_q, waiter->task);
}
@@ -307,6 +342,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock,
struct ww_acquire_ctx *ww_ctx,
struct ww_acquire_ctx *hold_ctx,
struct wake_q_head *wake_q)
+ __must_hold(&lock->WAIT_LOCK)
{
struct task_struct *owner = __ww_mutex_owner(lock);
@@ -339,15 +375,15 @@ static bool __ww_mutex_wound(struct MUTEX *lock,
*/
if (owner != current) {
/*
- * When waking up the task to wound, be sure to clear the
- * blocked_on pointer. Otherwise we can see circular
- * blocked_on relationships that can't resolve.
+ * When waking up the task to wound, be sure to set the
+ * blocked_on to PROXY_WAKING. Otherwise we can see
+ * circular blocked_on relationships that can't resolve.
*
* NOTE: We pass NULL here instead of lock, because we
* are waking the mutex owner, who may be currently
* blocked on a different mutex.
*/
- __clear_task_blocked_on(owner, NULL);
+ clear_task_blocked_on(owner, NULL);
wake_q_add(wake_q, owner);
}
return true;
@@ -371,6 +407,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock,
static void
__ww_mutex_check_waiters(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx,
struct wake_q_head *wake_q)
+ __must_hold(&lock->WAIT_LOCK)
{
struct MUTEX_WAITER *cur;
@@ -397,6 +434,7 @@ ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
{
DEFINE_WAKE_Q(wake_q);
unsigned long flags;
+ bool has_waiters;
ww_mutex_lock_acquired(lock, ctx);
@@ -418,7 +456,8 @@ ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
* __ww_mutex_add_waiter() and makes sure we either observe ww->ctx
* and/or !empty list.
*/
- if (likely(!__ww_mutex_has_waiters(&lock->base)))
+ has_waiters = data_race(__ww_mutex_has_waiters(&lock->base));
+ if (likely(!has_waiters))
return;
/*
@@ -464,6 +503,7 @@ __ww_mutex_kill(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx)
static inline int
__ww_mutex_check_kill(struct MUTEX *lock, struct MUTEX_WAITER *waiter,
struct ww_acquire_ctx *ctx)
+ __must_hold(&lock->WAIT_LOCK)
{
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx);
@@ -514,6 +554,7 @@ __ww_mutex_add_waiter(struct MUTEX_WAITER *waiter,
struct MUTEX *lock,
struct ww_acquire_ctx *ww_ctx,
struct wake_q_head *wake_q)
+ __must_hold(&lock->WAIT_LOCK)
{
struct MUTEX_WAITER *cur, *pos = NULL;
bool is_wait_die;
diff --git a/kernel/locking/ww_rt_mutex.c b/kernel/locking/ww_rt_mutex.c
index c7196de838ed..e07fb3b96bc3 100644
--- a/kernel/locking/ww_rt_mutex.c
+++ b/kernel/locking/ww_rt_mutex.c
@@ -90,6 +90,7 @@ ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
EXPORT_SYMBOL(ww_mutex_lock_interruptible);
void __sched ww_mutex_unlock(struct ww_mutex *lock)
+ __no_context_analysis
{
struct rt_mutex *rtm = &lock->base;
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-26 17:38:26 +0000