diff options
| -rw-r--r-- | kernel/locking/rwsem.c | 31 |
1 files changed, 26 insertions, 5 deletions
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index e1840b7c5310..ded96023f4dc 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -255,6 +255,14 @@ enum writer_wait_state { #define RWSEM_WAIT_TIMEOUT DIV_ROUND_UP(HZ, 250) /* + * Magic number to batch-wakeup waiting readers, even when writers are + * also present in the queue. This both limits the amount of work the + * waking thread must do and also prevents any potential counter overflow, + * however unlikely. + */ +#define MAX_READERS_WAKEUP 0x100 + +/* * handle the lock release when processes blocked on it that can now run * - if we come here from up_xxxx(), then the RWSEM_FLAG_WAITERS bit must * have been set. @@ -329,11 +337,17 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, } /* - * Grant an infinite number of read locks to the readers at the front - * of the queue. We know that woken will be at least 1 as we accounted + * Grant up to MAX_READERS_WAKEUP read locks to all the readers in the + * queue. We know that the woken will be at least 1 as we accounted * for above. Note we increment the 'active part' of the count by the * number of readers before waking any processes up. * + * This is an adaptation of the phase-fair R/W locks where at the + * reader phase (first waiter is a reader), all readers are eligible + * to acquire the lock at the same time irrespective of their order + * in the queue. The writers acquire the lock according to their + * order in the queue. + * * We have to do wakeup in 2 passes to prevent the possibility that * the reader count may be decremented before it is incremented. It * is because the to-be-woken waiter may not have slept yet. So it @@ -345,13 +359,20 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, * 2) For each waiters in the new list, clear waiter->task and * put them into wake_q to be woken up later. */ - list_for_each_entry(waiter, &sem->wait_list, list) { + INIT_LIST_HEAD(&wlist); + list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { if (waiter->type == RWSEM_WAITING_FOR_WRITE) - break; + continue; woken++; + list_move_tail(&waiter->list, &wlist); + + /* + * Limit # of readers that can be woken up per wakeup call. + */ + if (woken >= MAX_READERS_WAKEUP) + break; } - list_cut_before(&wlist, &sem->wait_list, &waiter->list); adjustment = woken * RWSEM_READER_BIAS - adjustment; lockevent_cond_inc(rwsem_wake_reader, woken); |