diff options
Diffstat (limited to 'kernel/futex')
| -rw-r--r-- | kernel/futex/core.c | 61 | ||||
| -rw-r--r-- | kernel/futex/requeue.c | 2 | ||||
| -rw-r--r-- | kernel/futex/waitwake.c | 8 |
3 files changed, 39 insertions, 32 deletions
diff --git a/kernel/futex/core.c b/kernel/futex/core.c index a061f54b606d..a240638f69e5 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -213,10 +213,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash * futex_rehash_private(fph, new); } new->state = FR_PERCPU; - scoped_guard(rcu) { - mmph->batches = get_state_synchronize_rcu(); - rcu_assign_pointer(mmph->hash, new); - } + rcu_assign_pointer(mmph->hash, new); + /* + * mmph->batches must reference a grace period which started after + * mmph->hash was assigned. See futex_ref_drop(). + */ + mmph->batches = get_state_synchronize_rcu(); kvfree_rcu(fph, rcu); return true; } @@ -437,45 +439,51 @@ struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper } /* - * Generate a machine wide unique identifier for this inode. + * Generate a machine wide unique identifier for this object (inode or mm). * * This relies on u64 not wrapping in the life-time of the machine; which with * 1ns resolution means almost 585 years. - * - * This further relies on the fact that a well formed program will not unmap - * the file while it has a (shared) futex waiting on it. This mapping will have - * a file reference which pins the mount and inode. - * - * If for some reason an inode gets evicted and read back in again, it will get - * a new sequence number and will _NOT_ match, even though it is the exact same - * file. - * - * It is important that futex_match() will never have a false-positive, esp. - * for PI futexes that can mess up the state. The above argues that false-negatives - * are only possible for malformed programs. */ -static u64 get_inode_sequence_number(struct inode *inode) +static u64 get_object_id(atomic64_t *object_seq) { - static atomic64_t i_seq; + static atomic64_t last_assigned_seq; u64 old; - /* Does the inode already have a sequence number? */ - old = atomic64_read(&inode->i_sequence); + /* Does the object already have a sequence number? */ + old = atomic64_read(object_seq); if (likely(old)) return old; for (;;) { - u64 new = atomic64_inc_return(&i_seq); + u64 new = atomic64_inc_return(&last_assigned_seq); if (WARN_ON_ONCE(!new)) continue; old = 0; - if (!atomic64_try_cmpxchg_relaxed(&inode->i_sequence, &old, new)) + if (!atomic64_try_cmpxchg_relaxed(object_seq, &old, new)) return old; return new; } } +/* + * This relies on the fact that a well formed program will not unmap + * the file while it has a (shared) futex waiting on it. This mapping will have + * a file reference which pins the mount and inode. + * + * If for some reason an inode gets evicted and read back in again, it will get + * a new sequence number and will _NOT_ match, even though it is the exact same + * file. + * + * It is important that futex_match() will never have a false-positive, esp. + * for PI futexes that can mess up the state. The above argues that false-negatives + * are only possible for malformed programs. + */ +static u64 get_inode_sequence_number(struct inode *inode) +{ + return get_object_id(&inode->i_sequence); +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -681,8 +689,8 @@ again: goto out; } - key->both.offset |= FUT_OFF_MMSHARED; /* ref taken on mm */ - key->private.mm = mm; + key->both.offset |= FUT_OFF_MMSHARED; + key->private.mm_seq = get_object_id(&mm->futex.unique_id); key->private.address = address; } else { @@ -2053,13 +2061,12 @@ static void futex_robust_unlock_init_mm(struct futex_mm_data *fd) static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { } #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ -#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) void futex_mm_init(struct mm_struct *mm) { + atomic64_set(&mm->futex.unique_id, 0); futex_hash_init_mm(&mm->futex); futex_robust_unlock_init_mm(&mm->futex); } -#endif int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) { diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index b3f4a4bccb12..842d852302dd 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -744,7 +744,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, /* Handle spurious wakeups gracefully */ ret = -EWOULDBLOCK; - if (timeout && !timeout->task) + if (timeout && !hrtimer_sleeper_task_get(timeout)) ret = -ETIMEDOUT; else if (signal_pending(current)) ret = -ERESTARTNOINTR; diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index d4483d15d30a..cf18309e5770 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -383,7 +383,7 @@ void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout) * flagged for rescheduling. Only call schedule if there * is no timeout, or if it has yet to expire. */ - if (!timeout || timeout->task) + if (!timeout || hrtimer_sleeper_task_get(timeout)) schedule(); } __set_current_state(TASK_RUNNING); @@ -539,7 +539,7 @@ retry: static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count, struct hrtimer_sleeper *to) { - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return; for (; count; count--, vs++) { @@ -590,7 +590,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, if (ret >= 0) return ret; - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return -ETIMEDOUT; else if (signal_pending(current)) return -ERESTARTSYS; @@ -725,7 +725,7 @@ retry: if (!futex_unqueue(&q)) return 0; - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return -ETIMEDOUT; /* |