diff options
Diffstat (limited to 'kernel/futex')
| -rw-r--r-- | kernel/futex/core.c | 58 |
1 files changed, 36 insertions, 22 deletions
diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 128c5752f225..8fef2f806cef 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -45,26 +45,23 @@ #include <linux/rseq.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#include <linux/kmemleak.h> #include <vdso/futex.h> +#include <asm/runtime-const.h> + #include "futex.h" #include "../locking/rtmutex_common.h" -/* - * The base of the bucket array and its size are always used together - * (after initialization only in futex_hash()), so ensure that they - * reside in the same cacheline. - */ -static struct { - unsigned long hashmask; - unsigned int hashshift; - struct futex_hash_bucket *queues[MAX_NUMNODES]; -} __futex_data __read_mostly __aligned(2*sizeof(long)); +static u32 __futex_mask __ro_after_init; +static u32 __futex_shift __ro_after_init; +static struct futex_hash_bucket **__futex_queues __ro_after_init; -#define futex_hashmask (__futex_data.hashmask) -#define futex_hashshift (__futex_data.hashshift) -#define futex_queues (__futex_data.queues) +static __always_inline struct futex_hash_bucket **futex_queues(void) +{ + return runtime_const_ptr(__futex_queues); +} struct futex_private_hash { int state; @@ -143,8 +140,14 @@ static bool futex_private_hash_get(struct futex_private_hash *fph) void futex_private_hash_put(struct futex_private_hash *fph) { - if (fph && futex_ref_put(fph)) - wake_up_var(fph->mm); + struct mm_struct *mm; + + if (!fph) + return; + + mm = fph->mm; + if (futex_ref_put(fph)) + wake_up_var(mm); } static struct futex_hash_bucket * @@ -395,13 +398,13 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_ * NOTE: this isn't perfectly uniform, but it is fast and * handles sparse node masks. */ - node = (hash >> futex_hashshift) % nr_node_ids; + node = runtime_const_shift_right_32(hash, __futex_shift) % nr_node_ids; if (!node_possible(node)) { node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } - return &futex_queues[node][hash & futex_hashmask]; + return &futex_queues()[node][runtime_const_mask_32(hash, __futex_mask)]; } /** @@ -520,7 +523,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * The futex address must be "naturally" aligned. */ key->both.offset = address % PAGE_SIZE; - if (unlikely((address % size) != 0)) + if (unlikely((address & (size-1)) != 0)) return -EINVAL; address -= key->both.offset; @@ -1954,7 +1957,7 @@ int futex_hash_allocate_default(void) * 16 <= threads * 4 <= global hash size */ buckets = roundup_pow_of_two(4 * threads); - buckets = clamp(buckets, 16, futex_hashmask + 1); + buckets = clamp(buckets, 16, __futex_mask + 1); if (current_buckets >= buckets) return 0; @@ -2052,10 +2055,22 @@ static int __init futex_init(void) hashsize = max(4, hashsize); hashsize = roundup_pow_of_two(hashsize); #endif - futex_hashshift = ilog2(hashsize); + __futex_mask = hashsize - 1; + __futex_shift = ilog2(hashsize); size = sizeof(struct futex_hash_bucket) * hashsize; order = get_order(size); + __futex_queues = kcalloc(nr_node_ids, sizeof(*__futex_queues), GFP_KERNEL); + kmemleak_not_leak(__futex_queues); + + runtime_const_init(shift, __futex_shift); + runtime_const_init(mask, __futex_mask); + runtime_const_init(ptr, __futex_queues); + + barrier(); + + BUG_ON(!futex_queues()); + for_each_node(n) { struct futex_hash_bucket *table; @@ -2069,10 +2084,9 @@ static int __init futex_init(void) for (i = 0; i < hashsize; i++) futex_hash_bucket_init(&table[i]); - futex_queues[n] = table; + futex_queues()[n] = table; } - futex_hashmask = hashsize - 1; pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n", hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024, order > MAX_PAGE_ORDER ? "vmalloc" : "linear"); |