diff options
Diffstat (limited to 'mm/mempool.c')
| -rw-r--r-- | mm/mempool.c | 62 |
1 files changed, 34 insertions, 28 deletions
diff --git a/mm/mempool.c b/mm/mempool.c index db23e0eef652..cb74e718b2c6 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -16,11 +16,28 @@ #include <linux/export.h> #include <linux/mempool.h> #include <linux/writeback.h> +#include <linux/static_key.h> +#include <linux/init.h> #include "slab.h" static DECLARE_FAULT_ATTR(fail_mempool_alloc); static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk); +/* + * Debugging support for mempool using static key. + * + * This allows enabling mempool debug at boot time via: + * mempool_debug + */ +static DEFINE_STATIC_KEY_FALSE(mempool_debug_enabled); + +static int __init mempool_debug_setup(char *str) +{ + static_branch_enable(&mempool_debug_enabled); + return 1; +} +__setup("mempool_debug", mempool_debug_setup); + static int __init mempool_faul_inject_init(void) { int error; @@ -37,7 +54,6 @@ static int __init mempool_faul_inject_init(void) } late_initcall(mempool_faul_inject_init); -#ifdef CONFIG_SLUB_DEBUG_ON static void poison_error(struct mempool *pool, void *element, size_t size, size_t byte) { @@ -140,14 +156,6 @@ static void poison_element(struct mempool *pool, void *element) #endif } } -#else /* CONFIG_SLUB_DEBUG_ON */ -static inline void check_element(struct mempool *pool, void *element) -{ -} -static inline void poison_element(struct mempool *pool, void *element) -{ -} -#endif /* CONFIG_SLUB_DEBUG_ON */ static __always_inline bool kasan_poison_element(struct mempool *pool, void *element) @@ -175,7 +183,10 @@ static void kasan_unpoison_element(struct mempool *pool, void *element) static __always_inline void add_element(struct mempool *pool, void *element) { BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr); - poison_element(pool, element); + + if (static_branch_unlikely(&mempool_debug_enabled)) + poison_element(pool, element); + if (kasan_poison_element(pool, element)) pool->elements[pool->curr_nr++] = element; } @@ -186,7 +197,9 @@ static void *remove_element(struct mempool *pool) BUG_ON(pool->curr_nr < 0); kasan_unpoison_element(pool, element); - check_element(pool, element); + + if (static_branch_unlikely(&mempool_debug_enabled)) + check_element(pool, element); return element; } @@ -419,12 +432,8 @@ static unsigned int mempool_alloc_from_pool(struct mempool *pool, void **elems, spin_lock_irqsave(&pool->lock, flags); if (unlikely(pool->curr_nr < count - allocated)) goto fail; - for (i = 0; i < count; i++) { - if (!elems[i]) { - elems[i] = remove_element(pool); - allocated++; - } - } + while (allocated < count) + elems[allocated++] = remove_element(pool); spin_unlock_irqrestore(&pool->lock, flags); /* Paired with rmb in mempool_free(), read comment there. */ @@ -479,22 +488,21 @@ static inline gfp_t mempool_adjust_gfp(gfp_t *gfp_mask) * @pool: pointer to the memory pool * @elems: partially or fully populated elements array * @count: number of entries in @elem that need to be allocated - * @allocated: number of entries in @elem already allocated * - * Allocate elements for each slot in @elem that is non-%NULL. This is done by - * first calling into the alloc_fn supplied at pool initialization time, and - * dipping into the reserved pool when alloc_fn fails to allocate an element. + * Allocate @count elements into @elems. This is done by first calling into the + * alloc_fn supplied at pool initialization time, and dipping into the reserved + * pool when alloc_fn fails to allocate an element. * * On return all @count elements in @elems will be populated. * * Return: Always 0. If it wasn't for %$#^$ alloc tags, it would return void. */ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems, - unsigned int count, unsigned int allocated) + unsigned int count) { gfp_t gfp_mask = GFP_KERNEL; gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask); - unsigned int i = 0; + unsigned int allocated = 0; VM_WARN_ON_ONCE(count > pool->min_nr); might_alloc(gfp_mask); @@ -514,11 +522,9 @@ repeat_alloc: * Try to allocate the elements using the allocation callback first as * that might succeed even when the caller's bulk allocation did not. */ - for (i = 0; i < count; i++) { - if (elems[i]) - continue; - elems[i] = pool->alloc(gfp_temp, pool->pool_data); - if (unlikely(!elems[i])) + while (allocated < count) { + elems[allocated] = pool->alloc(gfp_temp, pool->pool_data); + if (unlikely(!elems[allocated])) goto use_pool; allocated++; } |