Merge tag 'slab-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/slab

Pull slab updates from Harry Yoo:

 - Performance improvement of the sheaf prefill API, by allowing the
   barn to hold a partial sheaf and refilling sheaves from the barn.
   This improves will-it-scale mmap1 throughput by 24% on the maple_node
   cache (Hao Li)

 - Restore memcg accounting of ipc msg_msg allocations, lost with e.g.
   CONFIG_SLAB_BUCKETS=n where the allocations fall back to the general
   kmalloc caches (Kees Cook)

 - Fix an old race between kmem_cache_destroy() and slab_late_init(),
   where a cache destroyed late in the boot process tries to release a
   kobject that was never initialized (Imre Kaloz)

 - Remove slab_folio(), the last use of struct folio in slab code
   (Matthew Wilcox)

 - Reject kmalloc() requests above KMALLOC_MAX_SIZE explicitly with a
   warning and NULL return instead of relying on the page allocator's
   order check (Zi Yan)

 - Various minor fixes and cleanups (Hao Li, Karl Mehltretter, Kees
   Cook, Longlong Xia)

* tag 'slab-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/slab:
  mm/slub: initialize the free pointer of the bootstrap kmem_cache_node
  mm/slab: sample slab_state once in kmem_cache_destroy()
  ipc, msg: Account msg_msg allocations with GFP_KERNEL_ACCOUNT again
  mm/slab: Mark the kmem_buckets_create() context as a Context: section
  slab: Remove slab_folio()
  mm/slub: refill prefilled sheaves from the barn
  mm/slub: make the case handling in __slab_free() easier to follow
  mm/slub: unify slub stat definitions to simplify metric additions
  mm/slab_common: reject zero object_size before calculate_alignment
  mm/slab: reject unsupported kmalloc() sizes

4 files changed, 228 insertions, 155 deletions

diff --git a/ipc/msgutil.c b/ipc/msgutil.c
index e28f0cecb2ec..12663a40e9ae 100644
--- a/ipc/msgutil.c
+++ b/ipc/msgutil.c
@@ -58,7 +58,8 @@ static struct msg_msg *alloc_msg(size_t len)
 	size_t alen;
 
 	alen = min(len, DATALEN_MSG);
-	msg = kmem_buckets_alloc(msg_buckets, sizeof(*msg) + alen, GFP_KERNEL);
+	msg = kmem_buckets_alloc(msg_buckets, sizeof(*msg) + alen,
+				 GFP_KERNEL_ACCOUNT);
 	if (msg == NULL)
 		return NULL;
 
diff --git a/mm/slab.h b/mm/slab.h
index 8fd6835e4235..e94ac63d10dc 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -150,21 +150,6 @@ static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(struct freelist
 #endif
 
 /**
- * slab_folio - The folio allocated for a slab
- * @s: The slab.
- *
- * Slabs are allocated as folios that contain the individual objects and are
- * using some fields in the first struct page of the folio - those fields are
- * now accessed by struct slab. It is occasionally necessary to convert back to
- * a folio in order to communicate with the rest of the mm.  Please use this
- * helper function instead of casting yourself, as the implementation may change
- * in the future.
- */
-#define slab_folio(s)		(_Generic((s),				\
-	const struct slab *:	(const struct folio *)s,		\
-	struct slab *:		(struct folio *)s))
-
-/**
  * page_slab - Converts from struct page to its slab.
  * @page: A page which may or may not belong to a slab.
  *
@@ -184,15 +169,16 @@ static inline struct slab *page_slab(const struct page *page)
  * slab_page - The first struct page allocated for a slab
  * @s: The slab.
  *
- * A convenience wrapper for converting slab to the first struct page of the
- * underlying folio, to communicate with code not yet converted to folio or
- * struct slab.
+ * A convenience wrapper for returning the first page allocated to this slab.
+ * This is for the benefit of code not yet converted to struct slab.
  */
-#define slab_page(s) folio_page(slab_folio(s), 0)
+#define slab_page(s) 		(_Generic((s),				\
+	const struct slab *:	(const struct page *)s,			\
+	struct slab *:		(struct page *)s))
 
 static inline void *slab_address(const struct slab *slab)
 {
-	return folio_address(slab_folio(slab));
+	return page_address(slab_page(slab));
 }
 
 static inline int slab_nid(const struct slab *slab)
@@ -212,7 +198,7 @@ static inline struct slab *virt_to_slab(const void *addr)
 
 static inline int slab_order(const struct slab *slab)
 {
-	return folio_order(slab_folio(slab));
+	return compound_order(slab_page(slab));
 }
 
 static inline size_t slab_size(const struct slab *slab)
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 7223a7596dab..01c64ef661e7 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -102,7 +102,7 @@ static bool kmem_cache_is_duplicate_name(const char *name)
 
 static int kmem_cache_sanity_check(const char *name, unsigned int size)
 {
-	if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
+	if (!name || in_interrupt() || !size || size > KMALLOC_MAX_SIZE) {
 		pr_err("kmem_cache_create(%s) integrity check failed\n", name);
 		return -EINVAL;
 	}
@@ -311,7 +311,7 @@ __kmem_cache_alias(const char *name, unsigned int size, slab_flags_t flags,
  * &SLAB_TYPESAFE_BY_RCU - Slab page (not individual objects) freeing delayed
  * by a grace period - see the full description before using.
  *
- * Context: Cannot be called within a interrupt, but can be interrupted.
+ * Context: Cannot be called within an interrupt, but can be interrupted.
  *
  * Return: a pointer to the cache on success, NULL on failure.
  */
@@ -422,7 +422,7 @@ static struct kmem_cache *kmem_buckets_cache __ro_after_init;
  *		to/from userspace.
  * @ctor: A constructor for the objects, run when new allocations are made.
  *
- * Cannot be called within an interrupt, but can be interrupted.
+ * Context: Cannot be called within an interrupt, but can be interrupted.
  *
  * Return: a pointer to the cache on success, NULL on failure. When
  * CONFIG_SLAB_BUCKETS is not enabled, ZERO_SIZE_PTR is returned, and
@@ -515,10 +515,10 @@ EXPORT_SYMBOL(kmem_buckets_create);
  * and release of the kobject does not need slab_mutex or cpu_hotplug_lock
  * protection. So they are now done without holding those locks.
  */
-static void kmem_cache_release(struct kmem_cache *s)
+static void kmem_cache_release(struct kmem_cache *s, bool sysfs_ready)
 {
 	kfence_shutdown_cache(s);
-	if (__is_defined(SLAB_SUPPORTS_SYSFS) && slab_state >= FULL)
+	if (__is_defined(SLAB_SUPPORTS_SYSFS) && sysfs_ready)
 		sysfs_slab_release(s);
 	else
 		slab_kmem_cache_release(s);
@@ -533,6 +533,7 @@ void slab_kmem_cache_release(struct kmem_cache *s)
 
 void kmem_cache_destroy(struct kmem_cache *s)
 {
+	bool sysfs_ready;
 	int err;
 
 	if (unlikely(!s) || !kasan_check_byte(s))
@@ -580,10 +581,18 @@ void kmem_cache_destroy(struct kmem_cache *s)
 
 	list_del(&s->list);
 
+	/*
+	 * slab_late_init() sets slab_state to FULL under slab_mutex and adds
+	 * sysfs entries only for caches still on the list. Sample the state
+	 * here, so that a cache unlinked before that point is not handed to
+	 * sysfs_slab_release() with an uninitialized kobject.
+	 */
+	sysfs_ready = slab_state >= FULL;
+
 	mutex_unlock(&slab_mutex);
 	cpus_read_unlock();
 
-	if (slab_state >= FULL)
+	if (sysfs_ready)
 		sysfs_slab_unlink(s);
 	debugfs_slab_release(s);
 
@@ -593,7 +602,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
 	if (s->flags & SLAB_TYPESAFE_BY_RCU)
 		rcu_barrier();
 
-	kmem_cache_release(s);
+	kmem_cache_release(s, sysfs_ready);
 }
 EXPORT_SYMBOL(kmem_cache_destroy);
 
diff --git a/mm/slub.c b/mm/slub.c
index 544cff39762c..45e077c7526c 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -360,33 +360,64 @@ enum add_mode {
 	ADD_TO_TAIL,
 };
 
+#define STAT_ITEMS							\
+	/* Allocation from percpu sheaves */				\
+	ITEM_EXP(ALLOC_FASTPATH, alloc_fastpath)			\
+	/* Allocation from partial or new slab */			\
+	ITEM_EXP(ALLOC_SLOWPATH, alloc_slowpath)			\
+	/* Free to rcu_free sheaf */					\
+	ITEM_EXP(FREE_RCU_SHEAF, free_rcu_sheaf)			\
+	/* Failed to free to a rcu_free sheaf */			\
+	ITEM_EXP(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail)		\
+	/* Free to percpu sheaves */					\
+	ITEM_EXP(FREE_FASTPATH, free_fastpath)				\
+	/* Free to a slab */						\
+	ITEM_EXP(FREE_SLOWPATH, free_slowpath)				\
+	/* Freeing moves slab to partial list */			\
+	ITEM_EXP(FREE_ADD_PARTIAL, free_add_partial)			\
+	/* Freeing removes last object */				\
+	ITEM_EXP(FREE_REMOVE_PARTIAL, free_remove_partial)		\
+	/* New slab acquired from page allocator */			\
+	ITEM_EXP(ALLOC_SLAB, alloc_slab)				\
+	/* Requested node different from cpu sheaf */			\
+	ITEM_EXP(ALLOC_NODE_MISMATCH, alloc_node_mismatch)		\
+	/* Slab freed to the page allocator */				\
+	ITEM_EXP(FREE_SLAB, free_slab)					\
+	/* Number of times fallback was necessary */			\
+	ITEM_EXP(ORDER_FALLBACK, order_fallback)			\
+	/* Failures of slab freelist update */				\
+	ITEM_EXP(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail)		\
+	/* Objects flushed from a sheaf */				\
+	ITEM_EXP(SHEAF_FLUSH, sheaf_flush)				\
+	/* Objects refilled to a sheaf */				\
+	ITEM_EXP(SHEAF_REFILL, sheaf_refill)				\
+	/* Allocation of an empty sheaf including oversized ones */	\
+	ITEM_EXP(SHEAF_ALLOC, sheaf_alloc)				\
+	/* Freeing of an empty sheaf including oversized ones */	\
+	ITEM_EXP(SHEAF_FREE, sheaf_free)				\
+	/* Got full sheaf from barn */					\
+	ITEM_EXP(BARN_GET, barn_get)					\
+	/* Failed to get full sheaf from barn */			\
+	ITEM_EXP(BARN_GET_FAIL, barn_get_fail)				\
+	/* Put full sheaf to barn */					\
+	ITEM_EXP(BARN_PUT, barn_put)					\
+	/* Failed to put full sheaf to barn */				\
+	ITEM_EXP(BARN_PUT_FAIL, barn_put_fail)				\
+	/* Sheaf prefill grabbed the spare sheaf */			\
+	ITEM_EXP(SHEAF_PREFILL_FAST, sheaf_prefill_fast)		\
+	/* Sheaf prefill found no spare sheaf */			\
+	ITEM_EXP(SHEAF_PREFILL_SLOW, sheaf_prefill_slow)		\
+	/* Allocation of oversize sheaf for prefill */			\
+	ITEM_EXP(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize)	\
+	/* Sheaf return reattached spare sheaf */			\
+	ITEM_EXP(SHEAF_RETURN_FAST, sheaf_return_fast)			\
+	/* Sheaf return could not reattach spare */			\
+	ITEM_EXP(SHEAF_RETURN_SLOW, sheaf_return_slow)
+
 enum stat_item {
-	ALLOC_FASTPATH,		/* Allocation from percpu sheaves */
-	ALLOC_SLOWPATH,		/* Allocation from partial or new slab */
-	FREE_RCU_SHEAF,		/* Free to rcu_free sheaf */
-	FREE_RCU_SHEAF_FAIL,	/* Failed to free to a rcu_free sheaf */
-	FREE_FASTPATH,		/* Free to percpu sheaves */
-	FREE_SLOWPATH,		/* Free to a slab */
-	FREE_ADD_PARTIAL,	/* Freeing moves slab to partial list */
-	FREE_REMOVE_PARTIAL,	/* Freeing removes last object */
-	ALLOC_SLAB,		/* New slab acquired from page allocator */
-	ALLOC_NODE_MISMATCH,	/* Requested node different from cpu sheaf */
-	FREE_SLAB,		/* Slab freed to the page allocator */
-	ORDER_FALLBACK,		/* Number of times fallback was necessary */
-	CMPXCHG_DOUBLE_FAIL,	/* Failures of slab freelist update */
-	SHEAF_FLUSH,		/* Objects flushed from a sheaf */
-	SHEAF_REFILL,		/* Objects refilled to a sheaf */
-	SHEAF_ALLOC,		/* Allocation of an empty sheaf including oversized ones */
-	SHEAF_FREE,		/* Freeing of an empty sheaf including oversized ones */
-	BARN_GET,		/* Got full sheaf from barn */
-	BARN_GET_FAIL,		/* Failed to get full sheaf from barn */
-	BARN_PUT,		/* Put full sheaf to barn */
-	BARN_PUT_FAIL,		/* Failed to put full sheaf to barn */
-	SHEAF_PREFILL_FAST,	/* Sheaf prefill grabbed the spare sheaf */
-	SHEAF_PREFILL_SLOW,	/* Sheaf prefill found no spare sheaf */
-	SHEAF_PREFILL_OVERSIZE,	/* Allocation of oversize sheaf for prefill */
-	SHEAF_RETURN_FAST,	/* Sheaf return reattached spare sheaf */
-	SHEAF_RETURN_SLOW,	/* Sheaf return could not reattach spare */
+#define ITEM_EXP(name, unused) name,
+	STAT_ITEMS
+#undef ITEM_EXP
 	NR_SLUB_STAT_ITEMS
 };
 
@@ -422,6 +453,7 @@ struct node_barn {
 	spinlock_t lock;
 	struct list_head sheaves_full;
 	struct list_head sheaves_empty;
+	struct slab_sheaf *sheaf_partial;
 	unsigned int nr_full;
 	unsigned int nr_empty;
 };
@@ -3298,6 +3330,7 @@ static void barn_init(struct node_barn *barn)
 	spin_lock_init(&barn->lock);
 	INIT_LIST_HEAD(&barn->sheaves_full);
 	INIT_LIST_HEAD(&barn->sheaves_empty);
+	barn->sheaf_partial = NULL;
 	barn->nr_full = 0;
 	barn->nr_empty = 0;
 }
@@ -3315,6 +3348,10 @@ static void barn_shrink(struct kmem_cache *s, struct node_barn *barn)
 	barn->nr_full = 0;
 	list_splice_init(&barn->sheaves_empty, &empty_list);
 	barn->nr_empty = 0;
+	if (barn->sheaf_partial) {
+		list_add(&barn->sheaf_partial->barn_list, &full_list);
+		barn->sheaf_partial = NULL;
+	}
 
 	spin_unlock_irqrestore(&barn->lock, flags);
 
@@ -5073,12 +5110,87 @@ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int nod
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
 
+/*
+ * Refill @sheaf from the barn: from its partial sheaf first, then from its full
+ * sheaves. A sheaf that objects were copied from stays in the barn as the
+ * partial sheaf if it still holds objects, or goes on the empty list if it is
+ * empty.
+ *
+ * Returns true if the sheaf is now full, at s->sheaf_capacity.
+ * Returns false if the sheaf is still not full.
+ */
+static bool refill_sheaf_from_barn(struct kmem_cache *s,
+				   struct slab_sheaf *sheaf)
+{
+	struct node_barn *barn = get_barn(s);
+	struct slab_sheaf *src;
+	unsigned int to_move;
+	unsigned long flags;
+
+	if (!barn)
+		return false;
+
+	if (!data_race(barn->nr_full) && !data_race(barn->sheaf_partial))
+		return false;
+
+	spin_lock_irqsave(&barn->lock, flags);
+
+	/*
+	 * The partial sheaf can hold fewer objects than @sheaf needs to
+	 * reach capacity, and a sheaf on the full list is not necessarily
+	 * full (see the comment in rcu_free_sheaf()), so keep taking from
+	 * the barn until @sheaf is full or nothing is left.
+	 */
+	while (sheaf->size < s->sheaf_capacity) {
+		src = barn->sheaf_partial;
+		barn->sheaf_partial = NULL;
+		if (!src) {
+			if (!barn->nr_full)
+				break;
+			src = list_first_entry(&barn->sheaves_full,
+					       struct slab_sheaf, barn_list);
+			list_del(&src->barn_list);
+			barn->nr_full--;
+		}
+
+		to_move = min(s->sheaf_capacity - sheaf->size, src->size);
+		src->size -= to_move;
+		memcpy(&sheaf->objects[sheaf->size], &src->objects[src->size],
+		       to_move * sizeof(void *));
+		sheaf->size += to_move;
+
+		if (src->size) {
+			barn->sheaf_partial = src;
+		} else {
+			/*
+			 * No empty-limit check: the sheaf put on the empty list
+			 * was already in the barn, so the barn holds no more
+			 * sheaves than before. barn_replace_empty_sheaf() skips
+			 * the check for the same reason.
+			 */
+			list_add(&src->barn_list, &barn->sheaves_empty);
+			barn->nr_empty++;
+		}
+	}
+
+	spin_unlock_irqrestore(&barn->lock, flags);
+
+	if (sheaf->size < s->sheaf_capacity)
+		return false;
+
+	stat(s, BARN_GET);
+	return true;
+}
+
 static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
 				      struct slab_sheaf *sheaf, gfp_t gfp)
 {
 	gfp_t gfp_nomemalloc;
 	int ret;
 
+	if (refill_sheaf_from_barn(s, sheaf))
+		return 0;
+
 	gfp_nomemalloc = gfp | __GFP_NOMEMALLOC;
 	if (gfp_pfmemalloc_allowed(gfp))
 		gfp_nomemalloc |= __GFP_NOWARN;
@@ -5342,7 +5454,12 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
 	struct page *page;
 	void *ptr = NULL;
-	unsigned int order = get_order(size);
+	unsigned int order;
+
+	if (WARN_ON_ONCE_GFP(size > KMALLOC_MAX_SIZE, flags))
+		return NULL;
+
+	order = get_order(size);
 
 	if (unlikely(flags & GFP_SLAB_BUG_MASK))
 		flags = kmalloc_fix_flags(flags);
@@ -5751,76 +5868,80 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
 		new.inuse -= cnt;
 
 		/*
-		 * Might need to be taken off (due to becoming empty) or added
-		 * to (due to not being full anymore) the partial list.
-		 * Unless it's frozen.
+		 * partial->partial: if the slab was on the node partial list,
+		 * it stays there, and if it was off, it stays off, so we need
+		 * no list handling and no list_lock.
+		 *
+		 * Note that "continue;" in a do-while goes on to evaluate the
+		 * condition below, so we do perform the freelist update.
 		 */
-		if (!new.inuse || was_full) {
-
-			n = get_node(s, slab_nid(slab));
-			/*
-			 * Speculatively acquire the list_lock.
-			 * If the cmpxchg does not succeed then we may
-			 * drop the list_lock without any processing.
-			 *
-			 * Otherwise the list_lock will synchronize with
-			 * other processors updating the list of slabs.
-			 */
-			spin_lock_irqsave(&n->list_lock, flags);
+		if (!was_full && new.inuse)
+			continue;
 
-			on_node_partial = slab_test_node_partial(slab);
-		}
+		/*
+		 * The slab might need to be taken off (due to becoming empty)
+		 * or added to (due to not being full anymore) the partial
+		 * list.
+		 *
+		 * Speculatively acquire list_lock prior to cmpxchg(), as
+		 * performing cmpxchg() before lock acquisition races with
+		 * concurrent partial list operations.
+		 *
+		 * If the cmpxchg does not succeed then we will retry.
+		 */
+		n = get_node(s, slab_nid(slab));
+		spin_lock_irqsave(&n->list_lock, flags);
 
 	} while (!slab_update_freelist(s, slab, &old, &new, "__slab_free"));
 
-	if (likely(!n)) {
+	/* partial->partial: we didn't take the list_lock. */
+	if (likely(!n))
+		return;
+
+	on_node_partial = slab_test_node_partial(slab);
+
+	if (!was_full && !on_node_partial) {
 		/*
-		 * We didn't take the list_lock because the slab was already on
-		 * the partial list and will remain there.
+		 * partial->empty, offlist: a bulk refill has taken the slab
+		 * off the partial list and will put it back, so its list
+		 * handling is not ours to do.
 		 */
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		return;
 	}
 
 	/*
-	 * This slab was partially empty but not on the per-node partial list,
-	 * in which case we shouldn't manipulate its list, just return.
+	 * full/partial->empty, exceed: we have enough partial slabs already.
 	 */
-	if (!was_full && !on_node_partial) {
+	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
+		/* partial->empty, onlist, exceed */
+		if (likely(!was_full)) {
+			remove_partial(n, slab);
+			stat(s, FREE_REMOVE_PARTIAL);
+		}
+		/* else, full->empty, exceed: it is on no list to remove from */
+
 		spin_unlock_irqrestore(&n->list_lock, flags);
+		stat(s, FREE_SLAB);
+		discard_slab(s, slab);
 		return;
 	}
 
 	/*
-	 * If slab became empty, should we add/keep it on the partial list or we
-	 * have enough?
+	 * At this point, only three cases remain:
+	 *   full->partial
+	 *   full->empty, not exceed
+	 *   partial->empty, onlist, not exceed
 	 */
-	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
-		goto slab_empty;
 
-	/*
-	 * Objects left in the slab. If it was not on the partial list before
-	 * then add it.
-	 */
+	/* full->partial; full->empty, not exceed */
 	if (unlikely(was_full)) {
 		add_partial(n, slab, ADD_TO_TAIL);
 		stat(s, FREE_ADD_PARTIAL);
 	}
-	spin_unlock_irqrestore(&n->list_lock, flags);
-	return;
-
-slab_empty:
-	/*
-	 * The slab could have a single object and thus go from full to empty in
-	 * a single free, but more likely it was on the partial list. Remove it.
-	 */
-	if (likely(!was_full)) {
-		remove_partial(n, slab);
-		stat(s, FREE_REMOVE_PARTIAL);
-	}
+	/* else, partial->empty, onlist, not exceed: it stays on partial list */
 
 	spin_unlock_irqrestore(&n->list_lock, flags);
-	stat(s, FREE_SLAB);
-	discard_slab(s, slab);
 }
 
 /*
@@ -7778,6 +7899,7 @@ static void early_kmem_cache_node_alloc(int node)
 
 	n = next_slab_obj(kmem_cache_node, &iter);
 	BUG_ON(!n);
+	set_freepointer(kmem_cache_node, n, NULL);
 
 	slab->inuse = 1;
 	build_slab_freelist(kmem_cache_node, slab, &iter);
@@ -9515,34 +9637,12 @@ static ssize_t text##_store(struct kmem_cache *s,		\
 	clear_stat(s, si);					\
 	return length;						\
 }								\
-SLAB_ATTR(text);						\
-
-STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
-STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
-STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf);
-STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail);
-STAT_ATTR(FREE_FASTPATH, free_fastpath);
-STAT_ATTR(FREE_SLOWPATH, free_slowpath);
-STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
-STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
-STAT_ATTR(ALLOC_SLAB, alloc_slab);
-STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
-STAT_ATTR(FREE_SLAB, free_slab);
-STAT_ATTR(ORDER_FALLBACK, order_fallback);
-STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
-STAT_ATTR(SHEAF_FLUSH, sheaf_flush);
-STAT_ATTR(SHEAF_REFILL, sheaf_refill);
-STAT_ATTR(SHEAF_ALLOC, sheaf_alloc);
-STAT_ATTR(SHEAF_FREE, sheaf_free);
-STAT_ATTR(BARN_GET, barn_get);
-STAT_ATTR(BARN_GET_FAIL, barn_get_fail);
-STAT_ATTR(BARN_PUT, barn_put);
-STAT_ATTR(BARN_PUT_FAIL, barn_put_fail);
-STAT_ATTR(SHEAF_PREFILL_FAST, sheaf_prefill_fast);
-STAT_ATTR(SHEAF_PREFILL_SLOW, sheaf_prefill_slow);
-STAT_ATTR(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize);
-STAT_ATTR(SHEAF_RETURN_FAST, sheaf_return_fast);
-STAT_ATTR(SHEAF_RETURN_SLOW, sheaf_return_slow);
+SLAB_ATTR(text);
+
+#define ITEM_EXP(name, text) STAT_ATTR(name, text)
+STAT_ITEMS
+#undef ITEM_EXP
+
 #endif	/* CONFIG_SLUB_STATS */
 
 #ifdef CONFIG_KFENCE
@@ -9605,32 +9705,9 @@ static const struct attribute *const slab_attrs[] = {
 	&remote_node_defrag_ratio_attr.attr,
 #endif
 #ifdef CONFIG_SLUB_STATS
-	&alloc_fastpath_attr.attr,
-	&alloc_slowpath_attr.attr,
-	&free_rcu_sheaf_attr.attr,
-	&free_rcu_sheaf_fail_attr.attr,
-	&free_fastpath_attr.attr,
-	&free_slowpath_attr.attr,
-	&free_add_partial_attr.attr,
-	&free_remove_partial_attr.attr,
-	&alloc_slab_attr.attr,
-	&alloc_node_mismatch_attr.attr,
-	&free_slab_attr.attr,
-	&order_fallback_attr.attr,
-	&cmpxchg_double_fail_attr.attr,
-	&sheaf_flush_attr.attr,
-	&sheaf_refill_attr.attr,
-	&sheaf_alloc_attr.attr,
-	&sheaf_free_attr.attr,
-	&barn_get_attr.attr,
-	&barn_get_fail_attr.attr,
-	&barn_put_attr.attr,
-	&barn_put_fail_attr.attr,
-	&sheaf_prefill_fast_attr.attr,
-	&sheaf_prefill_slow_attr.attr,
-	&sheaf_prefill_oversize_attr.attr,
-	&sheaf_return_fast_attr.attr,
-	&sheaf_return_slow_attr.attr,
+#define ITEM_EXP(unused, text) &text##_attr.attr,
+	STAT_ITEMS
+#undef ITEM_EXP
 #endif
 #ifdef CONFIG_FAILSLAB
 	&failslab_attr.attr,