summaryrefslogtreecommitdiff
path: root/include/linux
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-11-17 11:51:40 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2026-11-17 11:51:40 -0800
commit63fdb511c27afcebce797e3db580b3534ddfa639 (patch)
tree0cbd5e51ca9cef10e99632878a6002e4ea078096 /include/linux
parent666539eecdf4db623f5e3c3e55cf9f2dfad24aa4 (diff)
parent25f5c92e1ac06496679ea229e5c825aa88968efd (diff)
Merge tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core
Pull core memory management fixes from David Hildenbrand: "The main changes are: - Prevent folio refcount overflows with pagecache folios, mostly relevant for large folios today. The networking changes for this will go through the net tree. - Fix out-of-bounds page table accesses with folded PMDs in 3 scenarios. - Fix write-sealing of memfd to detect GUP pins with concurrent (un)mapping more reliably, preventing writes through GUP pins after write-sealing. - Fix lost wakeups when splitting large folios. - Fix exposing uninitialized data through MADV_COLLAPSE to user space. The refcount changes also include a related iommufd fix for properly cleaning up when pinning memfd folios fails, which is now easier to trigger" * tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core: mm/gup: limit pagecache folio refcount when grabbing references iommufd: fix cleanup when adding memfd folio pins fails mm/mmu_gather: flush pending references early before the folio limit mm: limit pagecache folio refcount when splicing into pipes mm: limit pagecache folio refcount when mapping pages mm/huge_memory: prevent losing PG_waiters in __split_folio_to_order() mm/rmap: fix PTE table overruns during batched reference checks mm/rmap: fix PTE table overruns during batched unmap mm/userfaultfd: fix PTE table overruns in UFFDIO_MOVE mm/memfd: handle GUP pins more reliably when write-sealing mm/khugepaged: don't map folios that are not uptodate in MADV_COLLAPSE
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/mm.h89
-rw-r--r--include/linux/pgtable.h24
2 files changed, 99 insertions, 14 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 6af3b7008f1a..daf07b2d208f 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2249,9 +2249,54 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
* back into memory.
*/
-/* 127: arbitrary random number, small enough to assemble well */
-#define folio_ref_zero_or_close_to_overflow(folio) \
- ((unsigned int) folio_ref_count(folio) + 127u <= 127u)
+/* Leave separate reserves for administrators and concurrent reference gains. */
+#define FOLIO_REF_COUNT_LIMIT (INT_MAX - (1U << 26))
+#define FOLIO_REF_COUNT_ADMIN_LIMIT (INT_MAX - (1U << 25))
+#define FOLIO_REF_COUNT_FLUSH_LIMIT (INT_MAX - (1U << 27))
+
+bool folio_ref_count_over_limit_slowpath(const struct folio *folio,
+ unsigned int ref_count);
+
+/**
+ * __folio_ref_count_over_limit - Check whether to refuse more folio references.
+ * @folio: The folio.
+ * @ref_count: The sampled folio reference count.
+ *
+ * Check before adding page table mappings or other significant refcount users
+ * to pagecache folios (!hugetlb), either before acquiring references or
+ * afterwards with rollback if the operation is refused.
+ *
+ * Also required when pinning folios without a dedicated pincount that use
+ * GUP_PIN_COUNTING_BIAS.
+ *
+ * Checking other folios does no harm.
+ *
+ * This check is an approximation; we leave some headroom such that concurrent
+ * increments, batches, temporary references, or other bounded refcount
+ * consumers are not a problem.
+ *
+ * Return: True if the caller should refuse the operation, false otherwise.
+ */
+static inline bool __folio_ref_count_over_limit(const struct folio *folio,
+ unsigned int ref_count)
+{
+ if (likely(ref_count < FOLIO_REF_COUNT_LIMIT))
+ return false;
+ return folio_ref_count_over_limit_slowpath(folio, ref_count);
+}
+
+/**
+ * folio_ref_count_over_limit - Check whether to refuse more folio references.
+ * @folio: The folio.
+ *
+ * See __folio_ref_count_over_limit().
+ *
+ * Return: True if the caller should refuse the operation, false otherwise.
+ */
+static inline bool folio_ref_count_over_limit(const struct folio *folio)
+{
+ return __folio_ref_count_over_limit(folio, folio_ref_count(folio));
+}
/**
* folio_get - Increment the reference count on a folio.
@@ -2263,7 +2308,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
*/
static inline void folio_get(struct folio *folio)
{
- VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio);
+ VM_WARN_ON_FOLIO(folio_ref_count(folio) <= 0, folio);
folio_ref_inc(folio);
}
@@ -2277,15 +2322,35 @@ static inline void get_page(struct page *page)
folio_get(folio);
}
-static inline __must_check bool try_get_page(struct page *page)
+/**
+ * folio_get_checked - Try to acquire a reference to a live folio.
+ * @folio: The folio.
+ *
+ * The caller must already hold a reference to @folio. Warn and fail on a
+ * non-positive refcount, and refuse additional references at the limit.
+ *
+ * Use folio_try_get() for speculative reference acquisition where the
+ * refcount can legitimately be zero.
+ *
+ * Return: True if the refcount was incremented, false otherwise.
+ */
+static inline __must_check bool folio_get_checked(struct folio *folio)
{
- page = compound_head(page);
- if (WARN_ON_ONCE(page_ref_count(page) <= 0))
+ const int ref_count = folio_ref_count(folio);
+
+ if (WARN_ON_ONCE(ref_count <= 0))
+ return false;
+ if (unlikely(__folio_ref_count_over_limit(folio, ref_count)))
return false;
- page_ref_inc(page);
+ folio_ref_inc(folio);
return true;
}
+static inline __must_check bool try_get_page(struct page *page)
+{
+ return folio_get_checked(page_folio(page));
+}
+
/**
* folio_put - Decrement the reference count on a folio.
* @folio: The folio.
@@ -2925,12 +2990,8 @@ static inline bool folio_maybe_dma_pinned(struct folio *folio)
return atomic_read(&folio->_pincount) > 0;
/*
- * folio_ref_count() is signed. If that refcount overflows, then
- * folio_ref_count() returns a negative value, and callers will avoid
- * further incrementing the refcount.
- *
- * Here, for that overflow case, use the sign bit to count a little
- * bit higher via unsigned math, and thus still get an accurate result.
+ * Use unsigned math so even an unexpected negative refcount is
+ * treated as possibly pinned.
*/
return ((unsigned int)folio_ref_count(folio)) >=
GUP_PIN_COUNTING_BIAS;
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 780fe849ff8b..66d369870a74 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -12,6 +12,7 @@
#ifdef CONFIG_MMU
#include <linux/mm_types.h>
+#include <linux/minmax.h>
#include <linux/bug.h>
#include <linux/errno.h>
#include <asm-generic/pgtable_uffd.h>
@@ -1518,6 +1519,29 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
})
#endif
+/**
+ * pte_table_clamped_end() - Limit a range to a single PTE table.
+ * @addr: Start address.
+ * @end: Exclusive end address.
+ *
+ * Only use pmd_addr_end() when traversing page tables, in a pattern like::
+ *
+ * next = pmd_addr_end(addr, end);
+ *
+ * Higher levels already bound the range when PMDs are folded. Use this
+ * helper elsewhere to clamp the range regardless of PMD folding.
+ *
+ * Return: The exclusive end of the PTE table containing @addr, capped at @end.
+ */
+static inline unsigned long pte_table_clamped_end(unsigned long addr,
+ unsigned long end)
+{
+ const unsigned long pmd_end = (addr & PMD_MASK) + PMD_SIZE;
+
+ /* Compare inclusive ends to avoid wraparounds. */
+ return min(pmd_end - 1, end - 1) + 1;
+}
+
/*
* When walking page tables, we usually want to skip any p?d_none entries;
* and any p?d_bad entries - reporting the error before resetting to none.