diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-11-17 11:51:40 -0800 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-11-17 11:51:40 -0800 |
| commit | 63fdb511c27afcebce797e3db580b3534ddfa639 (patch) | |
| tree | 0cbd5e51ca9cef10e99632878a6002e4ea078096 /include/linux | |
| parent | 666539eecdf4db623f5e3c3e55cf9f2dfad24aa4 (diff) | |
| parent | 25f5c92e1ac06496679ea229e5c825aa88968efd (diff) | |
Merge tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core
Pull core memory management fixes from David Hildenbrand:
"The main changes are:
- Prevent folio refcount overflows with pagecache folios, mostly
relevant for large folios today. The networking changes for this
will go through the net tree.
- Fix out-of-bounds page table accesses with folded PMDs in 3
scenarios.
- Fix write-sealing of memfd to detect GUP pins with concurrent
(un)mapping more reliably, preventing writes through GUP pins after
write-sealing.
- Fix lost wakeups when splitting large folios.
- Fix exposing uninitialized data through MADV_COLLAPSE to user
space.
The refcount changes also include a related iommufd fix for properly
cleaning up when pinning memfd folios fails, which is now easier to
trigger"
* tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core:
mm/gup: limit pagecache folio refcount when grabbing references
iommufd: fix cleanup when adding memfd folio pins fails
mm/mmu_gather: flush pending references early before the folio limit
mm: limit pagecache folio refcount when splicing into pipes
mm: limit pagecache folio refcount when mapping pages
mm/huge_memory: prevent losing PG_waiters in __split_folio_to_order()
mm/rmap: fix PTE table overruns during batched reference checks
mm/rmap: fix PTE table overruns during batched unmap
mm/userfaultfd: fix PTE table overruns in UFFDIO_MOVE
mm/memfd: handle GUP pins more reliably when write-sealing
mm/khugepaged: don't map folios that are not uptodate in MADV_COLLAPSE
Diffstat (limited to 'include/linux')
| -rw-r--r-- | include/linux/mm.h | 89 | ||||
| -rw-r--r-- | include/linux/pgtable.h | 24 |
2 files changed, 99 insertions, 14 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h index 6af3b7008f1a..daf07b2d208f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2249,9 +2249,54 @@ vm_fault_t finish_fault(struct vm_fault *vmf); * back into memory. */ -/* 127: arbitrary random number, small enough to assemble well */ -#define folio_ref_zero_or_close_to_overflow(folio) \ - ((unsigned int) folio_ref_count(folio) + 127u <= 127u) +/* Leave separate reserves for administrators and concurrent reference gains. */ +#define FOLIO_REF_COUNT_LIMIT (INT_MAX - (1U << 26)) +#define FOLIO_REF_COUNT_ADMIN_LIMIT (INT_MAX - (1U << 25)) +#define FOLIO_REF_COUNT_FLUSH_LIMIT (INT_MAX - (1U << 27)) + +bool folio_ref_count_over_limit_slowpath(const struct folio *folio, + unsigned int ref_count); + +/** + * __folio_ref_count_over_limit - Check whether to refuse more folio references. + * @folio: The folio. + * @ref_count: The sampled folio reference count. + * + * Check before adding page table mappings or other significant refcount users + * to pagecache folios (!hugetlb), either before acquiring references or + * afterwards with rollback if the operation is refused. + * + * Also required when pinning folios without a dedicated pincount that use + * GUP_PIN_COUNTING_BIAS. + * + * Checking other folios does no harm. + * + * This check is an approximation; we leave some headroom such that concurrent + * increments, batches, temporary references, or other bounded refcount + * consumers are not a problem. + * + * Return: True if the caller should refuse the operation, false otherwise. + */ +static inline bool __folio_ref_count_over_limit(const struct folio *folio, + unsigned int ref_count) +{ + if (likely(ref_count < FOLIO_REF_COUNT_LIMIT)) + return false; + return folio_ref_count_over_limit_slowpath(folio, ref_count); +} + +/** + * folio_ref_count_over_limit - Check whether to refuse more folio references. + * @folio: The folio. + * + * See __folio_ref_count_over_limit(). + * + * Return: True if the caller should refuse the operation, false otherwise. + */ +static inline bool folio_ref_count_over_limit(const struct folio *folio) +{ + return __folio_ref_count_over_limit(folio, folio_ref_count(folio)); +} /** * folio_get - Increment the reference count on a folio. @@ -2263,7 +2308,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf); */ static inline void folio_get(struct folio *folio) { - VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); + VM_WARN_ON_FOLIO(folio_ref_count(folio) <= 0, folio); folio_ref_inc(folio); } @@ -2277,15 +2322,35 @@ static inline void get_page(struct page *page) folio_get(folio); } -static inline __must_check bool try_get_page(struct page *page) +/** + * folio_get_checked - Try to acquire a reference to a live folio. + * @folio: The folio. + * + * The caller must already hold a reference to @folio. Warn and fail on a + * non-positive refcount, and refuse additional references at the limit. + * + * Use folio_try_get() for speculative reference acquisition where the + * refcount can legitimately be zero. + * + * Return: True if the refcount was incremented, false otherwise. + */ +static inline __must_check bool folio_get_checked(struct folio *folio) { - page = compound_head(page); - if (WARN_ON_ONCE(page_ref_count(page) <= 0)) + const int ref_count = folio_ref_count(folio); + + if (WARN_ON_ONCE(ref_count <= 0)) + return false; + if (unlikely(__folio_ref_count_over_limit(folio, ref_count))) return false; - page_ref_inc(page); + folio_ref_inc(folio); return true; } +static inline __must_check bool try_get_page(struct page *page) +{ + return folio_get_checked(page_folio(page)); +} + /** * folio_put - Decrement the reference count on a folio. * @folio: The folio. @@ -2925,12 +2990,8 @@ static inline bool folio_maybe_dma_pinned(struct folio *folio) return atomic_read(&folio->_pincount) > 0; /* - * folio_ref_count() is signed. If that refcount overflows, then - * folio_ref_count() returns a negative value, and callers will avoid - * further incrementing the refcount. - * - * Here, for that overflow case, use the sign bit to count a little - * bit higher via unsigned math, and thus still get an accurate result. + * Use unsigned math so even an unexpected negative refcount is + * treated as possibly pinned. */ return ((unsigned int)folio_ref_count(folio)) >= GUP_PIN_COUNTING_BIAS; diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 780fe849ff8b..66d369870a74 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -12,6 +12,7 @@ #ifdef CONFIG_MMU #include <linux/mm_types.h> +#include <linux/minmax.h> #include <linux/bug.h> #include <linux/errno.h> #include <asm-generic/pgtable_uffd.h> @@ -1518,6 +1519,29 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) }) #endif +/** + * pte_table_clamped_end() - Limit a range to a single PTE table. + * @addr: Start address. + * @end: Exclusive end address. + * + * Only use pmd_addr_end() when traversing page tables, in a pattern like:: + * + * next = pmd_addr_end(addr, end); + * + * Higher levels already bound the range when PMDs are folded. Use this + * helper elsewhere to clamp the range regardless of PMD folding. + * + * Return: The exclusive end of the PTE table containing @addr, capped at @end. + */ +static inline unsigned long pte_table_clamped_end(unsigned long addr, + unsigned long end) +{ + const unsigned long pmd_end = (addr & PMD_MASK) + PMD_SIZE; + + /* Compare inclusive ends to avoid wraparounds. */ + return min(pmd_end - 1, end - 1) + 1; +} + /* * When walking page tables, we usually want to skip any p?d_none entries; * and any p?d_bad entries - reporting the error before resetting to none. |
