From 9ad472cfb1cbf97fd7f3ea859b54dafd0948602c Mon Sep 17 00:00:00 2001 From: Aristeu Rozanski Date: Thu, 27 Aug 2026 21:55:43 -0400 Subject: mm: gup: move pmd_protnone() into gup_fast_pmd_leaf() Make pmd handling match pud handling by calling pmd_protnone() inside gup_fast_pmd_leaf(). Suggested-by: David Hildenbrand Link: https://lore.kernel.org/all/85e760cf-b994-40db-8d13-221feee55c60@redhat.com/T/#u Cc: Andrew Morton Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Signed-off-by: Aristeu Rozanski Link: https://patch.msgid.link/20260828015542.245315718@ruivo.org Signed-off-by: David Hildenbrand (Arm) --- mm/gup.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index eb898ea1ee22..bd981ab39751 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2929,6 +2929,10 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, struct folio *folio; int refs; + /* See gup_fast_pte_range() */ + if (pmd_protnone(orig)) + return 0; + if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) return 0; @@ -3024,10 +3028,6 @@ static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, return 0; if (unlikely(pmd_leaf(pmd))) { - /* See gup_fast_pte_range() */ - if (pmd_protnone(pmd)) - return 0; - if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags, pages, nr)) return 0; -- cgit v1.2.3 From 49f7113dec7d68a7cd7f99f4ae318521950cf0b9 Mon Sep 17 00:00:00 2001 From: Aristeu Rozanski Date: Thu, 27 Aug 2026 21:55:44 -0400 Subject: mm: gup: cleanup the gup_fast_*() call chain Refactor gup_fast functions so each step of the way returns the number of pages pinned. Because the previous step of the chain knows what the number it should be, less indicates an error. This way there's no need to pass *nr along. Suggested-by: David Hildenbrand Link: https://lore.kernel.org/all/85e760cf-b994-40db-8d13-221feee55c60@redhat.com/T/#u Cc: Andrew Morton Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Signed-off-by: Aristeu Rozanski Link: https://patch.msgid.link/20260828015542.334186653@ruivo.org Signed-off-by: David Hildenbrand (Arm) --- mm/gup.c | 179 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 94 insertions(+), 85 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index bd981ab39751..a4036c02e213 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2826,11 +2826,11 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) * also check pmd here to make sure pmd doesn't change (corresponds to * pmdp_collapse_flush() in the THP collapse code path). */ -static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - int ret = 0; + unsigned long nr_pages = 0; pte_t *ptep, *ptem; ptem = ptep = pte_offset_map(&pmd, addr); @@ -2892,15 +2892,13 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, goto pte_unmap; } folio_set_referenced(folio); - pages[*nr] = page; - (*nr)++; + pages[nr_pages] = page; + nr_pages++; } while (ptep++, addr += PAGE_SIZE, addr != end); - ret = 1; - pte_unmap: pte_unmap(ptem); - return ret; + return nr_pages; } #else @@ -2913,21 +2911,21 @@ pte_unmap: * get_user_pages_fast_only implementation that can pin pages. Thus it's still * useful to have gup_fast_pmd_leaf even if we can't operate on ptes. */ -static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { return 0; } #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ -static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { struct page *page; struct folio *folio; - int refs; + unsigned long nr_pages, i; /* See gup_fast_pte_range() */ if (pmd_protnone(orig)) @@ -2939,42 +2937,40 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, if (pmd_special(orig)) return 0; - refs = (end - addr) >> PAGE_SHIFT; + nr_pages = (end - addr) >> PAGE_SHIFT; page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - folio = try_grab_folio_fast(page, refs, flags); + folio = try_grab_folio_fast(page, nr_pages, flags); if (!folio) return 0; if (unlikely(pmd_val(orig) != pmd_val(pmdp_get_lockless(pmdp)))) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!gup_fast_folio_allowed(folio, flags)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } - pages += *nr; - *nr += refs; - for (; refs; refs--) + for (i = 0; i < nr_pages; i++) *(pages++) = page++; folio_set_referenced(folio); - return 1; + return nr_pages; } -static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { struct page *page; struct folio *folio; - int refs; + unsigned long nr_pages, i; if (!pud_access_permitted(orig, flags & FOLL_WRITE)) return 0; @@ -2982,41 +2978,39 @@ static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, if (pud_special(orig)) return 0; - refs = (end - addr) >> PAGE_SHIFT; + nr_pages = (end - addr) >> PAGE_SHIFT; page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - folio = try_grab_folio_fast(page, refs, flags); + folio = try_grab_folio_fast(page, nr_pages, flags); if (!folio) return 0; if (unlikely(pud_val(orig) != pud_val(pudp_get(pudp)))) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!gup_fast_folio_allowed(folio, flags)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } - pages += *nr; - *nr += refs; - for (; refs; refs--) + for (i = 0; i < nr_pages; i++) *(pages++) = page++; folio_set_referenced(folio); - return 1; + return nr_pages; } -static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pmd_range(pud_t *pudp, pud_t pud, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pmd_t *pmdp; pmdp = pmd_offset_lockless(pudp, pud, addr); @@ -3025,26 +3019,30 @@ static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, next = pmd_addr_end(addr, end); if (!pmd_present(pmd)) - return 0; + break; if (unlikely(pmd_leaf(pmd))) { - if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags, - pages, nr)) - return 0; - - } else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags, - pages, nr)) - return 0; + chunk_nr_pages = gup_fast_pmd_leaf(pmd, pmdp, addr, + next, flags, + &pages[nr_pages]); + + } else + chunk_nr_pages = gup_fast_pte_range(pmd, pmdp, addr, + next, flags, + &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pmdp++, addr = next, addr != end); - return 1; + return nr_pages; } -static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pud_t *pudp; pudp = pud_offset_lockless(p4dp, p4d, addr); @@ -3053,24 +3051,27 @@ static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, next = pud_addr_end(addr, end); if (unlikely(!pud_present(pud))) - return 0; - if (unlikely(pud_leaf(pud))) { - if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags, - pages, nr)) - return 0; - } else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags, - pages, nr)) - return 0; + break; + if (unlikely(pud_leaf(pud))) + chunk_nr_pages = gup_fast_pud_leaf(pud, pudp, addr, + next, flags, + &pages[nr_pages]); + else + chunk_nr_pages = gup_fast_pmd_range(pudp, pud, addr, + next, flags, + &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pudp++, addr = next, addr != end); - return 1; + return nr_pages; } -static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; p4d_t *p4dp; p4dp = p4d_offset_lockless(pgdp, pgd, addr); @@ -3079,20 +3080,23 @@ static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, next = p4d_addr_end(addr, end); if (!p4d_present(p4d)) - return 0; + break; BUILD_BUG_ON(p4d_leaf(p4d)); - if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags, - pages, nr)) - return 0; + chunk_nr_pages = gup_fast_pud_range(p4dp, p4d, addr, next, + flags, &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (p4dp++, addr = next, addr != end); - return 1; + return nr_pages; } -static void gup_fast_pgd_range(unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static unsigned long gup_fast_pgd_range(unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pgd_t *pgdp; pgdp = pgd_offset(current->mm, addr); @@ -3101,17 +3105,23 @@ static void gup_fast_pgd_range(unsigned long addr, unsigned long end, next = pgd_addr_end(addr, end); if (pgd_none(pgd)) - return; + break; BUILD_BUG_ON(pgd_leaf(pgd)); - if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags, - pages, nr)) - return; + chunk_nr_pages = gup_fast_p4d_range(pgdp, pgd, addr, next, + flags, &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pgdp++, addr = next, addr != end); + + return nr_pages; } #else -static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static inline unsigned long gup_fast_pgd_range(unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages) { + return 0; } #endif /* CONFIG_HAVE_GUP_FAST */ @@ -3129,8 +3139,7 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end) static unsigned long gup_fast(unsigned long start, unsigned long end, unsigned int gup_flags, struct page **pages) { - unsigned long flags; - int nr_pinned = 0; + unsigned long flags, nr_pinned; unsigned seq; if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || @@ -3154,7 +3163,7 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, * that come from callers of tlb_remove_table_sync_one(). */ local_irq_save(flags); - gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned); + nr_pinned = gup_fast_pgd_range(start, end, gup_flags, pages); local_irq_restore(flags); /* -- cgit v1.2.3 From 44817a536aaa5bfac0d316242e93e489d0a69d5f Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Tue, 15 Sep 2026 15:55:24 +0530 Subject: mm/gup_test: safely calculate GUP batch size __gup_test_ioctl() calculates the end of a GUP batch using: next = addr + nr * PAGE_SIZE; If nr is too large, it can cause next to overflow and wrap around. If it wraps, the next > end check is bypassed and a large value of nr is passed to the GUP call, even though the pages array was allocated according to gup->size. This can lead to out of bounds writes. Also, when fewer than PAGE_SIZE bytes remain, the calculated batch contains zero pages. The code still calls GUP functions with pages + i, which can point past the allocated array. Calculate nr by taking the minimum of the number of pages per call and the pages remaining in the address range. Reject zero sized and non page aligned gup->size values and zero nr_pages_per_call value. Fixes: 64c349f4ae78 ("mm: add infrastructure for get_user_pages_fast() benchmarking") Reviewed-by: Kiryl Shutsemau (Meta) Signed-off-by: Sarthak Sharma Link: https://patch.msgid.link/20260915102524.125758-1-sarthak.sharma@arm.com Signed-off-by: David Hildenbrand (Arm) --- mm/gup_test.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mm/gup_test.c b/mm/gup_test.c index 44c1cdfb9c37..5bb940371a1e 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -116,7 +116,8 @@ static int __gup_test_ioctl(unsigned int cmd, bool needs_mmap_lock = cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; - if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX) + if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX || !gup->size || + !gup->nr_pages_per_call || !PAGE_ALIGNED(gup->size)) return -EINVAL; if (check_add_overflow((unsigned long)gup->addr, (unsigned long)gup->size, &end)) @@ -139,11 +140,9 @@ static int __gup_test_ioctl(unsigned int cmd, if (nr != gup->nr_pages_per_call) break; + nr = min_t(unsigned long, nr, (end - addr) / PAGE_SIZE); + next = addr + nr * PAGE_SIZE; - if (next > end) { - next = end; - nr = (next - addr) / PAGE_SIZE; - } switch (cmd) { case GUP_FAST_BENCHMARK: -- cgit v1.2.3 From 65a79b9b693d8c7bd7e5e4865c02cce922a7cc50 Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Tue, 1 Sep 2026 14:04:52 +0530 Subject: mm/gup_test: report actual pinned bytes __gup_test_ioctl() advances addr to the end of the current batch before checking if GUP pinned the entire requested batch. If GUP pins more than 0 pages but less than the requested batch size, addr still advances by the requested batch size. The next iteration detects the partial pinning and breaks out of the loop. Again gup->size is calculated using addr - gup->addr, so it also includes the unpinned pages of the requested batch. Calculate gup->size using the actual number of pages pinned multiplied by PAGE_SIZE. Fixes: 64c349f4ae78 ("mm: add infrastructure for get_user_pages_fast() benchmarking") Reviewed-by: Kiryl Shutsemau (Meta) Signed-off-by: Sarthak Sharma Link: https://patch.msgid.link/20260901083452.115365-3-sarthak.sharma@arm.com Signed-off-by: David Hildenbrand (Arm) --- mm/gup_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/gup_test.c b/mm/gup_test.c index 5bb940371a1e..ba74bf3f4104 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -187,7 +187,7 @@ static int __gup_test_ioctl(unsigned int cmd, nr_pages = i; gup->get_delta_usec = ktime_us_delta(end_time, start_time); - gup->size = addr - gup->addr; + gup->size = nr_pages * PAGE_SIZE; /* * Take an un-benchmark-timed moment to verify DMA pinned -- cgit v1.2.3 From da01399b90a73cd00b502f7903fa5acbfac4a103 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 21 Sep 2026 11:57:29 +0200 Subject: mm/gup: honour FOLL_PIN in NOMMU __get_user_pages_locked() The !CONFIG_MMU implementation of __get_user_pages_locked() takes a bare get_page() reference for each page regardless of foll_flags: if (pages[i]) get_page(pages[i]); This is reached from pin_user_pages*() with FOLL_PIN set. unpin_user_page() is shared between MMU and NOMMU configurations and unconditionally calls gup_put_folio(..., FOLL_PIN), which subtracts GUP_PIN_COUNTING_BIAS (1024) from the folio refcount. This means that pin adds 1, and then unpin will subtract 1024. If a user maps a page (refcount 1), registers it 1023 times as an io_uring fixed buffer (1023 pin_user_pages calls -> refcount 1024), then unregisters: the first unpin_user_page subtracts 1024, refcount hits 0, the page is freed and returned to the buddy allocator. The remaining 1022 unpins write into whatever was reallocated, and the user's VMA still maps the freed page (NOMMU has no MMU to invalidate it). Reallocating the page for an io_uring pbuf_ring then lets userspace corrupt the new owner's data through the stale mapping. Use try_grab_folio() which adds GUP_PIN_COUNTING_BIAS for FOLL_PIN and 1 for FOLL_GET, mirroring the CONFIG_MMU path so pin and unpin are symmetric. Keep supporting the traditional behavior where users specify a pages array but don't set FOLL_GET. While at it, don't return NULL pointers in the page array, as this is really not expected for GUP users; instead, just fail and return -EFAULT. [ david: support traditional behavior with no FOLL_GET, extend description ] Cc: Andrew Morton Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Lance Yang Reported-by: Anthropic Fixes: 3faa52c03f44 ("mm/gup: track FOLL_PIN pages") Assisted-by: gkh_clanker_t1000 Signed-off-by: Greg Kroah-Hartman Tested-by: Lance Yang Link: https://patch.msgid.link/20260921-nommu_gup_pin-v4-1-c410041cc479@kernel.org Signed-off-by: David Hildenbrand (Arm) --- mm/gup.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index a4036c02e213..e0fba1f04055 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1983,6 +1983,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, struct vm_area_struct *vma; bool must_unlock = false; vm_flags_t vm_flags; + int ret, err = -EFAULT; long i; if (!nr_pages) @@ -1999,6 +2000,10 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, *locked = 1; } + /* See the MMU variant: support the traditional behavior. */ + if (pages && !(foll_flags & FOLL_PIN)) + foll_flags |= FOLL_GET; + /* calculate required read or write permissions. * If FOLL_FORCE is set, we only require the "MAY" flags. */ @@ -2019,8 +2024,14 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, if (pages) { pages[i] = virt_to_page((void *)start); - if (pages[i]) - get_page(pages[i]); + if (!pages[i]) + break; + ret = try_grab_folio(page_folio(pages[i]), 1, foll_flags); + if (unlikely(ret)) { + pages[i] = NULL; + err = ret; + break; + } } start = (start + PAGE_SIZE) & PAGE_MASK; @@ -2031,7 +2042,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, *locked = 0; } - return i ? : -EFAULT; + return i ? : err; } #endif /* !CONFIG_MMU */ -- cgit v1.2.3 From 1a3fed3487f89cbd08de3fd0e94cf94c47e74646 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Sun, 20 Sep 2026 04:49:45 -0400 Subject: mm/memory_hotplug: make shrink_zone_span() more robust Let's make shrink_zone_span() more robust by checking in find_smallest_section_pfn() / find_biggest_section_pfn() that the start or end PFN of the subsection is within the zone. While at it, clean up the function by factoring the core check out into subsection_overlaps_zone(). There likely is no need to check the nid first. We require SPARSEMEM_VMEMMAP_ENABLE, where pfn_to_page() is cheap, and pfn_to_nid() on CONFIG_NUMA would call pfn_to_page() either way. So let's just drop that for now. Tested-by: Yuan Liu Reviewed-by: Wei Yang Signed-off-by: Yuan Liu Link: https://patch.msgid.link/20260920084946.3266279-2-yuan1.liu@intel.com Signed-off-by: David Hildenbrand (Arm) --- mm/memory_hotplug.c | 59 ++++++++++++++++++++++------------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 226ab9cb078a..9f19876ec3ec 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -425,49 +425,39 @@ int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, return err; } -/* find the smallest valid pfn in the range [start_pfn, end_pfn) */ -static unsigned long find_smallest_section_pfn(int nid, struct zone *zone, - unsigned long start_pfn, - unsigned long end_pfn) +static bool subsection_overlaps_zone(unsigned long pfn, struct zone *zone) { - for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) { - if (unlikely(!pfn_to_online_page(start_pfn))) - continue; + const unsigned long start_pfn = ALIGN_DOWN(pfn, PAGES_PER_SUBSECTION); + const unsigned long end_pfn = start_pfn + PAGES_PER_SUBSECTION - 1; - if (unlikely(pfn_to_nid(start_pfn) != nid)) - continue; + /* All pages in a subsection are either online or offline. */ + if (unlikely(!pfn_to_online_page(start_pfn))) + return false; - if (zone != page_zone(pfn_to_page(start_pfn))) - continue; + /* Checking start+end is sufficient. */ + return zone == page_zone(pfn_to_page(start_pfn)) || + zone == page_zone(pfn_to_page(end_pfn)); +} - return start_pfn; +/* find the smallest valid pfn in the range [start_pfn, end_pfn) */ +static unsigned long find_smallest_section_pfn(struct zone *zone, + unsigned long start_pfn, unsigned long end_pfn) +{ + for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) { + if (subsection_overlaps_zone(start_pfn, zone)) + return start_pfn; } - return 0; } /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ -static unsigned long find_biggest_section_pfn(int nid, struct zone *zone, - unsigned long start_pfn, - unsigned long end_pfn) +static unsigned long find_biggest_section_pfn(struct zone *zone, + unsigned long start_pfn, unsigned long end_pfn) { - unsigned long pfn; - - /* pfn is the end pfn of a memory section. */ - pfn = end_pfn - 1; - for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) { - if (unlikely(!pfn_to_online_page(pfn))) - continue; - - if (unlikely(pfn_to_nid(pfn) != nid)) - continue; - - if (zone != page_zone(pfn_to_page(pfn))) - continue; - - return pfn; + for (; end_pfn > start_pfn; end_pfn -= PAGES_PER_SUBSECTION) { + if (subsection_overlaps_zone(end_pfn - 1, zone)) + return end_pfn - 1; } - return 0; } @@ -475,7 +465,6 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { unsigned long pfn; - int nid = zone_to_nid(zone); if (zone->zone_start_pfn == start_pfn) { /* @@ -484,7 +473,7 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, * In this case, we find second smallest valid mem_section * for shrinking zone. */ - pfn = find_smallest_section_pfn(nid, zone, end_pfn, + pfn = find_smallest_section_pfn(zone, end_pfn, zone_end_pfn(zone)); if (pfn) { zone->spanned_pages = zone_end_pfn(zone) - pfn; @@ -500,7 +489,7 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, * In this case, we find second biggest valid mem_section for * shrinking zone. */ - pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn, + pfn = find_biggest_section_pfn(zone, zone->zone_start_pfn, start_pfn); if (pfn) zone->spanned_pages = pfn - zone->zone_start_pfn + 1; -- cgit v1.2.3 From 554e941ffb5fa25964bd58edad32f3868f1b3ec5 Mon Sep 17 00:00:00 2001 From: Yuan Liu Date: Sun, 20 Sep 2026 04:49:46 -0400 Subject: mm/memory_hotplug: optimize zone contiguous check when changing pfn range When move_pfn_range_to_zone() or remove_pfn_range_from_zone() updates a zone, set_zone_contiguous() rescans the entire zone pageblock-by-pageblock to rebuild zone->contiguous. For large zones this is a significant cost during memory hotplug and hot-unplug. Add a new zone member, pages_with_online_memmap, that tracks the number of pages within the zone span that have an online memory map, including present pages and memory holes whose memory map has been initialized and for which pfn_to_online_page() succeeds. For early boot memory, pages_with_online_memmap is calculated in memmap_init_zone_range(). PFNs initialized by memmap_init_range() are included in pages_with_online_memmap, and hole PFNs for which pfn_to_online_page() succeeds are also counted in init_unavailable_range(). For hotplugged memory, pages_with_online_memmap is updated through adjust_present_page_count(), which is called during memory online and offline operations. When spanned_pages == pages_with_online_memmap, every PFN in the zone span has a valid memmap entry, so pfn_to_page() can be called for any PFN within the zone span without an additional pfn_valid() check. Note: this counter may temporarily undercount when pages with an online memory map exist outside the current zone span. Such pages are only created during boot, when initializing the memory map of pages that do not fall into any zone span. The undercount itself can only happen after boot, during memory hotplug, when growing the zone to cover such pages and later shrinking it back, which may result in a "too small" value. This is safe: it merely prevents detecting a contiguous zone. Here is an example (page numbers are just for illustration purposes): after boot: [ zone span ] [ zone pages ] spanned=10, initialized=15, online=10 online == spanned -> contiguous growing after hotplug (hotplug 5): [ zone span ] [ zone pages ] [ zone pages ] spanned=30, initialized=20, online=15 online != spanned -> not contiguous shrinking after hotunplug (hotunplug 5 again): [ zone span ] [ zone pages ] spanned=15, initialized=15, online=10 online != spanned -> not contiguous although contiguous The contiguity check using pages_with_online_memmap is stricter than the old pageblock-by-pageblock scan. The old set_zone_contiguous() iterated at pageblock granularity via pageblock_pfn_to_page(), so a zone could be marked contiguous even if a subsection-sized hole existed within a pageblock. The new check requires spanned_pages == pages_with_online_memmap, meaning every PFN in the zone span must satisfy pfn_to_online_page(). The following test cases of memory hotplug for a VM [1], tested in the environment [2], show that this optimization can significantly reduce the memory hotplug time [3]. +----------------+------+---------------+--------------+----------------+ | | Size | Time (before) | Time (after) | Time Reduction | | +------+---------------+--------------+----------------+ | Plug Memory | 256G | 10s | 3s | 70% | | +------+---------------+--------------+----------------+ | | 512G | 36s | 7s | 81% | +----------------+------+---------------+--------------+----------------+ +----------------+------+---------------+--------------+----------------+ | | Size | Time (before) | Time (after) | Time Reduction | | +------+---------------+--------------+----------------+ | Unplug Memory | 256G | 11s | 4s | 64% | | +------+---------------+--------------+----------------+ | | 512G | 36s | 9s | 75% | +----------------+------+---------------+--------------+----------------+ [1] Qemu commands to hotplug 256G/512G memory for a VM: object_add memory-backend-ram,id=hotmem0,size=256G/512G,share=on device_add virtio-mem-pci,id=vmem1,memdev=hotmem0,bus=port1 qom-set vmem1 requested-size 256G/512G (Plug Memory) qom-set vmem1 requested-size 0G (Unplug Memory) [2] Hardware : Intel Icelake server Guest Kernel : v7.3-rc3 Qemu : v9.0.0 Launch VM : qemu-system-x86_64 -accel kvm -cpu host \ -drive file=./Centos10_cloud.qcow2,format=qcow2,if=virtio \ -drive file=./seed.img,format=raw,if=virtio \ -smp 3,cores=3,threads=1,sockets=1,maxcpus=3 \ -m 2G,slots=10,maxmem=2052472M \ -device pcie-root-port,id=port1,bus=pcie.0,slot=1,multifunction=on \ -device pcie-root-port,id=port2,bus=pcie.0,slot=2 \ -nographic -machine q35 \ -nic user,hostfwd=tcp::3000-:22 Guest kernel auto-onlines newly added memory blocks: echo online > /sys/devices/system/memory/auto_online_blocks [3] The time from typing the QEMU commands in [1] to when the output of 'grep MemTotal /proc/meminfo' on Guest reflects that all hotplugged memory is recognized. Reported-by: Nanhai Zou Reported-by: Chen Zhang Tested-by: Yuan Liu Reviewed-by: Jason Zeng Reviewed-by: Chen Yu Reviewed-by: Pan Deng Co-developed-by: Tianyou Li Signed-off-by: Tianyou Li Signed-off-by: Yuan Liu Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Wei Yang Link: https://patch.msgid.link/20260920084946.3266279-3-yuan1.liu@intel.com Signed-off-by: David Hildenbrand (Arm) --- Documentation/mm/physical_memory.rst | 6 ++++ drivers/base/memory.c | 7 +++- include/linux/mmzone.h | 69 ++++++++++++++++++++++++++++++++++++ mm/memory_hotplug.c | 12 ++----- mm/mm_init.c | 64 ++++++++++++++++++--------------- mm/mm_init.h | 6 ---- mm/page_alloc.h | 2 +- 7 files changed, 120 insertions(+), 46 deletions(-) diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst index a09407d72973..2e67e8b23a99 100644 --- a/Documentation/mm/physical_memory.rst +++ b/Documentation/mm/physical_memory.rst @@ -480,6 +480,12 @@ General ``present_pages`` should use ``get_online_mems()`` to get a stable value. It is initialized by ``calculate_node_totalpages()``. +``pages_with_online_memmap`` + Pages within the zone that have an online memory map: present pages and + memory holes whose memory map has been initialized and + ``pfn_to_online_page()`` succeeds. See the comment for + ``pages_with_online_memmap`` in ``include/linux/mmzone.h`` for more details. + ``present_early_pages`` The present pages existing within the zone located on memory available since early boot, excluding hotplugged memory. Defined only when diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 5eead3346f1e..28f9503f6a71 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -255,6 +255,7 @@ static int memory_block_online(struct memory_block *mem) nr_vmemmap_pages = mem->altmap->free; mem_hotplug_begin(); + clear_zone_contiguous(zone); if (nr_vmemmap_pages) { ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); if (ret) @@ -279,6 +280,7 @@ static int memory_block_online(struct memory_block *mem) mem->zone = zone; out: + set_zone_contiguous(zone); mem_hotplug_done(); return ret; } @@ -304,6 +306,7 @@ static int memory_block_offline(struct memory_block *mem) nr_vmemmap_pages = mem->altmap->free; mem_hotplug_begin(); + clear_zone_contiguous(mem->zone); if (nr_vmemmap_pages) adjust_present_page_count(pfn_to_page(start_pfn), mem->group, -nr_vmemmap_pages); @@ -321,8 +324,10 @@ static int memory_block_offline(struct memory_block *mem) if (nr_vmemmap_pages) mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); - mem->zone = NULL; out: + set_zone_contiguous(mem->zone); + if (!ret) + mem->zone = NULL; mem_hotplug_done(); return ret; } diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 94f9c3ff5416..7befe50223d1 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1043,6 +1043,42 @@ struct zone { * cma pages is present pages that are assigned for CMA use * (MIGRATE_CMA). * + * pages_with_online_memmap tracks pages within the zone that have + * an online memory map: present pages and memory holes whose + * memory map has been initialized and pfn_to_online_page() + * succeeds. When spanned_pages == pages_with_online_memmap, + * pfn_to_page() can be performed without further checks on any + * PFN within the zone span. + * + * Note: this counter may temporarily undercount when pages with an + * online memory map exist outside the current zone span. Such pages + * are only created during boot, when initializing the memory map of + * pages that do not fall into any zone span. The undercount itself + * can only happen after boot, during memory hotplug, when growing + * the zone to cover such pages and later shrinking it back, which + * may result in a "too small" value. This is safe: it merely + * prevents detecting a contiguous zone. + * + * Here is an example (page numbers are just for illustration + * purposes): + * after boot: + * [ zone span ] + * [ zone pages ] + * spanned=10, initialized=15, online=10 + * online == spanned -> contiguous + * + * growing after hotplug (hotplug 5): + * [ zone span ] + * [ zone pages ] [ zone pages ] + * spanned=30, initialized=20, online=15 + * online != spanned -> not contiguous + * + * shrinking after hotunplug (hotunplug 5 again): + * [ zone span ] + * [ zone pages ] + * spanned=15, initialized=15, online=10 + * online != spanned -> not contiguous although contiguous + * * So present_pages may be used by memory hotplug or memory power * management logic to figure out unmanaged pages by checking * (present_pages - managed_pages). And managed_pages should be used @@ -1067,6 +1103,7 @@ struct zone { atomic_long_t managed_pages; unsigned long spanned_pages; unsigned long present_pages; + unsigned long pages_with_online_memmap; #if defined(CONFIG_MEMORY_HOTPLUG) unsigned long present_early_pages; #endif @@ -1694,6 +1731,38 @@ static inline bool zone_is_zone_device(const struct zone *zone) } #endif +/** + * zone_is_contiguous - test whether a zone is contiguous + * @zone: the zone to test. + * + * In a contiguous zone, it is valid to call pfn_to_page() on any PFN in the + * spanned zone without requiring pfn_valid() or pfn_to_online_page() checks. + * + * Note that missing synchronization with memory offlining makes any PFN + * traversal prone to races. + * + * ZONE_DEVICE zones are always marked non-contiguous. + * + * Return: true if contiguous, otherwise false. + */ +static inline bool zone_is_contiguous(const struct zone *zone) +{ + return READ_ONCE(zone->contiguous); +} + +static inline void set_zone_contiguous(struct zone *zone) +{ + if (zone_is_zone_device(zone)) + return; + if (zone->spanned_pages == zone->pages_with_online_memmap) + WRITE_ONCE(zone->contiguous, true); +} + +static inline void clear_zone_contiguous(struct zone *zone) +{ + WRITE_ONCE(zone->contiguous, false); +} + /* * Returns true if a zone has pages managed by the buddy allocator. * All the reclaim decisions have to use this function rather than diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 9f19876ec3ec..100941d1b828 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -549,18 +549,13 @@ void remove_pfn_range_from_zone(struct zone *zone, /* * Zone shrinking code cannot properly deal with ZONE_DEVICE. So - * we will not try to shrink the zones - which is okay as - * set_zone_contiguous() cannot deal with ZONE_DEVICE either way. + * we will not try to shrink it. */ if (zone_is_zone_device(zone)) return; - clear_zone_contiguous(zone); - shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); update_pgdat_span(pgdat); - - set_zone_contiguous(zone); } /** @@ -738,8 +733,6 @@ void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, struct pglist_data *pgdat = zone->zone_pgdat; int nid = pgdat->node_id; - clear_zone_contiguous(zone); - if (zone_is_empty(zone)) init_currently_empty_zone(zone, start_pfn, nr_pages); resize_zone_range(zone, start_pfn, nr_pages); @@ -767,8 +760,6 @@ void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0, MEMINIT_HOTPLUG, altmap, migratetype, isolate_pageblock); - - set_zone_contiguous(zone); } struct auto_movable_stats { @@ -1064,6 +1055,7 @@ void adjust_present_page_count(struct page *page, struct memory_group *group, if (early_section(__pfn_to_section(page_to_pfn(page)))) zone->present_early_pages += nr_pages; zone->present_pages += nr_pages; + zone->pages_with_online_memmap += nr_pages; zone->zone_pgdat->node_present_pages += nr_pages; if (group && movable) diff --git a/mm/mm_init.c b/mm/mm_init.c index 1533aebafb68..259aab9ac941 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -816,23 +816,41 @@ void __meminit init_deferred_page(unsigned long pfn, int nid) * the zone boundary; the pages in such holes will be prepended to the * zone/node above the hole except for the trailing pages in the last * section that will be appended to the zone/node below. + * + * The function counts pages that should be added to + * zone->pages_with_online_memmap. With CONFIG_SPARSEMEM_VMEMMAP there could + * be offline subsections even though the entire memory map is valid and all + * the early sections are online. Count only pages in online subsections + * (see pfn_to_online_page()). */ -static void __init init_unavailable_range(unsigned long spfn, - unsigned long epfn, - int zone, int node) +static unsigned long __init init_unavailable_range(unsigned long spfn, + unsigned long epfn, + int zone, int node) { + unsigned long next_chunk_pfn __maybe_unused = spfn; unsigned long pfn; - u64 pgcnt = 0; + u64 online_pgcnt = 0, pgcnt = 0; + bool is_online = true; for_each_valid_pfn(pfn, spfn, epfn) { __init_single_page(pfn_to_page(pfn), pfn, zone, node); __SetPageReserved(pfn_to_page(pfn)); pgcnt++; + +#ifdef CONFIG_SPARSEMEM_VMEMMAP + if (pfn >= next_chunk_pfn) { + is_online = pfn_section_valid(__pfn_to_section(pfn), pfn); + next_chunk_pfn = min(SUBSECTION_ALIGN_UP(pfn + 1), epfn); + } +#endif + if (is_online) + online_pgcnt++; } if (pgcnt) pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n", node, zone_names[zone], pgcnt); + return online_pgcnt; } /* @@ -930,9 +948,21 @@ static void __init memmap_init_zone_range(struct zone *zone, memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE, false); + zone->pages_with_online_memmap += end_pfn - start_pfn; - if (*hole_pfn < start_pfn) - init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); + if (*hole_pfn < start_pfn) { + unsigned long hole_start_pfn = *hole_pfn; + unsigned long pgcnt; + + if (hole_start_pfn < zone_start_pfn) { + init_unavailable_range(hole_start_pfn, zone_start_pfn, + zone_id, nid); + hole_start_pfn = zone_start_pfn; + } + pgcnt = init_unavailable_range(hole_start_pfn, start_pfn, + zone_id, nid); + zone->pages_with_online_memmap += pgcnt; + } *hole_pfn = end_pfn; } @@ -2188,28 +2218,6 @@ void __init init_cma_pageblock(struct page *page) } #endif -void set_zone_contiguous(struct zone *zone) -{ - unsigned long block_start_pfn = zone->zone_start_pfn; - unsigned long block_end_pfn; - - block_end_pfn = pageblock_end_pfn(block_start_pfn); - for (; block_start_pfn < zone_end_pfn(zone); - block_start_pfn = block_end_pfn, - block_end_pfn += pageblock_nr_pages) { - - block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); - - if (!__pageblock_pfn_to_page(block_start_pfn, - block_end_pfn, zone)) - return; - cond_resched(); - } - - /* We confirm that there is no hole */ - zone->contiguous = true; -} - /* * Check if a PFN range intersects multiple zones on one or more * NUMA nodes. Specify the @nid argument if it is known that this diff --git a/mm/mm_init.h b/mm/mm_init.h index 39f75df9be1c..520d53ecc68d 100644 --- a/mm/mm_init.h +++ b/mm/mm_init.h @@ -20,15 +20,9 @@ struct vmem_altmap; /* perform sanity checks on struct pages being allocated or freed */ DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); -void set_zone_contiguous(struct zone *zone); bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, unsigned long nr_pages); -static inline void clear_zone_contiguous(struct zone *zone) -{ - zone->contiguous = false; -} - void memblock_free_pages(unsigned long pfn, unsigned int order); void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, diff --git a/mm/page_alloc.h b/mm/page_alloc.h index b9259deddb59..d4182f7cb7dd 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -214,7 +214,7 @@ extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, unsigned long end_pfn, struct zone *zone) { - if (zone->contiguous) + if (zone_is_contiguous(zone)) return pfn_to_page(start_pfn); return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); -- cgit v1.2.3 From c31bb839fb829958101250ee5acfa4f94fdb7ecd Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Thu, 1 Oct 2026 00:04:32 +0800 Subject: mm/memory_hotplug: fix missing rollback in __add_pages() __add_pages() returns on a sparse_add_section() failure without removing the sections already added in the same request. For memremap_pages(), the failed range is not counted in pgmap->nr_range, so memunmap_pages() skips it. The sections already added in that range retain their vmemmap mappings and subsection bits. Retrying a section-aligned range can then fail with -EEXIST. Save the initial PFN and remove [start_pfn, pfn) on failure. For a vmemmap population failure, section_activate() already cleans up the current section, so the rollback excludes it. If the first section fails, __remove_pages() receives an empty range and does nothing. Link: https://lore.kernel.org/all/BAD58999-1EDD-4A37-ABA0-DB1BD8AB3453@linux.dev/ Suggested-by: Muchun Song Signed-off-by: Lance Yang Fixes: ba72b4c8cf60 ("mm/sparsemem: support sub-section hotplug") Acked-by: Muchun Song Link: https://patch.msgid.link/20260930160432.5564-1-lance.yang@linux.dev Signed-off-by: David Hildenbrand (Arm) --- mm/memory_hotplug.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 100941d1b828..3f2bb2828ab1 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -384,6 +384,7 @@ EXPORT_SYMBOL_GPL(pfn_to_online_page); int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, struct mhp_params *params) { + const unsigned long start_pfn = pfn; const unsigned long end_pfn = pfn + nr_pages; unsigned long cur_nr_pages; int err; @@ -417,8 +418,11 @@ int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, SECTION_ALIGN_UP(pfn + 1) - pfn); err = sparse_add_section(nid, pfn, cur_nr_pages, altmap, params->pgmap); - if (err) + if (err) { + __remove_pages(start_pfn, pfn - start_pfn, altmap, + params->pgmap); break; + } cond_resched(); } vmemmap_populate_print_last(); -- cgit v1.2.3