diff options
| -rw-r--r-- | Documentation/mm/physical_memory.rst | 6 | ||||
| -rw-r--r-- | drivers/base/memory.c | 7 | ||||
| -rw-r--r-- | include/linux/mmzone.h | 69 | ||||
| -rw-r--r-- | mm/gup.c | 17 | ||||
| -rw-r--r-- | mm/memory_hotplug.c | 77 | ||||
| -rw-r--r-- | mm/mm_init.c | 64 | ||||
| -rw-r--r-- | mm/mm_init.h | 6 | ||||
| -rw-r--r-- | mm/page_alloc.h | 2 |
8 files changed, 163 insertions, 85 deletions
diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst index a09407d72973..2e67e8b23a99 100644 --- a/Documentation/mm/physical_memory.rst +++ b/Documentation/mm/physical_memory.rst @@ -480,6 +480,12 @@ General ``present_pages`` should use ``get_online_mems()`` to get a stable value. It is initialized by ``calculate_node_totalpages()``. +``pages_with_online_memmap`` + Pages within the zone that have an online memory map: present pages and + memory holes whose memory map has been initialized and + ``pfn_to_online_page()`` succeeds. See the comment for + ``pages_with_online_memmap`` in ``include/linux/mmzone.h`` for more details. + ``present_early_pages`` The present pages existing within the zone located on memory available since early boot, excluding hotplugged memory. Defined only when diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 5eead3346f1e..28f9503f6a71 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -255,6 +255,7 @@ static int memory_block_online(struct memory_block *mem) nr_vmemmap_pages = mem->altmap->free; mem_hotplug_begin(); + clear_zone_contiguous(zone); if (nr_vmemmap_pages) { ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); if (ret) @@ -279,6 +280,7 @@ static int memory_block_online(struct memory_block *mem) mem->zone = zone; out: + set_zone_contiguous(zone); mem_hotplug_done(); return ret; } @@ -304,6 +306,7 @@ static int memory_block_offline(struct memory_block *mem) nr_vmemmap_pages = mem->altmap->free; mem_hotplug_begin(); + clear_zone_contiguous(mem->zone); if (nr_vmemmap_pages) adjust_present_page_count(pfn_to_page(start_pfn), mem->group, -nr_vmemmap_pages); @@ -321,8 +324,10 @@ static int memory_block_offline(struct memory_block *mem) if (nr_vmemmap_pages) mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); - mem->zone = NULL; out: + set_zone_contiguous(mem->zone); + if (!ret) + mem->zone = NULL; mem_hotplug_done(); return ret; } diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 8e4e0bda3b58..d44b04589f48 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1019,6 +1019,42 @@ struct zone { * cma pages is present pages that are assigned for CMA use * (MIGRATE_CMA). * + * pages_with_online_memmap tracks pages within the zone that have + * an online memory map: present pages and memory holes whose + * memory map has been initialized and pfn_to_online_page() + * succeeds. When spanned_pages == pages_with_online_memmap, + * pfn_to_page() can be performed without further checks on any + * PFN within the zone span. + * + * Note: this counter may temporarily undercount when pages with an + * online memory map exist outside the current zone span. Such pages + * are only created during boot, when initializing the memory map of + * pages that do not fall into any zone span. The undercount itself + * can only happen after boot, during memory hotplug, when growing + * the zone to cover such pages and later shrinking it back, which + * may result in a "too small" value. This is safe: it merely + * prevents detecting a contiguous zone. + * + * Here is an example (page numbers are just for illustration + * purposes): + * after boot: + * [ zone span ] + * [ zone pages ] + * spanned=10, initialized=15, online=10 + * online == spanned -> contiguous + * + * growing after hotplug (hotplug 5): + * [ zone span ] + * [ zone pages ] [ zone pages ] + * spanned=30, initialized=20, online=15 + * online != spanned -> not contiguous + * + * shrinking after hotunplug (hotunplug 5 again): + * [ zone span ] + * [ zone pages ] + * spanned=15, initialized=15, online=10 + * online != spanned -> not contiguous although contiguous + * * So present_pages may be used by memory hotplug or memory power * management logic to figure out unmanaged pages by checking * (present_pages - managed_pages). And managed_pages should be used @@ -1043,6 +1079,7 @@ struct zone { atomic_long_t managed_pages; unsigned long spanned_pages; unsigned long present_pages; + unsigned long pages_with_online_memmap; #if defined(CONFIG_MEMORY_HOTPLUG) unsigned long present_early_pages; #endif @@ -1670,6 +1707,38 @@ static inline bool zone_is_zone_device(const struct zone *zone) } #endif +/** + * zone_is_contiguous - test whether a zone is contiguous + * @zone: the zone to test. + * + * In a contiguous zone, it is valid to call pfn_to_page() on any PFN in the + * spanned zone without requiring pfn_valid() or pfn_to_online_page() checks. + * + * Note that missing synchronization with memory offlining makes any PFN + * traversal prone to races. + * + * ZONE_DEVICE zones are always marked non-contiguous. + * + * Return: true if contiguous, otherwise false. + */ +static inline bool zone_is_contiguous(const struct zone *zone) +{ + return READ_ONCE(zone->contiguous); +} + +static inline void set_zone_contiguous(struct zone *zone) +{ + if (zone_is_zone_device(zone)) + return; + if (zone->spanned_pages == zone->pages_with_online_memmap) + WRITE_ONCE(zone->contiguous, true); +} + +static inline void clear_zone_contiguous(struct zone *zone) +{ + WRITE_ONCE(zone->contiguous, false); +} + /* * Returns true if a zone has pages managed by the buddy allocator. * All the reclaim decisions have to use this function rather than @@ -1983,6 +1983,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, struct vm_area_struct *vma; bool must_unlock = false; vm_flags_t vm_flags; + int ret, err = -EFAULT; long i; if (!nr_pages) @@ -1999,6 +2000,10 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, *locked = 1; } + /* See the MMU variant: support the traditional behavior. */ + if (pages && !(foll_flags & FOLL_PIN)) + foll_flags |= FOLL_GET; + /* calculate required read or write permissions. * If FOLL_FORCE is set, we only require the "MAY" flags. */ @@ -2019,8 +2024,14 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, if (pages) { pages[i] = virt_to_page((void *)start); - if (pages[i]) - get_page(pages[i]); + if (!pages[i]) + break; + ret = try_grab_folio(page_folio(pages[i]), 1, foll_flags); + if (unlikely(ret)) { + pages[i] = NULL; + err = ret; + break; + } } start = (start + PAGE_SIZE) & PAGE_MASK; @@ -2031,7 +2042,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, *locked = 0; } - return i ? : -EFAULT; + return i ? : err; } #endif /* !CONFIG_MMU */ diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index d7a59167bec4..dd9c8a519bfe 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -385,6 +385,7 @@ EXPORT_SYMBOL_GPL(pfn_to_online_page); int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, struct mhp_params *params) { + const unsigned long start_pfn = pfn; const unsigned long end_pfn = pfn + nr_pages; unsigned long cur_nr_pages; int err; @@ -418,57 +419,50 @@ int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, SECTION_ALIGN_UP(pfn + 1) - pfn); err = sparse_add_section(nid, pfn, cur_nr_pages, altmap, params->pgmap); - if (err) + if (err) { + __remove_pages(start_pfn, pfn - start_pfn, altmap, + params->pgmap); break; + } cond_resched(); } vmemmap_populate_print_last(); return err; } -/* find the smallest valid pfn in the range [start_pfn, end_pfn) */ -static unsigned long find_smallest_section_pfn(int nid, struct zone *zone, - unsigned long start_pfn, - unsigned long end_pfn) +static bool subsection_overlaps_zone(unsigned long pfn, struct zone *zone) { - for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) { - if (unlikely(!pfn_to_online_page(start_pfn))) - continue; + const unsigned long start_pfn = ALIGN_DOWN(pfn, PAGES_PER_SUBSECTION); + const unsigned long end_pfn = start_pfn + PAGES_PER_SUBSECTION - 1; - if (unlikely(pfn_to_nid(start_pfn) != nid)) - continue; + /* All pages in a subsection are either online or offline. */ + if (unlikely(!pfn_to_online_page(start_pfn))) + return false; - if (zone != page_zone(pfn_to_page(start_pfn))) - continue; + /* Checking start+end is sufficient. */ + return zone == page_zone(pfn_to_page(start_pfn)) || + zone == page_zone(pfn_to_page(end_pfn)); +} - return start_pfn; +/* find the smallest valid pfn in the range [start_pfn, end_pfn) */ +static unsigned long find_smallest_section_pfn(struct zone *zone, + unsigned long start_pfn, unsigned long end_pfn) +{ + for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) { + if (subsection_overlaps_zone(start_pfn, zone)) + return start_pfn; } - return 0; } /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ -static unsigned long find_biggest_section_pfn(int nid, struct zone *zone, - unsigned long start_pfn, - unsigned long end_pfn) +static unsigned long find_biggest_section_pfn(struct zone *zone, + unsigned long start_pfn, unsigned long end_pfn) { - unsigned long pfn; - - /* pfn is the end pfn of a memory section. */ - pfn = end_pfn - 1; - for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) { - if (unlikely(!pfn_to_online_page(pfn))) - continue; - - if (unlikely(pfn_to_nid(pfn) != nid)) - continue; - - if (zone != page_zone(pfn_to_page(pfn))) - continue; - - return pfn; + for (; end_pfn > start_pfn; end_pfn -= PAGES_PER_SUBSECTION) { + if (subsection_overlaps_zone(end_pfn - 1, zone)) + return end_pfn - 1; } - return 0; } @@ -476,7 +470,6 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { unsigned long pfn; - int nid = zone_to_nid(zone); if (zone->zone_start_pfn == start_pfn) { /* @@ -485,7 +478,7 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, * In this case, we find second smallest valid mem_section * for shrinking zone. */ - pfn = find_smallest_section_pfn(nid, zone, end_pfn, + pfn = find_smallest_section_pfn(zone, end_pfn, zone_end_pfn(zone)); if (pfn) { zone->spanned_pages = zone_end_pfn(zone) - pfn; @@ -501,7 +494,7 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, * In this case, we find second biggest valid mem_section for * shrinking zone. */ - pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn, + pfn = find_biggest_section_pfn(zone, zone->zone_start_pfn, start_pfn); if (pfn) zone->spanned_pages = pfn - zone->zone_start_pfn + 1; @@ -562,18 +555,13 @@ void remove_pfn_range_from_zone(struct zone *zone, /* * Zone shrinking code cannot properly deal with ZONE_DEVICE. So - * we will not try to shrink the zones - which is okay as - * set_zone_contiguous() cannot deal with ZONE_DEVICE either way. + * we will not try to shrink it. */ if (zone_is_zone_device(zone)) return; - clear_zone_contiguous(zone); - shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); update_pgdat_span(pgdat); - - set_zone_contiguous(zone); } /** @@ -751,8 +739,6 @@ void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, struct pglist_data *pgdat = zone->zone_pgdat; int nid = pgdat->node_id; - clear_zone_contiguous(zone); - if (zone_is_empty(zone)) init_currently_empty_zone(zone, start_pfn, nr_pages); resize_zone_range(zone, start_pfn, nr_pages); @@ -780,8 +766,6 @@ void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0, MEMINIT_HOTPLUG, altmap, migratetype, isolate_pageblock); - - set_zone_contiguous(zone); } struct auto_movable_stats { @@ -1077,6 +1061,7 @@ void adjust_present_page_count(struct page *page, struct memory_group *group, if (early_section(__pfn_to_section(page_to_pfn(page)))) zone->present_early_pages += nr_pages; zone->present_pages += nr_pages; + zone->pages_with_online_memmap += nr_pages; zone->zone_pgdat->node_present_pages += nr_pages; if (group && movable) diff --git a/mm/mm_init.c b/mm/mm_init.c index 56bb4567a494..2e55be145738 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -823,23 +823,41 @@ void __meminit init_deferred_page(unsigned long pfn, int nid) * the zone boundary; the pages in such holes will be prepended to the * zone/node above the hole except for the trailing pages in the last * section that will be appended to the zone/node below. + * + * The function counts pages that should be added to + * zone->pages_with_online_memmap. With CONFIG_SPARSEMEM_VMEMMAP there could + * be offline subsections even though the entire memory map is valid and all + * the early sections are online. Count only pages in online subsections + * (see pfn_to_online_page()). */ -static void __init init_unavailable_range(unsigned long spfn, - unsigned long epfn, - int zone, int node) +static unsigned long __init init_unavailable_range(unsigned long spfn, + unsigned long epfn, + int zone, int node) { + unsigned long next_chunk_pfn __maybe_unused = spfn; unsigned long pfn; - u64 pgcnt = 0; + u64 online_pgcnt = 0, pgcnt = 0; + bool is_online = true; for_each_valid_pfn(pfn, spfn, epfn) { __init_single_page(pfn_to_page(pfn), pfn, zone, node); __SetPageReserved(pfn_to_page(pfn)); pgcnt++; + +#ifdef CONFIG_SPARSEMEM_VMEMMAP + if (pfn >= next_chunk_pfn) { + is_online = pfn_section_valid(__pfn_to_section(pfn), pfn); + next_chunk_pfn = min(SUBSECTION_ALIGN_UP(pfn + 1), epfn); + } +#endif + if (is_online) + online_pgcnt++; } if (pgcnt) pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n", node, zone_names[zone], pgcnt); + return online_pgcnt; } /* @@ -942,9 +960,21 @@ static void __init memmap_init_zone_range(struct zone *zone, memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE, false); + zone->pages_with_online_memmap += end_pfn - start_pfn; - if (*hole_pfn < start_pfn) - init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); + if (*hole_pfn < start_pfn) { + unsigned long hole_start_pfn = *hole_pfn; + unsigned long pgcnt; + + if (hole_start_pfn < zone_start_pfn) { + init_unavailable_range(hole_start_pfn, zone_start_pfn, + zone_id, nid); + hole_start_pfn = zone_start_pfn; + } + pgcnt = init_unavailable_range(hole_start_pfn, start_pfn, + zone_id, nid); + zone->pages_with_online_memmap += pgcnt; + } *hole_pfn = end_pfn; } @@ -2236,28 +2266,6 @@ void __init init_cma_pageblock(struct page *page) } #endif -void set_zone_contiguous(struct zone *zone) -{ - unsigned long block_start_pfn = zone->zone_start_pfn; - unsigned long block_end_pfn; - - block_end_pfn = pageblock_end_pfn(block_start_pfn); - for (; block_start_pfn < zone_end_pfn(zone); - block_start_pfn = block_end_pfn, - block_end_pfn += pageblock_nr_pages) { - - block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); - - if (!__pageblock_pfn_to_page(block_start_pfn, - block_end_pfn, zone)) - return; - cond_resched(); - } - - /* We confirm that there is no hole */ - zone->contiguous = true; -} - /* * Check if a PFN range intersects multiple zones on one or more * NUMA nodes. Specify the @nid argument if it is known that this diff --git a/mm/mm_init.h b/mm/mm_init.h index c9fc35e7e9f1..393006c1c6ef 100644 --- a/mm/mm_init.h +++ b/mm/mm_init.h @@ -20,15 +20,9 @@ struct vmem_altmap; /* perform sanity checks on struct pages being allocated or freed */ DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); -void set_zone_contiguous(struct zone *zone); bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, unsigned long nr_pages); -static inline void clear_zone_contiguous(struct zone *zone) -{ - zone->contiguous = false; -} - void memblock_free_pages(unsigned long pfn, unsigned int order); void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, diff --git a/mm/page_alloc.h b/mm/page_alloc.h index b9259deddb59..d4182f7cb7dd 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -214,7 +214,7 @@ extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, unsigned long end_pfn, struct zone *zone) { - if (zone->contiguous) + if (zone_is_contiguous(zone)) return pfn_to_page(start_pfn); return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); |