diff options
| -rw-r--r-- | mm/internal.h | 6 | ||||
| -rw-r--r-- | mm/mm_init.c | 1 | ||||
| -rw-r--r-- | mm/page_alloc.c | 123 | ||||
| -rw-r--r-- | mm/vmscan.c | 6 |
4 files changed, 130 insertions, 6 deletions
diff --git a/mm/internal.h b/mm/internal.h index 88ac7be741ca..71d160437205 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -400,9 +400,15 @@ static inline void mminit_verify_zonelist(void) #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT #define __defermem_init __meminit #define __defer_init __meminit + +void deferred_init_memmap(int nid); #else #define __defermem_init #define __defer_init __init + +static inline void deferred_init_memmap(int nid) +{ +} #endif /* mminit_validate_memmodel_limits is independent of CONFIG_DEBUG_MEMORY_INIT */ diff --git a/mm/mm_init.c b/mm/mm_init.c index 5f420f7fafa1..28fbf87b20aa 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -11,6 +11,7 @@ #include <linux/export.h> #include <linux/memory.h> #include <linux/notifier.h> +#include <linux/sched.h> #include "internal.h" #ifdef CONFIG_DEBUG_MEMORY_INIT diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 7af45b2e8870..c30f5a0535fd 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -252,6 +252,14 @@ static inline bool __defermem_init early_page_uninitialised(unsigned long pfn) return false; } +static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid) +{ + if (pfn >= NODE_DATA(nid)->first_deferred_pfn) + return true; + + return false; +} + /* * Returns false when the remaining initialisation should be deferred until * later in the boot cycle when it can be parallelised. @@ -284,6 +292,11 @@ static inline bool early_page_uninitialised(unsigned long pfn) return false; } +static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid) +{ + return false; +} + static inline bool update_defer_init(pg_data_t *pgdat, unsigned long pfn, unsigned long zone_end, unsigned long *nr_initialised) @@ -866,20 +879,51 @@ static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone, return __init_single_page(pfn_to_page(pfn), pfn, zone, nid); } +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +static void init_reserved_page(unsigned long pfn) +{ + pg_data_t *pgdat; + int nid, zid; + + if (!early_page_uninitialised(pfn)) + return; + + nid = early_pfn_to_nid(pfn); + pgdat = NODE_DATA(nid); + + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + struct zone *zone = &pgdat->node_zones[zid]; + + if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) + break; + } + __init_single_pfn(pfn, zid, nid); +} +#else +static inline void init_reserved_page(unsigned long pfn) +{ +} +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + /* * Initialised pages do not have PageReserved set. This function is * called for each range allocated by the bootmem allocator and * marks the pages PageReserved. The remaining valid pages are later * sent to the buddy page allocator. */ -void reserve_bootmem_region(unsigned long start, unsigned long end) +void __meminit reserve_bootmem_region(unsigned long start, unsigned long end) { unsigned long start_pfn = PFN_DOWN(start); unsigned long end_pfn = PFN_UP(end); - for (; start_pfn < end_pfn; start_pfn++) - if (pfn_valid(start_pfn)) - SetPageReserved(pfn_to_page(start_pfn)); + for (; start_pfn < end_pfn; start_pfn++) { + if (pfn_valid(start_pfn)) { + struct page *page = pfn_to_page(start_pfn); + + init_reserved_page(start_pfn); + SetPageReserved(page); + } + } } static bool free_pages_prepare(struct page *page, unsigned int order) @@ -1017,6 +1061,74 @@ void __defer_init __free_pages_bootmem(struct page *page, unsigned long pfn, return __free_pages_boot_core(page, pfn, order); } +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +/* Initialise remaining memory on a node */ +void __defermem_init deferred_init_memmap(int nid) +{ + struct mminit_pfnnid_cache nid_init_state = { }; + unsigned long start = jiffies; + unsigned long nr_pages = 0; + unsigned long walk_start, walk_end; + int i, zid; + struct zone *zone; + pg_data_t *pgdat = NODE_DATA(nid); + unsigned long first_init_pfn = pgdat->first_deferred_pfn; + + if (first_init_pfn == ULONG_MAX) + return; + + /* Sanity check boundaries */ + BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); + BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); + pgdat->first_deferred_pfn = ULONG_MAX; + + /* Only the highest zone is deferred so find it */ + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + zone = pgdat->node_zones + zid; + if (first_init_pfn < zone_end_pfn(zone)) + break; + } + + for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) { + unsigned long pfn, end_pfn; + + end_pfn = min(walk_end, zone_end_pfn(zone)); + pfn = first_init_pfn; + if (pfn < walk_start) + pfn = walk_start; + if (pfn < zone->zone_start_pfn) + pfn = zone->zone_start_pfn; + + for (; pfn < end_pfn; pfn++) { + struct page *page; + + if (!pfn_valid(pfn)) + continue; + + if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) + continue; + + if (page->flags) { + VM_BUG_ON(page_zone(page) != zone); + continue; + } + + __init_single_page(page, pfn, zid, nid); + __free_pages_boot_core(page, pfn, 0); + nr_pages++; + cond_resched(); + } + first_init_pfn = max(end_pfn, first_init_pfn); + } + + /* Sanity check that the next zone really is unpopulated */ + WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); + + pr_info("kswapd %d initialised %lu pages in %ums\n", nid, nr_pages, + jiffies_to_msecs(jiffies - start)); +} +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + #ifdef CONFIG_CMA /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ void __init init_cma_reserved_pageblock(struct page *page) @@ -4329,6 +4441,9 @@ static void setup_zone_migrate_reserve(struct zone *zone) zone->nr_migrate_reserve_block = reserve; for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { + if (!early_page_nid_uninitialised(pfn, zone_to_nid(zone))) + return; + if (!pfn_valid(pfn)) continue; page = pfn_to_page(pfn); diff --git a/mm/vmscan.c b/mm/vmscan.c index e61445dce04e..f4a487110764 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3386,7 +3386,7 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx) * If there are applications that are active memory-allocators * (most normal use), this basically shouldn't matter. */ -static int kswapd(void *p) +static int __defermem_init kswapd(void *p) { unsigned long order, new_order; unsigned balanced_order; @@ -3421,6 +3421,8 @@ static int kswapd(void *p) tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD; set_freezable(); + deferred_init_memmap(pgdat->node_id); + order = new_order = 0; balanced_order = 0; classzone_idx = new_classzone_idx = pgdat->nr_zones - 1; @@ -3576,7 +3578,7 @@ static int cpu_callback(struct notifier_block *nfb, unsigned long action, * This kswapd start function will be called by init and node-hot-add. * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added. */ -int kswapd_run(int nid) +int __defermem_init kswapd_run(int nid) { pg_data_t *pgdat = NODE_DATA(nid); int ret = 0; |