diff options
| author | Kirill A. Shutemov <kirill.shutemov@linux.intel.com> | 2012-12-12 13:51:05 -0800 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2012-12-12 17:38:31 -0800 |
| commit | 78ca0e679203bbf74f8febd9725a1c8dd083d073 (patch) | |
| tree | 6b2541d8cb1ee2697eb12a299447e5afc78f830c /mm/huge_memory.c | |
| parent | 80371957f09814d25c38733d2d08de47f59a13c2 (diff) | |
thp: lazy huge zero page allocation
Instead of allocating huge zero page on hugepage_init() we can postpone it
until first huge zero page map. It saves memory if THP is not in use.
cmpxchg() is used to avoid race on huge_zero_pfn initialization.
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@linux.intel.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'mm/huge_memory.c')
| -rw-r--r-- | mm/huge_memory.c | 20 |
1 files changed, 10 insertions, 10 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index e1b6f4e13b91..9539d6654bb9 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -160,22 +160,24 @@ static int start_khugepaged(void) return err; } -static int __init init_huge_zero_page(void) +static int init_huge_zero_pfn(void) { struct page *hpage; + unsigned long pfn; hpage = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, HPAGE_PMD_ORDER); if (!hpage) return -ENOMEM; - - huge_zero_pfn = page_to_pfn(hpage); + pfn = page_to_pfn(hpage); + if (cmpxchg(&huge_zero_pfn, 0, pfn)) + __free_page(hpage); return 0; } static inline bool is_huge_zero_pfn(unsigned long pfn) { - return pfn == huge_zero_pfn; + return huge_zero_pfn && pfn == huge_zero_pfn; } static inline bool is_huge_zero_pmd(pmd_t pmd) @@ -564,10 +566,6 @@ static int __init hugepage_init(void) if (err) return err; - err = init_huge_zero_page(); - if (err) - goto out; - err = khugepaged_slab_init(); if (err) goto out; @@ -590,8 +588,6 @@ static int __init hugepage_init(void) return 0; out: - if (huge_zero_pfn) - __free_page(pfn_to_page(huge_zero_pfn)); hugepage_exit_sysfs(hugepage_kobj); return err; } @@ -735,6 +731,10 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, return VM_FAULT_OOM; if (!(flags & FAULT_FLAG_WRITE)) { pgtable_t pgtable; + if (unlikely(!huge_zero_pfn && init_huge_zero_pfn())) { + count_vm_event(THP_FAULT_FALLBACK); + goto out; + } pgtable = pte_alloc_one(mm, haddr); if (unlikely(!pgtable)) return VM_FAULT_OOM; |