From a9869b837c098732bad84939015c0eb391b23e41 Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Wed, 8 Sep 2010 10:19:37 +0900 Subject: hugetlb: move refcounting in hugepage allocation inside hugetlb_lock Currently alloc_huge_page() raises page refcount outside hugetlb_lock. but it causes race when dequeue_hwpoison_huge_page() runs concurrently with alloc_huge_page(). To avoid it, this patch moves set_page_refcounted() in hugetlb_lock. Signed-off-by: Naoya Horiguchi Signed-off-by: Wu Fengguang Acked-by: Mel Gorman Reviewed-by: Christoph Lameter Signed-off-by: Andi Kleen --- mm/hugetlb.c | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) (limited to 'mm/hugetlb.c') diff --git a/mm/hugetlb.c b/mm/hugetlb.c index deb7bebefe68..636be5d6aadd 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -509,6 +509,7 @@ static struct page *dequeue_huge_page_node(struct hstate *h, int nid) return NULL; page = list_entry(h->hugepage_freelists[nid].next, struct page, lru); list_del(&page->lru); + set_page_refcounted(page); h->free_huge_pages--; h->free_huge_pages_node[nid]--; return page; @@ -868,12 +869,6 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, int nid) spin_lock(&hugetlb_lock); if (page) { - /* - * This page is now managed by the hugetlb allocator and has - * no users -- drop the buddy allocator's reference. - */ - put_page_testzero(page); - VM_BUG_ON(page_count(page)); r_nid = page_to_nid(page); set_compound_page_dtor(page, free_huge_page); /* @@ -936,16 +931,13 @@ retry: spin_unlock(&hugetlb_lock); for (i = 0; i < needed; i++) { page = alloc_buddy_huge_page(h, NUMA_NO_NODE); - if (!page) { + if (!page) /* * We were not able to allocate enough pages to * satisfy the entire reservation so we free what * we've allocated so far. */ - spin_lock(&hugetlb_lock); - needed = 0; goto free; - } list_add(&page->lru, &surplus_list); } @@ -972,31 +964,31 @@ retry: needed += allocated; h->resv_huge_pages += delta; ret = 0; -free: + + spin_unlock(&hugetlb_lock); /* Free the needed pages to the hugetlb pool */ list_for_each_entry_safe(page, tmp, &surplus_list, lru) { if ((--needed) < 0) break; list_del(&page->lru); + /* + * This page is now managed by the hugetlb allocator and has + * no users -- drop the buddy allocator's reference. + */ + put_page_testzero(page); + VM_BUG_ON(page_count(page)); enqueue_huge_page(h, page); } /* Free unnecessary surplus pages to the buddy allocator */ +free: if (!list_empty(&surplus_list)) { - spin_unlock(&hugetlb_lock); list_for_each_entry_safe(page, tmp, &surplus_list, lru) { list_del(&page->lru); - /* - * The page has a reference count of zero already, so - * call free_huge_page directly instead of using - * put_page. This must be done with hugetlb_lock - * unlocked which is safe because free_huge_page takes - * hugetlb_lock before deciding how to free the page. - */ - free_huge_page(page); + put_page(page); } - spin_lock(&hugetlb_lock); } + spin_lock(&hugetlb_lock); return ret; } @@ -1123,7 +1115,6 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma, } } - set_page_refcounted(page); set_page_private(page, (unsigned long) mapping); vma_commit_reservation(h, vma, addr); -- cgit v1.2.3