diff options
Diffstat (limited to 'mm')
-rw-r--r-- | mm/internal.h | 3 | ||||
-rw-r--r-- | mm/swap.c | 33 |
2 files changed, 29 insertions, 7 deletions
diff --git a/mm/internal.h b/mm/internal.h index 684f7aa9692a..a85a3ab1f7ef 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -51,7 +51,8 @@ static inline void __get_page_tail_foll(struct page *page, VM_BUG_ON(page_mapcount(page) < 0); if (get_page_head) atomic_inc(&page->first_page->_count); - atomic_inc(&page->_mapcount); + if (compound_tail_refcounted(page->first_page)) + atomic_inc(&page->_mapcount); } /* diff --git a/mm/swap.c b/mm/swap.c index e2757fbb04ea..bba4aa5bf686 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -88,8 +88,9 @@ static void put_compound_page(struct page *page) /* * THP can not break up slab pages so avoid taking - * compound_lock(). Slab performs non-atomic bit ops - * on page->flags for better performance. In + * compound_lock() and skip the tail page refcounting + * (in _mapcount) too. Slab performs non-atomic bit + * ops on page->flags for better performance. In * particular slab_unlock() in slub used to be a hot * path. It is still hot on arches that do not support * this_cpu_cmpxchg_double(). @@ -102,7 +103,7 @@ static void put_compound_page(struct page *page) * PageTail clear after smp_rmb() and we'll treat it * as a single page. */ - if (PageSlab(page_head) || PageHeadHuge(page_head)) { + if (!__compound_tail_refcounted(page_head)) { /* * If "page" is a THP tail, we must read the tail page * flags after the head page flags. The @@ -117,10 +118,30 @@ static void put_compound_page(struct page *page) * cannot race here. */ VM_BUG_ON(!PageHead(page_head)); - VM_BUG_ON(page_mapcount(page) <= 0); - atomic_dec(&page->_mapcount); - if (put_page_testzero(page_head)) + VM_BUG_ON(page_mapcount(page) != 0); + if (put_page_testzero(page_head)) { + /* + * If this is the tail of a + * slab compound page, the + * tail pin must not be the + * last reference held on the + * page, because the PG_slab + * cannot be cleared before + * all tail pins (which skips + * the _mapcount tail + * refcounting) have been + * released. For hugetlbfs the + * tail pin may be the last + * reference on the page + * instead, because + * PageHeadHuge will not go + * away until the compound + * page enters the buddy + * allocator. + */ + VM_BUG_ON(PageSlab(page_head)); __put_compound_page(page_head); + } return; } else /* |