diff options
Diffstat (limited to 'arch/x86/mm/pat/set_memory.c')
| -rw-r--r-- | arch/x86/mm/pat/set_memory.c | 53 |
1 files changed, 41 insertions, 12 deletions
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index c38faf39ce15..4652487b5572 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -22,6 +22,7 @@ #include <linux/cc_platform.h> #include <linux/set_memory.h> #include <linux/memregion.h> +#include <linux/cleanup.h> #include <asm/e820/api.h> #include <asm/processor.h> @@ -49,7 +50,8 @@ struct cpa_data { unsigned int flags; unsigned int force_split : 1, force_static_prot : 1, - force_flush_all : 1; + force_flush_all : 1, + init_mm_read_locked : 1; struct page **pages; }; @@ -409,7 +411,7 @@ static void __cpa_flush_tlb(void *data) static int collapse_large_pages(unsigned long addr, struct list_head *pgtables); -static void cpa_collapse_large_pages(struct cpa_data *cpa) +static void __cpa_collapse_large_pages(struct cpa_data *cpa) { unsigned long start, addr, end; struct ptdesc *ptdesc, *tmp; @@ -439,10 +441,30 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) { list_del(&ptdesc->pt_list); - pagetable_free(ptdesc); + /* + * Only early alloc'd direct map should not be flagged PG_table + * here and those shouldn't be collapsed. However be abundantly + * cautious and handle the !PG_table case too. + */ + if (PageTable((ptdesc_page(ptdesc)))) + pagetable_dtor_free(ptdesc); + else + pagetable_free(ptdesc); } } +static void cpa_collapse_large_pages(struct cpa_data *cpa) +{ + /* + * Take the mmap write lock on init_mm to: + * - Avoid a use-after-free if raced by ptdump (which takes its own + * write lock on init_mm). + * - Serialise concurrent CPA walkers. + */ + scoped_guard(mmap_write_lock, &init_mm) + __cpa_collapse_large_pages(cpa); +} + static void cpa_flush(struct cpa_data *cpa, int cache) { unsigned int i; @@ -1120,11 +1142,10 @@ set: static int __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, - struct ptdesc *ptdesc) + pte_t *pbase) { unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1; - struct page *base = ptdesc_page(ptdesc); - pte_t *pbase = (pte_t *)page_address(base); + struct page *base = virt_to_page(pbase); unsigned int i, level; pgprot_t ref_prot; bool nx, rw; @@ -1224,16 +1245,20 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, static int split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address) { - struct ptdesc *ptdesc; + pte_t *pte; spin_unlock(&cpa_lock); - ptdesc = pagetable_alloc(GFP_KERNEL, 0); + if (cpa->init_mm_read_locked) + mmap_read_unlock(&init_mm); + pte = pte_alloc_one_kernel(&init_mm); + if (cpa->init_mm_read_locked) + mmap_read_lock(&init_mm); spin_lock(&cpa_lock); - if (!ptdesc) + if (!pte) return -ENOMEM; - if (__split_large_page(cpa, kpte, address, ptdesc)) - pagetable_free(ptdesc); + if (__split_large_page(cpa, kpte, address, pte)) + pte_free_kernel(&init_mm, pte); return 0; } @@ -2121,7 +2146,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, cpa.curpage = 0; cpa.force_split = force_split; - ret = __change_page_attr_set_clr(&cpa, 1); + /* Avoid race with concurrent CPA collapse. */ + cpa.init_mm_read_locked = true; + scoped_guard(mmap_read_lock, &init_mm) + ret = __change_page_attr_set_clr(&cpa, 1); + cpa.init_mm_read_locked = false; /* * Check whether we really changed something: |