diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/memory.c | 133 | ||||
| -rw-r--r-- | mm/mmap.c | 22 | ||||
| -rw-r--r-- | mm/mprotect.c | 2 | ||||
| -rw-r--r-- | mm/mremap.c | 2 | ||||
| -rw-r--r-- | mm/shmem.c | 14 | ||||
| -rw-r--r-- | mm/swapfile.c | 4 |
6 files changed, 107 insertions, 70 deletions
diff --git a/mm/memory.c b/mm/memory.c index 1e0561e1f190..3416b6e018d6 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1267,10 +1267,10 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, return addr; } -static unsigned long unmap_page_range(struct mmu_gather *tlb, - struct vm_area_struct *vma, - unsigned long addr, unsigned long end, - struct zap_details *details) +static void unmap_page_range(struct mmu_gather *tlb, + struct vm_area_struct *vma, + unsigned long addr, unsigned long end, + struct zap_details *details) { pgd_t *pgd; unsigned long next; @@ -1290,8 +1290,47 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb, } while (pgd++, addr = next, addr != end); tlb_end_vma(tlb, vma); mem_cgroup_uncharge_end(); +} - return addr; + +static void unmap_single_vma(struct mmu_gather *tlb, + struct vm_area_struct *vma, unsigned long start_addr, + unsigned long end_addr, unsigned long *nr_accounted, + struct zap_details *details) +{ + unsigned long start = max(vma->vm_start, start_addr); + unsigned long end; + + if (start >= vma->vm_end) + return; + end = min(vma->vm_end, end_addr); + if (end <= vma->vm_start) + return; + + if (vma->vm_flags & VM_ACCOUNT) + *nr_accounted += (end - start) >> PAGE_SHIFT; + + if (unlikely(is_pfn_mapping(vma))) + untrack_pfn_vma(vma, 0, 0); + + if (start != end) { + if (unlikely(is_vm_hugetlb_page(vma))) { + /* + * It is undesirable to test vma->vm_file as it + * should be non-null for valid hugetlb area. + * However, vm_file will be NULL in the error + * cleanup path of do_mmap_pgoff. When + * hugetlbfs ->mmap method fails, + * do_mmap_pgoff() nullifies vma->vm_file + * before calling this function to clean up. + * Since no pte has actually been setup, it is + * safe to do nothing in this case. + */ + if (vma->vm_file) + unmap_hugepage_range(vma, start, end, NULL); + } else + unmap_page_range(tlb, vma, start, end, details); + } } /** @@ -1303,8 +1342,6 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb, * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here * @details: details of nonlinear truncation or shared cache invalidation * - * Returns the end address of the unmapping (restart addr if interrupted). - * * Unmap all pages in the vma list. * * Only addresses between `start' and `end' will be unmapped. @@ -1316,55 +1353,18 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb, * ensure that any thus-far unmapped pages are flushed before unmap_vmas() * drops the lock and schedules. */ -unsigned long unmap_vmas(struct mmu_gather *tlb, +void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long start_addr, unsigned long end_addr, unsigned long *nr_accounted, struct zap_details *details) { - unsigned long start = start_addr; struct mm_struct *mm = vma->vm_mm; mmu_notifier_invalidate_range_start(mm, start_addr, end_addr); - for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) { - unsigned long end; - - start = max(vma->vm_start, start_addr); - if (start >= vma->vm_end) - continue; - end = min(vma->vm_end, end_addr); - if (end <= vma->vm_start) - continue; - - if (vma->vm_flags & VM_ACCOUNT) - *nr_accounted += (end - start) >> PAGE_SHIFT; - - if (unlikely(is_pfn_mapping(vma))) - untrack_pfn_vma(vma, 0, 0); - - while (start != end) { - if (unlikely(is_vm_hugetlb_page(vma))) { - /* - * It is undesirable to test vma->vm_file as it - * should be non-null for valid hugetlb area. - * However, vm_file will be NULL in the error - * cleanup path of do_mmap_pgoff. When - * hugetlbfs ->mmap method fails, - * do_mmap_pgoff() nullifies vma->vm_file - * before calling this function to clean up. - * Since no pte has actually been setup, it is - * safe to do nothing in this case. - */ - if (vma->vm_file) - unmap_hugepage_range(vma, start, end, NULL); - - start = end; - } else - start = unmap_page_range(tlb, vma, start, end, details); - } - } - + for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) + unmap_single_vma(tlb, vma, start_addr, end_addr, nr_accounted, + details); mmu_notifier_invalidate_range_end(mm, start_addr, end_addr); - return start; /* which is now the end (or restart) address */ } /** @@ -1373,8 +1373,34 @@ unsigned long unmap_vmas(struct mmu_gather *tlb, * @address: starting address of pages to zap * @size: number of bytes to zap * @details: details of nonlinear truncation or shared cache invalidation + * + * Caller must protect the VMA list + */ +void zap_page_range(struct vm_area_struct *vma, unsigned long address, + unsigned long size, struct zap_details *details) +{ + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather tlb; + unsigned long end = address + size; + unsigned long nr_accounted = 0; + + lru_add_drain(); + tlb_gather_mmu(&tlb, mm, 0); + update_hiwater_rss(mm); + unmap_vmas(&tlb, vma, address, end, &nr_accounted, details); + tlb_finish_mmu(&tlb, address, end); +} + +/** + * zap_page_range_single - remove user pages in a given range + * @vma: vm_area_struct holding the applicable pages + * @address: starting address of pages to zap + * @size: number of bytes to zap + * @details: details of nonlinear truncation or shared cache invalidation + * + * The range must fit into one VMA. */ -unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address, +static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, unsigned long size, struct zap_details *details) { struct mm_struct *mm = vma->vm_mm; @@ -1385,9 +1411,10 @@ unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address, lru_add_drain(); tlb_gather_mmu(&tlb, mm, 0); update_hiwater_rss(mm); - end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details); + mmu_notifier_invalidate_range_start(mm, address, end); + unmap_single_vma(&tlb, vma, address, end, &nr_accounted, details); + mmu_notifier_invalidate_range_end(mm, address, end); tlb_finish_mmu(&tlb, address, end); - return end; } /** @@ -1408,7 +1435,7 @@ int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, if (address < vma->vm_start || address + size > vma->vm_end || !(vma->vm_flags & VM_PFNMAP)) return -1; - zap_page_range(vma, address, size, NULL); + zap_page_range_single(vma, address, size, NULL); return 0; } EXPORT_SYMBOL_GPL(zap_vma_ptes); @@ -2755,7 +2782,7 @@ static void unmap_mapping_range_vma(struct vm_area_struct *vma, unsigned long start_addr, unsigned long end_addr, struct zap_details *details) { - zap_page_range(vma, start_addr, end_addr - start_addr, details); + zap_page_range_single(vma, start_addr, end_addr - start_addr, details); } static inline void unmap_mapping_range_tree(struct prio_tree_root *root, diff --git a/mm/mmap.c b/mm/mmap.c index 230f0bac06b6..a7bf6a31c9f6 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -935,6 +935,19 @@ void vm_stat_account(struct mm_struct *mm, unsigned long flags, #endif /* CONFIG_PROC_FS */ /* + * If a hint addr is less than mmap_min_addr change hint to be as + * low as possible but still greater than mmap_min_addr + */ +static inline unsigned long round_hint_to_min(unsigned long hint) +{ + hint &= PAGE_MASK; + if (((void *)hint != NULL) && + (hint < mmap_min_addr)) + return PAGE_ALIGN(mmap_min_addr); + return hint; +} + +/* * The caller must hold down_write(¤t->mm->mmap_sem). */ @@ -1234,7 +1247,7 @@ munmap_back: */ if (accountable_mapping(file, vm_flags)) { charged = len >> PAGE_SHIFT; - if (security_vm_enough_memory(charged)) + if (security_vm_enough_memory_mm(mm, charged)) return -ENOMEM; vm_flags |= VM_ACCOUNT; } @@ -2183,7 +2196,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len) if (mm->map_count > sysctl_max_map_count) return -ENOMEM; - if (security_vm_enough_memory(len >> PAGE_SHIFT)) + if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) return -ENOMEM; /* Can we just expand an old private anonymous mapping? */ @@ -2227,7 +2240,6 @@ void exit_mmap(struct mm_struct *mm) struct mmu_gather tlb; struct vm_area_struct *vma; unsigned long nr_accounted = 0; - unsigned long end; /* mm's last user has gone, and its about to be pulled down */ mmu_notifier_release(mm); @@ -2252,11 +2264,11 @@ void exit_mmap(struct mm_struct *mm) tlb_gather_mmu(&tlb, mm, 1); /* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use -1 here to ensure all VMAs in the mm are unmapped */ - end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL); + unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL); vm_unacct_memory(nr_accounted); free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, 0); - tlb_finish_mmu(&tlb, 0, end); + tlb_finish_mmu(&tlb, 0, -1); /* * Walk the list again, actually closing and freeing it, diff --git a/mm/mprotect.c b/mm/mprotect.c index c621e999cbf7..a40992610ab6 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -168,7 +168,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| VM_SHARED|VM_NORESERVE))) { charged = nrpages; - if (security_vm_enough_memory(charged)) + if (security_vm_enough_memory_mm(mm, charged)) return -ENOMEM; newflags |= VM_ACCOUNT; } diff --git a/mm/mremap.c b/mm/mremap.c index 87bb8393e7d2..db8d983b5a7d 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -329,7 +329,7 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, if (vma->vm_flags & VM_ACCOUNT) { unsigned long charged = (new_len - old_len) >> PAGE_SHIFT; - if (security_vm_enough_memory(charged)) + if (security_vm_enough_memory_mm(mm, charged)) goto Efault; *p = charged; } diff --git a/mm/shmem.c b/mm/shmem.c index 7cc80833b74a..f99ff3e50bd6 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -127,7 +127,7 @@ static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) static inline int shmem_acct_size(unsigned long flags, loff_t size) { return (flags & VM_NORESERVE) ? - 0 : security_vm_enough_memory_kern(VM_ACCT(size)); + 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); } static inline void shmem_unacct_size(unsigned long flags, loff_t size) @@ -145,7 +145,7 @@ static inline void shmem_unacct_size(unsigned long flags, loff_t size) static inline int shmem_acct_block(unsigned long flags) { return (flags & VM_NORESERVE) ? - security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0; + security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_CACHE_SIZE)) : 0; } static inline void shmem_unacct_blocks(unsigned long flags, long pages) @@ -2231,7 +2231,6 @@ static void shmem_put_super(struct super_block *sb) int shmem_fill_super(struct super_block *sb, void *data, int silent) { struct inode *inode; - struct dentry *root; struct shmem_sb_info *sbinfo; int err = -ENOMEM; @@ -2288,14 +2287,11 @@ int shmem_fill_super(struct super_block *sb, void *data, int silent) goto failed; inode->i_uid = sbinfo->uid; inode->i_gid = sbinfo->gid; - root = d_alloc_root(inode); - if (!root) - goto failed_iput; - sb->s_root = root; + sb->s_root = d_make_root(inode); + if (!sb->s_root) + goto failed; return 0; -failed_iput: - iput(inode); failed: shmem_put_super(sb); return err; diff --git a/mm/swapfile.c b/mm/swapfile.c index 21b56945c5d2..dae42f380d6e 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1561,6 +1561,8 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) if (!capable(CAP_SYS_ADMIN)) return -EPERM; + BUG_ON(!current->mm); + pathname = getname(specialfile); err = PTR_ERR(pathname); if (IS_ERR(pathname)) @@ -1588,7 +1590,7 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) spin_unlock(&swap_lock); goto out_dput; } - if (!security_vm_enough_memory(p->pages)) + if (!security_vm_enough_memory_mm(current->mm, p->pages)) vm_unacct_memory(p->pages); else { err = -ENOMEM; |