diff options
| -rw-r--r-- | fs/hugetlbfs/inode.c | 8 | ||||
| -rw-r--r-- | include/asm-generic/tlb.h | 3 | ||||
| -rw-r--r-- | include/linux/huge_mm.h | 5 | ||||
| -rw-r--r-- | include/linux/ksm.h | 6 | ||||
| -rw-r--r-- | include/linux/memfd.h | 9 | ||||
| -rw-r--r-- | include/linux/mmu_notifier.h | 3 | ||||
| -rw-r--r-- | include/linux/shmem_fs.h | 15 | ||||
| -rw-r--r-- | include/linux/userfaultfd_k.h | 2 | ||||
| -rw-r--r-- | kernel/fork.c | 20 | ||||
| -rw-r--r-- | mm/cma.c | 10 | ||||
| -rw-r--r-- | mm/filemap.c | 84 | ||||
| -rw-r--r-- | mm/gup.c | 68 | ||||
| -rw-r--r-- | mm/huge_memory.c | 146 | ||||
| -rw-r--r-- | mm/hugetlb.c | 14 | ||||
| -rw-r--r-- | mm/internal.h | 28 | ||||
| -rw-r--r-- | mm/khugepaged.c | 65 | ||||
| -rw-r--r-- | mm/ksm.c | 56 | ||||
| -rw-r--r-- | mm/madvise.c | 9 | ||||
| -rw-r--r-- | mm/memfd.c | 2 | ||||
| -rw-r--r-- | mm/memory-tiers.c | 29 | ||||
| -rw-r--r-- | mm/memory.c | 30 | ||||
| -rw-r--r-- | mm/mempolicy.c | 4 | ||||
| -rw-r--r-- | mm/mmap.c | 8 | ||||
| -rw-r--r-- | mm/mmu_notifier.c | 24 | ||||
| -rw-r--r-- | mm/mremap.c | 103 | ||||
| -rw-r--r-- | mm/nommu.c | 2 | ||||
| -rw-r--r-- | mm/rmap.c | 23 | ||||
| -rw-r--r-- | mm/shmem.c | 173 | ||||
| -rw-r--r-- | mm/shmem_quota.c | 28 | ||||
| -rw-r--r-- | mm/swapfile.c | 3 | ||||
| -rw-r--r-- | mm/truncate.c | 13 | ||||
| -rw-r--r-- | mm/userfaultfd.c | 75 | ||||
| -rw-r--r-- | mm/util.c | 13 | ||||
| -rw-r--r-- | mm/vma.c | 24 | ||||
| -rw-r--r-- | mm/vmscan.c | 4 |
35 files changed, 717 insertions, 392 deletions
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 01c4d6f44ebe..02e0a65d596d 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -28,6 +28,7 @@ #include <linux/folio_batch.h> #include <linux/fs_parser.h> #include <linux/mman.h> +#include <linux/memfd.h> #include <linux/slab.h> #include <linux/dnotify.h> #include <linux/statfs.h> @@ -840,6 +841,13 @@ static int hugetlbfs_setattr(const struct mnt_idmap *idmap, if (error) return error; + if (S_ISREG(inode->i_mode) && (ia_valid & ATTR_MODE)) { + error = memfd_check_seals_mode(info->seals, inode->i_mode, + attr->ia_mode); + if (error) + return error; + } + trace_hugetlbfs_setattr(inode, dentry, attr); if (ia_valid & ATTR_SIZE) { diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index fbe1114e9a44..3f4a180b48f3 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -350,8 +350,7 @@ struct mmu_gather { void tlb_flush_mmu(struct mmu_gather *tlb); static inline void __tlb_adjust_range(struct mmu_gather *tlb, - unsigned long address, - unsigned int range_size) + unsigned long address, unsigned long range_size) { tlb->start = min(tlb->start, address); tlb->end = max(tlb->end, address + range_size); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index c745f7ad2298..fac05120ab77 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -12,9 +12,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); bool huge_pmd_set_accessed(struct vm_fault *vmf); -int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, - pud_t *dst_pud, pud_t *src_pud, unsigned long addr, - struct vm_area_struct *vma); +int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + pud_t *dst_pud, pud_t *src_pud, unsigned long addr); #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud); diff --git a/include/linux/ksm.h b/include/linux/ksm.h index d39d0d5483a2..a0d358a1feb6 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -91,8 +91,8 @@ static inline void ksm_exit(struct mm_struct *mm) * We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE, * but what if the vma was unmerged while the page was swapped out? */ -struct folio *ksm_might_need_to_copy(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr); +struct folio *ksm_might_need_to_copy(struct folio *folio, struct page *page, + struct vm_area_struct *vma, unsigned long addr); void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc); void folio_migrate_ksm(struct folio *newfolio, struct folio *folio); @@ -145,7 +145,7 @@ static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start, } static inline struct folio *ksm_might_need_to_copy(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr) + struct page *page, struct vm_area_struct *vma, unsigned long addr) { return folio; } diff --git a/include/linux/memfd.h b/include/linux/memfd.h index c159e40e3f34..272479317cf1 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -2,10 +2,19 @@ #ifndef __LINUX_MEMFD_H #define __LINUX_MEMFD_H +#include <linux/fcntl.h> #include <linux/file.h> #define MEMFD_ANON_NAME "[memfd]" +static inline int memfd_check_seals_mode(unsigned int seals, umode_t old_mode, + umode_t new_mode) +{ + if ((seals & F_SEAL_EXEC) && ((old_mode ^ new_mode) & 0111)) + return -EPERM; + return 0; +} + #ifdef CONFIG_MEMFD_CREATE extern long memfd_fcntl(struct file *file, unsigned int cmd, unsigned int arg); struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx); diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index a11a44eef521..5db78e5761c5 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -483,7 +483,8 @@ mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range) /* * This version of mmu_notifier_invalidate_range_start() avoids blocking, but it * can return an error if a notifier can't proceed without blocking, in which - * case you're not allowed to modify PTEs in the specified range. + * case you're not allowed to modify PTEs in the specified range or call + * mmu_notifier_invalidate_range_end(). * * This is mainly intended for OOM handling. */ diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index a7c7a96a7cbf..f89b4b151133 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -117,8 +117,14 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts); #ifdef CONFIG_SHMEM +bool shmem_file(struct file *file); bool shmem_mapping(const struct address_space *mapping); #else +static inline bool shmem_file(struct file *file) +{ + return false; +} + static inline bool shmem_mapping(const struct address_space *mapping) { return false; @@ -193,15 +199,6 @@ static inline struct page *shmem_read_mapping_page( mapping_gfp_mask(mapping)); } -static inline bool shmem_file(struct file *file) -{ - if (!IS_ENABLED(CONFIG_SHMEM)) - return false; - if (!file || !file->f_mapping) - return false; - return shmem_mapping(file->f_mapping); -} - /* Must be called with inode lock taken exclusive. */ static inline void shmem_freeze(struct inode *inode, bool freeze) { diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index ebd86677b052..dff9a5c9a0df 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -153,8 +153,6 @@ extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, unsigned long len, bool enable_rwp); /* move_pages */ -void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); -void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2); int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, diff --git a/kernel/fork.c b/kernel/fork.c index 8b108ed37152..5efceb8d58a7 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1533,12 +1533,19 @@ static struct mm_struct *dup_mm(struct task_struct *tsk, if (!mm) goto fail_nomem; + uprobe_start_dup_mmap(); + if (mmap_write_lock_killable(oldmm)) { + free_mm(mm); + goto fail_uprobe; + } + memcpy(mm, oldmm, sizeof(*mm)); - if (!mm_init(mm, tsk)) - goto fail_nomem; + if (!mm_init(mm, tsk)) { + mmap_write_unlock(oldmm); + goto fail_uprobe; + } - uprobe_start_dup_mmap(); err = dup_mmap(mm, oldmm); if (err) goto free_pt; @@ -1557,9 +1564,10 @@ free_pt: mm->binfmt = NULL; mm_init_owner(mm, NULL); mmput(mm); - if (err) - uprobe_end_dup_mmap(); - + if (!err) + return NULL; +fail_uprobe: + uprobe_end_dup_mmap(); fail_nomem: return NULL; } @@ -1024,7 +1024,7 @@ bool cma_release(struct cma *cma, const struct page *pages, unsigned long count) { struct cma_memrange *cmr; - unsigned long ret = 0; + unsigned long leaked = 0; unsigned long i, pfn; cmr = find_cma_memrange(cma, pages, count); @@ -1033,9 +1033,13 @@ bool cma_release(struct cma *cma, const struct page *pages, pfn = page_to_pfn(pages); for (i = 0; i < count; i++, pfn++) - ret += !put_page_testzero(pfn_to_page(pfn)); + leaked += !page_ref_freeze(pfn_to_page(pfn), 1); - WARN(ret, "%lu pages are still in use!\n", ret); + if (leaked) { + pr_warn_ratelimited("%lu pages are still in use, not freeing CMA region!\n", + leaked); + return true; + } __cma_release_frozen(cma, cmr, pages, count); diff --git a/mm/filemap.c b/mm/filemap.c index b74bc1e5015c..b04be64a5069 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -227,18 +227,6 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) page_cache_delete(mapping, folio, shadow); } -static void filemap_free_folio(const struct address_space *mapping, - struct folio *folio) -{ - void (*free_folio)(struct folio *); - - free_folio = mapping->a_ops->free_folio; - if (free_folio) - free_folio(folio); - - folio_put_refs(folio, folio_nr_pages(folio)); -} - /** * filemap_remove_folio - Remove folio from page cache. * @folio: The folio. @@ -250,6 +238,7 @@ static void filemap_free_folio(const struct address_space *mapping, void filemap_remove_folio(struct folio *folio) { struct address_space *mapping = folio->mapping; + void (*free_folio)(struct folio *folio); BUG_ON(!folio_test_locked(folio)); spin_lock(&mapping->host->i_lock); @@ -258,9 +247,12 @@ void filemap_remove_folio(struct folio *folio) xa_unlock_irq(&mapping->i_pages); if (mapping_shrinkable(mapping)) inode_lru_list_add(mapping->host); + free_folio = mapping->a_ops->free_folio; spin_unlock(&mapping->host->i_lock); - filemap_free_folio(mapping, folio); + if (free_folio) + free_folio(folio); + folio_put_refs(folio, folio_nr_pages(folio)); } /* @@ -320,6 +312,7 @@ static void page_cache_delete_batch(struct address_space *mapping, void delete_from_page_cache_batch(struct address_space *mapping, struct folio_batch *fbatch) { + void (*free_folio)(struct folio *folio); int i; if (!folio_batch_count(fbatch)) @@ -337,10 +330,16 @@ void delete_from_page_cache_batch(struct address_space *mapping, xa_unlock_irq(&mapping->i_pages); if (mapping_shrinkable(mapping)) inode_lru_list_add(mapping->host); + free_folio = mapping->a_ops->free_folio; spin_unlock(&mapping->host->i_lock); - for (i = 0; i < folio_batch_count(fbatch); i++) - filemap_free_folio(mapping, fbatch->folios[i]); + for (i = 0; i < folio_batch_count(fbatch); i++) { + struct folio *folio = fbatch->folios[i]; + + if (free_folio) + free_folio(folio); + folio_put_refs(folio, folio_nr_pages(folio)); + } } int filemap_check_errors(struct address_space *mapping) @@ -1374,10 +1373,22 @@ repeat: * return value based on that state without races. * * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive - * waiter, but an exclusive one requires WQ_FLAG_DONE. + * waiter, but an exclusive one must also acquire the bit. */ - if (behavior == EXCLUSIVE) - return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; + if (behavior == EXCLUSIVE) { + if (wait->flags & WQ_FLAG_DONE) + return 0; + /* + * We might have just been woken up and the waker will not + * wake up anybody else. If we ignore that now, the remaining + * waiters may not get woken up: take the bit now if it is + * still free. + */ + if ((wait->flags & WQ_FLAG_WOKEN) && + !test_and_set_bit(bit_nr, folio_flags(folio, 0))) + return 0; + return -EINTR; + } return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; } @@ -3297,11 +3308,11 @@ unlock: * @folio - the folio to lock. * @fpin - the pointer to the file we may pin (or is already pinned). * - * This works similar to lock_folio_or_retry in that it can drop the - * mmap_lock. It differs in that it actually returns the folio locked - * if it returns 1 and 0 if it couldn't lock the folio. If we did have - * to drop the mmap_lock then fpin will point to the pinned file and - * needs to be fput()'ed at a later point. + * Returns 1 with the folio locked, or 0 if locking failed. + * + * On both entry and return, a non-NULL *fpin means the fault lock has + * already been dropped and the caller must eventually fput() the file. + * Otherwise the fault lock is still held, including on failure. */ static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, struct file **fpin) @@ -3318,21 +3329,9 @@ static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, return 0; *fpin = maybe_unlock_mmap_for_io(vmf, *fpin); - if (vmf->flags & FAULT_FLAG_KILLABLE) { - if (__folio_lock_killable(folio)) { - /* - * We didn't have the right flags to drop the - * fault lock, but all fault_handlers only check - * for fatal signals if we return VM_FAULT_RETRY, - * so we need to drop the fault lock here and - * return 0 if we don't have a fpin. - */ - if (*fpin == NULL) - release_fault_lock(vmf); - return 0; - } - } else - __folio_lock(folio); + if (vmf->flags & FAULT_FLAG_KILLABLE) + return !__folio_lock_killable(folio); + __folio_lock(folio); return 1; } @@ -3561,7 +3560,7 @@ static vm_fault_t filemap_fault_recheck_pte_none(struct vm_fault *vmf) * vma->vm_mm->mmap_lock must be held on entry. * * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock - * may be dropped before doing I/O or by lock_folio_maybe_drop_mmap(). + * may be dropped before doing I/O or when aborting an interrupted wait. * * If our return value does not have VM_FAULT_RETRY set, the mmap_lock * has not been released. @@ -3717,9 +3716,8 @@ page_not_uptodate: out_retry: /* - * We dropped the mmap_lock, we need to return to the fault handler to - * re-find the vma and come back and find our hopefully still populated - * page. + * With fpin set, the fault lock was already dropped. Otherwise it is + * still held. Release invalidate_lock before dropping either protection. */ if (!IS_ERR(folio)) folio_put(folio); @@ -3727,6 +3725,8 @@ out_retry: filemap_invalidate_unlock_shared(mapping); if (fpin) fput(fpin); + else if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + release_fault_lock(vmf); return ret | VM_FAULT_RETRY; } EXPORT_SYMBOL(filemap_fault); @@ -658,9 +658,6 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma, assert_spin_locked(pud_lockptr(mm, pudp)); - if (!pud_present(pud)) - return NULL; - if ((flags & FOLL_WRITE) && !can_follow_write_pud(pud, pfn_to_page(pfn), vma, flags)) return NULL; @@ -896,15 +893,17 @@ no_page: } static struct page *follow_pmd_mask(struct vm_area_struct *vma, - unsigned long address, pud_t *pudp, - unsigned int flags, - unsigned long *page_mask) + unsigned long address, pud_t *pudp, pud_t pud, + unsigned int flags, unsigned long *page_mask) { pmd_t *pmd, pmdval; spinlock_t *ptl; struct page *page; struct mm_struct *mm = vma->vm_mm; + if (unlikely(pud_bad(pud))) + return no_page_table(vma, flags, address); + pmd = pmd_offset(pudp, address); pmdval = pmdp_get_lockless(pmd); if (pmd_none(pmdval)) @@ -953,18 +952,23 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, pud = pudp_get(pudp); if (!pud_present(pud)) return no_page_table(vma, flags, address); - if (pud_leaf(pud)) { - ptl = pud_lock(mm, pudp); - page = follow_huge_pud(vma, address, pudp, flags, page_mask); + if (!pud_leaf(pud)) + return follow_pmd_mask(vma, address, pudp, pud, flags, page_mask); + + ptl = pud_lock(mm, pudp); + pud = *pudp; + if (unlikely(!pud_present(pud))) { spin_unlock(ptl); - if (page) - return page; return no_page_table(vma, flags, address); } - if (unlikely(pud_bad(pud))) - return no_page_table(vma, flags, address); + if (unlikely(!pud_leaf(pud))) { + spin_unlock(ptl); + return follow_pmd_mask(vma, address, pudp, pud, flags, page_mask); + } - return follow_pmd_mask(vma, address, pudp, flags, page_mask); + page = follow_huge_pud(vma, address, pudp, flags, page_mask); + spin_unlock(ptl); + return page; } static struct page *follow_p4d_mask(struct vm_area_struct *vma, @@ -3211,6 +3215,9 @@ static int gup_fast_fallback(unsigned long start, unsigned long nr_pages, FOLL_PCI_P2PDMA | FOLL_HONOR_NUMA_FAULT))) return -EINVAL; + if (unlikely(!nr_pages)) + return 0; + if (gup_flags & FOLL_PIN) mm_set_has_pinned_flag(current->mm); @@ -3218,8 +3225,8 @@ static int gup_fast_fallback(unsigned long start, unsigned long nr_pages, might_lock_read(¤t->mm->mmap_lock); start = untagged_addr(start) & PAGE_MASK; - len = nr_pages << PAGE_SHIFT; - if (check_add_overflow(start, len, &end)) + if (check_shl_overflow(nr_pages, PAGE_SHIFT, &len) || + check_add_overflow(start, len, &end)) return -EOVERFLOW; if (end > TASK_SIZE_MAX) return -EFAULT; @@ -3464,7 +3471,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, { unsigned int flags, nr_folios, nr_found; unsigned int i, pgshift = PAGE_SHIFT; - pgoff_t start_idx, end_idx; + pgoff_t start_idx, end_idx, retry_idx; struct folio *folio = NULL; struct folio_batch fbatch; struct hstate *h; @@ -3505,6 +3512,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, * reason, we try to allocate them and add them to the * page cache. */ + retry_idx = start_idx; nr_found = filemap_get_folios_contig(memfd->f_mapping, &start_idx, end_idx, @@ -3517,12 +3525,38 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, for (i = 0; i < nr_found; i++) { folio = fbatch.folios[i]; + /* + * The folio might have been removed from the + * page cache (e.g., truncated or collapsed) + * after the lookup. Recheck under the folio + * lock. + */ + folio_lock(folio); + if (unlikely(folio->mapping != memfd->f_mapping)) { + folio_unlock(folio); + start_idx = retry_idx; + break; + } + if (try_grab_folio(folio, 1, FOLL_PIN)) { + folio_unlock(folio); folio_batch_release(&fbatch); ret = -EINVAL; goto err; } + /* + * fallocate() can leave folios in the + * page cache that were never cleared. + */ + if (!folio_test_uptodate(folio)) { + folio_zero_range(folio, 0, folio_size(folio)); + flush_dcache_folio(folio); + folio_mark_uptodate(folio); + } + retry_idx = folio_next_index(folio); + folio_unlock(folio); + if (nr_folios == 0) *offset = offset_in_folio(folio, start); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index bd44d58e8684..eb913acb03b7 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2072,11 +2072,13 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr, update_mmu_cache_pud(vma, addr, pud); } -int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, - pud_t *dst_pud, pud_t *src_pud, unsigned long addr, - struct vm_area_struct *vma) +int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + pud_t *dst_pud, pud_t *src_pud, unsigned long addr) { + struct mm_struct *dst_mm = dst_vma->vm_mm; + struct mm_struct *src_mm = src_vma->vm_mm; spinlock_t *dst_ptl, *src_ptl; + struct page *page; pud_t pud; int ret; @@ -2089,11 +2091,20 @@ int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (unlikely(!pud_trans_huge(pud))) goto out_unlock; - /* - * TODO: once we support anonymous pages, use - * folio_try_dup_anon_rmap_*() and split if duplicating fails. - */ - if (vma_is_cow_mapping(vma) && pud_write(pud)) { + page = vm_normal_page_pud(src_vma, addr, pud); + if (page) { + struct folio *folio = page_folio(page); + + folio_get(folio); + /* + * TODO: once we support anonymous pages, use + * folio_try_dup_anon_rmap_*() and split if duplicating fails. + */ + folio_add_file_rmap_pud(folio, page, dst_vma); + add_mm_counter(dst_mm, mm_counter_file(folio), HPAGE_PUD_NR); + } + + if (vma_is_cow_mapping(src_vma) && pud_write(pud)) { pudp_set_wrprotect(src_mm, addr, src_pud); pud = pud_wrprotect(pud); } @@ -2482,11 +2493,7 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) static void zap_huge_pmd_folio(struct mm_struct *mm, struct vm_area_struct *vma, pmd_t pmdval, struct folio *folio, bool is_present) { - const bool is_device_private = folio_is_device_private(folio); - - /* Present and device private folios are rmappable. */ - if (is_present || is_device_private) - folio_remove_rmap_pmd(folio, &folio->page, vma); + const bool is_device_private = pmd_is_device_private_entry(pmdval); if (folio_test_anon(folio)) { add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); @@ -2501,6 +2508,13 @@ static void zap_huge_pmd_folio(struct mm_struct *mm, struct vm_area_struct *vma, folio_mark_accessed(folio); } + /* + * Present and device private folios are rmappable. Remove the + * rmap after transferring young+dirty to the folio. + */ + if (is_present || is_device_private) + folio_remove_rmap_pmd(folio, &folio->page, vma); + /* Device private folios are pinned. */ if (is_device_private) folio_put(folio); @@ -2632,15 +2646,11 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, return false; } - /* - * We don't have to worry about the ordering of src and dst - * ptlocks because exclusive mmap_lock prevents deadlock. - */ - old_ptl = __pmd_trans_huge_lock(old_pmd, vma); - if (old_ptl) { - new_ptl = pmd_lockptr(mm, new_pmd); - if (new_ptl != old_ptl) - spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); + old_ptl = pmd_lockptr(mm, old_pmd); + new_ptl = pmd_lockptr(mm, new_pmd); + + double_pt_lock(old_ptl, new_ptl); + if (likely(pmd_is_huge(*old_pmd))) { pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); if (pmd_present(pmd)) force_flush = true; @@ -2669,11 +2679,10 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, set_pmd_at(mm, new_addr, new_pmd, pmd); if (force_flush) flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); - if (new_ptl != old_ptl) - spin_unlock(new_ptl); - spin_unlock(old_ptl); + double_pt_unlock(old_ptl, new_ptl); return true; } + double_pt_unlock(old_ptl, new_ptl); return false; } @@ -3040,6 +3049,8 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, spinlock_t *ptl; pud_t orig_pud; + tlb_change_page_size(tlb, HPAGE_PUD_SIZE); + ptl = __pud_trans_huge_lock(pud, vma); if (!ptl) return 0; @@ -3495,16 +3506,66 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, __split_huge_pmd(vma, pmd, address, freeze); } -static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) +static bool pud_split_required(const struct vm_area_struct *vma, + unsigned long address) { + if (!IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) + return false; + if (IS_ALIGNED(address, HPAGE_PUD_SIZE)) + return false; /* - * If the new address isn't hpage aligned and it could previously - * contain an hugepage: check if we need to split an huge pmd. + * Only selected VMAs support mapping PUDs for now. Lift the + * restriction once this changes. */ - if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && - range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), - ALIGN(address, HPAGE_PMD_SIZE))) - split_huge_pmd_address(vma, address, false); + if (!vma_is_dax(vma) && !vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT)) + return false; + return range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PUD_SIZE), + ALIGN(address, HPAGE_PUD_SIZE)); +} + +static bool pmd_split_required(const struct vm_area_struct *vma, + unsigned long address) +{ + if (IS_ALIGNED(address, HPAGE_PMD_SIZE)) + return false; + + return range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), + ALIGN(address, HPAGE_PMD_SIZE)); +} + +static void split_huge_leaf(struct vm_area_struct *vma, + unsigned long address) +{ + const bool split_pud = pud_split_required(vma, address); + const bool split_pmd = pmd_split_required(vma, address); + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + /* + * mmap, VMA and rmap write locks protect the walk; the split helpers + * take the page table locks. + */ + if (!split_pud && !split_pmd) + return; + + pgd = pgd_offset(vma->vm_mm, address); + if (!pgd_present(*pgd)) + return; + + p4d = p4d_offset(pgd, address); + if (!p4d_present(*p4d)) + return; + + pud = pud_offset(p4d, address); + if (pud_leaf(*pud)) + __split_huge_pud(vma, pud, address); + if (!split_pmd || !pud_present(*pud)) + return; + + pmd = pmd_offset(pud, address); + __split_huge_pmd(vma, pmd, address, false); } void vma_adjust_trans_huge(struct vm_area_struct *vma, @@ -3513,14 +3574,14 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, struct vm_area_struct *next) { /* Check if we need to split start first. */ - split_huge_pmd_if_needed(vma, start); + split_huge_leaf(vma, start); /* Check if we need to split end next. */ - split_huge_pmd_if_needed(vma, end); + split_huge_leaf(vma, end); /* If we're incrementing next->vm_start, we might need to split it. */ if (next) - split_huge_pmd_if_needed(next, end); + split_huge_leaf(next, end); } /* @@ -4522,8 +4583,7 @@ int split_folio_to_list(struct folio *folio, struct list_head *list) * * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is * zero: because even when the list_lru lock is held, a non-empty - * _deferred_list might be in use on deferred_split_scan()'s unlocked - * on-stack list. + * _deferred_list might be in use on deferred_split_scan()'s on-stack list. * * The list_lru sublist is determined by folio's memcg: it is therefore * important to unqueue deferred split before changing folio memcg. @@ -4665,10 +4725,22 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, deferred_split_isolate, &dispose); list_for_each_entry_safe(folio, next, &dispose, _deferred_list) { + struct mem_cgroup *memcg; + struct list_lru_one *lru; bool did_split = false; bool underused = false; + /* + * Sync with deferred_split_folio(), which can requeue the + * folio once list_empty() is true, and sets PG_partially_mapped. + */ + rcu_read_lock(); + memcg = folio_memcg(folio); + lru = list_lru_lock_irq(&deferred_split_lru, folio_nid(folio), + &memcg); list_del_init(&folio->_deferred_list); + list_lru_unlock_irq(lru); + rcu_read_unlock(); if (!folio_test_partially_mapped(folio)) { /* diff --git a/mm/hugetlb.c b/mm/hugetlb.c index ec05e8c1d1f8..630c96cd1e88 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5157,15 +5157,9 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, spinlock_t *src_ptl, *dst_ptl; pte_t pte; - dst_ptl = huge_pte_lock(h, mm, dst_pte); + dst_ptl = huge_pte_lockptr(h, mm, dst_pte); src_ptl = huge_pte_lockptr(h, mm, src_pte); - - /* - * We don't have to worry about the ordering of src and dst ptlocks - * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock. - */ - if (src_ptl != dst_ptl) - spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); + double_pt_lock(dst_ptl, src_ptl); pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz); @@ -5194,9 +5188,7 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, set_huge_pte_at(mm, new_addr, dst_pte, pte, sz); } - if (src_ptl != dst_ptl) - spin_unlock(src_ptl); - spin_unlock(dst_ptl); + double_pt_unlock(dst_ptl, src_ptl); } unsigned long move_hugetlb_page_tables(struct vm_area_struct *vma, diff --git a/mm/internal.h b/mm/internal.h index 8b1263e81054..e3f92883bf23 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -626,6 +626,34 @@ static inline void force_page_cache_readahead(struct address_space *mapping, force_page_cache_ra(&ractl, nr_to_read); } +/* + * Take two page table locks at the same level in address order such that + * we cannot deadlock. + */ +static inline void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2) + __acquires(ptl1) + __acquires(ptl2) +{ + if (ptl1 > ptl2) + swap(ptl1, ptl2); + spin_lock(ptl1); + if (ptl1 != ptl2) + spin_lock_nested(ptl2, SINGLE_DEPTH_NESTING); + else + __acquire(ptl2); +} + +static inline void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2) + __releases(ptl1) + __releases(ptl2) +{ + spin_unlock(ptl1); + if (ptl1 != ptl2) + spin_unlock(ptl2); + else + __release(ptl2); +} + unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, diff --git a/mm/khugepaged.c b/mm/khugepaged.c index da91762f6358..4bbe319ef86f 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1904,7 +1904,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign result = SCAN_FAIL; start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); - if (!start_pte) /* mmap_lock + page lock should prevent this */ + if (!start_pte) goto drop_folio; /* step 1: check all mapped PTEs are to the right huge page */ @@ -1935,6 +1935,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign } pte_unmap_unlock(start_pte, ptl); + ptl = NULL; mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr, haddr + HPAGE_PMD_SIZE); mmu_notifier_invalidate_range_start(&range); @@ -1952,7 +1953,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign pml = pmd_lock(mm, pmd); start_pte = pte_offset_map_rw_nolock(mm, pmd, haddr, &pgt_pmd, &ptl); - if (!start_pte) /* mmap_lock + page lock should prevent this */ + if (!start_pte) goto abort; if (!pml) spin_lock(ptl); @@ -1994,9 +1995,10 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign /* * Must clear entry, or a racing truncate may re-remove it. * TLB flush can be left until pmdp_collapse_flush() does it. - * PTE dirty? Shmem page is already dirty; file is read-only. */ - clear_ptes(mm, addr, pte, nr_batch_ptes); + ptent = get_and_clear_ptes(mm, addr, pte, nr_batch_ptes); + if (pte_dirty(ptent)) + folio_mark_dirty(folio); folio_remove_rmap_ptes(folio, page, nr_batch_ptes, vma); nr_mapped_ptes += nr_batch_ptes; } @@ -2013,12 +2015,11 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign /* step 4: remove empty page table */ if (!pml) { pml = pmd_lock(mm, pmd); - if (ptl != pml) { + if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); - if (unlikely(!pmd_same(pgt_pmd, pmdp_get_lockless(pmd)))) { - flush_tlb_mm(mm); - goto unlock; - } + if (unlikely(!pmd_same(pgt_pmd, pmdp_get_lockless(pmd)))) { + flush_tlb_mm(mm); + goto unlock; } } pgt_pmd = pmdp_collapse_flush(vma, haddr, pmd); @@ -2230,6 +2231,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, struct file *file, pgoff_t start, struct collapse_control *cc) { struct address_space *mapping = file->f_mapping; + const bool is_shmem = shmem_mapping(mapping); struct page *dst; struct folio *folio, *tmp, *new_folio; unsigned long new_pfn = -1; @@ -2238,7 +2240,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); enum scan_result result = SCAN_SUCCEED; int nr_none = 0; - bool is_shmem = shmem_file(file); + bool do_readahead = false; + bool readahead_done = false; /* * MADV_COLLAPSE ignores shmem huge config, so do not check shmem @@ -2256,6 +2259,10 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, mapping_set_update(&xas, mapping); +retry: + result = SCAN_SUCCEED; + do_readahead = false; + xas_set_order(&xas, start, HPAGE_PMD_ORDER); __folio_set_locked(new_folio); if (is_shmem) __folio_set_swapbacked(new_folio); @@ -2320,17 +2327,9 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, } } else { /* !is_shmem */ if (!folio || xa_is_value(folio)) { - xas_unlock_irq(&xas); - page_cache_sync_readahead(mapping, &file->f_ra, - file, index, - end - index); - /* drain lru cache to help folio_isolate_lru() */ - lru_add_drain(); - folio = filemap_lock_folio(mapping, index); - if (IS_ERR(folio)) { - result = SCAN_FAIL; - goto xa_unlocked; - } + do_readahead = !readahead_done; + result = SCAN_FAIL; + goto xa_locked; } else if (folio_test_dirty(folio)) { /* * This page is dirty because it hasn't @@ -2657,6 +2656,30 @@ rollback: new_folio->mapping = NULL; folio_unlock(new_folio); + + /* Readahead may take invalidate_lock; source folios are unlocked. */ + if (do_readahead) { + for (index = start; index < end;) { + folio = filemap_lock_folio(mapping, index); + if (IS_ERR(folio)) { + page_cache_sync_readahead(mapping, &file->f_ra, + file, index, end - index); + folio = filemap_lock_folio(mapping, index); + if (IS_ERR(folio)) { + result = SCAN_FAIL; + goto out_put; + } + } + index = folio_next_index(folio); + folio_unlock(folio); + folio_put(folio); + } + /* Drain the local LRU batch before retrying isolation. */ + lru_add_drain(); + readahead_done = true; + goto retry; + } +out_put: folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); @@ -476,6 +476,7 @@ static void advisor_stop_scan(void) #ifdef CONFIG_NUMA /* Zeroed when merging across nodes is not allowed */ static unsigned int ksm_merge_across_nodes = 1; +/* Number of allocated stable and unstable tree roots. */ static int ksm_nr_node_ids = 1; #else #define ksm_merge_across_nodes 1U @@ -3146,24 +3147,52 @@ unlock: trace_ksm_exit(mm); } -struct folio *ksm_might_need_to_copy(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr) +static bool folio_anon_rmap_compatible(const struct folio *folio, + const struct page *page, struct vm_area_struct *vma, + unsigned long addr) { - struct page *page = folio_page(folio, 0); - struct anon_vma *anon_vma = folio_anon_vma(folio); - struct folio *new_folio; + const struct anon_vma *anon_vma = folio_anon_vma(folio); + struct anon_vma_chain *avc; - if (folio_test_large(folio)) - return folio; + VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); + VM_WARN_ON_ONCE_FOLIO(page_folio(page) != folio, folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_ksm(folio), folio); + vma_assert_stabilised(vma); + + if (!anon_vma) + return true; + if (page_pgoff(folio, page) != linear_anon_page_index(vma, addr)) + return false; + + /* + * Careful: anon_vma might have been freed, so we cannot safely + * dereference it. Add a fast path before we walk the anon_vma chain. + */ + if (anon_vma == vma->anon_vma || anon_vma == vma->anon_vma->root) + return true; + + list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) { + if (avc->anon_vma == anon_vma) + return true; + } + return false; +} +struct folio *ksm_might_need_to_copy(struct folio *folio, struct page *page, + struct vm_area_struct *vma, unsigned long addr) +{ + struct folio *new_folio; + + /* + * A KSM folio is small, but the slot that it left behind can be swapped + * in as a part of a large folio by one of the processes that shared it. + * So test, and copy, the page of the slot. + */ if (folio_test_ksm(folio)) { if (folio_stable_node(folio) && !(ksm_run & KSM_RUN_UNMERGE)) return folio; /* no need to copy it */ - } else if (!anon_vma) { - return folio; /* no need to copy it */ - } else if (folio->index == linear_anon_page_index(vma, addr) && - anon_vma->root == vma->anon_vma->root) { + } else if (folio_anon_rmap_compatible(folio, page, vma, addr)) { return folio; /* still no need to copy it */ } if (PageHWPoison(page)) @@ -3646,14 +3675,13 @@ static ssize_t merge_across_nodes_store(struct kobject *kobj, else { root_stable_tree = buf; root_unstable_tree = buf + nr_node_ids; + ksm_nr_node_ids = nr_node_ids; /* Stable tree is empty but not the unstable */ root_unstable_tree[0] = one_unstable_tree[0]; } } - if (!err) { + if (!err) ksm_merge_across_nodes = knob; - ksm_nr_node_ids = knob ? 1 : nr_node_ids; - } } mutex_unlock(&ksm_thread_mutex); diff --git a/mm/madvise.c b/mm/madvise.c index e9fa4f71efea..0214efd2a014 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -757,6 +757,15 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, if (!folio_trylock(folio)) continue; /* + * KSM folios can gain new mappings at any time, so we must + * not clear their dirty flag. + */ + if (folio_test_ksm(folio)) { + folio_unlock(folio); + continue; + } + + /* * If we have a large folio at this point, we know it is * fully mapped so if its mapcount is the same as its * number of pages, it must be exclusive. diff --git a/mm/memfd.c b/mm/memfd.c index 0f6fff004f5e..0e3ff3eebfa8 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -482,6 +482,8 @@ struct file *memfd_alloc_file(const char *name, unsigned int flags) } file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; + /* a regular file like any other: POSIX.1-2008/SUSv4 Section XSI 2.9.7 */ + file->f_mode |= FMODE_ATOMIC_POS; file->f_flags |= O_LARGEFILE; if (flags & MFD_NOEXEC_SEAL) { diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 25e121851b58..8eda313cf619 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -572,21 +572,14 @@ static struct memory_tier *set_node_memory_tier(int node) return memtier; } -static void destroy_memory_tier(struct memory_tier *memtier) +static struct memory_tier *clear_node_memory_tier(int node) { - list_del(&memtier->list); - device_unregister(&memtier->dev); -} - -static bool clear_node_memory_tier(int node) -{ - bool cleared = false; pg_data_t *pgdat; struct memory_tier *memtier; pgdat = NODE_DATA(node); if (!pgdat) - return false; + return NULL; /* * Make sure that anybody looking at NODE_DATA who finds @@ -604,14 +597,10 @@ static bool clear_node_memory_tier(int node) synchronize_rcu(); memtype = node_memory_types[node].memtype; node_clear(node, memtype->nodes); - if (nodes_empty(memtype->nodes)) { + if (nodes_empty(memtype->nodes)) list_del_init(&memtype->tier_sibling); - if (list_empty(&memtier->memory_types)) - destroy_memory_tier(memtier); - } - cleared = true; } - return cleared; + return memtier; } static void release_memtype(struct kref *kref) @@ -884,13 +873,21 @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self, { struct memory_tier *memtier; struct node_notify *nn = _arg; + bool empty = false; switch (action) { case NODE_REMOVED_LAST_MEMORY: mutex_lock(&memory_tier_lock); - if (clear_node_memory_tier(nn->nid)) + memtier = clear_node_memory_tier(nn->nid); + if (memtier) { + empty = list_empty(&memtier->memory_types); + if (empty) + list_del(&memtier->list); establish_demotion_targets(); + } mutex_unlock(&memory_tier_lock); + if (empty) + device_unregister(&memtier->dev); break; case NODE_ADDED_FIRST_MEMORY: mutex_lock(&memory_tier_lock); diff --git a/mm/memory.c b/mm/memory.c index b65b56fe9714..ec3d9d99ccee 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1463,7 +1463,6 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, unsigned long end) { struct mm_struct *dst_mm = dst_vma->vm_mm; - struct mm_struct *src_mm = src_vma->vm_mm; pud_t *src_pud, *dst_pud; unsigned long next; @@ -1477,8 +1476,7 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, int err; VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma); - err = copy_huge_pud(dst_mm, src_mm, - dst_pud, src_pud, addr, src_vma); + err = copy_huge_pud(dst_vma, src_vma, dst_pud, src_pud, addr); if (err == -ENOMEM) return -ENOMEM; if (!err) @@ -3769,6 +3767,13 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src, } /* + * __copy_from_user_inatomic() only works as expected for local faults. + * Reject remote faults. + */ + if (unlikely(mm != current->mm)) + return -EFAULT; + + /* * If the source page was a PFN mapping, we don't have * a "struct page" for it. We do a best-effort copy by * just copying from the original user address. If that @@ -4093,6 +4098,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) folio_put(old_folio); delayacct_wpcopy_end(); + if (err == -EFAULT) + return VM_FAULT_SIGBUS; return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0; } kmsan_copy_page_meta(&new_folio->page, vmf->page); @@ -4650,8 +4657,8 @@ EXPORT_SYMBOL_GPL(unmap_mapping_pages); void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { - pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT; - pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT; + pgoff_t hba = (u64)holebegin >> PAGE_SHIFT; + pgoff_t hlen = ((u64)holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; /* Check for overflow. */ if (sizeof(holelen) > sizeof(hlen)) { @@ -5125,7 +5132,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * folio->index of non-ksm folios would be nonlinear inside the * anon VMA -- the ksm flag is lost on actual swapout. */ - folio = ksm_might_need_to_copy(folio, vma, vmf->address); + folio = ksm_might_need_to_copy(folio, page, vma, vmf->address); if (unlikely(!folio)) { ret = VM_FAULT_OOM; folio = swapcache; @@ -5306,7 +5313,8 @@ check_folio: if (unlikely(folio != swapcache)) { folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); - folio_put_swap(swapcache, NULL); + folio_put_swap(swapcache, + folio_file_page(swapcache, swp_offset(entry))); } else if (!folio_test_anon(folio)) { /* * We currently only expect !anon folios that are fully @@ -6983,8 +6991,12 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, lru_gen_exit_fault(); - /* If the mapping is droppable, then errors due to OOM aren't fatal. */ - if (is_droppable) + /* + * If the mapping is droppable, then errors due to OOM aren't fatal: the + * task returns to user mode, handles its signals and tries again. A + * fault from kernel mode has no such return, so it must see the error. + */ + if (is_droppable && (flags & FAULT_FLAG_USER)) ret &= ~VM_FAULT_OOM; if (flags & FAULT_FLAG_USER) { diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 0884c629670d..4da1cb8777f6 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -3372,11 +3372,11 @@ void __init numa_policy_init(void) policy_cache = kmem_cache_create("numa_policy", sizeof(struct mempolicy), - 0, SLAB_PANIC, NULL); + 0, SLAB_PANIC | SLAB_ACCOUNT, NULL); sn_cache = kmem_cache_create("shared_policy_node", sizeof(struct sp_node), - 0, SLAB_PANIC, NULL); + 0, SLAB_PANIC | SLAB_ACCOUNT, NULL); for_each_node(nid) { preferred_node_policy[nid] = (struct mempolicy) { diff --git a/mm/mmap.c b/mm/mmap.c index 148ba01c0373..7618e72766a5 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1713,8 +1713,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) LIST_HEAD(uf); VMA_ITERATOR(vmi, mm, 0); - if (mmap_write_lock_killable(oldmm)) - return -EINTR; + mmap_assert_write_locked(oldmm); flush_cache_dup_mm(oldmm); uprobe_dup_mmap(oldmm, mm); /* @@ -1725,11 +1724,6 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) /* No ordering required: file already has been exposed. */ dup_mm_exe_file(mm, oldmm); - mm->total_vm = oldmm->total_vm; - mm->data_vm = oldmm->data_vm; - mm->exec_vm = oldmm->exec_vm; - mm->stack_vm = oldmm->stack_vm; - /* Use __mt_dup() to efficiently build an identical maple tree. */ retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL); if (unlikely(retval)) diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index df69ba6e797f..a7bae5de5fed 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -569,16 +569,25 @@ int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range) { struct mmu_notifier_subscriptions *subscriptions = range->mm->notifier_subscriptions; - int ret; + const bool has_itree = subscriptions->has_itree; + int ret = 0; - if (subscriptions->has_itree) { + if (has_itree) { ret = mn_itree_invalidate(subscriptions, range); if (ret) return ret; } if (!hlist_empty(&subscriptions->list)) - return mn_hlist_invalidate_range_start(subscriptions, range); - return 0; + ret = mn_hlist_invalidate_range_start(subscriptions, range); + + /* + * On error, the non-blocking caller is not allowed to call + * invalidate_range_end(), as in mn_itree_invalidate(). + */ + if (ret && has_itree && !mmu_notifier_range_blockable(range)) + mn_itree_inv_end(subscriptions); + + return ret; } static void @@ -938,7 +947,6 @@ static int __mmu_interval_notifier_insert( struct mmu_notifier_subscriptions *subscriptions, unsigned long start, unsigned long length, const struct mmu_interval_notifier_ops *ops) { - interval_sub->mm = mm; interval_sub->ops = ops; RB_CLEAR_NODE(&interval_sub->interval_tree.rb); interval_sub->interval_tree.start = start; @@ -955,8 +963,12 @@ static int __mmu_interval_notifier_insert( if (WARN_ON(atomic_read(&mm->mm_users) <= 0)) return -EINVAL; - /* pairs with mmdrop in mmu_interval_notifier_remove() */ + /* + * Pairs with mmdrop in mmu_interval_notifier_remove(). Drivers take a + * non-NULL ->mm for "registered", so set it only with the reference. + */ mmgrab(mm); + interval_sub->mm = mm; /* * If some invalidate_range_start/end region is going on in parallel diff --git a/mm/mremap.c b/mm/mremap.c index 0d06d7f37f19..70b7264aa6bf 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -87,7 +87,7 @@ static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr) return NULL; pud = pud_offset(p4d, addr); - if (pud_none_or_clear_bad(pud)) + if (!pud_trans_huge(*pud) && pud_none_or_clear_bad(pud)) return NULL; return pud; @@ -202,7 +202,7 @@ static int move_ptes(struct pagetable_move_control *pmc, struct mm_struct *mm = vma->vm_mm; pte_t *old_ptep, *new_ptep; pte_t old_pte, pte; - pmd_t dummy_pmdval; + pmd_t old_pmdval, new_pmdval; spinlock_t *old_ptl, *new_ptl; bool force_flush = false; unsigned long old_addr = pmc->old_addr; @@ -234,31 +234,30 @@ static int move_ptes(struct pagetable_move_control *pmc, if (pmc->need_rmap_locks) take_rmap_locks(vma); - /* - * We don't have to worry about the ordering of src and dst - * pte locks because exclusive mmap_lock prevents deadlock. - */ - old_ptep = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl); + old_ptep = pte_offset_map_rw_nolock(mm, old_pmd, old_addr, &old_pmdval, + &old_ptl); if (!old_ptep) { err = -EAGAIN; goto out; } - /* - * Now new_pte is none, so collapse_scan_file() path can not find - * this by traversing file->f_mapping, so there is no concurrency with - * retract_page_tables(). In addition, we already hold the exclusive - * mmap_lock, so this new_pte page is stable, so there is no need to get - * pmdval and do pmd_same() check. - */ - new_ptep = pte_offset_map_rw_nolock(mm, new_pmd, new_addr, &dummy_pmdval, - &new_ptl); + new_ptep = pte_offset_map_rw_nolock(mm, new_pmd, new_addr, &new_pmdval, + &new_ptl); if (!new_ptep) { - pte_unmap_unlock(old_ptep, old_ptl); + pte_unmap(old_ptep); + err = -EAGAIN; + goto out; + } + + double_pt_lock(old_ptl, new_ptl); + if (unlikely(!pmd_same(old_pmdval, pmdp_get_lockless(old_pmd)) || + !pmd_same(new_pmdval, pmdp_get_lockless(new_pmd)))) { + double_pt_unlock(old_ptl, new_ptl); + pte_unmap(new_ptep); + pte_unmap(old_ptep); err = -EAGAIN; goto out; } - if (new_ptl != old_ptl) - spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); + flush_tlb_batched_pending(vma->vm_mm); lazy_mmu_mode_enable(); @@ -317,10 +316,9 @@ static int move_ptes(struct pagetable_move_control *pmc, lazy_mmu_mode_disable(); if (force_flush) flush_tlb_range(vma, old_end - len, old_end); - if (new_ptl != old_ptl) - spin_unlock(new_ptl); + double_pt_unlock(old_ptl, new_ptl); pte_unmap(new_ptep - 1); - pte_unmap_unlock(old_ptep - 1, old_ptl); + pte_unmap(old_ptep - 1); out: if (pmc->need_rmap_locks) drop_rmap_locks(vma); @@ -395,14 +393,9 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, if (WARN_ON_ONCE(!pmd_none(*new_pmd))) return false; - /* - * We don't have to worry about the ordering of src and dst - * ptlocks because exclusive mmap_lock prevents deadlock. - */ - old_ptl = pmd_lock(mm, old_pmd); + old_ptl = pmd_lockptr(mm, old_pmd); new_ptl = pmd_lockptr(mm, new_pmd); - if (new_ptl != old_ptl) - spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); + double_pt_lock(old_ptl, new_ptl); pmd = *old_pmd; @@ -418,9 +411,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, pmd_populate(mm, new_pmd, pmd_pgtable(pmd)); flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE); out_unlock: - if (new_ptl != old_ptl) - spin_unlock(new_ptl); - spin_unlock(old_ptl); + double_pt_unlock(old_ptl, new_ptl); return res; } @@ -452,26 +443,24 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, if (WARN_ON_ONCE(!pud_none(*new_pud))) return false; - /* - * We don't have to worry about the ordering of src and dst - * ptlocks because exclusive mmap_lock prevents deadlock. - */ - old_ptl = pud_lock(mm, old_pud); + old_ptl = pud_lockptr(mm, old_pud); new_ptl = pud_lockptr(mm, new_pud); - if (new_ptl != old_ptl) - spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); - /* Clear the pud */ + double_pt_lock(old_ptl, new_ptl); + pud = *old_pud; + if (unlikely(pud_none(pud))) + goto out_unlock; + + /* Clear the pud */ pud_clear(old_pud); VM_BUG_ON(!pud_none(*new_pud)); pud_populate(mm, new_pud, pud_pgtable(pud)); flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE); - if (new_ptl != old_ptl) - spin_unlock(new_ptl); - spin_unlock(old_ptl); +out_unlock: + double_pt_unlock(old_ptl, new_ptl); return true; } @@ -499,18 +488,17 @@ static bool move_huge_pud(struct pagetable_move_control *pmc, if (WARN_ON_ONCE(!pud_none(*new_pud))) return false; - /* - * We don't have to worry about the ordering of src and dst - * ptlocks because exclusive mmap_lock prevents deadlock. - */ - old_ptl = pud_lock(mm, old_pud); + old_ptl = pud_lockptr(mm, old_pud); new_ptl = pud_lockptr(mm, new_pud); - if (new_ptl != old_ptl) - spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); + + double_pt_lock(old_ptl, new_ptl); + if (unlikely(!pud_trans_huge(*old_pud))) { + double_pt_unlock(old_ptl, new_ptl); + return false; + } /* Clear the pud */ - pud = *old_pud; - pud_clear(old_pud); + pud = pudp_huge_get_and_clear(mm, pmc->old_addr, old_pud); VM_BUG_ON(!pud_none(*new_pud)); @@ -518,9 +506,7 @@ static bool move_huge_pud(struct pagetable_move_control *pmc, /* mark soft_ditry when we add pud level soft dirty support */ set_pud_at(mm, pmc->new_addr, new_pud, pud); flush_pud_tlb_range(vma, pmc->old_addr, pmc->old_addr + HPAGE_PUD_SIZE); - if (new_ptl != old_ptl) - spin_unlock(new_ptl); - spin_unlock(old_ptl); + double_pt_unlock(old_ptl, new_ptl); return true; } @@ -842,11 +828,10 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) if (!new_pud) break; if (pud_trans_huge(*old_pud)) { - if (extent == HPAGE_PUD_SIZE) { - move_pgt_entry(pmc, HPAGE_PUD, old_pud, new_pud); - /* We ignore and continue on error? */ + if (extent == HPAGE_PUD_SIZE && + move_pgt_entry(pmc, HPAGE_PUD, old_pud, new_pud)) continue; - } + split_huge_pud(pmc->old, old_pud, pmc->old_addr); } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) { if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud)) continue; diff --git a/mm/nommu.c b/mm/nommu.c index 91ab65afda70..6c6f95329674 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1887,7 +1887,7 @@ subsys_initcall(init_admin_reserve); int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) { - mmap_write_lock(oldmm); + mmap_assert_write_locked(oldmm); dup_mm_exe_file(mm, oldmm); mmap_write_unlock(oldmm); return 0; diff --git a/mm/rmap.c b/mm/rmap.c index 1131b76bbbc2..0faef46806e5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -924,6 +924,7 @@ static bool folio_referenced_one(struct folio *folio, DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); int ptes = 0, referenced = 0; unsigned int nr; + pte_t pteval; while (page_vma_mapped_walk(&pvmw)) { address = pvmw.address; @@ -968,10 +969,12 @@ static bool folio_referenced_one(struct folio *folio, return false; } - if (pvmw.pte && folio_test_large(folio)) { + if (pvmw.pte) + pteval = ptep_get(pvmw.pte); + + if (pvmw.pte && pte_present(pteval) && folio_test_large(folio)) { const unsigned long end_addr = pmd_addr_end(address, vma->vm_end); const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT; - pte_t pteval = ptep_get(pvmw.pte); nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); } @@ -981,11 +984,23 @@ static bool folio_referenced_one(struct folio *folio, * are. —they could be on active/inactive lists or on MGLRU. So the * simplest approach is to disable this look-around optimization. */ - if (lru_gen_enabled() && !lru_gen_switching() && pvmw.pte) { + if (lru_gen_enabled() && !lru_gen_switching() && pvmw.pte && + pte_present(pteval)) { if (lru_gen_look_around(&pvmw, nr)) referenced++; } else if (pvmw.pte) { - if (clear_flush_young_ptes_notify(vma, address, pvmw.pte, nr)) + bool young = false; + + /* + * PFN swap PTEs, such as device-exclusive ones, that + * actually map pages are "old" from a CPU perspective. + * The MMU notifier takes care of any device aspects. + */ + if (pte_present(pteval)) + young = clear_flush_young_ptes(vma, address, pvmw.pte, nr); + young |= mmu_notifier_clear_flush_young(vma->vm_mm, address, + address + nr * PAGE_SIZE); + if (young) referenced++; } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { if (pmdp_clear_flush_young_notify(vma, address, diff --git a/mm/shmem.c b/mm/shmem.c index 3c0c7edd2020..b8399bb71997 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -77,6 +77,7 @@ static struct vfsmount *shm_mnt __ro_after_init; #include <linux/magic.h> #include <linux/syscalls.h> #include <linux/fcntl.h> +#include <linux/memfd.h> #include <uapi/linux/memfd.h> #include <linux/rmap.h> #include <linux/uuid.h> @@ -275,6 +276,11 @@ static const struct vm_operations_struct shmem_vm_ops; static const struct vm_operations_struct shmem_anon_vm_ops; static struct file_system_type shmem_fs_type; +bool shmem_file(struct file *file) +{ + return file && file_inode(file)->i_fop == &shmem_file_operations; +} + bool shmem_mapping(const struct address_space *mapping) { return mapping->a_ops == &shmem_aops; @@ -1176,12 +1182,11 @@ static void shmem_delete_from_page_cache(struct folio *folio, void *radswap) } /* - * Remove swap entry from page cache, free the swap and its page cache. Returns - * the number of pages being freed. 0 means entry not found in XArray (0 pages - * being freed). + * Remove a swap entry or swapin-error marker wholly within [index, end]. + * Return whether it was removed, counting freed swap pages in nr_swaps_freed. */ -static long shmem_free_swap(struct address_space *mapping, - pgoff_t index, pgoff_t end, void *radswap) +static bool shmem_free_swap(struct address_space *mapping, + pgoff_t index, pgoff_t end, void *radswap, long *nr_swaps_freed) { XA_STATE(xas, &mapping->i_pages, index); const softleaf_t swp = radix_to_swp_entry(radswap); @@ -1202,8 +1207,10 @@ static long shmem_free_swap(struct address_space *mapping, xas_unlock_irq(&xas); /* A swapin-error marker holds no swap slot, so just drop it. */ - if (nr_pages && softleaf_is_swap(swp)) + if (nr_pages && softleaf_is_swap(swp)) { swap_put_entries_direct(swp, nr_pages); + *nr_swaps_freed += nr_pages; + } return nr_pages; } @@ -1297,12 +1304,10 @@ void shmem_unlock_mapping(struct address_space *mapping) static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) { + const gfp_t gfp = mapping_gfp_mask(inode->i_mapping); struct folio *folio; - /* - * At first avoid shmem_get_folio(,,,SGP_READ): that fails - * beyond i_size, and reports fallocated folios as holes. - */ +retry: folio = filemap_get_entry(inode->i_mapping, index); if (!folio) return folio; @@ -1313,13 +1318,14 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) /* The folio has been swapped out */ folio_unlock(folio); folio_put(folio); + goto retry; } /* - * But read a folio back from swap if any of it is within i_size - * (although in some cases this is just a waste of time). + * Truncation can look up a large swap entry at or beyond i_size. + * Swap it in directly to bypass the size check in shmem_get_folio(). */ - folio = NULL; - shmem_get_folio(inode, index, 0, &folio, SGP_READ); + if (shmem_swapin_folio(inode, index, &folio, SGP_READ, gfp, NULL, NULL) == -EEXIST) + goto retry; return folio; } @@ -1358,8 +1364,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (xa_is_value(folio)) { if (unfalloc) continue; - nr_swaps_freed += shmem_free_swap(mapping, indices[i], - end - 1, folio); + shmem_free_swap(mapping, indices[i], end - 1, + folio, &nr_swaps_freed); continue; } @@ -1422,32 +1428,31 @@ whole_folios: if (xa_is_value(folio)) { int order; - long swaps_freed; if (unfalloc) continue; - swaps_freed = shmem_free_swap(mapping, indices[i], - end - 1, folio); - if (!swaps_freed) { + if (!shmem_free_swap(mapping, indices[i], end - 1, + folio, &nr_swaps_freed)) { pgoff_t base = indices[i]; order = shmem_confirm_swap(mapping, indices[i], radix_to_swp_entry(folio)); /* - * If found a large swap entry cross the end or start - * border, skip it as the truncate_inode_partial_folio - * above should have at least zerod its content once. + * Skip boundary entries when restarting + * the scan to avoid endless loops after + * swapin failure. */ if (order > 0) { base = round_down(base, 1 << order); - if (base < start || base + (1 << order) > end) + if (base < start || base + (1 << order) > end) { + start = max(start, base + (1 << order)); continue; + } } /* Swap was replaced by page or extended, retry */ index = base; break; } - nr_swaps_freed += swaps_freed; continue; } @@ -1463,23 +1468,35 @@ whole_folios: VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); - if (!folio_test_large(folio)) { + /* + * A shared mapping can have dirty PTEs while + * the folio is still clean. Mark the folio + * dirty so a failed split cannot discard data + * outside the range. + */ + if (!unfalloc && folio_test_large(folio)) + folio_mark_dirty(folio); + + if (unfalloc || !folio_test_large(folio)) { truncate_inode_folio(mapping, folio); } else if (truncate_inode_partial_folio(folio, lstart, lend, NULL, NULL)) { /* - * If we split a page, reset the loop so - * that we pick up the new sub pages. - * Otherwise the THP was entirely - * dropped or the target range was - * zeroed, so just continue the loop as - * is. + * The folio was split. Retry the scan + * to find the split folios. */ - if (!folio_test_large(folio)) { - folio_unlock(folio); - index = start; - break; - } + folio_unlock(folio); + index = start; + break; + } else if (folio->mapping) { + /* + * Splitting failed, but the part in the + * range is already zeroed. The folio + * was not dropped, so skip it when + * restarting the scan to avoid retrying + * it forever. + */ + start = max(start, folio_next_index(folio)); } } folio_unlock(folio); @@ -1546,10 +1563,11 @@ static int shmem_setattr(const struct mnt_idmap *idmap, if (error) return error; - if ((info->seals & F_SEAL_EXEC) && (attr->ia_valid & ATTR_MODE)) { - if ((inode->i_mode ^ attr->ia_mode) & 0111) { - return -EPERM; - } + if (attr->ia_valid & ATTR_MODE) { + error = memfd_check_seals_mode(info->seals, inode->i_mode, + attr->ia_mode); + if (error) + return error; } if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { @@ -1578,7 +1596,8 @@ static int shmem_setattr(const struct mnt_idmap *idmap, if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); - if (info->alloced) + /* Swapin-error markers are not counted in alloced. */ + if (info->alloced || !mapping_empty(inode->i_mapping)) shmem_truncate_range(inode, newsize, (loff_t)-1); /* unmap again to remove racily COWed private pages */ @@ -1614,6 +1633,17 @@ static int shmem_setattr(const struct mnt_idmap *idmap, return error; } +static void shmem_disallow_eviction(struct shmem_inode_info *info) +{ + atomic_inc(&info->stop_eviction); +} + +static void shmem_allow_eviction(struct shmem_inode_info *info) +{ + if (atomic_dec_and_test(&info->stop_eviction)) + wake_up_var(&info->stop_eviction); +} + static void shmem_evict_inode(struct inode *inode) { struct shmem_inode_info *info = SHMEM_I(inode); @@ -1626,16 +1656,21 @@ static void shmem_evict_inode(struct inode *inode) mapping_set_exiting(inode->i_mapping); shmem_truncate_range(inode, 0, (loff_t)-1); shmem_unused_huge_del(inode); - while (!list_empty(&info->swaplist)) { - /* Wait while shmem_unuse() is scanning this inode... */ + /* + * Writeout may still use the inode after swapoff has removed it + * from the swaplist, so always wait at least once. + */ + do { wait_var_event(&info->stop_eviction, !atomic_read(&info->stop_eviction)); + if (list_empty(&info->swaplist)) + break; spin_lock(&shmem_swaplist_lock); /* ...but beware of the race if we peeked too early */ if (!atomic_read(&info->stop_eviction)) list_del_init(&info->swaplist); spin_unlock(&shmem_swaplist_lock); - } + } while (!list_empty(&info->swaplist)); } simple_xattrs_free(&sbinfo->xa_cache, &info->xattrs, sbinfo->max_inodes ? &freed : NULL); @@ -1778,15 +1813,14 @@ start_over: * remove placeholder inode from swaplist, nor let it be freed * (igrab() would protect from unlink, but not from unmount). */ - atomic_inc(&info->stop_eviction); + shmem_disallow_eviction(info); spin_unlock(&shmem_swaplist_lock); error = shmem_unuse_inode(&info->vfs_inode, type); cond_resched(); spin_lock(&shmem_swaplist_lock); - if (atomic_dec_and_test(&info->stop_eviction)) - wake_up_var(&info->stop_eviction); + shmem_allow_eviction(info); if (error) break; if (list_empty(&info->swaplist)) @@ -1917,6 +1951,13 @@ try_split: spin_unlock(&shmem_swaplist_lock); } + /* + * Once the folio has left the page cache, its lock no longer + * keeps the inode from being evicted and freed: hold eviction + * off, in case the folio has to be put back below. + */ + shmem_disallow_eviction(info); + folio_dup_swap(folio, NULL); shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap)); @@ -1924,6 +1965,7 @@ try_split: error = swap_writeout(ctx, folio); if (error != AOP_WRITEPAGE_ACTIVATE) { /* folio has been unlocked */ + shmem_allow_eviction(info); return error; } @@ -1940,6 +1982,7 @@ try_split: shmem_recalc_inode(inode, 0, -nr_pages); folio_put_swap(folio, NULL); } + shmem_allow_eviction(info); /* * The swap_cache_del_folio() below could be left for @@ -2374,11 +2417,10 @@ static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index, folio_put_swap(folio, NULL); swap_cache_del_folio(folio); /* - * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks - * won't be 0 when inode is released and thus trigger WARN_ON(i_blocks) - * in shmem_evict_inode(). + * Let shmem_recalc_inode() reduce alloced and release the block charge + * for the freed swap pages. */ - shmem_recalc_inode(inode, -nr_pages, -nr_pages); + shmem_recalc_inode(inode, 0, -nr_pages); } static int shmem_split_large_entry(struct inode *inode, pgoff_t index, @@ -2519,6 +2561,8 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, else error = -ENOMEM; folio = NULL; + /* Make sure to validate the original value. */ + swap = index_entry; goto failed; } if (fault_type) { @@ -5408,12 +5452,35 @@ static const struct file_operations shmem_file_operations = { #endif }; +#ifdef CONFIG_TMPFS_POSIX_ACL +static int shmem_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + struct inode *inode = d_inode(dentry); + umode_t mode = inode->i_mode; + int error; + + if (type == ACL_TYPE_ACCESS) { + error = posix_acl_equiv_mode(acl, &mode); + if (error < 0) + return error; + error = memfd_check_seals_mode(SHMEM_I(inode)->seals, + inode->i_mode, mode); + if (error) + return error; + } + return simple_set_acl(idmap, dentry, acl, type); +} +#else +#define shmem_set_acl simple_set_acl +#endif + static const struct inode_operations shmem_inode_operations = { .getattr = shmem_getattr, .setattr = shmem_setattr, #ifdef CONFIG_TMPFS_XATTR .listxattr = shmem_listxattr, - .set_acl = simple_set_acl, + .set_acl = shmem_set_acl, .fileattr_get = shmem_fileattr_get, .fileattr_set = shmem_fileattr_set, #endif diff --git a/mm/shmem_quota.c b/mm/shmem_quota.c index a3fa4bd93e75..d191ebda3f00 100644 --- a/mm/shmem_quota.c +++ b/mm/shmem_quota.c @@ -239,7 +239,7 @@ out_unlock: return ret; } -static bool shmem_is_empty_dquot(struct dquot *dquot) +static bool shmem_is_default_dquot(struct dquot *dquot) { struct shmem_sb_info *sbinfo = dquot->dq_sb->s_fs_info; qsize_t bhardlimit; @@ -253,19 +253,21 @@ static bool shmem_is_empty_dquot(struct dquot *dquot) ihardlimit = sbinfo->qlimits.grpquota_ihardlimit; } - if (test_bit(DQ_FAKE_B, &dquot->dq_flags) || - (dquot->dq_dqb.dqb_curspace == 0 && - dquot->dq_dqb.dqb_curinodes == 0 && - dquot->dq_dqb.dqb_bhardlimit == bhardlimit && - dquot->dq_dqb.dqb_ihardlimit == ihardlimit)) - return true; - - return false; + /* + * Defaults are zero usage and soft limits, and the mount's hard limits. + * DQ_FAKE_B only means that *all* limits are zero, which need not match + * the mount defaults. + */ + return dquot->dq_dqb.dqb_curspace == 0 && + dquot->dq_dqb.dqb_curinodes == 0 && + dquot->dq_dqb.dqb_bsoftlimit == 0 && + dquot->dq_dqb.dqb_isoftlimit == 0 && + dquot->dq_dqb.dqb_bhardlimit == bhardlimit && + dquot->dq_dqb.dqb_ihardlimit == ihardlimit; } /* - * Store limits from dquot in the tree unless it's fake. If it is fake - * remove the id from the tree since there is no useful information in - * there. + * Remove the ID from the tree if it has no usage and its limits match + * the mount defaults. Otherwise, save its limits for the next acquire. */ static int shmem_release_dquot(struct dquot *dquot) { @@ -300,7 +302,7 @@ static int shmem_release_dquot(struct dquot *dquot) return -ENOENT; found: - if (shmem_is_empty_dquot(dquot)) { + if (shmem_is_default_dquot(dquot)) { /* Remove entry from the tree */ rb_erase(&entry->node, info->dqi_priv); kfree(entry); diff --git a/mm/swapfile.c b/mm/swapfile.c index 0f72a5b23a3a..2d2df53fc51a 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2514,7 +2514,8 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, return 0; swapcache = folio; - folio = ksm_might_need_to_copy(folio, vma, addr); + folio = ksm_might_need_to_copy(folio, + folio_file_page(folio, swp_offset(entry)), vma, addr); if (unlikely(!folio)) return -ENOMEM; else if (unlikely(folio == ERR_PTR(-EHWPOISON))) { diff --git a/mm/truncate.c b/mm/truncate.c index 733d2713a875..9e9bf029635c 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -475,8 +475,11 @@ void truncate_inode_pages_range(struct address_space *mapping, } } + if (start >= end) + return; + index = start; - while (index < end) { + while (true) { cond_resched(); if (!find_get_entries(mapping, &index, end - 1, &fbatch, indices)) { @@ -499,6 +502,7 @@ void truncate_inode_pages_range(struct address_space *mapping, folio_lock(folio); VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio); folio_wait_writeback(folio); + /* Do not handle errors, we'll do another lookup */ truncate_inode_folio(mapping, folio); folio_unlock(folio); } @@ -760,7 +764,12 @@ int invalidate_inode_pages2_range(struct address_space *mapping, folio_lock(folio); if (unlikely(folio->mapping != mapping)) { folio_unlock(folio); - continue; + /* + * Folio might have been replaced: look up + * again. + */ + index = indices[i]; + break; } VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio); folio_wait_writeback(folio); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 928ad95eeb2b..169f681f9849 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1206,33 +1206,6 @@ int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, return 0; } -void double_pt_lock(spinlock_t *ptl1, - spinlock_t *ptl2) - __acquires(ptl1) - __acquires(ptl2) -{ - if (ptl1 > ptl2) - swap(ptl1, ptl2); - /* lock in virtual address order to avoid lock inversion */ - spin_lock(ptl1); - if (ptl1 != ptl2) - spin_lock_nested(ptl2, SINGLE_DEPTH_NESTING); - else - __acquire(ptl2); -} - -void double_pt_unlock(spinlock_t *ptl1, - spinlock_t *ptl2) - __releases(ptl1) - __releases(ptl2) -{ - spin_unlock(ptl1); - if (ptl1 != ptl2) - spin_unlock(ptl2); - else - __release(ptl2); -} - static inline bool is_pte_pages_stable(pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst_pte, pte_t orig_src_pte, pmd_t *dst_pmd, pmd_t dst_pmdval) @@ -2270,6 +2243,9 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, return ret; } +static void wake_userfault_range(struct userfaultfd_ctx *ctx, + unsigned long start, unsigned long end); + /* Assumes mmap write lock taken, and mm_struct pinned. */ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, struct vm_area_struct *vma, @@ -2287,6 +2263,9 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, prev = vma; for_each_vma_range(vmi, vma, end) { + const bool any_mode_removed = vma->vm_flags & __VM_UFFD_FLAGS & ~vm_flags; + const vma_flags_t curr_vma_flags = vma_flags_and_mask(&vma->flags, __VMA_UFFD_FLAGS); + cond_resched(); VM_WARN_ON_ONCE(!vma_can_userfault(vma, vm_flags, wp_async)); @@ -2299,7 +2278,7 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, * userfaultfd and with the right tracking mode too. */ if (vma->vm_userfaultfd_ctx.ctx == ctx && - vma_test_all_mask(vma, vma_flags)) + vma_flags_same_pair(&curr_vma_flags, &vma_flags)) goto skip; /* @@ -2332,6 +2311,10 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, */ userfaultfd_set_ctx(vma, ctx, vm_flags); + /* Retry faults for removed modes while the VMA is write-locked. */ + if (any_mode_removed) + wake_userfault_range(ctx, start, vma_end); + if (vma_is_hugetlb(vma) && uffd_disable_huge_pmd_share(vma)) hugetlb_unshare_all_pmds(vma); @@ -3648,6 +3631,17 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, __wake_userfault(ctx, range); } +static void wake_userfault_range(struct userfaultfd_ctx *ctx, + unsigned long start, unsigned long end) +{ + struct userfaultfd_wake_range range = { + .start = start, + .len = end - start, + }; + + wake_userfault(ctx, &range); +} + static __always_inline int validate_unaligned_range( struct mm_struct *mm, __u64 start, __u64 len) { @@ -3977,6 +3971,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, ret = 0; for_each_vma_range(vmi, vma, end) { + struct userfaultfd_wake_range range; + cond_resched(); /* VMA not registered with userfaultfd. */ @@ -3991,18 +3987,17 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, start = vma->vm_start; vma_end = min(end, vma->vm_end); - if (userfaultfd_missing(vma)) { - /* - * Wake any concurrent pending userfault while - * we unregister, so they will not hang - * permanently and it avoids userland to call - * UFFDIO_WAKE explicitly. - */ - struct userfaultfd_wake_range range; - range.start = start; - range.len = vma_end - start; - wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range); - } + /* Prevent new userfaults from being queued after the wakeup. */ + vma_start_write(vma); + + /* + * Wake any concurrent pending userfault while we unregister, + * so they will not hang permanently and it avoids userland to + * call UFFDIO_WAKE explicitly. + */ + range.start = start; + range.len = vma_end - start; + wake_userfault(ctx, &range); vma = userfaultfd_clear_vma(&vmi, prev, vma, start, vma_end); diff --git a/mm/util.c b/mm/util.c index 8acfe2148ba5..80f6c714844f 100644 --- a/mm/util.c +++ b/mm/util.c @@ -697,6 +697,19 @@ void *vcalloc_noprof(size_t n, size_t size) } EXPORT_SYMBOL(vcalloc_noprof); +/** + * folio_anon_vma - Get a folio's raw anon_vma pointer. + * @folio: The folio. + * + * The returned pointer may be stale for an unmapped folio. The caller + * must ensure the anon_vma stays alive before dereferencing it. A folio + * reference or lock alone is insufficient; use folio_get_anon_vma() when + * the anon_vma's lifetime is not otherwise protected. Note that + * folio_get_anon_vma() is only useful for folios that are still mapped. + * + * Return: The raw anon_vma pointer for an anonymous, non-KSM folio, or + * %NULL otherwise. + */ struct anon_vma *folio_anon_vma(const struct folio *folio) { unsigned long mapping = (unsigned long)folio->mapping; @@ -516,6 +516,15 @@ void unmap_region(struct unmap_desc *unmap) tlb_finish_mmu(&tlb); } +static int check_split(struct vm_area_struct *vma, unsigned long addr) +{ + if (!vma->vm_ops) + return 0; + if (!vma->vm_ops->may_split) + return 0; + return vma->vm_ops->may_split(vma, addr); +} + /* * __split_vma() bypasses sysctl_max_map_count checking. We use this where it * has already been checked or doesn't make sense to fail. @@ -532,11 +541,9 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, WARN_ON(vma->vm_start >= addr); WARN_ON(vma->vm_end <= addr); - if (vma->vm_ops && vma->vm_ops->may_split) { - err = vma->vm_ops->may_split(vma, addr); - if (err) - return err; - } + err = check_split(vma, addr); + if (err) + return err; vma_start_write(vma); new = vm_area_dup(vma); @@ -969,6 +976,10 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle && !can_merge_remove_vma(middle)) return NULL; + /* If middle isn't removed then it's resized, i.e. splits. */ + if (!vmg->__remove_middle && check_split(middle, left_side ? end : start)) + return NULL; + /* * If we merge both VMAs, then next is also deleted. This implies * merge_will_delete_vma also. @@ -3406,7 +3417,8 @@ static int acct_stack_growth(struct vm_area_struct *vma, * Overcommit.. This must be the final test, as it will * update security statistics. */ - if (security_vm_enough_memory_mm(mm, grow)) + if (vma_test(vma, VMA_ACCOUNT_BIT) && + security_vm_enough_memory_mm(mm, grow)) return -ENOMEM; return 0; diff --git a/mm/vmscan.c b/mm/vmscan.c index 9959ca158ce8..6a1af9b92349 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3911,7 +3911,7 @@ restart: walk->mm_stats[MM_NONLEAF_FOUND]++; - if (!walk_pte_range(&val, addr, next, args)) + if (!walk_pte_range(pmd + i, addr, next, args)) continue; walk->mm_stats[MM_NONLEAF_ADDED]++; @@ -3947,7 +3947,7 @@ restart: if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; - walk_pmd_range(&val, addr, next, args); + walk_pmd_range(pud + i, addr, next, args); if (need_resched() || walk->batched >= MAX_LRU_BATCH) { end = (addr | ~PUD_MASK) + 1; |