diff options
Diffstat (limited to 'mm/hmm.c')
| -rw-r--r-- | mm/hmm.c | 275 |
1 files changed, 210 insertions, 65 deletions
@@ -17,6 +17,7 @@ #include <linux/slab.h> #include <linux/sched.h> #include <linux/mmzone.h> +#include <linux/oom.h> #include <linux/pagemap.h> #include <linux/leafops.h> #include <linux/hugetlb.h> @@ -32,9 +33,27 @@ struct hmm_vma_walk { struct hmm_range *range; + bool *locked; unsigned long last; + unsigned long end; + unsigned int required_fault; }; +/* + * Internal sentinel returned by walk callbacks when they need a page fault. + * The callback stores end/required_fault in hmm_vma_walk; the outer loop + * consumes the sentinel and never propagates it to the caller. + */ +#define HMM_FAULT_PENDING -EAGAIN + +/* + * Internal sentinel returned by hmm_do_fault() when handle_mm_fault() + * completes a page fault with the mmap lock dropped. hmm_do_fault() sets + * *locked = false; the outer loop consumes the sentinel and never propagates + * it to the caller. + */ +#define HMM_FAULT_UNLOCKED -ENOLCK + enum { HMM_NEED_FAULT = 1 << 0, HMM_NEED_WRITE_FAULT = 1 << 1, @@ -60,37 +79,25 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end, } /* - * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s) - * @addr: range virtual start address (inclusive) - * @end: range virtual end address (exclusive) - * @required_fault: HMM_NEED_* flags - * @walk: mm_walk structure - * Return: -EBUSY after page fault, or page fault error + * hmm_record_fault() - record a range that needs to be faulted in * - * This function will be called whenever pmd_none() or pte_none() returns true, - * or whenever there is no page directory covering the virtual address range. + * Called by the walk callbacks when they discover that part of the range + * needs a page fault. The callback records what to fault and returns + * HMM_FAULT_PENDING; the outer loop in hmm_range_fault_locked() drops + * back out of walk_page_range() and invokes handle_mm_fault() from a context + * where no page-table or hugetlb_vma_lock is held. */ -static int hmm_vma_fault(unsigned long addr, unsigned long end, - unsigned int required_fault, struct mm_walk *walk) +static int hmm_record_fault(unsigned long addr, unsigned long end, + unsigned int required_fault, + struct mm_walk *walk) { struct hmm_vma_walk *hmm_vma_walk = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned int fault_flags = FAULT_FLAG_REMOTE; WARN_ON_ONCE(!required_fault); hmm_vma_walk->last = addr; - - if (required_fault & HMM_NEED_WRITE_FAULT) { - if (!(vma->vm_flags & VM_WRITE)) - return -EPERM; - fault_flags |= FAULT_FLAG_WRITE; - } - - for (; addr < end; addr += PAGE_SIZE) - if (handle_mm_fault(vma, addr, fault_flags, NULL) & - VM_FAULT_ERROR) - return -EFAULT; - return -EBUSY; + hmm_vma_walk->end = end; + hmm_vma_walk->required_fault = required_fault; + return HMM_FAULT_PENDING; } static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, @@ -174,7 +181,7 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR); } if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); return hmm_pfns_fill(addr, end, range, 0); } @@ -209,7 +216,7 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags); if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { @@ -328,10 +335,10 @@ out: fault: pte_unmap(ptep); /* Fault any virtual address we were asked to fault */ - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, unsigned long end, unsigned long *hmm_pfns, pmd_t pmd) @@ -371,7 +378,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, npages, 0); if (required_fault) { if (softleaf_is_device_private(entry)) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); else return -EFAULT; } @@ -391,7 +398,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, return -EFAULT; return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, @@ -517,7 +524,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, npages, cpu_flags); if (required_fault) { spin_unlock(ptl); - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); @@ -564,21 +571,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, required_fault = hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags); if (required_fault) { - int ret; - spin_unlock(ptl); - hugetlb_vma_unlock_read(vma); - /* - * Avoid deadlock: drop the vma lock before calling - * hmm_vma_fault(), which will itself potentially take and - * drop the vma lock. This is also correct from a - * protection point of view, because there is no further - * use here of either pte or ptl after dropping the vma - * lock. - */ - ret = hmm_vma_fault(addr, end, required_fault, walk); - hugetlb_vma_lock_read(vma); - return ret; + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT); @@ -637,29 +631,70 @@ static const struct mm_walk_ops hmm_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; -/** - * hmm_range_fault - try to fault some address in a virtual address range - * @range: argument structure +/* + * hmm_do_fault - fault in a range recorded by a walk callback * - * Returns 0 on success or one of the following error codes: + * Called from the outer loop in hmm_range_fault_locked() after a callback + * returned HMM_FAULT_PENDING. At this point we hold only mmap_lock; + * the page-table spinlock and any hugetlb_vma_lock acquired by the walk + * framework have already been released by the unwind. * - * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma - * (e.g., device file vma). - * -ENOMEM: Out of memory. - * -EPERM: Invalid permission (e.g., asking for write and range is read - * only). - * -EBUSY: The range has been invalidated and the caller needs to wait for - * the invalidation to finish. - * -EFAULT: A page was requested to be valid and could not be made valid - * ie it has no backing VMA or it is illegal to access - * - * This is similar to get_user_pages(), except that it can read the page tables - * without mutating them (ie causing faults). + * Returns -EBUSY on success (all pages faulted, caller should re-walk). + * Returns a negative errno on failure. */ -int hmm_range_fault(struct hmm_range *range) +static int hmm_do_fault(struct mm_struct *mm, + struct hmm_vma_walk *hmm_vma_walk) +{ + unsigned long addr = hmm_vma_walk->last; + unsigned long end = hmm_vma_walk->end; + unsigned int required_fault = hmm_vma_walk->required_fault; + unsigned int fault_flags = FAULT_FLAG_REMOTE; + struct vm_area_struct *vma; + + if (hmm_vma_walk->locked) + fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + + vma = vma_lookup(mm, addr); + if (!vma) + return -EFAULT; + + if (required_fault & HMM_NEED_WRITE_FAULT) { + if (!(vma->vm_flags & VM_WRITE)) + return -EPERM; + fault_flags |= FAULT_FLAG_WRITE; + } + + for (; addr < end; addr += PAGE_SIZE) { + vm_fault_t ret; + + ret = handle_mm_fault(vma, addr, fault_flags, NULL); + + if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) { + if (hmm_vma_walk->locked) /* needed by sparse */ + *hmm_vma_walk->locked = false; + else + WARN_ON_ONCE(1); /* broken fault handler */ + return HMM_FAULT_UNLOCKED; + } + + if (ret & VM_FAULT_ERROR) { + int err = vm_fault_to_errno(ret, 0); + + if (WARN_ON(!err)) + err = -EINVAL; + + return err; + } + } + + return -EBUSY; +} + +static int hmm_range_fault_locked(struct hmm_range *range, bool *locked) { struct hmm_vma_walk hmm_vma_walk = { .range = range, + .locked = locked, .last = range->start, }; struct mm_struct *mm = range->notifier->mm; @@ -675,6 +710,22 @@ int hmm_range_fault(struct hmm_range *range) ret = walk_page_range(mm, hmm_vma_walk.last, range->end, &hmm_walk_ops, &hmm_vma_walk); /* + * When HMM_FAULT_PENDING is returned a walk callback + * recorded a range that needs handle_mm_fault(); + * hmm_do_fault() runs the fault outside walk_page_range() + * (so no page-table or hugetlb_vma_lock is held) and + * returns -EBUSY so the loop re-walks and picks up the + * now-present entries. + */ + if (ret == HMM_FAULT_PENDING) { + ret = hmm_do_fault(mm, &hmm_vma_walk); + if (ret == HMM_FAULT_UNLOCKED) { + if (fatal_signal_pending(current)) + return -EINTR; + return -EBUSY; + } + } + /* * When -EBUSY is returned the loop restarts with * hmm_vma_walk.last set to an address that has not been stored * in pfns. All entries < last in the pfn array are set to their @@ -683,9 +734,103 @@ int hmm_range_fault(struct hmm_range *range) } while (ret == -EBUSY); return ret; } + +/** + * hmm_range_fault - try to fault some address in a virtual address range + * @range: argument structure + * + * Returns 0 on success or one of the following error codes: + * + * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma + * (e.g., device file vma). + * -ENOMEM: Out of memory. + * -EPERM: Invalid permission (e.g., asking for write and range is read + * only). + * -EBUSY: The range has been invalidated and the caller needs to wait for + * the invalidation to finish. + * -EFAULT: A page was requested to be valid and could not be made valid + * ie it has no backing VMA or it is illegal to access + * + * This is similar to get_user_pages(), except that it can read the page tables + * without mutating them (ie causing faults). + * + * The mmap lock must be held by the caller and will remain held on return. + * New users should prefer hmm_range_fault_unlocked_timeout() unless they + * specifically need to keep the mmap lock held across the call. This helper + * cannot support VMAs whose fault handlers need to drop the mmap lock. + */ +int hmm_range_fault(struct hmm_range *range) +{ + return hmm_range_fault_locked(range, NULL); +} EXPORT_SYMBOL(hmm_range_fault); /** + * hmm_range_fault_unlocked_timeout - fault in a range with a retry timeout + * @range: argument structure + * @timeout: timeout in jiffies for internal -EBUSY retries, or 0 to retry + * indefinitely + * + * The caller must not hold the mmap lock. The function takes the mmap read + * lock internally and allows handle_mm_fault() to drop it during faults. If + * the mmap lock is dropped or the range is invalidated, the function refreshes + * range->notifier_seq and restarts the walk internally. + * + * Passing 0 for @timeout retries indefinitely. A non-zero @timeout is a caller + * policy limit for repeated mmu-notifier invalidation retries. HMM does not + * interrupt page fault handling when the timeout expires, but returns -EBUSY + * if the retry budget is exhausted before a stable range is obtained. + * + * Returns 0 on success or one of the error codes documented for + * hmm_range_fault(). -EINTR is returned if mmap_lock acquisition is + * interrupted or a fatal signal is pending during retry handling. + */ +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout) +{ + struct mm_struct *mm = range->notifier->mm; + unsigned long deadline = 0; + bool locked = false; + int ret; + + do { + /* + * If the previous fault dropped mmap_lock, then the fault + * handler made progress. Restart the retry timeout in that + * case, but keep the existing deadline for ordinary -EBUSY + * retries. + */ + if (timeout && !locked) + deadline = jiffies + timeout; + + range->notifier_seq = + mmu_interval_read_begin(range->notifier); + + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + + if (check_stable_address_space(mm)) { + mmap_read_unlock(mm); + return -EFAULT; + } + + if (timeout && time_after(jiffies, deadline)) { + mmap_read_unlock(mm); + return -EBUSY; + } + + locked = true; + ret = hmm_range_fault_locked(range, &locked); + if (locked) + mmap_read_unlock(mm); + } while (ret == -EBUSY); + + return ret; +} +EXPORT_SYMBOL(hmm_range_fault_unlocked_timeout); + +/** * hmm_dma_map_alloc - Allocate HMM map structure * @dev: device to allocate structure for * @map: HMM map to allocate @@ -709,7 +854,7 @@ int hmm_dma_map_alloc(struct device *dev, struct hmm_dma_map *map, * best approximation to ensure no swiotlb buffering happens. */ #ifdef CONFIG_DMA_NEED_SYNC - dma_need_sync = !dev->dma_skip_sync; + dma_need_sync = !dev_dma_skip_sync(dev); #endif /* CONFIG_DMA_NEED_SYNC */ if (dma_need_sync || dma_addressing_limited(dev)) return -EOPNOTSUPP; @@ -778,7 +923,7 @@ dma_addr_t hmm_dma_map_pfn(struct device *dev, struct hmm_dma_map *map, struct page *page = hmm_pfn_to_page(pfns[idx]); phys_addr_t paddr = hmm_pfn_to_phys(pfns[idx]); size_t offset = idx * map->dma_entry_size; - unsigned long attrs = 0; + unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT; dma_addr_t dma_addr; int ret; @@ -871,7 +1016,7 @@ bool hmm_dma_unmap_pfn(struct device *dev, struct hmm_dma_map *map, size_t idx) struct dma_iova_state *state = &map->state; dma_addr_t *dma_addrs = map->dma_list; unsigned long *pfns = map->pfn_list; - unsigned long attrs = 0; + unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT; if ((pfns[idx] & valid_dma) != valid_dma) return false; |