diff options
Diffstat (limited to 'lib/iov_iter.c')
| -rw-r--r-- | lib/iov_iter.c | 58 |
1 files changed, 49 insertions, 9 deletions
diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 0a63c7fba313..2072c04e99d0 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -277,7 +277,7 @@ static __always_inline size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); + return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len); } size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) @@ -296,7 +296,7 @@ static __always_inline size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_flushcache(to + progress, iter_from, len); + return copy_from_user_flushcache(to + progress, iter_from, len); } static __always_inline @@ -1224,13 +1224,13 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) { *new = *old; if (iov_iter_is_bvec(new)) - return new->bvec = kmemdup(new->bvec, - new->nr_segs * sizeof(struct bio_vec), + return new->bvec = kmemdup_array(new->bvec, + new->nr_segs, sizeof(struct bio_vec), flags); else if (iov_iter_is_kvec(new) || iter_is_iovec(new)) /* iovec and kvec have identical layout */ - return new->__iov = kmemdup(new->__iov, - new->nr_segs * sizeof(struct iovec), + return new->__iov = kmemdup_array(new->__iov, + new->nr_segs, sizeof(struct iovec), flags); return NULL; } @@ -1491,6 +1491,7 @@ void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) i->__iov -= state->nr_segs - i->nr_segs; i->nr_segs = state->nr_segs; } +EXPORT_SYMBOL_FOR_MODULES(iov_iter_restore, "vmw_vsock_virtio_transport_common"); /* * Extract a list of contiguous pages from an ITER_FOLIOQ iterator. This does @@ -1568,6 +1569,7 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, struct folio *folio; unsigned int nr = 0, offset; loff_t pos = i->xarray_start + i->iov_offset; + bool will_alloc = !*pages; XA_STATE(xas, i->xarray, pos >> PAGE_SHIFT); offset = pos & ~PAGE_MASK; @@ -1595,6 +1597,14 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, } rcu_read_unlock(); + if (!nr) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } + return 0; + } + maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize); iov_iter_advance(i, maxsize); return maxsize; @@ -1625,9 +1635,11 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, } bi.bi_idx = 0; bi.bi_size = maxsize; - bi.bi_bvec_done = skip; + bi.bi_offset = skip; maxpages = want_pages_array(pages, maxsize, skip, maxpages); + if (!maxpages) + return -ENOMEM; while (bi.bi_size && bi.bi_idx < i->nr_segs) { struct bio_vec bv = bvec_iter_bvec(i->bvec, bi); @@ -1745,6 +1757,7 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, unsigned long addr; unsigned int gup_flags = 0; size_t offset; + bool will_alloc = !*pages; int res; if (i->data_source == ITER_DEST) @@ -1761,8 +1774,14 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, if (!maxpages) return -ENOMEM; res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages); - if (unlikely(res <= 0)) + if (unlikely(res <= 0)) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } return res; + } + maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset); iov_iter_advance(i, maxsize); return maxsize; @@ -1886,6 +1905,8 @@ static unsigned int get_contig_folio_len(struct page **pages, * @max_size: maximum size to extract from @iter * @nr_vecs: number of vectors in @bv (on in and output) * @max_vecs: maximum vectors in @bv, including those filled before calling + * @mem_align_mask: reject with -EINVAL if the source address or + * length is not aligned to this mask * @extraction_flags: flags to qualify request * * Like iov_iter_extract_pages(), but returns physically contiguous ranges @@ -1897,7 +1918,8 @@ static unsigned int get_contig_folio_len(struct page **pages, */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags) + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags) { unsigned short entries_left = max_vecs - *nr_vecs; unsigned short nr_pages, i = 0; @@ -1906,6 +1928,24 @@ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, ssize_t size; /* + * DMA engines typically have both memory address and length alignment + * requirements, so check these against the alignment mask. For UBUF, + * IOVEC and KVEC, only the current segment will be extracted from; for + * everything else we might extract from multiple segments, so we need + * to check those too. + */ + if (likely(iter_is_ubuf(iter) || + iter_is_iovec(iter) || + iov_iter_is_kvec(iter))) { + unsigned long start = (unsigned long)iter_iov_addr(iter); + + if ((start | iter_iov_len(iter)) & mem_align_mask) + return -EINVAL; + } else if (iov_iter_alignment(iter) & mem_align_mask) { + return -EINVAL; + } + + /* * Move page array up in the allocated memory for the bio vecs as far as * possible so that we can start filling biovecs from the beginning * without overwriting the temporary page array. |