From 92a96b0a227e91dc42475265a1ce766b6cd044fa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 Aug 2025 23:09:18 +0100 Subject: io_uring: add request poisoning Poison various request fields on free. __io_req_caches_free() is a slow path, so can be done unconditionally, but gate it on kasan for io_req_add_to_cache(). Note that some fields are logically retained between cache allocations and can't be poisoned in io_req_add_to_cache(). Ideally, it'd be replaced with KASAN'ed caches, but that can't be enabled because of some synchronisation nuances. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7a78e8a7f5be434313c400650b862e36c211b312.1755459452.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 93633613a165..e511949086dd 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -179,6 +179,26 @@ static const struct ctl_table kernel_io_uring_disabled_table[] = { }; #endif +static void io_poison_cached_req(struct io_kiocb *req) +{ + req->ctx = IO_URING_PTR_POISON; + req->tctx = IO_URING_PTR_POISON; + req->file = IO_URING_PTR_POISON; + req->creds = IO_URING_PTR_POISON; + req->io_task_work.func = IO_URING_PTR_POISON; + req->apoll = IO_URING_PTR_POISON; +} + +static void io_poison_req(struct io_kiocb *req) +{ + io_poison_cached_req(req); + req->async_data = IO_URING_PTR_POISON; + req->kbuf = IO_URING_PTR_POISON; + req->comp_list.next = IO_URING_PTR_POISON; + req->file_node = IO_URING_PTR_POISON; + req->link = IO_URING_PTR_POISON; +} + static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) { return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); @@ -235,6 +255,8 @@ static inline void req_fail_link_node(struct io_kiocb *req, int res) static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) { + if (IS_ENABLED(CONFIG_KASAN)) + io_poison_cached_req(req); wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); } @@ -2767,6 +2789,7 @@ static __cold void __io_req_caches_free(struct io_ring_ctx *ctx) while (!io_req_cache_empty(ctx)) { req = io_extract_req(ctx); + io_poison_req(req); kmem_cache_free(req_cachep, req); nr++; } -- cgit v1.2.3 From ab3ea6eac5f45669b091309f592c4ea324003053 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Aug 2025 15:40:57 +0100 Subject: io_uring/zctx: check chained notif contexts Send zc only links ubuf_info for requests coming from the same context. There are some ambiguous syz reports, so let's check the assumption on notification completion. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/fd527d8638203fe0f1c5ff06ff2e1d8fd68f831b.1755179962.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/notif.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'io_uring') diff --git a/io_uring/notif.c b/io_uring/notif.c index 9a6f6e92d742..8c92e9cde2c6 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -14,10 +14,15 @@ static const struct ubuf_info_ops io_ubuf_ops; static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw) { struct io_notif_data *nd = io_notif_to_data(notif); + struct io_ring_ctx *ctx = notif->ctx; + + lockdep_assert_held(&ctx->uring_lock); do { notif = cmd_to_io_kiocb(nd); + if (WARN_ON_ONCE(ctx != notif->ctx)) + return; lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0); if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used)) -- cgit v1.2.3 From 5e73b402cbbea51bcab90fc5ee6c6d06af76ae1b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:30 -0600 Subject: io_uring/kbuf: drop 'issue_flags' from io_put_kbuf(s)() arguments Picking multiple buffers always requires the ring lock to be held across the operation, so there's no need to pass in the issue_flags to io_put_kbufs(). On the single buffer side, if the initial picking of a ring buffer was unlocked, then it will have been committed already. For legacy buffers, no locking is required, as they will simply be freed. Link: https://lore.kernel.org/r/20250821020750.598432-3-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 +- io_uring/kbuf.h | 5 ++--- io_uring/net.c | 14 ++++++-------- io_uring/rw.c | 10 +++++----- 4 files changed, 14 insertions(+), 17 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index e511949086dd..ee484a0e2c4f 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1007,7 +1007,7 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) lockdep_assert_held(&req->ctx->uring_lock); req_set_fail(req); - io_req_set_res(req, res, io_put_kbuf(req, res, IO_URING_F_UNLOCKED)); + io_req_set_res(req, res, io_put_kbuf(req, res)); if (def->fail) def->fail(req); io_req_complete_defer(req); diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index 723d0361898e..3d01778f378b 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -121,8 +121,7 @@ static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) return false; } -static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len, - unsigned issue_flags) +static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len) { if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED))) return 0; @@ -130,7 +129,7 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len, } static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len, - int nbufs, unsigned issue_flags) + int nbufs) { if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED))) return 0; diff --git a/io_uring/net.c b/io_uring/net.c index d69f2afa4f7a..3da253b80332 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -504,19 +504,18 @@ static int io_net_kbuf_recyle(struct io_kiocb *req, } static inline bool io_send_finish(struct io_kiocb *req, int *ret, - struct io_async_msghdr *kmsg, - unsigned issue_flags) + struct io_async_msghdr *kmsg) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); bool bundle_finished = *ret <= 0; unsigned int cflags; if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { - cflags = io_put_kbuf(req, *ret, issue_flags); + cflags = io_put_kbuf(req, *ret); goto finish; } - cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret), issue_flags); + cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret)); if (bundle_finished || req->flags & REQ_F_BL_EMPTY) goto finish; @@ -693,7 +692,7 @@ retry_bundle: else if (sr->done_io) ret = sr->done_io; - if (!io_send_finish(req, &ret, kmsg, issue_flags)) + if (!io_send_finish(req, &ret, kmsg)) goto retry_bundle; io_req_msg_cleanup(req, issue_flags); @@ -872,8 +871,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, if (sr->flags & IORING_RECVSEND_BUNDLE) { size_t this_ret = *ret - sr->done_io; - cflags |= io_put_kbufs(req, this_ret, io_bundle_nbufs(kmsg, this_ret), - issue_flags); + cflags |= io_put_kbufs(req, this_ret, io_bundle_nbufs(kmsg, this_ret)); if (sr->flags & IORING_RECV_RETRY) cflags = req->cqe.flags | (cflags & CQE_F_MASK); if (sr->mshot_len && *ret >= sr->mshot_len) @@ -895,7 +893,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, return false; } } else { - cflags |= io_put_kbuf(req, *ret, issue_flags); + cflags |= io_put_kbuf(req, *ret); } /* diff --git a/io_uring/rw.c b/io_uring/rw.c index 52a5b950b2e5..ae5229ae7dca 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -576,7 +576,7 @@ void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw) io_req_io_end(req); if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) - req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0); + req->cqe.flags |= io_put_kbuf(req, req->cqe.res); io_req_rw_cleanup(req, 0); io_req_task_complete(req, tw); @@ -659,7 +659,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, * from the submission path. */ io_req_io_end(req); - io_req_set_res(req, final_ret, io_put_kbuf(req, ret, issue_flags)); + io_req_set_res(req, final_ret, io_put_kbuf(req, ret)); io_req_rw_cleanup(req, issue_flags); return IOU_COMPLETE; } else { @@ -1057,7 +1057,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) if (ret < 0) req_set_fail(req); } else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { - cflags = io_put_kbuf(req, ret, issue_flags); + cflags = io_put_kbuf(req, ret); } else { /* * Any successful return value will keep the multishot read @@ -1065,7 +1065,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) * we fail to post a CQE, or multishot is no longer set, then * jump to the termination path. This request is then done. */ - cflags = io_put_kbuf(req, ret, issue_flags); + cflags = io_put_kbuf(req, ret); rw->len = 0; /* similarly to above, reset len to 0 */ if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { @@ -1362,7 +1362,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (!smp_load_acquire(&req->iopoll_completed)) break; nr_events++; - req->cqe.flags = io_put_kbuf(req, req->cqe.res, 0); + req->cqe.flags = io_put_kbuf(req, req->cqe.res); if (req->opcode != IORING_OP_URING_CMD) io_req_rw_cleanup(req, 0); } -- cgit v1.2.3 From 15ba5e51e689ceb1c2e921c5180a70c88cfdc8e9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:31 -0600 Subject: io_uring/net: don't use io_net_kbuf_recyle() for non-provided cases A previous commit used io_net_kbuf_recyle() for any network helper that did IO and needed partial retry. However, that's only needed if the opcode does buffer selection, which isnt support for sendzc, sendmsg_zc, or sendmsg. Just remove them - they don't do any harm, but it is a bit confusing when reading the code. Link: https://lore.kernel.org/r/20250821020750.598432-4-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/net.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'io_uring') diff --git a/io_uring/net.c b/io_uring/net.c index 3da253b80332..a8a6586bc416 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -570,7 +570,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_controllen = 0; kmsg->msg.msg_control = NULL; sr->done_io += ret; - return io_net_kbuf_recyle(req, kmsg, ret); + return -EAGAIN; } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -1503,7 +1503,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) zc->len -= ret; zc->buf += ret; zc->done_io += ret; - return io_net_kbuf_recyle(req, kmsg, ret); + return -EAGAIN; } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -1573,7 +1573,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; - return io_net_kbuf_recyle(req, kmsg, ret); + return -EAGAIN; } if (ret == -ERESTARTSYS) ret = -EINTR; -- cgit v1.2.3 From b22743f29b7d3dc68c68f9bd39a1b2600ec6434e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:32 -0600 Subject: io_uring/net: clarify io_recv_buf_select() return value It returns 0 on success, less than zero on error. Link: https://lore.kernel.org/r/20250821020750.598432-5-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/net.c b/io_uring/net.c index a8a6586bc416..381454ed41c0 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1197,7 +1197,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: if (io_do_buffer_select(req)) { ret = io_recv_buf_select(req, kmsg, &len, issue_flags); - if (unlikely(ret)) { + if (unlikely(ret < 0)) { kmsg->msg.msg_inq = -1; goto out_free; } -- cgit v1.2.3 From 1b5add75d7c894c62506c9b55f1d9eaadae50ef1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:33 -0600 Subject: io_uring/kbuf: pass in struct io_buffer_list to commit/recycle helpers Rather than have this implied being in the io_kiocb, pass it in directly so it's immediately obvious where these users of ->buf_list are coming from. Link: https://lore.kernel.org/r/20250821020750.598432-6-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 6 +++--- io_uring/kbuf.c | 9 +++++---- io_uring/kbuf.h | 22 +++++++++++++--------- io_uring/net.c | 28 ++++++++++++++-------------- io_uring/poll.c | 6 +++--- io_uring/rw.c | 14 +++++++------- 6 files changed, 45 insertions(+), 40 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index ee484a0e2c4f..27bc1486f07b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1007,7 +1007,7 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) lockdep_assert_held(&req->ctx->uring_lock); req_set_fail(req); - io_req_set_res(req, res, io_put_kbuf(req, res)); + io_req_set_res(req, res, io_put_kbuf(req, res, req->buf_list)); if (def->fail) def->fail(req); io_req_complete_defer(req); @@ -2025,11 +2025,11 @@ fail: switch (io_arm_poll_handler(req, 0)) { case IO_APOLL_READY: - io_kbuf_recycle(req, 0); + io_kbuf_recycle(req, req->buf_list, 0); io_req_task_queue(req); break; case IO_APOLL_ABORTED: - io_kbuf_recycle(req, 0); + io_kbuf_recycle(req, req->buf_list, 0); io_queue_iowq(req); break; case IO_APOLL_OK: diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index f2d2cc319faa..b8b2f6dee754 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -354,9 +354,9 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg) return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs); } -static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr) +static inline bool __io_put_kbuf_ring(struct io_kiocb *req, + struct io_buffer_list *bl, int len, int nr) { - struct io_buffer_list *bl = req->buf_list; bool ret = true; if (bl) @@ -366,7 +366,8 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr) return ret; } -unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs) +unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl, + int len, int nbufs) { unsigned int ret; @@ -377,7 +378,7 @@ unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs) return ret; } - if (!__io_put_kbuf_ring(req, len, nbufs)) + if (!__io_put_kbuf_ring(req, bl, len, nbufs)) ret |= IORING_CQE_F_BUF_MORE; return ret; } diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index 3d01778f378b..58451ab4ab8a 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -80,14 +80,16 @@ int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg); bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags); void io_kbuf_drop_legacy(struct io_kiocb *req); -unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs); +unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl, + int len, int nbufs); bool io_kbuf_commit(struct io_kiocb *req, struct io_buffer_list *bl, int len, int nr); struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx, unsigned int bgid); -static inline bool io_kbuf_recycle_ring(struct io_kiocb *req) +static inline bool io_kbuf_recycle_ring(struct io_kiocb *req, + struct io_buffer_list *bl) { /* * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear @@ -96,7 +98,7 @@ static inline bool io_kbuf_recycle_ring(struct io_kiocb *req) * The exception is partial io, that case we should increment bl->head * to monopolize the buffer. */ - if (req->buf_list) { + if (bl) { req->flags &= ~(REQ_F_BUFFER_RING|REQ_F_BUFFERS_COMMIT); return true; } @@ -110,29 +112,31 @@ static inline bool io_do_buffer_select(struct io_kiocb *req) return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)); } -static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) +static inline bool io_kbuf_recycle(struct io_kiocb *req, struct io_buffer_list *bl, + unsigned issue_flags) { if (req->flags & REQ_F_BL_NO_RECYCLE) return false; if (req->flags & REQ_F_BUFFER_SELECTED) return io_kbuf_recycle_legacy(req, issue_flags); if (req->flags & REQ_F_BUFFER_RING) - return io_kbuf_recycle_ring(req); + return io_kbuf_recycle_ring(req, bl); return false; } -static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len) +static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len, + struct io_buffer_list *bl) { if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED))) return 0; - return __io_put_kbufs(req, len, 1); + return __io_put_kbufs(req, bl, len, 1); } static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len, - int nbufs) + struct io_buffer_list *bl, int nbufs) { if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED))) return 0; - return __io_put_kbufs(req, len, nbufs); + return __io_put_kbufs(req, bl, len, nbufs); } #endif diff --git a/io_uring/net.c b/io_uring/net.c index 381454ed41c0..54e13d1b0d0b 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -494,12 +494,12 @@ static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret) return nbufs; } -static int io_net_kbuf_recyle(struct io_kiocb *req, +static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, struct io_async_msghdr *kmsg, int len) { req->flags |= REQ_F_BL_NO_RECYCLE; if (req->flags & REQ_F_BUFFERS_COMMIT) - io_kbuf_commit(req, req->buf_list, len, io_bundle_nbufs(kmsg, len)); + io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len)); return IOU_RETRY; } @@ -511,11 +511,11 @@ static inline bool io_send_finish(struct io_kiocb *req, int *ret, unsigned int cflags; if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { - cflags = io_put_kbuf(req, *ret); + cflags = io_put_kbuf(req, *ret, req->buf_list); goto finish; } - cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret)); + cflags = io_put_kbufs(req, *ret, req->buf_list, io_bundle_nbufs(kmsg, *ret)); if (bundle_finished || req->flags & REQ_F_BL_EMPTY) goto finish; @@ -681,7 +681,7 @@ retry_bundle: sr->len -= ret; sr->buf += ret; sr->done_io += ret; - return io_net_kbuf_recyle(req, kmsg, ret); + return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -871,7 +871,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, if (sr->flags & IORING_RECVSEND_BUNDLE) { size_t this_ret = *ret - sr->done_io; - cflags |= io_put_kbufs(req, this_ret, io_bundle_nbufs(kmsg, this_ret)); + cflags |= io_put_kbufs(req, this_ret, req->buf_list, io_bundle_nbufs(kmsg, this_ret)); if (sr->flags & IORING_RECV_RETRY) cflags = req->cqe.flags | (cflags & CQE_F_MASK); if (sr->mshot_len && *ret >= sr->mshot_len) @@ -893,7 +893,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, return false; } } else { - cflags |= io_put_kbuf(req, *ret); + cflags |= io_put_kbuf(req, *ret, req->buf_list); } /* @@ -1045,7 +1045,7 @@ retry_multishot: if (req->flags & REQ_F_APOLL_MULTISHOT) { ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); if (ret) { - io_kbuf_recycle(req, issue_flags); + io_kbuf_recycle(req, req->buf_list, issue_flags); return ret; } } @@ -1070,13 +1070,13 @@ retry_multishot: if (ret < min_ret) { if (ret == -EAGAIN && force_nonblock) { if (issue_flags & IO_URING_F_MULTISHOT) - io_kbuf_recycle(req, issue_flags); + io_kbuf_recycle(req, req->buf_list, issue_flags); return IOU_RETRY; } if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; - return io_net_kbuf_recyle(req, kmsg, ret); + return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -1090,7 +1090,7 @@ retry_multishot: else if (sr->done_io) ret = sr->done_io; else - io_kbuf_recycle(req, issue_flags); + io_kbuf_recycle(req, req->buf_list, issue_flags); if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags)) goto retry_multishot; @@ -1214,7 +1214,7 @@ retry_multishot: if (ret < min_ret) { if (ret == -EAGAIN && force_nonblock) { if (issue_flags & IO_URING_F_MULTISHOT) - io_kbuf_recycle(req, issue_flags); + io_kbuf_recycle(req, req->buf_list, issue_flags); return IOU_RETRY; } @@ -1222,7 +1222,7 @@ retry_multishot: sr->len -= ret; sr->buf += ret; sr->done_io += ret; - return io_net_kbuf_recyle(req, kmsg, ret); + return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -1238,7 +1238,7 @@ out_free: else if (sr->done_io) ret = sr->done_io; else - io_kbuf_recycle(req, issue_flags); + io_kbuf_recycle(req, req->buf_list, issue_flags); if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags)) goto retry_multishot; diff --git a/io_uring/poll.c b/io_uring/poll.c index c786e587563b..07ab22380c78 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -316,10 +316,10 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) ret = io_poll_check_events(req, tw); if (ret == IOU_POLL_NO_ACTION) { - io_kbuf_recycle(req, 0); + io_kbuf_recycle(req, req->buf_list, 0); return; } else if (ret == IOU_POLL_REQUEUE) { - io_kbuf_recycle(req, 0); + io_kbuf_recycle(req, req->buf_list, 0); __io_poll_execute(req, 0); return; } @@ -686,7 +686,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) req->flags |= REQ_F_POLLED; ipt.pt._qproc = io_async_queue_proc; - io_kbuf_recycle(req, issue_flags); + io_kbuf_recycle(req, req->buf_list, issue_flags); ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); if (ret) diff --git a/io_uring/rw.c b/io_uring/rw.c index ae5229ae7dca..7fe188872279 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -576,7 +576,7 @@ void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw) io_req_io_end(req); if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) - req->cqe.flags |= io_put_kbuf(req, req->cqe.res); + req->cqe.flags |= io_put_kbuf(req, req->cqe.res, req->buf_list); io_req_rw_cleanup(req, 0); io_req_task_complete(req, tw); @@ -659,7 +659,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, * from the submission path. */ io_req_io_end(req); - io_req_set_res(req, final_ret, io_put_kbuf(req, ret)); + io_req_set_res(req, final_ret, io_put_kbuf(req, ret, req->buf_list)); io_req_rw_cleanup(req, issue_flags); return IOU_COMPLETE; } else { @@ -1049,15 +1049,15 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) * Reset rw->len to 0 again to avoid clamping future mshot * reads, in case the buffer size varies. */ - if (io_kbuf_recycle(req, issue_flags)) + if (io_kbuf_recycle(req, req->buf_list, issue_flags)) rw->len = 0; return IOU_RETRY; } else if (ret <= 0) { - io_kbuf_recycle(req, issue_flags); + io_kbuf_recycle(req, req->buf_list, issue_flags); if (ret < 0) req_set_fail(req); } else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { - cflags = io_put_kbuf(req, ret); + cflags = io_put_kbuf(req, ret, req->buf_list); } else { /* * Any successful return value will keep the multishot read @@ -1065,7 +1065,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) * we fail to post a CQE, or multishot is no longer set, then * jump to the termination path. This request is then done. */ - cflags = io_put_kbuf(req, ret); + cflags = io_put_kbuf(req, ret, req->buf_list); rw->len = 0; /* similarly to above, reset len to 0 */ if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { @@ -1362,7 +1362,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (!smp_load_acquire(&req->iopoll_completed)) break; nr_events++; - req->cqe.flags = io_put_kbuf(req, req->cqe.res); + req->cqe.flags = io_put_kbuf(req, req->cqe.res, req->buf_list); if (req->opcode != IORING_OP_URING_CMD) io_req_rw_cleanup(req, 0); } -- cgit v1.2.3 From ab6559bdbb08f6bee606435cd014fc5ba0f7b750 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:34 -0600 Subject: io_uring/kbuf: introduce struct io_br_sel Rather than return addresses directly from buffer selection, add a struct around it. No functional changes in this patch, it's in preparation for storing more buffer related information locally, rather than in struct io_kiocb. Link: https://lore.kernel.org/r/20250821020750.598432-7-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/kbuf.c | 26 +++++++++++++------------- io_uring/kbuf.h | 19 +++++++++++++++++-- io_uring/net.c | 18 +++++++++--------- io_uring/rw.c | 34 ++++++++++++++++++++-------------- 4 files changed, 59 insertions(+), 38 deletions(-) (limited to 'io_uring') diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index b8b2f6dee754..61d9a8d439ba 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -151,18 +151,18 @@ static int io_provided_buffers_select(struct io_kiocb *req, size_t *len, return 1; } -static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, - struct io_buffer_list *bl, - unsigned int issue_flags) +static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, + struct io_buffer_list *bl, + unsigned int issue_flags) { struct io_uring_buf_ring *br = bl->buf_ring; __u16 tail, head = bl->head; + struct io_br_sel sel = { }; struct io_uring_buf *buf; - void __user *ret; tail = smp_load_acquire(&br->tail); if (unlikely(tail == head)) - return NULL; + return sel; if (head + 1 == tail) req->flags |= REQ_F_BL_EMPTY; @@ -173,7 +173,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT; req->buf_list = bl; req->buf_index = buf->bid; - ret = u64_to_user_ptr(buf->addr); + sel.addr = u64_to_user_ptr(buf->addr); if (issue_flags & IO_URING_F_UNLOCKED || !io_file_can_poll(req)) { /* @@ -189,27 +189,27 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, io_kbuf_commit(req, bl, *len, 1); req->buf_list = NULL; } - return ret; + return sel; } -void __user *io_buffer_select(struct io_kiocb *req, size_t *len, - unsigned buf_group, unsigned int issue_flags) +struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len, + unsigned buf_group, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; + struct io_br_sel sel = { }; struct io_buffer_list *bl; - void __user *ret = NULL; io_ring_submit_lock(req->ctx, issue_flags); bl = io_buffer_get_list(ctx, buf_group); if (likely(bl)) { if (bl->flags & IOBL_BUF_RING) - ret = io_ring_buffer_select(req, len, bl, issue_flags); + sel = io_ring_buffer_select(req, len, bl, issue_flags); else - ret = io_provided_buffer_select(req, len, bl); + sel.addr = io_provided_buffer_select(req, len, bl); } io_ring_submit_unlock(req->ctx, issue_flags); - return ret; + return sel; } /* cap it at a reasonable 256, will be one page even for 4K */ diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index 58451ab4ab8a..e14a43b125c3 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -62,8 +62,23 @@ struct buf_sel_arg { unsigned short partial_map; }; -void __user *io_buffer_select(struct io_kiocb *req, size_t *len, - unsigned buf_group, unsigned int issue_flags); +/* + * Return value from io_buffer_list selection. Just returns the error or + * user address for now, will be extended to return the buffer list in the + * future. + */ +struct io_br_sel { + /* + * Some selection parts return the user address, others return an error. + */ + union { + void __user *addr; + ssize_t val; + }; +}; + +struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len, + unsigned buf_group, unsigned int issue_flags); int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, unsigned int issue_flags); int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg); diff --git a/io_uring/net.c b/io_uring/net.c index 54e13d1b0d0b..12dcc21f2259 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1035,22 +1035,22 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: if (io_do_buffer_select(req)) { - void __user *buf; + struct io_br_sel sel; size_t len = sr->len; - buf = io_buffer_select(req, &len, sr->buf_group, issue_flags); - if (!buf) + sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); + if (!sel.addr) return -ENOBUFS; if (req->flags & REQ_F_APOLL_MULTISHOT) { - ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); + ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len); if (ret) { io_kbuf_recycle(req, req->buf_list, issue_flags); return ret; } } - iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len); + iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len); } kmsg->msg.msg_get_inq = 1; @@ -1153,13 +1153,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, arg.out_len); } else { - void __user *buf; + struct io_br_sel sel; *len = sr->len; - buf = io_buffer_select(req, len, sr->buf_group, issue_flags); - if (!buf) + sel = io_buffer_select(req, len, sr->buf_group, issue_flags); + if (!sel.addr) return -ENOBUFS; - sr->buf = buf; + sr->buf = sel.addr; sr->len = *len; map_ubuf: ret = import_ubuf(ITER_DEST, sr->buf, sr->len, diff --git a/io_uring/rw.c b/io_uring/rw.c index 7fe188872279..ef94cdde5f1f 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -107,34 +107,35 @@ static int io_import_vec(int ddir, struct io_kiocb *req, } static int __io_import_rw_buffer(int ddir, struct io_kiocb *req, - struct io_async_rw *io, - unsigned int issue_flags) + struct io_async_rw *io, struct io_br_sel *sel, + unsigned int issue_flags) { const struct io_issue_def *def = &io_issue_defs[req->opcode]; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - void __user *buf = u64_to_user_ptr(rw->addr); size_t sqe_len = rw->len; + sel->addr = u64_to_user_ptr(rw->addr); if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT)) - return io_import_vec(ddir, req, io, buf, sqe_len); + return io_import_vec(ddir, req, io, sel->addr, sqe_len); if (io_do_buffer_select(req)) { - buf = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags); - if (!buf) + *sel = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags); + if (!sel->addr) return -ENOBUFS; - rw->addr = (unsigned long) buf; + rw->addr = (unsigned long) sel->addr; rw->len = sqe_len; } - return import_ubuf(ddir, buf, sqe_len, &io->iter); + return import_ubuf(ddir, sel->addr, sqe_len, &io->iter); } static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, struct io_async_rw *io, + struct io_br_sel *sel, unsigned int issue_flags) { int ret; - ret = __io_import_rw_buffer(rw, req, io, issue_flags); + ret = __io_import_rw_buffer(rw, req, io, sel, issue_flags); if (unlikely(ret < 0)) return ret; @@ -306,10 +307,12 @@ static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, static int io_rw_do_import(struct io_kiocb *req, int ddir) { + struct io_br_sel sel = { }; + if (io_do_buffer_select(req)) return 0; - return io_import_rw_buffer(ddir, req, req->async_data, 0); + return io_import_rw_buffer(ddir, req, req->async_data, &sel, 0); } static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, @@ -899,7 +902,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type) return 0; } -static int __io_read(struct io_kiocb *req, unsigned int issue_flags) +static int __io_read(struct io_kiocb *req, struct io_br_sel *sel, + unsigned int issue_flags) { bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); @@ -913,7 +917,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(ret)) return ret; } else if (io_do_buffer_select(req)) { - ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags); + ret = io_import_rw_buffer(ITER_DEST, req, io, sel, issue_flags); if (unlikely(ret < 0)) return ret; } @@ -1015,9 +1019,10 @@ done: int io_read(struct io_kiocb *req, unsigned int issue_flags) { + struct io_br_sel sel = { }; int ret; - ret = __io_read(req, issue_flags); + ret = __io_read(req, &sel, issue_flags); if (ret >= 0) return kiocb_done(req, ret, issue_flags); @@ -1027,6 +1032,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + struct io_br_sel sel = { }; unsigned int cflags = 0; int ret; @@ -1038,7 +1044,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) /* make it sync, multishot doesn't support async execution */ rw->kiocb.ki_complete = NULL; - ret = __io_read(req, issue_flags); + ret = __io_read(req, &sel, issue_flags); /* * If we get -EAGAIN, recycle our buffer and just let normal poll -- cgit v1.2.3 From d8e1dec2f860ee40623609aa6c4f22e1ee45605d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:35 -0600 Subject: io_uring/rw: recycle buffers manually for non-mshot reads The mshot side of reads already does this, but the regular read path does not. This leads to needing recycling checks sprinkled in various spots in the "go async" path, like arming poll. In preparation for getting rid of those, ensure that read recycles appropriately. Link: https://lore.kernel.org/r/20250821020750.598432-8-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/rw.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'io_uring') diff --git a/io_uring/rw.c b/io_uring/rw.c index ef94cdde5f1f..2b106f644383 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -1026,6 +1026,8 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) if (ret >= 0) return kiocb_done(req, ret, issue_flags); + if (req->flags & REQ_F_BUFFERS_COMMIT) + io_kbuf_recycle(req, req->buf_list, issue_flags); return ret; } -- cgit v1.2.3 From 429884ff35f75a8ac3e8f822f483e220e3ea6394 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:36 -0600 Subject: io_uring/kbuf: use struct io_br_sel for multiple buffers picking The networking side uses bundles, which is picking multiple buffers at the same time. Pass in struct io_br_sel to those helpers. Link: https://lore.kernel.org/r/20250821020750.598432-9-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/kbuf.c | 5 +++-- io_uring/kbuf.h | 5 +++-- io_uring/net.c | 38 +++++++++++++++++++------------------- 3 files changed, 25 insertions(+), 23 deletions(-) (limited to 'io_uring') diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 61d9a8d439ba..21c12c437ab9 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -299,7 +299,7 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, } int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, - unsigned int issue_flags) + struct io_br_sel *sel, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_buffer_list *bl; @@ -331,7 +331,8 @@ out_unlock: return ret; } -int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg) +int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, + struct io_br_sel *sel) { struct io_ring_ctx *ctx = req->ctx; struct io_buffer_list *bl; diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index e14a43b125c3..b1723c2620da 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -80,8 +80,9 @@ struct io_br_sel { struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len, unsigned buf_group, unsigned int issue_flags); int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, - unsigned int issue_flags); -int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg); + struct io_br_sel *sel, unsigned int issue_flags); +int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, + struct io_br_sel *sel); void io_destroy_buffers(struct io_ring_ctx *ctx); int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); diff --git a/io_uring/net.c b/io_uring/net.c index 12dcc21f2259..8cff6a8244c0 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -586,17 +586,16 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) } static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, - struct io_async_msghdr *kmsg) + struct io_br_sel *sel, struct io_async_msghdr *kmsg) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); - - int ret; struct buf_sel_arg arg = { .iovs = &kmsg->fast_iov, .max_len = min_not_zero(sr->len, INT_MAX), .nr_iovs = 1, .buf_group = sr->buf_group, }; + int ret; if (kmsg->vec.iovec) { arg.nr_iovs = kmsg->vec.nr; @@ -609,7 +608,7 @@ static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, else arg.mode |= KBUF_MODE_EXPAND; - ret = io_buffers_select(req, &arg, issue_flags); + ret = io_buffers_select(req, &arg, sel, issue_flags); if (unlikely(ret < 0)) return ret; @@ -638,6 +637,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr *kmsg = req->async_data; + struct io_br_sel sel = { }; struct socket *sock; unsigned flags; int min_ret = 0; @@ -657,7 +657,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) retry_bundle: if (io_do_buffer_select(req)) { - ret = io_send_select_buffer(req, issue_flags, kmsg); + ret = io_send_select_buffer(req, issue_flags, &sel, kmsg); if (ret) return ret; } @@ -1015,6 +1015,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr *kmsg = req->async_data; + struct io_br_sel sel = { }; struct socket *sock; unsigned flags; int ret, min_ret = 0; @@ -1035,7 +1036,6 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: if (io_do_buffer_select(req)) { - struct io_br_sel sel; size_t len = sr->len; sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); @@ -1099,7 +1099,7 @@ retry_multishot: } static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg, - size_t *len, unsigned int issue_flags) + struct io_br_sel *sel, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); int ret; @@ -1124,15 +1124,15 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg arg.mode |= KBUF_MODE_FREE; } - if (*len) - arg.max_len = *len; + if (sel->val) + arg.max_len = sel->val; else if (kmsg->msg.msg_inq > 1) - arg.max_len = min_not_zero(*len, (size_t) kmsg->msg.msg_inq); + arg.max_len = min_not_zero(sel->val, (size_t) kmsg->msg.msg_inq); /* if mshot limited, ensure we don't go over */ if (sr->flags & IORING_RECV_MSHOT_LIM) arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len); - ret = io_buffers_peek(req, &arg); + ret = io_buffers_peek(req, &arg, sel); if (unlikely(ret < 0)) return ret; @@ -1153,14 +1153,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, arg.out_len); } else { - struct io_br_sel sel; + size_t len = sel->val; - *len = sr->len; - sel = io_buffer_select(req, len, sr->buf_group, issue_flags); - if (!sel.addr) + *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); + if (!sel->addr) return -ENOBUFS; - sr->buf = sel.addr; - sr->len = *len; + sr->buf = sel->addr; + sr->len = len; map_ubuf: ret = import_ubuf(ITER_DEST, sr->buf, sr->len, &kmsg->msg.msg_iter); @@ -1175,11 +1174,11 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr *kmsg = req->async_data; + struct io_br_sel sel = { }; struct socket *sock; unsigned flags; int ret, min_ret = 0; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; - size_t len = sr->len; bool mshot_finished; if (!(req->flags & REQ_F_POLLED) && @@ -1196,7 +1195,8 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: if (io_do_buffer_select(req)) { - ret = io_recv_buf_select(req, kmsg, &len, issue_flags); + sel.val = sr->len; + ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); if (unlikely(ret < 0)) { kmsg->msg.msg_inq = -1; goto out_free; -- cgit v1.2.3 From 58d815091890e83aa2f83a9cce1fdfe3af02c7b4 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:37 -0600 Subject: io_uring/net: use struct io_br_sel->val as the recv finish value Currently a pointer is passed in to the 'ret' in the receive handlers, but since we already have a value field in io_br_sel, just use that. This is also in preparation for needing to pass in struct io_br_sel to io_recv_finish() anyway. Link: https://lore.kernel.org/r/20250821020750.598432-10-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/net.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'io_uring') diff --git a/io_uring/net.c b/io_uring/net.c index 8cff6a8244c0..a7a4443e3ee7 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -845,9 +845,10 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) * Returns true if it is actually finished, or false if it should run * again (for multishot). */ -static inline bool io_recv_finish(struct io_kiocb *req, int *ret, +static inline bool io_recv_finish(struct io_kiocb *req, struct io_async_msghdr *kmsg, - bool mshot_finished, unsigned issue_flags) + struct io_br_sel *sel, bool mshot_finished, + unsigned issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); unsigned int cflags = 0; @@ -855,13 +856,13 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, if (kmsg->msg.msg_inq > 0) cflags |= IORING_CQE_F_SOCK_NONEMPTY; - if (*ret > 0 && sr->flags & IORING_RECV_MSHOT_LIM) { + if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) { /* * If sr->len hits zero, the limit has been reached. Mark * mshot as finished, and flag MSHOT_DONE as well to prevent * a potential bundle from being retried. */ - sr->mshot_total_len -= min_t(int, *ret, sr->mshot_total_len); + sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len); if (!sr->mshot_total_len) { sr->flags |= IORING_RECV_MSHOT_DONE; mshot_finished = true; @@ -869,12 +870,12 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, } if (sr->flags & IORING_RECVSEND_BUNDLE) { - size_t this_ret = *ret - sr->done_io; + size_t this_ret = sel->val - sr->done_io; cflags |= io_put_kbufs(req, this_ret, req->buf_list, io_bundle_nbufs(kmsg, this_ret)); if (sr->flags & IORING_RECV_RETRY) cflags = req->cqe.flags | (cflags & CQE_F_MASK); - if (sr->mshot_len && *ret >= sr->mshot_len) + if (sr->mshot_len && sel->val >= sr->mshot_len) sr->flags |= IORING_RECV_MSHOT_CAP; /* bundle with no more immediate buffers, we're done */ if (req->flags & REQ_F_BL_EMPTY) @@ -893,7 +894,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, return false; } } else { - cflags |= io_put_kbuf(req, *ret, req->buf_list); + cflags |= io_put_kbuf(req, sel->val, req->buf_list); } /* @@ -901,8 +902,8 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, * receive from this socket. */ if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished && - io_req_post_cqe(req, *ret, cflags | IORING_CQE_F_MORE)) { - *ret = IOU_RETRY; + io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { + sel->val = IOU_RETRY; io_mshot_prep_retry(req, kmsg); /* Known not-empty or unknown state, retry */ if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) { @@ -914,15 +915,15 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, sr->nr_multishot_loops = 0; sr->flags &= ~IORING_RECV_MSHOT_CAP; if (issue_flags & IO_URING_F_MULTISHOT) - *ret = IOU_REQUEUE; + sel->val = IOU_REQUEUE; } return true; } /* Finish the request / stop multishot. */ finish: - io_req_set_res(req, *ret, cflags); - *ret = IOU_COMPLETE; + io_req_set_res(req, sel->val, cflags); + sel->val = IOU_COMPLETE; io_req_msg_cleanup(req, issue_flags); return true; } @@ -1092,10 +1093,11 @@ retry_multishot: else io_kbuf_recycle(req, req->buf_list, issue_flags); - if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags)) + sel.val = ret; + if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) goto retry_multishot; - return ret; + return sel.val; } static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg, @@ -1240,10 +1242,11 @@ out_free: else io_kbuf_recycle(req, req->buf_list, issue_flags); - if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags)) + sel.val = ret; + if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) goto retry_multishot; - return ret; + return sel.val; } int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -- cgit v1.2.3 From 461382a51fb83a9c4b7c50e1f10d3ca94edff25e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:38 -0600 Subject: io_uring/net: use struct io_br_sel->val as the send finish value Currently a pointer is passed in to the 'ret' in the send mshot handler, but since we already have a value field in io_br_sel, just use that. This is also in preparation for needing to pass in struct io_br_sel to io_send_finish() anyway. Link: https://lore.kernel.org/r/20250821020750.598432-11-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/net.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'io_uring') diff --git a/io_uring/net.c b/io_uring/net.c index a7a4443e3ee7..4eb208d24169 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -503,19 +503,20 @@ static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, return IOU_RETRY; } -static inline bool io_send_finish(struct io_kiocb *req, int *ret, - struct io_async_msghdr *kmsg) +static inline bool io_send_finish(struct io_kiocb *req, + struct io_async_msghdr *kmsg, + struct io_br_sel *sel) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); - bool bundle_finished = *ret <= 0; + bool bundle_finished = sel->val <= 0; unsigned int cflags; if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { - cflags = io_put_kbuf(req, *ret, req->buf_list); + cflags = io_put_kbuf(req, sel->val, req->buf_list); goto finish; } - cflags = io_put_kbufs(req, *ret, req->buf_list, io_bundle_nbufs(kmsg, *ret)); + cflags = io_put_kbufs(req, sel->val, req->buf_list, io_bundle_nbufs(kmsg, sel->val)); if (bundle_finished || req->flags & REQ_F_BL_EMPTY) goto finish; @@ -524,15 +525,15 @@ static inline bool io_send_finish(struct io_kiocb *req, int *ret, * Fill CQE for this receive and see if we should keep trying to * receive from this socket. */ - if (io_req_post_cqe(req, *ret, cflags | IORING_CQE_F_MORE)) { + if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { io_mshot_prep_retry(req, kmsg); return false; } /* Otherwise stop bundle and use the current result. */ finish: - io_req_set_res(req, *ret, cflags); - *ret = IOU_COMPLETE; + io_req_set_res(req, sel->val, cflags); + sel->val = IOU_COMPLETE; return true; } @@ -692,11 +693,12 @@ retry_bundle: else if (sr->done_io) ret = sr->done_io; - if (!io_send_finish(req, &ret, kmsg)) + sel.val = ret; + if (!io_send_finish(req, kmsg, &sel)) goto retry_bundle; io_req_msg_cleanup(req, issue_flags); - return ret; + return sel.val; } static int io_recvmsg_mshot_prep(struct io_kiocb *req, -- cgit v1.2.3 From 5fda51255439addd1c9059098e30847a375a1008 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:39 -0600 Subject: io_uring/kbuf: switch to storing struct io_buffer_list locally Currently the buffer list is stored in struct io_kiocb. The buffer list can be of two types: 1) Classic/legacy buffer list. These don't need to get referenced after a buffer pick, and hence storing them in struct io_kiocb is perfectly fine. 2) Ring provided buffer lists. These DO need to be referenced after the initial buffer pick, as they need to get consumed later on. This can be either just incrementing the head of the ring, or it can be consuming parts of a buffer if incremental buffer consumptions has been configured. For case 2, io_uring needs to be careful not to access the buffer list after the initial pick-and-execute context. The core does recycling of these, but it's easy to make a mistake, because it's stored in the io_kiocb which does persist across multiple execution contexts. Either because it's a multishot request, or simply because it needed some kind of async trigger (eg poll) for retry purposes. Add a struct io_buffer_list to struct io_br_sel, which is always on stack for the various users of it. This prevents the buffer list from leaking outside of that execution context, and additionally it enables kbuf to not even pass back the struct io_buffer_list if the given context isn't appropriately locked already. This doesn't fix any bugs, it's simply a defensive measure to prevent any issues with reuse of a buffer list. Link: https://lore.kernel.org/r/20250821020750.598432-12-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 6 +++--- io_uring/kbuf.c | 27 +++++++++++++++------------ io_uring/kbuf.h | 16 ++++++---------- io_uring/net.c | 46 +++++++++++++++++----------------------------- io_uring/poll.c | 6 +++--- io_uring/rw.c | 22 +++++++++++----------- 6 files changed, 55 insertions(+), 68 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 27bc1486f07b..985b4681e513 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1007,7 +1007,7 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) lockdep_assert_held(&req->ctx->uring_lock); req_set_fail(req); - io_req_set_res(req, res, io_put_kbuf(req, res, req->buf_list)); + io_req_set_res(req, res, io_put_kbuf(req, res, NULL)); if (def->fail) def->fail(req); io_req_complete_defer(req); @@ -2025,11 +2025,11 @@ fail: switch (io_arm_poll_handler(req, 0)) { case IO_APOLL_READY: - io_kbuf_recycle(req, req->buf_list, 0); + io_kbuf_recycle(req, NULL, 0); io_req_task_queue(req); break; case IO_APOLL_ABORTED: - io_kbuf_recycle(req, req->buf_list, 0); + io_kbuf_recycle(req, NULL, 0); io_queue_iowq(req); break; case IO_APOLL_OK: diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 21c12c437ab9..3e9aab21af9d 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -171,8 +171,8 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, if (*len == 0 || *len > buf->len) *len = buf->len; req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT; - req->buf_list = bl; req->buf_index = buf->bid; + sel.buf_list = bl; sel.addr = u64_to_user_ptr(buf->addr); if (issue_flags & IO_URING_F_UNLOCKED || !io_file_can_poll(req)) { @@ -186,8 +186,8 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, * the transfer completes (or if we get -EAGAIN and must poll of * retry). */ - io_kbuf_commit(req, bl, *len, 1); - req->buf_list = NULL; + io_kbuf_commit(req, sel.buf_list, *len, 1); + sel.buf_list = NULL; } return sel; } @@ -294,7 +294,6 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, req->flags |= REQ_F_BL_EMPTY; req->flags |= REQ_F_BUFFER_RING; - req->buf_list = bl; return iov - arg->iovs; } @@ -302,16 +301,15 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, struct io_br_sel *sel, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; - struct io_buffer_list *bl; int ret = -ENOENT; io_ring_submit_lock(ctx, issue_flags); - bl = io_buffer_get_list(ctx, arg->buf_group); - if (unlikely(!bl)) + sel->buf_list = io_buffer_get_list(ctx, arg->buf_group); + if (unlikely(!sel->buf_list)) goto out_unlock; - if (bl->flags & IOBL_BUF_RING) { - ret = io_ring_buffers_peek(req, arg, bl); + if (sel->buf_list->flags & IOBL_BUF_RING) { + ret = io_ring_buffers_peek(req, arg, sel->buf_list); /* * Don't recycle these buffers if we need to go through poll. * Nobody else can use them anyway, and holding on to provided @@ -321,13 +319,16 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, */ if (ret > 0) { req->flags |= REQ_F_BUFFERS_COMMIT | REQ_F_BL_NO_RECYCLE; - io_kbuf_commit(req, bl, arg->out_len, ret); + io_kbuf_commit(req, sel->buf_list, arg->out_len, ret); } } else { - ret = io_provided_buffers_select(req, &arg->out_len, bl, arg->iovs); + ret = io_provided_buffers_select(req, &arg->out_len, sel->buf_list, arg->iovs); } out_unlock: - io_ring_submit_unlock(ctx, issue_flags); + if (issue_flags & IO_URING_F_UNLOCKED) { + sel->buf_list = NULL; + mutex_unlock(&ctx->uring_lock); + } return ret; } @@ -348,10 +349,12 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, ret = io_ring_buffers_peek(req, arg, bl); if (ret > 0) req->flags |= REQ_F_BUFFERS_COMMIT; + sel->buf_list = bl; return ret; } /* don't support multiple buffer selections for legacy */ + sel->buf_list = NULL; return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs); } diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index b1723c2620da..1a539969fc9c 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -63,11 +63,14 @@ struct buf_sel_arg { }; /* - * Return value from io_buffer_list selection. Just returns the error or - * user address for now, will be extended to return the buffer list in the - * future. + * Return value from io_buffer_list selection, to avoid stashing it in + * struct io_kiocb. For legacy/classic provided buffers, keeping a reference + * across execution contexts are fine. But for ring provided buffers, the + * list may go away as soon as ->uring_lock is dropped. As the io_kiocb + * persists, it's better to just keep the buffer local for those cases. */ struct io_br_sel { + struct io_buffer_list *buf_list; /* * Some selection parts return the user address, others return an error. */ @@ -107,13 +110,6 @@ struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx, static inline bool io_kbuf_recycle_ring(struct io_kiocb *req, struct io_buffer_list *bl) { - /* - * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear - * the flag and hence ensure that bl->head doesn't get incremented. - * If the tail has already been incremented, hang on to it. - * The exception is partial io, that case we should increment bl->head - * to monopolize the buffer. - */ if (bl) { req->flags &= ~(REQ_F_BUFFER_RING|REQ_F_BUFFERS_COMMIT); return true; diff --git a/io_uring/net.c b/io_uring/net.c index 4eb208d24169..b00cd2f59091 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -433,7 +433,6 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (req->opcode == IORING_OP_SENDMSG) return -EINVAL; sr->msg_flags |= MSG_WAITALL; - req->buf_list = NULL; req->flags |= REQ_F_MULTISHOT; } @@ -512,11 +511,11 @@ static inline bool io_send_finish(struct io_kiocb *req, unsigned int cflags; if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { - cflags = io_put_kbuf(req, sel->val, req->buf_list); + cflags = io_put_kbuf(req, sel->val, sel->buf_list); goto finish; } - cflags = io_put_kbufs(req, sel->val, req->buf_list, io_bundle_nbufs(kmsg, sel->val)); + cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); if (bundle_finished || req->flags & REQ_F_BL_EMPTY) goto finish; @@ -657,6 +656,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) flags |= MSG_DONTWAIT; retry_bundle: + sel.buf_list = NULL; if (io_do_buffer_select(req)) { ret = io_send_select_buffer(req, issue_flags, &sel, kmsg); if (ret) @@ -682,7 +682,7 @@ retry_bundle: sr->len -= ret; sr->buf += ret; sr->done_io += ret; - return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); + return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -795,18 +795,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->flags |= REQ_F_NOWAIT; if (sr->msg_flags & MSG_ERRQUEUE) req->flags |= REQ_F_CLEAR_POLLIN; - if (req->flags & REQ_F_BUFFER_SELECT) { - /* - * Store the buffer group for this multishot receive separately, - * as if we end up doing an io-wq based issue that selects a - * buffer, it has to be committed immediately and that will - * clear ->buf_list. This means we lose the link to the buffer - * list, and the eventual buffer put on completion then cannot - * restore it. - */ + if (req->flags & REQ_F_BUFFER_SELECT) sr->buf_group = req->buf_index; - req->buf_list = NULL; - } sr->mshot_total_len = sr->mshot_len = 0; if (sr->flags & IORING_RECV_MULTISHOT) { if (!(req->flags & REQ_F_BUFFER_SELECT)) @@ -874,7 +864,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, if (sr->flags & IORING_RECVSEND_BUNDLE) { size_t this_ret = sel->val - sr->done_io; - cflags |= io_put_kbufs(req, this_ret, req->buf_list, io_bundle_nbufs(kmsg, this_ret)); + cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret)); if (sr->flags & IORING_RECV_RETRY) cflags = req->cqe.flags | (cflags & CQE_F_MASK); if (sr->mshot_len && sel->val >= sr->mshot_len) @@ -896,7 +886,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, return false; } } else { - cflags |= io_put_kbuf(req, sel->val, req->buf_list); + cflags |= io_put_kbuf(req, sel->val, sel->buf_list); } /* @@ -1038,6 +1028,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) flags |= MSG_DONTWAIT; retry_multishot: + sel.buf_list = NULL; if (io_do_buffer_select(req)) { size_t len = sr->len; @@ -1048,7 +1039,7 @@ retry_multishot: if (req->flags & REQ_F_APOLL_MULTISHOT) { ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len); if (ret) { - io_kbuf_recycle(req, req->buf_list, issue_flags); + io_kbuf_recycle(req, sel.buf_list, issue_flags); return ret; } } @@ -1072,14 +1063,12 @@ retry_multishot: if (ret < min_ret) { if (ret == -EAGAIN && force_nonblock) { - if (issue_flags & IO_URING_F_MULTISHOT) - io_kbuf_recycle(req, req->buf_list, issue_flags); - + io_kbuf_recycle(req, sel.buf_list, issue_flags); return IOU_RETRY; } if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; - return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); + return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -1093,7 +1082,7 @@ retry_multishot: else if (sr->done_io) ret = sr->done_io; else - io_kbuf_recycle(req, req->buf_list, issue_flags); + io_kbuf_recycle(req, sel.buf_list, issue_flags); sel.val = ret; if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) @@ -1178,7 +1167,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr *kmsg = req->async_data; - struct io_br_sel sel = { }; + struct io_br_sel sel; struct socket *sock; unsigned flags; int ret, min_ret = 0; @@ -1198,6 +1187,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) flags |= MSG_DONTWAIT; retry_multishot: + sel.buf_list = NULL; if (io_do_buffer_select(req)) { sel.val = sr->len; ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); @@ -1217,16 +1207,14 @@ retry_multishot: ret = sock_recvmsg(sock, &kmsg->msg, flags); if (ret < min_ret) { if (ret == -EAGAIN && force_nonblock) { - if (issue_flags & IO_URING_F_MULTISHOT) - io_kbuf_recycle(req, req->buf_list, issue_flags); - + io_kbuf_recycle(req, sel.buf_list, issue_flags); return IOU_RETRY; } if (ret > 0 && io_net_retry(sock, flags)) { sr->len -= ret; sr->buf += ret; sr->done_io += ret; - return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); + return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); } if (ret == -ERESTARTSYS) ret = -EINTR; @@ -1242,7 +1230,7 @@ out_free: else if (sr->done_io) ret = sr->done_io; else - io_kbuf_recycle(req, req->buf_list, issue_flags); + io_kbuf_recycle(req, sel.buf_list, issue_flags); sel.val = ret; if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) diff --git a/io_uring/poll.c b/io_uring/poll.c index 07ab22380c78..f3852bf7627b 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -316,10 +316,10 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) ret = io_poll_check_events(req, tw); if (ret == IOU_POLL_NO_ACTION) { - io_kbuf_recycle(req, req->buf_list, 0); + io_kbuf_recycle(req, NULL, 0); return; } else if (ret == IOU_POLL_REQUEUE) { - io_kbuf_recycle(req, req->buf_list, 0); + io_kbuf_recycle(req, NULL, 0); __io_poll_execute(req, 0); return; } @@ -686,7 +686,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) req->flags |= REQ_F_POLLED; ipt.pt._qproc = io_async_queue_proc; - io_kbuf_recycle(req, req->buf_list, issue_flags); + io_kbuf_recycle(req, NULL, issue_flags); ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); if (ret) diff --git a/io_uring/rw.c b/io_uring/rw.c index 2b106f644383..906e869d330a 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -579,7 +579,7 @@ void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw) io_req_io_end(req); if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) - req->cqe.flags |= io_put_kbuf(req, req->cqe.res, req->buf_list); + req->cqe.flags |= io_put_kbuf(req, req->cqe.res, NULL); io_req_rw_cleanup(req, 0); io_req_task_complete(req, tw); @@ -648,7 +648,7 @@ static inline void io_rw_done(struct io_kiocb *req, ssize_t ret) } static int kiocb_done(struct io_kiocb *req, ssize_t ret, - unsigned int issue_flags) + struct io_br_sel *sel, unsigned int issue_flags) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); unsigned final_ret = io_fixup_rw_res(req, ret); @@ -662,7 +662,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, * from the submission path. */ io_req_io_end(req); - io_req_set_res(req, final_ret, io_put_kbuf(req, ret, req->buf_list)); + io_req_set_res(req, final_ret, io_put_kbuf(req, ret, sel->buf_list)); io_req_rw_cleanup(req, issue_flags); return IOU_COMPLETE; } else { @@ -1024,10 +1024,10 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = __io_read(req, &sel, issue_flags); if (ret >= 0) - return kiocb_done(req, ret, issue_flags); + return kiocb_done(req, ret, &sel, issue_flags); if (req->flags & REQ_F_BUFFERS_COMMIT) - io_kbuf_recycle(req, req->buf_list, issue_flags); + io_kbuf_recycle(req, sel.buf_list, issue_flags); return ret; } @@ -1057,15 +1057,15 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) * Reset rw->len to 0 again to avoid clamping future mshot * reads, in case the buffer size varies. */ - if (io_kbuf_recycle(req, req->buf_list, issue_flags)) + if (io_kbuf_recycle(req, sel.buf_list, issue_flags)) rw->len = 0; return IOU_RETRY; } else if (ret <= 0) { - io_kbuf_recycle(req, req->buf_list, issue_flags); + io_kbuf_recycle(req, sel.buf_list, issue_flags); if (ret < 0) req_set_fail(req); } else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { - cflags = io_put_kbuf(req, ret, req->buf_list); + cflags = io_put_kbuf(req, ret, sel.buf_list); } else { /* * Any successful return value will keep the multishot read @@ -1073,7 +1073,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) * we fail to post a CQE, or multishot is no longer set, then * jump to the termination path. This request is then done. */ - cflags = io_put_kbuf(req, ret, req->buf_list); + cflags = io_put_kbuf(req, ret, sel.buf_list); rw->len = 0; /* similarly to above, reset len to 0 */ if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { @@ -1202,7 +1202,7 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags) return -EAGAIN; } done: - return kiocb_done(req, ret2, issue_flags); + return kiocb_done(req, ret2, NULL, issue_flags); } else { ret_eagain: iov_iter_restore(&io->iter, &io->iter_state); @@ -1370,7 +1370,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (!smp_load_acquire(&req->iopoll_completed)) break; nr_events++; - req->cqe.flags = io_put_kbuf(req, req->cqe.res, req->buf_list); + req->cqe.flags = io_put_kbuf(req, req->cqe.res, NULL); if (req->opcode != IORING_OP_URING_CMD) io_req_rw_cleanup(req, 0); } -- cgit v1.2.3 From e973837b54024f070b2b48c7ee9725548548257a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:40 -0600 Subject: io_uring: remove async/poll related provided buffer recycles These aren't necessary anymore, get rid of them. Link: https://lore.kernel.org/r/20250821020750.598432-13-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 -- io_uring/poll.c | 4 ---- 2 files changed, 6 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 985b4681e513..5166f11f07c7 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2025,11 +2025,9 @@ fail: switch (io_arm_poll_handler(req, 0)) { case IO_APOLL_READY: - io_kbuf_recycle(req, NULL, 0); io_req_task_queue(req); break; case IO_APOLL_ABORTED: - io_kbuf_recycle(req, NULL, 0); io_queue_iowq(req); break; case IO_APOLL_OK: diff --git a/io_uring/poll.c b/io_uring/poll.c index f3852bf7627b..ea75c5cd81a0 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -316,10 +316,8 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) ret = io_poll_check_events(req, tw); if (ret == IOU_POLL_NO_ACTION) { - io_kbuf_recycle(req, NULL, 0); return; } else if (ret == IOU_POLL_REQUEUE) { - io_kbuf_recycle(req, NULL, 0); __io_poll_execute(req, 0); return; } @@ -686,8 +684,6 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) req->flags |= REQ_F_POLLED; ipt.pt._qproc = io_async_queue_proc; - io_kbuf_recycle(req, NULL, issue_flags); - ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); if (ret) return ret > 0 ? IO_APOLL_READY : IO_APOLL_ABORTED; -- cgit v1.2.3 From fe524b06843c19cf8d0025b644d56c4c31e60bc9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 20 Aug 2025 20:03:41 -0600 Subject: io_uring/kbuf: check for ring provided buffers first in recycling This is the most likely of paths if a provided buffer is used, so offer it up first and push the legacy buffers to later. Link: https://lore.kernel.org/r/20250821020750.598432-14-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/kbuf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'io_uring') diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index 1a539969fc9c..32f73adbe1e9 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -129,10 +129,10 @@ static inline bool io_kbuf_recycle(struct io_kiocb *req, struct io_buffer_list * { if (req->flags & REQ_F_BL_NO_RECYCLE) return false; - if (req->flags & REQ_F_BUFFER_SELECTED) - return io_kbuf_recycle_legacy(req, issue_flags); if (req->flags & REQ_F_BUFFER_RING) return io_kbuf_recycle_ring(req, bl); + if (req->flags & REQ_F_BUFFER_SELECTED) + return io_kbuf_recycle_legacy(req, issue_flags); return false; } -- cgit v1.2.3 From d589bcddaa3f8b1668499c3f0466863df3abe37a Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 21 Aug 2025 12:02:06 +0800 Subject: io-uring: move `struct io_br_sel` into io_uring_types.h Move `struct io_br_sel` into io_uring_types.h and prepare for supporting provided buffer on uring_cmd. Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20250821040210.1152145-2-ming.lei@redhat.com Signed-off-by: Jens Axboe --- io_uring/kbuf.h | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'io_uring') diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index 32f73adbe1e9..ada382ff38d7 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -62,24 +62,6 @@ struct buf_sel_arg { unsigned short partial_map; }; -/* - * Return value from io_buffer_list selection, to avoid stashing it in - * struct io_kiocb. For legacy/classic provided buffers, keeping a reference - * across execution contexts are fine. But for ring provided buffers, the - * list may go away as soon as ->uring_lock is dropped. As the io_kiocb - * persists, it's better to just keep the buffer local for those cases. - */ -struct io_br_sel { - struct io_buffer_list *buf_list; - /* - * Some selection parts return the user address, others return an error. - */ - union { - void __user *addr; - ssize_t val; - }; -}; - struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len, unsigned buf_group, unsigned int issue_flags); int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, -- cgit v1.2.3 From 620a50c927004f5c9420a7ca9b1a55673dbf3941 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 21 Aug 2025 12:02:07 +0800 Subject: io_uring: uring_cmd: add multishot support Add UAPI flag IORING_URING_CMD_MULTISHOT for supporting multishot uring_cmd operations with provided buffer. This enables drivers to post multiple completion events from a single uring_cmd submission, which is useful for: - Notifying userspace of device events (e.g., interrupt handling) - Supporting devices with multiple event sources (e.g., multi-queue devices) - Avoiding the need for device poll() support when events originate from multiple sources device-wide The implementation adds two new APIs: - io_uring_cmd_select_buffer(): selects a buffer from the provided buffer group for multishot uring_cmd - io_uring_mshot_cmd_post_cqe(): posts a CQE after event data is pushed to the provided buffer Multishot uring_cmd must be used with buffer select (IOSQE_BUFFER_SELECT) and is mutually exclusive with IORING_URING_CMD_FIXED for now. The ublk driver will be the first user of this functionality: https://github.com/ming1/linux/commits/ublk-devel/ Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20250821040210.1152145-3-ming.lei@redhat.com [axboe: fold in fix for !CONFIG_IO_URING] Signed-off-by: Jens Axboe --- io_uring/opdef.c | 1 + io_uring/uring_cmd.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 9568785810d9..932319633eac 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -413,6 +413,7 @@ const struct io_issue_def io_issue_defs[] = { #endif }, [IORING_OP_URING_CMD] = { + .buffer_select = 1, .needs_file = 1, .plug = 1, .iopoll = 1, diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 053bac89b6c0..3cfb5d51b88a 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -11,6 +11,7 @@ #include "io_uring.h" #include "alloc_cache.h" #include "rsrc.h" +#include "kbuf.h" #include "uring_cmd.h" #include "poll.h" @@ -194,8 +195,21 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (ioucmd->flags & ~IORING_URING_CMD_MASK) return -EINVAL; - if (ioucmd->flags & IORING_URING_CMD_FIXED) + if (ioucmd->flags & IORING_URING_CMD_FIXED) { + if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) + return -EINVAL; req->buf_index = READ_ONCE(sqe->buf_index); + } + + if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) { + if (ioucmd->flags & IORING_URING_CMD_FIXED) + return -EINVAL; + if (!(req->flags & REQ_F_BUFFER_SELECT)) + return -EINVAL; + } else { + if (req->flags & REQ_F_BUFFER_SELECT) + return -EINVAL; + } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); @@ -251,6 +265,10 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) } ret = file->f_op->uring_cmd(ioucmd, issue_flags); + if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) { + if (ret >= 0) + return IOU_ISSUE_SKIP_COMPLETE; + } if (ret == -EAGAIN) { ioucmd->flags |= IORING_URING_CMD_REISSUE; return ret; @@ -333,3 +351,54 @@ bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd, return false; return io_req_post_cqe32(req, cqe); } + +/* + * Work with io_uring_mshot_cmd_post_cqe() together for committing the + * provided buffer upfront + */ +struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, + unsigned buf_group, size_t *len, + unsigned int issue_flags) +{ + struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); + + if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT)) + return (struct io_br_sel) { .val = -EINVAL }; + + if (WARN_ON_ONCE(!io_do_buffer_select(req))) + return (struct io_br_sel) { .val = -EINVAL }; + + return io_buffer_select(req, len, buf_group, issue_flags); +} +EXPORT_SYMBOL_GPL(io_uring_cmd_buffer_select); + +/* + * Return true if this multishot uring_cmd needs to be completed, otherwise + * the event CQE is posted successfully. + * + * This function must use `struct io_br_sel` returned from + * io_uring_cmd_buffer_select() for committing the buffer in the same + * uring_cmd submission context. + */ +bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, + struct io_br_sel *sel, unsigned int issue_flags) +{ + struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); + unsigned int cflags = 0; + + if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT)) + return true; + + if (sel->val > 0) { + cflags = io_put_kbuf(req, sel->val, sel->buf_list); + if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) + return false; + } + + io_kbuf_recycle(req, sel->buf_list, issue_flags); + if (sel->val < 0) + req_set_fail(req); + io_req_set_res(req, sel->val, cflags); + return true; +} +EXPORT_SYMBOL_GPL(io_uring_mshot_cmd_post_cqe); -- cgit v1.2.3 From 3484f530f8d9da08b71d1e604dcd4ab8868d9919 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 21 Aug 2025 10:33:07 -0600 Subject: io_uring/cmd: deduplicate uring_cmd_flags checks io_uring_cmd_prep() currently has two checks for whether IORING_URING_CMD_FIXED and IORING_URING_CMD_MULTISHOT are both set in uring_cmd_flags. Remove the second check. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20250821163308.977915-3-csander@purestorage.com Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'io_uring') diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 3cfb5d51b88a..c8fd204f6892 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -202,8 +202,6 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) { - if (ioucmd->flags & IORING_URING_CMD_FIXED) - return -EINVAL; if (!(req->flags & REQ_F_BUFFER_SELECT)) return -EINVAL; } else { -- cgit v1.2.3 From e5c717e7953b688049cdc2a2a474e4905e0da3c0 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 21 Aug 2025 10:33:08 -0600 Subject: io_uring/cmd: consolidate REQ_F_BUFFER_SELECT checks io_uring_cmd_prep() checks that REQ_F_BUFFER_SELECT is set in the io_kiocb's flags iff IORING_URING_CMD_MULTISHOT is set in the SQE's uring_cmd_flags. Consolidate the IORING_URING_CMD_MULTISHOT and !IORING_URING_CMD_MULTISHOT branches into a single check that the IORING_URING_CMD_MULTISHOT flag matches the REQ_F_BUFFER_SELECT flag. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20250821163308.977915-4-csander@purestorage.com Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'io_uring') diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index c8fd204f6892..482cc5be1f8d 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -201,13 +201,9 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->buf_index = READ_ONCE(sqe->buf_index); } - if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) { - if (!(req->flags & REQ_F_BUFFER_SELECT)) - return -EINVAL; - } else { - if (req->flags & REQ_F_BUFFER_SELECT) - return -EINVAL; - } + if (!!(ioucmd->flags & IORING_URING_CMD_MULTISHOT) != + !!(req->flags & REQ_F_BUFFER_SELECT)) + return -EINVAL; ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); -- cgit v1.2.3 From 82ceb7fcc5fff5377b24c45c9b3c06bc98f91b55 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Aug 2025 14:09:57 -0600 Subject: io_uring/fdinfo: handle mixed sized CQEs Ensure that the CQ ring iteration handles differently sized CQEs, not just a fixed 16b or 32b size per ring. These CQEs aren't possible just yet, but prepare the fdinfo CQ ring dumping for handling them. Signed-off-by: Jens Axboe --- io_uring/fdinfo.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'io_uring') diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 9798d6fb4ec7..5c7339838769 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -65,15 +65,12 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) unsigned int sq_tail = READ_ONCE(r->sq.tail); unsigned int cq_head = READ_ONCE(r->cq.head); unsigned int cq_tail = READ_ONCE(r->cq.tail); - unsigned int cq_shift = 0; unsigned int sq_shift = 0; - unsigned int sq_entries, cq_entries; + unsigned int sq_entries; int sq_pid = -1, sq_cpu = -1; u64 sq_total_time = 0, sq_work_time = 0; unsigned int i; - if (ctx->flags & IORING_SETUP_CQE32) - cq_shift = 1; if (ctx->flags & IORING_SETUP_SQE128) sq_shift = 1; @@ -125,18 +122,23 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) seq_printf(m, "\n"); } seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head); - cq_entries = min(cq_tail - cq_head, ctx->cq_entries); - for (i = 0; i < cq_entries; i++) { - unsigned int entry = i + cq_head; - struct io_uring_cqe *cqe = &r->cqes[(entry & cq_mask) << cq_shift]; + while (cq_head < cq_tail) { + struct io_uring_cqe *cqe; + bool cqe32 = false; + cqe = &r->cqes[(cq_head & cq_mask)]; + if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) + cqe32 = true; seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x", - entry & cq_mask, cqe->user_data, cqe->res, + cq_head & cq_mask, cqe->user_data, cqe->res, cqe->flags); - if (cq_shift) + if (cqe32) seq_printf(m, ", extra1:%llu, extra2:%llu\n", cqe->big_cqe[0], cqe->big_cqe[1]); seq_printf(m, "\n"); + cq_head++; + if (cqe32) + cq_head++; } if (ctx->flags & IORING_SETUP_SQPOLL) { -- cgit v1.2.3 From e26dca67fde194340582cfbb0c0bf661825e9e46 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Aug 2025 14:14:41 -0600 Subject: io_uring: add support for IORING_SETUP_CQE_MIXED Normal rings support 16b CQEs for posting completions, while certain features require the ring to be configured with IORING_SETUP_CQE32, as they need to convey more information per completion. This, in turn, makes ALL the CQEs be 32b in size. This is somewhat wasteful and inefficient, particularly when only certain CQEs need to be of the bigger variant. This adds support for setting up a ring with mixed CQE sizes, using IORING_SETUP_CQE_MIXED. When setup in this mode, CQEs posted to the ring may be either 16b or 32b in size. If a CQE is 32b in size, then IORING_CQE_F_32 is set in the CQE flags to indicate that this is the case. If this flag isn't set, the CQE is the normal 16b variant. CQEs on these types of mixed rings may also have IORING_CQE_F_SKIP set. This can happen if the ring is one (small) CQE entry away from wrapping, and an attempt is made to post a 32b CQE. As CQEs must be contigious in the CQ ring, a 32b CQE cannot wrap the ring. For this case, a single dummy CQE is posted with the SKIP flag set. The application should simply ignore those. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 78 ++++++++++++++++++++++++++++++++++++++++++----------- io_uring/io_uring.h | 49 +++++++++++++++++++++++---------- io_uring/register.c | 3 ++- 3 files changed, 99 insertions(+), 31 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 5166f11f07c7..6c07efac977c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -620,27 +620,29 @@ static void io_cq_unlock_post(struct io_ring_ctx *ctx) static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool dying) { - size_t cqe_size = sizeof(struct io_uring_cqe); - lockdep_assert_held(&ctx->uring_lock); /* don't abort if we're dying, entries must get freed */ if (!dying && __io_cqring_events(ctx) == ctx->cq_entries) return; - if (ctx->flags & IORING_SETUP_CQE32) - cqe_size <<= 1; - io_cq_lock(ctx); while (!list_empty(&ctx->cq_overflow_list)) { + size_t cqe_size = sizeof(struct io_uring_cqe); struct io_uring_cqe *cqe; struct io_overflow_cqe *ocqe; + bool is_cqe32 = false; ocqe = list_first_entry(&ctx->cq_overflow_list, struct io_overflow_cqe, list); + if (ocqe->cqe.flags & IORING_CQE_F_32 || + ctx->flags & IORING_SETUP_CQE32) { + is_cqe32 = true; + cqe_size <<= 1; + } if (!dying) { - if (!io_get_cqe_overflow(ctx, &cqe, true)) + if (!io_get_cqe_overflow(ctx, &cqe, true, is_cqe32)) break; memcpy(cqe, &ocqe->cqe, cqe_size); } @@ -752,10 +754,12 @@ static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx, { struct io_overflow_cqe *ocqe; size_t ocq_size = sizeof(struct io_overflow_cqe); - bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32); + bool is_cqe32 = false; - if (is_cqe32) - ocq_size += sizeof(struct io_uring_cqe); + if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) { + is_cqe32 = true; + ocq_size <<= 1; + } ocqe = kzalloc(ocq_size, gfp | __GFP_ACCOUNT); trace_io_uring_cqe_overflow(ctx, cqe->user_data, cqe->res, cqe->flags, ocqe); @@ -773,12 +777,30 @@ static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx, return ocqe; } +/* + * Fill an empty dummy CQE, in case alignment is off for posting a 32b CQE + * because the ring is a single 16b entry away from wrapping. + */ +static bool io_fill_nop_cqe(struct io_ring_ctx *ctx, unsigned int off) +{ + if (__io_cqring_events(ctx) < ctx->cq_entries) { + struct io_uring_cqe *cqe = &ctx->rings->cqes[off]; + + cqe->user_data = 0; + cqe->res = 0; + cqe->flags = IORING_CQE_F_SKIP; + ctx->cached_cq_tail++; + return true; + } + return false; +} + /* * writes to the cq entry need to come after reading head; the * control dependency is enough as we're using WRITE_ONCE to * fill the cq entry */ -bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow) +bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32) { struct io_rings *rings = ctx->rings; unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1); @@ -792,12 +814,22 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow) if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))) return false; + /* + * Post dummy CQE if a 32b CQE is needed and there's only room for a + * 16b CQE before the ring wraps. + */ + if (cqe32 && off + 1 == ctx->cq_entries) { + if (!io_fill_nop_cqe(ctx, off)) + return false; + off = 0; + } + /* userspace may cheat modifying the tail, be safe and do min */ queued = min(__io_cqring_events(ctx), ctx->cq_entries); free = ctx->cq_entries - queued; /* we need a contiguous range, limit based on the current array offset */ len = min(free, ctx->cq_entries - off); - if (!len) + if (len < (cqe32 + 1)) return false; if (ctx->flags & IORING_SETUP_CQE32) { @@ -815,9 +847,9 @@ static bool io_fill_cqe_aux32(struct io_ring_ctx *ctx, { struct io_uring_cqe *cqe; - if (WARN_ON_ONCE(!(ctx->flags & IORING_SETUP_CQE32))) + if (WARN_ON_ONCE(!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)))) return false; - if (unlikely(!io_get_cqe(ctx, &cqe))) + if (unlikely(!io_get_cqe(ctx, &cqe, true))) return false; memcpy(cqe, src_cqe, 2 * sizeof(*cqe)); @@ -828,14 +860,15 @@ static bool io_fill_cqe_aux32(struct io_ring_ctx *ctx, static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags) { + bool cqe32 = cflags & IORING_CQE_F_32; struct io_uring_cqe *cqe; - if (likely(io_get_cqe(ctx, &cqe))) { + if (likely(io_get_cqe(ctx, &cqe, cqe32))) { WRITE_ONCE(cqe->user_data, user_data); WRITE_ONCE(cqe->res, res); WRITE_ONCE(cqe->flags, cflags); - if (ctx->flags & IORING_SETUP_CQE32) { + if (cqe32) { WRITE_ONCE(cqe->big_cqe[0], 0); WRITE_ONCE(cqe->big_cqe[1], 0); } @@ -2756,6 +2789,10 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries, if (check_shl_overflow(off, 1, &off)) return SIZE_MAX; } + if (flags & IORING_SETUP_CQE_MIXED) { + if (cq_entries < 2) + return SIZE_MAX; + } #ifdef CONFIG_SMP off = ALIGN(off, SMP_CACHE_BYTES); @@ -3680,6 +3717,14 @@ static int io_uring_sanitise_params(struct io_uring_params *p) !(flags & IORING_SETUP_SINGLE_ISSUER)) return -EINVAL; + /* + * Nonsensical to ask for CQE32 and mixed CQE support, it's not + * supported to post 16b CQEs on a ring setup with CQE32. + */ + if ((flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)) == + (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)) + return -EINVAL; + return 0; } @@ -3906,7 +3951,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY | - IORING_SETUP_NO_SQARRAY | IORING_SETUP_HYBRID_IOPOLL)) + IORING_SETUP_NO_SQARRAY | IORING_SETUP_HYBRID_IOPOLL | + IORING_SETUP_CQE_MIXED)) return -EINVAL; return io_uring_create(entries, &p, params); diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index abc6de227f74..2bcb565d9de6 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -75,7 +75,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq) unsigned long rings_size(unsigned int flags, unsigned int sq_entries, unsigned int cq_entries, size_t *sq_offset); int io_uring_fill_params(unsigned entries, struct io_uring_params *p); -bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow); +bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32); int io_run_task_work_sig(struct io_ring_ctx *ctx); void io_req_defer_failed(struct io_kiocb *req, s32 res); bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); @@ -169,25 +169,31 @@ static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) static inline bool io_get_cqe_overflow(struct io_ring_ctx *ctx, struct io_uring_cqe **ret, - bool overflow) + bool overflow, bool cqe32) { io_lockdep_assert_cq_locked(ctx); - if (unlikely(ctx->cqe_cached >= ctx->cqe_sentinel)) { - if (unlikely(!io_cqe_cache_refill(ctx, overflow))) + if (unlikely(ctx->cqe_sentinel - ctx->cqe_cached < (cqe32 + 1))) { + if (unlikely(!io_cqe_cache_refill(ctx, overflow, cqe32))) return false; } *ret = ctx->cqe_cached; ctx->cached_cq_tail++; ctx->cqe_cached++; - if (ctx->flags & IORING_SETUP_CQE32) + if (ctx->flags & IORING_SETUP_CQE32) { + ctx->cqe_cached++; + } else if (cqe32 && ctx->flags & IORING_SETUP_CQE_MIXED) { ctx->cqe_cached++; + ctx->cached_cq_tail++; + } + WARN_ON_ONCE(ctx->cqe_cached > ctx->cqe_sentinel); return true; } -static inline bool io_get_cqe(struct io_ring_ctx *ctx, struct io_uring_cqe **ret) +static inline bool io_get_cqe(struct io_ring_ctx *ctx, struct io_uring_cqe **ret, + bool cqe32) { - return io_get_cqe_overflow(ctx, ret, false); + return io_get_cqe_overflow(ctx, ret, false, cqe32); } static inline bool io_defer_get_uncommited_cqe(struct io_ring_ctx *ctx, @@ -196,25 +202,24 @@ static inline bool io_defer_get_uncommited_cqe(struct io_ring_ctx *ctx, io_lockdep_assert_cq_locked(ctx); ctx->submit_state.cq_flush = true; - return io_get_cqe(ctx, cqe_ret); + return io_get_cqe(ctx, cqe_ret, ctx->flags & IORING_SETUP_CQE_MIXED); } static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, struct io_kiocb *req) { + bool is_cqe32 = req->cqe.flags & IORING_CQE_F_32; struct io_uring_cqe *cqe; /* - * If we can't get a cq entry, userspace overflowed the - * submission (by quite a lot). Increment the overflow count in - * the ring. + * If we can't get a cq entry, userspace overflowed the submission + * (by quite a lot). */ - if (unlikely(!io_get_cqe(ctx, &cqe))) + if (unlikely(!io_get_cqe(ctx, &cqe, is_cqe32))) return false; - memcpy(cqe, &req->cqe, sizeof(*cqe)); - if (ctx->flags & IORING_SETUP_CQE32) { + if (is_cqe32) { memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); memset(&req->big_cqe, 0, sizeof(req->big_cqe)); } @@ -239,6 +244,22 @@ static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) req->cqe.flags = cflags; } +static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx) +{ + if (ctx->flags & IORING_SETUP_CQE_MIXED) + return IORING_CQE_F_32; + return 0; +} + +static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags, + __u64 extra1, __u64 extra2) +{ + req->cqe.res = res; + req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx); + req->big_cqe.extra1 = extra1; + req->big_cqe.extra2 = extra2; +} + static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache, struct io_kiocb *req) { diff --git a/io_uring/register.c b/io_uring/register.c index a59589249fce..a1a9b2884eae 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -396,7 +396,8 @@ static void io_register_free_rings(struct io_ring_ctx *ctx, #define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP) #define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \ - IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP) + IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \ + IORING_SETUP_CQE_MIXED) static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) { -- cgit v1.2.3 From 806ecb209aa86fcc1d92bc9f10323cf773f64d6d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Aug 2025 14:22:16 -0600 Subject: io_uring/nop: add support for IORING_SETUP_CQE_MIXED This adds support for setting IORING_NOP_CQE32 as a flag for a NOP command, in which case a 32b CQE will be posted rather than a regular one. This is the default if the ring has been setup with IORING_SETUP_CQE32. If the ring has been setup with IORING_SETUP_CQE_MIXED, then 16b CQEs will be posted without this flag set, and 32b CQEs if this flag is set. For the latter case, sqe->off is what will be posted as cqe->big_cqe[0] and sqe->addr is what will be posted as cqe->big_cqe[1]. Signed-off-by: Jens Axboe --- io_uring/nop.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'io_uring') diff --git a/io_uring/nop.c b/io_uring/nop.c index 20ed0f85b1c2..3caf07878f8a 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -17,11 +17,13 @@ struct io_nop { int result; int fd; unsigned int flags; + __u64 extra1; + __u64 extra2; }; #define NOP_FLAGS (IORING_NOP_INJECT_RESULT | IORING_NOP_FIXED_FILE | \ IORING_NOP_FIXED_BUFFER | IORING_NOP_FILE | \ - IORING_NOP_TW) + IORING_NOP_TW | IORING_NOP_CQE32) int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { @@ -41,6 +43,14 @@ int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) nop->fd = -1; if (nop->flags & IORING_NOP_FIXED_BUFFER) req->buf_index = READ_ONCE(sqe->buf_index); + if (nop->flags & IORING_NOP_CQE32) { + struct io_ring_ctx *ctx = req->ctx; + + if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) + return -EINVAL; + nop->extra1 = READ_ONCE(sqe->off); + nop->extra2 = READ_ONCE(sqe->addr); + } return 0; } @@ -68,7 +78,10 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) done: if (ret < 0) req_set_fail(req); - io_req_set_res(req, nop->result, 0); + if (nop->flags & IORING_NOP_CQE32) + io_req_set_res32(req, nop->result, 0, nop->extra1, nop->extra2); + else + io_req_set_res(req, nop->result, 0); if (nop->flags & IORING_NOP_TW) { req->io_task_work.func = io_req_task_complete; io_req_task_work_add(req); -- cgit v1.2.3 From 1e81bf1414127bfeab0a0bcd9f39f42ccb96b6b9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Aug 2025 14:24:18 -0600 Subject: io_uring/uring_cmd: add support for IORING_SETUP_CQE_MIXED Certain users of uring_cmd currently require fixed 32b CQE support, which is propagated through IO_URING_F_CQE32. Allow IORING_SETUP_CQE_MIXED to cover that case as well, so not all CQEs posted need to be 32b in size. Signed-off-by: Jens Axboe --- io_uring/cmd_net.c | 3 ++- io_uring/uring_cmd.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'io_uring') diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c index 3866fe6ff541..27a09aa4c9d0 100644 --- a/io_uring/cmd_net.c +++ b/io_uring/cmd_net.c @@ -4,6 +4,7 @@ #include #include "uring_cmd.h" +#include "io_uring.h" static inline int io_uring_cmd_getsockopt(struct socket *sock, struct io_uring_cmd *cmd, @@ -73,7 +74,7 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, cqe->user_data = 0; cqe->res = tskey; - cqe->flags = IORING_CQE_F_MORE; + cqe->flags = IORING_CQE_F_MORE | ctx_cqe32_flags(cmd_to_io_kiocb(cmd)->ctx); cqe->flags |= tstype << IORING_TIMESTAMP_TYPE_SHIFT; if (ret == SOF_TIMESTAMPING_TX_HARDWARE) cqe->flags |= IORING_CQE_F_TSTAMP_HW; diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 482cc5be1f8d..ff1d029633b8 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -242,7 +242,7 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) if (ctx->flags & IORING_SETUP_SQE128) issue_flags |= IO_URING_F_SQE128; - if (ctx->flags & IORING_SETUP_CQE32) + if (ctx->flags & (IORING_SETUP_CQE32 | IORING_SETUP_CQE_MIXED)) issue_flags |= IO_URING_F_CQE32; if (io_is_compat(ctx)) issue_flags |= IO_URING_F_COMPAT; -- cgit v1.2.3 From c986f7586b8d3381ab0ce764136c3b951c618381 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 7 Aug 2025 14:25:50 -0600 Subject: io_uring/zcrx: add support for IORING_SETUP_CQE_MIXED zcrx currently requires the ring to be set up with fixed 32b CQEs, allow it to use IORING_SETUP_CQE_MIXED as well. Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index e5ff49f3425e..51fd2350dbe9 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -554,8 +554,9 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, return -EPERM; /* mandatory io_uring features for zc rx */ - if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN && - ctx->flags & IORING_SETUP_CQE32)) + if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) + return -EINVAL; + if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) return -EINVAL; if (copy_from_user(®, arg, sizeof(reg))) return -EFAULT; @@ -920,17 +921,20 @@ static const struct memory_provider_ops io_uring_pp_zc_ops = { static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, struct io_zcrx_ifq *ifq, int off, int len) { + struct io_ring_ctx *ctx = req->ctx; struct io_uring_zcrx_cqe *rcqe; struct io_zcrx_area *area; struct io_uring_cqe *cqe; u64 offset; - if (!io_defer_get_uncommited_cqe(req->ctx, &cqe)) + if (!io_defer_get_uncommited_cqe(ctx, &cqe)) return false; cqe->user_data = req->cqe.user_data; cqe->res = len; cqe->flags = IORING_CQE_F_MORE; + if (ctx->flags & IORING_SETUP_CQE_MIXED) + cqe->flags |= IORING_CQE_F_32; area = io_zcrx_iov_to_area(niov); offset = off + (net_iov_idx(niov) << PAGE_SHIFT); -- cgit v1.2.3 From 4c0b26e23c79ecf934a92b2d9a516bffbb61c3e4 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 22 Aug 2025 08:19:56 -0600 Subject: io_uring: add async data clear/free helpers Futex recently had an issue where it mishandled how ->async_data and REQ_F_ASYNC_DATA is handled. To avoid future issues like that, add a set of helpers that either clear or clear-and-free the async data assigned to a struct io_kiocb. Convert existing manual handling of that to use the helpers. No intended functional changes in this patch. Reviewed-by: Caleb Sander Mateos Signed-off-by: Jens Axboe --- io_uring/futex.c | 13 ++++--------- io_uring/io_uring.h | 13 +++++++++++++ io_uring/net.c | 6 ++---- io_uring/rw.c | 6 ++---- io_uring/uring_cmd.c | 3 +-- io_uring/waitid.c | 4 +--- 6 files changed, 23 insertions(+), 22 deletions(-) (limited to 'io_uring') diff --git a/io_uring/futex.c b/io_uring/futex.c index 9113a44984f3..64f3bd51c84c 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -43,7 +43,6 @@ void io_futex_cache_free(struct io_ring_ctx *ctx) static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw) { - req->async_data = NULL; hlist_del_init(&req->hash_node); io_req_task_complete(req, tw); } @@ -54,6 +53,7 @@ static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw) io_tw_lock(ctx, tw); io_cache_free(&ctx->futex_cache, req->async_data); + io_req_async_data_clear(req, 0); __io_futex_complete(req, tw); } @@ -72,8 +72,7 @@ static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw) io_req_set_res(req, res, 0); } - kfree(req->async_data); - req->flags &= ~REQ_F_ASYNC_DATA; + io_req_async_data_free(req); __io_futex_complete(req, tw); } @@ -232,9 +231,7 @@ int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) io_ring_submit_unlock(ctx, issue_flags); req_set_fail(req); io_req_set_res(req, ret, 0); - kfree(futexv); - req->async_data = NULL; - req->flags &= ~REQ_F_ASYNC_DATA; + io_req_async_data_free(req); return IOU_COMPLETE; } @@ -310,9 +307,7 @@ done: if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); - req->async_data = NULL; - req->flags &= ~REQ_F_ASYNC_DATA; - kfree(ifd); + io_req_async_data_free(req); return IOU_COMPLETE; } diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 2bcb565d9de6..fa8a66b34d4e 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -281,6 +281,19 @@ static inline bool req_has_async_data(struct io_kiocb *req) return req->flags & REQ_F_ASYNC_DATA; } +static inline void io_req_async_data_clear(struct io_kiocb *req, + io_req_flags_t extra_flags) +{ + req->flags &= ~(REQ_F_ASYNC_DATA|extra_flags); + req->async_data = NULL; +} + +static inline void io_req_async_data_free(struct io_kiocb *req) +{ + kfree(req->async_data); + io_req_async_data_clear(req, 0); +} + static inline void io_put_file(struct io_kiocb *req) { if (!(req->flags & REQ_F_FIXED_FILE) && req->file) diff --git a/io_uring/net.c b/io_uring/net.c index b00cd2f59091..d2ca49ceb79d 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -178,10 +178,8 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP) io_vec_free(&hdr->vec); - if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) { - req->async_data = NULL; - req->flags &= ~(REQ_F_ASYNC_DATA|REQ_F_NEED_CLEANUP); - } + if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) + io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); } static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req) diff --git a/io_uring/rw.c b/io_uring/rw.c index 906e869d330a..dcde5bb7421a 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -154,10 +154,8 @@ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP) io_vec_free(&rw->vec); - if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) { - req->async_data = NULL; - req->flags &= ~REQ_F_ASYNC_DATA; - } + if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) + io_req_async_data_clear(req, 0); } static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index ff1d029633b8..f5a2642bb407 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -37,8 +37,7 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) { ioucmd->sqe = NULL; - req->async_data = NULL; - req->flags &= ~(REQ_F_ASYNC_DATA|REQ_F_NEED_CLEANUP); + io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); } } diff --git a/io_uring/waitid.c b/io_uring/waitid.c index e07a94694397..26c118f3918d 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -37,9 +37,7 @@ static void io_waitid_free(struct io_kiocb *req) struct io_waitid_async *iwa = req->async_data; put_pid(iwa->wo.wo_pid); - kfree(req->async_data); - req->async_data = NULL; - req->flags &= ~REQ_F_ASYNC_DATA; + io_req_async_data_free(req); } static bool io_waitid_compat_copy_si(struct io_waitid *iw, int signo) -- cgit v1.2.3 From 37500634d0a8f931e15879760fb70f9b6f5d5370 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 2 Sep 2025 05:19:42 -0600 Subject: io_uring/net: correct type for min_not_zero() cast The kernel test robot reports that after a recent change, the signedness of a min_not_zero() compare is now incorrect. Fix that up and cast to the right type. Fixes: 429884ff35f7 ("io_uring/kbuf: use struct io_br_sel for multiple buffers picking") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202509020426.WJtrdwOU-lkp@intel.com/ Signed-off-by: Jens Axboe --- io_uring/net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/net.c b/io_uring/net.c index d2ca49ceb79d..226fea2312b5 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1118,7 +1118,7 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg if (sel->val) arg.max_len = sel->val; else if (kmsg->msg.msg_inq > 1) - arg.max_len = min_not_zero(sel->val, (size_t) kmsg->msg.msg_inq); + arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq); /* if mshot limited, ensure we don't go over */ if (sr->flags & IORING_RECV_MSHOT_LIM) -- cgit v1.2.3 From 8b9c9a2e7da11e50a1109a1f38bca0aecf25b185 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Tue, 2 Sep 2025 15:51:07 -0600 Subject: io_uring/register: drop redundant submitter_task check For IORING_SETUP_SINGLE_ISSUER io_ring_ctx's, io_register_resize_rings() checks that the current task is the ctx's submitter_task. However, its caller __io_uring_register() already checks this. Drop the redundant check in io_register_resize_rings(). Signed-off-by: Caleb Sander Mateos Link: https://lore.kernel.org/r/20250902215108.1925105-1-csander@purestorage.com Signed-off-by: Jens Axboe --- io_uring/register.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'io_uring') diff --git a/io_uring/register.c b/io_uring/register.c index a1a9b2884eae..aa5f56ad8358 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -408,10 +408,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) struct io_uring_params p; int ret; - /* for single issuer, must be owner resizing */ - if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && - current != ctx->submitter_task) - return -EEXIST; /* limited to DEFER_TASKRUN for now */ if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) return -EINVAL; -- cgit v1.2.3 From df3a7762ee24ba6a33d4215244e329ca300f4819 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Tue, 2 Sep 2025 10:06:56 -0600 Subject: io_uring/uring_cmd: add io_uring_cmd_tw_t type alias Introduce a function pointer type alias io_uring_cmd_tw_t for the uring_cmd task work callback. This avoids repeating the signature in several places. Also name both arguments to the callback to clarify what they represent. Signed-off-by: Caleb Sander Mateos Reviewed-by: Keith Busch Link: https://lore.kernel.org/r/20250902160657.1726828-1-csander@purestorage.com Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index f5a2642bb407..d76d6d27765c 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -126,7 +126,7 @@ static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw) } void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, - void (*task_work_cb)(struct io_uring_cmd *, unsigned), + io_uring_cmd_tw_t task_work_cb, unsigned flags) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); -- cgit v1.2.3 From dd386b0d5e61556927189cd7b59a628d22cb6851 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 1 Sep 2025 19:26:07 -0600 Subject: io_uring/uring_cmd: correct io_uring_cmd_done() ret type io_uring_cmd_done() takes the result code for the CQE as a ssize_t ret argument. However, the CQE res field is a s32 value, as is the argument to io_req_set_res(). To clarify that only s32 values can be faithfully represented without truncation, change io_uring_cmd_done()'s ret argument type to s32. Signed-off-by: Caleb Sander Mateos Link: https://lore.kernel.org/r/20250902012609.1513123-1-csander@purestorage.com Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index d76d6d27765c..5562e8491c5b 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -151,7 +151,7 @@ static inline void io_req_set_cqe32_extra(struct io_kiocb *req, * Called by consumers of io_uring_cmd, if they originally returned * -EIOCBQUEUED upon receiving the command. */ -void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, u64 res2, +void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, unsigned issue_flags) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); -- cgit v1.2.3 From c2685729fa5463e5fc493244a2ba7e260217fc76 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 4 Sep 2025 10:12:22 -0600 Subject: io_uring: remove WRITE_ONCE() in io_uring_create() There's no need to use WRITE_ONCE() to set ctx->submitter_task in io_uring_create() since no other task can access the io_ring_ctx until a file descriptor is associated with it. So use a normal assignment instead of WRITE_ONCE(). Signed-off-by: Caleb Sander Mateos Link: https://lore.kernel.org/r/20250904161223.2600435-1-csander@purestorage.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 6c07efac977c..20dfa5ef75dc 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3891,8 +3891,13 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, } if (ctx->flags & IORING_SETUP_SINGLE_ISSUER - && !(ctx->flags & IORING_SETUP_R_DISABLED)) - WRITE_ONCE(ctx->submitter_task, get_task_struct(current)); + && !(ctx->flags & IORING_SETUP_R_DISABLED)) { + /* + * Unlike io_register_enable_rings(), don't need WRITE_ONCE() + * since ctx isn't yet accessible from other tasks + */ + ctx->submitter_task = get_task_struct(current); + } file = io_uring_get_file(ctx); if (IS_ERR(file)) { -- cgit v1.2.3 From da8bc3c81c71eb8906dafca805db1a2639665116 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 8 Sep 2025 00:02:58 +0100 Subject: io_uring: add helper for *REGISTER_SEND_MSG_RING Move handling of IORING_REGISTER_SEND_MSG_RING into a separate function in preparation to growing io_uring_register_blind(). Signed-off-by: Pavel Begunkov Reviewed-by: Martin K. Petersen Signed-off-by: Jens Axboe --- io_uring/register.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) (limited to 'io_uring') diff --git a/io_uring/register.c b/io_uring/register.c index aa5f56ad8358..50ce87928be0 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -874,6 +874,23 @@ struct file *io_uring_register_get_file(unsigned int fd, bool registered) return ERR_PTR(-EOPNOTSUPP); } +static int io_uring_register_send_msg_ring(void __user *arg, unsigned int nr_args) +{ + struct io_uring_sqe sqe; + + if (!arg || nr_args != 1) + return -EINVAL; + if (copy_from_user(&sqe, arg, sizeof(sqe))) + return -EFAULT; + /* no flags supported */ + if (sqe.flags) + return -EINVAL; + if (sqe.opcode != IORING_OP_MSG_RING) + return -EINVAL; + + return io_uring_sync_msg_ring(&sqe); +} + /* * "blind" registration opcodes are ones where there's no ring given, and * hence the source fd must be -1. @@ -882,21 +899,9 @@ static int io_uring_register_blind(unsigned int opcode, void __user *arg, unsigned int nr_args) { switch (opcode) { - case IORING_REGISTER_SEND_MSG_RING: { - struct io_uring_sqe sqe; - - if (!arg || nr_args != 1) - return -EINVAL; - if (copy_from_user(&sqe, arg, sizeof(sqe))) - return -EFAULT; - /* no flags supported */ - if (sqe.flags) - return -EINVAL; - if (sqe.opcode == IORING_OP_MSG_RING) - return io_uring_sync_msg_ring(&sqe); - } + case IORING_REGISTER_SEND_MSG_RING: + return io_uring_register_send_msg_ring(arg, nr_args); } - return -EINVAL; } -- cgit v1.2.3 From 63805d0a9b9670ade00c4f49e4fe093668b31ba5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 8 Sep 2025 00:02:59 +0100 Subject: io_uring: add macros for avaliable flags Add constants for supported setup / request / feature flags as well as the feature mask. They'll be used in the next patch. Signed-off-by: Pavel Begunkov Reviewed-by: Martin K. Petersen Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 32 +++--------------------------- io_uring/io_uring.h | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 29 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 20dfa5ef75dc..252a0021cd43 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -108,9 +108,6 @@ #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \ IOSQE_IO_HARDLINK | IOSQE_ASYNC) -#define SQE_VALID_FLAGS (SQE_COMMON_FLAGS | IOSQE_BUFFER_SELECT | \ - IOSQE_IO_DRAIN | IOSQE_CQE_SKIP_SUCCESS) - #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK) #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ @@ -3462,12 +3459,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, struct file *file; long ret; - if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | - IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG | - IORING_ENTER_REGISTERED_RING | - IORING_ENTER_ABS_TIMER | - IORING_ENTER_EXT_ARG_REG | - IORING_ENTER_NO_IOWAIT))) + if (unlikely(flags & ~IORING_ENTER_FLAGS)) return -EINVAL; /* @@ -3875,15 +3867,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, if (ret) goto err; - p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | - IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | - IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | - IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED | - IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS | - IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP | - IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING | - IORING_FEAT_RECVSEND_BUNDLE | IORING_FEAT_MIN_TIMEOUT | - IORING_FEAT_RW_ATTR | IORING_FEAT_NO_IOWAIT; + p->features = IORING_FEAT_FLAGS; if (copy_to_user(params, p, sizeof(*p))) { ret = -EFAULT; @@ -3948,18 +3932,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) return -EINVAL; } - if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | - IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE | - IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ | - IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL | - IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | - IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | - IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | - IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY | - IORING_SETUP_NO_SQARRAY | IORING_SETUP_HYBRID_IOPOLL | - IORING_SETUP_CQE_MIXED)) + if (p.flags & ~IORING_SETUP_FLAGS) return -EINVAL; - return io_uring_create(entries, &p, params); } diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index fa8a66b34d4e..a1d8d69411ff 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -18,6 +18,63 @@ #include #endif +#define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ + IORING_FEAT_NODROP |\ + IORING_FEAT_SUBMIT_STABLE |\ + IORING_FEAT_RW_CUR_POS |\ + IORING_FEAT_CUR_PERSONALITY |\ + IORING_FEAT_FAST_POLL |\ + IORING_FEAT_POLL_32BITS |\ + IORING_FEAT_SQPOLL_NONFIXED |\ + IORING_FEAT_EXT_ARG |\ + IORING_FEAT_NATIVE_WORKERS |\ + IORING_FEAT_RSRC_TAGS |\ + IORING_FEAT_CQE_SKIP |\ + IORING_FEAT_LINKED_FILE |\ + IORING_FEAT_REG_REG_RING |\ + IORING_FEAT_RECVSEND_BUNDLE |\ + IORING_FEAT_MIN_TIMEOUT |\ + IORING_FEAT_RW_ATTR |\ + IORING_FEAT_NO_IOWAIT) + +#define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\ + IORING_SETUP_SQPOLL |\ + IORING_SETUP_SQ_AFF |\ + IORING_SETUP_CQSIZE |\ + IORING_SETUP_CLAMP |\ + IORING_SETUP_ATTACH_WQ |\ + IORING_SETUP_R_DISABLED |\ + IORING_SETUP_SUBMIT_ALL |\ + IORING_SETUP_COOP_TASKRUN |\ + IORING_SETUP_TASKRUN_FLAG |\ + IORING_SETUP_SQE128 |\ + IORING_SETUP_CQE32 |\ + IORING_SETUP_SINGLE_ISSUER |\ + IORING_SETUP_DEFER_TASKRUN |\ + IORING_SETUP_NO_MMAP |\ + IORING_SETUP_REGISTERED_FD_ONLY |\ + IORING_SETUP_NO_SQARRAY |\ + IORING_SETUP_HYBRID_IOPOLL |\ + IORING_SETUP_CQE_MIXED) + +#define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\ + IORING_ENTER_SQ_WAKEUP |\ + IORING_ENTER_SQ_WAIT |\ + IORING_ENTER_EXT_ARG |\ + IORING_ENTER_REGISTERED_RING |\ + IORING_ENTER_ABS_TIMER |\ + IORING_ENTER_EXT_ARG_REG |\ + IORING_ENTER_NO_IOWAIT) + + +#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE |\ + IOSQE_IO_DRAIN |\ + IOSQE_IO_LINK |\ + IOSQE_IO_HARDLINK |\ + IOSQE_ASYNC |\ + IOSQE_BUFFER_SELECT |\ + IOSQE_CQE_SKIP_SUCCESS) + enum { IOU_COMPLETE = 0, -- cgit v1.2.3 From c265ae75f900cea4e415230a77b5d152377627dd Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 8 Sep 2025 00:03:00 +0100 Subject: io_uring: introduce io_uring querying There are many parameters users might want to query about io_uring like available request types or the ring sizes. This patch introduces an interface for such slow path queries. It was written with several requirements in mind: - Can be used with or without an io_uring instance. Asking for supported setup flags before creating an instance as well as qeurying info about an already created ring are valid use cases. - Should be moderately fast. For example, users might use it to periodically retrieve ring attributes at runtime. As a consequence, it should be able to query multiple attributes in a single syscall. - Backward and forward compatible. - Should be reasobably easy to use. - Reduce the kernel code size for introducing new query types. It's implemented as a new registration opcode IORING_REGISTER_QUERY. The user passes one or more query strutctures linked together, each represented by struct io_uring_query_hdr. The header stores common control fields needed for processing and points to query type specific information. The header contains - The query type - The result field, which on return contains the error code for the query - Pointer to the query type specific information - The size of the query structure. The kernel will only populate up to the size, which helps with backward compatibility. The kernel can also reduce the size, so if the current kernel is older than the inteface the user tries to use, it'll get only the supported bits. - next_entry field is used to chain multiple queries. Apart from common registeration syscall failures, it can only immediately return an error code in case when the headers are incorrect or any other addresses and invalid. That usually mean that the userspace doesn't use the API right and should be corrected. All query type specific errors are returned in the header's result field. As an example, the patch adds a single query type for now, i.e. IO_URING_QUERY_OPCODES, which tells what register / request / etc. opcodes are supported, but there are particular plans to extend it. Note: there is a request probing interface via IORING_REGISTER_PROBE, but it's a mess. It requires the user to create a ring first, it only works for requests, and requires dynamic allocations. Reviewed-by: Martin K. Petersen Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/Makefile | 2 +- io_uring/query.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++ io_uring/query.h | 9 ++++++ io_uring/register.c | 6 ++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 io_uring/query.c create mode 100644 io_uring/query.h (limited to 'io_uring') diff --git a/io_uring/Makefile b/io_uring/Makefile index b3f1bd492804..bc4e4a3fa0a5 100644 --- a/io_uring/Makefile +++ b/io_uring/Makefile @@ -13,7 +13,7 @@ obj-$(CONFIG_IO_URING) += io_uring.o opdef.o kbuf.o rsrc.o notif.o \ sync.o msg_ring.o advise.o openclose.o \ statx.o timeout.o cancel.o \ waitid.o register.o truncate.o \ - memmap.o alloc_cache.o + memmap.o alloc_cache.o query.o obj-$(CONFIG_IO_URING_ZCRX) += zcrx.o obj-$(CONFIG_IO_WQ) += io-wq.o obj-$(CONFIG_FUTEX) += futex.o diff --git a/io_uring/query.c b/io_uring/query.c new file mode 100644 index 000000000000..9eed0f371956 --- /dev/null +++ b/io_uring/query.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "linux/io_uring/query.h" + +#include "query.h" +#include "io_uring.h" + +#define IO_MAX_QUERY_SIZE (sizeof(struct io_uring_query_opcode)) + +static ssize_t io_query_ops(void *data) +{ + struct io_uring_query_opcode *e = data; + + BUILD_BUG_ON(sizeof(*e) > IO_MAX_QUERY_SIZE); + + e->nr_request_opcodes = IORING_OP_LAST; + e->nr_register_opcodes = IORING_REGISTER_LAST; + e->feature_flags = IORING_FEAT_FLAGS; + e->ring_setup_flags = IORING_SETUP_FLAGS; + e->enter_flags = IORING_ENTER_FLAGS; + e->sqe_flags = SQE_VALID_FLAGS; + return sizeof(*e); +} + +static int io_handle_query_entry(struct io_ring_ctx *ctx, + void *data, void __user *uhdr, + u64 *next_entry) +{ + struct io_uring_query_hdr hdr; + size_t usize, res_size = 0; + ssize_t ret = -EINVAL; + void __user *udata; + + if (copy_from_user(&hdr, uhdr, sizeof(hdr))) + return -EFAULT; + usize = hdr.size; + hdr.size = min(hdr.size, IO_MAX_QUERY_SIZE); + udata = u64_to_user_ptr(hdr.query_data); + + if (hdr.query_op >= __IO_URING_QUERY_MAX) { + ret = -EOPNOTSUPP; + goto out; + } + if (!mem_is_zero(hdr.__resv, sizeof(hdr.__resv)) || hdr.result || !hdr.size) + goto out; + if (copy_from_user(data, udata, hdr.size)) + return -EFAULT; + + switch (hdr.query_op) { + case IO_URING_QUERY_OPCODES: + ret = io_query_ops(data); + break; + } + + if (ret >= 0) { + if (WARN_ON_ONCE(ret > IO_MAX_QUERY_SIZE)) + return -EFAULT; + res_size = ret; + ret = 0; + } +out: + hdr.result = ret; + hdr.size = min_t(size_t, usize, res_size); + + if (copy_struct_to_user(udata, usize, data, hdr.size, NULL)) + return -EFAULT; + if (copy_to_user(uhdr, &hdr, sizeof(hdr))) + return -EFAULT; + *next_entry = hdr.next_entry; + return 0; +} + +int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) +{ + char entry_buffer[IO_MAX_QUERY_SIZE]; + void __user *uhdr = arg; + int ret; + + memset(entry_buffer, 0, sizeof(entry_buffer)); + + if (nr_args) + return -EINVAL; + + while (uhdr) { + u64 next_hdr; + + ret = io_handle_query_entry(ctx, entry_buffer, uhdr, &next_hdr); + if (ret) + return ret; + uhdr = u64_to_user_ptr(next_hdr); + } + return 0; +} diff --git a/io_uring/query.h b/io_uring/query.h new file mode 100644 index 000000000000..171d47ccaaba --- /dev/null +++ b/io_uring/query.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef IORING_QUERY_H +#define IORING_QUERY_H + +#include + +int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args); + +#endif diff --git a/io_uring/register.c b/io_uring/register.c index 50ce87928be0..9c31a8afb83d 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -31,6 +31,7 @@ #include "msg_ring.h" #include "memmap.h" #include "zcrx.h" +#include "query.h" #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) @@ -832,6 +833,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, break; ret = io_register_mem_region(ctx, arg); break; + case IORING_REGISTER_QUERY: + ret = io_query(ctx, arg, nr_args); + break; default: ret = -EINVAL; break; @@ -901,6 +905,8 @@ static int io_uring_register_blind(unsigned int opcode, void __user *arg, switch (opcode) { case IORING_REGISTER_SEND_MSG_RING: return io_uring_register_send_msg_ring(arg, nr_args); + case IORING_REGISTER_QUERY: + return io_query(NULL, arg, nr_args); } return -EINVAL; } -- cgit v1.2.3 From 7b0604d77a41192316347618cce1d9c795613adb Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Fri, 5 Sep 2025 12:18:17 +0200 Subject: io_uring: Replace kzalloc() + copy_from_user() with memdup_user() Replace kzalloc() followed by copy_from_user() with memdup_user() to improve and simplify io_probe(). No functional changes intended. Signed-off-by: Thorsten Blum Signed-off-by: Jens Axboe --- io_uring/register.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'io_uring') diff --git a/io_uring/register.c b/io_uring/register.c index 9c31a8afb83d..f4c76db27683 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -47,13 +47,9 @@ static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, nr_args = IORING_OP_LAST; size = struct_size(p, ops, nr_args); - p = kzalloc(size, GFP_KERNEL); - if (!p) - return -ENOMEM; - - ret = -EFAULT; - if (copy_from_user(p, arg, size)) - goto out; + p = memdup_user(arg, size); + if (IS_ERR(p)) + return PTR_ERR(p); ret = -EINVAL; if (memchr_inv(p, 0, size)) goto out; -- cgit v1.2.3 From 5d4c52bfa8cdc1dc1ff701246e662be3f43a3fe1 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 4 Sep 2025 11:08:58 -0600 Subject: io_uring: don't include filetable.h in io_uring.h io_uring/io_uring.h doesn't use anything declared in io_uring/filetable.h, so drop the unnecessary #include. Add filetable.h includes in .c files previously relying on the transitive include from io_uring.h. Signed-off-by: Caleb Sander Mateos Signed-off-by: Jens Axboe --- io_uring/cancel.c | 1 + io_uring/fdinfo.c | 2 +- io_uring/io_uring.c | 1 + io_uring/io_uring.h | 1 - io_uring/net.c | 1 + io_uring/openclose.c | 1 + io_uring/register.c | 1 + io_uring/rsrc.c | 1 + io_uring/rw.c | 1 + io_uring/splice.c | 1 + 10 files changed, 9 insertions(+), 2 deletions(-) (limited to 'io_uring') diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 6d57602304df..64b51e82baa2 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -11,6 +11,7 @@ #include +#include "filetable.h" #include "io_uring.h" #include "tctx.h" #include "poll.h" diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 5c7339838769..ff3364531c77 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -9,7 +9,7 @@ #include -#include "io_uring.h" +#include "filetable.h" #include "sqpoll.h" #include "fdinfo.h" #include "cancel.h" diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 252a0021cd43..347403862487 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -79,6 +79,7 @@ #include "io-wq.h" +#include "filetable.h" #include "io_uring.h" #include "opdef.h" #include "refs.h" diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index a1d8d69411ff..e92099d8fcb7 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -11,7 +11,6 @@ #include "alloc_cache.h" #include "io-wq.h" #include "slist.h" -#include "filetable.h" #include "opdef.h" #ifndef CREATE_TRACE_POINTS diff --git a/io_uring/net.c b/io_uring/net.c index 226fea2312b5..f99b90c762fc 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -10,6 +10,7 @@ #include +#include "filetable.h" #include "io_uring.h" #include "kbuf.h" #include "alloc_cache.h" diff --git a/io_uring/openclose.c b/io_uring/openclose.c index d70700e5cef8..bfeb91b31bba 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -14,6 +14,7 @@ #include "../fs/internal.h" +#include "filetable.h" #include "io_uring.h" #include "rsrc.h" #include "openclose.h" diff --git a/io_uring/register.c b/io_uring/register.c index f4c76db27683..96e9cac12823 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -18,6 +18,7 @@ #include #include +#include "filetable.h" #include "io_uring.h" #include "opdef.h" #include "tctx.h" diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index f75f5e43fa4a..2d15b8785a95 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -13,6 +13,7 @@ #include +#include "filetable.h" #include "io_uring.h" #include "openclose.h" #include "rsrc.h" diff --git a/io_uring/rw.c b/io_uring/rw.c index dcde5bb7421a..ab6b4afccec3 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -15,6 +15,7 @@ #include +#include "filetable.h" #include "io_uring.h" #include "opdef.h" #include "kbuf.h" diff --git a/io_uring/splice.c b/io_uring/splice.c index 35ce4e60b495..e81ebbb91925 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -11,6 +11,7 @@ #include +#include "filetable.h" #include "io_uring.h" #include "splice.h" -- cgit v1.2.3 From 2f076a453f75de691a081c89bce31b530153d53b Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 4 Sep 2025 11:08:59 -0600 Subject: io_uring/rsrc: respect submitter_task in io_register_clone_buffers() io_ring_ctx's enabled with IORING_SETUP_SINGLE_ISSUER are only allowed a single task submitting to the ctx. Although the documentation only mentions this restriction applying to io_uring_enter() syscalls, commit d7cce96c449e ("io_uring: limit registration w/ SINGLE_ISSUER") extends it to io_uring_register(). Ensuring only one task interacts with the io_ring_ctx will be important to allow this task to avoid taking the uring_lock. There is, however, one gap in these checks: io_register_clone_buffers() may take the uring_lock on a second (source) io_ring_ctx, but __io_uring_register() only checks the current thread against the *destination* io_ring_ctx's submitter_task. Fail the IORING_REGISTER_CLONE_BUFFERS with -EEXIST if the source io_ring_ctx has a registered submitter_task other than the current task. Signed-off-by: Caleb Sander Mateos Signed-off-by: Jens Axboe --- io_uring/rsrc.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'io_uring') diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 2d15b8785a95..d787c16dc1c3 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1300,10 +1300,17 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) if (src_ctx != ctx) { mutex_unlock(&ctx->uring_lock); lock_two_rings(ctx, src_ctx); + + if (src_ctx->submitter_task && + src_ctx->submitter_task != current) { + ret = -EEXIST; + goto out; + } } ret = io_clone_buffers(ctx, src_ctx, &buf); +out: if (src_ctx != ctx) mutex_unlock(&src_ctx->uring_lock); -- cgit v1.2.3 From 8577441d4a9c8fb5c6da1cde31708c3abb8d4cf8 Mon Sep 17 00:00:00 2001 From: Marco Crivellari Date: Fri, 5 Sep 2025 11:02:39 +0200 Subject: io_uring: replace use of system_wq with system_percpu_wq Currently if a user enqueue a work item using schedule_delayed_work() the used wq is "system_wq" (per-cpu wq) while queue_delayed_work() use WORK_CPU_UNBOUND (used when a cpu is not specified). The same applies to schedule_work() that is using system_wq and queue_work(), that makes use again of WORK_CPU_UNBOUND. This lack of consistentcy cannot be addressed without refactoring the API. system_wq is a per-CPU worqueue, yet nothing in its name tells about that CPU affinity constraint, which is very often not required by users. Make it clear by adding a system_percpu_wq. queue_work() / queue_delayed_work() mod_delayed_work() will now use the new per-cpu wq: whether the user still stick on the old name a warn will be printed along a wq redirect to the new one. This patch add the new system_percpu_wq except for mm, fs and net subsystem, whom are handled in separated patches. The old wq will be kept for a few release cylces. Suggested-by: Tejun Heo Signed-off-by: Marco Crivellari Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 347403862487..6fc5defcb156 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3106,7 +3106,7 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) * Use system_unbound_wq to avoid spawning tons of event kworkers * if we're exiting a ton of rings at the same time. It just adds * noise and overhead, there's no discernable change in runtime - * over using system_wq. + * over using system_percpu_wq. */ queue_work(iou_wq, &ctx->exit_work); } -- cgit v1.2.3 From 9f5f69d98efbcd4edb1c191cf91418688145e0d5 Mon Sep 17 00:00:00 2001 From: Marco Crivellari Date: Fri, 5 Sep 2025 11:02:40 +0200 Subject: io_uring: replace use of system_unbound_wq with system_dfl_wq Currently if a user enqueue a work item using schedule_delayed_work() the used wq is "system_wq" (per-cpu wq) while queue_delayed_work() use WORK_CPU_UNBOUND (used when a cpu is not specified). The same applies to schedule_work() that is using system_wq and queue_work(), that makes use again of WORK_CPU_UNBOUND. This lack of consistentcy cannot be addressed without refactoring the API. system_unbound_wq should be the default workqueue so as not to enforce locality constraints for random work whenever it's not required. Adding system_dfl_wq to encourage its use when unbound work should be used. queue_work() / queue_delayed_work() / mod_delayed_work() will now use the new unbound wq: whether the user still use the old wq a warn will be printed along with a wq redirect to the new one. The old system_unbound_wq will be kept for a few release cycles. Suggested-by: Tejun Heo Signed-off-by: Marco Crivellari Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 6fc5defcb156..6d62f10416eb 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3103,7 +3103,7 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) INIT_WORK(&ctx->exit_work, io_ring_exit_work); /* - * Use system_unbound_wq to avoid spawning tons of event kworkers + * Use system_dfl_wq to avoid spawning tons of event kworkers * if we're exiting a ton of rings at the same time. It just adds * noise and overhead, there's no discernable change in runtime * over using system_percpu_wq. -- cgit v1.2.3 From 9adc6669a60a7ec9e5a4b2b524791b0f4b7f3e66 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Sep 2025 09:50:30 -0600 Subject: io_uring: correct size of overflow CQE calculation If a 32b CQE is required, don't double the size of the overflow struct, just add the size of the io_uring_cqe addition that is needed. This avoids allocating too much memory, as the io_overflow_cqe size includes the list member required to queue them too. Fixes: e26dca67fde1 ("io_uring: add support for IORING_SETUP_CQE_MIXED") Reviewed-by: Caleb Sander Mateos Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 6d62f10416eb..1bfa124565f7 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -756,7 +756,7 @@ static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx, if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) { is_cqe32 = true; - ocq_size <<= 1; + ocq_size += sizeof(struct io_uring_cqe); } ocqe = kzalloc(ocq_size, gfp | __GFP_ACCOUNT); -- cgit v1.2.3 From 9eb3c571787d1ef7e2c3393c153b1a6b103a26e3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:44 +0100 Subject: io_uring/zcrx: improve rqe cache alignment Refill queue entries are 16B structures, but because of the ring header placement, they're 8B aligned but not naturally / 16B aligned, which means some of them span across 2 cache lines. Push rqes to a new cache line. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 51fd2350dbe9..c02045e4c1b6 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -352,7 +352,7 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, void *ptr; int ret; - off = sizeof(struct io_uring); + off = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES); size = off + sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries; if (size > rd->size) return -EINVAL; @@ -367,6 +367,10 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, ptr = io_region_get_ptr(&ifq->region); ifq->rq_ring = (struct io_uring *)ptr; ifq->rqes = (struct io_uring_zcrx_rqe *)(ptr + off); + + reg->offsets.head = offsetof(struct io_uring, head); + reg->offsets.tail = offsetof(struct io_uring, tail); + reg->offsets.rqes = off; return 0; } @@ -618,9 +622,6 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, goto err; ifq->if_rxq = reg.if_rxq; - reg.offsets.rqes = sizeof(struct io_uring); - reg.offsets.head = offsetof(struct io_uring, head); - reg.offsets.tail = offsetof(struct io_uring, tail); reg.zcrx_id = id; scoped_guard(mutex, &ctx->mmap_lock) { -- cgit v1.2.3 From bdc0d478a1632a72afa6d359d7fdd49dd08c0b25 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:45 +0100 Subject: io_uring/zcrx: replace memchar_inv with is_zero memchr_inv() is more ambiguous than mem_is_zero(), so use the latter for zero checks. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index c02045e4c1b6..a4a0560e8269 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -566,7 +566,7 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, return -EFAULT; if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd))) return -EFAULT; - if (memchr_inv(®.__resv, 0, sizeof(reg.__resv)) || + if (!mem_is_zero(®.__resv, sizeof(reg.__resv)) || reg.__resv2 || reg.zcrx_id) return -EINVAL; if (reg.if_rxq == -1 || !reg.rq_entries || reg.flags) -- cgit v1.2.3 From d5e31db9a950f1edfa20a59e7105e9cc78135493 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:46 +0100 Subject: io_uring/zcrx: use page_pool_unref_and_test() page_pool_unref_and_test() tries to better follow usuall refcount semantics, use it instead of page_pool_unref_netmem(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index a4a0560e8269..bd2fb3688432 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -787,7 +787,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp, continue; netmem = net_iov_to_netmem(niov); - if (page_pool_unref_netmem(netmem, 1) != 0) + if (!page_pool_unref_and_test(netmem)) continue; if (unlikely(niov->pp != pp)) { -- cgit v1.2.3 From c49606fc4be78da6c7a7c623566f6cf7663ba740 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:47 +0100 Subject: io_uring/zcrx: remove extra io_zcrx_drop_netdev io_close_queue() already detaches the netdev, don't unnecessary call io_zcrx_drop_netdev() right after. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 1 - 1 file changed, 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index bd2fb3688432..7a46e6fc2ee7 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -517,7 +517,6 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) { io_close_queue(ifq); - io_zcrx_drop_netdev(ifq); if (ifq->area) io_zcrx_free_area(ifq->area); -- cgit v1.2.3 From d425f13146af0ef10b8f1dc7cc9fd700ce7c759e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:48 +0100 Subject: io_uring/zcrx: don't pass slot to io_zcrx_create_area Don't pass a pointer to a pointer where an area should be stored to io_zcrx_create_area(), and let it handle finding the right place for a new area. It's more straightforward and will be needed to support multiple areas. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 7a46e6fc2ee7..c64b8c7ddedf 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -397,8 +397,16 @@ static void io_zcrx_free_area(struct io_zcrx_area *area) #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) +static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, + struct io_zcrx_area *area) +{ + if (ifq->area) + return -EINVAL; + ifq->area = area; + return 0; +} + static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, - struct io_zcrx_area **res, struct io_uring_zcrx_area_reg *area_reg) { struct io_zcrx_area *area; @@ -455,8 +463,10 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, area->area_id = 0; area_reg->rq_area_token = (u64)area->area_id << IORING_ZCRX_AREA_SHIFT; spin_lock_init(&area->freelist_lock); - *res = area; - return 0; + + ret = io_zcrx_append_area(ifq, area); + if (!ret) + return 0; err: if (area) io_zcrx_free_area(area); @@ -610,7 +620,7 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, } get_device(ifq->dev); - ret = io_zcrx_create_area(ifq, &ifq->area, &area); + ret = io_zcrx_create_area(ifq, &area); if (ret) goto err; -- cgit v1.2.3 From 01464ea405e13789bf4f14c7d4e9fa97f0885d46 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:49 +0100 Subject: io_uring/zcrx: move area reg checks into io_import_area io_import_area() is responsible for importing memory and parsing io_uring_zcrx_area_reg, so move all area reg structure checks into the function. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index c64b8c7ddedf..ef8d60b92646 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -26,6 +26,8 @@ #include "zcrx.h" #include "rsrc.h" +#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) + #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp) @@ -231,6 +233,13 @@ static int io_import_area(struct io_zcrx_ifq *ifq, { int ret; + if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS) + return -EINVAL; + if (area_reg->rq_area_token) + return -EINVAL; + if (area_reg->__resv2[0] || area_reg->__resv2[1]) + return -EINVAL; + ret = io_validate_user_buf_range(area_reg->addr, area_reg->len); if (ret) return ret; @@ -395,8 +404,6 @@ static void io_zcrx_free_area(struct io_zcrx_area *area) kfree(area); } -#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) - static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { @@ -413,13 +420,6 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, unsigned nr_iovs; int i, ret; - if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS) - return -EINVAL; - if (area_reg->rq_area_token) - return -EINVAL; - if (area_reg->__resv2[0] || area_reg->__resv2[1]) - return -EINVAL; - ret = -ENOMEM; area = kzalloc(sizeof(*area), GFP_KERNEL); if (!area) -- cgit v1.2.3 From d7ae46b454eb05e3df0d46c2ac9c61416a4d9057 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:50 +0100 Subject: io_uring/zcrx: check all niovs filled with dma addresses Add a warning if io_populate_area_dma() can't fill in all net_iovs, it should never happen. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index ef8d60b92646..0f15e0fa5467 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -77,6 +77,9 @@ static int io_populate_area_dma(struct io_zcrx_ifq *ifq, niov_idx++; } } + + if (WARN_ON_ONCE(niov_idx != area->nia.num_niovs)) + return -EFAULT; return 0; } -- cgit v1.2.3 From 02bb047b5f42ed30ca97010069cb36cd3afb74e1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:51 +0100 Subject: io_uring/zcrx: pass ifq to io_zcrx_alloc_fallback() io_zcrx_copy_chunk() doesn't and shouldn't care from which area the buffer is allocated, don't try to resolve the area in it but pass the ifq to io_zcrx_alloc_fallback() and let it handle it. Also rename it for more clarity. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 0f15e0fa5467..16bf036c7b24 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -957,10 +957,14 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, return true; } -static struct net_iov *io_zcrx_alloc_fallback(struct io_zcrx_area *area) +static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) { + struct io_zcrx_area *area = ifq->area; struct net_iov *niov = NULL; + if (area->mem.is_dmabuf) + return NULL; + spin_lock_bh(&area->freelist_lock); if (area->free_count) niov = __io_zcrx_get_free_niov(area); @@ -1020,19 +1024,15 @@ static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq, struct page *src_page, unsigned int src_offset, size_t len) { - struct io_zcrx_area *area = ifq->area; size_t copied = 0; int ret = 0; - if (area->mem.is_dmabuf) - return -EFAULT; - while (len) { struct io_copy_cache cc; struct net_iov *niov; size_t n; - niov = io_zcrx_alloc_fallback(area); + niov = io_alloc_fallback_niov(ifq); if (!niov) { ret = -ENOMEM; break; -- cgit v1.2.3 From 439a98b972fbb1991819b5367f482cd4161ba39c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:52 +0100 Subject: io_uring/zcrx: deduplicate area mapping With a common type for storing dma addresses and io_populate_area_dma(), type-specific area mapping helpers are trivial, so open code them and deduplicate the call to io_populate_area_dma(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 16bf036c7b24..bba92774c801 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -157,14 +157,6 @@ err: return ret; } -static int io_zcrx_map_area_dmabuf(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) -{ - if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) - return -EINVAL; - return io_populate_area_dma(ifq, area, area->mem.sgt, - area->mem.dmabuf_offset); -} - static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pages) { struct folio *last_folio = NULL; @@ -275,30 +267,29 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, } } -static unsigned io_zcrx_map_area_umem(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) -{ - int ret; - - ret = dma_map_sgtable(ifq->dev, &area->mem.page_sg_table, - DMA_FROM_DEVICE, IO_DMA_ATTR); - if (ret < 0) - return ret; - return io_populate_area_dma(ifq, area, &area->mem.page_sg_table, 0); -} - static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { + unsigned long offset; + struct sg_table *sgt; int ret; guard(mutex)(&ifq->dma_lock); if (area->is_mapped) return 0; - if (area->mem.is_dmabuf) - ret = io_zcrx_map_area_dmabuf(ifq, area); - else - ret = io_zcrx_map_area_umem(ifq, area); + if (!area->mem.is_dmabuf) { + ret = dma_map_sgtable(ifq->dev, &area->mem.page_sg_table, + DMA_FROM_DEVICE, IO_DMA_ATTR); + if (ret < 0) + return ret; + sgt = &area->mem.page_sg_table; + offset = 0; + } else { + sgt = area->mem.sgt; + offset = area->mem.dmabuf_offset; + } + ret = io_populate_area_dma(ifq, area, sgt, offset); if (ret == 0) area->is_mapped = true; return ret; -- cgit v1.2.3 From 6c185117291a85937fa67d402efc4f11b2891c6a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:53 +0100 Subject: io_uring/zcrx: remove dmabuf_offset It was removed from uapi, so now it's always 0 and can be removed together with offset handling in io_populate_area_dma(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 13 ++----------- io_uring/zcrx.h | 1 - 2 files changed, 2 insertions(+), 12 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index bba92774c801..bcefb302aadf 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -53,7 +53,7 @@ static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) static int io_populate_area_dma(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area, - struct sg_table *sgt, unsigned long off) + struct sg_table *sgt) { struct scatterlist *sg; unsigned i, niov_idx = 0; @@ -61,11 +61,6 @@ static int io_populate_area_dma(struct io_zcrx_ifq *ifq, for_each_sgtable_dma_sg(sgt, sg, i) { dma_addr_t dma = sg_dma_address(sg); unsigned long sg_len = sg_dma_len(sg); - unsigned long sg_off = min(sg_len, off); - - off -= sg_off; - sg_len -= sg_off; - dma += sg_off; while (sg_len && niov_idx < area->nia.num_niovs) { struct net_iov *niov = &area->nia.niovs[niov_idx]; @@ -149,7 +144,6 @@ static int io_import_dmabuf(struct io_zcrx_ifq *ifq, goto err; } - mem->dmabuf_offset = off; mem->size = len; return 0; err: @@ -269,7 +263,6 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { - unsigned long offset; struct sg_table *sgt; int ret; @@ -283,13 +276,11 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) if (ret < 0) return ret; sgt = &area->mem.page_sg_table; - offset = 0; } else { sgt = area->mem.sgt; - offset = area->mem.dmabuf_offset; } - ret = io_populate_area_dma(ifq, area, sgt, offset); + ret = io_populate_area_dma(ifq, area, sgt); if (ret == 0) area->is_mapped = true; return ret; diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 109c4ca36434..24ed473632c6 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -20,7 +20,6 @@ struct io_zcrx_mem { struct dma_buf_attachment *attach; struct dma_buf *dmabuf; struct sg_table *sgt; - unsigned long dmabuf_offset; }; struct io_zcrx_area { -- cgit v1.2.3 From 5d93f7bade0b1eb60d0f395ad72b35581d28a896 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:54 +0100 Subject: io_uring/zcrx: set sgt for umem area Set struct io_zcrx_mem::sgt for umem areas as well to simplify looking up the current sg table. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 14 ++++++-------- io_uring/zcrx.h | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index bcefb302aadf..764723bf04d6 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -52,9 +52,9 @@ static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) } static int io_populate_area_dma(struct io_zcrx_ifq *ifq, - struct io_zcrx_area *area, - struct sg_table *sgt) + struct io_zcrx_area *area) { + struct sg_table *sgt = area->mem.sgt; struct scatterlist *sg; unsigned i, niov_idx = 0; @@ -197,6 +197,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq, if (ret < 0) mem->account_pages = 0; + mem->sgt = &mem->page_sg_table; mem->pages = pages; mem->nr_folios = nr_pages; mem->size = area_reg->len; @@ -211,7 +212,8 @@ static void io_release_area_mem(struct io_zcrx_mem *mem) } if (mem->pages) { unpin_user_pages(mem->pages, mem->nr_folios); - sg_free_table(&mem->page_sg_table); + sg_free_table(mem->sgt); + mem->sgt = NULL; kvfree(mem->pages); } } @@ -263,7 +265,6 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { - struct sg_table *sgt; int ret; guard(mutex)(&ifq->dma_lock); @@ -275,12 +276,9 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) DMA_FROM_DEVICE, IO_DMA_ATTR); if (ret < 0) return ret; - sgt = &area->mem.page_sg_table; - } else { - sgt = area->mem.sgt; } - ret = io_populate_area_dma(ifq, area, sgt); + ret = io_populate_area_dma(ifq, area); if (ret == 0) area->is_mapped = true; return ret; diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 24ed473632c6..27d7cf28a04e 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -16,10 +16,10 @@ struct io_zcrx_mem { unsigned long nr_folios; struct sg_table page_sg_table; unsigned long account_pages; + struct sg_table *sgt; struct dma_buf_attachment *attach; struct dma_buf *dmabuf; - struct sg_table *sgt; }; struct io_zcrx_area { -- cgit v1.2.3 From d8d135dfe3e8e306d9edfcccf28dbe75c6a85567 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:55 +0100 Subject: io_uring/zcrx: make niov size variable Instead of using PAGE_SIZE for the niov size add a niov_shift field to ifq, and patch up all important places. Copy fallback still assumes PAGE_SIZE, so it'll be wasting some memory for now. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 30 ++++++++++++++++++++---------- io_uring/zcrx.h | 1 + 2 files changed, 21 insertions(+), 10 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 764723bf04d6..85832f60d68a 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -45,15 +45,18 @@ static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *nio static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) { struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); + unsigned niov_pages_shift; lockdep_assert(!area->mem.is_dmabuf); - return area->mem.pages[net_iov_idx(niov)]; + niov_pages_shift = area->ifq->niov_shift - PAGE_SHIFT; + return area->mem.pages[net_iov_idx(niov) << niov_pages_shift]; } static int io_populate_area_dma(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { + unsigned niov_size = 1U << ifq->niov_shift; struct sg_table *sgt = area->mem.sgt; struct scatterlist *sg; unsigned i, niov_idx = 0; @@ -62,13 +65,16 @@ static int io_populate_area_dma(struct io_zcrx_ifq *ifq, dma_addr_t dma = sg_dma_address(sg); unsigned long sg_len = sg_dma_len(sg); + if (WARN_ON_ONCE(sg_len % niov_size)) + return -EINVAL; + while (sg_len && niov_idx < area->nia.num_niovs) { struct net_iov *niov = &area->nia.niovs[niov_idx]; if (net_mp_niov_set_dma_addr(niov, dma)) return -EFAULT; - sg_len -= PAGE_SIZE; - dma += PAGE_SIZE; + sg_len -= niov_size; + dma += niov_size; niov_idx++; } } @@ -284,18 +290,21 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) return ret; } -static void io_zcrx_sync_for_device(const struct page_pool *pool, +static void io_zcrx_sync_for_device(struct page_pool *pool, struct net_iov *niov) { #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) dma_addr_t dma_addr; + unsigned niov_size; + if (!dma_dev_need_sync(pool->p.dev)) return; + niov_size = 1U << io_pp_to_ifq(pool)->niov_shift; dma_addr = page_pool_get_dma_addr_netmem(net_iov_to_netmem(niov)); __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset, - PAGE_SIZE, pool->p.dma_dir); + niov_size, pool->p.dma_dir); #endif } @@ -413,7 +422,8 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, if (ret) goto err; - nr_iovs = area->mem.size >> PAGE_SHIFT; + ifq->niov_shift = PAGE_SHIFT; + nr_iovs = area->mem.size >> ifq->niov_shift; area->nia.num_niovs = nr_iovs; ret = -ENOMEM; @@ -764,7 +774,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp, unsigned niov_idx, area_idx; area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; - niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> PAGE_SHIFT; + niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; if (unlikely(rqe->__pad || area_idx)) continue; @@ -854,8 +864,8 @@ static int io_pp_zc_init(struct page_pool *pp) return -EINVAL; if (WARN_ON_ONCE(!pp->dma_map)) return -EOPNOTSUPP; - if (pp->p.order != 0) - return -EOPNOTSUPP; + if (pp->p.order + PAGE_SHIFT != ifq->niov_shift) + return -EINVAL; if (pp->p.dma_dir != DMA_FROM_DEVICE) return -EOPNOTSUPP; @@ -930,7 +940,7 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, cqe->flags |= IORING_CQE_F_32; area = io_zcrx_iov_to_area(niov); - offset = off + (net_iov_idx(niov) << PAGE_SHIFT); + offset = off + (net_iov_idx(niov) << ifq->niov_shift); rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1); rcqe->off = offset + ((u64)area->area_id << IORING_ZCRX_AREA_SHIFT); rcqe->__pad = 0; diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 27d7cf28a04e..7604f1f85ccb 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -41,6 +41,7 @@ struct io_zcrx_area { struct io_zcrx_ifq { struct io_ring_ctx *ctx; struct io_zcrx_area *area; + unsigned niov_shift; spinlock_t rq_lock ____cacheline_aligned_in_smp; struct io_uring *rq_ring; -- cgit v1.2.3 From 4f602f3112c8271e32bea358dd2a8005d32a5bd5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:56 +0100 Subject: io_uring/zcrx: rename dma lock In preparation for reusing the lock for other purposes, rename it to "pp_lock". As before, it can be taken deeper inside the networking stack by page pool, and so the syscall io_uring must avoid holding it while doing queue reconfiguration or anything that can result in immediate pp init/destruction. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 8 ++++---- io_uring/zcrx.h | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 85832f60d68a..0deb41b74b7c 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -253,7 +253,7 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, { int i; - guard(mutex)(&ifq->dma_lock); + guard(mutex)(&ifq->pp_lock); if (!area->is_mapped) return; area->is_mapped = false; @@ -273,7 +273,7 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { int ret; - guard(mutex)(&ifq->dma_lock); + guard(mutex)(&ifq->pp_lock); if (area->is_mapped) return 0; @@ -478,7 +478,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) ifq->ctx = ctx; spin_lock_init(&ifq->lock); spin_lock_init(&ifq->rq_lock); - mutex_init(&ifq->dma_lock); + mutex_init(&ifq->pp_lock); return ifq; } @@ -527,7 +527,7 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) put_device(ifq->dev); io_free_rbuf_ring(ifq); - mutex_destroy(&ifq->dma_lock); + mutex_destroy(&ifq->pp_lock); kfree(ifq); } diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 7604f1f85ccb..3f89a34e5282 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -54,7 +54,12 @@ struct io_zcrx_ifq { struct net_device *netdev; netdevice_tracker netdev_tracker; spinlock_t lock; - struct mutex dma_lock; + + /* + * Page pool and net configuration lock, can be taken deeper in the + * net stack. + */ + struct mutex pp_lock; struct io_mapped_region region; }; -- cgit v1.2.3 From 20dda449c0b6297ed7c13a23a1207ed072655bff Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:57 +0100 Subject: io_uring/zcrx: protect netdev with pp_lock Remove ifq->lock and reuse pp_lock to protect the netdev pointer. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 23 +++++++++++------------ io_uring/zcrx.h | 1 - 2 files changed, 11 insertions(+), 13 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 0deb41b74b7c..6a5b6f32edc3 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -476,7 +476,6 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) ifq->if_rxq = -1; ifq->ctx = ctx; - spin_lock_init(&ifq->lock); spin_lock_init(&ifq->rq_lock); mutex_init(&ifq->pp_lock); return ifq; @@ -484,12 +483,12 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq) { - spin_lock(&ifq->lock); - if (ifq->netdev) { - netdev_put(ifq->netdev, &ifq->netdev_tracker); - ifq->netdev = NULL; - } - spin_unlock(&ifq->lock); + guard(mutex)(&ifq->pp_lock); + + if (!ifq->netdev) + return; + netdev_put(ifq->netdev, &ifq->netdev_tracker); + ifq->netdev = NULL; } static void io_close_queue(struct io_zcrx_ifq *ifq) @@ -504,11 +503,11 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) if (ifq->if_rxq == -1) return; - spin_lock(&ifq->lock); - netdev = ifq->netdev; - netdev_tracker = ifq->netdev_tracker; - ifq->netdev = NULL; - spin_unlock(&ifq->lock); + scoped_guard(mutex, &ifq->pp_lock) { + netdev = ifq->netdev; + netdev_tracker = ifq->netdev_tracker; + ifq->netdev = NULL; + } if (netdev) { net_mp_close_rxq(netdev, ifq->if_rxq, &p); diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 3f89a34e5282..a48871b5adad 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -53,7 +53,6 @@ struct io_zcrx_ifq { struct device *dev; struct net_device *netdev; netdevice_tracker netdev_tracker; - spinlock_t lock; /* * Page pool and net configuration lock, can be taken deeper in the -- cgit v1.2.3 From 73fa880effc5644aaf746596acb1b1efa44606df Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:58 +0100 Subject: io_uring/zcrx: reduce netmem scope in refill Reduce the scope of a local var netmem in io_zcrx_ring_refill. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 6a5b6f32edc3..5f99fc7b43ee 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -755,7 +755,6 @@ static void io_zcrx_ring_refill(struct page_pool *pp, { unsigned int mask = ifq->rq_entries - 1; unsigned int entries; - netmem_ref netmem; spin_lock_bh(&ifq->rq_lock); @@ -771,6 +770,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp, struct io_zcrx_area *area; struct net_iov *niov; unsigned niov_idx, area_idx; + netmem_ref netmem; area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; -- cgit v1.2.3 From c95257f336556de05f26dc88a890fb2a59364939 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:27:59 +0100 Subject: io_uring/zcrx: use guards for the refill lock Use guards for rq_lock in io_zcrx_ring_refill(), makes it a tad simpler. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 5f99fc7b43ee..630b19ebb47e 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -756,14 +756,12 @@ static void io_zcrx_ring_refill(struct page_pool *pp, unsigned int mask = ifq->rq_entries - 1; unsigned int entries; - spin_lock_bh(&ifq->rq_lock); + guard(spinlock_bh)(&ifq->rq_lock); entries = io_zcrx_rqring_entries(ifq); entries = min_t(unsigned, entries, PP_ALLOC_CACHE_REFILL - pp->alloc.count); - if (unlikely(!entries)) { - spin_unlock_bh(&ifq->rq_lock); + if (unlikely(!entries)) return; - } do { struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(ifq, mask); @@ -801,7 +799,6 @@ static void io_zcrx_ring_refill(struct page_pool *pp, } while (--entries); smp_store_release(&ifq->rq_ring->head, ifq->cached_rq_head); - spin_unlock_bh(&ifq->rq_lock); } static void io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq) -- cgit v1.2.3 From 5a8b6e7c1d7b5863faaf392eafa089bd599a8973 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:28:00 +0100 Subject: io_uring/zcrx: don't adjust free cache space The cache should be empty when io_pp_zc_alloc_netmems() is called, that's promised by page pool and further checked, so there is no need to recalculate the available space in io_zcrx_ring_refill(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 630b19ebb47e..a805f744c774 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -759,7 +759,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp, guard(spinlock_bh)(&ifq->rq_lock); entries = io_zcrx_rqring_entries(ifq); - entries = min_t(unsigned, entries, PP_ALLOC_CACHE_REFILL - pp->alloc.count); + entries = min_t(unsigned, entries, PP_ALLOC_CACHE_REFILL); if (unlikely(!entries)) return; -- cgit v1.2.3 From 8fd08d8dda3c6c4e9f0b73acdcf8a1cf391b0c8f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:28:01 +0100 Subject: io_uring/zcrx: introduce io_parse_rqe() Add a helper for verifying a rqe and extracting a niov out of it. It'll be reused in following patches. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index a805f744c774..81d4aa75a69f 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -750,6 +750,28 @@ static struct io_uring_zcrx_rqe *io_zcrx_get_rqe(struct io_zcrx_ifq *ifq, return &ifq->rqes[idx]; } +static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, + struct io_zcrx_ifq *ifq, + struct net_iov **ret_niov) +{ + unsigned niov_idx, area_idx; + struct io_zcrx_area *area; + + area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; + niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; + + if (unlikely(rqe->__pad || area_idx)) + return false; + area = ifq->area; + + if (unlikely(niov_idx >= area->nia.num_niovs)) + return false; + niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs); + + *ret_niov = &area->nia.niovs[niov_idx]; + return true; +} + static void io_zcrx_ring_refill(struct page_pool *pp, struct io_zcrx_ifq *ifq) { @@ -765,23 +787,11 @@ static void io_zcrx_ring_refill(struct page_pool *pp, do { struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(ifq, mask); - struct io_zcrx_area *area; struct net_iov *niov; - unsigned niov_idx, area_idx; netmem_ref netmem; - area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; - niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; - - if (unlikely(rqe->__pad || area_idx)) + if (!io_parse_rqe(rqe, ifq, &niov)) continue; - area = ifq->area; - - if (unlikely(niov_idx >= area->nia.num_niovs)) - continue; - niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs); - - niov = &area->nia.niovs[niov_idx]; if (!io_zcrx_put_niov_uref(niov)) continue; -- cgit v1.2.3 From 705d2ac7b2044f1ca05ba6033183151a04dbff4d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:28:02 +0100 Subject: io_uring/zcrx: allow synchronous buffer return Returning buffers via a ring is performant and convenient, but it becomes a problem when/if the user misconfigured the ring size and it becomes full. Add a synchronous way to return buffers back to the page pool via a new register opcode. It's supposed to be a reliable slow path for refilling. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/register.c | 3 +++ io_uring/zcrx.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++ io_uring/zcrx.h | 7 ++++++ 3 files changed, 78 insertions(+) (limited to 'io_uring') diff --git a/io_uring/register.c b/io_uring/register.c index 96e9cac12823..43f04c47522c 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -833,6 +833,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, case IORING_REGISTER_QUERY: ret = io_query(ctx, arg, nr_args); break; + case IORING_REGISTER_ZCRX_REFILL: + ret = io_zcrx_return_bufs(ctx, arg, nr_args); + break; default: ret = -EINVAL; break; diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 81d4aa75a69f..07a114f9a542 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -927,6 +927,74 @@ static const struct memory_provider_ops io_uring_pp_zc_ops = { .uninstall = io_pp_uninstall, }; +#define IO_ZCRX_MAX_SYS_REFILL_BUFS (1 << 16) +#define IO_ZCRX_SYS_REFILL_BATCH 32 + +static void io_return_buffers(struct io_zcrx_ifq *ifq, + struct io_uring_zcrx_rqe *rqes, unsigned nr) +{ + int i; + + for (i = 0; i < nr; i++) { + struct net_iov *niov; + netmem_ref netmem; + + if (!io_parse_rqe(&rqes[i], ifq, &niov)) + continue; + + scoped_guard(spinlock_bh, &ifq->rq_lock) { + if (!io_zcrx_put_niov_uref(niov)) + continue; + } + + netmem = net_iov_to_netmem(niov); + if (!page_pool_unref_and_test(netmem)) + continue; + io_zcrx_return_niov(niov); + } +} + +int io_zcrx_return_bufs(struct io_ring_ctx *ctx, + void __user *arg, unsigned nr_arg) +{ + struct io_uring_zcrx_rqe rqes[IO_ZCRX_SYS_REFILL_BATCH]; + struct io_uring_zcrx_rqe __user *user_rqes; + struct io_uring_zcrx_sync_refill zr; + struct io_zcrx_ifq *ifq; + unsigned nr, i; + + if (nr_arg) + return -EINVAL; + if (copy_from_user(&zr, arg, sizeof(zr))) + return -EFAULT; + if (!zr.nr_entries || zr.nr_entries > IO_ZCRX_MAX_SYS_REFILL_BUFS) + return -EINVAL; + if (!mem_is_zero(&zr.__resv, sizeof(zr.__resv))) + return -EINVAL; + + ifq = xa_load(&ctx->zcrx_ctxs, zr.zcrx_id); + if (!ifq) + return -EINVAL; + nr = zr.nr_entries; + user_rqes = u64_to_user_ptr(zr.rqes); + + for (i = 0; i < nr;) { + unsigned batch = min(nr - i, IO_ZCRX_SYS_REFILL_BATCH); + size_t size = batch * sizeof(rqes[0]); + + if (copy_from_user(rqes, user_rqes + i, size)) + return i ? i : -EFAULT; + io_return_buffers(ifq, rqes, batch); + + i += batch; + + if (fatal_signal_pending(current)) + return i; + cond_resched(); + } + return nr; +} + static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, struct io_zcrx_ifq *ifq, int off, int len) { diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index a48871b5adad..33ef61503092 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -63,6 +63,8 @@ struct io_zcrx_ifq { }; #if defined(CONFIG_IO_URING_ZCRX) +int io_zcrx_return_bufs(struct io_ring_ctx *ctx, + void __user *arg, unsigned nr_arg); int io_register_zcrx_ifq(struct io_ring_ctx *ctx, struct io_uring_zcrx_ifq_reg __user *arg); void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); @@ -95,6 +97,11 @@ static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ct { return NULL; } +static inline int io_zcrx_return_bufs(struct io_ring_ctx *ctx, + void __user *arg, unsigned nr_arg) +{ + return -EOPNOTSUPP; +} #endif int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); -- cgit v1.2.3 From 31bf77dcc3810e08bcc7d15470e92cdfffb7f7f1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 16 Sep 2025 15:28:03 +0100 Subject: io_uring/zcrx: account niov arrays to cgroup net_iov / freelist / etc. arrays can be quite long, make sure they're accounted. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'io_uring') diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 07a114f9a542..6799b5f33c96 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -428,17 +428,17 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, ret = -ENOMEM; area->nia.niovs = kvmalloc_array(nr_iovs, sizeof(area->nia.niovs[0]), - GFP_KERNEL | __GFP_ZERO); + GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (!area->nia.niovs) goto err; area->freelist = kvmalloc_array(nr_iovs, sizeof(area->freelist[0]), - GFP_KERNEL | __GFP_ZERO); + GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (!area->freelist) goto err; area->user_refs = kvmalloc_array(nr_iovs, sizeof(area->user_refs[0]), - GFP_KERNEL | __GFP_ZERO); + GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (!area->user_refs) goto err; -- cgit v1.2.3 From 2408d1783204920880f929a7a3087c76f5a59c13 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 19 Sep 2025 12:11:56 +0100 Subject: io_uring/query: prevent infinite loops If the query chain forms a cycle, the interface will loop indefinitely. Make sure it handles fatal signals, so the user can kill the process and hence break out of the infinite loop. Fixes: c265ae75f900 ("io_uring: introduce io_uring querying") Reported-by: Jens Axboe Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/query.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'io_uring') diff --git a/io_uring/query.c b/io_uring/query.c index 9eed0f371956..c2183daf5a46 100644 --- a/io_uring/query.c +++ b/io_uring/query.c @@ -88,6 +88,10 @@ int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) if (ret) return ret; uhdr = u64_to_user_ptr(next_hdr); + + if (fatal_signal_pending(current)) + return -EINTR; + cond_resched(); } return 0; } -- cgit v1.2.3 From 7ea24326e72dad7cd326bedd8442c162ae23df9d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 19 Sep 2025 12:11:57 +0100 Subject: io_uring/query: cap number of queries If a query chain forms a cycle, it'll be looping in the kernel until the process is killed. It might be fine as any such mistake can be easily uncovered during testing, but it's still nicer to let it break out of the syscall if it executed too many queries. Suggested-by: Jens Axboe Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/query.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'io_uring') diff --git a/io_uring/query.c b/io_uring/query.c index c2183daf5a46..645301bd2c82 100644 --- a/io_uring/query.c +++ b/io_uring/query.c @@ -6,6 +6,7 @@ #include "io_uring.h" #define IO_MAX_QUERY_SIZE (sizeof(struct io_uring_query_opcode)) +#define IO_MAX_QUERY_ENTRIES 1000 static ssize_t io_query_ops(void *data) { @@ -74,7 +75,7 @@ int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { char entry_buffer[IO_MAX_QUERY_SIZE]; void __user *uhdr = arg; - int ret; + int ret, nr = 0; memset(entry_buffer, 0, sizeof(entry_buffer)); @@ -89,6 +90,9 @@ int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) return ret; uhdr = u64_to_user_ptr(next_hdr); + /* Have some limit to avoid a potential cycle */ + if (++nr >= IO_MAX_QUERY_ENTRIES) + return -ERANGE; if (fatal_signal_pending(current)) return -EINTR; cond_resched(); -- cgit v1.2.3 From 79525b51acc1c8e331ab47eb131a99f5370a76c2 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 19 Sep 2025 12:38:58 -0700 Subject: io_uring: fix nvme's 32b cqes on mixed cq The nvme uring_cmd only uses 32b CQEs. If the ring uses a mixed CQ, then we need to make sure we flag the completion as a 32b CQE. On the other hand, if nvme uring_cmd was using a dedicated 32b CQE, the posting was missing the extra memcpy because it only applied to bit CQEs on a mixed CQ. Fixes: e26dca67fde1943 ("io_uring: add support for IORING_SETUP_CQE_MIXED") Signed-off-by: Keith Busch Signed-off-by: Jens Axboe --- io_uring/io_uring.h | 2 +- io_uring/uring_cmd.c | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index e92099d8fcb7..af4c11310652 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -275,7 +275,7 @@ static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, return false; memcpy(cqe, &req->cqe, sizeof(*cqe)); - if (is_cqe32) { + if (ctx->flags & IORING_SETUP_CQE32 || is_cqe32) { memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); memset(&req->big_cqe, 0, sizeof(req->big_cqe)); } diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 5562e8491c5b..a688c9f1a21c 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -151,8 +151,8 @@ static inline void io_req_set_cqe32_extra(struct io_kiocb *req, * Called by consumers of io_uring_cmd, if they originally returned * -EIOCBQUEUED upon receiving the command. */ -void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, - unsigned issue_flags) +void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, + unsigned issue_flags, bool is_cqe32) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); @@ -165,8 +165,11 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, req_set_fail(req); io_req_set_res(req, ret, 0); - if (req->ctx->flags & IORING_SETUP_CQE32) + if (is_cqe32) { + if (req->ctx->flags & IORING_SETUP_CQE_MIXED) + req->cqe.flags |= IORING_CQE_F_32; io_req_set_cqe32_extra(req, res2, 0); + } io_req_uring_cleanup(req, issue_flags); if (req->ctx->flags & IORING_SETUP_IOPOLL) { /* order with io_iopoll_req_issued() checking ->iopoll_complete */ @@ -180,7 +183,7 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, io_req_task_work_add(req); } } -EXPORT_SYMBOL_GPL(io_uring_cmd_done); +EXPORT_SYMBOL_GPL(__io_uring_cmd_done); int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { -- cgit v1.2.3