diff options
Diffstat (limited to 'fs/fuse/dev.c')
| -rw-r--r-- | fs/fuse/dev.c | 1503 |
1 files changed, 604 insertions, 899 deletions
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 0b0241f47170..4fec31fc0b84 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -1,14 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ +#include "dev.h" +#include "args.h" #include "dev_uring_i.h" -#include "fuse_i.h" -#include "fuse_dev_i.h" #include <linux/init.h> #include <linux/module.h> @@ -30,124 +28,33 @@ MODULE_ALIAS_MISCDEV(FUSE_MINOR); MODULE_ALIAS("devname:fuse"); -static struct kmem_cache *fuse_req_cachep; - -const unsigned long fuse_timeout_timer_freq = - secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ); - -bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list) -{ - struct fuse_req *req; - - req = list_first_entry_or_null(list, struct fuse_req, list); - if (!req) - return false; - return time_is_before_jiffies(req->create_time + fc->timeout.req_timeout); -} - -static bool fuse_fpq_processing_expired(struct fuse_conn *fc, struct list_head *processing) -{ - int i; - - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - if (fuse_request_expired(fc, &processing[i])) - return true; +static DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq); - return false; -} - -/* - * Check if any requests aren't being completed by the time the request timeout - * elapses. To do so, we: - * - check the fiq pending list - * - check the bg queue - * - check the fpq io and processing lists - * - * To make this fast, we only check against the head request on each list since - * these are generally queued in order of creation time (eg newer requests get - * queued to the tail). We might miss a few edge cases (eg requests transitioning - * between lists, re-sent requests at the head of the pending list having a - * later creation time than other requests on that list, etc.) but that is fine - * since if the request never gets fulfilled, it will eventually be caught. - */ -void fuse_check_timeout(struct work_struct *work) -{ - struct delayed_work *dwork = to_delayed_work(work); - struct fuse_conn *fc = container_of(dwork, struct fuse_conn, - timeout.work); - struct fuse_iqueue *fiq = &fc->iq; - struct fuse_dev *fud; - struct fuse_pqueue *fpq; - bool expired = false; - - if (!atomic_read(&fc->num_waiting)) - goto out; - - spin_lock(&fiq->lock); - expired = fuse_request_expired(fc, &fiq->pending); - spin_unlock(&fiq->lock); - if (expired) - goto abort_conn; - - spin_lock(&fc->bg_lock); - expired = fuse_request_expired(fc, &fc->bg_queue); - spin_unlock(&fc->bg_lock); - if (expired) - goto abort_conn; - - spin_lock(&fc->lock); - if (!fc->connected) { - spin_unlock(&fc->lock); - return; - } - list_for_each_entry(fud, &fc->devices, entry) { - fpq = &fud->pq; - spin_lock(&fpq->lock); - if (fuse_request_expired(fc, &fpq->io) || - fuse_fpq_processing_expired(fc, fpq->processing)) { - spin_unlock(&fpq->lock); - spin_unlock(&fc->lock); - goto abort_conn; - } - - spin_unlock(&fpq->lock); - } - spin_unlock(&fc->lock); - - if (fuse_uring_request_expired(fc)) - goto abort_conn; - -out: - queue_delayed_work(system_percpu_wq, &fc->timeout.work, - fuse_timeout_timer_freq); - return; - -abort_conn: - fuse_abort_conn(fc); -} +static struct kmem_cache *fuse_req_cachep; -static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req) +static void fuse_request_init(struct fuse_chan *fch, struct fuse_req *req) { INIT_LIST_HEAD(&req->list); INIT_LIST_HEAD(&req->intr_entry); init_waitqueue_head(&req->waitq); refcount_set(&req->count, 1); __set_bit(FR_PENDING, &req->flags); - req->fm = fm; + req->chan = fch; req->create_time = jiffies; } -static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags) +static struct fuse_req *fuse_request_alloc(struct fuse_chan *fch, gfp_t flags) { struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags); if (req) - fuse_request_init(fm, req); + fuse_request_init(fch, req); return req; } static void fuse_request_free(struct fuse_req *req) { + WARN_ON(!list_empty(&req->intr_entry)); kmem_cache_free(fuse_req_cachep, req); } @@ -162,110 +69,89 @@ static void __fuse_put_request(struct fuse_req *req) refcount_dec(&req->count); } -void fuse_set_initialized(struct fuse_conn *fc) +void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param) { - /* Make sure stores before this are seen on another CPU */ - smp_wmb(); - fc->initialized = 1; + if (param) { + fch->minor = param->minor; + fch->max_write = param->max_write; + fch->max_pages = param->max_pages; + + if (param->io_uring_enabled) + fuse_uring_conn_init(fch); + } + + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&fch->initialized, 1); + wake_up_all(&fch->blocked_waitq); } -static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background) +static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - return !fc->initialized || (for_background && fc->blocked) || - (fc->io_uring && fc->connected && !fuse_uring_ready(fc)); + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized)) + return true; + + return (for_background && fch->blocked) || + (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } -static void fuse_drop_waiting(struct fuse_conn *fc) +static void fuse_drop_waiting(struct fuse_chan *fch) { /* - * lockess check of fc->connected is okay, because atomic_dec_and_test() - * provides a memory barrier matched with the one in fuse_wait_aborted() + * lockess check of fch->connected is okay, because atomic_dec_and_test() + * provides a memory barrier matched with the one in fuse_chan_wait_aborted() * to ensure no wake-up is missed. */ - if (atomic_dec_and_test(&fc->num_waiting) && - !READ_ONCE(fc->connected)) { + if (atomic_dec_and_test(&fch->num_waiting) && + !READ_ONCE(fch->connected)) { /* wake up aborters */ - wake_up_all(&fc->blocked_waitq); + wake_up_all(&fch->blocked_waitq); } } static void fuse_put_request(struct fuse_req *req); -static struct fuse_req *fuse_get_req(struct mnt_idmap *idmap, - struct fuse_mount *fm, - bool for_background) +static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background) { - struct fuse_conn *fc = fm->fc; struct fuse_req *req; - bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP); - kuid_t fsuid; - kgid_t fsgid; int err; - atomic_inc(&fc->num_waiting); + atomic_inc(&fch->num_waiting); - if (fuse_block_alloc(fc, for_background)) { + if (fuse_block_alloc(fch, for_background)) { err = -EINTR; - if (wait_event_state_exclusive(fc->blocked_waitq, - !fuse_block_alloc(fc, for_background), + if (wait_event_state_exclusive(fch->blocked_waitq, + !fuse_block_alloc(fch, for_background), (TASK_KILLABLE | TASK_FREEZABLE))) goto out; } - /* Matches smp_wmb() in fuse_set_initialized() */ - smp_rmb(); err = -ENOTCONN; - if (!fc->connected) - goto out; - - err = -ECONNREFUSED; - if (fc->conn_error) + if (!fch->connected) goto out; - req = fuse_request_alloc(fm, GFP_KERNEL); + req = fuse_request_alloc(fch, GFP_KERNEL); err = -ENOMEM; if (!req) { if (for_background) - wake_up(&fc->blocked_waitq); + wake_up(&fch->blocked_waitq); goto out; } - req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); - __set_bit(FR_WAITING, &req->flags); if (for_background) __set_bit(FR_BACKGROUND, &req->flags); - /* - * Keep the old behavior when idmappings support was not - * declared by a FUSE server. - * - * For those FUSE servers who support idmapped mounts, - * we send UID/GID only along with "inode creation" - * fuse requests, otherwise idmap == &invalid_mnt_idmap and - * req->in.h.{u,g}id will be equal to FUSE_INVALID_UIDGID. - */ - fsuid = no_idmap ? current_fsuid() : mapped_fsuid(idmap, fc->user_ns); - fsgid = no_idmap ? current_fsgid() : mapped_fsgid(idmap, fc->user_ns); - req->in.h.uid = from_kuid(fc->user_ns, fsuid); - req->in.h.gid = from_kgid(fc->user_ns, fsgid); - - if (no_idmap && unlikely(req->in.h.uid == ((uid_t)-1) || - req->in.h.gid == ((gid_t)-1))) { - fuse_put_request(req); - return ERR_PTR(-EOVERFLOW); - } - return req; out: - fuse_drop_waiting(fc); + fuse_drop_waiting(fch); return ERR_PTR(err); } static void fuse_put_request(struct fuse_req *req) { - struct fuse_conn *fc = req->fm->fc; + struct fuse_chan *fch = req->chan; if (refcount_dec_and_test(&req->count)) { if (test_bit(FR_BACKGROUND, &req->flags)) { @@ -273,15 +159,15 @@ static void fuse_put_request(struct fuse_req *req) * We get here in the unlikely case that a background * request was allocated but not sent */ - spin_lock(&fc->bg_lock); - if (!fc->blocked) - wake_up(&fc->blocked_waitq); - spin_unlock(&fc->bg_lock); + spin_lock(&fch->bg_lock); + if (!fch->blocked) + wake_up(&fch->blocked_waitq); + spin_unlock(&fch->bg_lock); } if (test_bit(FR_WAITING, &req->flags)) { __clear_bit(FR_WAITING, &req->flags); - fuse_drop_waiting(fc); + fuse_drop_waiting(fch); } fuse_request_free(req); @@ -327,14 +213,22 @@ EXPORT_SYMBOL_GPL(fuse_req_hash); /* * A new request is available, wake fiq->waitq */ -static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq) +static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock) { - wake_up(&fiq->waitq); + if (sync) + wake_up_sync(&fiq->waitq); + else + wake_up(&fiq->waitq); kill_fasync(&fiq->fasync, SIGIO, POLL_IN); spin_unlock(&fiq->lock); } +struct fuse_forget_link *fuse_alloc_forget(void) +{ + return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT); +} + void fuse_dev_queue_forget(struct fuse_iqueue *fiq, struct fuse_forget_link *forget) { @@ -342,7 +236,7 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, if (fiq->connected) { fiq->forget_list_tail->next = forget; fiq->forget_list_tail = forget; - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } else { kfree(forget); spin_unlock(&fiq->lock); @@ -352,7 +246,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) { spin_lock(&fiq->lock); - if (list_empty(&req->intr_entry)) { + /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */ + if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) { list_add_tail(&req->intr_entry, &fiq->interrupts); /* * Pairs with smp_mb() implied by test_and_set_bit() @@ -363,7 +258,7 @@ void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) list_del_init(&req->intr_entry); spin_unlock(&fiq->lock); } else { - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } } else { spin_unlock(&fiq->lock); @@ -393,11 +288,13 @@ EXPORT_SYMBOL_GPL(fuse_request_assign_unique); static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req) { + bool sync = test_and_clear_bit(FR_SYNC_WAKEUP, &req->flags); + spin_lock(&fiq->lock); if (fiq->connected) { fuse_request_assign_unique_locked(fiq, req); list_add_tail(&req->list, &fiq->pending); - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, sync); } else { spin_unlock(&fiq->lock); req->out.h.error = -ENOTCONN; @@ -406,12 +303,267 @@ static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req) } } -const struct fuse_iqueue_ops fuse_dev_fiq_ops = { +static const struct fuse_iqueue_ops fuse_dev_fiq_ops = { .send_forget = fuse_dev_queue_forget, .send_interrupt = fuse_dev_queue_interrupt, .send_req = fuse_dev_queue_req, }; -EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops); + +void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv) +{ + spin_lock_init(&fiq->lock); + init_waitqueue_head(&fiq->waitq); + INIT_LIST_HEAD(&fiq->pending); + INIT_LIST_HEAD(&fiq->interrupts); + fiq->forget_list_tail = &fiq->forget_list_head; + fiq->connected = 1; + fiq->ops = ops; + fiq->priv = priv; +} +EXPORT_SYMBOL_GPL(fuse_iqueue_init); + +void fuse_chan_release(struct fuse_chan *fch) +{ + struct fuse_iqueue *fiq = &fch->iq; + + if (fiq->ops->release) + fiq->ops->release(fiq); + + if (fch->timeout.req_timeout) + cancel_delayed_work_sync(&fch->timeout.work); +} + +void fuse_chan_free(struct fuse_chan *fch) +{ + WARN_ON(!list_empty(&fch->devices)); + kfree(fch->pq_prealloc); + kfree(fch); +} +EXPORT_SYMBOL_GPL(fuse_chan_free); + +struct fuse_chan *fuse_chan_new(void) +{ + struct fuse_chan *fch = kzalloc_obj(struct fuse_chan); + if (!fch) + return NULL; + + spin_lock_init(&fch->lock); + INIT_LIST_HEAD(&fch->devices); + spin_lock_init(&fch->bg_lock); + INIT_LIST_HEAD(&fch->bg_queue); + init_waitqueue_head(&fch->blocked_waitq); + atomic_set(&fch->num_waiting, 0); + fch->max_background = FUSE_DEFAULT_MAX_BACKGROUND; + fch->initialized = 0; + fch->blocked = 0; + fch->connected = 1; + fch->timeout.req_timeout = 0; + + return fch; +} +EXPORT_SYMBOL_GPL(fuse_chan_new); + +struct list_head *fuse_pqueue_alloc(void) +{ + struct list_head *pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE); + + if (pq) { + for (int i = 0; i < FUSE_PQ_HASH_SIZE; i++) + INIT_LIST_HEAD(&pq[i]); + } + return pq; +} + +struct fuse_chan *fuse_dev_chan_new(void) +{ + struct fuse_chan *fch __free(kfree) = fuse_chan_new(); + if (!fch) + return NULL; + + fch->pq_prealloc = fuse_pqueue_alloc(); + if (!fch->pq_prealloc) + return NULL; + + fuse_iqueue_init(&fch->iq, &fuse_dev_fiq_ops, NULL); + + return no_free_ptr(fch); +} +EXPORT_SYMBOL_GPL(fuse_dev_chan_new); + +unsigned int fuse_chan_num_background(struct fuse_chan *fch) +{ + return READ_ONCE(fch->num_background); +} + +unsigned int fuse_chan_max_background(struct fuse_chan *fch) +{ + return READ_ONCE(fch->max_background); +} + +void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val) +{ + spin_lock(&fch->bg_lock); + fch->max_background = val; + fch->blocked = fch->num_background >= fch->max_background; + if (!fch->blocked) + wake_up_nr(&fch->blocked_waitq, + fch->max_background - fch->num_background); + spin_unlock(&fch->bg_lock); +} + +unsigned int fuse_chan_num_waiting(struct fuse_chan *fch) +{ + return atomic_read(&fch->num_waiting); +} + +void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc) +{ + fch->conn = fc; +} + +void fuse_pqueue_init(struct fuse_pqueue *fpq) +{ + spin_lock_init(&fpq->lock); + INIT_LIST_HEAD(&fpq->io); + fpq->connected = 1; + fpq->processing = NULL; +} + +static struct fuse_dev *fuse_dev_alloc_no_pq(void) +{ + struct fuse_dev *fud; + + fud = kzalloc_obj(struct fuse_dev); + if (!fud) + return NULL; + + refcount_set(&fud->ref, 1); + fuse_pqueue_init(&fud->pq); + + return fud; +} + +struct fuse_dev *fuse_dev_alloc(void) +{ + struct fuse_dev *fud __free(kfree) = fuse_dev_alloc_no_pq(); + if (!fud) + return NULL; + + fud->pq.processing = fuse_pqueue_alloc(); + if (!fud->pq.processing) + return NULL; + + return no_free_ptr(fud); +} +EXPORT_SYMBOL_GPL(fuse_dev_alloc); + +/* + * Installs @fch into @fud, return true on success. "Consumes" @pq in either case. + */ +static bool fuse_dev_install_with_pq(struct fuse_dev *fud, struct fuse_chan *fch, + struct list_head *pq) +{ + struct fuse_chan *old_fch; + + guard(spinlock)(&fch->lock); + /* + * Pairs with: + * - xchg() in fuse_dev_release() + * - smp_load_acquire() in fuse_dev_fc_get() + */ + old_fch = cmpxchg(&fud->chan, NULL, fch); + if (old_fch) { + /* + * failed to set fud->chan because + * - it was already set to a different fc + * - it was set to disconneted + */ + kfree(pq); + return false; + } + if (pq) { + WARN_ON(fud->pq.processing); + fud->pq.processing = pq; + } + list_add_tail(&fud->entry, &fch->devices); + fuse_conn_get(fch->conn); + wake_up_all(&fuse_dev_waitq); + return true; +} + +void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch) +{ + struct list_head *pq = fch->pq_prealloc; + + fch->pq_prealloc = NULL; + if (!fuse_dev_install_with_pq(fud, fch, pq)) { + /* Channel is not usable without a dev */ + fuse_chan_abort(fch, false); + } +} +EXPORT_SYMBOL_GPL(fuse_dev_install); + +struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch) +{ + struct fuse_dev *fud; + + fud = fuse_dev_alloc_no_pq(); + if (!fud) + return NULL; + + fuse_dev_install(fud, fch); + return fud; +} +EXPORT_SYMBOL_GPL(fuse_dev_alloc_install); + +void fuse_dev_put(struct fuse_dev *fud) +{ + struct fuse_chan *fch; + + if (!refcount_dec_and_test(&fud->ref)) + return; + + fch = fuse_dev_chan_get(fud); + if (fch && fch != FUSE_DEV_CHAN_DISCONNECTED) { + /* This is the virtiofs case (fuse_dev_release() not called) */ + spin_lock(&fch->lock); + list_del(&fud->entry); + spin_unlock(&fch->lock); + + fuse_conn_put(fch->conn); + } + kfree(fud->pq.processing); + kfree(fud); +} +EXPORT_SYMBOL_GPL(fuse_dev_put); + +bool fuse_dev_is_installed(struct fuse_dev *fud) +{ + struct fuse_chan *fch = fuse_dev_chan_get(fud); + + return fch != NULL && fch != FUSE_DEV_CHAN_DISCONNECTED; +} + +/* + * Checks if @fc matches the one installed in @fud + */ +bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch) +{ + return fuse_dev_chan_get(fud) == fch; +} + +bool fuse_dev_is_sync_init(struct fuse_dev *fud) +{ + return fud->sync_init; +} + +struct fuse_dev *fuse_dev_grab(struct file *file) +{ + struct fuse_dev *fud = fuse_file_to_fud(file); + + refcount_inc(&fud->ref); + return fud; +} static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req) { @@ -421,10 +573,10 @@ static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req) fiq->ops->send_req(fiq, req); } -void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, - u64 nodeid, u64 nlookup) +void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget, + u64 nodeid, u64 nlookup) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fch->iq; forget->forget_one.nodeid = nodeid; forget->forget_one.nlookup = nlookup; @@ -432,21 +584,44 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, fiq->ops->send_forget(fiq, forget); } -static void flush_bg_queue(struct fuse_conn *fc) +static void flush_bg_queue(struct fuse_chan *fch) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fch->iq; - while (fc->active_background < fc->max_background && - !list_empty(&fc->bg_queue)) { + while (fch->active_background < fch->max_background && + !list_empty(&fch->bg_queue)) { struct fuse_req *req; - req = list_first_entry(&fc->bg_queue, struct fuse_req, list); + req = list_first_entry(&fch->bg_queue, struct fuse_req, list); list_del(&req->list); - fc->active_background++; + fch->active_background++; fuse_send_one(fiq, req); } } +void fuse_request_bg_finish(struct fuse_chan *fch, struct fuse_req *req) +{ + lockdep_assert_held(&fch->bg_lock); + + clear_bit(FR_BACKGROUND, &req->flags); + if (fch->num_background == fch->max_background) { + fch->blocked = 0; + wake_up(&fch->blocked_waitq); + } else if (!fch->blocked) { + /* + * Wake up next waiter, if any. It's okay to use + * waitqueue_active(), as we've already synced up + * fch->blocked with waiters with the wake_up() call + * above. + */ + if (waitqueue_active(&fch->blocked_waitq)) + wake_up(&fch->blocked_waitq); + } + + fch->num_background--; + fch->active_background--; +} + /* * This function is called when a request is finished. Either a reply * has arrived or it was aborted (and not yet sent) or some error @@ -457,9 +632,8 @@ static void flush_bg_queue(struct fuse_conn *fc) */ void fuse_request_end(struct fuse_req *req) { - struct fuse_mount *fm = req->fm; - struct fuse_conn *fc = fm->fc; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_chan *fch = req->chan; + struct fuse_iqueue *fiq = &fch->iq; if (test_and_set_bit(FR_FINISHED, &req->flags)) goto put_request; @@ -478,33 +652,17 @@ void fuse_request_end(struct fuse_req *req) WARN_ON(test_bit(FR_PENDING, &req->flags)); WARN_ON(test_bit(FR_SENT, &req->flags)); if (test_bit(FR_BACKGROUND, &req->flags)) { - spin_lock(&fc->bg_lock); - clear_bit(FR_BACKGROUND, &req->flags); - if (fc->num_background == fc->max_background) { - fc->blocked = 0; - wake_up(&fc->blocked_waitq); - } else if (!fc->blocked) { - /* - * Wake up next waiter, if any. It's okay to use - * waitqueue_active(), as we've already synced up - * fc->blocked with waiters with the wake_up() call - * above. - */ - if (waitqueue_active(&fc->blocked_waitq)) - wake_up(&fc->blocked_waitq); - } - - fc->num_background--; - fc->active_background--; - flush_bg_queue(fc); - spin_unlock(&fc->bg_lock); + spin_lock(&fch->bg_lock); + fuse_request_bg_finish(fch, req); + flush_bg_queue(fch); + spin_unlock(&fch->bg_lock); } else { /* Wake up waiter sleeping in request_wait_answer() */ wake_up(&req->waitq); } if (test_bit(FR_ASYNC, &req->flags)) - req->args->end(fm, req->args, req->out.h.error); + req->args->end(req->args, req->out.h.error); put_request: fuse_put_request(req); } @@ -512,7 +670,7 @@ EXPORT_SYMBOL_GPL(fuse_request_end); static int queue_interrupt(struct fuse_req *req) { - struct fuse_iqueue *fiq = &req->fm->fc->iq; + struct fuse_iqueue *fiq = &req->chan->iq; /* Check for we've sent request to interrupt this req */ if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) @@ -543,11 +701,11 @@ bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock) static void request_wait_answer(struct fuse_req *req) { - struct fuse_conn *fc = req->fm->fc; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_chan *fch = req->chan; + struct fuse_iqueue *fiq = &fch->iq; int err; - if (!fc->no_interrupt) { + if (!fch->no_interrupt) { /* Any signal may interrupt this */ err = wait_event_interruptible(req->waitq, test_bit(FR_FINISHED, &req->flags)); @@ -570,6 +728,11 @@ static void request_wait_answer(struct fuse_req *req) if (!err) return; + if (req->args->abort_on_kill) { + fuse_chan_abort(fch, false); + goto wait_for_finish; + } + if (test_bit(FR_URING, &req->flags)) removed = fuse_uring_remove_pending_req(req); else @@ -578,6 +741,7 @@ static void request_wait_answer(struct fuse_req *req) return; } +wait_for_finish: /* * Either request is already in userspace, or it was forced. * Wait it out. @@ -587,13 +751,18 @@ static void request_wait_answer(struct fuse_req *req) static void __fuse_request_send(struct fuse_req *req) { - struct fuse_iqueue *fiq = &req->fm->fc->iq; + struct fuse_iqueue *fiq = &req->chan->iq; BUG_ON(test_bit(FR_BACKGROUND, &req->flags)); /* acquire extra reference, since request is still needed after fuse_request_end() */ __fuse_get_request(req); + /* + * This is a synchronous request: the caller will block waiting for + * the answer. Hint the scheduler via wake_up_sync(). + */ + set_bit(FR_SYNC_WAKEUP, &req->flags); fuse_send_one(fiq, req); request_wait_answer(req); @@ -601,12 +770,12 @@ static void __fuse_request_send(struct fuse_req *req) smp_rmb(); } -static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args) +static void fuse_adjust_compat(struct fuse_chan *fch, struct fuse_args *args) { - if (fc->minor < 4 && args->opcode == FUSE_STATFS) + if (fch->minor < 4 && args->opcode == FUSE_STATFS) args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE; - if (fc->minor < 9) { + if (fch->minor < 9) { switch (args->opcode) { case FUSE_LOOKUP: case FUSE_CREATE: @@ -622,7 +791,7 @@ static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args) break; } } - if (fc->minor < 12) { + if (fch->minor < 12) { switch (args->opcode) { case FUSE_CREATE: args->in_args[0].size = sizeof(struct fuse_open_in); @@ -634,25 +803,13 @@ static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args) } } -static void fuse_force_creds(struct fuse_req *req) -{ - struct fuse_conn *fc = req->fm->fc; - - if (!req->fm->sb || req->fm->sb->s_iflags & SB_I_NOIDMAP) { - req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid()); - req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid()); - } else { - req->in.h.uid = FUSE_INVALID_UIDGID; - req->in.h.gid = FUSE_INVALID_UIDGID; - } - - req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); -} - static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) { req->in.h.opcode = args->opcode; req->in.h.nodeid = args->nodeid; + req->in.h.uid = args->uid; + req->in.h.gid = args->gid; + req->in.h.pid = args->pid; req->args = args; if (args->is_ext) req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; @@ -660,32 +817,26 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) __set_bit(FR_ASYNC, &req->flags); } -ssize_t __fuse_simple_request(struct mnt_idmap *idmap, - struct fuse_mount *fm, - struct fuse_args *args) +ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args) { - struct fuse_conn *fc = fm->fc; struct fuse_req *req; ssize_t ret; if (args->force) { - atomic_inc(&fc->num_waiting); - req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL); - - if (!args->nocreds) - fuse_force_creds(req); + atomic_inc(&fch->num_waiting); + req = fuse_request_alloc(fch, GFP_KERNEL | __GFP_NOFAIL); __set_bit(FR_WAITING, &req->flags); - __set_bit(FR_FORCE, &req->flags); + if (!args->abort_on_kill) + __set_bit(FR_FORCE, &req->flags); } else { - WARN_ON(args->nocreds); - req = fuse_get_req(idmap, fm, false); + req = fuse_get_req(fch, false); if (IS_ERR(req)) return PTR_ERR(req); } - /* Needs to be done after fuse_get_req() so that fc->minor is valid */ - fuse_adjust_compat(fc, args); + /* Needs to be done after fuse_get_req() so that fch->minor is valid */ + fuse_adjust_compat(fch, args); fuse_args_to_req(req, args); if (!args->noreply) @@ -702,10 +853,9 @@ ssize_t __fuse_simple_request(struct mnt_idmap *idmap, } #ifdef CONFIG_FUSE_IO_URING -static bool fuse_request_queue_background_uring(struct fuse_conn *fc, - struct fuse_req *req) +static bool fuse_request_queue_background_uring(struct fuse_req *req) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &req->chan->iq; req->in.h.len = sizeof(struct fuse_in_header) + fuse_len_args(req->args->in_numargs, @@ -721,50 +871,46 @@ static bool fuse_request_queue_background_uring(struct fuse_conn *fc, */ static int fuse_request_queue_background(struct fuse_req *req) { - struct fuse_mount *fm = req->fm; - struct fuse_conn *fc = fm->fc; + struct fuse_chan *fch = req->chan; bool queued = false; WARN_ON(!test_bit(FR_BACKGROUND, &req->flags)); if (!test_bit(FR_WAITING, &req->flags)) { __set_bit(FR_WAITING, &req->flags); - atomic_inc(&fc->num_waiting); + atomic_inc(&fch->num_waiting); } __set_bit(FR_ISREPLY, &req->flags); #ifdef CONFIG_FUSE_IO_URING - if (fuse_uring_ready(fc)) - return fuse_request_queue_background_uring(fc, req); + if (fuse_uring_ready(fch)) + return fuse_request_queue_background_uring(req); #endif - spin_lock(&fc->bg_lock); - if (likely(fc->connected)) { - fc->num_background++; - if (fc->num_background == fc->max_background) - fc->blocked = 1; - list_add_tail(&req->list, &fc->bg_queue); - flush_bg_queue(fc); + spin_lock(&fch->bg_lock); + if (likely(fch->connected)) { + fch->num_background++; + if (fch->num_background == fch->max_background) + fch->blocked = 1; + list_add_tail(&req->list, &fch->bg_queue); + flush_bg_queue(fch); queued = true; } - spin_unlock(&fc->bg_lock); + spin_unlock(&fch->bg_lock); return queued; } -int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, - gfp_t gfp_flags) +int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags) { struct fuse_req *req; if (args->force) { - WARN_ON(!args->nocreds); - req = fuse_request_alloc(fm, gfp_flags); + req = fuse_request_alloc(fch, gfp_flags); if (!req) return -ENOMEM; __set_bit(FR_BACKGROUND, &req->flags); } else { - WARN_ON(args->nocreds); - req = fuse_get_req(&invalid_mnt_idmap, fm, true); + req = fuse_get_req(fch, true); if (IS_ERR(req)) return PTR_ERR(req); } @@ -778,15 +924,13 @@ int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, return 0; } -EXPORT_SYMBOL_GPL(fuse_simple_background); -static int fuse_simple_notify_reply(struct fuse_mount *fm, - struct fuse_args *args, u64 unique) +int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique) { struct fuse_req *req; - struct fuse_iqueue *fiq = &fm->fc->iq; + struct fuse_iqueue *fiq = &fch->iq; - req = fuse_get_req(&invalid_mnt_idmap, fm, false); + req = fuse_get_req(fch, false); if (IS_ERR(req)) return PTR_ERR(req); @@ -1011,6 +1155,9 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop folio_clear_uptodate(newfolio); folio_clear_mappedtodisk(newfolio); + if (folio_test_large(newfolio)) + goto out_fallback_unlock; + if (fuse_check_folio(newfolio) != 0) goto out_fallback_unlock; @@ -1028,6 +1175,10 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop if (WARN_ON(folio_test_mlocked(oldfolio))) goto out_fallback_unlock; + err = lock_request(cs->req); + if (err) + goto out_fallback_unlock; + replace_page_cache_folio(oldfolio, newfolio); folio_get(newfolio); @@ -1041,20 +1192,7 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop */ pipe_buf_release(cs->pipe, buf); - err = 0; - spin_lock(&cs->req->waitq.lock); - if (test_bit(FR_ABORTED, &cs->req->flags)) - err = -ENOENT; - else - *foliop = newfolio; - spin_unlock(&cs->req->waitq.lock); - - if (err) { - folio_unlock(newfolio); - folio_put(newfolio); - goto out_put_old; - } - + *foliop = newfolio; folio_unlock(oldfolio); /* Drop ref for ap->pages[] array */ folio_put(oldfolio); @@ -1106,15 +1244,15 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio, cs->nr_segs++; cs->len = 0; - return 0; + return lock_request(cs->req); } /* * Copy a folio in the request to/from the userspace buffer. Must be * done atomically */ -static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, - unsigned offset, unsigned count, int zeroing) +int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, + unsigned offset, unsigned count, int zeroing) { int err; struct folio *folio = *foliop; @@ -1122,11 +1260,25 @@ static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, if (folio) { size = folio_size(folio); - if (zeroing && count < size) - folio_zero_range(folio, 0, size); + if (zeroing && count < size) { + /* + * When the copy is skipped the folio already holds the + * payload, so only the bytes outside [offset, offset + + * count) may be zeroed. + * + * Otherwise, the whole folio is cleared first so that a + * failed copy leaves zeros rather than stale folio + * contents. + */ + if (cs->skip_folio_copy) + folio_zero_segments(folio, 0, offset, + offset + count, size); + else + folio_zero_range(folio, 0, size); + } } - while (count) { + while (!cs->skip_folio_copy && count) { if (cs->write && cs->pipebufs && folio) { /* * Can't control lifetime of pipe buffers, so always @@ -1195,7 +1347,7 @@ static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes, } /* Copy a single argument in the request to/from userspace buffer */ -static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) +int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) { while (size) { if (!cs->len) { @@ -1219,6 +1371,10 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, for (i = 0; !err && i < numargs; i++) { struct fuse_arg *arg = &args[i]; if (i == numargs - 1 && argpages) + /* + * if cs->skip_folio_copy is set, this just does any + * needed zeroing. No copying is involved. + */ err = fuse_copy_folios(cs, arg->size, zeroing); else err = fuse_copy_one(cs, arg->value, arg->size); @@ -1245,34 +1401,29 @@ static int request_pending(struct fuse_iqueue *fiq) * * Called with fiq->lock held, releases it */ -static int fuse_read_interrupt(struct fuse_iqueue *fiq, - struct fuse_copy_state *cs, - size_t nbytes, struct fuse_req *req) +static int fuse_read_interrupt(struct fuse_iqueue *fiq, struct fuse_copy_state *cs) __releases(fiq->lock) { - struct fuse_in_header ih; - struct fuse_interrupt_in arg; - unsigned reqsize = sizeof(ih) + sizeof(arg); + struct fuse_req *req = list_first_entry(&fiq->interrupts, struct fuse_req, intr_entry); + struct fuse_interrupt_in arg = { + .unique = req->in.h.unique, + }; + struct fuse_in_header ih = { + .opcode = FUSE_INTERRUPT, + .unique = (req->in.h.unique | FUSE_INT_REQ_BIT), + .len = sizeof(ih) + sizeof(arg), + }; int err; list_del_init(&req->intr_entry); - memset(&ih, 0, sizeof(ih)); - memset(&arg, 0, sizeof(arg)); - ih.len = reqsize; - ih.opcode = FUSE_INTERRUPT; - ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT); - arg.unique = req->in.h.unique; - spin_unlock(&fiq->lock); - if (nbytes < reqsize) - return -EINVAL; err = fuse_copy_one(cs, &ih, sizeof(ih)); if (!err) err = fuse_copy_one(cs, &arg, sizeof(arg)); fuse_copy_finish(cs); - return err ? err : reqsize; + return err ? err : ih.len; } static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq, @@ -1298,8 +1449,7 @@ static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq, } static int fuse_read_single_forget(struct fuse_iqueue *fiq, - struct fuse_copy_state *cs, - size_t nbytes) + struct fuse_copy_state *cs) __releases(fiq->lock) { int err; @@ -1316,8 +1466,6 @@ __releases(fiq->lock) spin_unlock(&fiq->lock); kfree(forget); - if (nbytes < ih.len) - return -EINVAL; err = fuse_copy_one(cs, &ih, sizeof(ih)); if (!err) @@ -1345,11 +1493,6 @@ __releases(fiq->lock) .len = sizeof(ih) + sizeof(arg), }; - if (nbytes < ih.len) { - spin_unlock(&fiq->lock); - return -EINVAL; - } - max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one); head = fuse_dequeue_forget(fiq, max_forgets, &count); spin_unlock(&fiq->lock); @@ -1379,13 +1522,13 @@ __releases(fiq->lock) return ih.len; } -static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq, +static int fuse_read_forget(struct fuse_chan *fch, struct fuse_iqueue *fiq, struct fuse_copy_state *cs, size_t nbytes) __releases(fiq->lock) { - if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL) - return fuse_read_single_forget(fiq, cs, nbytes); + if (fch->minor < 16 || fiq->forget_list_head.next->next == NULL) + return fuse_read_single_forget(fiq, cs); else return fuse_read_batch_forget(fiq, cs, nbytes); } @@ -1403,8 +1546,8 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, struct fuse_copy_state *cs, size_t nbytes) { ssize_t err; - struct fuse_conn *fc = fud->fc; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_chan *fch = fud->chan; + struct fuse_iqueue *fiq = &fch->iq; struct fuse_pqueue *fpq = &fud->pq; struct fuse_req *req; struct fuse_args *args; @@ -1426,7 +1569,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER, sizeof(struct fuse_in_header) + sizeof(struct fuse_write_in) + - fc->max_write)) + fch->max_write)) return -EINVAL; restart: @@ -1445,19 +1588,16 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, } if (!fiq->connected) { - err = fc->aborted ? -ECONNABORTED : -ENODEV; + err = fch->abort_with_err ? -ECONNABORTED : -ENODEV; goto err_unlock; } - if (!list_empty(&fiq->interrupts)) { - req = list_entry(fiq->interrupts.next, struct fuse_req, - intr_entry); - return fuse_read_interrupt(fiq, cs, nbytes, req); - } + if (!list_empty(&fiq->interrupts)) + return fuse_read_interrupt(fiq, cs); if (forget_pending(fiq)) { if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0) - return fuse_read_forget(fc, fiq, cs, nbytes); + return fuse_read_forget(fch, fiq, cs, nbytes); if (fiq->forget_batch <= -8) fiq->forget_batch = 16; @@ -1483,12 +1623,11 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, spin_lock(&fpq->lock); /* * Must not put request on fpq->io queue after having been shut down by - * fuse_abort_conn() + * fuse_chan_abort() */ if (!fpq->connected) { req->out.h.error = err = -ECONNABORTED; goto out_end; - } list_add(&req->list, &fpq->io); spin_unlock(&fpq->lock); @@ -1501,7 +1640,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, spin_lock(&fpq->lock); clear_bit(FR_LOCKED, &req->flags); if (!fpq->connected) { - err = fc->aborted ? -ECONNABORTED : -ENODEV; + err = fch->abort_with_err ? -ECONNABORTED : -ENODEV; goto out_end; } if (err) { @@ -1516,6 +1655,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, list_move_tail(&req->list, &fpq->processing[hash]); __fuse_get_request(req); set_bit(FR_SENT, &req->flags); + trace_fuse_request_sent(req); spin_unlock(&fpq->lock); /* matches barrier in request_wait_answer() */ smp_mb__after_atomic(); @@ -1539,31 +1679,29 @@ out_end: static int fuse_dev_open(struct inode *inode, struct file *file) { - /* - * The fuse device's file's private_data is used to hold - * the fuse_conn(ection) when it is mounted, and is used to - * keep track of whether the file has been mounted already. - */ - file->private_data = NULL; + struct fuse_dev *fud = fuse_dev_alloc_no_pq(); + + if (!fud) + return -ENOMEM; + + file->private_data = fud; return 0; } struct fuse_dev *fuse_get_dev(struct file *file) { - struct fuse_dev *fud = __fuse_get_dev(file); + struct fuse_dev *fud = fuse_file_to_fud(file); int err; - if (likely(fud)) - return fud; + if (unlikely(!fuse_dev_chan_get(fud))) { + /* only block waiting for mount if sync init was requested */ + if (!fud->sync_init) + return ERR_PTR(-EPERM); - err = wait_event_interruptible(fuse_dev_waitq, - READ_ONCE(file->private_data) != FUSE_DEV_SYNC_INIT); - if (err) - return ERR_PTR(err); - - fud = __fuse_get_dev(file); - if (!fud) - return ERR_PTR(-EPERM); + err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_chan_get(fud) != NULL); + if (err) + return ERR_PTR(err); + } return fud; } @@ -1635,346 +1773,6 @@ out: return ret; } -static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_poll_wakeup_out outarg; - int err; - - if (size != sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - fuse_copy_finish(cs); - return fuse_notify_poll_wakeup(fc, &outarg); -} - -static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_inval_inode_out outarg; - int err; - - if (size != sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - fuse_copy_finish(cs); - - down_read(&fc->killsb); - err = fuse_reverse_inval_inode(fc, outarg.ino, - outarg.off, outarg.len); - up_read(&fc->killsb); - return err; -} - -static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_inval_entry_out outarg; - int err; - char *buf; - struct qstr name; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (outarg.namelen > fc->name_max) - return -ENAMETOOLONG; - - err = -EINVAL; - if (size != sizeof(outarg) + outarg.namelen + 1) - return -EINVAL; - - buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - name.name = buf; - name.len = outarg.namelen; - err = fuse_copy_one(cs, buf, outarg.namelen + 1); - if (err) - goto err; - fuse_copy_finish(cs); - buf[outarg.namelen] = 0; - - down_read(&fc->killsb); - err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags); - up_read(&fc->killsb); -err: - kfree(buf); - return err; -} - -static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_delete_out outarg; - int err; - char *buf; - struct qstr name; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (outarg.namelen > fc->name_max) - return -ENAMETOOLONG; - - if (size != sizeof(outarg) + outarg.namelen + 1) - return -EINVAL; - - buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - name.name = buf; - name.len = outarg.namelen; - err = fuse_copy_one(cs, buf, outarg.namelen + 1); - if (err) - goto err; - fuse_copy_finish(cs); - buf[outarg.namelen] = 0; - - down_read(&fc->killsb); - err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0); - up_read(&fc->killsb); -err: - kfree(buf); - return err; -} - -static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_store_out outarg; - struct inode *inode; - struct address_space *mapping; - u64 nodeid; - int err; - pgoff_t index; - unsigned int offset; - unsigned int num; - loff_t file_size; - loff_t end; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (size - sizeof(outarg) != outarg.size) - return -EINVAL; - - nodeid = outarg.nodeid; - - down_read(&fc->killsb); - - err = -ENOENT; - inode = fuse_ilookup(fc, nodeid, NULL); - if (!inode) - goto out_up_killsb; - - mapping = inode->i_mapping; - index = outarg.offset >> PAGE_SHIFT; - offset = outarg.offset & ~PAGE_MASK; - file_size = i_size_read(inode); - end = outarg.offset + outarg.size; - if (end > file_size) { - file_size = end; - fuse_write_update_attr(inode, file_size, outarg.size); - } - - num = outarg.size; - while (num) { - struct folio *folio; - unsigned int folio_offset; - unsigned int nr_bytes; - unsigned int nr_pages; - - folio = filemap_grab_folio(mapping, index); - err = PTR_ERR(folio); - if (IS_ERR(folio)) - goto out_iput; - - folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; - nr_bytes = min(num, folio_size(folio) - folio_offset); - nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; - - err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0); - if (!folio_test_uptodate(folio) && !err && offset == 0 && - (nr_bytes == folio_size(folio) || file_size == end)) { - folio_zero_segment(folio, nr_bytes, folio_size(folio)); - folio_mark_uptodate(folio); - } - folio_unlock(folio); - folio_put(folio); - - if (err) - goto out_iput; - - num -= nr_bytes; - offset = 0; - index += nr_pages; - } - - err = 0; - -out_iput: - iput(inode); -out_up_killsb: - up_read(&fc->killsb); - return err; -} - -struct fuse_retrieve_args { - struct fuse_args_pages ap; - struct fuse_notify_retrieve_in inarg; -}; - -static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, - int error) -{ - struct fuse_retrieve_args *ra = - container_of(args, typeof(*ra), ap.args); - - release_pages(ra->ap.folios, ra->ap.num_folios); - kfree(ra); -} - -static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, - struct fuse_notify_retrieve_out *outarg) -{ - int err; - struct address_space *mapping = inode->i_mapping; - pgoff_t index; - loff_t file_size; - unsigned int num; - unsigned int offset; - size_t total_len = 0; - unsigned int num_pages; - struct fuse_conn *fc = fm->fc; - struct fuse_retrieve_args *ra; - size_t args_size = sizeof(*ra); - struct fuse_args_pages *ap; - struct fuse_args *args; - - offset = outarg->offset & ~PAGE_MASK; - file_size = i_size_read(inode); - - num = min(outarg->size, fc->max_write); - if (outarg->offset > file_size) - num = 0; - else if (outarg->offset + num > file_size) - num = file_size - outarg->offset; - - num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; - num_pages = min(num_pages, fc->max_pages); - num = min(num, num_pages << PAGE_SHIFT); - - args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); - - ra = kzalloc(args_size, GFP_KERNEL); - if (!ra) - return -ENOMEM; - - ap = &ra->ap; - ap->folios = (void *) (ra + 1); - ap->descs = (void *) (ap->folios + num_pages); - - args = &ap->args; - args->nodeid = outarg->nodeid; - args->opcode = FUSE_NOTIFY_REPLY; - args->in_numargs = 3; - args->in_pages = true; - args->end = fuse_retrieve_end; - - index = outarg->offset >> PAGE_SHIFT; - - while (num && ap->num_folios < num_pages) { - struct folio *folio; - unsigned int folio_offset; - unsigned int nr_bytes; - unsigned int nr_pages; - - folio = filemap_get_folio(mapping, index); - if (IS_ERR(folio)) - break; - - folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; - nr_bytes = min(folio_size(folio) - folio_offset, num); - nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; - - ap->folios[ap->num_folios] = folio; - ap->descs[ap->num_folios].offset = folio_offset; - ap->descs[ap->num_folios].length = nr_bytes; - ap->num_folios++; - - offset = 0; - num -= nr_bytes; - total_len += nr_bytes; - index += nr_pages; - } - ra->inarg.offset = outarg->offset; - ra->inarg.size = total_len; - fuse_set_zero_arg0(args); - args->in_args[1].size = sizeof(ra->inarg); - args->in_args[1].value = &ra->inarg; - args->in_args[2].size = total_len; - - err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); - if (err) - fuse_retrieve_end(fm, args, err); - - return err; -} - -static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_retrieve_out outarg; - struct fuse_mount *fm; - struct inode *inode; - u64 nodeid; - int err; - - if (size != sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - fuse_copy_finish(cs); - - down_read(&fc->killsb); - err = -ENOENT; - nodeid = outarg.nodeid; - - inode = fuse_ilookup(fc, nodeid, &fm); - if (inode) { - err = fuse_retrieve(fm, inode, &outarg); - iput(inode); - } - up_read(&fc->killsb); - - return err; -} - /* * Resending all processing queue requests. * @@ -1988,143 +1786,56 @@ static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, * if the FUSE daemon takes careful measures to avoid processing duplicated * non-idempotent requests. */ -static void fuse_resend(struct fuse_conn *fc) +void fuse_chan_resend(struct fuse_chan *fch) { struct fuse_dev *fud; - struct fuse_req *req, *next; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_req *req; + struct fuse_iqueue *fiq = &fch->iq; LIST_HEAD(to_queue); unsigned int i; - spin_lock(&fc->lock); - if (!fc->connected) { - spin_unlock(&fc->lock); + spin_lock(&fch->lock); + if (!fch->connected) { + spin_unlock(&fch->lock); return; } - list_for_each_entry(fud, &fc->devices, entry) { + list_for_each_entry(fud, &fch->devices, entry) { struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - list_splice_tail_init(&fpq->processing[i], &to_queue); - spin_unlock(&fpq->lock); - } - spin_unlock(&fc->lock); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + struct list_head *this_queue = &fpq->processing[i]; - list_for_each_entry_safe(req, next, &to_queue, list) { - set_bit(FR_PENDING, &req->flags); - clear_bit(FR_SENT, &req->flags); - /* mark the request as resend request */ - req->in.h.unique |= FUSE_UNIQUE_RESEND; + list_for_each_entry(req, this_queue, list) + clear_bit(FR_SENT, &req->flags); + list_splice_tail_init(this_queue, &to_queue); + } + spin_unlock(&fpq->lock); } + spin_unlock(&fch->lock); spin_lock(&fiq->lock); if (!fiq->connected) { spin_unlock(&fiq->lock); - list_for_each_entry(req, &to_queue, list) - clear_bit(FR_PENDING, &req->flags); fuse_dev_end_requests(&to_queue); return; } - /* iq and pq requests are both oldest to newest */ - list_splice(&to_queue, &fiq->pending); - fuse_dev_wake_and_unlock(fiq); -} - -static int fuse_notify_resend(struct fuse_conn *fc) -{ - fuse_resend(fc); - return 0; -} - -/* - * Increments the fuse connection epoch. This will result of dentries from - * previous epochs to be invalidated. Additionally, if inval_wq is set, a work - * queue is scheduled to trigger the invalidation. - */ -static int fuse_notify_inc_epoch(struct fuse_conn *fc) -{ - atomic_inc(&fc->epoch); - if (inval_wq) - schedule_work(&fc->epoch_work); - - return 0; -} - -static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_prune_out outarg; - const unsigned int batch = 512; - u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL); - unsigned int num, i; - int err; - - if (!nodeids) - return -ENOMEM; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (size - sizeof(outarg) != outarg.count * sizeof(u64)) - return -EINVAL; - - for (; outarg.count; outarg.count -= num) { - num = min(batch, outarg.count); - err = fuse_copy_one(cs, nodeids, num * sizeof(u64)); - if (err) - return err; - - scoped_guard(rwsem_read, &fc->killsb) { - for (i = 0; i < num; i++) - fuse_try_prune_one_inode(fc, nodeids[i]); - } - } - return 0; -} - -static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, - unsigned int size, struct fuse_copy_state *cs) -{ - /* Don't try to move folios (yet) */ - cs->move_folios = false; - - switch (code) { - case FUSE_NOTIFY_POLL: - return fuse_notify_poll(fc, size, cs); - - case FUSE_NOTIFY_INVAL_INODE: - return fuse_notify_inval_inode(fc, size, cs); - - case FUSE_NOTIFY_INVAL_ENTRY: - return fuse_notify_inval_entry(fc, size, cs); - - case FUSE_NOTIFY_STORE: - return fuse_notify_store(fc, size, cs); - - case FUSE_NOTIFY_RETRIEVE: - return fuse_notify_retrieve(fc, size, cs); - - case FUSE_NOTIFY_DELETE: - return fuse_notify_delete(fc, size, cs); - - case FUSE_NOTIFY_RESEND: - return fuse_notify_resend(fc); - - case FUSE_NOTIFY_INC_EPOCH: - return fuse_notify_inc_epoch(fc); - - case FUSE_NOTIFY_PRUNE: - return fuse_notify_prune(fc, size, cs); + /* + * Remove interrupt entries for resent requests to prevent stale + * intr_entry on fiq->interrupts after the request is re-queued. + */ + list_for_each_entry(req, &to_queue, list) { + set_bit(FR_PENDING, &req->flags); + /* mark the request as resend request */ + req->in.h.unique |= FUSE_UNIQUE_RESEND; - default: - return -EINVAL; + if (test_bit(FR_INTERRUPTED, &req->flags)) + list_del_init(&req->intr_entry); } + /* iq and pq requests are both oldest to newest */ + list_splice(&to_queue, &fiq->pending); + fuse_dev_wake_and_unlock(fiq, false); } /* Look up request on processing list by unique ID */ @@ -2179,7 +1890,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, struct fuse_copy_state *cs, size_t nbytes) { int err; - struct fuse_conn *fc = fud->fc; + struct fuse_chan *fch = fud->chan; struct fuse_pqueue *fpq = &fud->pq; struct fuse_req *req; struct fuse_out_header oh; @@ -2201,7 +1912,19 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, * and error contains notification code. */ if (!oh.unique) { - err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); + /* + * Only allow notifications during while the connection is in an + * initialized and connected state + */ + err = -EINVAL; + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized) || !fch->connected) + goto copy_finish; + + /* Don't try to move folios (yet) */ + cs->move_folios = false; + + err = fuse_notify(fch->conn, oh.error, nbytes - sizeof(oh), cs); goto copy_finish; } @@ -2229,7 +1952,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, if (nbytes != sizeof(struct fuse_out_header)) err = -EINVAL; else if (oh.error == -ENOSYS) - fc->no_interrupt = 1; + fch->no_interrupt = 1; else if (oh.error == -EAGAIN) err = queue_interrupt(req); @@ -2389,7 +2112,7 @@ static __poll_t fuse_dev_poll(struct file *file, poll_table *wait) if (IS_ERR(fud)) return EPOLLERR; - fiq = &fud->fc->iq; + fiq = &fud->chan->iq; poll_wait(file, &fiq->waitq, wait); spin_lock(&fiq->lock); @@ -2415,21 +2138,6 @@ void fuse_dev_end_requests(struct list_head *head) } } -static void end_polls(struct fuse_conn *fc) -{ - struct rb_node *p; - - p = rb_first(&fc->polled_files); - - while (p) { - struct fuse_file *ff; - ff = rb_entry(p, struct fuse_file, polled_node); - wake_up_interruptible_all(&ff->poll_wait); - - p = rb_next(p); - } -} - /* * Abort all requests. * @@ -2448,27 +2156,29 @@ static void end_polls(struct fuse_conn *fc) * is OK, the request will in that case be removed from the list before we touch * it. */ -void fuse_abort_conn(struct fuse_conn *fc) +void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fch->iq; + + fch->abort_with_err = abort_with_err; - spin_lock(&fc->lock); - if (fc->connected) { + spin_lock(&fch->lock); + if (fch->connected) { struct fuse_dev *fud; struct fuse_req *req, *next; LIST_HEAD(to_end); unsigned int i; - if (fc->timeout.req_timeout) - cancel_delayed_work(&fc->timeout.work); + if (fch->timeout.req_timeout) + cancel_delayed_work(&fch->timeout.work); - /* Background queuing checks fc->connected under bg_lock */ - spin_lock(&fc->bg_lock); - fc->connected = 0; - spin_unlock(&fc->bg_lock); + /* Background queuing checks fch->connected under bg_lock */ + spin_lock(&fch->bg_lock); + fch->connected = 0; + spin_unlock(&fch->bg_lock); - fuse_set_initialized(fc); - list_for_each_entry(fud, &fc->devices, entry) { + fuse_chan_set_initialized(fch, NULL); + list_for_each_entry(fud, &fch->devices, entry) { struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); @@ -2489,11 +2199,11 @@ void fuse_abort_conn(struct fuse_conn *fc) &to_end); spin_unlock(&fpq->lock); } - spin_lock(&fc->bg_lock); - fc->blocked = 0; - fc->max_background = UINT_MAX; - flush_bg_queue(fc); - spin_unlock(&fc->bg_lock); + spin_lock(&fch->bg_lock); + fch->blocked = 0; + fch->max_background = UINT_MAX; + flush_bg_queue(fch); + spin_unlock(&fch->bg_lock); spin_lock(&fiq->lock); fiq->connected = 0; @@ -2505,57 +2215,66 @@ void fuse_abort_conn(struct fuse_conn *fc) wake_up_all(&fiq->waitq); spin_unlock(&fiq->lock); kill_fasync(&fiq->fasync, SIGIO, POLL_IN); - end_polls(fc); - wake_up_all(&fc->blocked_waitq); - spin_unlock(&fc->lock); + fuse_end_polls(fch->conn); + wake_up_all(&fch->blocked_waitq); + spin_unlock(&fch->lock); fuse_dev_end_requests(&to_end); /* - * fc->lock must not be taken to avoid conflicts with io-uring + * fch->lock must not be taken to avoid conflicts with io-uring * locks */ - fuse_uring_abort(fc); + fuse_uring_abort(fch); } else { - spin_unlock(&fc->lock); + spin_unlock(&fch->lock); } } -EXPORT_SYMBOL_GPL(fuse_abort_conn); +EXPORT_SYMBOL_GPL(fuse_chan_abort); -void fuse_wait_aborted(struct fuse_conn *fc) +void fuse_chan_wait_aborted(struct fuse_chan *fch) { /* matches implicit memory barrier in fuse_drop_waiting() */ smp_mb(); - wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0); + wait_event(fch->blocked_waitq, fuse_chan_num_waiting(fch) == 0); - fuse_uring_wait_stopped_queues(fc); + fuse_uring_wait_stopped_queues(fch); } int fuse_dev_release(struct inode *inode, struct file *file) { - struct fuse_dev *fud = __fuse_get_dev(file); + struct fuse_dev *fud = fuse_file_to_fud(file); + /* Pairs with cmpxchg() in fuse_dev_install() */ + struct fuse_chan *fch = xchg(&fud->chan, FUSE_DEV_CHAN_DISCONNECTED); - if (fud) { - struct fuse_conn *fc = fud->fc; + if (fch) { struct fuse_pqueue *fpq = &fud->pq; LIST_HEAD(to_end); unsigned int i; + bool last; + /* Make sure fuse_dev_install_with_pq() has finished */ + spin_lock(&fch->lock); spin_lock(&fpq->lock); WARN_ON(!list_empty(&fpq->io)); for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) list_splice_init(&fpq->processing[i], &to_end); spin_unlock(&fpq->lock); + list_del(&fud->entry); + /* Are we the last open device? */ + last = list_empty(&fch->devices); + spin_unlock(&fch->lock); + fuse_dev_end_requests(&to_end); - /* Are we the last open device? */ - if (atomic_dec_and_test(&fc->dev_count)) { - WARN_ON(fc->iq.fasync != NULL); - fuse_abort_conn(fc); + if (last) { + WARN_ON(fch->iq.fasync != NULL); + fuse_chan_abort(fch, false); } - fuse_dev_free(fud); + fuse_conn_put(fch->conn); } + fuse_dev_put(fud); return 0; } EXPORT_SYMBOL_GPL(fuse_dev_release); @@ -2568,31 +2287,14 @@ static int fuse_dev_fasync(int fd, struct file *file, int on) return PTR_ERR(fud); /* No locking - fasync_helper does its own locking */ - return fasync_helper(fd, file, on, &fud->fc->iq.fasync); -} - -static int fuse_device_clone(struct fuse_conn *fc, struct file *new) -{ - struct fuse_dev *fud; - - if (__fuse_get_dev(new)) - return -EINVAL; - - fud = fuse_dev_alloc_install(fc); - if (!fud) - return -ENOMEM; - - new->private_data = fud; - atomic_inc(&fc->dev_count); - - return 0; + return fasync_helper(fd, file, on, &fud->chan->iq.fasync); } static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) { - int res; int oldfd; - struct fuse_dev *fud = NULL; + struct fuse_dev *fud, *new_fud; + struct list_head *pq; if (get_user(oldfd, argp)) return -EFAULT; @@ -2605,17 +2307,22 @@ static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) * Check against file->f_op because CUSE * uses the same ioctl handler. */ - if (fd_file(f)->f_op == file->f_op) - fud = __fuse_get_dev(fd_file(f)); + if (fd_file(f)->f_op != file->f_op) + return -EINVAL; - res = -EINVAL; - if (fud) { - mutex_lock(&fuse_mutex); - res = fuse_device_clone(fud->fc, file); - mutex_unlock(&fuse_mutex); - } + fud = fuse_get_dev(fd_file(f)); + if (IS_ERR(fud)) + return PTR_ERR(fud); + + pq = fuse_pqueue_alloc(); + if (!pq) + return -ENOMEM; + + new_fud = fuse_file_to_fud(file); + if (!fuse_dev_install_with_pq(new_fud, fud->chan, pq)) + return -EINVAL; - return res; + return 0; } static long fuse_dev_ioctl_backing_open(struct file *file, @@ -2633,7 +2340,7 @@ static long fuse_dev_ioctl_backing_open(struct file *file, if (copy_from_user(&map, argp, sizeof(map))) return -EFAULT; - return fuse_backing_open(fud->fc, &map); + return fuse_backing_open(fud->chan->conn, &map); } static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp) @@ -2650,20 +2357,18 @@ static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp) if (get_user(backing_id, argp)) return -EFAULT; - return fuse_backing_close(fud->fc, backing_id); + return fuse_backing_close(fud->chan->conn, backing_id); } static long fuse_dev_ioctl_sync_init(struct file *file) { - int err = -EINVAL; + struct fuse_dev *fud = fuse_file_to_fud(file); - mutex_lock(&fuse_mutex); - if (!__fuse_get_dev(file)) { - WRITE_ONCE(file->private_data, FUSE_DEV_SYNC_INIT); - err = 0; - } - mutex_unlock(&fuse_mutex); - return err; + if (fuse_dev_chan_get(fud)) + return -EINVAL; + + fud->sync_init = true; + return 0; } static long fuse_dev_ioctl(struct file *file, unsigned int cmd, @@ -2696,7 +2401,7 @@ static void fuse_dev_show_fdinfo(struct seq_file *seq, struct file *file) if (!fud) return; - seq_printf(seq, "fuse_connection:\t%u\n", fud->fc->dev); + seq_printf(seq, "fuse_connection:\t%u\n", fuse_conn_get_id(fud->chan->conn)); } #endif |