diff options
Diffstat (limited to 'fs')
755 files changed, 29033 insertions, 18376 deletions
diff --git a/fs/9p/fid.c b/fs/9p/fid.c index f84412290a30..76242d450aa7 100644 --- a/fs/9p/fid.c +++ b/fs/9p/fid.c @@ -20,7 +20,9 @@ static inline void __add_fid(struct dentry *dentry, struct p9_fid *fid) { - hlist_add_head(&fid->dlist, (struct hlist_head *)&dentry->d_fsdata); + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + hlist_add_head(&fid->dlist, &v9fs_dentry->head); } @@ -112,6 +114,7 @@ void v9fs_open_fid_add(struct inode *inode, struct p9_fid **pfid) static struct p9_fid *v9fs_fid_find(struct dentry *dentry, kuid_t uid, int any) { + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); struct p9_fid *fid, *ret; p9_debug(P9_DEBUG_VFS, " dentry: %pd (%p) uid %d any %d\n", @@ -119,11 +122,9 @@ static struct p9_fid *v9fs_fid_find(struct dentry *dentry, kuid_t uid, int any) any); ret = NULL; /* we'll recheck under lock if there's anything to look in */ - if (dentry->d_fsdata) { - struct hlist_head *h = (struct hlist_head *)&dentry->d_fsdata; - + if (!hlist_empty(&v9fs_dentry->head)) { spin_lock(&dentry->d_lock); - hlist_for_each_entry(fid, h, dlist) { + hlist_for_each_entry(fid, &v9fs_dentry->head, dlist) { if (any || uid_eq(fid->uid, uid)) { ret = fid; p9_fid_get(ret); diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index acda42499ca9..274c5157135d 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -24,6 +24,9 @@ #include "v9fs_vfs.h" #include "cache.h" +/* cache=loose default negative dentry retention time is 24hours */ +#define CACHE_LOOSE_NDENTRY_TIMEOUT_DEFAULT (24 * 60 * 60 * 1000) + static DEFINE_SPINLOCK(v9fs_sessionlist_lock); static LIST_HEAD(v9fs_sessionlist); struct kmem_cache *v9fs_inode_cache; @@ -39,7 +42,7 @@ enum { * source if we rejected it as EINVAL */ Opt_source, /* Options that take integer arguments */ - Opt_debug, Opt_dfltuid, Opt_dfltgid, Opt_afid, + Opt_debug, Opt_dfltuid, Opt_dfltgid, Opt_afid, Opt_negtimeout, /* String options */ Opt_uname, Opt_remotename, Opt_cache, Opt_cachetag, /* Options that take no arguments */ @@ -93,6 +96,7 @@ const struct fs_parameter_spec v9fs_param_spec[] = { fsparam_string ("access", Opt_access), fsparam_flag ("posixacl", Opt_posixacl), fsparam_u32 ("locktimeout", Opt_locktimeout), + fsparam_s32 ("negtimeout", Opt_negtimeout), /* client options */ fsparam_u32 ("msize", Opt_msize), @@ -159,6 +163,9 @@ int v9fs_show_options(struct seq_file *m, struct dentry *root) from_kgid_munged(&init_user_ns, v9ses->dfltgid)); if (v9ses->afid != ~0) seq_printf(m, ",afid=%u", v9ses->afid); + if (v9ses->flags & V9FS_NDENTRY_TIMEOUT_SET) + seq_printf(m, ",negtimeout=%d", + (int)v9ses->ndentry_timeout_ms); if (strcmp(v9ses->uname, V9FS_DEFUSER) != 0) seq_printf(m, ",uname=%s", v9ses->uname); if (strcmp(v9ses->aname, V9FS_DEFANAME) != 0) @@ -337,6 +344,16 @@ int v9fs_parse_param(struct fs_context *fc, struct fs_parameter *param) session_opts->session_lock_timeout = (long)result.uint_32 * HZ; break; + case Opt_negtimeout: + session_opts->flags |= V9FS_NDENTRY_TIMEOUT_SET; + if (result.int_32 < 0) { + session_opts->ndentry_timeout_ms = + NDENTRY_TIMEOUT_NEVER; + } else { + session_opts->ndentry_timeout_ms = result.int_32; + } + break; + /* Options for client */ case Opt_msize: if (result.uint_32 < 4096) { @@ -426,6 +443,14 @@ static void v9fs_apply_options(struct v9fs_session_info *v9ses, v9ses->cache = ctx->session_opts.cache; v9ses->uid = ctx->session_opts.uid; v9ses->session_lock_timeout = ctx->session_opts.session_lock_timeout; + v9ses->ndentry_timeout_ms = ctx->session_opts.ndentry_timeout_ms; + + /* If negative dentry timeout has not been overridden set default for + * cache=loose + */ + if (!(v9ses->flags & V9FS_NDENTRY_TIMEOUT_SET) && + (v9ses->cache & CACHE_LOOSE)) + v9ses->ndentry_timeout_ms = CACHE_LOOSE_NDENTRY_TIMEOUT_DEFAULT; } /** diff --git a/fs/9p/v9fs.h b/fs/9p/v9fs.h index 6a12445d3858..a462bcbfc7da 100644 --- a/fs/9p/v9fs.h +++ b/fs/9p/v9fs.h @@ -24,6 +24,8 @@ * @V9FS_ACCESS_ANY: use a single attach for all users * @V9FS_ACCESS_MASK: bit mask of different ACCESS options * @V9FS_POSIX_ACL: POSIX ACLs are enforced + * @V9FS_NDENTRY_TIMEOUT_SET: Has negative dentry timeout retention time been + * overridden by negtimeout mount option * * Session flags reflect options selected by users at mount time */ @@ -34,16 +36,17 @@ #define V9FS_ACL_MASK V9FS_POSIX_ACL enum p9_session_flags { - V9FS_PROTO_2000U = 0x01, - V9FS_PROTO_2000L = 0x02, - V9FS_ACCESS_SINGLE = 0x04, - V9FS_ACCESS_USER = 0x08, - V9FS_ACCESS_CLIENT = 0x10, - V9FS_POSIX_ACL = 0x20, - V9FS_NO_XATTR = 0x40, - V9FS_IGNORE_QV = 0x80, /* ignore qid.version for cache hints */ - V9FS_DIRECT_IO = 0x100, - V9FS_SYNC = 0x200 + V9FS_PROTO_2000U = 0x01, + V9FS_PROTO_2000L = 0x02, + V9FS_ACCESS_SINGLE = 0x04, + V9FS_ACCESS_USER = 0x08, + V9FS_ACCESS_CLIENT = 0x10, + V9FS_POSIX_ACL = 0x20, + V9FS_NO_XATTR = 0x40, + V9FS_IGNORE_QV = 0x80, /* ignore qid.version for cache hints */ + V9FS_DIRECT_IO = 0x100, + V9FS_SYNC = 0x200, + V9FS_NDENTRY_TIMEOUT_SET = 0x400, }; /** @@ -91,6 +94,7 @@ enum p9_cache_bits { * @debug: debug level * @afid: authentication handle * @cache: cache mode of type &p9_cache_bits + * @ndentry_timeout: Negative dentry lookup cache retention time in ms * @cachetag: the tag of the cache associated with this session * @fscache: session cookie associated with FS-Cache * @uname: string user name to mount hierarchy as @@ -101,6 +105,7 @@ enum p9_cache_bits { * @uid: if %V9FS_ACCESS_SINGLE, the numeric uid which mounted the hierarchy * @clnt: reference to 9P network client instantiated for this session * @slist: reference to list of registered 9p sessions + * @ndentry_timeout_ms: Negative dentry caching retention time * * This structure holds state for each session instance established during * a sys_mount() . @@ -116,6 +121,7 @@ struct v9fs_session_info { unsigned short debug; unsigned int afid; unsigned int cache; + unsigned int ndentry_timeout_ms; #ifdef CONFIG_9P_FSCACHE char *cachetag; struct fscache_volume *fscache; @@ -133,6 +139,8 @@ struct v9fs_session_info { long session_lock_timeout; /* retry interval for blocking locks */ }; +#define NDENTRY_TIMEOUT_NEVER (-1U) + /* cache_validity flags */ #define V9FS_INO_INVALID_ATTR 0x01 diff --git a/fs/9p/v9fs_vfs.h b/fs/9p/v9fs_vfs.h index d3aefbec4de6..1856d91f8703 100644 --- a/fs/9p/v9fs_vfs.h +++ b/fs/9p/v9fs_vfs.h @@ -28,6 +28,19 @@ /* flags for v9fs_stat2inode() & v9fs_stat2inode_dotl() */ #define V9FS_STAT2INODE_KEEP_ISIZE 1 +/** + * struct v9fs_dentry - v9fs specific dentry data + * @head: List of fid associated with this dentry + * @expire_time: Lookup cache expiration time for negative dentries + * @rcu: used by kfree_rcu to schedule clean up job + */ +struct v9fs_dentry { + struct hlist_head head; + u64 expire_time; + struct rcu_head rcu; +}; +#define to_v9fs_dentry(d) ((struct v9fs_dentry *)((d)->d_fsdata)) + extern struct file_system_type v9fs_fs_type; extern const struct address_space_operations v9fs_addr_operations; extern const struct file_operations v9fs_file_operations; @@ -35,6 +48,8 @@ extern const struct file_operations v9fs_file_operations_dotl; extern const struct file_operations v9fs_dir_operations; extern const struct file_operations v9fs_dir_operations_dotl; extern const struct dentry_operations v9fs_dentry_operations; +extern void v9fs_ndentry_refresh_timeout(struct dentry *dentry); +extern void v9fs_dentry_fid_remove(struct dentry *dentry); extern const struct dentry_operations v9fs_cached_dentry_operations; extern struct kmem_cache *v9fs_inode_cache; @@ -75,17 +90,4 @@ static inline void v9fs_invalidate_inode_attr(struct inode *inode) int v9fs_open_to_dotl_flags(int flags); -static inline void v9fs_i_size_write(struct inode *inode, loff_t i_size) -{ - /* - * 32-bit need the lock, concurrent updates could break the - * sequences and make i_size_read() loop forever. - * 64-bit updates are atomic and can skip the locking. - */ - if (sizeof(i_size) > sizeof(long)) - spin_lock(&inode->i_lock); - i_size_write(inode, i_size); - if (sizeof(i_size) > sizeof(long)) - spin_unlock(&inode->i_lock); -} #endif diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c index c21d33830f5f..1ac0b3dcc077 100644 --- a/fs/9p/vfs_addr.c +++ b/fs/9p/vfs_addr.c @@ -70,11 +70,34 @@ static void v9fs_issue_read(struct netfs_io_subrequest *subreq) { struct netfs_io_request *rreq = subreq->rreq; struct p9_fid *fid = rreq->netfs_priv; + char *target; unsigned long long pos = subreq->start + subreq->transferred; - int total, err; - - total = p9_client_read(fid, pos, &subreq->io_iter, &err); + int total = 0, err, len, n; + + if (S_ISLNK(rreq->inode->i_mode)) { + /* p9_client_readlink() must not be called for legacy protocols + * 9p2000 or 9p2000.u. + */ + BUG_ON(!p9_is_proto_dotl(fid->clnt)); + if (WARN_ON_ONCE(pos)) { + /* reading a link at a non null offset should + * not happen + */ + err = -EIO; + goto fill_subreq; + } + err = p9_client_readlink(fid, &target); + if (err != 0) + goto fill_subreq; + len = strlen(target); + n = copy_to_iter(target, len, &subreq->io_iter); + kfree(target); + total = n; + } else { + total = p9_client_read(fid, pos, &subreq->io_iter, &err); + } +fill_subreq: /* if we just extended the file size, any portion not in * cache won't be on server and is zeroes */ if (subreq->rreq->origin != NETFS_UNBUFFERED_READ && @@ -99,6 +122,7 @@ static void v9fs_issue_read(struct netfs_io_subrequest *subreq) static int v9fs_init_request(struct netfs_io_request *rreq, struct file *file) { struct p9_fid *fid; + struct dentry *dentry; bool writing = (rreq->origin == NETFS_READ_FOR_WRITE || rreq->origin == NETFS_WRITETHROUGH || rreq->origin == NETFS_UNBUFFERED_WRITE || @@ -115,6 +139,14 @@ static int v9fs_init_request(struct netfs_io_request *rreq, struct file *file) if (!fid) goto no_fid; p9_fid_get(fid); + } else if (S_ISLNK(rreq->inode->i_mode)) { + dentry = d_find_any_alias(rreq->inode); + if (!dentry) + goto no_fid; + fid = v9fs_fid_lookup(dentry); + dput(dentry); + if (IS_ERR(fid)) + goto no_fid; } else { fid = v9fs_fid_find_inode(rreq->inode, writing, INVALID_UID, true); if (!fid) diff --git a/fs/9p/vfs_dentry.c b/fs/9p/vfs_dentry.c index c5bf74d547e8..e549e222602e 100644 --- a/fs/9p/vfs_dentry.c +++ b/fs/9p/vfs_dentry.c @@ -24,6 +24,46 @@ #include "fid.h" /** + * v9fs_ndentry_is_expired - Check if negative dentry lookup has expired + * + * This should be called to know if a negative dentry should be removed from + * cache. + * + * @dentry: dentry in question + * + */ +static bool v9fs_ndentry_is_expired(struct dentry const *dentry) +{ + struct v9fs_session_info *v9ses = v9fs_dentry2v9ses(dentry); + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + if (v9ses->ndentry_timeout_ms == NDENTRY_TIMEOUT_NEVER) + return false; + + return time_before_eq64(v9fs_dentry->expire_time, get_jiffies_64()); +} + +/** + * v9fs_ndentry_refresh_timeout - Refresh negative dentry lookup cache timeout + * + * This should be called when a look up yields a negative entry. + * + * @dentry: dentry in question + * + */ +void v9fs_ndentry_refresh_timeout(struct dentry *dentry) +{ + struct v9fs_session_info *v9ses = v9fs_dentry2v9ses(dentry); + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + if (v9ses->ndentry_timeout_ms == NDENTRY_TIMEOUT_NEVER) + return; + + v9fs_dentry->expire_time = get_jiffies_64() + + msecs_to_jiffies(v9ses->ndentry_timeout_ms); +} + +/** * v9fs_cached_dentry_delete - called when dentry refcount equals 0 * @dentry: dentry in question * @@ -33,20 +73,15 @@ static int v9fs_cached_dentry_delete(const struct dentry *dentry) p9_debug(P9_DEBUG_VFS, " dentry: %pd (%p)\n", dentry, dentry); - /* Don't cache negative dentries */ - if (d_really_is_negative(dentry)) - return 1; - return 0; -} + if (!d_really_is_negative(dentry)) + return 0; -/** - * v9fs_dentry_release - called when dentry is going to be freed - * @dentry: dentry that is being release - * - */ + return v9fs_ndentry_is_expired(dentry); +} -static void v9fs_dentry_release(struct dentry *dentry) +static void __v9fs_dentry_fid_remove(struct dentry *dentry) { + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); struct hlist_node *p, *n; struct hlist_head head; @@ -54,13 +89,54 @@ static void v9fs_dentry_release(struct dentry *dentry) dentry, dentry); spin_lock(&dentry->d_lock); - hlist_move_list((struct hlist_head *)&dentry->d_fsdata, &head); + hlist_move_list(&v9fs_dentry->head, &head); spin_unlock(&dentry->d_lock); hlist_for_each_safe(p, n, &head) p9_fid_put(hlist_entry(p, struct p9_fid, dlist)); } +/** + * v9fs_dentry_fid_remove - Release all dentry's fids + * @dentry: dentry in question + * + */ +void v9fs_dentry_fid_remove(struct dentry *dentry) +{ + __v9fs_dentry_fid_remove(dentry); +} + +/** + * v9fs_dentry_init - Initialize v9fs dentry data + * @dentry: dentry in question + * + */ +static int v9fs_dentry_init(struct dentry *dentry) +{ + struct v9fs_dentry *v9fs_dentry = kzalloc(sizeof(*v9fs_dentry), + GFP_KERNEL); + + if (!v9fs_dentry) + return -ENOMEM; + + INIT_HLIST_HEAD(&v9fs_dentry->head); + dentry->d_fsdata = (void *)v9fs_dentry; + return 0; +} + +/** + * v9fs_dentry_release - called when dentry is going to be freed + * @dentry: dentry that is being released + * + */ +static void v9fs_dentry_release(struct dentry *dentry) +{ + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + __v9fs_dentry_fid_remove(dentry); + kfree_rcu(v9fs_dentry, rcu); +} + static int __v9fs_lookup_revalidate(struct dentry *dentry, unsigned int flags) { struct p9_fid *fid; @@ -72,7 +148,7 @@ static int __v9fs_lookup_revalidate(struct dentry *dentry, unsigned int flags) inode = d_inode(dentry); if (!inode) - goto out_valid; + return !v9fs_ndentry_is_expired(dentry); v9inode = V9FS_I(inode); if (v9inode->cache_validity & V9FS_INO_INVALID_ATTR) { @@ -112,7 +188,6 @@ static int __v9fs_lookup_revalidate(struct dentry *dentry, unsigned int flags) return retval; } } -out_valid: p9_debug(P9_DEBUG_VFS, "dentry: %pd (%p) is valid\n", dentry, dentry); return 1; } @@ -139,12 +214,14 @@ const struct dentry_operations v9fs_cached_dentry_operations = { .d_revalidate = v9fs_lookup_revalidate, .d_weak_revalidate = __v9fs_lookup_revalidate, .d_delete = v9fs_cached_dentry_delete, + .d_init = v9fs_dentry_init, .d_release = v9fs_dentry_release, .d_unalias_trylock = v9fs_dentry_unalias_trylock, .d_unalias_unlock = v9fs_dentry_unalias_unlock, }; const struct dentry_operations v9fs_dentry_operations = { + .d_init = v9fs_dentry_init, .d_release = v9fs_dentry_release, .d_unalias_trylock = v9fs_dentry_unalias_trylock, .d_unalias_unlock = v9fs_dentry_unalias_unlock, diff --git a/fs/9p/vfs_dir.c b/fs/9p/vfs_dir.c index e0d34e4e9076..323f85352f6a 100644 --- a/fs/9p/vfs_dir.c +++ b/fs/9p/vfs_dir.c @@ -27,6 +27,7 @@ * struct p9_rdir - readdir accounting * @head: start offset of current dirread buffer * @tail: end offset of current dirread buffer + * @offset: file position the data at @head corresponds to * @buf: dirread buffer * * private structure for keeping track of readdir @@ -36,6 +37,7 @@ struct p9_rdir { int head; int tail; + loff_t offset; uint8_t buf[]; }; @@ -70,7 +72,7 @@ static struct p9_rdir *v9fs_alloc_rdir_buf(struct file *filp, int buflen) struct p9_fid *fid = filp->private_data; if (!fid->rdir) - fid->rdir = kzalloc(sizeof(struct p9_rdir) + buflen, GFP_KERNEL); + fid->rdir = kvzalloc(sizeof(struct p9_rdir) + buflen, GFP_KERNEL); return fid->rdir; } @@ -102,6 +104,9 @@ static int v9fs_dir_readdir(struct file *file, struct dir_context *ctx) kvec.iov_base = rdir->buf; kvec.iov_len = buflen; + if (rdir->head < rdir->tail && rdir->offset != ctx->pos) + rdir->head = rdir->tail = 0; + while (1) { if (rdir->tail == rdir->head) { struct iov_iter to; @@ -117,6 +122,7 @@ static int v9fs_dir_readdir(struct file *file, struct dir_context *ctx) rdir->head = 0; rdir->tail = n; + rdir->offset = ctx->pos; } while (rdir->head < rdir->tail) { err = p9stat_read(fid->clnt, rdir->buf + rdir->head, @@ -134,6 +140,7 @@ static int v9fs_dir_readdir(struct file *file, struct dir_context *ctx) rdir->head += err; ctx->pos += err; + rdir->offset = ctx->pos; } } } @@ -161,6 +168,9 @@ static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) if (!rdir) return -ENOMEM; + if (rdir->head < rdir->tail && rdir->offset != ctx->pos) + rdir->head = rdir->tail = 0; + while (1) { if (rdir->tail == rdir->head) { err = p9_client_readdir(fid, rdir->buf, buflen, @@ -170,6 +180,7 @@ static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) rdir->head = 0; rdir->tail = err; + rdir->offset = ctx->pos; } while (rdir->head < rdir->tail) { @@ -190,6 +201,7 @@ static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) ctx->pos = curdirent.d_off; rdir->head += err; + rdir->offset = ctx->pos; } } } diff --git a/fs/9p/vfs_file.c b/fs/9p/vfs_file.c index c5e73c37baea..cddfb4b7ce4e 100644 --- a/fs/9p/vfs_file.c +++ b/fs/9p/vfs_file.c @@ -198,7 +198,7 @@ static int v9fs_file_do_lock(struct file *filp, int cmd, struct file_lock *fl) res = -EAGAIN; break; default: - WARN_ONCE(1, "unknown lock status code: %d\n", status); + p9_debug(P9_DEBUG_ERROR, "unknown lock status code: %d\n", status); fallthrough; case P9_LOCK_ERROR: case P9_LOCK_GRACE: diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index d1508b1fe109..5783d0336f96 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -302,10 +302,12 @@ int v9fs_init_inode(struct v9fs_session_info *v9ses, goto error; } - if (v9fs_proto_dotl(v9ses)) + if (v9fs_proto_dotl(v9ses)) { inode->i_op = &v9fs_symlink_inode_operations_dotl; - else + inode_nohighmem(inode); + } else { inode->i_op = &v9fs_symlink_inode_operations; + } break; case S_IFDIR: @@ -488,10 +490,19 @@ static int v9fs_at_to_dotl_flags(int flags) * - ext4 (with dir_nlink feature enabled) sets nlink to 1 if a dir has more * than EXT4_LINK_MAX (65000) links. * + * In cacheless mode the server is the source of truth for nlink and the + * inode is going away immediately, so locally adjusting i_nlink buys + * nothing and races with concurrent metadata fetches that may already + * have observed the post-unlink value (nlink == 0). + * * @inode: inode whose nlink is being dropped */ static void v9fs_dec_count(struct inode *inode) { + struct v9fs_session_info *v9ses = v9fs_inode2v9ses(inode); + + if (!(v9ses->cache & (CACHE_META | CACHE_LOOSE))) + return; if (!S_ISDIR(inode->i_mode) || inode->i_nlink > 2) drop_nlink(inode); } @@ -549,7 +560,7 @@ static int v9fs_remove(struct inode *dir, struct dentry *dentry, int flags) /* invalidate all fids associated with dentry */ /* NOTE: This will not include open fids */ - dentry->d_op->d_release(dentry); + v9fs_dentry_fid_remove(dentry); } return retval; } @@ -672,27 +683,20 @@ v9fs_vfs_create(struct mnt_idmap *idmap, struct inode *dir, static struct dentry *v9fs_vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode) { - int err; u32 perm; struct p9_fid *fid; struct v9fs_session_info *v9ses; p9_debug(P9_DEBUG_VFS, "name %pd\n", dentry); - err = 0; v9ses = v9fs_inode2v9ses(dir); perm = unixmode2p9mode(v9ses, mode | S_IFDIR); fid = v9fs_create(v9ses, dir, dentry, NULL, perm, P9_OREAD); - if (IS_ERR(fid)) { - err = PTR_ERR(fid); - fid = NULL; - } else { - inc_nlink(dir); - v9fs_invalidate_inode_attr(dir); - } - - if (fid) - p9_fid_put(fid); - return ERR_PTR(err); + if (IS_ERR(fid)) + return ERR_CAST(fid); + inc_nlink(dir); + v9fs_invalidate_inode_attr(dir); + p9_fid_put(fid); + return NULL; } /** @@ -732,14 +736,16 @@ struct dentry *v9fs_vfs_lookup(struct inode *dir, struct dentry *dentry, name = dentry->d_name.name; fid = p9_client_walk(dfid, 1, &name, 1); p9_fid_put(dfid); - if (fid == ERR_PTR(-ENOENT)) + if (fid == ERR_PTR(-ENOENT)) { inode = NULL; - else if (IS_ERR(fid)) + v9fs_ndentry_refresh_timeout(dentry); + } else if (IS_ERR(fid)) { inode = ERR_CAST(fid); - else if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) + } else if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) { inode = v9fs_get_inode_from_fid(v9ses, fid, dir->i_sb); - else + } else { inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + } /* * If we had a rename on the server and a parallel lookup * for the new name, then make sure we instantiate with @@ -1141,11 +1147,13 @@ v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, mode |= inode->i_mode & ~S_IALLUGO; inode->i_mode = mode; - v9inode->netfs.remote_i_size = stat->length; + spin_lock(&inode->i_lock); + netfs_write_remote_i_size(inode, stat->length); if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE)) - v9fs_i_size_write(inode, stat->length); + i_size_write(inode, stat->length); /* not real number of blocks, but 512 byte ones ... */ inode->i_blocks = (stat->length + 512 - 1) >> 9; + spin_unlock(&inode->i_lock); v9inode->cache_validity &= ~V9FS_INO_INVALID_ATTR; } diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 71796a89bcf4..f7396d20cb6c 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -349,7 +349,7 @@ static struct dentry *v9fs_vfs_mkdir_dotl(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t omode) { - int err; + int err = 0; struct v9fs_session_info *v9ses; struct p9_fid *fid = NULL, *dfid = NULL; kgid_t gid; @@ -412,7 +412,7 @@ error: p9_fid_put(fid); v9fs_put_acl(dacl, pacl); p9_fid_put(dfid); - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; } static int @@ -634,10 +634,12 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode, mode |= inode->i_mode & ~S_IALLUGO; inode->i_mode = mode; - v9inode->netfs.remote_i_size = stat->st_size; + spin_lock(&inode->i_lock); + netfs_write_remote_i_size(inode, stat->st_size); if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE)) - v9fs_i_size_write(inode, stat->st_size); + i_size_write(inode, stat->st_size); inode->i_blocks = stat->st_blocks; + spin_unlock(&inode->i_lock); } else { if (stat->st_result_mask & P9_STATS_ATIME) { inode_set_atime(inode, stat->st_atime_sec, @@ -662,13 +664,15 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode, mode |= inode->i_mode & ~S_IALLUGO; inode->i_mode = mode; } + spin_lock(&inode->i_lock); if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE) && stat->st_result_mask & P9_STATS_SIZE) { - v9inode->netfs.remote_i_size = stat->st_size; - v9fs_i_size_write(inode, stat->st_size); + netfs_write_remote_i_size(inode, stat->st_size); + i_size_write(inode, stat->st_size); } if (stat->st_result_mask & P9_STATS_BLOCKS) inode->i_blocks = stat->st_blocks; + spin_unlock(&inode->i_lock); } if (stat->st_result_mask & P9_STATS_GEN) inode->i_generation = stat->st_gen; @@ -686,9 +690,11 @@ v9fs_vfs_symlink_dotl(struct mnt_idmap *idmap, struct inode *dir, int err; kgid_t gid; const unsigned char *name; + struct v9fs_session_info *v9ses; struct p9_qid qid; struct p9_fid *dfid; struct p9_fid *fid = NULL; + struct inode *inode; name = dentry->d_name.name; p9_debug(P9_DEBUG_VFS, "%llu,%s,%s\n", dir->i_ino, name, symname); @@ -712,6 +718,26 @@ v9fs_vfs_symlink_dotl(struct mnt_idmap *idmap, struct inode *dir, v9fs_invalidate_inode_attr(dir); + /* instantiate inode and assign the unopened fid to the dentry */ + fid = p9_client_walk(dfid, 1, &name, 1); + if (IS_ERR(fid)) { + err = PTR_ERR(fid); + p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n", + err); + goto error; + } + + v9ses = v9fs_inode2v9ses(dir); + inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + p9_debug(P9_DEBUG_VFS, "inode creation failed %d\n", + err); + goto error; + } + v9fs_fid_add(dentry, &fid); + d_instantiate(dentry, inode); + err = 0; error: p9_fid_put(fid); p9_fid_put(dfid); @@ -853,16 +879,18 @@ error: } /** - * v9fs_vfs_get_link_dotl - follow a symlink path + * v9fs_vfs_get_link_nocache_dotl - Resolve a symlink directly. + * + * To be used when symlink caching is not enabled. + * * @dentry: dentry for symlink * @inode: inode for symlink * @done: destructor for return value */ - static const char * -v9fs_vfs_get_link_dotl(struct dentry *dentry, - struct inode *inode, - struct delayed_call *done) +v9fs_vfs_get_link_nocache_dotl(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) { struct p9_fid *fid; char *target; @@ -884,6 +912,26 @@ v9fs_vfs_get_link_dotl(struct dentry *dentry, return target; } +/** + * v9fs_vfs_get_link_dotl - follow a symlink path + * @dentry: dentry for symlink + * @inode: inode for symlink + * @done: destructor for return value + */ +static const char * +v9fs_vfs_get_link_dotl(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct v9fs_session_info *v9ses; + + v9ses = v9fs_inode2v9ses(inode); + if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) + return page_get_link(dentry, inode, done); + + return v9fs_vfs_get_link_nocache_dotl(dentry, inode, done); +} + int v9fs_refresh_inode_dotl(struct p9_fid *fid, struct inode *inode) { struct p9_stat_dotl *st; diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c index 431f24938a1d..94d6b02c221b 100644 --- a/fs/9p/vfs_super.c +++ b/fs/9p/vfs_super.c @@ -330,6 +330,7 @@ static int v9fs_init_fs_context(struct fs_context *fc) ctx->session_opts.uid = INVALID_UID; ctx->session_opts.dfltuid = V9FS_DEFUID; ctx->session_opts.dfltgid = V9FS_DEFGID; + ctx->session_opts.ndentry_timeout_ms = 0; /* initialize client options */ ctx->client_opts.proto_version = p9_proto_2000L; diff --git a/fs/Kconfig b/fs/Kconfig index 43cb06de297f..cf6ae64776e6 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -78,7 +78,7 @@ config FS_DAX --map=mem: https://docs.pmem.io/ndctl-user-guide/ndctl-man-pages/ndctl-create-namespace - For ndctl to work CONFIG_DEV_DAX needs to be enabled as well. For most + For ndctl to work CONFIG_DEV_DAX needs to be enabled as well. For most file systems DAX support needs to be manually enabled globally or per-inode using a mount option as well. See the file documentation in Documentation/filesystems/dax.rst for details. @@ -116,8 +116,8 @@ config FILE_LOCKING default y help This option enables standard file locking support, required - for filesystems like NFS and for the flock() system - call. Disabling this option saves about 11k. + for filesystems like NFS and for the flock() system + call. Disabling this option saves about 11k. source "fs/crypto/Kconfig" diff --git a/fs/Makefile b/fs/Makefile index ae1b07f9c6a0..89a8a9d207d1 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -27,7 +27,6 @@ obj-y += anon_inodes.o obj-$(CONFIG_SIGNALFD) += signalfd.o obj-$(CONFIG_TIMERFD) += timerfd.o obj-$(CONFIG_EVENTFD) += eventfd.o -obj-$(CONFIG_USERFAULTFD) += userfaultfd.o obj-$(CONFIG_AIO) += aio.o obj-$(CONFIG_FS_DAX) += dax.o obj-$(CONFIG_FS_ENCRYPTION) += crypto/ diff --git a/fs/affs/affs.h b/fs/affs/affs.h index a0caf6ace860..44a3f69d275f 100644 --- a/fs/affs/affs.h +++ b/fs/affs/affs.h @@ -227,11 +227,6 @@ static inline bool affs_validblock(struct super_block *sb, int block) block < AFFS_SB(sb)->s_partition_size); } -static inline void -affs_set_blocksize(struct super_block *sb, int size) -{ - sb_set_blocksize(sb, size); -} static inline struct buffer_head * affs_bread(struct super_block *sb, int block) { diff --git a/fs/affs/super.c b/fs/affs/super.c index 079f36e1ddec..b232251aa7bb 100644 --- a/fs/affs/super.c +++ b/fs/affs/super.c @@ -358,7 +358,8 @@ static int affs_fill_super(struct super_block *sb, struct fs_context *fc) size = bdev_nr_sectors(sb->s_bdev); pr_debug("initial blocksize=%d, #blocks=%d\n", 512, size); - affs_set_blocksize(sb, PAGE_SIZE); + if (!sb_set_blocksize(sb, PAGE_SIZE)) + return -EINVAL; /* Try to find root block. Its location depends on the block size. */ i = bdev_logical_block_size(sb->s_bdev); @@ -374,7 +375,8 @@ static int affs_fill_super(struct super_block *sb, struct fs_context *fc) if (ctx->root_block < 0) sbi->s_root_block = (ctx->reserved + size - 1) / 2; pr_debug("setting blocksize to %d\n", blocksize); - affs_set_blocksize(sb, blocksize); + if (!sb_set_blocksize(sb, blocksize)) + return -EINVAL; sbi->s_partition_size = size; /* The root block location that was calculated above is not diff --git a/fs/afs/Makefile b/fs/afs/Makefile index b49b8fe682f3..0d8f1982d596 100644 --- a/fs/afs/Makefile +++ b/fs/afs/Makefile @@ -30,6 +30,7 @@ kafs-y := \ server.o \ server_list.o \ super.o \ + symlink.o \ validation.o \ vlclient.o \ vl_alias.o \ diff --git a/fs/afs/callback.c b/fs/afs/callback.c index 894d2bad6b6c..61354003c006 100644 --- a/fs/afs/callback.c +++ b/fs/afs/callback.c @@ -113,16 +113,12 @@ static struct afs_volume *afs_lookup_volume_rcu(struct afs_cell *cell, { struct afs_volume *volume = NULL; struct rb_node *p; - int seq = 1; - for (;;) { + scoped_seqlock_read(&cell->volume_lock, ss_lock) { /* Unfortunately, rbtree walking doesn't give reliable results * under just the RCU read lock, so we have to check for * changes. */ - seq++; /* 2 on the 1st/lockless path, otherwise odd */ - read_seqbegin_or_lock(&cell->volume_lock, &seq); - p = rcu_dereference_raw(cell->volumes.rb_node); while (p) { volume = rb_entry(p, struct afs_volume, cell_node); @@ -138,12 +134,9 @@ static struct afs_volume *afs_lookup_volume_rcu(struct afs_cell *cell, if (volume && afs_try_get_volume(volume, afs_volume_trace_get_callback)) break; - if (!need_seqretry(&cell->volume_lock, seq)) - break; - seq |= 1; /* Want a lock next time */ + volume = NULL; } - done_seqretry(&cell->volume_lock, seq); return volume; } @@ -221,7 +214,11 @@ static void afs_break_some_callbacks(struct afs_server *server, rcu_read_lock(); volume = afs_lookup_volume_rcu(server->cell, vid); - if (cbb->fid.vnode == 0 && cbb->fid.unique == 0) { + if (!volume) { + /* Ignore breaks on unknown volumes. */ + rcu_read_unlock(); + *_count = 0; + } else if (cbb->fid.vnode == 0 && cbb->fid.unique == 0) { afs_break_volume_callback(server, volume); *_count -= 1; if (*_count) diff --git a/fs/afs/cell.c b/fs/afs/cell.c index 9738684dbdd2..47a2645768d7 100644 --- a/fs/afs/cell.c +++ b/fs/afs/cell.c @@ -206,11 +206,6 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net, cell->dns_status = vllist->status; smp_store_release(&cell->dns_lookup_count, 1); /* vs source/status */ atomic_inc(&net->cells_outstanding); - ret = idr_alloc_cyclic(&net->cells_dyn_ino, cell, - 2, INT_MAX / 2, GFP_KERNEL); - if (ret < 0) - goto error; - cell->dynroot_ino = ret; cell->debug_id = atomic_inc_return(&cell_debug_id); trace_afs_cell(cell->debug_id, 1, 0, afs_cell_trace_alloc); @@ -304,6 +299,13 @@ struct afs_cell *afs_lookup_cell(struct afs_net *net, goto cell_already_exists; } + ret = idr_alloc_cyclic(&net->cells_dyn_ino, candidate, + 2, INT_MAX / 2, GFP_KERNEL); + if (ret < 0) + goto cant_alloc_ino; + candidate->dynroot_ino = ret; + set_bit(AFS_CELL_FL_HAVE_INO, &candidate->flags); + cell = candidate; candidate = NULL; afs_use_cell(cell, trace); @@ -378,6 +380,11 @@ no_wait: _leave(" = %p [cell]", cell); return cell; +cant_alloc_ino: + up_write(&net->cells_lock); + afs_put_cell(candidate, afs_cell_trace_put_candidate); + goto error_noput; + cell_already_exists: _debug("cell exists"); cell = cursor; @@ -547,6 +554,8 @@ static int afs_update_cell(struct afs_cell *cell) rcu_assign_pointer(cell->vl_servers, vllist); cell->dns_source = vllist->source; old = p; + } else { + old = vllist; } write_unlock(&cell->vl_servers_lock); afs_put_vlserverlist(cell->net, old); @@ -577,7 +586,6 @@ static void afs_cell_destroy(struct rcu_head *rcu) afs_put_vlserverlist(net, rcu_access_pointer(cell->vl_servers)); afs_unuse_cell(cell->alias_of, afs_cell_trace_unuse_alias); key_put(cell->anonymous_key); - idr_remove(&net->cells_dyn_ino, cell->dynroot_ino); kfree(cell->name - 1); kfree(cell); @@ -592,6 +600,13 @@ static void afs_destroy_cell_work(struct work_struct *work) afs_see_cell(cell, afs_cell_trace_destroy); timer_delete_sync(&cell->management_timer); cancel_work_sync(&cell->manager); + + if (test_bit(AFS_CELL_FL_HAVE_INO, &cell->flags)) { + down_write(&cell->net->cells_lock); + idr_remove(&cell->net->cells_dyn_ino, cell->dynroot_ino); + up_write(&cell->net->cells_lock); + } + call_rcu(&cell->rcu, afs_cell_destroy); } diff --git a/fs/afs/cm_security.c b/fs/afs/cm_security.c index edcbd249d202..103168c70dd4 100644 --- a/fs/afs/cm_security.c +++ b/fs/afs/cm_security.c @@ -101,7 +101,8 @@ void afs_process_oob_queue(struct work_struct *work) struct sk_buff *oob; enum rxrpc_oob_type type; - while ((oob = rxrpc_kernel_dequeue_oob(net->socket, &type))) { + while (READ_ONCE(net->live) && + (oob = rxrpc_kernel_dequeue_oob(net->socket, &type))) { switch (type) { case RXRPC_OOB_CHALLENGE: afs_respond_to_challenge(oob); diff --git a/fs/afs/cmservice.c b/fs/afs/cmservice.c index 5540ae1cad59..db394f101fc6 100644 --- a/fs/afs/cmservice.c +++ b/fs/afs/cmservice.c @@ -334,7 +334,6 @@ static int afs_deliver_cb_init_call_back_state3(struct afs_call *call) ret = afs_extract_data(call, false); switch (ret) { case 0: break; - case -EAGAIN: return 0; default: return ret; } @@ -364,6 +363,11 @@ static int afs_deliver_cb_init_call_back_state3(struct afs_call *call) if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) return afs_io_error(call, afs_io_error_cm_reply); + if (!call->server) { + trace_afs_cm_no_server_u(call, call->request); + return 0; + } + if (memcmp(call->request, &call->server->_uuid, sizeof(call->server->_uuid)) != 0) { pr_notice("Callback UUID does not match fileserver UUID\n"); trace_afs_cm_no_server_u(call, call->request); @@ -451,7 +455,6 @@ static int afs_deliver_cb_probe_uuid(struct afs_call *call) ret = afs_extract_data(call, false); switch (ret) { case 0: break; - case -EAGAIN: return 0; default: return ret; } diff --git a/fs/afs/dir.c b/fs/afs/dir.c index aaaa55878ffd..6df56fe9163f 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -28,9 +28,11 @@ static int afs_d_revalidate(struct inode *dir, const struct qstr *name, static int afs_d_delete(const struct dentry *dentry); static void afs_d_iput(struct dentry *dentry, struct inode *inode); static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen, - loff_t fpos, u64 ino, unsigned dtype); + u64 ino, u32 uniquifier); +#define AFS_LOOKUP_ONE ((filldir_t)0x123UL) static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen, - loff_t fpos, u64 ino, unsigned dtype); + u64 ino, u32 uniquifier); +#define AFS_LOOKUP ((filldir_t)0x137UL) static int afs_create(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, bool excl); static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir, @@ -44,6 +46,8 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir, static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags); +static int afs_dir_writepages(struct address_space *mapping, + struct writeback_control *wbc); const struct file_operations afs_dir_file_operations = { .open = afs_dir_open, @@ -68,7 +72,7 @@ const struct inode_operations afs_dir_inode_operations = { }; const struct address_space_operations afs_dir_aops = { - .writepages = afs_single_writepages, + .writepages = afs_dir_writepages, }; const struct dentry_operations afs_fs_dentry_operations = { @@ -233,22 +237,13 @@ static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file) struct iov_iter iter; ssize_t ret; loff_t i_size; - bool is_dir = (S_ISDIR(dvnode->netfs.inode.i_mode) && - !test_bit(AFS_VNODE_MOUNTPOINT, &dvnode->flags)); i_size = i_size_read(&dvnode->netfs.inode); - if (is_dir) { - if (i_size < AFS_DIR_BLOCK_SIZE) - return afs_bad(dvnode, afs_file_error_dir_small); - if (i_size > AFS_DIR_BLOCK_SIZE * 1024) { - trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big); - return -EFBIG; - } - } else { - if (i_size > AFSPATHMAX) { - trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big); - return -EFBIG; - } + if (i_size < AFS_DIR_BLOCK_SIZE) + return afs_bad(dvnode, afs_file_error_dir_small); + if (i_size > AFS_DIR_BLOCK_SIZE * 1024) { + trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big); + return -EFBIG; } /* Expand the storage. TODO: Shrink the storage too. */ @@ -277,24 +272,18 @@ static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file) * buffer. */ ret = -ESTALE; - } else if (is_dir) { + } else { int ret2 = afs_dir_check(dvnode); if (ret2 < 0) ret = ret2; - } else if (i_size < folioq_folio_size(dvnode->directory, 0)) { - /* NUL-terminate a symlink. */ - char *symlink = kmap_local_folio(folioq_folio(dvnode->directory, 0), 0); - - symlink[i_size] = 0; - kunmap_local(symlink); } } return ret; } -ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file) +static ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file) { ssize_t ret; @@ -434,11 +423,18 @@ static int afs_dir_iterate_block(struct afs_vnode *dvnode, } /* found the next entry */ - if (!dir_emit(ctx, dire->u.name, nlen, - ntohl(dire->u.vnode), - (ctx->actor == afs_lookup_filldir || - ctx->actor == afs_lookup_one_filldir)? - ntohl(dire->u.unique) : DT_UNKNOWN)) { + if (ctx->actor == AFS_LOOKUP) { + if (!afs_lookup_filldir(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), + ntohl(dire->u.unique))) + return 0; + } else if (ctx->actor == AFS_LOOKUP_ONE) { + if (!afs_lookup_one_filldir(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), + ntohl(dire->u.unique))) + return 0; + } else if (!dir_emit(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), DT_UNKNOWN)) { _leave(" = 0 [full]"); return 0; } @@ -558,6 +554,7 @@ static int afs_readdir(struct file *file, struct dir_context *ctx) { afs_dataversion_t dir_version; + ctx->dt_flags_mask = UINT_MAX; return afs_dir_iterate(file_inode(file), ctx, file, &dir_version); } @@ -567,14 +564,14 @@ static int afs_readdir(struct file *file, struct dir_context *ctx) * uniquifier through dtype */ static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, - int nlen, loff_t fpos, u64 ino, unsigned dtype) + int nlen, u64 ino, u32 uniquifier) { struct afs_lookup_one_cookie *cookie = container_of(ctx, struct afs_lookup_one_cookie, ctx); _enter("{%s,%u},%s,%u,,%llu,%u", cookie->name.name, cookie->name.len, name, nlen, - (unsigned long long) ino, dtype); + (unsigned long long) ino, uniquifier); /* insanity checks first */ BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048); @@ -587,7 +584,7 @@ static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, } cookie->fid.vnode = ino; - cookie->fid.unique = dtype; + cookie->fid.unique = uniquifier; cookie->found = 1; _leave(" = false [found]"); @@ -604,7 +601,7 @@ static int afs_do_lookup_one(struct inode *dir, const struct qstr *name, { struct afs_super_info *as = dir->i_sb->s_fs_info; struct afs_lookup_one_cookie cookie = { - .ctx.actor = afs_lookup_one_filldir, + .ctx.actor = AFS_LOOKUP_ONE, .name = *name, .fid.vid = as->volume->vid }; @@ -635,14 +632,14 @@ static int afs_do_lookup_one(struct inode *dir, const struct qstr *name, * uniquifier through dtype */ static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, - int nlen, loff_t fpos, u64 ino, unsigned dtype) + int nlen, u64 ino, u32 uniquifier) { struct afs_lookup_cookie *cookie = container_of(ctx, struct afs_lookup_cookie, ctx); _enter("{%s,%u},%s,%u,,%llu,%u", cookie->name.name, cookie->name.len, name, nlen, - (unsigned long long) ino, dtype); + (unsigned long long) ino, uniquifier); /* insanity checks first */ BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048); @@ -650,7 +647,7 @@ static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, if (cookie->nr_fids < 50) { cookie->fids[cookie->nr_fids].vnode = ino; - cookie->fids[cookie->nr_fids].unique = dtype; + cookie->fids[cookie->nr_fids].unique = uniquifier; cookie->nr_fids++; } @@ -791,7 +788,7 @@ static struct inode *afs_do_lookup(struct inode *dir, struct dentry *dentry) for (i = 0; i < ARRAY_SIZE(cookie->fids); i++) cookie->fids[i].vid = dvnode->fid.vid; - cookie->ctx.actor = afs_lookup_filldir; + cookie->ctx.actor = AFS_LOOKUP; cookie->name = dentry->d_name; cookie->nr_fids = 2; /* slot 1 is saved for the fid we actually want * and slot 0 for the directory */ @@ -1763,13 +1760,20 @@ error: return ret; } +static void afs_symlink_put(struct afs_operation *op) +{ + kfree(op->create.symlink); + op->create.symlink = NULL; + afs_create_put(op); +} + static const struct afs_operation_ops afs_symlink_operation = { .issue_afs_rpc = afs_fs_symlink, .issue_yfs_rpc = yfs_fs_symlink, .success = afs_create_success, .aborted = afs_check_for_remote_deletion, .edit_dir = afs_create_edit_dir, - .put = afs_create_put, + .put = afs_symlink_put, }; /* @@ -1779,7 +1783,9 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, const char *content) { struct afs_operation *op; + struct afs_symlink *symlink; struct afs_vnode *dvnode = AFS_FS_I(dir); + size_t clen = strlen(content); int ret; _enter("{%llx:%llu},{%pd},%s", @@ -1791,12 +1797,20 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir, goto error; ret = -EINVAL; - if (strlen(content) >= AFSPATHMAX) + if (clen >= AFSPATHMAX) + goto error; + + ret = -ENOMEM; + symlink = kmalloc_flex(struct afs_symlink, content, clen + 1, GFP_KERNEL); + if (!symlink) goto error; + refcount_set(&symlink->ref, 1); + memcpy(symlink->content, content, clen + 1); op = afs_alloc_operation(NULL, dvnode->volume); if (IS_ERR(op)) { ret = PTR_ERR(op); + kfree(symlink); goto error; } @@ -1808,7 +1822,7 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir, op->dentry = dentry; op->ops = &afs_symlink_operation; op->create.reason = afs_edit_dir_for_symlink; - op->create.symlink = content; + op->create.symlink = symlink; op->mtime = current_time(dir); ret = afs_do_sync_operation(op); afs_dir_unuse_cookie(dvnode, ret); @@ -2192,28 +2206,33 @@ error: } /* - * Write the file contents to the cache as a single blob. + * Write the directory contents to the cache as a single blob. */ -int afs_single_writepages(struct address_space *mapping, - struct writeback_control *wbc) +static int afs_dir_writepages(struct address_space *mapping, + struct writeback_control *wbc) { struct afs_vnode *dvnode = AFS_FS_I(mapping->host); struct iov_iter iter; - bool is_dir = (S_ISDIR(dvnode->netfs.inode.i_mode) && - !test_bit(AFS_VNODE_MOUNTPOINT, &dvnode->flags)); int ret = 0; /* Need to lock to prevent the folio queue and folios from being thrown * away. */ - down_read(&dvnode->validate_lock); + if (!down_read_trylock(&dvnode->validate_lock)) { + if (wbc->sync_mode == WB_SYNC_NONE) { + /* The VFS will have undirtied the inode. */ + netfs_single_mark_inode_dirty(&dvnode->netfs.inode); + return 0; + } + down_read(&dvnode->validate_lock); + } - if (is_dir ? - test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) : - atomic64_read(&dvnode->cb_expires_at) != AFS_NO_CB_PROMISE) { + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) { iov_iter_folio_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0, i_size_read(&dvnode->netfs.inode)); ret = netfs_writeback_single(mapping, wbc, &iter); + if (ret == 1) + ret = 0; /* Skipped write due to lock conflict. */ } up_read(&dvnode->validate_lock); diff --git a/fs/afs/dir_silly.c b/fs/afs/dir_silly.c index a748fd133faf..982bb6ec15f0 100644 --- a/fs/afs/dir_silly.c +++ b/fs/afs/dir_silly.c @@ -248,13 +248,11 @@ int afs_silly_iput(struct dentry *dentry, struct inode *inode) struct dentry *alias; int ret; - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); - _enter("%p{%pd},%llx", dentry, dentry, vnode->fid.vnode); down_read(&dvnode->rmdir_lock); - alias = d_alloc_parallel(dentry->d_parent, &dentry->d_name, &wq); + alias = d_alloc_parallel(dentry->d_parent, &dentry->d_name); if (IS_ERR(alias)) { up_read(&dvnode->rmdir_lock); return 0; diff --git a/fs/afs/dynroot.c b/fs/afs/dynroot.c index 1d5e33bc7502..6e3c8c691ba9 100644 --- a/fs/afs/dynroot.c +++ b/fs/afs/dynroot.c @@ -278,7 +278,7 @@ static struct dentry *afs_lookup_atcell(struct inode *dir, struct dentry *dentry } /* - * Transcribe the cell database into readdir content under the RCU read lock. + * Transcribe the cell database into readdir content under net->cells_lock. * Each cell produces two entries, one prefixed with a dot and one not. */ static int afs_dynroot_readdir_cells(struct afs_net *net, struct dir_context *ctx) diff --git a/fs/afs/file.c b/fs/afs/file.c index 85696ac984cc..0467742bfeee 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -427,21 +427,35 @@ static void afs_free_request(struct netfs_io_request *rreq) afs_put_wb_key(rreq->netfs_priv2); } -static void afs_update_i_size(struct inode *inode, loff_t new_i_size) +/* + * Set the file size and block count, taking ->cb_lock and ->i_lock to maintain + * coherency and prevent 64-bit tearing on 32-bit arches. + * + * Also, estimate the number of 512 bytes blocks used, rounded up to nearest 1K + * for consistency with other AFS clients. + */ +void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size) { - struct afs_vnode *vnode = AFS_FS_I(inode); + struct inode *inode = &vnode->netfs.inode; loff_t i_size; write_seqlock(&vnode->cb_lock); - i_size = i_size_read(&vnode->netfs.inode); + spin_lock(&inode->i_lock); + i_size = i_size_read(inode); if (new_i_size > i_size) { - i_size_write(&vnode->netfs.inode, new_i_size); - inode_set_bytes(&vnode->netfs.inode, new_i_size); + i_size_write(inode, new_i_size); + inode_set_bytes(inode, round_up(new_i_size, 1024)); } + spin_unlock(&inode->i_lock); write_sequnlock(&vnode->cb_lock); fscache_update_cookie(afs_vnode_cache(vnode), NULL, &new_i_size); } +static void afs_update_i_size(struct inode *inode, loff_t new_i_size) +{ + afs_set_i_size(AFS_FS_I(inode), new_i_size); +} + static void afs_netfs_invalidate_cache(struct netfs_io_request *wreq) { struct afs_vnode *vnode = AFS_FS_I(wreq->inode); diff --git a/fs/afs/fs_operation.c b/fs/afs/fs_operation.c index c0dbbc6d3716..20801b29521d 100644 --- a/fs/afs/fs_operation.c +++ b/fs/afs/fs_operation.c @@ -348,7 +348,7 @@ int afs_put_operation(struct afs_operation *op) for (i = 0; i < op->nr_files - 2; i++) if (op->more_files[i].put_vnode) iput(&op->more_files[i].vnode->netfs.inode); - kfree(op->more_files); + kvfree(op->more_files); } if (op->estate) { diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c index 95494d5f2b8a..a2ffd60889f8 100644 --- a/fs/afs/fsclient.c +++ b/fs/afs/fsclient.c @@ -886,7 +886,7 @@ void afs_fs_symlink(struct afs_operation *op) namesz = name->len; padsz = (4 - (namesz & 3)) & 3; - c_namesz = strlen(op->create.symlink); + c_namesz = strlen(op->create.symlink->content); c_padsz = (4 - (c_namesz & 3)) & 3; reqsz = (6 * 4) + namesz + padsz + c_namesz + c_padsz + (6 * 4); @@ -910,7 +910,7 @@ void afs_fs_symlink(struct afs_operation *op) bp = (void *) bp + padsz; } *bp++ = htonl(c_namesz); - memcpy(bp, op->create.symlink, c_namesz); + memcpy(bp, op->create.symlink->content, c_namesz); bp = (void *) bp + c_namesz; if (c_padsz > 0) { memset(bp, 0, c_padsz); diff --git a/fs/afs/inode.c b/fs/afs/inode.c index a5173434f786..14f39a9bea6c 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -25,96 +25,6 @@ #include "internal.h" #include "afs_fs.h" -void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op) -{ - size_t size = strlen(op->create.symlink) + 1; - size_t dsize = 0; - char *p; - - if (netfs_alloc_folioq_buffer(NULL, &vnode->directory, &dsize, size, - mapping_gfp_mask(vnode->netfs.inode.i_mapping)) < 0) - return; - - vnode->directory_size = dsize; - p = kmap_local_folio(folioq_folio(vnode->directory, 0), 0); - memcpy(p, op->create.symlink, size); - kunmap_local(p); - set_bit(AFS_VNODE_DIR_READ, &vnode->flags); - netfs_single_mark_inode_dirty(&vnode->netfs.inode); -} - -static void afs_put_link(void *arg) -{ - struct folio *folio = virt_to_folio(arg); - - kunmap_local(arg); - folio_put(folio); -} - -const char *afs_get_link(struct dentry *dentry, struct inode *inode, - struct delayed_call *callback) -{ - struct afs_vnode *vnode = AFS_FS_I(inode); - struct folio *folio; - char *content; - ssize_t ret; - - if (!dentry) { - /* RCU pathwalk. */ - if (!test_bit(AFS_VNODE_DIR_READ, &vnode->flags) || !afs_check_validity(vnode)) - return ERR_PTR(-ECHILD); - goto good; - } - - if (test_bit(AFS_VNODE_DIR_READ, &vnode->flags)) - goto fetch; - - ret = afs_validate(vnode, NULL); - if (ret < 0) - return ERR_PTR(ret); - - if (!test_and_clear_bit(AFS_VNODE_ZAP_DATA, &vnode->flags) && - test_bit(AFS_VNODE_DIR_READ, &vnode->flags)) - goto good; - -fetch: - ret = afs_read_single(vnode, NULL); - if (ret < 0) - return ERR_PTR(ret); - set_bit(AFS_VNODE_DIR_READ, &vnode->flags); - -good: - folio = folioq_folio(vnode->directory, 0); - folio_get(folio); - content = kmap_local_folio(folio, 0); - set_delayed_call(callback, afs_put_link, content); - return content; -} - -int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen) -{ - DEFINE_DELAYED_CALL(done); - const char *content; - int len; - - content = afs_get_link(dentry, d_inode(dentry), &done); - if (IS_ERR(content)) { - do_delayed_call(&done); - return PTR_ERR(content); - } - - len = umin(strlen(content), buflen); - if (copy_to_user(buffer, content, len)) - len = -EFAULT; - do_delayed_call(&done); - return len; -} - -static const struct inode_operations afs_symlink_inode_operations = { - .get_link = afs_get_link, - .readlink = afs_readlink, -}; - static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *parent_vnode) { static unsigned long once_only; @@ -142,9 +52,9 @@ static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *paren /* * Set parameters for the netfs library */ -static void afs_set_netfs_context(struct afs_vnode *vnode) +static void afs_set_netfs_context(struct afs_vnode *vnode, bool is_file) { - netfs_inode_init(&vnode->netfs, &afs_req_ops, true); + netfs_inode_init(&vnode->netfs, &afs_req_ops, is_file); } /* @@ -183,6 +93,10 @@ static int afs_inode_init_from_status(struct afs_operation *op, inode->i_gid = make_kgid(&init_user_ns, status->group); set_nlink(&vnode->netfs.inode, status->nlink); + i_size_write(inode, status->size); + inode_set_bytes(inode, status->size); + afs_set_netfs_context(vnode, status->type == AFS_FTYPE_FILE); + switch (status->type) { case AFS_FTYPE_FILE: inode->i_mode = S_IFREG | (status->mode & S_IALLUGO); @@ -214,9 +128,8 @@ static int afs_inode_init_from_status(struct afs_operation *op, inode->i_mode = S_IFLNK | status->mode; inode->i_op = &afs_symlink_inode_operations; } - inode->i_mapping->a_ops = &afs_dir_aops; + inode->i_mapping->a_ops = &afs_symlink_aops; inode_nohighmem(inode); - mapping_set_release_always(inode->i_mapping); break; default: dump_vnode(vnode, op->file[0].vnode != vnode ? op->file[0].vnode : NULL); @@ -224,9 +137,6 @@ static int afs_inode_init_from_status(struct afs_operation *op, return afs_protocol_error(NULL, afs_eproto_file_type); } - afs_set_i_size(vnode, status->size); - afs_set_netfs_context(vnode); - vnode->invalid_before = status->data_version; trace_afs_set_dv(vnode, status->data_version); inode_set_iversion_raw(&vnode->netfs.inode, status->data_version); @@ -253,7 +163,8 @@ static void afs_apply_status(struct afs_operation *op, { struct afs_file_status *status = &vp->scb.status; struct afs_vnode *vnode = vp->vnode; - struct inode *inode = &vnode->netfs.inode; + struct netfs_inode *ictx = &vnode->netfs; + struct inode *inode = &ictx->inode; struct timespec64 t; umode_t mode; bool unexpected_jump = false; @@ -336,6 +247,8 @@ static void afs_apply_status(struct afs_operation *op, } if (data_changed) { + unsigned long long zero_point, size = status->size; + inode_set_iversion_raw(inode, status->data_version); /* Only update the size if the data version jumped. If the @@ -343,16 +256,25 @@ static void afs_apply_status(struct afs_operation *op, * idea of what the size should be that's not the same as * what's on the server. */ - vnode->netfs.remote_i_size = status->size; - if (change_size || status->size > i_size_read(inode)) { - afs_set_i_size(vnode, status->size); + spin_lock(&inode->i_lock); + + if (change_size || size > i_size_read(inode)) { + /* We can read the sizes directly as we hold i_lock. */ + zero_point = ictx->_zero_point; + if (unexpected_jump) - vnode->netfs.zero_point = status->size; + zero_point = size; + netfs_write_sizes(inode, size, size, zero_point); + inode_set_bytes(inode, size); inode_set_ctime_to_ts(inode, t); inode_set_atime_to_ts(inode, t); + } else { + netfs_write_remote_i_size(inode, size); } + spin_unlock(&inode->i_lock); + if (op->ops == &afs_fetch_data_operation) - op->fetch.subreq->rreq->i_size = status->size; + op->fetch.subreq->rreq->i_size = size; } } @@ -643,7 +565,6 @@ struct inode *afs_root_iget(struct super_block *sb, struct key *key) vnode = AFS_FS_I(inode); vnode->cb_v_check = atomic_read(&as->volume->cb_v_break); - afs_set_netfs_context(vnode); op = afs_alloc_operation(key, as->volume); if (IS_ERR(op)) { @@ -709,7 +630,7 @@ int afs_getattr(struct mnt_idmap *idmap, const struct path *path, * it, but we need to give userspace the server's size. */ if (S_ISDIR(inode->i_mode)) - stat->size = vnode->netfs.remote_i_size; + stat->size = netfs_read_remote_i_size(inode); } while (read_seqretry(&vnode->cb_lock, seq)); return 0; @@ -756,12 +677,15 @@ void afs_evict_inode(struct inode *inode) .range_end = LLONG_MAX, }; - afs_single_writepages(inode->i_mapping, &wbc); + inode->i_mapping->a_ops->writepages(inode->i_mapping, &wbc); } + flush_delayed_work(&vnode->lock_work); netfs_wait_for_outstanding_io(inode); truncate_inode_pages_final(&inode->i_data); netfs_free_folioq_buffer(vnode->directory); + if (vnode->symlink) + afs_evict_symlink(vnode); afs_set_cache_aux(vnode, &aux); netfs_clear_inode_writeback(inode, &aux); @@ -889,7 +813,7 @@ int afs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if (!(attr->ia_valid & (supported & ~ATTR_SIZE & ~ATTR_MTIME)) && attr->ia_size < i_size && - attr->ia_size > vnode->netfs.remote_i_size) { + attr->ia_size > netfs_read_remote_i_size(inode)) { truncate_setsize(inode, attr->ia_size); netfs_resize_file(&vnode->netfs, size, false); fscache_resize_cookie(afs_vnode_cache(vnode), diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 599353c33337..601f01e5c15f 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -388,6 +388,7 @@ struct afs_cell { #define AFS_CELL_FL_NO_GC 0 /* The cell was added manually, don't auto-gc */ #define AFS_CELL_FL_DO_LOOKUP 1 /* DNS lookup requested */ #define AFS_CELL_FL_CHECK_ALIAS 2 /* Need to check for aliases */ +#define AFS_CELL_FL_HAVE_INO 3 /* Have dynroot_ino */ enum afs_cell_state state; short error; enum dns_record_source dns_source:8; /* Latest source of data from lookup */ @@ -710,6 +711,7 @@ struct afs_vnode { #define AFS_VNODE_DIR_READ 11 /* Set if we've read a dir's contents */ struct folio_queue *directory; /* Directory contents */ + struct afs_symlink __rcu *symlink; /* Symlink content */ struct list_head wb_keys; /* List of keys available for writeback */ struct list_head pending_locks; /* locks waiting to be granted */ struct list_head granted_locks; /* locks granted on this file */ @@ -749,8 +751,6 @@ static inline void afs_vnode_set_cache(struct afs_vnode *vnode, { #ifdef CONFIG_AFS_FSCACHE vnode->netfs.cache = cookie; - if (cookie) - mapping_set_release_always(vnode->netfs.inode.i_mapping); #endif } @@ -777,6 +777,15 @@ struct afs_permits { }; /* + * Copy of symlink content for normal use. + */ +struct afs_symlink { + struct rcu_head rcu; + refcount_t ref; + char content[]; +}; + +/* * Error prioritisation and accumulation. */ struct afs_error { @@ -887,7 +896,7 @@ struct afs_operation { struct { int reason; /* enum afs_edit_dir_reason */ mode_t mode; - const char *symlink; + struct afs_symlink *symlink; } create; struct { bool need_rehash; @@ -1098,13 +1107,10 @@ extern const struct inode_operations afs_dir_inode_operations; extern const struct address_space_operations afs_dir_aops; extern const struct dentry_operations afs_fs_dentry_operations; -ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file); ssize_t afs_read_dir(struct afs_vnode *dvnode, struct file *file) __acquires(&dvnode->validate_lock); extern void afs_d_release(struct dentry *); extern void afs_check_for_remote_deletion(struct afs_operation *); -int afs_single_writepages(struct address_space *mapping, - struct writeback_control *wbc); /* * dir_edit.c @@ -1157,6 +1163,7 @@ extern int afs_open(struct inode *, struct file *); extern int afs_release(struct inode *, struct file *); void afs_fetch_data_async_rx(struct work_struct *work); void afs_fetch_data_immediate_cancel(struct afs_call *call); +void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size); /* * flock.c @@ -1246,10 +1253,6 @@ extern void afs_fs_probe_cleanup(struct afs_net *); */ extern const struct afs_operation_ops afs_fetch_status_operation; -void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op); -const char *afs_get_link(struct dentry *dentry, struct inode *inode, - struct delayed_call *callback); -int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen); extern void afs_vnode_commit_status(struct afs_operation *, struct afs_vnode_param *); extern int afs_fetch_status(struct afs_vnode *, struct key *, bool, afs_access_t *); extern int afs_ilookup5_test_by_fid(struct inode *, void *); @@ -1600,6 +1603,21 @@ extern int __init afs_fs_init(void); extern void afs_fs_exit(void); /* + * symlink.c + */ +extern const struct inode_operations afs_symlink_inode_operations; +extern const struct address_space_operations afs_symlink_aops; + +void afs_invalidate_symlink(struct afs_vnode *vnode); +void afs_evict_symlink(struct afs_vnode *vnode); +void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op); +const char *afs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback); +int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen); +int afs_symlink_writepages(struct address_space *mapping, + struct writeback_control *wbc); + +/* * validation.c */ bool afs_check_validity(const struct afs_vnode *vnode); @@ -1759,16 +1777,6 @@ static inline void afs_update_dentry_version(struct afs_operation *op, } /* - * Set the file size and block count. Estimate the number of 512 bytes blocks - * used, rounded up to nearest 1K for consistency with other AFS clients. - */ -static inline void afs_set_i_size(struct afs_vnode *vnode, u64 size) -{ - i_size_write(&vnode->netfs.inode, size); - vnode->netfs.inode.i_blocks = ((size + 1023) >> 10) << 1; -} - -/* * Check for a conflicting operation on a directory that we just unlinked from. * If someone managed to sneak a link or an unlink in on the file we just * unlinked, we won't be able to trust nlink on an AFS file (but not YFS). diff --git a/fs/afs/rxrpc.c b/fs/afs/rxrpc.c index 588f8de51167..d82916657a3d 100644 --- a/fs/afs/rxrpc.c +++ b/fs/afs/rxrpc.c @@ -127,8 +127,15 @@ void afs_close_socket(struct afs_net *net) { _enter(""); + cancel_work_sync(&net->charge_preallocation_work); + cancel_work_sync(&net->rx_oob_work); + /* Future work items should now see ->live is false. */ + kernel_listen(net->socket, 0); + + /* Make sure work items are no longer running. */ flush_workqueue(afs_async_calls); + cancel_work_sync(&net->charge_preallocation_work); if (net->spare_incoming_call) { afs_put_call(net->spare_incoming_call); @@ -142,6 +149,7 @@ void afs_close_socket(struct afs_net *net) kernel_sock_shutdown(net->socket, SHUT_RDWR); flush_workqueue(afs_async_calls); + cancel_work_sync(&net->rx_oob_work); net->socket->sk->sk_user_data = NULL; sock_release(net->socket); key_put(net->fs_cm_token_key); @@ -742,7 +750,7 @@ void afs_charge_preallocation(struct work_struct *work) container_of(work, struct afs_net, charge_preallocation_work); struct afs_call *call = net->spare_incoming_call; - for (;;) { + while (READ_ONCE(net->live)) { if (!call) { call = afs_alloc_call(net, &afs_RXCMxxxx, GFP_KERNEL); if (!call) @@ -792,7 +800,8 @@ static void afs_rx_new_call(struct sock *sk, struct rxrpc_call *rxcall, if (!call->server) trace_afs_cm_no_server(call, rxrpc_kernel_remote_srx(call->peer)); - queue_work(afs_wq, &net->charge_preallocation_work); + if (net->live) + queue_work(afs_wq, &net->charge_preallocation_work); } /* @@ -982,5 +991,6 @@ static void afs_rx_notify_oob(struct sock *sk, struct sk_buff *oob) { struct afs_net *net = sk->sk_user_data; - schedule_work(&net->rx_oob_work); + if (READ_ONCE(net->live)) + queue_work(afs_wq, &net->rx_oob_work); } diff --git a/fs/afs/super.c b/fs/afs/super.c index 942f3e9800d7..82bb713825a0 100644 --- a/fs/afs/super.c +++ b/fs/afs/super.c @@ -587,7 +587,8 @@ static int afs_get_tree(struct fs_context *fc) } fc->root = dget(sb->s_root); - trace_afs_get_tree(as->cell, as->volume); + if (!ctx->dyn_root) + trace_afs_get_tree(as->cell, as->volume); _leave(" = 0 [%p]", sb); return 0; @@ -659,7 +660,6 @@ static void afs_i_init_once(void *_vnode) INIT_LIST_HEAD(&vnode->wb_keys); INIT_LIST_HEAD(&vnode->pending_locks); INIT_LIST_HEAD(&vnode->granted_locks); - INIT_DELAYED_WORK(&vnode->lock_work, afs_lock_work); INIT_LIST_HEAD(&vnode->cb_mmap_link); seqlock_init(&vnode->cb_lock); } @@ -693,6 +693,7 @@ static struct inode *afs_alloc_inode(struct super_block *sb) init_rwsem(&vnode->rmdir_lock); INIT_WORK(&vnode->cb_work, afs_invalidate_mmap_work); + INIT_DELAYED_WORK(&vnode->lock_work, afs_lock_work); _leave(" = %p", &vnode->netfs.inode); return &vnode->netfs.inode; diff --git a/fs/afs/symlink.c b/fs/afs/symlink.c new file mode 100644 index 000000000000..16b4823cb7b7 --- /dev/null +++ b/fs/afs/symlink.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS filesystem symbolic link handling + * + * Copyright (C) 2026 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/kernel.h> +#include <linux/fs.h> +#include <linux/namei.h> +#include <linux/pagemap.h> +#include <linux/iov_iter.h> +#include "internal.h" + +static void afs_put_symlink(struct afs_symlink *symlink) +{ + if (refcount_dec_and_test(&symlink->ref)) + kfree_rcu(symlink, rcu); +} + +static void afs_replace_symlink(struct afs_vnode *vnode, struct afs_symlink *symlink) +{ + struct afs_symlink *old; + + old = rcu_replace_pointer(vnode->symlink, symlink, + lockdep_is_held(&vnode->validate_lock)); + if (old) + afs_put_symlink(old); +} + +/* + * In the event that a third-party update of a symlink occurs, dispose of the + * copy of the old contents. Called under ->validate_lock. + */ +void afs_invalidate_symlink(struct afs_vnode *vnode) +{ + afs_replace_symlink(vnode, NULL); +} + +/* + * Dispose of a symlink copy during inode deletion. + */ +void afs_evict_symlink(struct afs_vnode *vnode) +{ + struct afs_symlink *old; + + old = rcu_replace_pointer(vnode->symlink, NULL, true); + if (old) + afs_put_symlink(old); + +} + +/* + * Set up a locally created symlink inode for immediate write to the cache. + */ +void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op) +{ + struct afs_symlink *symlink = op->create.symlink; + size_t dsize = 0; + size_t size = strlen(symlink->content) + 1; + char *p; + + rcu_assign_pointer(vnode->symlink, symlink); + op->create.symlink = NULL; + + if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs))) + return; + + if (netfs_alloc_folioq_buffer(NULL, &vnode->directory, &dsize, size, + mapping_gfp_mask(vnode->netfs.inode.i_mapping)) < 0) + return; + + vnode->directory_size = dsize; + p = kmap_local_folio(folioq_folio(vnode->directory, 0), 0); + memcpy(p, symlink->content, size); + kunmap_local(p); + netfs_single_mark_inode_dirty(&vnode->netfs.inode); +} + +/* + * Read a symlink in a single download. + */ +static ssize_t afs_do_read_symlink(struct afs_vnode *vnode) +{ + struct afs_symlink *symlink; + struct iov_iter iter; + ssize_t ret; + loff_t i_size; + + i_size = i_size_read(&vnode->netfs.inode); + if (i_size > PAGE_SIZE - 1) { + trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big); + return -EFBIG; + } + + if (!vnode->directory) { + size_t cur_size = 0; + + ret = netfs_alloc_folioq_buffer(NULL, + &vnode->directory, &cur_size, PAGE_SIZE, + mapping_gfp_mask(vnode->netfs.inode.i_mapping)); + vnode->directory_size = PAGE_SIZE - 1; + if (ret < 0) + return ret; + } + + iov_iter_folio_queue(&iter, ITER_DEST, vnode->directory, 0, 0, PAGE_SIZE); + + /* AFS requires us to perform the read of a symlink as a single unit to + * avoid issues with the content being changed between reads. + */ + ret = netfs_read_single(&vnode->netfs.inode, NULL, &iter); + if (ret >= 0) { + i_size = ret; + if (i_size > PAGE_SIZE - 1) { + trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big); + return -EFBIG; + } + vnode->directory_size = i_size; + + /* Copy the symlink. */ + symlink = kmalloc_flex(struct afs_symlink, content, i_size + 1, + GFP_KERNEL); + if (!symlink) + return -ENOMEM; + + refcount_set(&symlink->ref, 1); + symlink->content[i_size] = 0; + + const char *s = kmap_local_folio(folioq_folio(vnode->directory, 0), 0); + + memcpy(symlink->content, s, i_size); + kunmap_local(s); + + afs_replace_symlink(vnode, symlink); + } + + if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs))) { + netfs_free_folioq_buffer(vnode->directory); + vnode->directory = NULL; + vnode->directory_size = 0; + } + + return ret; +} + +static ssize_t afs_read_symlink(struct afs_vnode *vnode) +{ + ssize_t ret; + + fscache_use_cookie(afs_vnode_cache(vnode), false); + ret = afs_do_read_symlink(vnode); + fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL); + return ret; +} + +static void afs_put_link(void *arg) +{ + afs_put_symlink(arg); +} + +const char *afs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + struct afs_symlink *symlink; + struct afs_vnode *vnode = AFS_FS_I(inode); + ssize_t ret; + + if (!dentry) { + /* RCU pathwalk. */ + symlink = rcu_dereference(vnode->symlink); + if (!symlink || !afs_check_validity(vnode)) + return ERR_PTR(-ECHILD); + set_delayed_call(callback, NULL, NULL); + return symlink->content; + } + + if (vnode->symlink) { + ret = afs_validate(vnode, NULL); + if (ret < 0) + return ERR_PTR(ret); + + down_read(&vnode->validate_lock); + if (vnode->symlink) + goto good; + up_read(&vnode->validate_lock); + } + + if (down_write_killable(&vnode->validate_lock) < 0) + return ERR_PTR(-ERESTARTSYS); + if (!vnode->symlink) { + ret = afs_read_symlink(vnode); + if (ret < 0) { + up_write(&vnode->validate_lock); + return ERR_PTR(ret); + } + } + + downgrade_write(&vnode->validate_lock); + +good: + symlink = rcu_dereference_protected(vnode->symlink, + lockdep_is_held(&vnode->validate_lock)); + refcount_inc(&symlink->ref); + up_read(&vnode->validate_lock); + + set_delayed_call(callback, afs_put_link, symlink); + return symlink->content; +} + +int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen) +{ + DEFINE_DELAYED_CALL(done); + const char *content; + int len; + + content = afs_get_link(dentry, d_inode(dentry), &done); + if (IS_ERR(content)) { + do_delayed_call(&done); + return PTR_ERR(content); + } + + len = umin(strlen(content), buflen); + if (copy_to_user(buffer, content, len)) + len = -EFAULT; + do_delayed_call(&done); + return len; +} + +/* + * Write the symlink contents to the cache as a single blob. We then throw + * away the page we used to receive it. + */ +int afs_symlink_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct afs_vnode *vnode = AFS_FS_I(mapping->host); + struct iov_iter iter; + int ret = 0; + + if (!down_read_trylock(&vnode->validate_lock)) { + if (wbc->sync_mode == WB_SYNC_NONE) { + /* The VFS will have undirtied the inode. */ + netfs_single_mark_inode_dirty(&vnode->netfs.inode); + return 0; + } + down_read(&vnode->validate_lock); + } + + if (vnode->directory && + atomic64_read(&vnode->cb_expires_at) != AFS_NO_CB_PROMISE) { + iov_iter_folio_queue(&iter, ITER_SOURCE, vnode->directory, 0, 0, + i_size_read(&vnode->netfs.inode)); + ret = netfs_writeback_single(mapping, wbc, &iter); + } + + if (ret == 0) { + netfs_wb_begin(&vnode->netfs, false); + netfs_free_folioq_buffer(vnode->directory); + vnode->directory = NULL; + vnode->directory_size = 0; + netfs_wb_end(&vnode->netfs); + } else if (ret == 1) { + ret = 0; /* Skipped write due to lock conflict. */ + } + + up_read(&vnode->validate_lock); + return ret; +} + +const struct inode_operations afs_symlink_inode_operations = { + .get_link = afs_get_link, + .readlink = afs_readlink, +}; + +const struct address_space_operations afs_symlink_aops = { + .writepages = afs_symlink_writepages, +}; diff --git a/fs/afs/validation.c b/fs/afs/validation.c index 0ba8336c9025..e997563af658 100644 --- a/fs/afs/validation.c +++ b/fs/afs/validation.c @@ -465,11 +465,17 @@ int afs_validate(struct afs_vnode *vnode, struct key *key) vnode->cb_ro_snapshot = cb_ro_snapshot; vnode->cb_scrub = cb_scrub; - /* if the vnode's data version number changed then its contents are - * different */ + /* If the vnode's data version number changed then its contents are + * different. Note that afs_apply_status() doesn't set ZAP_DATA on + * directories. + */ zap |= test_and_clear_bit(AFS_VNODE_ZAP_DATA, &vnode->flags); - if (zap) - afs_zap_data(vnode); + if (zap) { + if (S_ISREG(vnode->netfs.inode.i_mode)) + afs_zap_data(vnode); + else if (S_ISLNK(vnode->netfs.inode.i_mode)) + afs_invalidate_symlink(vnode); + } up_write(&vnode->validate_lock); _leave(" = 0"); return 0; diff --git a/fs/afs/vl_list.c b/fs/afs/vl_list.c index 3e4966915ea4..c1dac5dbed0d 100644 --- a/fs/afs/vl_list.c +++ b/fs/afs/vl_list.c @@ -92,7 +92,7 @@ static struct afs_addr_list *afs_extract_vl_addrs(struct afs_net *net, { struct afs_addr_list *alist; const u8 *b = *_b; - int ret = -EINVAL; + int ret; alist = afs_alloc_addrlist(nr_addrs); if (!alist) @@ -110,6 +110,7 @@ static struct afs_addr_list *afs_extract_vl_addrs(struct afs_net *net, case DNS_ADDRESS_IS_IPV4: if (end - b < 4) { _leave(" = -EINVAL [short inet]"); + ret = -EINVAL; goto error; } memcpy(x, b, 4); @@ -122,6 +123,7 @@ static struct afs_addr_list *afs_extract_vl_addrs(struct afs_net *net, case DNS_ADDRESS_IS_IPV6: if (end - b < 16) { _leave(" = -EINVAL [short inet6]"); + ret = -EINVAL; goto error; } memcpy(x, b, 16); @@ -198,6 +200,8 @@ struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *cell, b += sizeof(*hdr); while (end - b >= sizeof(bs)) { + int nlen; + bs.name_len = afs_extract_le16(&b); bs.priority = afs_extract_le16(&b); bs.weight = afs_extract_le16(&b); @@ -207,10 +211,12 @@ struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *cell, bs.protocol = *b++; bs.nr_addrs = *b++; + nlen = min3(bs.name_len, end - b, 255); + _debug("extract %u %u %u %u %u %u %*.*s", bs.name_len, bs.priority, bs.weight, bs.port, bs.protocol, bs.nr_addrs, - bs.name_len, bs.name_len, b); + bs.name_len, nlen, b); if (end - b < bs.name_len) break; @@ -287,8 +293,20 @@ struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *cell, afs_put_addrlist(old, afs_alist_trace_put_vlserver_old); } + /* Check for duplicates in the server list */ + for (j = 0; j < vllist->nr_servers; j++) { + struct afs_vlserver *s = vllist->servers[j].server; - /* TODO: Might want to check for duplicates */ + if (s->name_len == server->name_len && + s->port == server->port && + strncasecmp(s->name, server->name, server->name_len) == 0) { + afs_put_vlserver(cell->net, server); + server = NULL; + break; + } + } + if (!server) + continue; /* Insertion-sort by priority and weight */ for (j = 0; j < vllist->nr_servers; j++) { diff --git a/fs/afs/volume.c b/fs/afs/volume.c index 9ae5c8ad2e04..4f79d25ec37f 100644 --- a/fs/afs/volume.c +++ b/fs/afs/volume.c @@ -40,7 +40,7 @@ static struct afs_volume *afs_insert_volume_into_cell(struct afs_cell *cell, goto found; } - set_bit(AFS_VOLUME_RM_TREE, &volume->flags); + set_bit(AFS_VOLUME_RM_TREE, &p->flags); rb_replace_node_rcu(&p->cell_node, &volume->cell_node, &cell->volumes); } } diff --git a/fs/afs/write.c b/fs/afs/write.c index fcfed9d24e0a..7f34b939706a 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -142,7 +142,7 @@ static void afs_issue_write_worker(struct work_struct *work) afs_begin_vnode_operation(op); op->store.write_iter = &subreq->io_iter; - op->store.i_size = umax(pos + len, vnode->netfs.remote_i_size); + op->store.i_size = umax(pos + len, netfs_read_remote_i_size(&vnode->netfs.inode)); op->mtime = inode_get_mtime(&vnode->netfs.inode); afs_wait_for_operation(op); diff --git a/fs/afs/yfsclient.c b/fs/afs/yfsclient.c index 24fb562ebd33..d941179730a9 100644 --- a/fs/afs/yfsclient.c +++ b/fs/afs/yfsclient.c @@ -960,7 +960,7 @@ void yfs_fs_symlink(struct afs_operation *op) _enter(""); - contents_sz = strlen(op->create.symlink); + contents_sz = strlen(op->create.symlink->content); call = afs_alloc_flat_call(op->net, &yfs_RXYFSSymlink, sizeof(__be32) + sizeof(struct yfs_xdr_RPCFlags) + @@ -981,7 +981,7 @@ void yfs_fs_symlink(struct afs_operation *op) bp = xdr_encode_u32(bp, 0); /* RPC flags */ bp = xdr_encode_YFSFid(bp, &dvp->fid); bp = xdr_encode_name(bp, name); - bp = xdr_encode_string(bp, op->create.symlink, contents_sz); + bp = xdr_encode_string(bp, op->create.symlink->content, contents_sz); bp = xdr_encode_YFSStoreStatus(bp, &mode, &op->mtime); yfs_check_req(call, bp); @@ -318,7 +318,6 @@ static int aio_init_fs_context(struct fs_context *fc) pfc = init_pseudo(fc, AIO_RING_MAGIC); if (!pfc) return -ENOMEM; - fc->s_iflags |= SB_I_NOEXEC; pfc->ops = &aio_super_operations; return 0; } diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index b8381c7fb636..a7b9b948e33d 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -86,8 +86,6 @@ static int anon_inodefs_init_fs_context(struct fs_context *fc) struct pseudo_fs_context *ctx = init_pseudo(fc, ANON_INODE_FS_MAGIC); if (!ctx) return -ENOMEM; - fc->s_iflags |= SB_I_NOEXEC; - fc->s_iflags |= SB_I_NODEV; ctx->dops = &anon_inodefs_dentry_operations; return 0; } diff --git a/fs/attr.c b/fs/attr.c index ded221defae6..4f437fabb7f0 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -547,7 +547,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, * breaking the delegation in this case. */ if (!(ia_valid & ATTR_DELEG)) { - error = try_break_deleg(inode, delegated_inode); + error = try_break_deleg(inode, 0, delegated_inode); if (error) return error; } diff --git a/fs/backing-file.c b/fs/backing-file.c index 1f3bbfc75882..080c99696cd0 100644 --- a/fs/backing-file.c +++ b/fs/backing-file.c @@ -18,17 +18,18 @@ /** * backing_file_open - open a backing file for kernel internal use - * @user_path: path that the user reuqested to open + * @user_file: file the user requested to open * @flags: open flags * @real_path: path of the backing file * @cred: credentials for open * * Open a backing file for a stackable filesystem (e.g., overlayfs). - * @user_path may be on the stackable filesystem and @real_path on the - * underlying filesystem. In this case, we want to be able to return the - * @user_path of the stackable filesystem. This is done by embedding the - * returned file into a container structure that also stores the stacked - * file's path, which can be retrieved using backing_file_user_path(). + * @user_file->f_path may be on the stackable filesystem and @real_path + * on the underlying filesystem. In this case, we want to be able to + * return the path of the stackable filesystem. This is done by + * embedding the returned file into a container structure that also + * stores the stacked file's path, which can be retrieved using + * backing_file_user_path(). */ struct file *backing_file_open(const struct file *user_file, int flags, const struct path *real_path, diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c index c12caae9a967..ee0cbae521b9 100644 --- a/fs/befs/linuxvfs.c +++ b/fs/befs/linuxvfs.c @@ -860,7 +860,8 @@ befs_fill_super(struct super_block *sb, struct fs_context *fc) */ sb->s_magic = BEFS_SUPER_MAGIC; /* Set real blocksize of fs */ - sb_set_blocksize(sb, (ulong) befs_sb->block_size); + if (!sb_set_blocksize(sb, (ulong) befs_sb->block_size)) + goto unacquire_priv_sbp; sb->s_op = &befs_sops; sb->s_export_op = &befs_export_operations; sb->s_time_min = 0; diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c index 19e49c8cf750..e41efdd35db9 100644 --- a/fs/bfs/inode.c +++ b/fs/bfs/inode.c @@ -311,7 +311,7 @@ void bfs_dump_imap(const char *prefix, struct super_block *s) { #ifdef DEBUG int i; - char *tmpbuf = (char *)get_zeroed_page(GFP_KERNEL); + char *tmpbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!tmpbuf) return; @@ -323,7 +323,7 @@ void bfs_dump_imap(const char *prefix, struct super_block *s) strcat(tmpbuf, "0"); } printf("%s: lasti=%08lx <%s>\n", prefix, BFS_SB(s)->si_lasti, tmpbuf); - free_page((unsigned long)tmpbuf); + kfree(tmpbuf); #endif } @@ -346,7 +346,8 @@ static int bfs_fill_super(struct super_block *s, struct fs_context *fc) s->s_time_min = 0; s->s_time_max = U32_MAX; - sb_set_blocksize(s, BFS_BSIZE); + if (!sb_set_blocksize(s, BFS_BSIZE)) + goto out; sbh = sb_bread(s, 0); if (!sbh) diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c index b3d8fd70e8b1..84349fcb93f1 100644 --- a/fs/binfmt_misc.c +++ b/fs/binfmt_misc.c @@ -704,7 +704,7 @@ bm_entry_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) ssize_t res; char *page; - page = (char *) __get_free_page(GFP_KERNEL); + page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page) return -ENOMEM; @@ -712,7 +712,7 @@ bm_entry_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) res = simple_read_from_buffer(buf, nbytes, ppos, page, strlen(page)); - free_page((unsigned long) page); + kfree(page); return res; } diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c index e4e51a1d0de2..f1863a891db6 100644 --- a/fs/bpf_fs_kfuncs.c +++ b/fs/bpf_fs_kfuncs.c @@ -100,7 +100,7 @@ static bool match_security_bpf_prefix(const char *name__str) static int bpf_xattr_read_permission(const char *name, struct inode *inode) { - if (WARN_ON(!inode)) + if (!inode) return -EINVAL; /* Allow reading xattr with user. and security.bpf. prefix */ @@ -170,7 +170,7 @@ __bpf_kfunc_end_defs(); static int bpf_xattr_write_permission(const char *name, struct inode *inode) { - if (WARN_ON(!inode)) + if (!inode) return -EINVAL; /* Only allow setting and removing security.bpf. xattrs */ @@ -200,7 +200,7 @@ int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str, const struct bpf_dynptr *value_p, int flags) { - struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + const struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; struct inode *inode = d_inode(dentry); const void *value; u32 value_len; @@ -289,6 +289,9 @@ __bpf_kfunc int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__st struct inode *inode = d_inode(dentry); int ret; + if (!inode) + return -EINVAL; + inode_lock(inode); ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags); inode_unlock(inode); @@ -314,6 +317,9 @@ __bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name_ struct inode *inode = d_inode(dentry); int ret; + if (!inode) + return -EINVAL; + inode_lock(inode); ret = bpf_remove_dentry_xattr_locked(dentry, name__str); inode_unlock(inode); @@ -353,6 +359,26 @@ __bpf_kfunc int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__s } #endif /* CONFIG_CGROUPS */ +/** + * bpf_real_data_inode - get the real inode hosting a file's data + * @file: file to resolve + * + * Resolve @file to the inode that hosts its data. For a regular file on a + * union/overlay filesystem this is the underlying (upper or lower) inode that + * stores the data, not the overlay inode. + * + * Data resolution only applies to regular files. For a non-regular file (e.g. + * a device node, fifo or socket) on a union/overlay filesystem the overlay + * inode itself is returned; for any file on a non-union filesystem the inode + * attached to @file is returned. + * + * Return: The inode hosting @file's data, or NULL. + */ +__bpf_kfunc struct inode *bpf_real_data_inode(struct file *file) +{ + return d_real_inode(file_dentry(file)); +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_fs_kfunc_set_ids) @@ -363,6 +389,7 @@ BTF_ID_FLAGS(func, bpf_get_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_real_data_inode, KF_SLEEPABLE | KF_RET_NULL) BTF_KFUNCS_END(bpf_fs_kfunc_set_ids) static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id) diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 5e75438e0b73..9de04c37e11a 100644 --- a/fs/btrfs/Kconfig +++ b/fs/btrfs/Kconfig @@ -93,10 +93,6 @@ config BTRFS_EXPERIMENTAL Current list: - - COW fixup worker warning - last warning before removing the - functionality catching out-of-band page - dirtying, not necessary since 5.8 - - RAID mirror read policy - additional read policies for balancing reading from redundant block group profiles (currently: pid, round-robin, @@ -110,7 +106,9 @@ config BTRFS_EXPERIMENTAL - extent tree v2 - complex rework of extent tracking - - large folio and block size (> page size) support + - block size > page size support + + - huge folios for data - folios can be as large as 2MiB now - asynchronous checksum generation for data writes diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 273924ca912c..23c3eeb58dc1 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1256,7 +1256,7 @@ static bool lookup_backref_shared_cache(struct btrfs_backref_share_check_ctx *ct * realizing. We cache results only for extent buffers that lead from * the root node down to the leaf with the file extent item. */ - ASSERT(level >= 0); + ASSERT(level >= 0, "level=%d", level); entry = &ctx->path_cache_entries[level]; @@ -1327,7 +1327,7 @@ static void store_backref_shared_cache(struct btrfs_backref_share_check_ctx *ctx * realizing. We cache results only for extent buffers that lead from * the root node down to the leaf with the file extent item. */ - ASSERT(level >= 0); + ASSERT(level >= 0, "level=%d", level); if (is_shared) gen = btrfs_get_last_root_drop_gen(fs_info); @@ -2367,7 +2367,7 @@ int tree_backref_for_extent(unsigned long *ptr, struct extent_buffer *eb, info = (struct btrfs_tree_block_info *)(ei + 1); *out_level = btrfs_tree_block_level(eb, info); } else { - ASSERT(key->type == BTRFS_METADATA_ITEM_KEY); + ASSERT(key->type == BTRFS_METADATA_ITEM_KEY, "key->type=%hhu", key->type); *out_level = (u8)key->offset; } @@ -2814,26 +2814,17 @@ struct inode_fs_paths *init_ipath(s32 total_bytes, struct btrfs_root *fs_root, return ifp; } -struct btrfs_backref_iter *btrfs_backref_iter_alloc(struct btrfs_fs_info *fs_info) +int btrfs_backref_iter_init(struct btrfs_backref_iter *iter) { - struct btrfs_backref_iter *ret; - - ret = kzalloc_obj(*ret, GFP_NOFS); - if (!ret) - return NULL; - - ret->path = btrfs_alloc_path(); - if (!ret->path) { - kfree(ret); - return NULL; - } + iter->path = btrfs_alloc_path(); + if (!iter->path) + return -ENOMEM; /* Current backref iterator only supports iteration in commit root */ - ret->path->search_commit_root = true; - ret->path->skip_locking = true; - ret->fs_info = fs_info; + iter->path->search_commit_root = true; + iter->path->skip_locking = true; - return ret; + return 0; } static void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) @@ -2846,9 +2837,8 @@ static void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) memset(&iter->cur_key, 0, sizeof(iter->cur_key)); } -int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr) +int btrfs_backref_iter_start(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter, u64 bytenr) { - struct btrfs_fs_info *fs_info = iter->fs_info; struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bytenr); struct btrfs_path *path = iter->path; struct btrfs_extent_item *ei; @@ -2963,7 +2953,7 @@ static bool btrfs_backref_iter_is_inline_ref(struct btrfs_backref_iter *iter) * Return >0 if there is no extra backref for this bytenr. * Return <0 if there is something wrong happened. */ -int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) +int btrfs_backref_iter_next(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter) { struct extent_buffer *eb = iter->path->nodes[0]; struct btrfs_root *extent_root; @@ -2974,7 +2964,9 @@ int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) if (btrfs_backref_iter_is_inline_ref(iter)) { /* We're still inside the inline refs */ - ASSERT(iter->cur_ptr < iter->end_ptr); + ASSERT(iter->cur_ptr < iter->end_ptr, + "iter->cur_ptr=%u iter->end_ptr=%u", + iter->cur_ptr, iter->end_ptr); if (btrfs_backref_has_tree_block_info(iter)) { /* First tree block info */ @@ -2997,10 +2989,9 @@ int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) } /* We're at keyed items, there is no inline item, go to the next one */ - extent_root = btrfs_extent_root(iter->fs_info, iter->bytenr); + extent_root = btrfs_extent_root(fs_info, iter->bytenr); if (unlikely(!extent_root)) { - btrfs_err(iter->fs_info, - "missing extent root for extent at bytenr %llu", + btrfs_err(fs_info, "missing extent root for extent at bytenr %llu", iter->bytenr); return -EUCLEAN; } @@ -3041,7 +3032,7 @@ struct btrfs_backref_node *btrfs_backref_alloc_node( { struct btrfs_backref_node *node; - ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL); + ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL, "level=%d", level); node = kzalloc_obj(*node, GFP_NOFS); if (!node) return node; @@ -3063,7 +3054,7 @@ void btrfs_backref_free_node(struct btrfs_backref_cache *cache, if (node) { ASSERT(list_empty(&node->list)); ASSERT(list_empty(&node->lower)); - ASSERT(node->eb == NULL); + ASSERT(node->eb == NULL, "node->eb->start=%llu", node->eb->start); cache->nr_nodes--; btrfs_put_root(node->root); kfree(node); @@ -3166,15 +3157,18 @@ void btrfs_backref_release_cache(struct btrfs_backref_cache *cache) ASSERT(list_empty(&cache->pending_edge)); ASSERT(list_empty(&cache->useless_node)); - ASSERT(!cache->nr_nodes); - ASSERT(!cache->nr_edges); + ASSERT(!cache->nr_nodes, "cache->nr_nodes=%d", cache->nr_nodes); + ASSERT(!cache->nr_edges, "cache->nr_edges=%d", cache->nr_edges); } static void btrfs_backref_link_edge(struct btrfs_backref_edge *edge, struct btrfs_backref_node *lower, struct btrfs_backref_node *upper) { - ASSERT(upper && lower && upper->level == lower->level + 1); + ASSERT(upper != NULL); + ASSERT(lower != NULL); + ASSERT(upper->level == lower->level + 1, "upper->level=%d lower->level=%d", + upper->level, lower->level); edge->node[LOWER] = lower; edge->node[UPPER] = upper; list_add_tail(&edge->list[LOWER], &lower->upper); @@ -3199,7 +3193,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, struct btrfs_backref_node *upper; struct rb_node *rb_node; - ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY); + ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY, "ref_key->type=%hhu", ref_key->type); /* Only reloc root uses backref pointing to itself */ if (ref_key->objectid == ref_key->offset) { @@ -3294,7 +3288,9 @@ static int handle_indirect_tree_backref(struct btrfs_trans_handle *trans, if (btrfs_root_level(&root->root_item) == cur->level) { /* Tree root */ - ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr); + ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr, + "root_bytenr=%llu cur->bytenr=%llu", + btrfs_root_bytenr(&root->root_item), cur->bytenr); /* * For reloc backref cache, we may ignore reloc root. But for * general purpose backref cache, we can't rely on @@ -3344,8 +3340,9 @@ static int handle_indirect_tree_backref(struct btrfs_trans_handle *trans, /* Add all nodes and edges in the path */ for (; level < BTRFS_MAX_LEVEL; level++) { if (!path->nodes[level]) { - ASSERT(btrfs_root_bytenr(&root->root_item) == - lower->bytenr); + ASSERT(btrfs_root_bytenr(&root->root_item) == lower->bytenr, + "root_bytenr=%llu lower->bytenr=%llu", + btrfs_root_bytenr(&root->root_item), lower->bytenr); /* Same as previous should_ignore_reloc_root() call */ if (btrfs_should_ignore_reloc_root(root) && cache->is_reloc) { @@ -3454,7 +3451,7 @@ int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, struct btrfs_backref_node *exist; int ret; - ret = btrfs_backref_iter_start(iter, cur->bytenr); + ret = btrfs_backref_iter_start(trans->fs_info, iter, cur->bytenr); if (ret < 0) return ret; /* @@ -3462,7 +3459,7 @@ int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, * stored in it, but fetch it from the tree block */ if (btrfs_backref_has_tree_block_info(iter)) { - ret = btrfs_backref_iter_next(iter); + ret = btrfs_backref_iter_next(trans->fs_info, iter); if (ret < 0) goto out; /* No extra backref? This means the tree block is corrupted */ @@ -3492,7 +3489,7 @@ int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, exist = NULL; } - for (; ret == 0; ret = btrfs_backref_iter_next(iter)) { + for (; ret == 0; ret = btrfs_backref_iter_next(trans->fs_info, iter)) { struct extent_buffer *eb; struct btrfs_key key; int type; diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 1d009b0f4c69..179791de6b19 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -278,15 +278,12 @@ struct prelim_ref { struct btrfs_backref_iter { u64 bytenr; struct btrfs_path *path; - struct btrfs_fs_info *fs_info; struct btrfs_key cur_key; u32 item_ptr; u32 cur_ptr; u32 end_ptr; }; -struct btrfs_backref_iter *btrfs_backref_iter_alloc(struct btrfs_fs_info *fs_info); - /* * For metadata with EXTENT_ITEM key (non-skinny) case, the first inline data * is btrfs_tree_block_info, without a btrfs_extent_inline_ref header. @@ -302,9 +299,11 @@ static inline bool btrfs_backref_has_tree_block_info( return false; } -int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr); +int btrfs_backref_iter_init(struct btrfs_backref_iter *iter); + +int btrfs_backref_iter_start(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter, u64 bytenr); -int btrfs_backref_iter_next(struct btrfs_backref_iter *iter); +int btrfs_backref_iter_next(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter); /* * Backref cache related structures diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index e6f5a17a13e3..ab76a5173272 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -22,6 +22,34 @@ #include "accessors.h" #include "extent-tree.h" +static struct kmem_cache *block_group_cache; +static struct kmem_cache *free_space_ctl_cache; + +int __init btrfs_init_block_group(void) +{ + block_group_cache = kmem_cache_create("btrfs_block_group", + sizeof(struct btrfs_block_group), + 0, 0, NULL); + if (!block_group_cache) + return -ENOMEM; + + free_space_ctl_cache = kmem_cache_create("btrfs_free_space_ctl", + sizeof(struct btrfs_free_space_ctl), + 0, 0, NULL); + if (!free_space_ctl_cache) { + kmem_cache_destroy(block_group_cache); + return -ENOMEM; + } + + return 0; +} + +void __cold btrfs_exit_block_group(void) +{ + kmem_cache_destroy(block_group_cache); + kmem_cache_destroy(free_space_ctl_cache); +} + #ifdef CONFIG_BTRFS_DEBUG int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group) { @@ -180,9 +208,9 @@ void btrfs_put_block_group(struct btrfs_block_group *cache) btrfs_discard_cancel_work(&cache->fs_info->discard_ctl, cache); - kfree(cache->free_space_ctl); + kmem_cache_free(free_space_ctl_cache, cache->free_space_ctl); btrfs_free_chunk_map(cache->physical_map); - kfree(cache); + kmem_cache_free(block_group_cache, cache); } } @@ -2371,13 +2399,13 @@ static struct btrfs_block_group *btrfs_create_block_group( { struct btrfs_block_group *cache; - cache = kzalloc_obj(*cache, GFP_NOFS); + cache = kmem_cache_zalloc(block_group_cache, GFP_NOFS); if (!cache) return NULL; - cache->free_space_ctl = kzalloc_obj(*cache->free_space_ctl, GFP_NOFS); + cache->free_space_ctl = kmem_cache_zalloc(free_space_ctl_cache, GFP_NOFS); if (!cache->free_space_ctl) { - kfree(cache); + kmem_cache_free(block_group_cache, cache); return NULL; } @@ -2412,29 +2440,25 @@ static struct btrfs_block_group *btrfs_create_block_group( */ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) { - u64 start = 0; + struct rb_node *node; int ret = 0; - while (1) { + /* + * This is called during mount from btrfs_read_block_groups(), before + * any background threads are started, so no concurrent writers can + * modify the mapping_tree. No lock is needed here. + */ + for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) { struct btrfs_chunk_map *map; struct btrfs_block_group *bg; - /* - * btrfs_find_chunk_map() will return the first chunk map - * intersecting the range, so setting @length to 1 is enough to - * get the first chunk. - */ - map = btrfs_find_chunk_map(fs_info, start, 1); - if (!map) - break; - + map = rb_entry(node, struct btrfs_chunk_map, rb_node); bg = btrfs_lookup_block_group(fs_info, map->start); if (unlikely(!bg)) { btrfs_err(fs_info, "chunk start=%llu len=%llu doesn't have corresponding block group", map->start, map->chunk_len); ret = -EUCLEAN; - btrfs_free_chunk_map(map); break; } if (unlikely(bg->start != map->start || bg->length != map->chunk_len || @@ -2447,12 +2471,9 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) bg->start, bg->length, bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK); ret = -EUCLEAN; - btrfs_free_chunk_map(map); btrfs_put_block_group(bg); break; } - start = map->start + map->chunk_len; - btrfs_free_chunk_map(map); btrfs_put_block_group(bg); } return ret; @@ -2461,7 +2482,7 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) static int read_one_block_group(struct btrfs_fs_info *info, struct btrfs_block_group_item_v2 *bgi, const struct btrfs_key *key, - int need_clear) + bool need_clear) { struct btrfs_block_group *cache; const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS); @@ -2642,7 +2663,7 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) struct btrfs_block_group *cache; struct btrfs_space_info *space_info; struct btrfs_key key; - int need_clear = 0; + bool need_clear = false; u64 cache_gen; /* @@ -2667,9 +2688,9 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) cache_gen = btrfs_super_cache_generation(info->super_copy); if (btrfs_test_opt(info, SPACE_CACHE) && btrfs_super_generation(info->super_copy) != cache_gen) - need_clear = 1; + need_clear = true; if (btrfs_test_opt(info, CLEAR_CACHE)) - need_clear = 1; + need_clear = true; while (1) { struct btrfs_block_group_item_v2 bgi; @@ -4096,7 +4117,7 @@ int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type) struct btrfs_space_info *space_info; space_info = btrfs_find_space_info(trans->fs_info, type); - if (!space_info) { + if (unlikely(!space_info)) { DEBUG_WARN(); return -EINVAL; } diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 0504cb357992..790c2d467af5 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -122,6 +122,7 @@ struct btrfs_block_group { struct btrfs_fs_info *fs_info; struct btrfs_inode *inode; spinlock_t lock; + unsigned int ro; u64 start; u64 length; u64 pinned; @@ -134,7 +135,8 @@ struct btrfs_block_group { u64 global_root_id; u64 remap_bytes; u32 identity_remap_count; - + /* The last commited identity_remap_count value of this block group. */ + u32 last_identity_remap_count; /* * The last committed used bytes of this block group, if the above @used * is still the same as @last_used, we don't need to update block @@ -143,8 +145,6 @@ struct btrfs_block_group { u64 last_used; /* The last committed remap_bytes value of this block group. */ u64 last_remap_bytes; - /* The last commited identity_remap_count value of this block group. */ - u32 last_identity_remap_count; /* The last committed flags value for this block group. */ u64 last_flags; @@ -171,12 +171,10 @@ struct btrfs_block_group { unsigned long full_stripe_len; unsigned long runtime_flags; - unsigned int ro; - - int disk_cache_state; + enum btrfs_disk_cache_state disk_cache_state; /* Cache tracking stuff */ - int cached; + enum btrfs_caching_type cached; struct btrfs_caching_control *caching_ctl; struct btrfs_space_info *space_info; @@ -193,6 +191,16 @@ struct btrfs_block_group { refcount_t refs; /* + * When non-zero it means the block group's logical address and its + * device extents can not be reused for future block group allocations + * until the counter goes down to 0. This is to prevent them from being + * reused while some task is still using the block group after it was + * deleted - we want to make sure they can only be reused for new block + * groups after that task is done with the deleted block group. + */ + atomic_t frozen; + + /* * List of struct btrfs_free_clusters for this block group. * Today it will only have one thing on it, but that may change */ @@ -211,22 +219,12 @@ struct btrfs_block_group { /* For read-only block groups */ struct list_head ro_list; - /* - * When non-zero it means the block group's logical address and its - * device extents can not be reused for future block group allocations - * until the counter goes down to 0. This is to prevent them from being - * reused while some task is still using the block group after it was - * deleted - we want to make sure they can only be reused for new block - * groups after that task is done with the deleted block group. - */ - atomic_t frozen; - /* For discard operations */ struct list_head discard_list; int discard_index; + enum btrfs_discard_state discard_state; u64 discard_eligible_time; u64 discard_cursor; - enum btrfs_discard_state discard_state; /* For dirty block groups */ struct list_head dirty_list; @@ -263,6 +261,8 @@ struct btrfs_block_group { /* Protected by @free_space_lock. */ bool using_free_space_bitmaps_cached; + enum btrfs_block_group_size_class size_class:8; + /* * Number of extents in this block group used for swap files. * All accesses protected by the spinlock 'lock'. @@ -281,7 +281,6 @@ struct btrfs_block_group { struct list_head active_bg_list; struct work_struct zone_finish_work; struct extent_buffer *last_eb; - enum btrfs_block_group_size_class size_class; u64 reclaim_mark; }; @@ -320,6 +319,9 @@ static inline u64 btrfs_block_group_available_space(const struct btrfs_block_gro int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group); #endif +int __init btrfs_init_block_group(void); +void __cold btrfs_exit_block_group(void); + struct btrfs_block_group *btrfs_lookup_first_block_group( struct btrfs_fs_info *info, u64 bytenr); struct btrfs_block_group *btrfs_lookup_block_group( diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 55c272fe5d92..d5d81f9546c3 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -128,15 +128,6 @@ struct btrfs_inode { /* which subvolume this inode belongs to */ struct btrfs_root *root; -#if BITS_PER_LONG == 32 - /* - * The objectid of the corresponding BTRFS_INODE_ITEM_KEY. - * On 64 bits platforms we can get it from vfs_inode.i_ino, which is an - * unsigned long and therefore 64 bits on such platforms. - */ - u64 objectid; -#endif - /* Cached value of inode property 'compression'. */ u8 prop_compress; @@ -372,30 +363,11 @@ static inline unsigned long btrfs_inode_hash(u64 objectid, return (unsigned long)h; } -#if BITS_PER_LONG == 32 - -/* - * On 32 bit systems the i_ino of struct inode is 32 bits (unsigned long), so - * we use the inode's location objectid which is a u64 to avoid truncation. - */ -static inline u64 btrfs_ino(const struct btrfs_inode *inode) -{ - u64 ino = inode->objectid; - - if (test_bit(BTRFS_INODE_ROOT_STUB, &inode->runtime_flags)) - ino = inode->vfs_inode.i_ino; - return ino; -} - -#else - static inline u64 btrfs_ino(const struct btrfs_inode *inode) { return inode->vfs_inode.i_ino; } -#endif - static inline void btrfs_get_inode_key(const struct btrfs_inode *inode, struct btrfs_key *key) { @@ -406,9 +378,6 @@ static inline void btrfs_get_inode_key(const struct btrfs_inode *inode, static inline void btrfs_set_inode_number(struct btrfs_inode *inode, u64 ino) { -#if BITS_PER_LONG == 32 - inode->objectid = ino; -#endif inode->vfs_inode.i_ino = ino; } @@ -531,12 +500,9 @@ static inline void btrfs_set_inode_mapping_order(struct btrfs_inode *inode) /* Metadata inode should not reach here. */ ASSERT(is_data_inode(inode)); - /* We only allow BITS_PER_LONGS blocks for each bitmap. */ -#ifdef CONFIG_BTRFS_EXPERIMENTAL mapping_set_folio_order_range(inode->vfs_inode.i_mapping, inode->root->fs_info->block_min_order, inode->root->fs_info->block_max_order); -#endif } void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info, @@ -569,6 +535,8 @@ int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, unsigned int extra_bits, struct extent_state **cached_state); +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state); struct btrfs_new_inode_args { /* Input */ @@ -630,7 +598,6 @@ int btrfs_prealloc_file_range_trans(struct inode *inode, loff_t actual_len, u64 *alloc_hint); int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio, u64 start, u64 end, struct writeback_control *wbc); -int btrfs_writepage_cow_fixup(struct folio *folio); int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info, int compress_type); int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index b2393a48a8fe..ffb6b52863a7 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -355,21 +355,16 @@ struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, } /* - * Add extra pages in the same compressed file extent so that we don't need to + * Add extra folios in the same compressed file extent so that we don't need to * re-read the same extent again and again. * - * NOTE: this won't work well for subpage, as for subpage read, we lock the - * full page then submit bio for each compressed/regular extents. - * - * This means, if we have several sectors in the same page points to the same - * on-disk compressed data, we will re-read the same extent many times and - * this function can only help for the next page. + * If in the same folio, we have several non-contiguous blocks which are pointing + * to the same on-disk compressed data, we will re-read the same extent many + * times, as this function can only help cross folio situations. */ -static noinline int add_ra_bio_pages(struct inode *inode, - u64 compressed_end, - struct compressed_bio *cb, - int *memstall, unsigned long *pflags, - bool direct_reclaim) +static noinline int add_ra_bio_folios(struct inode *inode, u64 compressed_end, + struct compressed_bio *cb, int *memstall, + unsigned long *pflags, bool direct_reclaim) { struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); pgoff_t end_index; @@ -391,38 +386,24 @@ static noinline int add_ra_bio_pages(struct inode *inode, if (isize == 0) return 0; - /* - * For current subpage support, we only support 64K page size, - * which means maximum compressed extent size (128K) is just 2x page - * size. - * This makes readahead less effective, so here disable readahead for - * subpage for now, until full compressed write is supported. - */ - if (fs_info->sectorsize < PAGE_SIZE) - return 0; - /* For bs > ps cases, we don't support readahead for compressed folios for now. */ if (fs_info->block_min_order) return 0; end_index = (i_size_read(inode) - 1) >> PAGE_SHIFT; - /* - * Avoid direct reclaim when the caller does not allow it. Since - * add_ra_bio_pages() is always speculative, suppress allocation warnings - * in either case. - */ + /* Avoid direct reclaim when the caller does not allow it. */ + constraint_gfp = ~__GFP_FS; + cache_gfp = GFP_NOFS | __GFP_NOWARN; if (!direct_reclaim) { - constraint_gfp = ~(__GFP_FS | __GFP_DIRECT_RECLAIM) | __GFP_NOWARN; - cache_gfp = (GFP_NOFS & ~__GFP_DIRECT_RECLAIM) | __GFP_NOWARN; - } else { - constraint_gfp = (~__GFP_FS) | __GFP_NOWARN; - cache_gfp = GFP_NOFS | __GFP_NOWARN; + constraint_gfp &= ~__GFP_DIRECT_RECLAIM; + cache_gfp &= ~__GFP_DIRECT_RECLAIM; } while (cur < compressed_end) { - pgoff_t page_end; + u64 folio_end; pgoff_t pg_index = cur >> PAGE_SHIFT; + gfp_t masked_constraint_gfp; u32 add_size; if (pg_index > end_index) @@ -442,21 +423,27 @@ static noinline int add_ra_bio_pages(struct inode *inode, break; /* - * Jump to next page start as we already have page for - * current offset. + * Jump to the next folio as we already have a folio for + * the current offset. */ cur += (folio_sz - offset); continue; } - folio = filemap_alloc_folio(mapping_gfp_constraint(mapping, constraint_gfp), - 0, NULL); + /* + * Since add_ra_bio_pages() is always speculative, suppress + * allocation warnings. + */ + masked_constraint_gfp = mapping_gfp_constraint(mapping, constraint_gfp); + masked_constraint_gfp |= __GFP_NOWARN; + + folio = filemap_alloc_folio(masked_constraint_gfp, 0, NULL); if (!folio) break; if (filemap_add_folio(mapping, folio, pg_index, cache_gfp)) { - /* There is already a page, skip to page end */ - cur += folio_size(folio); + /* There is already a folio, skip to the folio end. */ + cur += folio_size(folio) - offset_in_folio(folio, cur); folio_put(folio); continue; } @@ -473,14 +460,14 @@ static noinline int add_ra_bio_pages(struct inode *inode, break; } - page_end = (pg_index << PAGE_SHIFT) + folio_size(folio) - 1; - btrfs_lock_extent(tree, cur, page_end, NULL); + folio_end = folio_next_pos(folio) - 1; + btrfs_lock_extent(tree, cur, folio_end, NULL); read_lock(&em_tree->lock); - em = btrfs_lookup_extent_mapping(em_tree, cur, page_end + 1 - cur); + em = btrfs_lookup_extent_mapping(em_tree, cur, folio_end + 1 - cur); read_unlock(&em_tree->lock); /* - * At this point, we have a locked page in the page cache for + * At this point, we have a locked folio in the page cache for * these bytes in the file. But, we have to make sure they map * to this compressed extent on disk. */ @@ -489,14 +476,14 @@ static noinline int add_ra_bio_pages(struct inode *inode, (btrfs_extent_map_block_start(em) >> SECTOR_SHIFT) != orig_bio->bi_iter.bi_sector) { btrfs_free_extent_map(em); - btrfs_unlock_extent(tree, cur, page_end, NULL); + btrfs_unlock_extent(tree, cur, folio_end, NULL); folio_unlock(folio); folio_put(folio); break; } - add_size = min(btrfs_extent_map_end(em), page_end + 1) - cur; + add_size = min(btrfs_extent_map_end(em), folio_end + 1) - cur; btrfs_free_extent_map(em); - btrfs_unlock_extent(tree, cur, page_end, NULL); + btrfs_unlock_extent(tree, cur, folio_end, NULL); if (folio_contains(folio, end_index)) { size_t zero_offset = offset_in_folio(folio, isize); @@ -514,13 +501,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, folio_put(folio); break; } - /* - * If it's subpage, we also need to increase its - * subpage::readers number, as at endio we will decrease - * subpage::readers and to unlock the page. - */ - if (fs_info->sectorsize < PAGE_SIZE) - btrfs_folio_set_lock(fs_info, folio, cur, add_size); + btrfs_folio_set_lock(fs_info, folio, cur, add_size); folio_put(folio); cur += add_size; } @@ -611,8 +592,8 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) } ASSERT(cb->bbio.bio.bi_iter.bi_size == compressed_len); - add_ra_bio_pages(&inode->vfs_inode, em_start + em_len, cb, &memstall, - &pflags, !(bbio->bio.bi_opf & REQ_RAHEAD)); + add_ra_bio_folios(&inode->vfs_inode, em_start + em_len, cb, &memstall, + &pflags, !(bbio->bio.bi_opf & REQ_RAHEAD)); cb->len = bbio->bio.bi_iter.bi_size; cb->bbio.bio.bi_iter.bi_sector = bbio->bio.bi_iter.bi_sector; @@ -1190,22 +1171,6 @@ void __cold btrfs_exit_compress(void) } /* - * The bvec is a single page bvec from a bio that contains folios from a filemap. - * - * Since the folio may be a large one, and if the bv_page is not a head page of - * a large folio, then page->index is unreliable. - * - * Thus we need this helper to grab the proper file offset. - */ -static u64 file_offset_from_bvec(const struct bio_vec *bvec) -{ - const struct page *page = bvec->bv_page; - const struct folio *folio = page_folio(page); - - return (page_pgoff(folio, page) << PAGE_SHIFT) + bvec->bv_offset; -} - -/* * Copy decompressed data from working buffer to pages. * * @buf: The decompressed data buffer @@ -1257,7 +1222,7 @@ int btrfs_decompress_buf2page(const char *buf, u32 buf_len, * cb->start may underflow, but subtracting that value can still * give us correct offset inside the full decompressed extent. */ - bvec_offset = file_offset_from_bvec(&bvec) - cb->start; + bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start; /* Haven't reached the bvec range, exit */ if (decompressed + buf_len <= bvec_offset) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index d70da290bedf..49fb6b816aa9 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -475,13 +475,10 @@ int btrfs_force_cow_block(struct btrfs_trans_handle *trans, struct extent_buffer *cow; int level, ret; int last_ref = 0; - int unlock_orig = 0; + const bool unlock_orig = (*cow_ret == buf); u64 parent_start = 0; u64 reloc_src_root = 0; - if (*cow_ret == buf) - unlock_orig = 1; - btrfs_assert_tree_write_locked(buf); WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && @@ -1497,17 +1494,11 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, if (p->reada == READA_FORWARD_ALWAYS) reada_for_search(fs_info, p, parent_level, slot, key->objectid); - /* first we do an atomic uptodate check */ - if (btrfs_buffer_uptodate(tmp, check.transid, NULL) > 0) { - /* - * Do extra check for first_key, eb can be stale due to - * being cached, read from scrub, or have multiple - * parents (shared tree blocks). - */ - if (unlikely(btrfs_verify_level_key(tmp, &check))) { - ret = -EUCLEAN; - goto out; - } + /* Check if the cached eb is uptodate. */ + ret = btrfs_buffer_uptodate(tmp, check.transid, &check); + if (unlikely(ret < 0)) + goto out; + if (ret > 0) { *eb_ret = tmp; tmp = NULL; ret = 0; @@ -2075,7 +2066,7 @@ again: } while (b) { - int dec = 0; + bool dec = false; int ret2; level = btrfs_header_level(b); @@ -2158,7 +2149,7 @@ cow_done: prev_cmp = ret; if (ret && slot > 0) { - dec = 1; + dec = true; slot--; } p->slots[level] = slot; @@ -2288,7 +2279,7 @@ again: p->locks[level] = BTRFS_READ_LOCK; while (b) { - int dec = 0; + bool dec = false; int ret2; level = btrfs_header_level(b); @@ -2313,7 +2304,7 @@ again: } if (ret && slot > 0) { - dec = 1; + dec = true; slot--; } p->slots[level] = slot; @@ -3674,7 +3665,7 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans, int wret; int split; int num_doubles = 0; - int tried_avoid_double = 0; + bool tried_avoid_double = false; l = path->nodes[0]; slot = path->slots[0]; @@ -3836,7 +3827,7 @@ again: push_for_double: push_for_double_split(trans, root, path, data_size); - tried_avoid_double = 1; + tried_avoid_double = true; if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) return 0; goto again; diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index 7e2db5d3a4d4..f0c6758b7055 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -859,23 +859,6 @@ again: if (IS_ERR(folio)) return folio; - /* - * Since we can defragment files opened read-only, we can encounter - * transparent huge pages here (see CONFIG_READ_ONLY_THP_FOR_FS). - * - * The IO for such large folios is not fully tested, thus return - * an error to reject such folios unless it's an experimental build. - * - * Filesystem transparent huge pages are typically only used for - * executables that explicitly enable them, so this isn't very - * restrictive. - */ - if (!IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL) && folio_test_large(folio)) { - folio_unlock(folio); - folio_put(folio); - return ERR_PTR(-ETXTBSY); - } - ret = set_folio_extent_mapped(folio); if (ret < 0) { folio_unlock(folio); @@ -1179,7 +1162,6 @@ static int defrag_one_locked_target(struct btrfs_inode *inode, if (start >= folio_next_pos(folio) || start + len <= folio_pos(folio)) continue; - btrfs_folio_clamp_clear_checked(fs_info, folio, start, len); btrfs_folio_clamp_set_dirty(fs_info, folio, start, len); } btrfs_delalloc_release_extents(inode, len); diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c index 0970799d0aa4..d357ed7efd99 100644 --- a/fs/btrfs/delalloc-space.c +++ b/fs/btrfs/delalloc-space.c @@ -134,6 +134,8 @@ int btrfs_alloc_data_chunk_ondemand(const struct btrfs_inode *inode, u64 bytes) if (btrfs_is_free_space_inode(inode)) flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE; + else if (btrfs_is_zoned(fs_info) && btrfs_is_data_reloc_root(root)) + flush = BTRFS_RESERVE_FLUSH_ZONED_RELOCATION; return btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), bytes, flush); } @@ -279,7 +281,7 @@ static void btrfs_calculate_inode_block_rsv_size(struct btrfs_fs_info *fs_info, * * This is overestimating in most cases. */ - qgroup_rsv_size = (u64)outstanding_extents * fs_info->nodesize; + qgroup_rsv_size = ((u64)outstanding_extents << fs_info->nodesize_bits); spin_lock(&block_rsv->lock); block_rsv->size = reserve_size; @@ -309,7 +311,7 @@ static void calc_inode_reservations(struct btrfs_inode *inode, * for an inode update. */ *meta_reserve += inode_update; - *qgroup_reserve = nr_extents * fs_info->nodesize; + *qgroup_reserve = (nr_extents << fs_info->nodesize_bits); } int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes, diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 605858c2d9a9..8bc1929237f7 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -615,6 +615,9 @@ static bool insert_delayed_ref(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_node *exist; int mod; + ASSERT(ref->action == BTRFS_ADD_DELAYED_REF || + ref->action == BTRFS_DROP_DELAYED_REF); + spin_lock(&href->lock); exist = tree_insert(&href->ref_tree, ref); if (!exist) { @@ -641,7 +644,7 @@ static bool insert_delayed_ref(struct btrfs_trans_handle *trans, ASSERT(!list_empty(&exist->add_list)); list_del_init(&exist->add_list); } else { - ASSERT(0); + DEBUG_WARN(); } } else mod = -ref->ref_mod; diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index 8f8fa14886de..318ddb790429 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -307,6 +307,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, device->bdev_file = bdev_file; set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + /* Check the comment in btrfs_init_new_device() for the reason. */ + atomic_inc(&device->dev_stats_ccnt); device->dev_stats_valid = 1; set_blocksize(bdev_file, BTRFS_BDEV_BLOCKSIZE); device->fs_devices = fs_devices; @@ -1013,8 +1015,15 @@ error: /* write back the superblocks */ trans = btrfs_start_transaction(root, 0); - if (!IS_ERR(trans)) + if (!IS_ERR(trans)) { + /* + * Ignore any error here, if we failed to remove the DEV_STATS + * item for devid 0, it's not a big deal. We have other ways + * to address it. + */ + btrfs_remove_dev_stat_item(trans, BTRFS_DEV_REPLACE_DEVID); btrfs_commit_transaction(trans); + } mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 57167d56dc72..460326d34143 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -15,10 +15,12 @@ struct btrfs_dio_data { ssize_t submitted; + loff_t old_isize; struct extent_changeset *data_reserved; struct btrfs_ordered_extent *ordered; bool data_space_reserved; bool nocow_done; + bool updated_isize; }; struct btrfs_dio_private { @@ -228,6 +230,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, bool space_reserved = false; u64 len = *lenp; u64 prev_len; + loff_t old_isize; int ret = 0; /* @@ -341,8 +344,14 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, * Need to update the i_size under the extent lock so buffered * readers will get the updated i_size when we unlock. */ - if (start + len > i_size_read(inode)) + old_isize = i_size_read(inode); + if (start + len > old_isize) { + if (!dio_data->updated_isize) { + dio_data->old_isize = old_isize; + dio_data->updated_isize = true; + } i_size_write(inode, start + len); + } out: if (ret && space_reserved) { btrfs_delalloc_release_extents(BTRFS_I(inode), len); @@ -624,12 +633,55 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, if (submitted < length) { pos += submitted; length -= submitted; - if (write) + if (write) { + /* + * Got a short write and have updated the isize, need to + * revert the isize change. + * + * Normally we need to update isize with extent lock hold, + * but we're safe due to the following factors: + * + * - Only a single writer can be enlarging isize + * Enlarging isize will take the exclusive inode lock. + * + * - Buffered readers need to wait for the OE we're holding + * Buffered readers will lock extent and wait for OE + * of the folio range, and since page cache is invalidated + * the OE wait can not be skipped. + * + * So here we are safe to revert the isize before + * finishing the OE, and no reader of the remaining range + * can see the enlarged size. + * + * TODO: Extend the DIO_LOCKED lifespan for direct writes, + * and only enlarge isize after a successful write. + */ + if (dio_data->updated_isize) { + u64 new_isize; + + if (submitted == 0) + new_isize = dio_data->old_isize; + else + new_isize = max(dio_data->old_isize, pos); + i_size_write(inode, new_isize); + dio_data->updated_isize = false; + } + /* + * We have a short write, if there is any range + * that is submitted properly, that part will have + * its own OE split from the original one. + * + * So for the OE at dio_data->ordered, it's the part + * that is not submitted, and should be marked + * as fully truncated. + */ + btrfs_mark_ordered_extent_truncated(dio_data->ordered, 0); btrfs_finish_ordered_extent(dio_data->ordered, - pos, length, false); - else + pos, length, true); + } else { btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1, NULL); + } ret = -ENOTBLK; } if (write) { @@ -926,7 +978,7 @@ again: if (ret > 0) written = ret; - if (iov_iter_count(from) > 0 && (ret == -EFAULT || ret > 0)) { + if (iov_iter_count(from) > 0 && (ret == -EFAULT || ret >= 0)) { const size_t left = iov_iter_count(from); /* * We have more data left to write. Try to fault in as many as diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 8a11be02eeb9..274e2f0826b6 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -215,7 +215,7 @@ int btrfs_read_extent_buffer(struct extent_buffer *eb, const struct btrfs_tree_parent_check *check) { struct btrfs_fs_info *fs_info = eb->fs_info; - int failed = 0; + bool failed = false; int ret; int num_copies = 0; int mirror_num = 0; @@ -234,7 +234,7 @@ int btrfs_read_extent_buffer(struct extent_buffer *eb, break; if (!failed_mirror) { - failed = 1; + failed = true; failed_mirror = eb->read_mirror; } @@ -491,10 +491,34 @@ static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags) static void btree_invalidate_folio(struct folio *folio, size_t offset, size_t length) { - struct extent_io_tree *tree; + struct extent_io_tree *tree = &folio_to_inode(folio)->io_tree; + struct extent_state *cached_state = NULL; + const u64 start = folio_pos(folio); + const u64 end = folio_next_pos(folio) - 1; + + /* + * The range must cover the full @folio. + * Btree inode is never exposed to regular file operations, thus there + * is no partial truncation. + * The folio is only invalidated when the btree inode is evicted. + */ + ASSERT(offset == 0, "folio=%llu offset=%zu", folio_pos(folio), offset); + ASSERT(length == folio_size(folio), "folio=%llu folio_size=%zu length=%zu", + folio_pos(folio), folio_size(folio), length); + + /* This function is only called for the btree inode */ + ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO); + + btrfs_lock_extent(tree, start, end, &cached_state); + folio_wait_writeback(folio); + + /* + * Currently for btree io tree, only EXTENT_LOCKED is utilized, + * so here we only need to unlock the extent range to free any + * existing extent state. + */ + btrfs_unlock_extent(tree, start, end, &cached_state); - tree = &folio_to_inode(folio)->io_tree; - extent_invalidate_folio(tree, folio, offset); btree_release_folio(folio, GFP_NOFS); if (folio_get_private(folio)) { btrfs_warn(folio_to_fs_info(folio), @@ -539,7 +563,7 @@ static bool btree_dirty_folio(struct address_space *mapping, continue; } spin_unlock_irqrestore(&subpage->lock, flags); - cur = page_start + cur_bit * fs_info->sectorsize; + cur = page_start + (cur_bit << fs_info->sectorsize_bits); eb = find_extent_buffer(fs_info, cur); ASSERT(eb); @@ -1736,7 +1760,6 @@ static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) /* helper to cleanup workers */ static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) { - btrfs_destroy_workqueue(fs_info->fixup_workers); btrfs_destroy_workqueue(fs_info->delalloc_workers); btrfs_destroy_workqueue(fs_info->workers); if (fs_info->endio_workers) @@ -1928,7 +1951,7 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) { u32 max_active = fs_info->thread_pool_size; unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND; - unsigned int ordered_flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_PERCPU; + unsigned int ordered_flags = WQ_MEM_RECLAIM | WQ_FREEZABLE; fs_info->workers = btrfs_alloc_workqueue(fs_info, "worker", flags, max_active, 16); @@ -1944,9 +1967,6 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) fs_info->caching_workers = btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0); - fs_info->fixup_workers = - btrfs_alloc_ordered_workqueue(fs_info, "fixup", ordered_flags); - fs_info->endio_workers = alloc_workqueue("btrfs-endio", flags, max_active); fs_info->endio_meta_workers = @@ -1972,7 +1992,7 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) fs_info->endio_workers && fs_info->endio_meta_workers && fs_info->endio_write_workers && fs_info->endio_freespace_worker && fs_info->rmw_workers && - fs_info->caching_workers && fs_info->fixup_workers && + fs_info->caching_workers && fs_info->delayed_workers && fs_info->qgroup_rescan_workers && fs_info->discard_ctl.discard_workers)) { return -ENOMEM; @@ -2776,6 +2796,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) mutex_init(&fs_info->unused_bg_unpin_mutex); mutex_init(&fs_info->reclaim_bgs_lock); mutex_init(&fs_info->reloc_mutex); + spin_lock_init(&fs_info->reloc_ctl_lock); mutex_init(&fs_info->delalloc_root_mutex); mutex_init(&fs_info->zoned_meta_io_lock); mutex_init(&fs_info->zoned_data_reloc_io_lock); @@ -3276,6 +3297,64 @@ static bool fs_is_full_ro(const struct btrfs_fs_info *fs_info) return false; } +/* + * Try to wait for any metadata readahead, and invalidate all btree folios. + * + * If the invalidation failed, report any dirty/held extent buffers. + */ +static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) +{ + unsigned long index = 0; + struct extent_buffer *eb; + int ret; + + ret = invalidate_inode_pages2(fs_info->btree_inode->i_mapping); + if (likely(ret == 0)) + return; + + /* + * Some btree pages can not be invalidated, this happens when some tree + * blocks are still held (either by readahead or some task is holding a ref). + */ + rcu_read_lock(); + xa_for_each(&fs_info->buffer_tree, index, eb) { + /* Increase the ref so that the eb won't disappear. */ + if (!refcount_inc_not_zero(&eb->refs)) + continue; + rcu_read_unlock(); + + /* Wait for any readahead first. */ + if (test_bit(EXTENT_BUFFER_READING, &eb->bflags)) + wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, + TASK_UNINTERRUPTIBLE); + /* + * The refs threshold is 2, one held by us at the beginning + * of the loop, one for the ownership in the buffer tree. + */ + if (unlikely(refcount_read(&eb->refs) > 2 || extent_buffer_under_io(eb))) { + WARN_ON_ONCE(IS_ENABLED(CONFIG_BTRFS_DEBUG)); + btrfs_warn(fs_info, + "unable to release extent buffer %llu owner %llu gen %llu refs %u flags 0x%lx", + eb->start, btrfs_header_owner(eb), + btrfs_header_generation(eb), + refcount_read(&eb->refs), eb->bflags); + } + free_extent_buffer(eb); + rcu_read_lock(); + } + rcu_read_unlock(); + invalidate_inode_pages2(fs_info->btree_inode->i_mapping); +} + +static u32 calc_block_max_order(u32 sectorsize_bits) +{ + u32 max_size; + + max_size = min(BTRFS_MAX_BLOCKS_PER_FOLIO << sectorsize_bits, + BTRFS_MAX_FOLIO_SIZE); + return ilog2(round_up(max_size, PAGE_SIZE) >> PAGE_SHIFT); +} + int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices) { u32 sectorsize; @@ -3398,7 +3477,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device fs_info->sectorsize = sectorsize; fs_info->sectorsize_bits = ilog2(sectorsize); fs_info->block_min_order = ilog2(round_up(sectorsize, PAGE_SIZE) >> PAGE_SHIFT); - fs_info->block_max_order = ilog2((BITS_PER_LONG << fs_info->sectorsize_bits) >> PAGE_SHIFT); + fs_info->block_max_order = calc_block_max_order(fs_info->sectorsize_bits); fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size; fs_info->stripesize = stripesize; fs_info->fs_devices->fs_info = fs_info; @@ -3451,7 +3530,16 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device /* Update the values for the current filesystem. */ sb->s_blocksize = sectorsize; sb->s_blocksize_bits = blksize_bits(sectorsize); - memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE); + /* + * When temp_fsid is active, fs_devices->fsid is assigned a random UUID + * at mount. This inconsistent UUID causes issues for layered filesystems + * like OverlayFS. Since metadata_uuid may or may not be set, provide the + * on-disk UUID directly from the super_copy. + */ + if (fs_info->fs_devices->temp_fsid) + memcpy(&sb->s_uuid, fs_info->super_copy->fsid, BTRFS_FSID_SIZE); + else + memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE); mutex_lock(&fs_info->chunk_mutex); ret = btrfs_read_sys_array(fs_info); @@ -3591,6 +3679,13 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device } } + ret = btrfs_init_writeback_bio_size(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to get optimum writeback size: %d", + ret); + goto fail_sysfs; + } + btrfs_free_zone_cache(fs_info); btrfs_check_active_zone_reservation(fs_info); @@ -3706,7 +3801,7 @@ fail_tree_roots: if (fs_info->data_reloc_root) btrfs_drop_and_free_fs_root(fs_info, fs_info->data_reloc_root); free_root_pointers(fs_info, true); - invalidate_inode_pages2(fs_info->btree_inode->i_mapping); + invalidate_and_check_btree_folios(fs_info); fail_sb_buffer: btrfs_stop_all_workers(fs_info); @@ -4209,7 +4304,6 @@ static void warn_about_uncommitted_trans(struct btrfs_fs_info *fs_info) list_del_init(&trans->list); btrfs_put_transaction(trans); - trace_btrfs_transaction_commit(fs_info); } ASSERT(!found); } @@ -4280,16 +4374,6 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) btrfs_error_commit_super(fs_info); /* - * Wait for any fixup workers to complete. - * If we don't wait for them here and they are still running by the time - * we call kthread_stop() against the cleaner kthread further below, we - * get an use-after-free on the cleaner because the fixup worker adds an - * inode to the list of delayed iputs and then attempts to wakeup the - * cleaner kthread, which was already stopped and destroyed. We parked - * already the cleaner, but below we run all pending delayed iputs. - */ - btrfs_flush_workqueue(fs_info->fixup_workers); - /* * Similar case here, we have to wait for delalloc workers before we * proceed below and stop the cleaner kthread, otherwise we trigger a * use-after-tree on the cleaner kthread task_struct when a delalloc @@ -4412,7 +4496,7 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) ASSERT(list_empty(&fs_info->delayed_iputs)); set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags); - if (btrfs_check_quota_leak(fs_info)) { + if (unlikely(btrfs_check_quota_leak(fs_info))) { DEBUG_WARN("qgroup reserved space leaked"); btrfs_err(fs_info, "qgroup reserved space leaked"); } @@ -4445,7 +4529,7 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) * We must make sure there is not any read request to * submit after we stop all workers. */ - invalidate_inode_pages2(fs_info->btree_inode->i_mapping); + invalidate_and_check_btree_folios(fs_info); btrfs_stop_all_workers(fs_info); /* @@ -4686,6 +4770,7 @@ static void btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info, free_extent_buffer_stale(eb); } } + btrfs_extent_io_tree_release(dirty_pages); } static void btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info, @@ -4880,7 +4965,6 @@ static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info) spin_unlock(&fs_info->trans_lock); btrfs_put_transaction(t); - trace_btrfs_transaction_commit(fs_info); spin_lock(&fs_info->trans_lock); } spin_unlock(&fs_info->trans_lock); diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c index 626702244809..c18ea5ef2974 100644 --- a/fs/btrfs/extent-io-tree.c +++ b/fs/btrfs/extent-io-tree.c @@ -1763,7 +1763,7 @@ u64 btrfs_count_range_bits(struct extent_io_tree *tree, u64 cur_start = *start; u64 total_bytes = 0; u64 last = 0; - int found = 0; + bool found = false; if (WARN_ON(search_end < cur_start)) return 0; @@ -1817,7 +1817,7 @@ search: break; if (!found) { *start = max(cur_start, state->start); - found = 1; + found = true; } last = state->end; } else if (contig && found) { diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 391fad41c3b6..624d76e0ca01 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -385,7 +385,7 @@ int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb, return type; } } else { - ASSERT(is_data == BTRFS_REF_TYPE_ANY); + ASSERT(is_data == BTRFS_REF_TYPE_ANY, "is_data=%d", is_data); return type; } } @@ -1699,13 +1699,13 @@ static int run_delayed_extent_op(struct btrfs_trans_handle *trans, struct extent_buffer *leaf; u32 item_size; int ret; - int metadata = 1; + bool metadata = true; if (TRANS_ABORTED(trans)) return 0; if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA)) - metadata = 0; + metadata = false; path = btrfs_alloc_path(); if (!path) @@ -1745,7 +1745,7 @@ again: } if (ret > 0) { btrfs_release_path(path); - metadata = 0; + metadata = false; key.objectid = head->bytenr; key.type = BTRFS_EXTENT_ITEM_KEY; @@ -2108,7 +2108,8 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, locked_ref = btrfs_select_ref_head(fs_info, delayed_refs); if (IS_ERR_OR_NULL(locked_ref)) { if (PTR_ERR(locked_ref) == -EAGAIN) { - continue; + count++; + goto again; } else { break; } @@ -2156,7 +2157,7 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, * Either success case or btrfs_run_delayed_refs_for_head * returned -EAGAIN, meaning we need to select another head */ - +again: locked_ref = NULL; cond_resched(); } while ((min_bytes != U64_MAX && bytes_processed < min_bytes) || @@ -2531,8 +2532,11 @@ int btrfs_cross_ref_exist(struct btrfs_inode *inode, u64 offset, struct btrfs_key key; btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); - ASSERT(key.objectid == bytenr); - ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY); + ASSERT(key.objectid == bytenr, + "key.objectid=%llu bytenr=%llu", + key.objectid, bytenr); + ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY, "key.type=%u", + key.type); } } @@ -3279,7 +3283,7 @@ static int __btrfs_free_extent(struct btrfs_trans_handle *trans, int ret; int is_data; int extent_slot = 0; - int found_extent = 0; + bool found_extent = false; int num_to_del = 1; int refs_to_drop = node->ref_mod; u32 item_size; @@ -3335,12 +3339,12 @@ static int __btrfs_free_extent(struct btrfs_trans_handle *trans, break; if (key.type == BTRFS_EXTENT_ITEM_KEY && key.offset == num_bytes) { - found_extent = 1; + found_extent = true; break; } if (key.type == BTRFS_METADATA_ITEM_KEY && key.offset == owner_objectid) { - found_extent = 1; + found_extent = true; break; } @@ -4598,10 +4602,12 @@ static noinline int find_free_extent(struct btrfs_root *root, /* Use dedicated sub-space_info for dedicated block group users. */ if (ffe_ctl->for_data_reloc) { space_info = space_info->sub_group[0]; - ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC); + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC, + "space_info->subgroup_id=%d", space_info->subgroup_id); } else if (ffe_ctl->for_treelog) { space_info = space_info->sub_group[0]; - ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_TREELOG); + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_TREELOG, + "space_info->subgroup_id=%d", space_info->subgroup_id); } } if (!space_info) { @@ -5781,16 +5787,21 @@ static int check_next_block_uptodate(struct btrfs_trans_handle *trans, generation = btrfs_node_ptr_generation(path->nodes[level], path->slots[level]); - if (btrfs_buffer_uptodate(next, generation, NULL)) - return 0; - check.level = level - 1; check.transid = generation; check.owner_root = btrfs_root_id(root); check.has_first_key = true; btrfs_node_key_to_cpu(path->nodes[level], &check.first_key, path->slots[level]); + ret = btrfs_buffer_uptodate(next, generation, &check); + if (ret > 0) + return 0; btrfs_tree_unlock(next); + if (ret < 0) { + free_extent_buffer(next); + return ret; + } + if (level == 1) reada_walk_down(trans, root, wc, path); ret = btrfs_read_extent_buffer(next, &check); @@ -6613,12 +6624,16 @@ static int btrfs_trim_free_extents_throttle(struct btrfs_device *device, *trimmed = 0; - /* Discard not supported = nothing to do. */ - if (!bdev_max_discard_sectors(device->bdev)) + /* + * The caller only filters out MISSING devices, but a device that was + * missing at mount and later rescanned has MISSING cleared while bdev + * is still NULL and WRITEABLE is still unset. Skip those here. + */ + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) || !device->bdev) return 0; - /* Not writable = nothing to do. */ - if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) + /* Discard not supported = nothing to do. */ + if (!bdev_max_discard_sectors(device->bdev)) return 0; /* No free space = nothing to do. */ @@ -6645,7 +6660,7 @@ static int btrfs_trim_free_extents_throttle(struct btrfs_device *device, start = max(start, cur_start); /* Check if there are any CHUNK_* bits left */ - if (start > device->total_bytes) { + if (unlikely(start > device->total_bytes)) { DEBUG_WARN(); btrfs_warn(fs_info, "ignoring attempt to trim beyond device size: offset %llu length %llu device %s device size %llu", diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 2275189b7860..7d604524e83c 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -130,7 +130,8 @@ struct btrfs_bio_ctrl { * extent_writepage_io(). * This is to avoid touching ranges covered by compression/inline. */ - unsigned long submit_bitmap; + unsigned long submit_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)]; + struct readahead_control *ractl; /* @@ -250,8 +251,6 @@ static void process_one_folio(struct btrfs_fs_info *fs_info, ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX); len = end + 1 - start; - if (page_ops & PAGE_SET_ORDERED) - btrfs_folio_clamp_set_ordered(fs_info, folio, start, len); if (page_ops & PAGE_START_WRITEBACK) { btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len); btrfs_folio_clamp_set_writeback(fs_info, folio, start, len); @@ -381,7 +380,7 @@ noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, bool found; struct extent_state *cached_state = NULL; int ret; - int loops = 0; + bool loops = false; /* Caller should pass a valid @end to indicate the search range end */ ASSERT(orig_end > orig_start); @@ -438,7 +437,7 @@ again: cached_state = NULL; if (!loops) { max_bytes = fs_info->sectorsize; - loops = 1; + loops = true; goto again; } else { return false; @@ -530,8 +529,6 @@ static void end_bbio_data_write(struct btrfs_bio *bbio) u32 len = fi.length; bio_size += len; - ASSERT(btrfs_folio_test_ordered(fs_info, folio, start, len)); - btrfs_folio_clear_ordered(fs_info, folio, start, len); btrfs_folio_clear_writeback(fs_info, folio, start, len); } @@ -620,24 +617,24 @@ static void end_bbio_data_read(struct btrfs_bio *bbio) } /* - * Populate every free slot in a provided array with folios using GFP_NOFS. + * Populate every free slot in a provided array with folios. * - * @nr_folios: number of folios to allocate - * @order: the order of the folios to be allocated - * @folio_array: the array to fill with folios; any existing non-NULL entries in - * the array will be skipped + * @nr_folios: number of folios to allocate + * @order: folio order + * @folio_array: array to fill with folios; non-NULL entries are skipped + * @gfp: GFP flags for the allocation * * Return: 0 if all folios were able to be allocated; * -ENOMEM otherwise, the partially allocated folios would be freed and * the array slots zeroed */ int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order, - struct folio **folio_array) + struct folio **folio_array, gfp_t gfp) { for (int i = 0; i < nr_folios; i++) { if (folio_array[i]) continue; - folio_array[i] = folio_alloc(GFP_NOFS, order); + folio_array[i] = folio_alloc(gfp, order); if (!folio_array[i]) goto error; } @@ -652,21 +649,18 @@ error: } /* - * Populate every free slot in a provided array with pages, using GFP_NOFS. + * Populate every free slot in a provided array with pages. * - * @nr_pages: number of pages to allocate - * @page_array: the array to fill with pages; any existing non-null entries in - * the array will be skipped - * @nofail: whether using __GFP_NOFAIL flag + * @nr_pages: number of pages to allocate + * @page_array: array to fill; non-NULL entries are skipped + * @gfp: GFP flags for the allocation * * Return: 0 if all pages were able to be allocated; * -ENOMEM otherwise, the partially allocated pages would be freed and * the array slots zeroed */ -int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, - bool nofail) +int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, gfp_t gfp) { - const gfp_t gfp = nofail ? (GFP_NOFS | __GFP_NOFAIL) : GFP_NOFS; unsigned int allocated; for (allocated = 0; allocated < nr_pages;) { @@ -690,13 +684,13 @@ int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, * * For now, the folios populated are always in order 0 (aka, single page). */ -static int alloc_eb_folio_array(struct extent_buffer *eb, bool nofail) +static int alloc_eb_folio_array(struct extent_buffer *eb, gfp_t gfp) { struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 }; int num_pages = num_extent_pages(eb); int ret; - ret = btrfs_alloc_page_array(num_pages, page_array, nofail); + ret = btrfs_alloc_page_array(num_pages, page_array, gfp); if (ret < 0) return ret; @@ -729,9 +723,9 @@ static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl, bio_end_sector(bio) == sector; } -static void alloc_new_bio(struct btrfs_inode *inode, - struct btrfs_bio_ctrl *bio_ctrl, - u64 disk_bytenr, u64 file_offset) +static int alloc_new_bio(struct btrfs_inode *inode, + struct btrfs_bio_ctrl *bio_ctrl, + u64 disk_bytenr, u64 file_offset) { struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_bio *bbio; @@ -748,13 +742,25 @@ static void alloc_new_bio(struct btrfs_inode *inode, if (bio_ctrl->wbc) { struct btrfs_ordered_extent *ordered; + /* This must be a write for data inodes. */ + ASSERT(btrfs_op(&bio_ctrl->bbio->bio) == BTRFS_MAP_WRITE); + ASSERT(is_data_inode(inode)); + ordered = btrfs_lookup_ordered_extent(inode, file_offset); - if (ordered) { - bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, - ordered->file_offset + - ordered->disk_num_bytes - file_offset); - bbio->ordered = ordered; + if (unlikely(!ordered)) { + bio_ctrl->bbio = NULL; + bio_ctrl->next_file_offset = 0; + bio_put(&bbio->bio); + btrfs_err_rl(fs_info, + "root %lld ino %llu file offset %llu is marked dirty without notifying the fs", + btrfs_root_id(inode->root), btrfs_ino(inode), + file_offset); + return -EUCLEAN; } + bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, + ordered->file_offset + + ordered->disk_num_bytes - file_offset); + bbio->ordered = ordered; /* * Pick the last added device to support cgroup writeback. For @@ -765,6 +771,7 @@ static void alloc_new_bio(struct btrfs_inode *inode, bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev); wbc_init_bio(bio_ctrl->wbc, &bbio->bio); } + return 0; } /* @@ -780,14 +787,19 @@ static void alloc_new_bio(struct btrfs_inode *inode, * new one in @bio_ctrl->bbio. * The mirror number for this IO should already be initialized in * @bio_ctrl->mirror_num. + * + * Return the number of bytes that are queued into a bio. + * If the returned bytes is smaller than @size, it means we hit a critical error + * for data write, where there is no ordered extent for the range. */ -static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, - u64 disk_bytenr, struct folio *folio, - size_t size, unsigned long pg_offset, - u64 read_em_generation) +static unsigned int submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, + u64 disk_bytenr, struct folio *folio, + size_t size, unsigned long pg_offset, + u64 read_em_generation) { struct btrfs_inode *inode = folio_to_inode(folio); loff_t file_offset = folio_pos(folio) + pg_offset; + unsigned int queued = 0; ASSERT(pg_offset + size <= folio_size(folio)); ASSERT(bio_ctrl->end_io_func); @@ -800,8 +812,13 @@ static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, u32 len = size; /* Allocate new bio if needed */ - if (!bio_ctrl->bbio) - alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset); + if (!bio_ctrl->bbio) { + int ret; + + ret = alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset); + if (ret < 0) + break; + } /* Cap to the current ordered extent boundary if there is one. */ if (len > bio_ctrl->len_to_oe_boundary) { @@ -829,6 +846,7 @@ static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, pg_offset += len; disk_bytenr += len; file_offset += len; + queued += len; /* * len_to_oe_boundary defaults to U32_MAX, which isn't folio or @@ -857,7 +875,18 @@ static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, /* Ordered extent boundary: move on to a new bio. */ if (bio_ctrl->len_to_oe_boundary == 0) submit_one_bio(bio_ctrl); + /* + * If we have accumulated decent amount of IO, send it to the + * block layer so that IO can run while we are accumulating + * more folios to write. + */ + else if (bio_ctrl->wbc && + bio_ctrl->bbio->bio.bi_iter.bi_size >= + inode->root->fs_info->writeback_bio_size) + submit_one_bio(bio_ctrl); + } while (size); + return queued; } static int attach_extent_buffer_folio(struct extent_buffer *eb, @@ -922,6 +951,17 @@ void clear_folio_extent_mapped(struct folio *folio) struct btrfs_fs_info *fs_info; ASSERT(folio->mapping); + /* + * The folio should not have writeback nor dirty flag set. + * + * If dirty flag is set, the folio can be written back again and we + * expect the private flag set for the folio. + * + * If writeback flag is set, the endio may need to utilize the + * private for btrfs_folio_state. + */ + ASSERT(!folio_test_dirty(folio)); + ASSERT(!folio_test_writeback(folio)); if (!folio_test_private(folio)) return; @@ -1030,6 +1070,7 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, u64 disk_bytenr; u64 block_start; u64 em_gen; + unsigned int queued; ASSERT(IS_ALIGNED(cur, fs_info->sectorsize)); if (cur >= last_byte) { @@ -1143,8 +1184,10 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, if (force_bio_submit) submit_one_bio(bio_ctrl); - submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize, - pg_offset, em_gen); + queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize, + pg_offset, em_gen); + /* Read submission should not fail. */ + ASSERT(queued == blocksize); } return 0; } @@ -1422,7 +1465,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, const u64 page_start = folio_pos(folio); const u64 page_end = page_start + folio_size(folio) - 1; const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); - unsigned long delalloc_bitmap = 0; + unsigned long delalloc_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)] = { 0 }; /* * Save the last found delalloc end. As the delalloc end can go beyond * page boundary, thus we cannot rely on subpage bitmap to locate the @@ -1447,14 +1490,9 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, int ret = 0; /* Save the dirty bitmap as our submission bitmap will be a subset of it. */ - if (btrfs_is_subpage(fs_info, folio)) { - ASSERT(blocks_per_folio > 1); - btrfs_get_subpage_dirty_bitmap(fs_info, folio, &bio_ctrl->submit_bitmap); - } else { - bio_ctrl->submit_bitmap = 1; - } + btrfs_copy_subpage_dirty_bitmap(fs_info, folio, bio_ctrl->submit_bitmap); - for_each_set_bitrange(start_bit, end_bit, &bio_ctrl->submit_bitmap, + for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap, blocks_per_folio) { u64 start = page_start + (start_bit << fs_info->sectorsize_bits); u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits; @@ -1470,7 +1508,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, delalloc_start = delalloc_end + 1; continue; } - set_delalloc_bitmap(folio, &delalloc_bitmap, delalloc_start, + set_delalloc_bitmap(folio, delalloc_bitmap, delalloc_start, min(delalloc_end, page_end) + 1 - delalloc_start); last_delalloc_end = delalloc_end; delalloc_start = delalloc_end + 1; @@ -1496,7 +1534,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, found_len = last_delalloc_end + 1 - found_start; found = true; } else { - found = find_next_delalloc_bitmap(folio, &delalloc_bitmap, + found = find_next_delalloc_bitmap(folio, delalloc_bitmap, delalloc_start, &found_start, &found_len); } if (!found) @@ -1530,7 +1568,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, btrfs_ino(inode), folio_pos(folio), blocks_per_folio, - &bio_ctrl->submit_bitmap, + bio_ctrl->submit_bitmap, found_start, found_len, ret); } else { /* @@ -1555,7 +1593,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, fs_info->sectorsize_bits; unsigned int end_bit = (min(page_end + 1, found_start + found_len) - page_start) >> fs_info->sectorsize_bits; - bitmap_clear(&bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit); + bitmap_clear(bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit); } /* * Above btrfs_run_delalloc_range() may have unlocked the folio, @@ -1576,12 +1614,11 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, fs_info->sectorsize_bits, blocks_per_folio); - for_each_set_bitrange(start_bit, end_bit, &bio_ctrl->submit_bitmap, + for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap, bitmap_size) { u64 start = page_start + (start_bit << fs_info->sectorsize_bits); u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits; - btrfs_folio_clear_ordered(fs_info, folio, start, len); btrfs_mark_ordered_io_finished(inode, start, len, false); } return ret; @@ -1602,7 +1639,7 @@ out: * If all ranges are submitted asynchronously, we just need to account * for them here. */ - if (bitmap_empty(&bio_ctrl->submit_bitmap, blocks_per_folio)) { + if (bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio)) { wbc->nr_to_write -= delalloc_to_write; return 1; } @@ -1637,6 +1674,7 @@ static int submit_one_sector(struct btrfs_inode *inode, u64 extent_offset; u64 em_end; const u32 sectorsize = fs_info->sectorsize; + unsigned int queued; ASSERT(IS_ALIGNED(filepos, sectorsize)); @@ -1658,7 +1696,6 @@ static int submit_one_sector(struct btrfs_inode *inode, * ordered extent. */ btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize); - btrfs_folio_clear_ordered(fs_info, folio, filepos, sectorsize); btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize); btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize); @@ -1703,8 +1740,14 @@ static int submit_one_sector(struct btrfs_inode *inode, */ ASSERT(folio_test_writeback(folio)); - submit_extent_folio(bio_ctrl, disk_bytenr, folio, - sectorsize, filepos - folio_pos(folio), 0); + queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio, + sectorsize, filepos - folio_pos(folio), 0); + if (unlikely(queued < sectorsize)) { + btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize); + btrfs_mark_ordered_io_finished(inode, filepos, fs_info->sectorsize, + false); + return -EUCLEAN; + } return 0; } @@ -1723,7 +1766,6 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, loff_t i_size) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - unsigned long range_bitmap = 0; bool submitted_io = false; int found_error = 0; const u64 end = start + len; @@ -1738,28 +1780,18 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, ASSERT(end <= folio_end, "start=%llu len=%u folio_start=%llu folio_size=%zu", start, len, folio_start, folio_size(folio)); - ret = btrfs_writepage_cow_fixup(folio); - if (ret == -EAGAIN) { - /* Fixup worker will requeue */ - folio_redirty_for_writepage(bio_ctrl->wbc, folio); - folio_unlock(folio); - return 1; - } - if (ret < 0) { - btrfs_folio_clear_dirty(fs_info, folio, start, len); - btrfs_folio_set_writeback(fs_info, folio, start, len); - btrfs_folio_clear_writeback(fs_info, folio, start, len); - return ret; - } - - bitmap_set(&range_bitmap, (start - folio_pos(folio)) >> fs_info->sectorsize_bits, - len >> fs_info->sectorsize_bits); - bitmap_and(&bio_ctrl->submit_bitmap, &bio_ctrl->submit_bitmap, &range_bitmap, - blocks_per_folio); + /* Truncate the submit bitmap to the current range. */ + if (start > folio_start) + bitmap_clear(bio_ctrl->submit_bitmap, 0, + (start - folio_start) >> fs_info->sectorsize_bits); + if (start + len < folio_end) + bitmap_clear(bio_ctrl->submit_bitmap, + (end - folio_start) >> fs_info->sectorsize_bits, + (folio_end - end) >> fs_info->sectorsize_bits); bio_ctrl->end_io_func = end_bbio_data_write; - for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) { + for_each_set_bit(bit, bio_ctrl->submit_bitmap, blocks_per_folio) { cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits); if (cur >= i_size) { @@ -1779,7 +1811,6 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, spin_unlock(&inode->ordered_tree_lock); btrfs_put_ordered_extent(ordered); - btrfs_folio_clear_ordered(fs_info, folio, cur, fs_info->sectorsize); btrfs_mark_ordered_io_finished(inode, cur, fs_info->sectorsize, true); /* * This range is beyond i_size, thus we don't need to @@ -1819,6 +1850,25 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, return found_error; } +static void bio_ctrl_init_submit_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + struct btrfs_bio_ctrl *bio_ctrl) +{ + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + + ASSERT(blocks_per_folio <= BTRFS_MAX_BLOCKS_PER_FOLIO); + + /* + * Default to unlock the whole folio. + * The proper bitmap is not initialized until writepage_delalloc(). + * + * We're safe just to set the bitmap range [0, blocks_per_folio), as + * all later usage of the bitmap will follow the same range limit. + * Any bits beyond blocks_per_folio will be ignored. + */ + bitmap_set(bio_ctrl->submit_bitmap, 0, blocks_per_folio); +} + /* * the writepage semantics are similar to regular writepage. extent * records are inserted to lock ranges in the tree, and as dirty areas @@ -1853,12 +1903,7 @@ static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl if (folio_contains(folio, end_index)) folio_zero_range(folio, pg_offset, folio_size(folio) - pg_offset); - /* - * Default to unlock the whole folio. - * The proper bitmap can only be initialized until writepage_delalloc(). - */ - bio_ctrl->submit_bitmap = (unsigned long)-1; - + bio_ctrl_init_submit_bitmap(fs_info, folio, bio_ctrl); /* * If the page is dirty but without private set, it's marked dirty * without informing the fs. @@ -1867,13 +1912,9 @@ static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl * * So here we check if the page has private set to rule out such * case. - * But we also have a long history of relying on the COW fixup, - * so here we only enable this check for experimental builds until - * we're sure it's safe. */ - if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL) && - unlikely(!folio_test_private(folio))) { - WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG)); + if (unlikely(!folio_test_private(folio))) { + DEBUG_WARN(); btrfs_err_rl(fs_info, "root %lld ino %llu folio %llu is marked dirty without notifying the fs", btrfs_root_id(inode->root), @@ -1901,7 +1942,7 @@ static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl "failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %d", btrfs_root_id(inode->root), btrfs_ino(inode), folio_pos(folio), blocks_per_folio, - &bio_ctrl->submit_bitmap, ret); + bio_ctrl->submit_bitmap, ret); bio_ctrl->wbc->nr_to_write--; @@ -2318,13 +2359,13 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb struct btrfs_eb_write_context ctx = { .wbc = wbc }; struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); int ret = 0; - int done = 0; + bool done = false; int nr_to_write_done = 0; struct eb_batch batch; unsigned int nr_ebs; unsigned long index; unsigned long end; - int scanned = 0; + bool scanned = false; xa_mark_t tag; eb_batch_init(&batch); @@ -2341,7 +2382,7 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb index = (wbc->range_start >> fs_info->nodesize_bits); end = (wbc->range_end >> fs_info->nodesize_bits); - scanned = 1; + scanned = true; } if (wbc->sync_mode == WB_SYNC_ALL) tag = PAGECACHE_TAG_TOWRITE; @@ -2364,7 +2405,7 @@ retry: ret = 0; if (ret) { - done = 1; + done = true; break; } continue; @@ -2390,7 +2431,7 @@ retry: * We hit the last page and there is more work to be done: wrap * back to the start of the file */ - scanned = 1; + scanned = true; index = 0; goto retry; } @@ -2430,15 +2471,15 @@ static int extent_write_cache_pages(struct address_space *mapping, struct writeback_control *wbc = bio_ctrl->wbc; struct inode *inode = mapping->host; int ret = 0; - int done = 0; + bool done = false; int nr_to_write_done = 0; struct folio_batch fbatch; unsigned int nr_folios; pgoff_t index; pgoff_t end; /* Inclusive */ pgoff_t done_index; - int range_whole = 0; - int scanned = 0; + bool range_whole = false; + bool scanned = false; xa_mark_t tag; /* @@ -2466,8 +2507,8 @@ static int extent_write_cache_pages(struct address_space *mapping, index = wbc->range_start >> PAGE_SHIFT; end = wbc->range_end >> PAGE_SHIFT; if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) - range_whole = 1; - scanned = 1; + range_whole = true; + scanned = true; } /* @@ -2544,14 +2585,14 @@ retry: } if (folio_test_writeback(folio) || - !folio_clear_dirty_for_io(folio)) { + !folio_test_dirty(folio)) { folio_unlock(folio); continue; } ret = extent_writepage(folio, bio_ctrl); if (ret < 0) { - done = 1; + done = true; break; } @@ -2571,7 +2612,7 @@ retry: * We hit the last page and there is more work to be done: wrap * back to the start of the file */ - scanned = 1; + scanned = true; index = 0; /* @@ -2648,7 +2689,7 @@ void extent_write_locked_range(struct inode *inode, const struct folio *locked_f * Set the submission bitmap to submit all sectors. * extent_writepage_io() will do the truncation correctly. */ - bio_ctrl.submit_bitmap = (unsigned long)-1; + bio_ctrl_init_submit_bitmap(fs_info, folio, &bio_ctrl); ret = extent_writepage_io(BTRFS_I(inode), folio, cur, cur_len, &bio_ctrl, i_size); if (ret == 1) @@ -2717,38 +2758,6 @@ void btrfs_readahead(struct readahead_control *rac) } /* - * basic invalidate_folio code, this waits on any locked or writeback - * ranges corresponding to the folio, and then deletes any extent state - * records from the tree - */ -int extent_invalidate_folio(struct extent_io_tree *tree, - struct folio *folio, size_t offset) -{ - struct extent_state *cached_state = NULL; - u64 start = folio_pos(folio); - u64 end = start + folio_size(folio) - 1; - size_t blocksize = folio_to_fs_info(folio)->sectorsize; - - /* This function is only called for the btree inode */ - ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO); - - start += ALIGN(offset, blocksize); - if (start > end) - return 0; - - btrfs_lock_extent(tree, start, end, &cached_state); - folio_wait_writeback(folio); - - /* - * Currently for btree io tree, only EXTENT_LOCKED is utilized, - * so here we only need to unlock the extent range to free any - * existing extent state. - */ - btrfs_unlock_extent(tree, start, end, &cached_state); - return 0; -} - -/* * A helper for struct address_space_operations::release_folio, this tests for * areas of the folio that are locked or under IO and drops the related state * bits if it is safe to drop the folio. @@ -2877,12 +2886,6 @@ next: return try_release_extent_state(io_tree, folio); } -static int extent_buffer_under_io(const struct extent_buffer *eb) -{ - return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || - test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); -} - static bool folio_range_has_eb(struct folio *folio) { struct btrfs_folio_state *bfs; @@ -3097,7 +3100,7 @@ struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) */ set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags); - ret = alloc_eb_folio_array(new, false); + ret = alloc_eb_folio_array(new, GFP_NOFS); if (ret) goto release_eb; @@ -3138,7 +3141,7 @@ struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, if (!eb) return NULL; - ret = alloc_eb_folio_array(eb, false); + ret = alloc_eb_folio_array(eb, GFP_NOFS); if (ret) goto release_eb; @@ -3394,8 +3397,8 @@ retry: finish: spin_lock(&mapping->i_private_lock); if (existing_folio && btrfs_meta_is_subpage(fs_info)) { - /* We're going to reuse the existing page, can drop our folio now. */ - __free_page(folio_page(eb->folios[i], 0)); + /* We're going to reuse the existing folio, can drop our folio now. */ + folio_put(eb->folios[i]); eb->folios[i] = existing_folio; } else if (existing_folio) { struct extent_buffer *existing_eb; @@ -3410,7 +3413,7 @@ finish: return 1; } /* The extent buffer no longer exists, we can reuse the folio. */ - __free_page(folio_page(eb->folios[i], 0)); + folio_put(eb->folios[i]); eb->folios[i] = existing_folio; } eb->folio_size = folio_size(eb->folios[i]); @@ -3441,7 +3444,7 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, struct btrfs_folio_state *prealloc = NULL; u64 lockdep_owner = owner_root; bool page_contig = true; - int uptodate = 1; + bool uptodate = true; int ret; if (check_eb_alignment(fs_info, start)) @@ -3491,8 +3494,12 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, } reallocate: - /* Allocate all pages first. */ - ret = alloc_eb_folio_array(eb, true); + /* + * Allocate all pages first. These will be attached to btree_inode->i_mapping + * below (added to LRU, served by btree_migrate_folio), so request + * __GFP_MOVABLE so the page allocator places them in MOVABLE pageblocks. + */ + ret = alloc_eb_folio_array(eb, GFP_NOFS | __GFP_NOFAIL | __GFP_MOVABLE); if (ret < 0) { btrfs_free_folio_state(prealloc); goto out; @@ -3551,7 +3558,7 @@ reallocate: page_contig = false; if (!btrfs_meta_folio_test_uptodate(folio, eb)) - uptodate = 0; + uptodate = false; /* * We can't unlock the pages just yet since the extent buffer @@ -3739,17 +3746,6 @@ void free_extent_buffer_stale(struct extent_buffer *eb) release_extent_buffer(eb); } -static void btree_clear_folio_dirty_tag(struct folio *folio) -{ - ASSERT(!folio_test_dirty(folio)); - ASSERT(folio_test_locked(folio)); - xa_lock_irq(&folio->mapping->i_pages); - if (!folio_test_dirty(folio)) - __xa_clear_mark(&folio->mapping->i_pages, folio->index, - PAGECACHE_TAG_DIRTY); - xa_unlock_irq(&folio->mapping->i_pages); -} - void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, struct extent_buffer *eb) { @@ -3790,7 +3786,7 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, folio_lock(folio); last = btrfs_meta_folio_clear_and_test_dirty(folio, eb); if (last) - btree_clear_folio_dirty_tag(folio); + btrfs_clear_folio_dirty_tag(folio); folio_unlock(folio); } WARN_ON(refcount_read(&eb->refs) == 0); @@ -3981,15 +3977,14 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num, return 0; } -static bool report_eb_range(const struct extent_buffer *eb, unsigned long start, - unsigned long len) +/* Never inlined to decrease code size, as this is called in a cold path. */ +static noinline void report_eb_range(const struct extent_buffer *eb, + unsigned long start, unsigned long len) { btrfs_warn(eb->fs_info, "access to eb bytenr %llu len %u out of range start %lu len %lu", eb->start, eb->len, start, len); DEBUG_WARN(); - - return true; } /* @@ -3999,14 +3994,16 @@ static bool report_eb_range(const struct extent_buffer *eb, unsigned long start, * * Caller should not touch the dst/src memory if this function returns error. */ -static inline int check_eb_range(const struct extent_buffer *eb, - unsigned long start, unsigned long len) +static inline bool check_eb_range(const struct extent_buffer *eb, + unsigned long start, unsigned long len) { unsigned long offset; /* start, start + len should not go beyond eb->len nor overflow */ - if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len)) - return report_eb_range(eb, start, len); + if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len)) { + report_eb_range(eb, start, len); + return true; + } return false; } @@ -4660,7 +4657,7 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, if (IS_ERR(eb)) return; - if (btrfs_buffer_uptodate(eb, gen, NULL)) { + if (btrfs_buffer_uptodate(eb, gen, &check)) { free_extent_buffer(eb); return; } diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index b310a5145cf6..9896e15ddc40 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -55,7 +55,6 @@ enum { /* Page starts writeback, clear dirty bit and set writeback bit */ ENUM_BIT(PAGE_START_WRITEBACK), ENUM_BIT(PAGE_END_WRITEBACK), - ENUM_BIT(PAGE_SET_ORDERED), }; /* @@ -327,6 +326,12 @@ static inline bool extent_buffer_uptodate(const struct extent_buffer *eb) return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); } +static inline bool extent_buffer_under_io(const struct extent_buffer *eb) +{ + return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || + test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); +} + int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, unsigned long start, unsigned long len); void read_extent_buffer(const struct extent_buffer *eb, void *dst, @@ -381,15 +386,23 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, const struct folio *locked_folio, struct extent_state **cached, u32 bits_to_clear, unsigned long page_ops); -int extent_invalidate_folio(struct extent_io_tree *tree, - struct folio *folio, size_t offset); void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, struct extent_buffer *buf); -int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, - bool nofail); +static inline void btrfs_clear_folio_dirty_tag(struct folio *folio) +{ + ASSERT(!folio_test_dirty(folio)); + ASSERT(folio_test_locked(folio)); + ASSERT(folio->mapping); + xa_lock_irq(&folio->mapping->i_pages); + __xa_clear_mark(&folio->mapping->i_pages, folio->index, + PAGECACHE_TAG_DIRTY); + xa_unlock_irq(&folio->mapping->i_pages); +} + +int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, gfp_t gfp); int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order, - struct folio **folio_array); + struct folio **folio_array, gfp_t gfp); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS bool find_lock_delalloc_range(struct inode *inode, diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c index 9284c0a81bef..fce9c5cc0122 100644 --- a/fs/btrfs/extent_map.c +++ b/fs/btrfs/extent_map.c @@ -717,7 +717,7 @@ int btrfs_add_extent_mapping(struct btrfs_inode *inode, * file offset. Here just do a sanity check. */ if (em->disk_bytenr == EXTENT_MAP_INLINE) - ASSERT(em->start == 0); + ASSERT(em->start == 0, "em->start=%llu", em->start); ret = add_extent_mapping(inode, em, false); /* it is possible that someone inserted the extent into the tree @@ -761,7 +761,7 @@ int btrfs_add_extent_mapping(struct btrfs_inode *inode, } } - ASSERT(ret == 0 || ret == -EEXIST); + ASSERT(ret == 0 || ret == -EEXIST, "ret=%d", ret); return ret; } @@ -943,7 +943,7 @@ void btrfs_drop_extent_map_range(struct btrfs_inode *inode, u64 start, u64 end, ret = add_extent_mapping(inode, split, modified); /* Logic error, shouldn't happen. */ - ASSERT(ret == 0); + ASSERT(ret == 0, "ret=%d", ret); if (WARN_ON(ret != 0) && modified) btrfs_set_inode_full_sync(inode); } @@ -1246,7 +1246,9 @@ static struct btrfs_inode *find_first_inode_to_shrink(struct btrfs_root *root, write_unlock(&tree->lock); next: from = btrfs_ino(inode) + 1; - cond_resched_lock(&root->inodes.xa_lock); + xa_unlock(&root->inodes); + cond_resched(); + xa_lock(&root->inodes); } xa_unlock(&root->inodes); diff --git a/fs/btrfs/fiemap.c b/fs/btrfs/fiemap.c index 27d361c7adc4..6263e837093e 100644 --- a/fs/btrfs/fiemap.c +++ b/fs/btrfs/fiemap.c @@ -112,7 +112,7 @@ static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, u64 cache_end; /* Set at the end of extent_fiemap(). */ - ASSERT((flags & FIEMAP_EXTENT_LAST) == 0); + ASSERT((flags & FIEMAP_EXTENT_LAST) == 0, "flags=0x%u", flags); if (!cache->cached) goto assign; diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index d72249390030..cf50fd623f41 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -325,7 +325,9 @@ static int search_csum_tree(struct btrfs_fs_info *fs_info, csum_start = key.offset; csum_len = (itemsize / csum_size) * sectorsize; - ASSERT(in_range(disk_bytenr, csum_start, csum_len)); + ASSERT(in_range(disk_bytenr, csum_start, csum_len), + "disk_bytenr=%llu csum_start=%llu csum_len=%llu", + disk_bytenr, csum_start, csum_len); found: ret = (min(csum_start + csum_len, disk_bytenr + len) - @@ -356,6 +358,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits; int ret = 0; u32 bio_offset = 0; + bool using_commit_root = false; if ((inode->flags & BTRFS_INODE_NODATASUM) || test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state)) @@ -429,6 +432,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) * from across transactions. */ if (bbio->csum_search_commit_root) { + using_commit_root = true; path->search_commit_root = true; path->skip_locking = true; down_read(&fs_info->commit_root_sem); @@ -461,6 +465,28 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) * assume this is the case. */ if (count == 0) { + /* + * If an extent is relocated in the current transaction + * then relocation writes a new csum without updating + * the extent map generation. Until the next commit, we + * will see a hole in that case, so we need to fallback + * to searching the transaction csum root. + * + * Note that a commit root lookup of a referenced extent can + * only miss, not return a stale csum. A freed extent's csum + * is deleted in the same transaction and its bytenr is not + * reusable until that transaction has committed and the + * extent is unpinned. + */ + if (using_commit_root) { + up_read(&fs_info->commit_root_sem); + using_commit_root = false; + path->search_commit_root = false; + path->skip_locking = false; + btrfs_release_path(path); + continue; + } + memset(csum_dst, 0, csum_size); count = 1; @@ -479,7 +505,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) bio_offset += count * sectorsize; } - if (bbio->csum_search_commit_root) + if (using_commit_root) up_read(&fs_info->commit_root_sem); return ret; } @@ -1307,7 +1333,7 @@ found: index += ins_size; ins_size /= csum_size; - total_bytes += ins_size * fs_info->sectorsize; + total_bytes += (ins_size << fs_info->sectorsize_bits); if (total_bytes < sums->len) { btrfs_release_path(path); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index cf1cb5c4db75..a2a2df2df786 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -49,14 +49,6 @@ static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio, u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start; ASSERT(block_len <= U32_MAX); - /* - * Folio checked is some magic around finding folios that have been - * modified without going through btrfs_dirty_folio(). Clear it here. - * There should be no need to mark the pages accessed as - * prepare_one_folio() should have marked them accessed in - * prepare_one_folio() via find_or_create_page() - */ - btrfs_folio_clamp_clear_checked(fs_info, folio, block_start, block_len); folio_unlock(folio); folio_put(folio); } @@ -65,7 +57,7 @@ static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio, * After copy_folio_from_iter_atomic(), update the following things for delalloc: * - Mark newly dirtied folio as DELALLOC in the io tree. * Used to advise which range is to be written back. - * - Mark modified folio as Uptodate/Dirty and not needing COW fixup + * - Mark modified folio as Uptodate/Dirty * - Update inode size for past EOF write */ int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos, @@ -93,21 +85,12 @@ int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos end_of_last_block = start_pos + num_bytes - 1; - /* - * The pages may have already been dirty, clear out old accounting so - * we can set things up properly - */ - btrfs_clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - cached); - - ret = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block, - extra_bits, cached); + ret = btrfs_reset_extent_delalloc(inode, start_pos, end_of_last_block, + extra_bits, cached); if (ret) return ret; btrfs_folio_clamp_set_uptodate(fs_info, folio, start_pos, num_bytes); - btrfs_folio_clamp_clear_checked(fs_info, folio, start_pos, num_bytes); btrfs_folio_clamp_set_dirty(fs_info, folio, start_pos, num_bytes); /* @@ -158,7 +141,7 @@ int btrfs_drop_extents(struct btrfs_trans_handle *trans, int ret; int modify_tree = -1; int update_refs; - int found = 0; + bool found = false; struct btrfs_path *path = args->path; args->bytes_found = 0; @@ -266,7 +249,7 @@ next_slot: goto next_slot; } - found = 1; + found = true; search_start = max(key.offset, args->start); if (recow || !modify_tree) { modify_tree = -1; @@ -633,7 +616,7 @@ again: trans->transid); btrfs_set_file_extent_num_bytes(leaf, fi, end - other_start); - return 0; + goto mark_dirty; } } @@ -661,7 +644,7 @@ again: other_end - start); btrfs_set_file_extent_offset(leaf, fi, start - orig_offset); - return 0; + goto mark_dirty; } } @@ -788,7 +771,12 @@ again: } } - return 0; +mark_dirty: + ret = btrfs_inode_set_file_extent_range(inode, start, end - start); + if (ret) + btrfs_abort_transaction(trans, ret); + + return ret; } /* @@ -1227,8 +1215,11 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, return ret; reserved_len = ret; /* Write range must be inside the reserved range. */ - ASSERT(reserved_start <= start); - ASSERT(start + write_bytes <= reserved_start + reserved_len); + ASSERT(reserved_start <= start, "reserved_start=%llu start=%llu", + reserved_start, start); + ASSERT(start + write_bytes <= reserved_start + reserved_len, + "start=%llu write_bytes=%zu reserved_start=%llu reserved_len=%llu", + start, write_bytes, reserved_start, reserved_len); again: ret = balance_dirty_pages_ratelimited_flags(inode->vfs_inode.i_mapping, @@ -1573,7 +1564,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) btrfs_assert_inode_locked(inode); } - trace_btrfs_sync_file(file, datasync); + trace_btrfs_sync_file_enter(file, datasync); btrfs_init_log_ctx(&ctx, inode); @@ -1697,14 +1688,6 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) * reason, it's no longer relevant. */ clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); - /* - * An ordered extent might have started before and completed - * already with io errors, in which case the inode was not - * updated and we end up here. So check the inode's mapping - * for any errors that might have happened since we last - * checked called fsync. - */ - ret = filemap_check_wb_err(inode->vfs_inode.i_mapping, file->f_wb_err); goto out_release_extents; } @@ -1819,10 +1802,17 @@ out: free_extent_buffer(ctx.scratch_eb); ASSERT(list_empty(&ctx.list)); ASSERT(list_empty(&ctx.conflict_inodes)); + ASSERT(ret <= 0, "ret=%d", ret); + /* + * Ordered extents might have started and completed before this fsync, + * so check for any io errors and advance the writeback error sequence. + */ err = file_check_and_advance_wb_err(file); if (!ret) ret = err; - return ret > 0 ? -EIO : ret; + trace_btrfs_sync_file_exit(file, ret); + + return ret; out_release_extents: btrfs_release_log_ctx_extents(&ctx); @@ -1961,18 +1951,7 @@ again: } } - /* - * page_mkwrite gets called when the page is firstly dirtied after it's - * faulted in, but write(2) could also dirty a page and set delalloc - * bits, thus in this case for space account reason, we still need to - * clear any delalloc bits within this page range since we have to - * reserve data&meta space before lock_page() (see above comments). - */ - btrfs_clear_extent_bit(io_tree, page_start, end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | - EXTENT_DEFRAG, &cached_state); - - ret = btrfs_set_extent_delalloc(inode, page_start, end, 0, &cached_state); + ret = btrfs_reset_extent_delalloc(inode, page_start, end, 0, &cached_state); if (ret < 0) { btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state); goto out_unlock; @@ -1987,7 +1966,6 @@ again: if (zero_start != fsize) folio_zero_range(folio, zero_start, folio_size(folio) - zero_start); - btrfs_folio_clear_checked(fs_info, folio, page_start, fsize); btrfs_folio_set_dirty(fs_info, folio, page_start, end + 1 - page_start); btrfs_folio_set_uptodate(fs_info, folio, page_start, end + 1 - page_start); @@ -2093,6 +2071,10 @@ static int fill_holes(struct btrfs_trans_handle *trans, struct btrfs_file_extent_item *fi; struct extent_map *hole_em; struct btrfs_key key; + int modify_slot = -1; + int del_slot = -1; + bool update_offset = false; + u64 num_bytes = 0; int ret; if (btrfs_fs_incompat(fs_info, NO_HOLES)) @@ -2102,7 +2084,7 @@ static int fill_holes(struct btrfs_trans_handle *trans, key.type = BTRFS_EXTENT_DATA_KEY; key.offset = offset; - ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); if (ret <= 0) { /* * We should have dropped this offset, so if we find it then @@ -2115,33 +2097,44 @@ static int fill_holes(struct btrfs_trans_handle *trans, leaf = path->nodes[0]; if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) { - u64 num_bytes; - - path->slots[0]--; - fi = btrfs_item_ptr(leaf, path->slots[0], + fi = btrfs_item_ptr(leaf, path->slots[0] - 1, struct btrfs_file_extent_item); num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end - offset; - btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); - btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes); - btrfs_set_file_extent_offset(leaf, fi, 0); - btrfs_set_file_extent_generation(leaf, fi, trans->transid); - goto out; + modify_slot = path->slots[0] - 1; } - if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) { - u64 num_bytes; - - key.offset = offset; - btrfs_set_item_key_safe(trans, path, &key); fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); - num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end - - offset; + if (modify_slot != -1) { + num_bytes += btrfs_file_extent_num_bytes(leaf, fi); + del_slot = path->slots[0]; + } else { + num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + + end - offset; + modify_slot = path->slots[0]; + update_offset = true; + } + } + if (modify_slot >= 0) { + fi = btrfs_item_ptr(leaf, modify_slot, + struct btrfs_file_extent_item); btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes); + if (update_offset) { + key.offset = offset; + btrfs_set_item_key_safe(trans, path, &key); + } btrfs_set_file_extent_offset(leaf, fi, 0); btrfs_set_file_extent_generation(leaf, fi, trans->transid); + if (del_slot >= 0) { + ret = btrfs_del_items(trans, root, path, del_slot, 1); + if (ret) { + btrfs_abort_transaction(trans, ret); + btrfs_release_path(path); + return ret; + } + } goto out; } btrfs_release_path(path); @@ -2402,7 +2395,7 @@ int btrfs_replace_file_extents(struct btrfs_inode *inode, struct btrfs_drop_extents_args drop_args = { 0 }; struct btrfs_root *root = inode->root; struct btrfs_fs_info *fs_info = root->fs_info; - u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1); + const u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1); u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize); struct btrfs_trans_handle *trans = NULL; struct btrfs_block_rsv rsv; @@ -2415,7 +2408,7 @@ int btrfs_replace_file_extents(struct btrfs_inode *inode, return -EINVAL; btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP); - rsv.size = btrfs_calc_insert_metadata_size(fs_info, 1); + rsv.size = min_size; rsv.failfast = true; /* diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index ab22e4f9ffdd..e2af75a205ea 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -433,10 +433,6 @@ static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl) for (i = 0; i < io_ctl->num_pages; i++) { if (io_ctl->pages[i]) { - btrfs_folio_clear_checked(io_ctl->fs_info, - page_folio(io_ctl->pages[i]), - page_offset(io_ctl->pages[i]), - PAGE_SIZE); unlock_page(io_ctl->pages[i]); put_page(io_ctl->pages[i]); } @@ -555,6 +551,9 @@ static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index) u32 crc = ~(u32)0; unsigned offset = 0; + if (index >= io_ctl->num_pages) + return -EIO; + if (index == 0) offset = sizeof(u32) * io_ctl->num_pages; @@ -690,11 +689,12 @@ static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl, static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) { struct btrfs_block_group *block_group = ctl->block_group; + const int unit = block_group->fs_info->sectorsize; u64 max_bytes; u64 bitmap_bytes; u64 extent_bytes; u64 size = block_group->length; - u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; + u64 bytes_per_bg = BITS_PER_BITMAP * unit; u64 max_bitmaps = div64_u64(size + bytes_per_bg - 1, bytes_per_bg); max_bitmaps = max_t(u64, max_bitmaps, 1); @@ -703,7 +703,7 @@ static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) btrfs_err(block_group->fs_info, "invalid free space control: bg start=%llu len=%llu total_bitmaps=%u unit=%u max_bitmaps=%llu bytes_per_bg=%llu", block_group->start, block_group->length, - ctl->total_bitmaps, ctl->unit, max_bitmaps, + ctl->total_bitmaps, unit, max_bitmaps, bytes_per_bg); ASSERT(ctl->total_bitmaps <= max_bitmaps); @@ -718,7 +718,7 @@ static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) else max_bytes = MAX_CACHE_BYTES_PER_GIG * div_u64(size, SZ_1G); - bitmap_bytes = ctl->total_bitmaps * ctl->unit; + bitmap_bytes = ctl->total_bitmaps * unit; /* * we want the extent entry threshold to always be at most 1/2 the max @@ -896,8 +896,7 @@ free_cache: goto out; } -static int copy_free_space_cache(struct btrfs_block_group *block_group, - struct btrfs_free_space_ctl *ctl) +static int copy_free_space_cache(struct btrfs_free_space_ctl *ctl) { struct btrfs_free_space *info; struct rb_node *n; @@ -912,17 +911,17 @@ static int copy_free_space_cache(struct btrfs_block_group *block_group, unlink_free_space(ctl, info, true); spin_unlock(&ctl->tree_lock); kmem_cache_free(btrfs_free_space_cachep, info); - ret = btrfs_add_free_space(block_group, offset, bytes); + ret = btrfs_add_free_space(ctl->block_group, offset, bytes); spin_lock(&ctl->tree_lock); } else { u64 offset = info->offset; - u64 bytes = ctl->unit; + u64 bytes = ctl->block_group->fs_info->sectorsize; ret = search_bitmap(ctl, info, &offset, &bytes, false); if (ret == 0) { bitmap_clear_bits(ctl, info, offset, bytes, true); spin_unlock(&ctl->tree_lock); - ret = btrfs_add_free_space(block_group, offset, + ret = btrfs_add_free_space(ctl->block_group, offset, bytes); spin_lock(&ctl->tree_lock); } else { @@ -1025,7 +1024,7 @@ int load_free_space_cache(struct btrfs_block_group *block_group) if (matched) { spin_lock(&tmp_ctl.tree_lock); - ret = copy_free_space_cache(block_group, &tmp_ctl); + ret = copy_free_space_cache(&tmp_ctl); spin_unlock(&tmp_ctl.tree_lock); /* * ret == 1 means we successfully loaded the free space cache, @@ -1068,12 +1067,12 @@ out: static noinline_for_stack int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl, - struct btrfs_free_space_ctl *ctl, struct btrfs_block_group *block_group, int *entries, int *bitmaps, struct list_head *bitmap_list) { int ret; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; struct btrfs_free_cluster *cluster = NULL; struct btrfs_free_cluster *cluster_locked = NULL; struct rb_node *node = rb_first(&ctl->free_space_offset); @@ -1367,17 +1366,17 @@ int btrfs_wait_cache_io(struct btrfs_trans_handle *trans, * or an errno if it was not. */ static int __btrfs_write_out_cache(struct inode *inode, - struct btrfs_free_space_ctl *ctl, struct btrfs_block_group *block_group, struct btrfs_trans_handle *trans) { + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; struct btrfs_io_ctl *io_ctl = &block_group->io_ctl; struct extent_state *cached_state = NULL; LIST_HEAD(bitmap_list); int entries = 0; int bitmaps = 0; int ret; - int must_iput = 0; + bool must_iput = false; int i_size; if (!i_size_read(inode)) @@ -1397,7 +1396,7 @@ static int __btrfs_write_out_cache(struct inode *inode, up_write(&block_group->data_rwsem); BTRFS_I(inode)->generation = 0; ret = 0; - must_iput = 1; + must_iput = true; goto out; } spin_unlock(&block_group->lock); @@ -1416,8 +1415,7 @@ static int __btrfs_write_out_cache(struct inode *inode, mutex_lock(&ctl->cache_writeout_mutex); /* Write out the extent entries in the free space cache */ spin_lock(&ctl->tree_lock); - ret = write_cache_extent_entries(io_ctl, ctl, - block_group, &entries, &bitmaps, + ret = write_cache_extent_entries(io_ctl, block_group, &entries, &bitmaps, &bitmap_list); if (ret) goto out_nospc_locked; @@ -1516,7 +1514,6 @@ int btrfs_write_out_cache(struct btrfs_trans_handle *trans, struct btrfs_path *path) { struct btrfs_fs_info *fs_info = trans->fs_info; - struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; struct inode *inode; int ret = 0; @@ -1531,7 +1528,7 @@ int btrfs_write_out_cache(struct btrfs_trans_handle *trans, if (IS_ERR(inode)) return 0; - ret = __btrfs_write_out_cache(inode, ctl, block_group, trans); + ret = __btrfs_write_out_cache(inode, block_group, trans); if (ret) { btrfs_debug(fs_info, "failed to write free space cache for block group %llu error %d", @@ -1571,11 +1568,11 @@ static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, u64 bitmap_start; u64 bytes_per_bitmap; - bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; - bitmap_start = offset - ctl->start; + bytes_per_bitmap = BITS_PER_BITMAP * ctl->block_group->fs_info->sectorsize; + bitmap_start = offset - ctl->block_group->start; bitmap_start = div64_u64(bitmap_start, bytes_per_bitmap); bitmap_start *= bytes_per_bitmap; - bitmap_start += ctl->start; + bitmap_start += ctl->block_group->start; return bitmap_start; } @@ -1702,6 +1699,7 @@ tree_search_offset(struct btrfs_free_space_ctl *ctl, { struct rb_node *n = ctl->free_space_offset.rb_node; struct btrfs_free_space *entry = NULL, *prev = NULL; + const int unit = ctl->block_group->fs_info->sectorsize; lockdep_assert_held(&ctl->tree_lock); @@ -1785,7 +1783,7 @@ tree_search_offset(struct btrfs_free_space_ctl *ctl, prev->offset + prev->bytes > offset) return prev; } - if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) + if (entry->offset + BITS_PER_BITMAP * unit > offset) return entry; } else if (entry->offset + entry->bytes > offset) return entry; @@ -1799,8 +1797,7 @@ tree_search_offset(struct btrfs_free_space_ctl *ctl, return NULL; entry = rb_entry(n, struct btrfs_free_space, offset_index); if (entry->bitmap) { - if (entry->offset + BITS_PER_BITMAP * - ctl->unit > offset) + if (entry->offset + BITS_PER_BITMAP * unit > offset) break; } else { if (entry->offset + entry->bytes > offset) @@ -1875,18 +1872,19 @@ static inline void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, u64 offset, u64 bytes, bool update_stat) { + const int unit = ctl->block_group->fs_info->sectorsize; unsigned long start, count, end; int extent_delta = -1; - start = offset_to_bit(info->offset, ctl->unit, offset); - count = bytes_to_bits(bytes, ctl->unit); + start = offset_to_bit(info->offset, unit, offset); + count = bytes_to_bits(bytes, unit); end = start + count; ASSERT(end <= BITS_PER_BITMAP); bitmap_clear(info->bitmap, start, count); info->bytes -= bytes; - if (info->max_extent_size > ctl->unit) + if (info->max_extent_size > unit) info->max_extent_size = 0; relink_bitmap_entry(ctl, info); @@ -1911,11 +1909,12 @@ static void btrfs_bitmap_set_bits(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, u64 offset, u64 bytes) { + const int unit = ctl->block_group->fs_info->sectorsize; unsigned long start, count, end; int extent_delta = 1; - start = offset_to_bit(info->offset, ctl->unit, offset); - count = bytes_to_bits(bytes, ctl->unit); + start = offset_to_bit(info->offset, unit, offset); + count = bytes_to_bits(bytes, unit); end = start + count; ASSERT(end <= BITS_PER_BITMAP); @@ -1952,6 +1951,7 @@ static int search_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *bitmap_info, u64 *offset, u64 *bytes, bool for_alloc) { + const int unit = ctl->block_group->fs_info->sectorsize; unsigned long found_bits = 0; unsigned long max_bits = 0; unsigned long bits, i; @@ -1969,9 +1969,9 @@ static int search_bitmap(struct btrfs_free_space_ctl *ctl, return -1; } - i = offset_to_bit(bitmap_info->offset, ctl->unit, + i = offset_to_bit(bitmap_info->offset, unit, max_t(u64, *offset, bitmap_info->offset)); - bits = bytes_to_bits(*bytes, ctl->unit); + bits = bytes_to_bits(*bytes, unit); for_each_set_bit_from(i, bitmap_info->bitmap, BITS_PER_BITMAP) { if (for_alloc && bits == 1) { @@ -1991,12 +1991,12 @@ static int search_bitmap(struct btrfs_free_space_ctl *ctl, } if (found_bits) { - *offset = (u64)(i * ctl->unit) + bitmap_info->offset; - *bytes = (u64)(found_bits) * ctl->unit; + *offset = (u64)(i * unit) + bitmap_info->offset; + *bytes = (u64)(found_bits) * unit; return 0; } - *bytes = (u64)(max_bits) * ctl->unit; + *bytes = (u64)(max_bits) * unit; bitmap_info->max_extent_size = *bytes; relink_bitmap_entry(ctl, bitmap_info); return -1; @@ -2054,9 +2054,9 @@ again: * to match our requested alignment */ if (*bytes >= align) { - tmp = entry->offset - ctl->start + align - 1; + tmp = entry->offset - ctl->block_group->start + align - 1; tmp = div64_u64(tmp, align); - tmp = tmp * align + ctl->start; + tmp = tmp * align + ctl->block_group->start; align_off = tmp - entry->offset; } else { align_off = 0; @@ -2148,12 +2148,13 @@ static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *bitmap_info, u64 *offset, u64 *bytes) { + const int unit = ctl->block_group->fs_info->sectorsize; u64 end; u64 search_start, search_bytes; int ret; again: - end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; + end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * unit) - 1; /* * We need to search for bits in this bitmap. We could only cover some @@ -2162,7 +2163,7 @@ again: * go searching for the next bit. */ search_start = *offset; - search_bytes = ctl->unit; + search_bytes = unit; search_bytes = min(search_bytes, end - search_start + 1); ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, false); @@ -2208,7 +2209,7 @@ again: * everything over again. */ search_start = *offset; - search_bytes = ctl->unit; + search_bytes = unit; ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, false); if (ret < 0 || search_start != *offset) @@ -2225,6 +2226,7 @@ static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, u64 offset, u64 bytes, enum btrfs_trim_state trim_state) { + const int unit = ctl->block_group->fs_info->sectorsize; u64 bytes_to_set = 0; u64 end; @@ -2241,7 +2243,7 @@ static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED; } - end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); + end = info->offset + (u64)(BITS_PER_BITMAP * unit); bytes_to_set = min(end - offset, bytes); @@ -2251,7 +2253,8 @@ static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, } -static bool use_bitmap(struct btrfs_free_space_ctl *ctl, +EXPORT_FOR_TESTS +bool btrfs_use_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info) { struct btrfs_block_group *block_group = ctl->block_group; @@ -2295,22 +2298,18 @@ static bool use_bitmap(struct btrfs_free_space_ctl *ctl, * so allow those block groups to still be allowed to have a bitmap * entry. */ - if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length) + if (((BITS_PER_BITMAP * fs_info->sectorsize) >> 1) > block_group->length) return false; return true; } -static const struct btrfs_free_space_op free_space_op = { - .use_bitmap = use_bitmap, -}; - static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info) { struct btrfs_free_space *bitmap_info; - struct btrfs_block_group *block_group = NULL; - int added = 0; + struct btrfs_block_group *block_group = ctl->block_group; + bool added = false; u64 bytes, offset, bytes_added; enum btrfs_trim_state trim_state; int ret; @@ -2319,18 +2318,20 @@ static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, offset = info->offset; trim_state = info->trim_state; - if (!ctl->op->use_bitmap(ctl, info)) - return 0; - - if (ctl->op == &free_space_op) - block_group = ctl->block_group; + if (btrfs_is_testing(block_group->fs_info)) { + if (!block_group->fs_info->use_bitmap(ctl, info)) + return 0; + } else { + if (!btrfs_use_bitmap(ctl, info)) + return 0; + } again: /* * Since we link bitmaps right into the cluster we need to see if we * have a cluster here, and if so and it has our bitmap we need to add * the free space to that bitmap. */ - if (block_group && !list_empty(&block_group->cluster_list)) { + if (!list_empty(&block_group->cluster_list)) { struct btrfs_free_cluster *cluster; struct rb_node *node; struct btrfs_free_space *entry; @@ -2367,7 +2368,7 @@ no_cluster_bitmap: bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), 1, 0); if (!bitmap_info) { - ASSERT(added == 0); + ASSERT(!added); goto new_bitmap; } @@ -2375,7 +2376,7 @@ no_cluster_bitmap: trim_state); bytes -= bytes_added; offset += bytes_added; - added = 0; + added = false; if (!bytes) { ret = 1; @@ -2386,7 +2387,7 @@ no_cluster_bitmap: new_bitmap: if (info && info->bitmap) { add_new_bitmap(ctl, info, offset); - added = 1; + added = true; info = NULL; goto again; } else { @@ -2494,6 +2495,7 @@ static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, bool update_stat) { + const int unit = ctl->block_group->fs_info->sectorsize; struct btrfs_free_space *bitmap; unsigned long i; unsigned long j; @@ -2505,11 +2507,11 @@ static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, if (!bitmap) return false; - i = offset_to_bit(bitmap->offset, ctl->unit, end); + i = offset_to_bit(bitmap->offset, unit, end); j = find_next_zero_bit(bitmap->bitmap, BITS_PER_BITMAP, i); if (j == i) return false; - bytes = (j - i) * ctl->unit; + bytes = (j - i) * unit; info->bytes += bytes; /* See try_merge_free_space() comment. */ @@ -2528,6 +2530,7 @@ static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, bool update_stat) { + const int unit = ctl->block_group->fs_info->sectorsize; struct btrfs_free_space *bitmap; u64 bitmap_offset; unsigned long i; @@ -2547,7 +2550,7 @@ static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, if (!bitmap) return false; - i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; + i = offset_to_bit(bitmap->offset, unit, info->offset) - 1; j = 0; prev_j = (unsigned long)-1; for_each_clear_bit_from(j, bitmap->bitmap, BITS_PER_BITMAP) { @@ -2559,9 +2562,9 @@ static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, return false; if (prev_j == (unsigned long)-1) - bytes = (i + 1) * ctl->unit; + bytes = (i + 1) * unit; else - bytes = (i - prev_j) * ctl->unit; + bytes = (i - prev_j) * unit; info->offset -= bytes; info->bytes += bytes; @@ -2947,13 +2950,8 @@ void btrfs_dump_free_space(struct btrfs_block_group *block_group, void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group, struct btrfs_free_space_ctl *ctl) { - struct btrfs_fs_info *fs_info = block_group->fs_info; - spin_lock_init(&ctl->tree_lock); - ctl->unit = fs_info->sectorsize; - ctl->start = block_group->start; ctl->block_group = block_group; - ctl->op = &free_space_op; ctl->free_space_bytes = RB_ROOT_CACHED; INIT_LIST_HEAD(&ctl->trimming_ranges); mutex_init(&ctl->cache_writeout_mutex); @@ -3327,6 +3325,7 @@ static int btrfs_bitmap_cluster(struct btrfs_block_group *block_group, u64 cont1_bytes, u64 min_bytes) { struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + const int unit = block_group->fs_info->sectorsize; unsigned long next_zero; unsigned long i; unsigned long want_bits; @@ -3339,10 +3338,10 @@ static int btrfs_bitmap_cluster(struct btrfs_block_group *block_group, lockdep_assert_held(&ctl->tree_lock); - i = offset_to_bit(entry->offset, ctl->unit, + i = offset_to_bit(entry->offset, unit, max_t(u64, offset, entry->offset)); - want_bits = bytes_to_bits(bytes, ctl->unit); - min_bits = bytes_to_bits(min_bytes, ctl->unit); + want_bits = bytes_to_bits(bytes, unit); + min_bits = bytes_to_bits(min_bytes, unit); /* * Don't bother looking for a cluster in this bitmap if it's heavily @@ -3368,7 +3367,7 @@ again: } if (!found_bits) { - entry->max_extent_size = (u64)max_bits * ctl->unit; + entry->max_extent_size = (u64)max_bits * unit; return -ENOSPC; } @@ -3379,15 +3378,15 @@ again: total_found += found_bits; - if (cluster->max_size < found_bits * ctl->unit) - cluster->max_size = found_bits * ctl->unit; + if (cluster->max_size < found_bits * unit) + cluster->max_size = found_bits * unit; if (total_found < want_bits || cluster->max_size < cont1_bytes) { i = next_zero + 1; goto again; } - cluster->window_start = start * ctl->unit + entry->offset; + cluster->window_start = start * unit + entry->offset; rb_erase(&entry->offset_index, &ctl->free_space_offset); rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); @@ -3403,8 +3402,7 @@ again: ret = tree_insert_offset(ctl, cluster, entry); ASSERT(!ret); /* -EEXIST; Logic error */ - trace_btrfs_setup_cluster(block_group, cluster, - total_found * ctl->unit, 1); + trace_btrfs_setup_cluster(block_group, cluster, total_found * unit, 1); return 0; } @@ -4044,7 +4042,9 @@ static int trim_bitmaps(struct btrfs_block_group *block_group, } next: if (next_bitmap) { - offset += BITS_PER_BITMAP * ctl->unit; + const int unit = block_group->fs_info->sectorsize; + + offset += BITS_PER_BITMAP * unit; start = offset; } else { start += bytes; @@ -4071,6 +4071,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, u64 *trimmed, u64 start, u64 end, u64 minlen) { struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + const int unit = block_group->fs_info->sectorsize; int ret; u64 rem = 0; @@ -4091,7 +4092,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, goto out; ret = trim_bitmaps(block_group, trimmed, start, end, minlen, 0, false); - div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem); + div64_u64_rem(end, BITS_PER_BITMAP * unit, &rem); /* If we ended in the middle of a bitmap, reset the trimming flag */ if (rem) reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); @@ -4310,6 +4311,7 @@ int test_check_exists(struct btrfs_block_group *cache, u64 offset, u64 bytes) { struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; + const int unit = cache->fs_info->sectorsize; struct btrfs_free_space *info; int ret = 0; @@ -4329,7 +4331,7 @@ have_info: struct btrfs_free_space *tmp; bit_off = offset; - bit_bytes = ctl->unit; + bit_bytes = unit; ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); if (!ret) { if (bit_off == offset) { diff --git a/fs/btrfs/free-space-cache.h b/fs/btrfs/free-space-cache.h index 33fc3b245648..53fe8e293af1 100644 --- a/fs/btrfs/free-space-cache.h +++ b/fs/btrfs/free-space-cache.h @@ -74,28 +74,20 @@ enum { }; struct btrfs_free_space_ctl { - spinlock_t tree_lock; struct rb_root free_space_offset; struct rb_root_cached free_space_bytes; - u64 free_space; + spinlock_t tree_lock; int extents_thresh; int free_extents; int total_bitmaps; - int unit; - u64 start; + u64 free_space; s32 discardable_extents[BTRFS_STAT_NR_ENTRIES]; s64 discardable_bytes[BTRFS_STAT_NR_ENTRIES]; - const struct btrfs_free_space_op *op; struct btrfs_block_group *block_group; struct mutex cache_writeout_mutex; struct list_head trimming_ranges; }; -struct btrfs_free_space_op { - bool (*use_bitmap)(struct btrfs_free_space_ctl *ctl, - struct btrfs_free_space *info); -}; - struct btrfs_io_ctl { void *cur, *orig; struct page *page; @@ -172,6 +164,8 @@ bool btrfs_free_space_cache_v1_active(struct btrfs_fs_info *fs_info); int btrfs_set_free_space_cache_v1_active(struct btrfs_fs_info *fs_info, bool active); /* Support functions for running our sanity tests */ #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +bool btrfs_use_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); int test_add_free_space_entry(struct btrfs_block_group *cache, u64 offset, u64 bytes, bool bitmap); int test_check_exists(struct btrfs_block_group *cache, u64 offset, u64 bytes); diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index 472b3060e5ac..1b3d82ae3de8 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -109,7 +109,7 @@ struct btrfs_free_space_info *btrfs_search_free_space_info( ret = btrfs_search_slot(trans, root, &key, path, 0, cow); if (ret < 0) return ERR_PTR(ret); - if (ret != 0) { + if (unlikely(ret != 0)) { btrfs_warn(fs_info, "missing free space info for %llu", block_group->start); DEBUG_WARN(); @@ -209,7 +209,8 @@ int btrfs_convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans, u64 bitmap_range, i; u32 bitmap_size, flags, expected_extent_count; u32 extent_count = 0; - int done = 0, nr; + bool done = false; + int nr; int ret; bitmap_size = free_space_bitmap_size(fs_info, block_group->length); @@ -240,7 +241,7 @@ int btrfs_convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans, if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { ASSERT(found_key.objectid == block_group->start); ASSERT(found_key.offset == block_group->length); - done = 1; + done = true; break; } else if (found_key.type == BTRFS_FREE_SPACE_EXTENT_KEY) { u64 first, last; @@ -353,7 +354,8 @@ int btrfs_convert_free_space_to_extents(struct btrfs_trans_handle *trans, u32 bitmap_size, flags, expected_extent_count; unsigned long nrbits, start_bit, end_bit; u32 extent_count = 0; - int done = 0, nr; + bool done = false; + int nr; int ret; bitmap_size = free_space_bitmap_size(fs_info, block_group->length); @@ -384,7 +386,7 @@ int btrfs_convert_free_space_to_extents(struct btrfs_trans_handle *trans, if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { ASSERT(found_key.objectid == block_group->start); ASSERT(found_key.offset == block_group->length); - done = 1; + done = true; break; } else if (found_key.type == BTRFS_FREE_SPACE_BITMAP_KEY) { unsigned long ptr; @@ -1473,7 +1475,8 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, struct btrfs_key key, found_key; struct extent_buffer *leaf; u64 start, end; - int done = 0, nr; + bool done = false; + int nr; int ret; if (!btrfs_fs_compat_ro(trans->fs_info, FREE_SPACE_TREE)) @@ -1514,7 +1517,7 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { ASSERT(found_key.objectid == block_group->start); ASSERT(found_key.offset == block_group->length); - done = 1; + done = true; nr++; path->slots[0]--; break; @@ -1545,6 +1548,29 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, return 0; } +static int validate_free_space_key(struct btrfs_block_group *block_group, + const struct btrfs_key *key, u8 expected_type) +{ + const u64 end = btrfs_block_group_end(block_group); + + if (unlikely(key->type != expected_type)) { + btrfs_err(block_group->fs_info, + "block group %llu has unexpected free space key type %u, expected %u", + block_group->start, key->type, expected_type); + return -EUCLEAN; + } + + if (unlikely(key->objectid + key->offset > end)) { + btrfs_err(block_group->fs_info, + "block group %llu has invalid free space key (%llu %u %llu)", + block_group->start, key->objectid, key->type, + key->offset); + return -EUCLEAN; + } + + return 0; +} + static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, struct btrfs_path *path, u32 expected_extent_count) @@ -1576,8 +1602,9 @@ static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, if (key.type == BTRFS_FREE_SPACE_INFO_KEY) break; - ASSERT(key.type == BTRFS_FREE_SPACE_BITMAP_KEY); - ASSERT(key.objectid < end && key.objectid + key.offset <= end); + ret = validate_free_space_key(block_group, &key, BTRFS_FREE_SPACE_BITMAP_KEY); + if (unlikely(ret)) + return ret; offset = key.objectid; while (offset < key.objectid + key.offset) { @@ -1633,7 +1660,6 @@ static int load_free_space_extents(struct btrfs_caching_control *caching_ctl, struct btrfs_fs_info *fs_info = block_group->fs_info; struct btrfs_root *root; struct btrfs_key key; - const u64 end = btrfs_block_group_end(block_group); u64 total_found = 0; u32 extent_count = 0; int ret; @@ -1654,8 +1680,9 @@ static int load_free_space_extents(struct btrfs_caching_control *caching_ctl, if (key.type == BTRFS_FREE_SPACE_INFO_KEY) break; - ASSERT(key.type == BTRFS_FREE_SPACE_EXTENT_KEY); - ASSERT(key.objectid < end && key.objectid + key.offset <= end); + ret = validate_free_space_key(block_group, &key, BTRFS_FREE_SPACE_EXTENT_KEY); + if (unlikely(ret)) + return ret; ret = btrfs_add_new_free_space(block_group, key.objectid, key.objectid + key.offset, diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index a4758d94b32e..5f0cfb0b5466 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -50,20 +50,26 @@ struct btrfs_subpage_info; struct btrfs_stripe_hash_table; struct btrfs_space_info; +/* Minimum data and metadata block size. */ +#define BTRFS_MIN_BLOCKSIZE (SZ_4K) +#define BTRFS_MAX_BLOCKSIZE (SZ_64K) + +/* The maximum folio size btrfs supports. */ +#define BTRFS_MAX_FOLIO_SIZE (SZ_2M) +static_assert(BTRFS_MAX_FOLIO_SIZE > PAGE_SIZE); + /* - * Minimum data and metadata block size. + * The maximum number of blocks a huge folio can support. * - * Normally it's 4K, but for testing subpage block size on 4K page systems, we - * allow DEBUG builds to accept 2K page size. + * Depending on the filesystem block size, the real maximum blocks per folio + * may also be limited by the above BTRFS_MAX_FOLIO_SIZE. */ -#ifdef CONFIG_BTRFS_DEBUG -#define BTRFS_MIN_BLOCKSIZE (SZ_2K) +#ifdef CONFIG_BTRFS_EXPERIMENTAL +#define BTRFS_MAX_BLOCKS_PER_FOLIO (512) #else -#define BTRFS_MIN_BLOCKSIZE (SZ_4K) +#define BTRFS_MAX_BLOCKS_PER_FOLIO (BITS_PER_LONG) #endif -#define BTRFS_MAX_BLOCKSIZE (SZ_64K) - #define BTRFS_MAX_EXTENT_SIZE SZ_128M /* @@ -89,6 +95,10 @@ static_assert(sizeof(struct btrfs_super_block) == BTRFS_SUPER_INFO_SIZE); #define BTRFS_KEY_FMT "(%llu %u %llu)" #define BTRFS_KEY_FMT_VALUE(key) (key)->objectid, (key)->type, (key)->offset +#define BTRFS_QGROUP_FMT "%hu/%llu" +#define BTRFS_QGROUP_FMT_VALUE(qgroup) btrfs_qgroup_level((qgroup)->qgroupid), \ + btrfs_qgroup_subvolid((qgroup)->qgroupid) + /* * Number of metadata items necessary for an unlink operation: * @@ -155,6 +165,7 @@ enum { BTRFS_FS_LOG_RECOVERING, BTRFS_FS_OPEN, BTRFS_FS_QUOTA_ENABLED, + BTRFS_FS_SQUOTA_ENABLING, BTRFS_FS_UPDATE_UUID_TREE_GEN, BTRFS_FS_CREATING_FREE_SPACE_TREE, BTRFS_FS_BTREE_ERR, @@ -485,6 +496,9 @@ struct btrfs_delayed_root { wait_queue_head_t wait; }; +struct btrfs_free_space_ctl; +struct btrfs_free_space; + struct btrfs_fs_info { u8 chunk_tree_uuid[BTRFS_UUID_SIZE]; unsigned long flags; @@ -643,6 +657,8 @@ struct btrfs_fs_info { * to protect us from the relocation code. */ struct mutex reloc_mutex; + /* Protects setting, clearing and getting fs_info->reloc_ctl. */ + spinlock_t reloc_ctl_lock; struct list_head trans_list; struct list_head dead_roots; @@ -697,13 +713,6 @@ struct btrfs_fs_info { struct btrfs_workqueue *endio_write_workers; struct btrfs_workqueue *endio_freespace_worker; struct btrfs_workqueue *caching_workers; - - /* - * Fixup workers take dirty pages that didn't properly go through the - * cow mechanism and make them safe to write. It happens for the - * sys_munmap function call path. - */ - struct btrfs_workqueue *fixup_workers; struct btrfs_workqueue *delayed_workers; struct task_struct *transaction_kthread; @@ -880,6 +889,7 @@ struct btrfs_fs_info { u32 block_min_order; u32 block_max_order; u32 stripesize; + u32 writeback_bio_size; u32 csum_size; u32 csums_per_leaf; u32 csum_type; @@ -957,6 +967,10 @@ struct btrfs_fs_info { spinlock_t eb_leak_lock; struct list_head allocated_ebs; #endif + + /* Used by self tests only. */ + bool (*use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); }; #define folio_to_inode(_folio) (BTRFS_I(_Generic((_folio), \ @@ -1206,14 +1220,6 @@ static inline void btrfs_force_shutdown(struct btrfs_fs_info *fs_info) } } -/* - * We use folio flag owner_2 to indicate there is an ordered extent with - * unfinished IO. - */ -#define folio_test_ordered(folio) folio_test_owner_2(folio) -#define folio_set_ordered(folio) folio_set_owner_2(folio) -#define folio_clear_ordered(folio) folio_clear_owner_2(folio) - #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS #define EXPORT_FOR_TESTS diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 906d5c21ebc4..272598f6ae77 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -226,9 +226,7 @@ static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off u32 item_size; int ret; - mutex_lock(&fs_info->reloc_mutex); logical = btrfs_get_reloc_bg_bytenr(fs_info); - mutex_unlock(&fs_info->reloc_mutex); if (logical == U64_MAX) { btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation"); @@ -401,28 +399,6 @@ void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags) static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, u64 offset, u64 bytes) { - pgoff_t index = offset >> PAGE_SHIFT; - const pgoff_t end_index = (offset + bytes - 1) >> PAGE_SHIFT; - struct folio *folio; - - while (index <= end_index) { - folio = filemap_get_folio(inode->vfs_inode.i_mapping, index); - if (IS_ERR(folio)) { - index++; - continue; - } - - index = folio_next_index(folio); - /* - * Here we just clear all Ordered bits for every page in the - * range, then btrfs_mark_ordered_io_finished() will handle - * the ordered extent accounting for the range. - */ - btrfs_folio_clamp_clear_ordered(inode->root->fs_info, folio, - offset, bytes); - folio_put(folio); - } - return btrfs_mark_ordered_io_finished(inode, offset, bytes, false); } @@ -755,7 +731,7 @@ static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, { struct btrfs_fs_info *fs_info = inode->root->fs_info; - if (!btrfs_inode_can_compress(inode)) { + if (unlikely(!btrfs_inode_can_compress(inode))) { DEBUG_WARN("BTRFS: unexpected compression for ino %llu", btrfs_ino(inode)); return 0; } @@ -842,7 +818,7 @@ static struct folio *compressed_bio_last_folio(struct compressed_bio *cb) ASSERT(bio->bi_vcnt); bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; - paddr = page_to_phys(bvec->bv_page) + bvec->bv_offset + bvec->bv_len - 1; + paddr = bvec_phys(bvec) + bvec->bv_len - 1; return page_folio(phys_to_page(paddr)); } @@ -1406,7 +1382,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode, * setup for writepage. */ page_ops = ((flags & COW_FILE_RANGE_KEEP_LOCKED) ? 0 : PAGE_UNLOCK); - page_ops |= PAGE_SET_ORDERED; /* * Relocation relies on the relocated extents to have exactly the same @@ -1972,8 +1947,7 @@ static int nocow_one_range(struct btrfs_inode *inode, struct folio *locked_folio goto error; extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached, EXTENT_LOCKED | EXTENT_DELALLOC | - EXTENT_CLEAR_DATA_RESV, - PAGE_SET_ORDERED); + EXTENT_CLEAR_DATA_RESV, 0); return ret; error: @@ -2317,7 +2291,7 @@ error: static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end) { if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) { - if (inode->defrag_bytes && + if (data_race(inode->defrag_bytes) && btrfs_test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG)) return false; return true; @@ -2605,8 +2579,7 @@ void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *s lockdep_assert_held(&inode->io_tree.lock); - if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC)) - WARN_ON(1); + WARN_ON((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC)); /* * set_bit and clear bit hooks normally require _irqsave/restore * but in this case, we are only testing for the DELALLOC @@ -2810,7 +2783,13 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, unsigned int extra_bits, struct extent_state **cached_state) { - WARN_ON(PAGE_ALIGNED(end)); + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); if (start >= i_size_read(&inode->vfs_inode) && !(inode->flags & BTRFS_INODE_PREALLOC)) { @@ -2833,206 +2812,50 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, EXTENT_DELALLOC | extra_bits, cached_state); } -/* see btrfs_writepage_start_hook for details on why this is required */ -struct btrfs_writepage_fixup { - struct folio *folio; - struct btrfs_inode *inode; - struct btrfs_work work; -}; - -static void btrfs_writepage_fixup_worker(struct btrfs_work *work) -{ - struct btrfs_writepage_fixup *fixup = - container_of(work, struct btrfs_writepage_fixup, work); - struct btrfs_ordered_extent *ordered; - struct extent_state *cached_state = NULL; - struct extent_changeset *data_reserved = NULL; - struct folio *folio = fixup->folio; - struct btrfs_inode *inode = fixup->inode; - struct btrfs_fs_info *fs_info = inode->root->fs_info; - u64 page_start = folio_pos(folio); - u64 page_end = folio_next_pos(folio) - 1; - int ret = 0; - bool free_delalloc_space = true; - - /* - * This is similar to page_mkwrite, we need to reserve the space before - * we take the folio lock. - */ - ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, - folio_size(folio)); -again: - folio_lock(folio); - - /* - * Before we queued this fixup, we took a reference on the folio. - * folio->mapping may go NULL, but it shouldn't be moved to a different - * address space. - */ - if (!folio->mapping || !folio_test_dirty(folio) || - !folio_test_checked(folio)) { - /* - * Unfortunately this is a little tricky, either - * - * 1) We got here and our folio had already been dealt with and - * we reserved our space, thus ret == 0, so we need to just - * drop our space reservation and bail. This can happen the - * first time we come into the fixup worker, or could happen - * while waiting for the ordered extent. - * 2) Our folio was already dealt with, but we happened to get an - * ENOSPC above from the btrfs_delalloc_reserve_space. In - * this case we obviously don't have anything to release, but - * because the folio was already dealt with we don't want to - * mark the folio with an error, so make sure we're resetting - * ret to 0. This is why we have this check _before_ the ret - * check, because we do not want to have a surprise ENOSPC - * when the folio was already properly dealt with. - */ - if (!ret) { - btrfs_delalloc_release_extents(inode, folio_size(folio)); - btrfs_delalloc_release_space(inode, data_reserved, - page_start, folio_size(folio), - true); - } - ret = 0; - goto out_page; - } - - /* - * We can't mess with the folio state unless it is locked, so now that - * it is locked bail if we failed to make our space reservation. - */ - if (ret) - goto out_page; - - btrfs_lock_extent(&inode->io_tree, page_start, page_end, &cached_state); - - /* already ordered? We're done */ - if (folio_test_ordered(folio)) - goto out_reserved; - - ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); - if (ordered) { - btrfs_unlock_extent(&inode->io_tree, page_start, page_end, - &cached_state); - folio_unlock(folio); - btrfs_start_ordered_extent(ordered); - btrfs_put_ordered_extent(ordered); - goto again; - } - - ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, - &cached_state); - if (ret) - goto out_reserved; - - /* - * Everything went as planned, we're now the owner of a dirty page with - * delayed allocation bits set and space reserved for our COW - * destination. - * - * The page was dirty when we started, nothing should have cleaned it. - */ - BUG_ON(!folio_test_dirty(folio)); - free_delalloc_space = false; -out_reserved: - btrfs_delalloc_release_extents(inode, PAGE_SIZE); - if (free_delalloc_space) - btrfs_delalloc_release_space(inode, data_reserved, page_start, - PAGE_SIZE, true); - btrfs_unlock_extent(&inode->io_tree, page_start, page_end, &cached_state); -out_page: - if (ret) { - /* - * We hit ENOSPC or other errors. Update the mapping and page - * to reflect the errors and clean the page. - */ - mapping_set_error(folio->mapping, ret); - btrfs_folio_clear_ordered(fs_info, folio, page_start, - folio_size(folio)); - btrfs_mark_ordered_io_finished(inode, page_start, - folio_size(folio), !ret); - folio_clear_dirty_for_io(folio); - } - btrfs_folio_clear_checked(fs_info, folio, page_start, PAGE_SIZE); - folio_unlock(folio); - folio_put(folio); - kfree(fixup); - extent_changeset_free(data_reserved); - /* - * As a precaution, do a delayed iput in case it would be the last iput - * that could need flushing space. Recursing back to fixup worker would - * deadlock. - */ - btrfs_add_delayed_iput(inode); -} - /* - * There are a few paths in the higher layers of the kernel that directly - * set the folio dirty bit without asking the filesystem if it is a - * good idea. This causes problems because we want to make sure COW - * properly happens and the data=ordered rules are followed. + * Clear the old accounting flags and set EXTENT_DELALLOC for the range. * - * In our case any range that doesn't have the ORDERED bit set - * hasn't been properly setup for IO. We kick off an async process - * to fix it up. The async helper will wait for ordered extents, set - * the delalloc bit and make it safe to write the folio. + * Return <0 for error, in that case no range has EXTENT_DELALLOC bit cleared or set. */ -int btrfs_writepage_cow_fixup(struct folio *folio) +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state) { - struct inode *inode = folio->mapping->host; - struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_writepage_fixup *fixup; - - /* This folio has ordered extent covering it already */ - if (folio_test_ordered(folio)) - return 0; - - /* - * For experimental build, we error out instead of EAGAIN. - * - * We should not hit such out-of-band dirty folios anymore. - */ - if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL)) { - DEBUG_WARN(); - btrfs_err_rl(fs_info, - "root %lld ino %llu folio %llu is marked dirty without notifying the fs", - btrfs_root_id(BTRFS_I(inode)->root), - btrfs_ino(BTRFS_I(inode)), - folio_pos(folio)); - return -EUCLEAN; - } + const u32 blocksize = inode->root->fs_info->sectorsize; - /* - * folio_checked is set below when we create a fixup worker for this - * folio, don't try to create another one if we're already - * folio_test_checked. - * - * The extent_io writepage code will redirty the foio if we send back - * EAGAIN. - */ - if (folio_test_checked(folio)) - return -EAGAIN; + /* The @extra_bits can only be EXTENT_NORESERVE for now. */ + ASSERT(!(extra_bits & ~EXTENT_NORESERVE), "extra_bits=0x%x", extra_bits); - fixup = kzalloc_obj(*fixup, GFP_NOFS); - if (!fixup) - return -EAGAIN; + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); /* - * We are already holding a reference to this inode from - * write_cache_pages. We need to hold it because the space reservation - * takes place outside of the folio lock, and we can't trust - * folio->mapping outside of the folio lock. + * Check and set DELALLOC_NEW flag, this needs to search tree thus can + * fail early. Thus we want to do this before clearing EXTENT_DELALLOC. */ - ihold(inode); - btrfs_folio_set_checked(fs_info, folio, folio_pos(folio), folio_size(folio)); - folio_get(folio); - btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL); - fixup->folio = folio; - fixup->inode = BTRFS_I(inode); - btrfs_queue_work(fs_info->fixup_workers, &fixup->work); + if (start >= i_size_read(&inode->vfs_inode) && + !(inode->flags & BTRFS_INODE_PREALLOC)) { + /* + * There can't be any extents following EOF in this case so just + * set the delalloc new bit for the range directly. + */ + extra_bits |= EXTENT_DELALLOC_NEW; + } else { + int ret; - return -EAGAIN; + ret = btrfs_find_new_delalloc_bytes(inode, start, end + 1 - start, + NULL); + if (unlikely(ret)) + return ret; + } + /* Clear the old accounting as the range may already be dirty. */ + btrfs_clear_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | + EXTENT_DEFRAG, cached_state); + return btrfs_set_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | extra_bits, cached_state); } static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, @@ -3798,7 +3621,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) if (!inode && root == fs_info->tree_root) { struct btrfs_root *dead_root; - int is_dead_root = 0; + bool is_dead_root = false; /* * This is an orphan in the tree root. Currently these @@ -3820,7 +3643,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, (unsigned long)found_key.objectid); if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) - is_dead_root = 1; + is_dead_root = true; spin_unlock(&fs_info->fs_roots_radix_lock); if (is_dead_root) { @@ -4745,7 +4568,7 @@ static void btrfs_prune_dentries(struct btrfs_root *root) inode = btrfs_find_first_inode(root, min_ino); while (inode) { - if (icount_read(&inode->vfs_inode) > 1) + if (icount_read_once(&inode->vfs_inode) > 1) d_prune_aliases(&inode->vfs_inode); min_ino = btrfs_ino(inode) + 1; @@ -5181,12 +5004,7 @@ again: goto again; } - btrfs_clear_extent_bit(&inode->io_tree, block_start, block_end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - &cached_state); - - ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, - &cached_state); + ret = btrfs_reset_extent_delalloc(inode, block_start, block_end, 0, &cached_state); if (ret) { btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); goto out_unlock; @@ -5211,8 +5029,6 @@ again: folio_zero_range(folio, zero_start - folio_pos(folio), zero_end - zero_start + 1); - btrfs_folio_clear_checked(fs_info, folio, block_start, - block_end + 1 - block_start); btrfs_folio_set_dirty(fs_info, folio, block_start, block_end + 1 - block_start); @@ -7657,12 +7473,6 @@ static int btrfs_migrate_folio(struct address_space *mapping, if (ret) return ret; - - if (folio_test_ordered(src)) { - folio_clear_ordered(src); - folio_set_ordered(dst); - } - return 0; } #else @@ -7751,18 +7561,20 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, page_end); ASSERT(range_end + 1 - cur < U32_MAX); range_len = range_end + 1 - cur; - if (!btrfs_folio_test_ordered(fs_info, folio, cur, range_len)) { - /* - * If Ordered is cleared, it means endio has - * already been executed for the range. - * We can't delete the extent states as - * btrfs_finish_ordered_io() may still use some of them. - */ + /* + * If the range is not dirty, the range has been submitted and + * since we have waited for the writeback, endio has been + * executed, thus we must skip the range to avoid double + * accounting for the ordered extent. + */ + if (!btrfs_folio_test_dirty(fs_info, folio, cur, range_len)) goto next; - } - btrfs_folio_clear_ordered(fs_info, folio, cur, range_len); /* + * The range is dirty meaning it has not been submitted. + * Here we need to truncate the OE range as the range will never + * be submitted. + * * IO on this page will never be started, so we need to account * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW * here, must leave that up for the ordered extent completion. @@ -7776,11 +7588,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, &cached_state); - spin_lock(&inode->ordered_tree_lock); - set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); - ordered->truncated_len = min(ordered->truncated_len, - cur - ordered->file_offset); - spin_unlock(&inode->ordered_tree_lock); + btrfs_mark_ordered_extent_truncated(ordered, cur - ordered->file_offset); /* * If the ordered extent has finished, we're safe to delete all @@ -7823,13 +7631,8 @@ next: &cached_state); cur = range_end + 1; } - /* - * We have iterated through all ordered extents of the page, the page - * should not have Ordered anymore, or the above iteration - * did something wrong. - */ - ASSERT(!folio_test_ordered(folio)); - btrfs_folio_clear_checked(fs_info, folio, folio_pos(folio), folio_size(folio)); + btrfs_folio_clear_dirty(fs_info, folio, page_start, folio_size(folio)); + btrfs_clear_folio_dirty_tag(folio); if (!inode_evicting) __btrfs_release_folio(folio, GFP_NOFS); clear_folio_extent_mapped(folio); @@ -9299,10 +9102,38 @@ next: if (!(mode & FALLOC_FL_KEEP_SIZE) && (actual_len > inode->i_size) && (cur_offset > inode->i_size)) { + u64 range_start; + u64 range_end; + if (cur_offset > actual_len) i_size = actual_len; else i_size = cur_offset; + + /* + * Make sure the file_extent_tree covers the entire + * range [old_i_size, new_i_size) before we update + * disk_i_size. Without this, a previous KEEP_SIZE + * prealloc that extended past i_size (and was lost + * across umount/mount because file_extent_tree is + * only populated up to round_up(i_size) on inode + * load) can leave a gap inside this range. That gap + * would cause btrfs_inode_safe_disk_i_size_write() + * (via find_contiguous_extent_bit() starting at 0) + * to truncate disk_i_size to the start of the gap, + * making the persisted size smaller than i_size. + */ + range_start = round_down(inode->i_size, fs_info->sectorsize); + range_end = round_up(i_size, fs_info->sectorsize); + ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), + range_start, range_end - range_start); + if (ret) { + btrfs_abort_transaction(trans, ret); + if (own_trans) + btrfs_end_transaction(trans); + break; + } + i_size_write(inode, i_size); btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); } @@ -9659,7 +9490,7 @@ ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, struct iov_iter *iter, pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); if (!pages) return -ENOMEM; - ret = btrfs_alloc_page_array(nr_pages, pages, false); + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); if (ret) { ret = -ENOMEM; goto out; @@ -10671,7 +10502,9 @@ struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino) break; from = btrfs_ino(inode) + 1; - cond_resched_lock(&root->inodes.xa_lock); + xa_unlock(&root->inodes); + cond_resched(); + xa_lock(&root->inodes); } xa_unlock(&root->inodes); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index a39460bf68a7..81e87bc39828 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -56,6 +56,7 @@ #include "uuid-tree.h" #include "ioctl.h" #include "file.h" +#include "file-item.h" #include "scrub.h" #include "super.h" @@ -82,6 +83,30 @@ struct btrfs_ioctl_received_subvol_args_32 { #define BTRFS_IOC_SET_RECEIVED_SUBVOL_32 _IOWR(BTRFS_IOCTL_MAGIC, 37, \ struct btrfs_ioctl_received_subvol_args_32) + +struct btrfs_ioctl_get_subvol_info_args_32 { + __u64 treeid; + char name[BTRFS_VOL_NAME_MAX + 1]; + __u64 parent_id; + __u64 dirid; + __u64 generation; + __u64 flags; + __u8 uuid[BTRFS_UUID_SIZE]; + __u8 parent_uuid[BTRFS_UUID_SIZE]; + __u8 received_uuid[BTRFS_UUID_SIZE]; + __u64 ctransid; + __u64 otransid; + __u64 stransid; + __u64 rtransid; + struct btrfs_ioctl_timespec_32 ctime; + struct btrfs_ioctl_timespec_32 otime; + struct btrfs_ioctl_timespec_32 stime; + struct btrfs_ioctl_timespec_32 rtime; + __u64 reserved[8]; +} __attribute__ ((__packed__)); + +#define BTRFS_IOC_GET_SUBVOL_INFO_32 _IOR(BTRFS_IOCTL_MAGIC, 60, \ + struct btrfs_ioctl_get_subvol_info_args_32) #endif #if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) @@ -264,6 +289,7 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap, int ret; const char *comp = NULL; u32 inode_flags; + bool prop_set = false; if (btrfs_root_readonly(root)) return -EROFS; @@ -376,16 +402,15 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap, if (comp) { ret = btrfs_set_prop(trans, inode, "btrfs.compression", comp, strlen(comp), 0); - if (unlikely(ret)) { - btrfs_abort_transaction(trans, ret); + if (ret) goto out_end_trans; - } + prop_set = true; } else { ret = btrfs_set_prop(trans, inode, "btrfs.compression", NULL, 0, 0); - if (unlikely(ret && ret != -ENODATA)) { - btrfs_abort_transaction(trans, ret); + prop_set = (ret == 0); + /* If ret == -ENODATA ignore and proceed to update inode item. */ + if (ret && ret != -ENODATA) goto out_end_trans; - } } update_flags: @@ -395,6 +420,12 @@ update_flags: inode_inc_iversion(&inode->vfs_inode); inode_set_ctime_current(&inode->vfs_inode); ret = btrfs_update_inode(trans, inode); + /* + * If we set a property or deleted one, we must abort if we fail to + * update the inode, to avoid persisting an inconsistent state. + */ + if (unlikely(ret && prop_set)) + btrfs_abort_transaction(trans, ret); out_end_trans: btrfs_end_transaction(trans); @@ -707,7 +738,7 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, { struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); struct inode *inode; - struct btrfs_pending_snapshot *pending_snapshot; + struct btrfs_pending_snapshot AUTO_KFREE(pending_snapshot); unsigned int trans_num_items; struct btrfs_trans_handle *trans; struct btrfs_block_rsv *block_rsv; @@ -816,7 +847,6 @@ free_pending: free_anon_bdev(pending_snapshot->anon_dev); kfree(pending_snapshot->root_item); btrfs_free_path(pending_snapshot->path); - kfree(pending_snapshot); return ret; } @@ -961,7 +991,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, u64 new_size; u64 old_size; u64 devid = 1; - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); struct btrfs_device *device = NULL; char *sizestr; char *devstr = NULL; @@ -987,13 +1017,13 @@ static noinline int btrfs_ioctl_resize(struct file *file, } ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out_free; + goto out_drop; sizestr = vol_args->name; cancel = (strcmp("cancel", sizestr) == 0); ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_RESIZE, cancel); if (ret) - goto out_free; + goto out_drop; /* Exclusive operation is now claimed */ devstr = strchr(sizestr, ':'); @@ -1100,8 +1130,6 @@ static noinline int btrfs_ioctl_resize(struct file *file, old_size, new_size); out_finish: btrfs_exclop_finish(fs_info); -out_free: - kfree(vol_args); out_drop: mnt_drop_write_file(file); return ret; @@ -1114,7 +1142,7 @@ static noinline int __btrfs_ioctl_snap_create(struct file *file, struct btrfs_qgroup_inherit *inherit) { int ret; - struct qstr qname = QSTR_INIT(name, strlen(name)); + struct qstr qname = QSTR(name); if (!S_ISDIR(file_inode(file)->i_mode)) return -ENOTDIR; @@ -1179,7 +1207,7 @@ out_drop_write: static noinline int btrfs_ioctl_snap_create(struct file *file, void __user *arg, bool subvol) { - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); int ret; if (!S_ISDIR(file_inode(file)->i_mode)) @@ -1190,24 +1218,20 @@ static noinline int btrfs_ioctl_snap_create(struct file *file, return PTR_ERR(vol_args); ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out; - - ret = __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), - vol_args->name, vol_args->fd, subvol, - false, NULL); + return ret; -out: - kfree(vol_args); - return ret; + return __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), + vol_args->name, vol_args->fd, subvol, + false, NULL); } static noinline int btrfs_ioctl_snap_create_v2(struct file *file, void __user *arg, bool subvol) { - struct btrfs_ioctl_vol_args_v2 *vol_args; + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args); + struct btrfs_qgroup_inherit AUTO_KFREE(inherit); int ret; bool readonly = false; - struct btrfs_qgroup_inherit *inherit = NULL; if (!S_ISDIR(file_inode(file)->i_mode)) return -ENOTDIR; @@ -1217,44 +1241,32 @@ static noinline int btrfs_ioctl_snap_create_v2(struct file *file, return PTR_ERR(vol_args); ret = btrfs_check_ioctl_vol_args2_subvol_name(vol_args); if (ret < 0) - goto free_args; + return ret; - if (vol_args->flags & ~BTRFS_SUBVOL_CREATE_ARGS_MASK) { - ret = -EOPNOTSUPP; - goto free_args; - } + if (vol_args->flags & ~BTRFS_SUBVOL_CREATE_ARGS_MASK) + return -EOPNOTSUPP; if (vol_args->flags & BTRFS_SUBVOL_RDONLY) readonly = true; if (vol_args->flags & BTRFS_SUBVOL_QGROUP_INHERIT) { struct btrfs_fs_info *fs_info = inode_to_fs_info(file_inode(file)); - if (vol_args->size < sizeof(*inherit) || - vol_args->size > PAGE_SIZE) { - ret = -EINVAL; - goto free_args; - } + if (vol_args->size < sizeof(*inherit) || vol_args->size > PAGE_SIZE) + return -EINVAL; + inherit = memdup_user(vol_args->qgroup_inherit, vol_args->size); if (IS_ERR(inherit)) { - ret = PTR_ERR(inherit); - goto free_args; + return PTR_ERR(inherit); } ret = btrfs_qgroup_check_inherit(fs_info, inherit, vol_args->size); if (ret < 0) - goto free_inherit; + return ret; } - ret = __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), - vol_args->name, vol_args->fd, subvol, - readonly, inherit); - if (ret) - goto free_inherit; -free_inherit: - kfree(inherit); -free_args: - kfree(vol_args); - return ret; + return __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), + vol_args->name, vol_args->fd, subvol, + readonly, inherit); } static noinline int btrfs_ioctl_subvol_getflags(struct btrfs_inode *inode, @@ -1865,7 +1877,7 @@ out_put: static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, void __user *argp) { - struct btrfs_ioctl_ino_lookup_args *args; + struct btrfs_ioctl_ino_lookup_args AUTO_KFREE(args); int ret = 0; args = memdup_user(argp, sizeof(*args)); @@ -1895,9 +1907,8 @@ static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, out: if (ret == 0 && copy_to_user(argp, args, sizeof(*args))) - ret = -EFAULT; + return -EFAULT; - kfree(args); return ret; } @@ -1915,7 +1926,7 @@ out: */ static int btrfs_ioctl_ino_lookup_user(struct file *file, void __user *argp) { - struct btrfs_ioctl_ino_lookup_user_args *args; + struct btrfs_ioctl_ino_lookup_user_args AUTO_KFREE(args); struct inode *inode; int ret; @@ -1931,7 +1942,6 @@ static int btrfs_ioctl_ino_lookup_user(struct file *file, void __user *argp) * The subvolume does not exist under fd with which this is * called */ - kfree(args); return -EACCES; } @@ -1940,14 +1950,13 @@ static int btrfs_ioctl_ino_lookup_user(struct file *file, void __user *argp) if (ret == 0 && copy_to_user(argp, args, sizeof(*args))) ret = -EFAULT; - kfree(args); return ret; } /* Get the subvolume information in BTRFS_ROOT_ITEM and BTRFS_ROOT_BACKREF */ -static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) +static int _btrfs_ioctl_get_subvol_info(struct inode *inode, + struct btrfs_ioctl_get_subvol_info_args *subvol_info) { - struct btrfs_ioctl_get_subvol_info_args *subvol_info; struct btrfs_fs_info *fs_info; struct btrfs_root *root; struct btrfs_path *path; @@ -1956,7 +1965,6 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) struct btrfs_root_ref *rref; struct extent_buffer *leaf; unsigned long item_off; - unsigned long item_len; int slot; int ret = 0; @@ -1964,12 +1972,6 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) if (!path) return -ENOMEM; - subvol_info = kzalloc_obj(*subvol_info); - if (!subvol_info) { - btrfs_free_path(path); - return -ENOMEM; - } - fs_info = BTRFS_I(inode)->root->fs_info; /* Get root_item of inode's subvolume */ @@ -2031,33 +2033,91 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) btrfs_item_key_to_cpu(leaf, &key, slot); if (key.objectid == subvol_info->treeid && key.type == BTRFS_ROOT_BACKREF_KEY) { + u16 name_len; + subvol_info->parent_id = key.offset; rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); subvol_info->dirid = btrfs_root_ref_dirid(leaf, rref); - item_off = btrfs_item_ptr_offset(leaf, slot) - + sizeof(struct btrfs_root_ref); - item_len = btrfs_item_size(leaf, slot) - - sizeof(struct btrfs_root_ref); + item_off = btrfs_item_ptr_offset(leaf, slot) + sizeof(*rref); read_extent_buffer(leaf, subvol_info->name, - item_off, item_len); + item_off, name_len); } else { ret = -ENOENT; goto out; } } - btrfs_free_path(path); - path = NULL; - if (copy_to_user(argp, subvol_info, sizeof(*subvol_info))) - ret = -EFAULT; - out: btrfs_put_root(root); out_free: btrfs_free_path(path); - kfree(subvol_info); + return ret; +} + +#ifdef CONFIG_64BIT +static int btrfs_ioctl_get_subvol_info_32(struct inode *inode, void __user *argp) +{ + struct btrfs_ioctl_get_subvol_info_args AUTO_KFREE(subvol_info); + struct btrfs_ioctl_get_subvol_info_args_32 AUTO_KFREE(subvol_info_32); + int ret; + + subvol_info = kzalloc_obj(*subvol_info); + if (!subvol_info) + return -ENOMEM; + + subvol_info_32 = kzalloc_obj(*subvol_info_32); + if (!subvol_info_32) + return -ENOMEM; + + ret = _btrfs_ioctl_get_subvol_info(inode, subvol_info); + if (ret) + return ret; + + subvol_info_32->treeid = subvol_info->treeid; + memcpy(subvol_info_32->name, subvol_info->name, sizeof(subvol_info_32->name)); + subvol_info_32->parent_id = subvol_info->parent_id; + subvol_info_32->dirid = subvol_info->dirid; + subvol_info_32->generation = subvol_info->generation; + subvol_info_32->flags = subvol_info->flags; + memcpy(subvol_info_32->uuid, subvol_info->uuid, BTRFS_UUID_SIZE); + memcpy(subvol_info_32->parent_uuid, subvol_info->parent_uuid, BTRFS_UUID_SIZE); + memcpy(subvol_info_32->received_uuid, subvol_info->received_uuid, BTRFS_UUID_SIZE); + subvol_info_32->ctransid = subvol_info->ctransid; + subvol_info_32->otransid = subvol_info->otransid; + subvol_info_32->stransid = subvol_info->stransid; + subvol_info_32->rtransid = subvol_info->rtransid; + subvol_info_32->ctime.sec = subvol_info->ctime.sec; + subvol_info_32->ctime.nsec = subvol_info->ctime.nsec; + subvol_info_32->otime.sec = subvol_info->otime.sec; + subvol_info_32->otime.nsec = subvol_info->otime.nsec; + subvol_info_32->stime.sec = subvol_info->stime.sec; + subvol_info_32->stime.nsec = subvol_info->stime.nsec; + subvol_info_32->rtime.sec = subvol_info->rtime.sec; + subvol_info_32->rtime.nsec = subvol_info->rtime.nsec; + + if (copy_to_user(argp, subvol_info_32, sizeof(*subvol_info_32))) + ret = -EFAULT; + + return ret; +} +#endif + +static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) +{ + struct btrfs_ioctl_get_subvol_info_args AUTO_KFREE(subvol_info); + int ret; + + subvol_info = kzalloc_obj(*subvol_info); + if (!subvol_info) + return -ENOMEM; + + ret = _btrfs_ioctl_get_subvol_info(inode, subvol_info); + if (!ret && copy_to_user(argp, subvol_info, sizeof(*subvol_info))) + ret = -EFAULT; + return ret; } @@ -2068,7 +2128,7 @@ out_free: static int btrfs_ioctl_get_subvol_rootref(struct btrfs_root *root, void __user *argp) { - struct btrfs_ioctl_get_subvol_rootref_args *rootrefs; + struct btrfs_ioctl_get_subvol_rootref_args AUTO_KFREE(rootrefs); struct btrfs_root_ref *rref; struct btrfs_path *path; struct btrfs_key key; @@ -2151,8 +2211,6 @@ out: ret = -EFAULT; } - kfree(rootrefs); - return ret; } @@ -2167,8 +2225,8 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, struct inode *inode; struct btrfs_root *root = BTRFS_I(dir)->root; struct btrfs_root *dest = NULL; - struct btrfs_ioctl_vol_args *vol_args = NULL; - struct btrfs_ioctl_vol_args_v2 *vol_args2 = NULL; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args2); struct mnt_idmap *idmap = file_mnt_idmap(file); char *subvol_name, *subvol_name_ptr = NULL; int ret = 0; @@ -2186,10 +2244,8 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, if (IS_ERR(vol_args2)) return PTR_ERR(vol_args2); - if (vol_args2->flags & ~BTRFS_SUBVOL_DELETE_ARGS_MASK) { - ret = -EOPNOTSUPP; - goto out; - } + if (vol_args2->flags & ~BTRFS_SUBVOL_DELETE_ARGS_MASK) + return -EOPNOTSUPP; /* * If SPEC_BY_ID is not set, we are looking for the subvolume by @@ -2198,23 +2254,21 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, if (!(vol_args2->flags & BTRFS_SUBVOL_SPEC_BY_ID)) { ret = btrfs_check_ioctl_vol_args2_subvol_name(vol_args2); if (ret < 0) - goto out; + return ret; subvol_name = vol_args2->name; ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; } else { struct inode *old_dir; - if (vol_args2->subvolid < BTRFS_FIRST_FREE_OBJECTID) { - ret = -EINVAL; - goto out; - } + if (vol_args2->subvolid < BTRFS_FIRST_FREE_OBJECTID) + return -EINVAL; ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; dentry = btrfs_get_dentry(fs_info->sb, BTRFS_FIRST_FREE_OBJECTID, @@ -2284,13 +2338,13 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out; + return ret; subvol_name = vol_args->name; ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; } if (strchr(subvol_name, '/') || @@ -2371,9 +2425,6 @@ free_parent: dput(parent); out_drop_write: mnt_drop_write_file(file); -out: - kfree(vol_args2); - kfree(vol_args); return ret; } @@ -2461,7 +2512,7 @@ out: static long btrfs_ioctl_add_dev(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); bool restore_op = false; int ret; @@ -2501,15 +2552,13 @@ static long btrfs_ioctl_add_dev(struct btrfs_fs_info *fs_info, void __user *arg) ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out_free; + goto out; ret = btrfs_init_new_device(fs_info, vol_args->name); if (!ret) btrfs_info(fs_info, "disk added %s", vol_args->name); -out_free: - kfree(vol_args); out: if (restore_op) btrfs_exclop_balance(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED); @@ -2523,7 +2572,7 @@ static long btrfs_ioctl_rm_dev_v2(struct file *file, void __user *arg) BTRFS_DEV_LOOKUP_ARGS(args); struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_vol_args_v2 *vol_args; + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args); struct file *bdev_file = NULL; int ret; bool cancel = false; @@ -2582,7 +2631,6 @@ err_drop: bdev_fput(bdev_file); out: btrfs_put_dev_args_from_path(&args); - kfree(vol_args); return ret; } @@ -2591,7 +2639,7 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) BTRFS_DEV_LOOKUP_ARGS(args); struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); struct file *bdev_file = NULL; int ret; bool cancel = false; @@ -2605,7 +2653,7 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out_free; + return ret; if (!strcmp("cancel", vol_args->name)) { cancel = true; @@ -2633,19 +2681,16 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) bdev_fput(bdev_file); out: btrfs_put_dev_args_from_path(&args); -out_free: - kfree(vol_args); return ret; } static long btrfs_ioctl_fs_info(const struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_fs_info_args *fi_args; + struct btrfs_ioctl_fs_info_args AUTO_KFREE(fi_args); struct btrfs_device *device; struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; u64 flags_in; - int ret = 0; fi_args = memdup_user(arg, sizeof(*fi_args)); if (IS_ERR(fi_args)) @@ -2686,17 +2731,16 @@ static long btrfs_ioctl_fs_info(const struct btrfs_fs_info *fs_info, } if (copy_to_user(arg, fi_args, sizeof(*fi_args))) - ret = -EFAULT; + return -EFAULT; - kfree(fi_args); - return ret; + return 0; } static long btrfs_ioctl_dev_info(const struct btrfs_fs_info *fs_info, void __user *arg) { BTRFS_DEV_LOOKUP_ARGS(args); - struct btrfs_ioctl_dev_info_args *di_args; + struct btrfs_ioctl_dev_info_args AUTO_KFREE(di_args); struct btrfs_device *dev; int ret = 0; @@ -2730,7 +2774,6 @@ out: if (ret == 0 && copy_to_user(arg, di_args, sizeof(*di_args))) ret = -EFAULT; - kfree(di_args); return ret; } @@ -2792,9 +2835,13 @@ static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) if (IS_ERR_OR_NULL(di)) { btrfs_release_path(path); btrfs_end_transaction(trans); + if (di) + ret = PTR_ERR(di); + else + ret = -ENOENT; btrfs_err(fs_info, - "Umm, you don't have the default diritem, this isn't going to work"); - ret = -ENOENT; + "could not find default diritem for dir %llu: %d", + dir_id, ret); goto out_free; } @@ -3011,7 +3058,7 @@ static noinline long btrfs_ioctl_wait_sync(struct btrfs_fs_info *fs_info, static long btrfs_ioctl_scrub(struct file *file, void __user *arg) { struct btrfs_fs_info *fs_info = inode_to_fs_info(file_inode(file)); - struct btrfs_ioctl_scrub_args *sa; + struct btrfs_ioctl_scrub_args AUTO_KFREE(sa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3026,15 +3073,13 @@ static long btrfs_ioctl_scrub(struct file *file, void __user *arg) if (IS_ERR(sa)) return PTR_ERR(sa); - if (sa->flags & ~BTRFS_SCRUB_SUPPORTED_FLAGS) { - ret = -EOPNOTSUPP; - goto out; - } + if (sa->flags & ~BTRFS_SCRUB_SUPPORTED_FLAGS) + return -EOPNOTSUPP; if (!(sa->flags & BTRFS_SCRUB_READONLY)) { ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; } ret = btrfs_scrub_dev(fs_info, sa->devid, sa->start, sa->end, @@ -3058,8 +3103,7 @@ static long btrfs_ioctl_scrub(struct file *file, void __user *arg) if (!(sa->flags & BTRFS_SCRUB_READONLY)) mnt_drop_write_file(file); -out: - kfree(sa); + return ret; } @@ -3074,7 +3118,7 @@ static long btrfs_ioctl_scrub_cancel(struct btrfs_fs_info *fs_info) static long btrfs_ioctl_scrub_progress(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_scrub_args *sa; + struct btrfs_ioctl_scrub_args AUTO_KFREE(sa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3087,40 +3131,36 @@ static long btrfs_ioctl_scrub_progress(struct btrfs_fs_info *fs_info, ret = btrfs_scrub_progress(fs_info, sa->devid, &sa->progress); if (ret == 0 && copy_to_user(arg, sa, sizeof(*sa))) - ret = -EFAULT; + return -EFAULT; - kfree(sa); return ret; } static long btrfs_ioctl_get_dev_stats(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_get_dev_stats *sa; + struct btrfs_ioctl_get_dev_stats AUTO_KFREE(sa); int ret; sa = memdup_user(arg, sizeof(*sa)); if (IS_ERR(sa)) return PTR_ERR(sa); - if ((sa->flags & BTRFS_DEV_STATS_RESET) && !capable(CAP_SYS_ADMIN)) { - kfree(sa); + if ((sa->flags & BTRFS_DEV_STATS_RESET) && !capable(CAP_SYS_ADMIN)) return -EPERM; - } ret = btrfs_get_dev_stats(fs_info, sa); if (ret == 0 && copy_to_user(arg, sa, sizeof(*sa))) - ret = -EFAULT; + return -EFAULT; - kfree(sa); return ret; } static long btrfs_ioctl_dev_replace(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_dev_replace_args *p; + struct btrfs_ioctl_dev_replace_args AUTO_KFREE(p); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3137,10 +3177,8 @@ static long btrfs_ioctl_dev_replace(struct btrfs_fs_info *fs_info, switch (p->cmd) { case BTRFS_IOCTL_DEV_REPLACE_CMD_START: - if (sb_rdonly(fs_info->sb)) { - ret = -EROFS; - goto out; - } + if (sb_rdonly(fs_info->sb)) + return -EROFS; if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) { ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; } else { @@ -3162,9 +3200,8 @@ static long btrfs_ioctl_dev_replace(struct btrfs_fs_info *fs_info, } if ((ret == 0 || ret == -ECANCELED) && copy_to_user(arg, p, sizeof(*p))) - ret = -EFAULT; -out: - kfree(p); + return -EFAULT; + return ret; } @@ -3174,7 +3211,7 @@ static long btrfs_ioctl_ino_to_path(struct btrfs_root *root, void __user *arg) int i; u64 rel_ptr; int size; - struct btrfs_ioctl_ino_path_args *ipa = NULL; + struct btrfs_ioctl_ino_path_args AUTO_KFREE(ipa); struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL; struct btrfs_path *path; @@ -3223,7 +3260,6 @@ static long btrfs_ioctl_ino_to_path(struct btrfs_root *root, void __user *arg) out: btrfs_free_path(path); - kfree(ipa); return ret; } @@ -3233,8 +3269,8 @@ static long btrfs_ioctl_logical_to_ino(struct btrfs_fs_info *fs_info, { int ret = 0; int size; - struct btrfs_ioctl_logical_ino_args *loi; - struct btrfs_data_container *inodes = NULL; + struct btrfs_ioctl_logical_ino_args AUTO_KFREE(loi); + struct btrfs_data_container AUTO_KVFREE(inodes); bool ignore_offset; if (!capable(CAP_SYS_ADMIN)) @@ -3249,41 +3285,32 @@ static long btrfs_ioctl_logical_to_ino(struct btrfs_fs_info *fs_info, size = min_t(u32, loi->size, SZ_64K); } else { /* All reserved bits must be 0 for now */ - if (memchr_inv(loi->reserved, 0, sizeof(loi->reserved))) { - ret = -EINVAL; - goto out_loi; - } + if (memchr_inv(loi->reserved, 0, sizeof(loi->reserved))) + return -EINVAL; + /* Only accept flags we have defined so far */ - if (loi->flags & ~(BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET)) { - ret = -EINVAL; - goto out_loi; - } + if (loi->flags & ~(BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET)) + return -EINVAL; + ignore_offset = loi->flags & BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET; size = min_t(u32, loi->size, SZ_16M); } inodes = init_data_container(size); - if (IS_ERR(inodes)) { - ret = PTR_ERR(inodes); - goto out_loi; - } + if (IS_ERR(inodes)) + return PTR_ERR(inodes); ret = iterate_inodes_from_logical(loi->logical, fs_info, inodes, ignore_offset); if (ret == -EINVAL) - ret = -ENOENT; + return -ENOENT; if (ret < 0) - goto out; + return ret; ret = copy_to_user((void __user *)(unsigned long)loi->inodes, inodes, size); if (ret) ret = -EFAULT; -out: - kvfree(inodes); -out_loi: - kfree(loi); - return ret; } @@ -3380,7 +3407,7 @@ static long btrfs_ioctl_balance(struct file *file, void __user *arg) { struct btrfs_root *root = BTRFS_I(file_inode(file))->root; struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_ioctl_balance_args *bargs; + struct btrfs_ioctl_balance_args AUTO_KFREE(bargs); struct btrfs_balance_control *bctl; bool need_unlock = true; int ret; @@ -3465,7 +3492,6 @@ out_unlock: btrfs_exclop_finish(fs_info); out: mnt_drop_write_file(file); - kfree(bargs); return ret; } @@ -3518,7 +3544,7 @@ static long btrfs_ioctl_quota_ctl(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_quota_ctl_args *sa; + struct btrfs_ioctl_quota_ctl_args AUTO_KFREE(sa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3577,7 +3603,6 @@ static long btrfs_ioctl_quota_ctl(struct file *file, void __user *arg) break; } - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3588,8 +3613,8 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); struct btrfs_root *root = BTRFS_I(inode)->root; - struct btrfs_ioctl_qgroup_assign_args *sa; - struct btrfs_qgroup_list *prealloc = NULL; + struct btrfs_ioctl_qgroup_assign_args AUTO_KFREE(sa); + struct btrfs_qgroup_list AUTO_KFREE(prealloc); struct btrfs_trans_handle *trans; int ret; int err; @@ -3614,7 +3639,7 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) prealloc = kzalloc_obj(*prealloc); if (!prealloc) { ret = -ENOMEM; - goto out; + goto drop_write; } } @@ -3622,7 +3647,7 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - goto out; + goto drop_write; } /* @@ -3648,9 +3673,6 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) if (err && !ret) ret = err; -out: - kfree(prealloc); - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3660,7 +3682,7 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_root *root = BTRFS_I(inode)->root; - struct btrfs_ioctl_qgroup_create_args *sa; + struct btrfs_ioctl_qgroup_create_args AUTO_KFREE(sa); struct btrfs_trans_handle *trans; int ret; int err; @@ -3683,12 +3705,12 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) if (!sa->qgroupid) { ret = -EINVAL; - goto out; + goto drop_write; } if (sa->create && btrfs_is_fstree(sa->qgroupid)) { ret = -EINVAL; - goto out; + goto drop_write; } /* @@ -3698,7 +3720,7 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - goto out; + goto drop_write; } if (sa->create) { @@ -3711,8 +3733,6 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) if (err && !ret) ret = err; -out: - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3722,7 +3742,7 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_root *root = BTRFS_I(inode)->root; - struct btrfs_ioctl_qgroup_limit_args *sa; + struct btrfs_ioctl_qgroup_limit_args AUTO_KFREE(sa); struct btrfs_trans_handle *trans; int ret; int err; @@ -3748,7 +3768,7 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) trans = btrfs_start_transaction(root, 1); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - goto out; + goto drop_write; } qgroupid = sa->qgroupid; @@ -3763,8 +3783,6 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) if (err && !ret) ret = err; -out: - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3774,7 +3792,7 @@ static long btrfs_ioctl_quota_rescan(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_quota_rescan_args *qsa; + struct btrfs_ioctl_quota_rescan_args AUTO_KFREE(qsa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3795,13 +3813,11 @@ static long btrfs_ioctl_quota_rescan(struct file *file, void __user *arg) if (qsa->flags) { ret = -EINVAL; - goto out; + goto drop_write; } ret = btrfs_qgroup_rescan(fs_info); -out: - kfree(qsa); drop_write: mnt_drop_write_file(file); return ret; @@ -3946,8 +3962,8 @@ out: static long btrfs_ioctl_set_received_subvol_32(struct file *file, void __user *arg) { - struct btrfs_ioctl_received_subvol_args_32 *args32 = NULL; - struct btrfs_ioctl_received_subvol_args *args64 = NULL; + struct btrfs_ioctl_received_subvol_args_32 AUTO_KFREE(args32); + struct btrfs_ioctl_received_subvol_args AUTO_KFREE(args64); int ret = 0; args32 = memdup_user(arg, sizeof(*args32)); @@ -3955,10 +3971,8 @@ static long btrfs_ioctl_set_received_subvol_32(struct file *file, return PTR_ERR(args32); args64 = kmalloc_obj(*args64); - if (!args64) { - ret = -ENOMEM; - goto out; - } + if (!args64) + return -ENOMEM; memcpy(args64->uuid, args32->uuid, BTRFS_UUID_SIZE); args64->stransid = args32->stransid; @@ -3971,7 +3985,7 @@ static long btrfs_ioctl_set_received_subvol_32(struct file *file, ret = _btrfs_ioctl_set_received_subvol(file, file_mnt_idmap(file), args64); if (ret) - goto out; + return ret; memcpy(args32->uuid, args64->uuid, BTRFS_UUID_SIZE); args32->stransid = args64->stransid; @@ -3984,19 +3998,16 @@ static long btrfs_ioctl_set_received_subvol_32(struct file *file, ret = copy_to_user(arg, args32, sizeof(*args32)); if (ret) - ret = -EFAULT; + return -EFAULT; -out: - kfree(args32); - kfree(args64); - return ret; + return 0; } #endif static long btrfs_ioctl_set_received_subvol(struct file *file, void __user *arg) { - struct btrfs_ioctl_received_subvol_args *sa = NULL; + struct btrfs_ioctl_received_subvol_args AUTO_KFREE(sa); int ret = 0; sa = memdup_user(arg, sizeof(*sa)); @@ -4004,17 +4015,14 @@ static long btrfs_ioctl_set_received_subvol(struct file *file, return PTR_ERR(sa); ret = _btrfs_ioctl_set_received_subvol(file, file_mnt_idmap(file), sa); - if (ret) - goto out; + return ret; ret = copy_to_user(arg, sa, sizeof(*sa)); if (ret) - ret = -EFAULT; + return -EFAULT; -out: - kfree(sa); - return ret; + return 0; } static int btrfs_ioctl_get_fslabel(struct btrfs_fs_info *fs_info, @@ -4254,11 +4262,11 @@ out_drop_write: static int _btrfs_ioctl_send(struct btrfs_root *root, void __user *argp, bool compat) { - struct btrfs_ioctl_send_args *arg; - int ret; + struct btrfs_ioctl_send_args AUTO_KFREE(arg); if (compat) { #if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + int ret; struct btrfs_ioctl_send_args_32 args32 = { 0 }; ret = copy_from_user(&args32, argp, sizeof(args32)); @@ -4283,9 +4291,7 @@ static int _btrfs_ioctl_send(struct btrfs_root *root, void __user *argp, bool co if (IS_ERR(arg)) return PTR_ERR(arg); } - ret = btrfs_ioctl_send(root, arg); - kfree(arg); - return ret; + return btrfs_ioctl_send(root, arg); } static int btrfs_ioctl_encoded_read(struct file *file, void __user *argp, @@ -4621,7 +4627,7 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); if (!pages) return -ENOMEM; - ret = btrfs_alloc_page_array(nr_pages, pages, 0); + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); if (ret) { ret = -ENOMEM; goto out_fail; @@ -5092,7 +5098,8 @@ static int btrfs_ioctl_subvol_sync(struct btrfs_fs_info *fs_info, void __user *a return -ENOENT; wait_for_deletion = true; - ASSERT(root_flags & BTRFS_ROOT_SUBVOL_DEAD); + ASSERT(root_flags & BTRFS_ROOT_SUBVOL_DEAD, "root_flags=0x%llx", + root_flags); sched_ret = schedule_timeout_interruptible(HZ); /* Early wake up or error. */ if (sched_ret != 0) @@ -5140,6 +5147,342 @@ static int btrfs_ioctl_shutdown(struct btrfs_fs_info *fs_info, unsigned long arg return ret; } +#define GET_CSUMS_BUF_MAX SZ_16M + +static int copy_csums_to_user(struct btrfs_fs_info *fs_info, u64 disk_bytenr, + u64 len, u8 __user *buf) +{ + struct btrfs_root *csum_root; + struct btrfs_ordered_sum *sums; + LIST_HEAD(list); + const u32 csum_size = fs_info->csum_size; + int ret; + + csum_root = btrfs_csum_root(fs_info, disk_bytenr); + if (unlikely(!csum_root)) { + btrfs_err(fs_info, "missing csum root for extent at bytenr %llu", disk_bytenr); + return -EUCLEAN; + } + + ret = btrfs_lookup_csums_list(csum_root, disk_bytenr, + disk_bytenr + len - 1, &list, false); + if (ret < 0) + return ret; + + ret = 0; + while (!list_empty(&list)) { + u64 offset; + size_t copy_size; + + sums = list_first_entry(&list, struct btrfs_ordered_sum, list); + list_del(&sums->list); + + offset = ((sums->logical - disk_bytenr) >> fs_info->sectorsize_bits) * csum_size; + copy_size = (sums->len >> fs_info->sectorsize_bits) * csum_size; + + if (copy_to_user(buf + offset, sums->sums, copy_size)) { + kfree(sums); + ret = -EFAULT; + goto out; + } + + kfree(sums); + } + +out: + while (!list_empty(&list)) { + sums = list_first_entry(&list, struct btrfs_ordered_sum, list); + list_del(&sums->list); + kfree(sums); + } + return ret; +} + +static int btrfs_ioctl_get_csums(struct file *file, void __user *argp) +{ + struct inode *vfs_inode = file_inode(file); + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_root *root = inode->root; + struct btrfs_ioctl_get_csums_args args; + BTRFS_PATH_AUTO_FREE(path); + const u64 ino = btrfs_ino(inode); + const u32 csum_size = fs_info->csum_size; + u8 __user *ubuf; + u64 buf_limit; + u64 buf_used = 0; + u64 cur_offset; + u64 end_offset; + u64 prev_extent_end; + struct btrfs_key key; + int ret; + + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + + if (!S_ISREG(vfs_inode->i_mode)) + return -EINVAL; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + if (!IS_ALIGNED(args.offset, fs_info->sectorsize) || + !IS_ALIGNED(args.length, fs_info->sectorsize)) + return -EINVAL; + if (args.length == 0) + return -EINVAL; + if (args.offset + args.length < args.offset) + return -EOVERFLOW; + if (args.flags != 0) + return -EINVAL; + if (args.buf_size < sizeof(struct btrfs_ioctl_get_csums_entry)) + return -EINVAL; + + buf_limit = min_t(u64, args.buf_size, GET_CSUMS_BUF_MAX); + ubuf = (u8 __user *)(argp + offsetof(struct btrfs_ioctl_get_csums_args, buf)); + + if (clear_user(ubuf, buf_limit)) + return -EFAULT; + + cur_offset = args.offset; + end_offset = args.offset + args.length; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_wait_ordered_range(inode, cur_offset, args.length); + if (ret) + return ret; + + ret = down_read_interruptible(&vfs_inode->i_rwsem); + if (ret) + return ret; + + ret = btrfs_wait_ordered_range(inode, cur_offset, args.length); + if (ret) + goto out_unlock; + + /* NODATASUM early exit. */ + if (inode->flags & BTRFS_INODE_NODATASUM) { + struct btrfs_ioctl_get_csums_entry entry = { + .offset = cur_offset, + .length = end_offset - cur_offset, + .type = BTRFS_GET_CSUMS_NODATASUM, + }; + + if (copy_to_user(ubuf, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + + buf_used = sizeof(entry); + cur_offset = end_offset; + goto done; + } + + prev_extent_end = cur_offset; + + while (cur_offset < end_offset) { + struct btrfs_file_extent_item *ei; + struct extent_buffer *leaf; + struct btrfs_ioctl_get_csums_entry entry = { 0 }; + u64 extent_end; + u64 disk_bytenr = 0; + u64 extent_offset = 0; + u64 range_start, range_len; + u64 entry_csum_size; + u64 key_offset; + int extent_type; + u8 compression; + u8 encryption; + + /* Search for the extent at or before cur_offset. */ + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = cur_offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out_unlock; + + if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) { + path->slots[0]--; + if (btrfs_file_extent_end(path) <= cur_offset) + path->slots[0]++; + } + } + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out_unlock; + if (ret > 0) { + ret = 0; + btrfs_release_path(path); + break; + } + } + + leaf = path->nodes[0]; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) { + btrfs_release_path(path); + break; + } + + extent_end = btrfs_file_extent_end(path); + key_offset = key.offset; + + /* Read extent fields before releasing the path. */ + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, ei); + compression = btrfs_file_extent_compression(leaf, ei); + encryption = btrfs_file_extent_encryption(leaf, ei); + + if (extent_type != BTRFS_FILE_EXTENT_INLINE) { + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + if (disk_bytenr && compression == BTRFS_COMPRESS_NONE) + extent_offset = btrfs_file_extent_offset(leaf, ei); + } + + btrfs_release_path(path); + + /* Implicit hole (NO_HOLES feature). */ + if (prev_extent_end < key_offset) { + u64 hole_end = min(key_offset, end_offset); + u64 hole_len = hole_end - prev_extent_end; + + if (prev_extent_end >= cur_offset) { + entry.offset = prev_extent_end; + entry.length = hole_len; + entry.type = BTRFS_GET_CSUMS_ZEROED; + + if (buf_used + sizeof(entry) > buf_limit) + goto done; + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + cur_offset = hole_end; + } + + if (key_offset >= end_offset) { + cur_offset = end_offset; + break; + } + } + + /* Clamp to our query range. */ + range_start = max(cur_offset, key_offset); + range_len = min(extent_end, end_offset) - range_start; + + entry.offset = range_start; + entry.length = range_len; + + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + entry.type = BTRFS_GET_CSUMS_INLINE; + if (compression != BTRFS_COMPRESS_NONE) + entry.type |= BTRFS_GET_CSUMS_COMPRESSED; + if (encryption != 0) + entry.type |= BTRFS_GET_CSUMS_ENCRYPTED; + entry_csum_size = 0; + } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + entry.type = BTRFS_GET_CSUMS_ZEROED; + entry_csum_size = 0; + } else { + /* BTRFS_FILE_EXTENT_REG */ + if (disk_bytenr == 0) { + /* Explicit hole. */ + entry.type = BTRFS_GET_CSUMS_ZEROED; + entry_csum_size = 0; + } else if (encryption != 0 || compression != BTRFS_COMPRESS_NONE) { + entry.type = 0; + if (encryption != 0) + entry.type |= BTRFS_GET_CSUMS_ENCRYPTED; + if (compression != BTRFS_COMPRESS_NONE) + entry.type |= BTRFS_GET_CSUMS_COMPRESSED; + entry_csum_size = 0; + } else { + entry.type = BTRFS_GET_CSUMS_HAS_CSUMS; + entry_csum_size = (range_len >> fs_info->sectorsize_bits) * csum_size; + } + } + + /* Check if this entry (+ csum data) fits in the buffer. */ + if (buf_used + sizeof(entry) + entry_csum_size > buf_limit) { + if (buf_used == 0) { + ret = -EOVERFLOW; + goto out_unlock; + } + goto done; + } + + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + + if (entry.type == BTRFS_GET_CSUMS_HAS_CSUMS) { + ret = copy_csums_to_user(fs_info, + disk_bytenr + extent_offset + (range_start - key_offset), + range_len, ubuf + buf_used); + if (ret) + goto out_unlock; + buf_used += entry_csum_size; + } + + cur_offset = range_start + range_len; + prev_extent_end = extent_end; + + if (fatal_signal_pending(current)) { + if (buf_used == 0) { + ret = -EINTR; + goto out_unlock; + } + goto done; + } + + cond_resched(); + } + + /* Handle trailing implicit hole. */ + if (cur_offset < end_offset) { + struct btrfs_ioctl_get_csums_entry entry = { + .offset = prev_extent_end, + .length = end_offset - prev_extent_end, + .type = BTRFS_GET_CSUMS_ZEROED, + }; + + if (buf_used + sizeof(entry) <= buf_limit) { + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + cur_offset = end_offset; + } + } + +done: + args.offset = cur_offset; + args.length = (cur_offset < end_offset) ? end_offset - cur_offset : 0; + args.buf_size = buf_used; + + if (copy_to_user(argp, &args, sizeof(args))) + ret = -EFAULT; + +out_unlock: + up_read(&vfs_inode->i_rwsem); + return ret; +} + long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -5273,6 +5616,10 @@ long btrfs_ioctl(struct file *file, unsigned int return btrfs_ioctl_set_features(file, argp); case BTRFS_IOC_GET_SUBVOL_INFO: return btrfs_ioctl_get_subvol_info(inode, argp); +#ifdef CONFIG_64BIT + case BTRFS_IOC_GET_SUBVOL_INFO_32: + return btrfs_ioctl_get_subvol_info_32(inode, argp); +#endif case BTRFS_IOC_GET_SUBVOL_ROOTREF: return btrfs_ioctl_get_subvol_rootref(root, argp); case BTRFS_IOC_INO_LOOKUP_USER: @@ -5297,6 +5644,8 @@ long btrfs_ioctl(struct file *file, unsigned int return btrfs_ioctl_subvol_sync(fs_info, argp); case BTRFS_IOC_SHUTDOWN: return btrfs_ioctl_shutdown(fs_info, arg); + case BTRFS_IOC_GET_CSUMS: + return btrfs_ioctl_get_csums(file, argp); } return -ENOTTY; diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c index 2de18c7b563a..1531adb117d1 100644 --- a/fs/btrfs/lzo.c +++ b/fs/btrfs/lzo.c @@ -491,6 +491,17 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb) return -EIO; } + /* The segment must not extend beyond the compressed input. */ + if (unlikely(cur_in + seg_len > compressed_len)) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(fs_info, + "lzo segment overflows compressed input, root %llu inode %llu offset %llu cur_in %u len %u compressed len %u", + btrfs_root_id(inode->root), btrfs_ino(inode), + cb->start, cur_in, seg_len, compressed_len); + return -EUCLEAN; + } + /* Copy the compressed segment payload into workspace */ copy_compressed_segment(cb, &fi, &cur_folio_index, workspace->cbuf, seg_len, &cur_in); @@ -541,17 +552,26 @@ int lzo_decompress(struct list_head *ws, const u8 *data_in, size_t max_segment_len = workspace_buf_length(fs_info); int ret; - if (unlikely(srclen < LZO_LEN || srclen > max_segment_len + LZO_LEN * 2)) + if (unlikely(srclen < LZO_LEN || srclen > max_segment_len + LZO_LEN * 2)) { + btrfs_err(fs_info, "invalid lzo header length, has %zu expect (%u, %zu)", + srclen, LZO_LEN, max_segment_len + LZO_LEN * 2); return -EUCLEAN; + } in_len = get_unaligned_le32(data_in); - if (unlikely(in_len != srclen)) + if (unlikely(in_len != srclen)) { + btrfs_err(fs_info, "invalid lzo header length, has %zu expect %zu", + in_len, srclen); return -EUCLEAN; + } data_in += LZO_LEN; in_len = get_unaligned_le32(data_in); - if (unlikely(in_len != srclen - LZO_LEN * 2)) + if (unlikely(in_len != srclen - LZO_LEN * 2)) { + btrfs_err(fs_info, "invalid lzo segment length, has %zu expect %zu", + in_len, srclen - LZO_LEN * 2); return -EUCLEAN; + } data_in += LZO_LEN; out_len = sectorsize; diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index e5a24b3ff95e..b32d4eabe0ab 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -157,7 +157,8 @@ static struct btrfs_ordered_extent *alloc_ordered_extent( ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))); /* Only one type flag can be set. */ - ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS)); + ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS), + "flags=0x%lx", flags); /* DIRECT cannot be set with COMPRESSED nor ENCODED. */ if (test_bit(BTRFS_ORDERED_DIRECT, &flags)) { @@ -302,7 +303,7 @@ struct btrfs_ordered_extent *btrfs_alloc_ordered_extent( { struct btrfs_ordered_extent *entry; - ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0); + ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0, "flags=0x%lx", flags); /* * For regular writes, we just use the members in @file_extent. @@ -357,6 +358,18 @@ void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered) mapping_set_error(ordered->inode->vfs_inode.i_mapping, -EIO); } +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len) +{ + struct btrfs_inode *inode = ordered->inode; + + ASSERT(truncate_len <= ordered->num_bytes); + spin_lock(&inode->ordered_tree_lock); + set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); + ordered->truncated_len = min(ordered->truncated_len, truncate_len); + spin_unlock(&inode->ordered_tree_lock); +} + static void finish_ordered_fn(struct btrfs_work *work) { struct btrfs_ordered_extent *ordered_extent; @@ -1238,7 +1251,7 @@ struct btrfs_ordered_extent *btrfs_split_ordered_extent( trace_btrfs_ordered_extent_split(inode, ordered); - ASSERT(!(flags & (1U << BTRFS_ORDERED_COMPRESSED))); + ASSERT(!(flags & (1U << BTRFS_ORDERED_COMPRESSED)), "flags=0x%lx", flags); /* * The entire bio must be covered by the ordered extent, but we can't @@ -1260,7 +1273,7 @@ struct btrfs_ordered_extent *btrfs_split_ordered_extent( } /* We cannot split partially completed ordered extents. */ if (ordered->bytes_left) { - ASSERT(!(flags & ~BTRFS_ORDERED_TYPE_FLAGS)); + ASSERT(!(flags & ~BTRFS_ORDERED_TYPE_FLAGS), "flags=0x%lx", flags); if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes)) return ERR_PTR(-EINVAL); } @@ -1307,7 +1320,8 @@ struct btrfs_ordered_extent *btrfs_split_ordered_extent( ordered->ram_bytes -= len; if (test_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags)) { - ASSERT(ordered->bytes_left == 0); + ASSERT(ordered->bytes_left == 0, "ordered->bytes_left=%llu", + ordered->bytes_left); new->bytes_left = 0; } else { ordered->bytes_left -= len; diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 03e12380a2fd..8d5d5ba1e02f 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -226,6 +226,8 @@ bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end, struct btrfs_ordered_extent *btrfs_split_ordered_extent( struct btrfs_ordered_extent *ordered, u64 len); void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered); +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len); int __init ordered_data_init(void); void __cold ordered_data_exit(void); diff --git a/fs/btrfs/print-tree.c b/fs/btrfs/print-tree.c index 87e60a2d4bd8..53e726119ca7 100644 --- a/fs/btrfs/print-tree.c +++ b/fs/btrfs/print-tree.c @@ -449,9 +449,9 @@ void btrfs_print_leaf(const struct extent_buffer *l) nr = btrfs_header_nritems(l); btrfs_info(fs_info, - "leaf %llu gen %llu total ptrs %d free space %d owner %llu", + "leaf %llu gen %llu total ptrs %d free space %d owner %lld", btrfs_header_bytenr(l), btrfs_header_generation(l), nr, - btrfs_leaf_free_space(l), btrfs_header_owner(l)); + btrfs_leaf_free_space(l), (s64)btrfs_header_owner(l)); print_eb_refs_lock(l); for (i = 0 ; i < nr ; i++) { char key_buf[KEY_TYPE_BUF_SIZE]; @@ -600,10 +600,10 @@ void btrfs_print_tree(const struct extent_buffer *c, bool follow) return; } btrfs_info(fs_info, - "node %llu level %d gen %llu total ptrs %d free spc %u owner %llu", + "node %llu level %d gen %llu total ptrs %d free spc %u owner %lld", btrfs_header_bytenr(c), level, btrfs_header_generation(c), nr, (u32)BTRFS_NODEPTRS_PER_BLOCK(fs_info) - nr, - btrfs_header_owner(c)); + (s64)btrfs_header_owner(c)); print_eb_refs_lock(c); for (i = 0; i < nr; i++) { btrfs_node_key_to_cpu(c, &key, i); diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c index adc956432d2f..bb77d46376d4 100644 --- a/fs/btrfs/props.c +++ b/fs/btrfs/props.c @@ -127,14 +127,24 @@ int btrfs_set_prop(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, return ret; } + ret = handler->validate(inode, value, value_len); + if (ret) + return ret; ret = btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, value, value_len, flags); if (ret) return ret; ret = handler->apply(inode, value, value_len); - if (ret) { - btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, NULL, - 0, flags); + /* We validated before, so it should not fail here. */ + ASSERT(ret == 0); + if (unlikely(ret)) { + int ret2; + + /* Try to delete xattr, if not possible abort transaction. */ + ret2 = btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, + NULL, 0, flags); + if (unlikely(ret2)) + btrfs_abort_transaction(trans, ret2); return ret; } diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index cdf736d3a4e5..502fb4a55cb2 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -373,10 +373,9 @@ static bool squota_check_parent_usage(struct btrfs_fs_info *fs_info, struct btrf parent->excl_cmpr != excl_cmpr_sum || parent->rfer_cmpr != rfer_cmpr_sum); WARN(mismatch, - "parent squota qgroup %hu/%llu has mismatched usage from its %d members. " + "parent squota qgroup " BTRFS_QGROUP_FMT " has mismatched usage from its %d members. " "%llu %llu %llu %llu vs %llu %llu %llu %llu\n", - btrfs_qgroup_level(parent->qgroupid), - btrfs_qgroup_subvolid(parent->qgroupid), nr_members, parent->excl, + BTRFS_QGROUP_FMT_VALUE(parent), nr_members, parent->excl, parent->rfer, parent->excl_cmpr, parent->rfer_cmpr, excl_sum, rfer_sum, excl_cmpr_sum, rfer_cmpr_sum); return mismatch; @@ -652,9 +651,8 @@ bool btrfs_check_quota_leak(const struct btrfs_fs_info *fs_info) if (qgroup->rsv.values[i]) { ret = true; btrfs_warn(fs_info, - "qgroup %hu/%llu has unreleased space, type %d rsv %llu", - btrfs_qgroup_level(qgroup->qgroupid), - btrfs_qgroup_subvolid(qgroup->qgroupid), + "qgroup " BTRFS_QGROUP_FMT " has unreleased space, type %d rsv %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), i, qgroup->rsv.values[i]); } } @@ -1107,7 +1105,13 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info, if (simple) { fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE; btrfs_set_fs_incompat(fs_info, SIMPLE_QUOTA); - btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid); + /* + * Set the enable generation to the next transaction, as we cannot + * ensure that extents written during this transaction will see any + * state we have set here. So we should treat all extents of the + * transaction as coming in before squotas was enabled. + */ + btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid + 1); } else { fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; } @@ -1210,7 +1214,15 @@ out_add_root: goto out_free_path; } - fs_info->qgroup_enable_gen = trans->transid; + /* + * Set fs_info->qgroup_enable_gen and BTRFS_FS_SQUOTA_ENABLING + * under the transaction handle. We want to ensure that all extents in + * the next transaction definitely see them. + */ + if (simple) { + fs_info->qgroup_enable_gen = trans->transid + 1; + set_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags); + } mutex_unlock(&fs_info->qgroup_ioctl_lock); /* @@ -1224,9 +1236,15 @@ out_add_root: */ ret = btrfs_commit_transaction(trans); trans = NULL; + mutex_lock(&fs_info->qgroup_ioctl_lock); - if (ret) + if (ret) { + if (simple) { + clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags); + fs_info->qgroup_enable_gen = 0; + } goto out_free_path; + } /* * Set quota enabled flag after committing the transaction, to avoid @@ -1236,6 +1254,8 @@ out_add_root: spin_lock(&fs_info->qgroup_lock); fs_info->quota_root = quota_root; set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + if (simple) + clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags); spin_unlock(&fs_info->qgroup_lock); /* Skip rescan for simple qgroups. */ @@ -1715,32 +1735,24 @@ out: return ret; } -static bool can_delete_parent_qgroup(struct btrfs_qgroup *qgroup) - +static bool can_delete_parent_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup) { ASSERT(btrfs_qgroup_level(qgroup->qgroupid)); + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + squota_check_parent_usage(fs_info, qgroup); return list_empty(&qgroup->members); } /* - * Return true if we can delete the squota qgroup and false otherwise. - * - * Rules for whether we can delete: - * - * A subvolume qgroup can be removed iff the subvolume is fully deleted, which - * is iff there is 0 usage in the qgroup. - * - * A higher level qgroup can be removed iff it has no members. - * Note: We audit its usage to warn on inconsitencies without blocking deletion. + * Because a shared extent can outlive its owning subvolume, we cannot delete a + * subvol squota qgroup until all of the extents it owns are gone, even if the + * subvolume itself has been deleted. */ -static bool can_delete_squota_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup) +static bool can_delete_squota_subvol_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup) { ASSERT(btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE); - - if (btrfs_qgroup_level(qgroup->qgroupid) > 0) { - squota_check_parent_usage(fs_info, qgroup); - return can_delete_parent_qgroup(qgroup); - } + ASSERT(btrfs_qgroup_level(qgroup->qgroupid) == 0); return !(qgroup->rfer || qgroup->excl || qgroup->rfer_cmpr || qgroup->excl_cmpr); } @@ -1754,14 +1766,11 @@ static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup { struct btrfs_key key; BTRFS_PATH_AUTO_FREE(path); - - /* Since squotas cannot be inconsistent, they have special rules for deletion. */ - if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) - return can_delete_squota_qgroup(fs_info, qgroup); + int ret; /* For higher level qgroup, we can only delete it if it has no child. */ if (btrfs_qgroup_level(qgroup->qgroupid)) - return can_delete_parent_qgroup(qgroup); + return can_delete_parent_qgroup(fs_info, qgroup); /* * For level-0 qgroups, we can only delete it if it has no subvolume @@ -1777,10 +1786,21 @@ static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup return -ENOMEM; /* - * The @ret from btrfs_find_root() exactly matches our definition for - * the return value, thus can be returned directly. + * Any subvol qgroup, regardless of mode, cannot be deleted if the + * subvol still exists. */ - return btrfs_find_root(fs_info->tree_root, &key, path, NULL, NULL); + ret = btrfs_find_root(fs_info->tree_root, &key, path, NULL, NULL); + /* + * btrfs_find_root returns <0 on error, 0 if found, and >0 if not, + * so the "found" and "error" cases match our desired return values. + */ + if (ret <= 0) + return ret; + + /* Squotas require additional checks, even if the subvol is deleted. */ + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + return can_delete_squota_subvol_qgroup(fs_info, qgroup); + return 1; } int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) @@ -1836,14 +1856,13 @@ int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) * Thus its reserved space should all be zero, no matter if qgroup * is consistent or the mode. */ - if (qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA] || - qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC] || - qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS]) { + if (unlikely(qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA] || + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC] || + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS])) { DEBUG_WARN(); btrfs_warn_rl(fs_info, -"to be deleted qgroup %u/%llu has non-zero numbers, data %llu meta prealloc %llu meta pertrans %llu", - btrfs_qgroup_level(qgroup->qgroupid), - btrfs_qgroup_subvolid(qgroup->qgroupid), +"to be deleted qgroup " BTRFS_QGROUP_FMT " has non-zero numbers, data %llu meta prealloc %llu meta pertrans %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA], qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC], qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS]); @@ -1857,13 +1876,12 @@ int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) */ if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL && !(fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT)) { - if (qgroup->rfer || qgroup->excl || - qgroup->rfer_cmpr || qgroup->excl_cmpr) { + if (unlikely(qgroup->rfer || qgroup->excl || + qgroup->rfer_cmpr || qgroup->excl_cmpr)) { DEBUG_WARN(); qgroup_mark_inconsistent(fs_info, - "to be deleted qgroup %u/%llu has non-zero numbers, rfer %llu rfer_cmpr %llu excl %llu excl_cmpr %llu", - btrfs_qgroup_level(qgroup->qgroupid), - btrfs_qgroup_subvolid(qgroup->qgroupid), +"to be deleted qgroup " BTRFS_QGROUP_FMT " has non-zero numbers, rfer %llu rfer_cmpr %llu excl %llu excl_cmpr %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), qgroup->rfer, qgroup->rfer_cmpr, qgroup->excl, qgroup->excl_cmpr); } @@ -4800,9 +4818,9 @@ int btrfs_qgroup_add_swapped_blocks(struct btrfs_root *subvol_root, entry = rb_entry(node, struct btrfs_qgroup_swapped_block, node); - if (entry->subvol_generation != block->subvol_generation || - entry->reloc_bytenr != block->reloc_bytenr || - entry->reloc_generation != block->reloc_generation) { + if (unlikely(entry->subvol_generation != block->subvol_generation || + entry->reloc_bytenr != block->reloc_bytenr || + entry->reloc_generation != block->reloc_generation)) { /* * Duplicated but mismatch entry found. Shouldn't happen. * Marking qgroup inconsistent should be enough for end @@ -4922,7 +4940,8 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info, u64 num_bytes = delta->num_bytes; const int sign = (delta->is_inc ? 1 : -1); - if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE) + if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE && + !test_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags)) return 0; if (!btrfs_is_fstree(root)) @@ -4934,8 +4953,9 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info, spin_lock(&fs_info->qgroup_lock); qgroup = find_qgroup_rb(fs_info, root); - if (!qgroup) { - ret = -ENOENT; + if (WARN_ON_ONCE(!qgroup)) { + btrfs_warn(fs_info, "squota failed to find qgroup for root %llu", root); + ret = 0; goto out; } @@ -4944,8 +4964,18 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info, list_for_each_entry(qg, &qgroup_list, iterator) { struct btrfs_qgroup_list *glist; - qg->excl += num_bytes * sign; - qg->rfer += num_bytes * sign; + ASSERT(qg->excl == qg->rfer); + if (WARN_ON_ONCE(sign < 0 && qg->excl < num_bytes)) { + btrfs_warn(fs_info, + "squota underflow qg " BTRFS_QGROUP_FMT " excl %llu num_bytes %llu", + BTRFS_QGROUP_FMT_VALUE(qg), + qg->excl, num_bytes); + qg->excl = 0; + qg->rfer = 0; + } else { + qg->excl += num_bytes * sign; + qg->rfer += num_bytes * sign; + } qgroup_dirty(fs_info, qg); list_for_each_entry(glist, &qg->groups, next_group) diff --git a/fs/btrfs/raid-stripe-tree.c b/fs/btrfs/raid-stripe-tree.c index 4b0186c83ad1..454a95bf542a 100644 --- a/fs/btrfs/raid-stripe-tree.c +++ b/fs/btrfs/raid-stripe-tree.c @@ -272,7 +272,9 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le &key, key.offset - length, length); - ASSERT(key.offset - diff_end == length); + ASSERT(key.offset - diff_end == length, + "key.offset=%llu diff_end=%llu length=%llu", + key.offset, diff_end, length); break; } diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 08ee8f316d96..00a01b97cc0c 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -466,7 +466,7 @@ static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio) int bucket = rbio_bucket(rbio); struct btrfs_stripe_hash_table *table; struct btrfs_stripe_hash *h; - int freeit = 0; + bool freeit = false; /* * check the bit again under the hash table lock. @@ -491,7 +491,7 @@ static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio) if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) { list_del_init(&rbio->stripe_cache); table->cache_size -= 1; - freeit = 1; + freeit = true; /* if the bio list isn't empty, this rbio is * still involved in an IO. We take it out @@ -855,7 +855,7 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio) { int bucket; struct btrfs_stripe_hash *h; - int keep_cache = 0; + bool keep_cache = false; bucket = rbio_bucket(rbio); h = rbio->bioc->fs_info->stripe_hash_table->table + bucket; @@ -874,7 +874,7 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio) */ if (list_empty(&rbio->plug_list) && test_bit(RBIO_CACHE_BIT, &rbio->flags)) { - keep_cache = 1; + keep_cache = true; clear_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); BUG_ON(!bio_list_empty(&rbio->bio_list)); goto done; @@ -1123,7 +1123,7 @@ static int alloc_rbio_pages(struct btrfs_raid_bio *rbio) { int ret; - ret = btrfs_alloc_page_array(rbio->nr_pages, rbio->stripe_pages, false); + ret = btrfs_alloc_page_array(rbio->nr_pages, rbio->stripe_pages, GFP_NOFS); if (ret < 0) return ret; /* Mapping all sectors */ @@ -1138,7 +1138,7 @@ static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio) int ret; ret = btrfs_alloc_page_array(rbio->nr_pages - data_pages, - rbio->stripe_pages + data_pages, false); + rbio->stripe_pages + data_pages, GFP_NOFS); if (ret < 0) return ret; @@ -1410,7 +1410,7 @@ static void generate_pq_vertical_step(struct btrfs_raid_bio *rbio, unsigned int rbio_qstripe_paddr(rbio, sector_nr, step_nr)); assert_rbio(rbio); - raid6_call.gen_syndrome(rbio->real_stripes, step, pointers); + raid6_gen_syndrome(rbio->real_stripes, step, pointers); } else { /* raid5 */ memcpy(pointers[rbio->nr_data], pointers[0], step); @@ -1732,7 +1732,7 @@ static int alloc_rbio_data_pages(struct btrfs_raid_bio *rbio) const int data_pages = rbio->nr_data * rbio->stripe_npages; int ret; - ret = btrfs_alloc_page_array(data_pages, rbio->stripe_pages, false); + ret = btrfs_alloc_page_array(data_pages, rbio->stripe_pages, GFP_NOFS); if (ret < 0) return ret; @@ -1987,10 +1987,10 @@ static void recover_vertical_step(struct btrfs_raid_bio *rbio, } if (failb == rbio->real_stripes - 2) { - raid6_datap_recov(rbio->real_stripes, step, + raid6_recov_datap(rbio->real_stripes, step, faila, pointers); } else { - raid6_2data_recov(rbio->real_stripes, step, + raid6_recov_2data(rbio->real_stripes, step, faila, failb, pointers); } } else { @@ -2644,7 +2644,7 @@ static bool verify_one_parity_step(struct btrfs_raid_bio *rbio, if (has_qstripe) { assert_rbio(rbio); /* RAID6, call the library function to fill in our P/Q. */ - raid6_call.gen_syndrome(rbio->real_stripes, step, pointers); + raid6_gen_syndrome(rbio->real_stripes, step, pointers); } else { /* RAID5. */ memcpy(pointers[nr_data], pointers[0], step); @@ -2695,7 +2695,7 @@ static int finish_parity_scrub(struct btrfs_raid_bio *rbio) phys_addr_t p_paddr = INVALID_PADDR; phys_addr_t q_paddr = INVALID_PADDR; struct bio_list bio_list; - int is_replace = 0; + bool is_replace = false; int ret; bio_list_init(&bio_list); @@ -2712,7 +2712,7 @@ static int finish_parity_scrub(struct btrfs_raid_bio *rbio) * need to duplicate the final write to replace target. */ if (bioc->replace_nr_stripes && bioc->replace_stripe_src == rbio->scrubp) { - is_replace = 1; + is_replace = true; bitmap_copy(pbitmap, &rbio->dbitmap, rbio->stripe_nsectors); } diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 49865a463780..9a49d2ecb949 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -69,7 +69,8 @@ static int copy_inline_to_page(struct btrfs_inode *inode, struct address_space *mapping = inode->vfs_inode.i_mapping; int ret; - ASSERT(IS_ALIGNED(file_offset, block_size)); + ASSERT(IS_ALIGNED(file_offset, block_size), "file_offset=%llu block_size=%u", + file_offset, block_size); /* * We have flushed and locked the ranges of the source and destination @@ -94,9 +95,7 @@ static int copy_inline_to_page(struct btrfs_inode *inode, if (ret < 0) goto out_unlock; - btrfs_clear_extent_bit(&inode->io_tree, file_offset, range_end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, NULL); - ret = btrfs_set_extent_delalloc(inode, file_offset, range_end, 0, NULL); + ret = btrfs_reset_extent_delalloc(inode, file_offset, range_end, 0, NULL); if (ret) goto out_unlock; @@ -141,7 +140,6 @@ static int copy_inline_to_page(struct btrfs_inode *inode, folio_zero_range(folio, datal, block_size - datal); btrfs_folio_set_uptodate(fs_info, folio, file_offset, block_size); - btrfs_folio_clear_checked(fs_info, folio, file_offset, block_size); btrfs_folio_set_dirty(fs_info, folio, file_offset, block_size); out_unlock: if (!IS_ERR(folio)) { @@ -181,10 +179,12 @@ static int clone_copy_inline_extent(struct btrfs_inode *inode, struct btrfs_drop_extents_args drop_args = { 0 }; int ret; struct btrfs_key key; + bool copied_inline_to_page = false; if (new_key->offset > 0) { ret = copy_inline_to_page(inode, new_key->offset, inline_data, size, datal, comp_type); + copied_inline_to_page = (ret == 0); goto out; } @@ -290,6 +290,60 @@ copy_inline_extent: btrfs_abort_transaction(trans, ret); out: if (!ret && !trans) { + if (copied_inline_to_page && + new_key->offset + datal > i_size_read(&inode->vfs_inode)) { + /* + * If we copied the inline extent data to a page/folio + * beyond the i_size of the destination inode, then we + * need to increase the i_size before we start a + * transaction to update the inode item. This is to + * prevent a deadlock when the flushoncommit mount + * option is used, which happens like this: + * + * 1) Task A clones an inline extent from inode X to an + * offset of inode Y that is beyond Y's current + * i_size. This means we copied the inline extent's + * data to a folio of inode Y that is beyond its EOF, + * using the call above to copy_inline_to_page(); + * + * 2) Task B starts a transaction commit and calls + * btrfs_start_delalloc_flush() to flush delalloc; + * + * 3) The delalloc flushing sees the new dirty folio of + * inode Y and when it attempts to flush it, it ends + * up at extent_writepage() and sees that the offset + * of the folio is beyond the i_size of inode Y, so + * it attempts to invalidate the folio by calling + * folio_invalidate(), which ends up at btrfs' folio + * invalidate callback - btrfs_invalidate_folio(). + * There it tries to lock the folio's range in inode + * Y's extent io tree, but it blocks since it's + * currently locked by task A - during reflink we + * lock the inodes and the source and destination + * ranges after flushing all delalloc and waiting for + * ordered extent completion - after that we don't + * expect to have dirty folios in the ranges, the + * exception is if we have to copy an inline extent's + * data (because the destination offset is not zero); + * + * 4) Task A then does the 'goto out' below and attempts + * to start a transaction to update the inode item, + * and then it's blocked since the current + * transaction is in the TRANS_STATE_COMMIT_START + * state. Therefore task A has to wait for the + * current transaction to become unblocked (its + * state >= TRANS_STATE_UNBLOCKED). + * + * This leads to a deadlock - the task committing the + * transaction waiting for the delalloc flushing which + * is blocked during folio invalidation on the inode's + * extent lock and the reflink task waiting for the + * current transaction to be unblocked so that it can + * start a new one to update the inode item (while + * holding the extent lock). + */ + i_size_write(&inode->vfs_inode, new_key->offset + datal); + } /* * No transaction here means we copied the inline extent into a * page of the destination inode. @@ -322,50 +376,7 @@ copy_to_page: ret = copy_inline_to_page(inode, new_key->offset, inline_data, size, datal, comp_type); - - /* - * If we copied the inline extent data to a page/folio beyond the i_size - * of the destination inode, then we need to increase the i_size before - * we start a transaction to update the inode item. This is to prevent a - * deadlock when the flushoncommit mount option is used, which happens - * like this: - * - * 1) Task A clones an inline extent from inode X to an offset of inode - * Y that is beyond Y's current i_size. This means we copied the - * inline extent's data to a folio of inode Y that is beyond its EOF, - * using the call above to copy_inline_to_page(); - * - * 2) Task B starts a transaction commit and calls - * btrfs_start_delalloc_flush() to flush delalloc; - * - * 3) The delalloc flushing sees the new dirty folio of inode Y and when - * it attempts to flush it, it ends up at extent_writepage() and sees - * that the offset of the folio is beyond the i_size of inode Y, so - * it attempts to invalidate the folio by calling folio_invalidate(), - * which ends up at btrfs' folio invalidate callback - - * btrfs_invalidate_folio(). There it tries to lock the folio's range - * in inode Y's extent io tree, but it blocks since it's currently - * locked by task A - during reflink we lock the inodes and the - * source and destination ranges after flushing all delalloc and - * waiting for ordered extent completion - after that we don't expect - * to have dirty folios in the ranges, the exception is if we have to - * copy an inline extent's data (because the destination offset is - * not zero); - * - * 4) Task A then does the 'goto out' below and attempts to start a - * transaction to update the inode item, and then it's blocked since - * the current transaction is in the TRANS_STATE_COMMIT_START state. - * Therefore task A has to wait for the current transaction to become - * unblocked (its state >= TRANS_STATE_UNBLOCKED). - * - * This leads to a deadlock - the task committing the transaction - * waiting for the delalloc flushing which is blocked during folio - * invalidation on the inode's extent lock and the reflink task waiting - * for the current transaction to be unblocked so that it can start a - * a new one to update the inode item (while holding the extent lock). - */ - if (ret == 0 && new_key->offset + datal > i_size_read(&inode->vfs_inode)) - i_size_write(&inode->vfs_inode, new_key->offset + datal); + copied_inline_to_page = (ret == 0); goto out; } @@ -459,7 +470,7 @@ process_slot: key.objectid != btrfs_ino(BTRFS_I(src))) break; - ASSERT(key.type == BTRFS_EXTENT_DATA_KEY); + ASSERT(key.type == BTRFS_EXTENT_DATA_KEY, "key.type=%u", key.type); extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 3ebaf5880125..0d63d117db59 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -178,8 +178,101 @@ struct reloc_control { bool create_reloc_tree; bool merge_reloc_tree; bool found_file_extent; + + refcount_t refs; }; +static struct reloc_control *get_reloc_control(struct btrfs_fs_info *fs_info) +{ + struct reloc_control *rc; + + /* Quick path, avoid lock contention on fs_info->reloc_ctl_lock. */ + if (!data_race(fs_info->reloc_ctl)) + return NULL; + + spin_lock(&fs_info->reloc_ctl_lock); + rc = fs_info->reloc_ctl; + if (rc) + refcount_inc(&rc->refs); + spin_unlock(&fs_info->reloc_ctl_lock); + + return rc; +} + +static void __del_reloc_root(struct btrfs_root *root); + +static noinline_for_stack void free_reloc_roots(struct list_head *list) +{ + struct btrfs_root *reloc_root, *tmp; + + list_for_each_entry_safe(reloc_root, tmp, list, root_list) + __del_reloc_root(reloc_root); +} + +static void put_reloc_control(struct reloc_control *rc) +{ + if (refcount_dec_and_test(&rc->refs)) { + struct mapping_node *node, *tmp; + + if (rc->extent_root) + ASSERT(rc->extent_root->fs_info->reloc_ctl != rc); + + free_reloc_roots(&rc->reloc_roots); + rbtree_postorder_for_each_entry_safe(node, tmp, + &rc->reloc_root_tree.rb_root, + rb_node) + kfree(node); + + if (rc->block_group) + btrfs_put_block_group(rc->block_group); + + kfree(rc); + } +} + +/* Helper to delete the 'address of tree root -> reloc tree' mapping. */ +static void __del_reloc_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *rb_node; + struct mapping_node AUTO_KFREE(node); + struct reloc_control *rc; + bool put_ref = false; + + rc = get_reloc_control(fs_info); + if (rc && root->node) { + spin_lock(&rc->reloc_root_tree.lock); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, + root->commit_root->start); + if (rb_node) { + node = rb_entry(rb_node, struct mapping_node, rb_node); + rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); + RB_CLEAR_NODE(&node->rb_node); + } + spin_unlock(&rc->reloc_root_tree.lock); + ASSERT(!node || (struct btrfs_root *)node->data == root); + } + + /* + * We only put the reloc root here if it's on the list. There's a lot + * of places where the pattern is to splice the rc->reloc_roots, process + * the reloc roots, and then add the reloc root back onto + * rc->reloc_roots. If we call __del_reloc_root while it's off of the + * list we don't want the reference being dropped, because the guy + * messing with the list is in charge of the reference. + */ + spin_lock(&fs_info->trans_lock); + if (!list_empty(&root->root_list)) { + put_ref = true; + list_del_init(&root->root_list); + } + spin_unlock(&fs_info->trans_lock); + if (put_ref) + btrfs_put_root(root); + if (rc) + put_reloc_control(rc); +} + static void mark_block_processed(struct reloc_control *rc, struct btrfs_backref_node *node) { @@ -407,7 +500,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( struct reloc_control *rc, struct btrfs_key *node_key, int level, u64 bytenr) { - struct btrfs_backref_iter *iter; + struct btrfs_backref_iter iter; struct btrfs_backref_cache *cache = &rc->backref_cache; /* For searching parent of TREE_BLOCK_REF */ struct btrfs_path *path; @@ -416,9 +509,9 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( struct btrfs_backref_edge *edge; int ret; - iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info); - if (!iter) - return ERR_PTR(-ENOMEM); + ret = btrfs_backref_iter_init(&iter); + if (ret < 0) + return ERR_PTR(ret); path = btrfs_alloc_path(); if (!path) { ret = -ENOMEM; @@ -435,7 +528,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( /* Breadth-first search to build backref cache */ do { - ret = btrfs_backref_add_tree_node(trans, cache, path, iter, + ret = btrfs_backref_add_tree_node(trans, cache, path, &iter, node_key, cur); if (ret < 0) goto out; @@ -460,8 +553,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( if (handle_useless_nodes(rc, node)) node = NULL; out: - btrfs_free_path(iter->path); - kfree(iter); + btrfs_free_path(iter.path); btrfs_free_path(path); if (ret) { btrfs_backref_error_cleanup(cache, node); @@ -476,12 +568,11 @@ out: /* * helper to add 'address of tree root -> reloc tree' mapping */ -static int __add_reloc_root(struct btrfs_root *root) +static int __add_reloc_root(struct btrfs_root *root, struct reloc_control *rc) { struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *rb_node; struct mapping_node *node; - struct reloc_control *rc = fs_info->reloc_ctl; node = kmalloc_obj(*node, GFP_NOFS); if (!node) @@ -505,49 +596,6 @@ static int __add_reloc_root(struct btrfs_root *root) } /* - * helper to delete the 'address of tree root -> reloc tree' - * mapping - */ -static void __del_reloc_root(struct btrfs_root *root) -{ - struct btrfs_fs_info *fs_info = root->fs_info; - struct rb_node *rb_node; - struct mapping_node AUTO_KFREE(node); - struct reloc_control *rc = fs_info->reloc_ctl; - bool put_ref = false; - - if (rc && root->node) { - spin_lock(&rc->reloc_root_tree.lock); - rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, - root->commit_root->start); - if (rb_node) { - node = rb_entry(rb_node, struct mapping_node, rb_node); - rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); - RB_CLEAR_NODE(&node->rb_node); - } - spin_unlock(&rc->reloc_root_tree.lock); - ASSERT(!node || (struct btrfs_root *)node->data == root); - } - - /* - * We only put the reloc root here if it's on the list. There's a lot - * of places where the pattern is to splice the rc->reloc_roots, process - * the reloc roots, and then add the reloc root back onto - * rc->reloc_roots. If we call __del_reloc_root while it's off of the - * list we don't want the reference being dropped, because the guy - * messing with the list is in charge of the reference. - */ - spin_lock(&fs_info->trans_lock); - if (!list_empty(&root->root_list)) { - put_ref = true; - list_del_init(&root->root_list); - } - spin_unlock(&fs_info->trans_lock); - if (put_ref) - btrfs_put_root(root); -} - -/* * helper to update the 'address of tree root -> reloc tree' * mapping */ @@ -590,7 +638,7 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_key root_key; int ret = 0; - root_item = kmalloc(sizeof(*root_item), GFP_NOFS); + root_item = kmalloc_obj(*root_item, GFP_NOFS); if (!root_item) return ERR_PTR(-ENOMEM); @@ -671,21 +719,19 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, ret = btrfs_insert_root(trans, fs_info->tree_root, &root_key, root_item); - if (ret) - goto abort; + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ERR_PTR(ret); + } reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key); if (IS_ERR(reloc_root)) { - ret = PTR_ERR(reloc_root); - goto abort; + btrfs_abort_transaction(trans, PTR_ERR(reloc_root)); + return ERR_CAST(reloc_root); } set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); btrfs_set_root_last_trans(reloc_root, trans->transid); return reloc_root; - -abort: - btrfs_abort_transaction(trans, ret); - return ERR_PTR(ret); } /* @@ -700,11 +746,12 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; - struct reloc_control *rc = fs_info->reloc_ctl; + struct reloc_control *rc; struct btrfs_block_rsv *rsv; - int clear_rsv = 0; - int ret; + bool clear_rsv = false; + int ret = 0; + rc = get_reloc_control(fs_info); if (!rc) return 0; @@ -713,7 +760,7 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, * create/update the dead reloc tree */ if (reloc_root_is_dead(root)) - return 0; + goto out; /* * This is subtle but important. We do not do @@ -724,9 +771,8 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, * in. */ if (root->reloc_root) { - reloc_root = root->reloc_root; - btrfs_set_root_last_trans(reloc_root, trans->transid); - return 0; + btrfs_set_root_last_trans(root->reloc_root, trans->transid); + goto out; } /* @@ -734,28 +780,33 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, * reloc trees never need their own reloc tree. */ if (!rc->create_reloc_tree || btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) - return 0; + goto out; if (!trans->reloc_reserved) { rsv = trans->block_rsv; trans->block_rsv = rc->block_rsv; - clear_rsv = 1; + clear_rsv = true; } reloc_root = create_reloc_root(trans, root, btrfs_root_id(root)); if (clear_rsv) trans->block_rsv = rsv; - if (IS_ERR(reloc_root)) - return PTR_ERR(reloc_root); + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto out; + } - ret = __add_reloc_root(reloc_root); + ret = __add_reloc_root(reloc_root, rc); ASSERT(ret != -EEXIST); if (ret) { /* Pairs with create_reloc_root */ btrfs_put_root(reloc_root); - return ret; + goto out; } root->reloc_root = btrfs_grab_root(reloc_root); - return 0; +out: + put_reloc_control(rc); + + return ret; } /* @@ -767,6 +818,7 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; struct btrfs_root_item *root_item; + struct reloc_control *rc; int ret; if (!have_reloc_root(root)) @@ -782,9 +834,9 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, */ btrfs_grab_root(reloc_root); + rc = get_reloc_control(fs_info); /* root->reloc_root will stay until current relocation finished */ - if (fs_info->reloc_ctl && fs_info->reloc_ctl->merge_reloc_tree && - btrfs_root_refs(root_item) == 0) { + if (rc && rc->merge_reloc_tree && btrfs_root_refs(root_item) == 0) { set_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); /* * Mark the tree as dead before we change reloc_root so @@ -804,6 +856,9 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, ret = btrfs_update_root(trans, fs_info->tree_root, &reloc_root->root_key, root_item); btrfs_put_root(reloc_root); + if (rc) + put_reloc_control(rc); + return ret; } @@ -814,6 +869,7 @@ static int get_new_location(struct inode *reloc_inode, u64 *new_bytenr, u64 bytenr, u64 num_bytes) { struct btrfs_root *root = BTRFS_I(reloc_inode)->root; + struct btrfs_fs_info *fs_info = root->fs_info; BTRFS_PATH_AUTO_FREE(path); struct btrfs_file_extent_item *fi; struct extent_buffer *leaf; @@ -835,10 +891,23 @@ static int get_new_location(struct inode *reloc_inode, u64 *new_bytenr, fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); - BUG_ON(btrfs_file_extent_offset(leaf, fi) || - btrfs_file_extent_compression(leaf, fi) || - btrfs_file_extent_encryption(leaf, fi) || - btrfs_file_extent_other_encoding(leaf, fi)); + /* + * The cluster-boundary key searched above is always written by + * relocation with offset 0: either by insert_prealloc_file_extent() + * (memsets the stack item to 0) or by the front portion of a partial + * writeback (offset=0 by construction). A non-zero value here means + * the on-disk leaf does not match what relocation wrote, i.e. + * corruption. The other encoding fields are caught earlier by + * tree-checker's check_extent_data_item(). + */ + if (unlikely(btrfs_file_extent_offset(leaf, fi))) { + btrfs_print_leaf(leaf); + btrfs_err(fs_info, +"unexpected non-zero offset in file extent item for data reloc inode %llu key offset %llu offset %llu", + btrfs_ino(BTRFS_I(reloc_inode)), bytenr, + btrfs_file_extent_offset(leaf, fi)); + return -EUCLEAN; + } if (num_bytes != btrfs_file_extent_disk_num_bytes(leaf, fi)) return -EINVAL; @@ -869,7 +938,7 @@ int replace_file_extents(struct btrfs_trans_handle *trans, u32 nritems; u32 i; int ret = 0; - int first = 1; + bool first = true; if (rc->stage != UPDATE_DATA_PTRS) return 0; @@ -907,7 +976,7 @@ int replace_file_extents(struct btrfs_trans_handle *trans, if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID) { if (first) { inode = btrfs_find_first_inode(root, key.objectid); - first = 0; + first = false; } else if (inode && btrfs_ino(inode) < key.objectid) { btrfs_add_delayed_iput(inode); inode = btrfs_find_first_inode(root, key.objectid); @@ -1021,7 +1090,7 @@ int replace_path(struct btrfs_trans_handle *trans, struct reloc_control *rc, u64 new_ptr_gen; u64 last_snapshot; u32 blocksize; - int cow = 0; + bool cow = false; int level; int ret; int slot; @@ -1127,7 +1196,7 @@ again: if (!cow) { btrfs_tree_unlock(parent); free_extent_buffer(parent); - cow = 1; + cow = true; goto again; } @@ -1515,7 +1584,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, int reserve_level; int level; int max_level; - int replaced = 0; + bool replaced = false; int ret = 0; u32 min_reserved; @@ -1561,7 +1630,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, * and * 2 since we have two trees to COW. */ reserve_level = max_t(int, 1, btrfs_root_level(root_item)); - min_reserved = fs_info->nodesize * reserve_level * 2; + min_reserved = (reserve_level << fs_info->nodesize_bits) * 2; memset(&next_key, 0, sizeof(next_key)); while (1) { @@ -1590,7 +1659,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, btrfs_set_root_last_trans(reloc_root, trans->transid); trans->block_rsv = rc->block_rsv; - replaced = 0; + replaced = false; max_level = level; ret = walk_down_reloc_tree(reloc_root, path, &level); @@ -1612,7 +1681,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, level = ret; btrfs_node_key_to_cpu(path->nodes[level], &key, path->slots[level]); - replaced = 1; + replaced = true; } ret = walk_up_reloc_tree(reloc_root, path, &level); @@ -1795,22 +1864,13 @@ again: } static noinline_for_stack -void free_reloc_roots(struct list_head *list) -{ - struct btrfs_root *reloc_root, *tmp; - - list_for_each_entry_safe(reloc_root, tmp, list, root_list) - __del_reloc_root(reloc_root); -} - -static noinline_for_stack void merge_reloc_roots(struct reloc_control *rc) { struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_root *root; struct btrfs_root *reloc_root; LIST_HEAD(reloc_roots); - int found = 0; + bool found = false; int ret = 0; again: root = rc->extent_root; @@ -1826,7 +1886,7 @@ again: mutex_unlock(&fs_info->reloc_mutex); while (!list_empty(&reloc_roots)) { - found = 1; + found = true; reloc_root = list_first_entry(&reloc_roots, struct btrfs_root, root_list); root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, @@ -1850,6 +1910,7 @@ again: * corruption, e.g. bad reloc tree key offset. */ ret = -EINVAL; + btrfs_put_root(root); goto out; } ret = merge_reloc_root(rc, root); @@ -1879,7 +1940,7 @@ again: } if (found) { - found = 0; + found = false; goto again; } out: @@ -1907,7 +1968,7 @@ out: * do the reloc_dirty_list afterwards. Meanwhile the root->reloc_root * will be cleaned up on unmount. * - * The remaining nodes will be cleaned up by free_reloc_control. + * The remaining nodes will be cleaned up by put_reloc_control(). */ } @@ -2559,7 +2620,7 @@ static int relocate_cowonly_block(struct btrfs_trans_handle *trans, nr_levels = max(btrfs_header_level(root->node) - block->level, 0) + 1; - num_bytes = fs_info->nodesize * nr_levels; + num_bytes = (nr_levels << fs_info->nodesize_bits); ret = refill_metadata_space(trans, rc, num_bytes); if (ret) { btrfs_put_root(root); @@ -2944,7 +3005,7 @@ static int relocate_file_extent_cluster(struct reloc_control *rc) if (!cluster->nr) return 0; - ra = kzalloc(sizeof(*ra), GFP_NOFS); + ra = kzalloc_obj(*ra, GFP_NOFS); if (!ra) return -ENOMEM; @@ -3187,13 +3248,12 @@ again: goto again; } } - if (ret) { + if (WARN_ON(ret)) { ASSERT(ret == 1); btrfs_print_leaf(path->nodes[0]); btrfs_err(fs_info, "tree block extent item (%llu) is not found in extent tree", bytenr); - WARN_ON(1); return -EINVAL; } @@ -3421,7 +3481,9 @@ static void set_reloc_control(struct reloc_control *rc) struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; mutex_lock(&fs_info->reloc_mutex); + spin_lock(&fs_info->reloc_ctl_lock); fs_info->reloc_ctl = rc; + spin_unlock(&fs_info->reloc_ctl_lock); mutex_unlock(&fs_info->reloc_mutex); } @@ -3430,7 +3492,9 @@ static void unset_reloc_control(struct reloc_control *rc) struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; mutex_lock(&fs_info->reloc_mutex); + spin_lock(&fs_info->reloc_ctl_lock); fs_info->reloc_ctl = NULL; + spin_unlock(&fs_info->reloc_ctl_lock); mutex_unlock(&fs_info->reloc_mutex); } @@ -3815,19 +3879,9 @@ static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) rc->reloc_root_tree.rb_root = RB_ROOT; spin_lock_init(&rc->reloc_root_tree.lock); btrfs_extent_io_tree_init(fs_info, &rc->processed_blocks, IO_TREE_RELOC_BLOCKS); - return rc; -} - -static void free_reloc_control(struct reloc_control *rc) -{ - struct mapping_node *node, *tmp; + refcount_set(&rc->refs, 1); - free_reloc_roots(&rc->reloc_roots); - rbtree_postorder_for_each_entry_safe(node, tmp, - &rc->reloc_root_tree.rb_root, rb_node) - kfree(node); - - kfree(rc); + return rc; } /* @@ -3863,7 +3917,7 @@ static int add_remap_tree_entries(struct btrfs_trans_handle *trans, struct btrfs max_items = BTRFS_LEAF_DATA_SIZE(trans->fs_info) / sizeof(struct btrfs_item); - data_sizes = kzalloc(sizeof(u32) * min_t(u32, num_entries, max_items), GFP_NOFS); + data_sizes = kzalloc_objs(u32, min_t(u32, num_entries, max_items), GFP_NOFS); if (!data_sizes) return -ENOMEM; @@ -4038,7 +4092,7 @@ static int copy_remapped_data(struct btrfs_fs_info *fs_info, u64 old_addr, if (!pages) return -ENOMEM; - ret = btrfs_alloc_page_array(nr_pages, pages, 0); + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); if (ret) { ret = -ENOMEM; goto end; @@ -4454,7 +4508,7 @@ static int create_remap_tree_entries(struct btrfs_trans_handle *trans, btrfs_release_path(path); - space_runs = kmalloc(sizeof(*space_runs) * extent_count, GFP_NOFS); + space_runs = kmalloc_objs(*space_runs, extent_count, GFP_NOFS); if (!space_runs) { mutex_unlock(&bg->free_space_lock); return -ENOMEM; @@ -4543,7 +4597,7 @@ static int create_remap_tree_entries(struct btrfs_trans_handle *trans, mutex_unlock(&bg->free_space_lock); max_entries = extent_count + 2; - entries = kmalloc(sizeof(*entries) * max_entries, GFP_NOFS); + entries = kmalloc_objs(*entries, max_entries, GFP_NOFS); if (!entries) { ret = -ENOMEM; goto out; @@ -5367,13 +5421,14 @@ int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start, return -ENOMEM; } - ret = reloc_chunk_start(fs_info); - if (ret < 0) - goto out_put_bg; - rc->extent_root = extent_root; + /* Block group ref now owned by rc, put_reloc_control() will drop it. */ rc->block_group = bg; + ret = reloc_chunk_start(fs_info); + if (ret < 0) + goto out_put_rc; + ret = btrfs_inc_block_group_ro(rc->block_group, true); if (ret) goto out; @@ -5441,9 +5496,8 @@ out: iput(rc->data_inode); btrfs_free_path(path); reloc_chunk_end(fs_info); -out_put_bg: - btrfs_put_block_group(bg); - free_reloc_control(rc); +out_put_rc: + put_reloc_control(rc); return ret; } @@ -5598,7 +5652,7 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) goto out_unset; } - ret = __add_reloc_root(reloc_root); + ret = __add_reloc_root(reloc_root, rc); ASSERT(ret != -EEXIST); if (ret) { list_add_tail(&reloc_root->root_list, &reloc_roots); @@ -5632,7 +5686,7 @@ out_unset: unset_reloc_control(rc); reloc_chunk_end(fs_info); out_end: - free_reloc_control(rc); + put_reloc_control(rc); out: free_reloc_roots(&reloc_roots); @@ -5712,11 +5766,11 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = root->fs_info; struct reloc_control *rc; struct btrfs_backref_node *node; - int first_cow = 0; + bool first_cow = false; int level; int ret = 0; - rc = fs_info->reloc_ctl; + rc = get_reloc_control(fs_info); if (!rc) return 0; @@ -5725,7 +5779,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, level = btrfs_header_level(buf); if (btrfs_header_generation(buf) <= btrfs_root_last_snapshot(&root->root_item)) - first_cow = 1; + first_cow = true; if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID && rc->create_reloc_tree) { WARN_ON(!first_cow && level == 0); @@ -5741,7 +5795,8 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, btrfs_err(fs_info, "bytenr %llu was found but our backref cache was expecting %llu or %llu", buf->start, node->bytenr, node->new_bytenr); - return -EUCLEAN; + ret = -EUCLEAN; + goto out; } btrfs_backref_drop_node_buffer(node); @@ -5764,6 +5819,9 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, if (level == 0 && first_cow && rc->stage == UPDATE_DATA_PTRS) ret = replace_file_extents(trans, rc, root, cow); +out: + put_reloc_control(rc); + return ret; } @@ -5812,13 +5870,16 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, struct btrfs_root *root = pending->root; struct btrfs_root *reloc_root; struct btrfs_root *new_root; - struct reloc_control *rc = root->fs_info->reloc_ctl; - int ret; + struct reloc_control *rc; + int ret = 0; - if (!rc || !have_reloc_root(root)) + rc = get_reloc_control(trans->fs_info); + if (!rc) return 0; - rc = root->fs_info->reloc_ctl; + if (!have_reloc_root(root)) + goto out; + rc->merging_rsv_size += rc->nodes_relocated; if (rc->merge_reloc_tree) { @@ -5826,23 +5887,28 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, rc->block_rsv, rc->nodes_relocated, true); if (ret) - return ret; + goto out; } new_root = pending->snap; reloc_root = create_reloc_root(trans, root->reloc_root, btrfs_root_id(new_root)); - if (IS_ERR(reloc_root)) - return PTR_ERR(reloc_root); + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto out; + } - ret = __add_reloc_root(reloc_root); + ret = __add_reloc_root(reloc_root, rc); ASSERT(ret != -EEXIST); if (ret) { /* Pairs with create_reloc_root */ btrfs_put_root(reloc_root); - return ret; + goto out; } new_root->reloc_root = btrfs_grab_root(reloc_root); - return 0; +out: + put_reloc_control(rc); + + return ret; } /* @@ -5850,14 +5916,15 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, * * Return U64_MAX if no running relocation. */ -u64 btrfs_get_reloc_bg_bytenr(const struct btrfs_fs_info *fs_info) +u64 btrfs_get_reloc_bg_bytenr(struct btrfs_fs_info *fs_info) { u64 logical = U64_MAX; - lockdep_assert_held(&fs_info->reloc_mutex); - + mutex_lock(&fs_info->reloc_mutex); if (fs_info->reloc_ctl && fs_info->reloc_ctl->block_group) logical = fs_info->reloc_ctl->block_group->start; + mutex_unlock(&fs_info->reloc_mutex); + return logical; } diff --git a/fs/btrfs/relocation.h b/fs/btrfs/relocation.h index d647823b5d13..bb7a86e7dbe3 100644 --- a/fs/btrfs/relocation.h +++ b/fs/btrfs/relocation.h @@ -41,7 +41,7 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, int btrfs_should_cancel_balance(const struct btrfs_fs_info *fs_info); struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr); bool btrfs_should_ignore_reloc_root(const struct btrfs_root *root); -u64 btrfs_get_reloc_bg_bytenr(const struct btrfs_fs_info *fs_info); +u64 btrfs_get_reloc_bg_bytenr(struct btrfs_fs_info *fs_info); int btrfs_translate_remap(struct btrfs_fs_info *fs_info, u64 *logical, u64 *length); int btrfs_remove_extent_from_remap_tree(struct btrfs_trans_handle *trans, struct btrfs_path *path, diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c index d85a09ae1733..90659b287d90 100644 --- a/fs/btrfs/root-tree.c +++ b/fs/btrfs/root-tree.c @@ -27,20 +27,20 @@ static void btrfs_read_root_item(struct extent_buffer *eb, int slot, struct btrfs_root_item *item) { u32 len; - int need_reset = 0; + bool need_reset = false; len = btrfs_item_size(eb, slot); read_extent_buffer(eb, item, btrfs_item_ptr_offset(eb, slot), min_t(u32, len, sizeof(*item))); if (len < sizeof(*item)) - need_reset = 1; + need_reset = true; if (!need_reset && btrfs_root_generation(item) != btrfs_root_generation_v2(item)) { if (btrfs_root_generation_v2(item) != 0) { btrfs_warn(eb->fs_info, "mismatching generation and generation_v2 found in root item. This root was probably mounted with an older kernel. Resetting all new fields."); } - need_reset = 1; + need_reset = true; } if (need_reset) { /* Clear all members from generation_v2 onwards. */ diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 1ac609239cbe..d2f7ac5b6e96 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -369,7 +369,8 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, ASSERT(BTRFS_STRIPE_LEN >> min_folio_shift <= SCRUB_STRIPE_MAX_FOLIOS); ret = btrfs_alloc_folio_array(BTRFS_STRIPE_LEN >> min_folio_shift, - fs_info->block_min_order, stripe->folios); + fs_info->block_min_order, stripe->folios, + GFP_NOFS); if (ret < 0) goto error; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 89d72d8cb85f..3ae480c7474b 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -17,6 +17,7 @@ #include <linux/compat.h> #include <linux/crc32c.h> #include <linux/fsverity.h> +#include <linux/cleanup.h> #include "send.h" #include "ctree.h" #include "backref.h" @@ -72,6 +73,8 @@ struct fs_path { #define FS_PATH_INLINE_SIZE \ sizeof_field(struct fs_path, inline_buf) +static void fs_path_free(struct fs_path *p); +DEFINE_FREE(fs_path_free, struct fs_path *, fs_path_free(_T)) /* reused for each extent */ struct clone_root { @@ -981,7 +984,7 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, struct btrfs_inode_ref *iref; struct btrfs_inode_extref *extref; BTRFS_PATH_AUTO_FREE(tmp_path); - struct fs_path *p; + struct fs_path *p __free(fs_path_free) = NULL; u32 cur = 0; u32 total; int slot = path->slots[0]; @@ -998,11 +1001,8 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, return -ENOMEM; tmp_path = alloc_path_for_send(); - if (!tmp_path) { - fs_path_free(p); + if (!tmp_path) return -ENOMEM; - } - if (found_key->type == BTRFS_INODE_REF_KEY) { ptr = (unsigned long)btrfs_item_ptr(eb, slot, @@ -1034,30 +1034,27 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, start = btrfs_ref_to_path(root, tmp_path, name_len, name_off, eb, dir, p->buf, p->buf_len); - if (IS_ERR(start)) { - ret = PTR_ERR(start); - goto out; - } + if (IS_ERR(start)) + return PTR_ERR(start); + if (start < p->buf) { /* overflow , try again with larger buffer */ ret = fs_path_ensure_buf(p, p->buf_len + p->buf - start); if (ret < 0) - goto out; + return ret; start = btrfs_ref_to_path(root, tmp_path, name_len, name_off, eb, dir, p->buf, p->buf_len); - if (IS_ERR(start)) { - ret = PTR_ERR(start); - goto out; - } + if (IS_ERR(start)) + return PTR_ERR(start); + if (unlikely(start < p->buf)) { btrfs_err(root->fs_info, "send: path ref buffer underflow for key " BTRFS_KEY_FMT, BTRFS_KEY_FMT_VALUE(found_key)); - ret = -EINVAL; - goto out; + return -EINVAL; } } p->start = start; @@ -1065,17 +1062,15 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, ret = fs_path_add_from_extent_buffer(p, eb, name_off, name_len); if (ret < 0) - goto out; + return ret; } cur += elem_size + name_len; ret = iterate(dir, p, ctx); if (ret) - goto out; + return ret; } -out: - fs_path_free(p); return ret; } @@ -2028,7 +2023,7 @@ static int is_first_ref(struct btrfs_root *root, const char *name, int name_len) { int ret; - struct fs_path *tmp_name; + struct fs_path *tmp_name __free(fs_path_free) = NULL; u64 tmp_dir; tmp_name = fs_path_alloc(); @@ -2037,17 +2032,13 @@ static int is_first_ref(struct btrfs_root *root, ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name); if (ret < 0) - goto out; + return ret; - if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) { - ret = 0; - goto out; - } + if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) + return 0; ret = !memcmp(tmp_name->start, name, name_len); -out: - fs_path_free(tmp_name); return ret; } @@ -2196,13 +2187,13 @@ static int did_overwrite_ref(struct send_ctx *sctx, */ static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen) { - int ret = 0; - struct fs_path *name = NULL; + int ret; + struct fs_path *name __free(fs_path_free) = NULL; u64 dir; u64 dir_gen; if (!sctx->parent_root) - goto out; + return 0; name = fs_path_alloc(); if (!name) @@ -2210,14 +2201,10 @@ static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen) ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name); if (ret < 0) - goto out; - - ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen, - name->start, fs_path_len(name)); + return ret; -out: - fs_path_free(name); - return ret; + return did_overwrite_ref(sctx, dir, dir_gen, ino, gen, + name->start, fs_path_len(name)); } static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx, @@ -2375,10 +2362,10 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, struct fs_path *dest) { int ret = 0; - struct fs_path *name = NULL; + struct fs_path *name __free(fs_path_free) = NULL; u64 parent_inode = 0; u64 parent_gen = 0; - int stop = 0; + bool stop = false; const bool is_cur_inode = (ino == sctx->cur_ino && gen == sctx->cur_inode_gen); if (is_cur_inode && fs_path_len(&sctx->cur_inode_path) > 0) { @@ -2389,10 +2376,8 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, } name = fs_path_alloc(); - if (!name) { - ret = -ENOMEM; - goto out; - } + if (!name) + return -ENOMEM; dest->reversed = 1; fs_path_reset(dest); @@ -2413,7 +2398,7 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, wdm = get_waiting_dir_move(sctx, ino); if (wdm && wdm->orphanized) { ret = gen_unique_name(sctx, ino, gen, name); - stop = 1; + stop = true; } else if (wdm) { ret = get_first_ref(sctx->parent_root, ino, &parent_inode, &parent_gen, name); @@ -2422,7 +2407,7 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, &parent_inode, &parent_gen, name); if (ret) - stop = 1; + stop = true; } if (ret < 0) @@ -2437,7 +2422,6 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, } out: - fs_path_free(name); if (!ret) { fs_path_unreverse(dest); if (is_cur_inode && dest != &sctx->cur_inode_path) @@ -2787,7 +2771,7 @@ static int trim_dir_utimes_cache(struct send_ctx *sctx) static int send_create_inode(struct send_ctx *sctx, u64 ino) { int ret = 0; - struct fs_path *p; + struct fs_path *p __free(fs_path_free) = NULL; int cmd; struct btrfs_inode_info info; u64 gen; @@ -2801,7 +2785,7 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) if (ino != sctx->cur_ino) { ret = get_inode_info(sctx->send_root, ino, &info); if (ret < 0) - goto out; + return ret; gen = info.gen; mode = info.mode; rdev = info.rdev; @@ -2826,17 +2810,16 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) } else { btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o", (int)(mode & S_IFMT)); - ret = -EOPNOTSUPP; - goto out; + return -EOPNOTSUPP; } ret = begin_cmd(sctx, cmd); if (ret < 0) - goto out; + return ret; ret = gen_unique_name(sctx, ino, gen, p); if (ret < 0) - goto out; + return ret; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino); @@ -2845,7 +2828,7 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) fs_path_reset(p); ret = read_symlink(sctx->send_root, ino, p); if (ret < 0) - goto out; + return ret; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p); } else if (S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || S_ISSOCK(mode)) { @@ -2855,12 +2838,9 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) ret = send_cmd(sctx); if (ret < 0) - goto out; - + return ret; tlv_put_failure: -out: - fs_path_free(p); return ret; } @@ -3039,7 +3019,7 @@ static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, struct fs_path *path) { int ret; - struct fs_path *orphan; + struct fs_path *orphan __free(fs_path_free) = NULL; orphan = fs_path_alloc(); if (!orphan) @@ -3047,17 +3027,15 @@ static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, ret = gen_unique_name(sctx, ino, gen, orphan); if (ret < 0) - goto out; + return ret; ret = send_rename(sctx, path, orphan); if (ret < 0) - goto out; + return ret; if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen) ret = fs_path_copy(&sctx->cur_inode_path, orphan); -out: - fs_path_free(orphan); return ret; } @@ -3349,7 +3327,7 @@ static int add_pending_dir_move(struct send_ctx *sctx, struct rb_node *parent = NULL; struct pending_dir_move *entry = NULL, *pm; struct recorded_ref *cur; - int exists = 0; + bool exists = false; int ret; pm = kmalloc_obj(*pm); @@ -3370,7 +3348,7 @@ static int add_pending_dir_move(struct send_ctx *sctx, } else if (parent_ino > entry->parent_ino) { p = &(*p)->rb_right; } else { - exists = 1; + exists = true; break; } } @@ -3467,9 +3445,9 @@ static int path_loop(struct send_ctx *sctx, struct fs_path *name, static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) { - struct fs_path *from_path = NULL; - struct fs_path *to_path = NULL; - struct fs_path *name = NULL; + struct fs_path *from_path __free(fs_path_free) = NULL; + struct fs_path *to_path __free(fs_path_free) = NULL; + struct fs_path *name __free(fs_path_free) = NULL; u64 orig_progress = sctx->send_progress; struct recorded_ref *cur; u64 parent_ino, parent_gen; @@ -3482,10 +3460,8 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) name = fs_path_alloc(); from_path = fs_path_alloc(); - if (!name || !from_path) { - ret = -ENOMEM; - goto out; - } + if (!name || !from_path) + return -ENOMEM; dm = get_waiting_dir_move(sctx, pm->ino); ASSERT(dm); @@ -3599,9 +3575,6 @@ finish: } out: - fs_path_free(name); - fs_path_free(from_path); - fs_path_free(to_path); sctx->send_progress = orig_progress; return ret; @@ -4157,7 +4130,7 @@ static int rename_current_inode(struct send_ctx *sctx, /* * This does all the move/link/unlink/rmdir magic. */ -static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) +static int process_recorded_refs(struct send_ctx *sctx, bool *pending_move) { struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; int ret = 0; @@ -4417,7 +4390,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) goto out; if (ret == 1) { can_rename = false; - *pending_move = 1; + *pending_move = true; } } @@ -4428,7 +4401,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) goto out; if (ret == 1) { can_rename = false; - *pending_move = 1; + *pending_move = true; } } @@ -4793,7 +4766,7 @@ static int process_all_refs(struct send_ctx *sctx, struct btrfs_key key; struct btrfs_key found_key; iterate_inode_ref_t cb; - int pending_move = 0; + bool pending_move = false; path = alloc_path_for_send(); if (!path) @@ -6524,8 +6497,7 @@ static int process_all_extents(struct send_ctx *sctx) } static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end, - int *pending_move, - int *refs_processed) + bool *pending_move, bool *refs_processed) { int ret; @@ -6543,7 +6515,7 @@ static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end, if (ret < 0) return ret; - *refs_processed = 1; + *refs_processed = true; return 0; } @@ -6559,12 +6531,12 @@ static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end) u64 right_uid; u64 right_gid; u64 right_fileattr; - int need_chmod = 0; - int need_chown = 0; + bool need_chmod = false; + bool need_chown = false; bool need_fileattr = false; - int need_truncate = 1; - int pending_move = 0; - int refs_processed = 0; + bool need_truncate = true; + bool pending_move = false; + bool refs_processed = false; if (sctx->ignore_cur_inode) return 0; @@ -6602,11 +6574,11 @@ static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end) left_fileattr = info.fileattr; if (!sctx->parent_root || sctx->cur_inode_new) { - need_chown = 1; + need_chown = true; if (!S_ISLNK(sctx->cur_inode_mode)) - need_chmod = 1; + need_chmod = true; if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size) - need_truncate = 0; + need_truncate = false; } else { u64 old_size; @@ -6620,15 +6592,15 @@ static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end) right_fileattr = info.fileattr; if (left_uid != right_uid || left_gid != right_gid) - need_chown = 1; + need_chown = true; if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode) - need_chmod = 1; + need_chmod = true; if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr) need_fileattr = true; if ((old_size == sctx->cur_inode_size) || (sctx->cur_inode_size > old_size && sctx->cur_inode_next_write_offset == sctx->cur_inode_size)) - need_truncate = 0; + need_truncate = false; } if (S_ISREG(sctx->cur_inode_mode)) { @@ -7986,7 +7958,7 @@ long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_sen u64 *clone_sources_tmp = NULL; int clone_sources_to_rollback = 0; size_t alloc_size; - int sort_clone_roots = 0; + bool sort_clone_roots = false; struct btrfs_lru_cache_entry *entry; struct btrfs_lru_cache_entry *tmp; @@ -8209,7 +8181,7 @@ long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_sen sort(sctx->clone_roots, sctx->clone_roots_cnt, sizeof(*sctx->clone_roots), __clone_root_cmp_sort, NULL); - sort_clone_roots = 1; + sort_clone_roots = true; ret = flush_delalloc_roots(sctx); if (ret) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index f0436eea1544..e6641597b321 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1411,6 +1411,13 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) * This is where we reclaim all of the pinned space generated by running the * iputs * + * RECLAIM_ZONES + * This state only works for the zoned mode. We scan the block groups in the + * reclaim_bgs_list and check if we can relocate them. If yes perform the + * relocation to garbage collect the zone. On each of these runs + * BTRFS_ZONED_SYNC_RECLAIM_BATCH (5) block-groups will be reclaimed, after all + * unused block-groups have been deleted. + * * RESET_ZONES * This state works only for the zoned mode. We scan the unused block group * list and reset the zones and reuse the block group. @@ -1698,6 +1705,7 @@ static int handle_reserve_ticket(struct btrfs_space_info *space_info, ARRAY_SIZE(evict_flush_states)); break; case BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE: + case BTRFS_RESERVE_FLUSH_ZONED_RELOCATION: priority_reclaim_data_space(space_info, ticket); break; default: @@ -1961,6 +1969,7 @@ int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes, ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA || flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE || + flush == BTRFS_RESERVE_FLUSH_ZONED_RELOCATION || flush == BTRFS_RESERVE_NO_FLUSH, "flush=%d", flush); ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA, "current->journal_info=0x%lx flush=%d", diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 24f45072ca4b..aa836e8a9d4a 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -78,6 +78,17 @@ enum btrfs_reserve_flush_enum { BTRFS_RESERVE_FLUSH_ALL_STEAL, /* + * This is for relocation on zoned filesystems only. We need to use + * priority flushing for this, because otherwise we can deadlock on + * waiting for a ticket, that cannot be granted, because we cannot do + * any allocations. + * + * Apart from being specific to zoned relocation, it is equal to + * BTRFS_FLUSH_FREE_SPACE_INODE. + */ + BTRFS_RESERVE_FLUSH_ZONED_RELOCATION, + + /* * This is for btrfs_use_block_rsv only. We have exhausted our block * rsv and our global block rsv. This can happen for things like * delalloc where we are overwriting a lot of extents with a single diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index f82e71f5d88b..56060acac2e9 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -10,41 +10,13 @@ * * Limitations: * - * - Only support 64K page size for now - * This is to make metadata handling easier, as 64K page would ensure - * all nodesize would fit inside one page, thus we don't need to handle - * cases where a tree block crosses several pages. + * - Metadata must be fully aligned to node size + * So when nodesize <= page size, the metadata can never cross folio boundaries. * - * - Only metadata read-write for now - * The data read-write part is in development. - * - * - Metadata can't cross 64K page boundary - * btrfs-progs and kernel have done that for a while, thus only ancient - * filesystems could have such problem. For such case, do a graceful - * rejection. - * - * Special behavior: - * - * - Metadata - * Metadata read is fully supported. - * Meaning when reading one tree block will only trigger the read for the - * needed range, other unrelated range in the same page will not be touched. - * - * Metadata write support is partial. - * The writeback is still for the full page, but we will only submit - * the dirty extent buffers in the page. - * - * This means, if we have a metadata page like this: - * - * Page offset - * 0 16K 32K 48K 64K - * |/////////| |///////////| - * \- Tree block A \- Tree block B - * - * Even if we just want to writeback tree block A, we will also writeback - * tree block B if it's also dirty. - * - * This may cause extra metadata writeback which results more COW. + * - Only support blocks per folio <= min(BTRFS_MAX_FOLIO_SIZE / fs block size, + * BTRFS_MAX_BLOCKS_PER_FOLIO) + * This is to ensure we can afford an on-stack bitmap, without the need to allocate + * bitmap memory at runtime. * * Implementation: * @@ -224,11 +196,8 @@ static bool btrfs_subpage_end_and_test_lock(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len) { struct btrfs_folio_state *bfs = folio_get_private(folio); - const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len); const int nbits = (len >> fs_info->sectorsize_bits); unsigned long flags; - unsigned int cleared = 0; - int bit = start_bit; bool last; btrfs_subpage_assert(fs_info, folio, start, len); @@ -245,15 +214,10 @@ static bool btrfs_subpage_end_and_test_lock(const struct btrfs_fs_info *fs_info, spin_unlock_irqrestore(&bfs->lock, flags); return true; } - - for_each_set_bit_from(bit, bfs->bitmaps, start_bit + nbits) { - clear_bit(bit, bfs->bitmaps); - cleared++; - } - ASSERT(atomic_read(&bfs->nr_locked) >= cleared, - "atomic_read(&bfs->nr_locked)=%d cleared=%d", - atomic_read(&bfs->nr_locked), cleared); - last = atomic_sub_and_test(cleared, &bfs->nr_locked); + ASSERT(atomic_read(&bfs->nr_locked) >= nbits, + "atomic_read(&bfs->nr_locked)=%d nbits=%d", + atomic_read(&bfs->nr_locked), nbits); + last = atomic_sub_and_test(nbits, &bfs->nr_locked); spin_unlock_irqrestore(&bfs->lock, flags); return last; } @@ -305,15 +269,13 @@ void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, } void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, - struct folio *folio, unsigned long bitmap) + struct folio *folio, unsigned long *bitmap) { struct btrfs_folio_state *bfs = folio_get_private(folio); const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); - const int start_bit = blocks_per_folio * btrfs_bitmap_nr_locked; + const unsigned int nbits = bitmap_weight(bitmap, blocks_per_folio); unsigned long flags; bool last = false; - int cleared = 0; - int bit; if (!btrfs_is_subpage(fs_info, folio)) { folio_unlock(folio); @@ -327,14 +289,10 @@ void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, } spin_lock_irqsave(&bfs->lock, flags); - for_each_set_bit(bit, &bitmap, blocks_per_folio) { - if (test_and_clear_bit(bit + start_bit, bfs->bitmaps)) - cleared++; - } - ASSERT(atomic_read(&bfs->nr_locked) >= cleared, - "atomic_read(&bfs->nr_locked)=%d cleared=%d", - atomic_read(&bfs->nr_locked), cleared); - last = atomic_sub_and_test(cleared, &bfs->nr_locked); + ASSERT(atomic_read(&bfs->nr_locked) >= nbits, + "atomic_read(&bfs->nr_locked)=%d nbits=%d", + atomic_read(&bfs->nr_locked), nbits); + last = atomic_sub_and_test(nbits, &bfs->nr_locked); spin_unlock_irqrestore(&bfs->lock, flags); if (last) folio_unlock(folio); @@ -479,64 +437,6 @@ void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, spin_unlock_irqrestore(&bfs->lock, flags); } -void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - ordered, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - folio_set_ordered(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - -void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - ordered, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - if (subpage_test_bitmap_all_zero(fs_info, folio, ordered)) - folio_clear_ordered(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - -void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - checked, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - if (subpage_test_bitmap_all_set(fs_info, folio, checked)) - folio_set_checked(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - -void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - checked, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - folio_clear_checked(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - /* * Unlike set/clear which is dependent on each page status, for test all bits * are tested in the same way. @@ -560,8 +460,6 @@ bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); -IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); -IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked); /* * Note that, in selftests (extent-io-tests), we can have empty fs_info passed @@ -657,30 +555,55 @@ IMPLEMENT_BTRFS_PAGE_OPS(dirty, folio_mark_dirty, folio_clear_dirty_for_io, folio_test_dirty); IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback, folio_test_writeback); -IMPLEMENT_BTRFS_PAGE_OPS(ordered, folio_set_ordered, folio_clear_ordered, - folio_test_ordered); -IMPLEMENT_BTRFS_PAGE_OPS(checked, folio_set_checked, folio_clear_checked, - folio_test_checked); -#define GET_SUBPAGE_BITMAP(fs_info, folio, name, dst) \ -{ \ - const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ - const struct btrfs_folio_state *__bfs = folio_get_private(folio); \ - \ - ASSERT(__bpf <= BITS_PER_LONG); \ - *dst = bitmap_read(__bfs->bitmaps, \ - __bpf * btrfs_bitmap_nr_##name, __bpf); \ +#define DEFINE_GET_SUBPAGE_BITMAP(name) \ +static inline unsigned long get_bitmap_value_##name( \ + const struct btrfs_fs_info *fs_info, \ + struct folio *folio) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + const struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + unsigned long value; \ + \ + ASSERT(__bpf <= BITS_PER_LONG); \ + value = bitmap_read(__bfs->bitmaps, __bpf * btrfs_bitmap_nr_##name, \ + __bpf); \ + return value; \ +} \ +static inline const unsigned long *get_bitmap_pointer_##name( \ + const struct btrfs_fs_info *fs_info, \ + struct folio *folio) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + unsigned long *pointer; \ + \ + ASSERT(__bpf >= BITS_PER_LONG); \ + ASSERT(IS_ALIGNED(__bpf, BITS_PER_LONG)); \ + pointer = __bfs->bitmaps + (BIT_WORD(__bpf) * btrfs_bitmap_nr_##name); \ + return pointer; \ } -#define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len) \ -{ \ - unsigned long bitmap; \ - const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ - \ - GET_SUBPAGE_BITMAP(fs_info, folio, name, &bitmap); \ - btrfs_warn(fs_info, \ - "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ - start, len, folio_pos(folio), __bpf, &bitmap); \ +DEFINE_GET_SUBPAGE_BITMAP(uptodate); +DEFINE_GET_SUBPAGE_BITMAP(dirty); +DEFINE_GET_SUBPAGE_BITMAP(writeback); + +#define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + \ + if (__bpf <= BITS_PER_LONG) { \ + unsigned long bitmap = get_bitmap_value_##name(fs_info, folio); \ + \ + btrfs_warn(fs_info, \ + "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ + start, len, folio_pos(folio), __bpf, &bitmap); \ + } else { \ + btrfs_warn(fs_info, \ + "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ + start, len, folio_pos(folio), __bpf, \ + get_bitmap_pointer_##name(fs_info, folio)); \ + } \ } /* @@ -728,7 +651,6 @@ void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, { struct btrfs_folio_state *bfs; unsigned long flags; - unsigned int start_bit; unsigned int nbits; int ret; @@ -737,15 +659,8 @@ void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, return; bfs = folio_get_private(folio); - start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len); nbits = len >> fs_info->sectorsize_bits; spin_lock_irqsave(&bfs->lock, flags); - /* Target range should not yet be locked. */ - if (unlikely(!bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits))) { - SUBPAGE_DUMP_BITMAP(fs_info, folio, locked, start, len); - ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); - } - bitmap_set(bfs->bitmaps, start_bit, nbits); ret = atomic_add_return(nbits, &bfs->nr_locked); ASSERT(ret <= btrfs_blocks_per_folio(fs_info, folio)); spin_unlock_irqrestore(&bfs->lock, flags); @@ -778,51 +693,74 @@ void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, { struct btrfs_folio_state *bfs; const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); - unsigned long uptodate_bitmap; - unsigned long dirty_bitmap; - unsigned long writeback_bitmap; - unsigned long ordered_bitmap; - unsigned long checked_bitmap; - unsigned long locked_bitmap; unsigned long flags; ASSERT(folio_test_private(folio) && folio_get_private(folio)); ASSERT(blocks_per_folio > 1); bfs = folio_get_private(folio); - spin_lock_irqsave(&bfs->lock, flags); - GET_SUBPAGE_BITMAP(fs_info, folio, uptodate, &uptodate_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, dirty, &dirty_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, writeback, &writeback_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, ordered, &ordered_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, checked, &checked_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, locked, &locked_bitmap); - spin_unlock_irqrestore(&bfs->lock, flags); - dump_page(folio_page(folio, 0), "btrfs folio state dump"); + + if (blocks_per_folio <= BITS_PER_LONG) { + unsigned long uptodate; + unsigned long dirty; + unsigned long writeback; + + spin_lock_irqsave(&bfs->lock, flags); + uptodate = get_bitmap_value_uptodate(fs_info, folio); + dirty = get_bitmap_value_dirty(fs_info, folio); + writeback = get_bitmap_value_writeback(fs_info, folio); + + spin_unlock_irqrestore(&bfs->lock, flags); + + btrfs_warn(fs_info, +"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", + start, len, folio_pos(folio), + blocks_per_folio, &uptodate, + blocks_per_folio, &dirty, + blocks_per_folio, &writeback); + return; + } + + spin_lock_irqsave(&bfs->lock, flags); btrfs_warn(fs_info, -"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl locked=%*pbl writeback=%*pbl ordered=%*pbl checked=%*pbl", +"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", start, len, folio_pos(folio), - blocks_per_folio, &uptodate_bitmap, - blocks_per_folio, &dirty_bitmap, - blocks_per_folio, &locked_bitmap, - blocks_per_folio, &writeback_bitmap, - blocks_per_folio, &ordered_bitmap, - blocks_per_folio, &checked_bitmap); + blocks_per_folio, get_bitmap_pointer_uptodate(fs_info, folio), + blocks_per_folio, get_bitmap_pointer_dirty(fs_info, folio), + blocks_per_folio, get_bitmap_pointer_writeback(fs_info, folio)); + spin_unlock_irqrestore(&bfs->lock, flags); } -void btrfs_get_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, - struct folio *folio, - unsigned long *ret_bitmap) +void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + unsigned long *dst) { struct btrfs_folio_state *bfs; + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); unsigned long flags; + unsigned long value; + + if (blocks_per_folio == 1) { + value = 1; + bitmap_copy(dst, &value, 1); + return; + } ASSERT(folio_test_private(folio) && folio_get_private(folio)); - ASSERT(btrfs_blocks_per_folio(fs_info, folio) > 1); + ASSERT(blocks_per_folio > 1); bfs = folio_get_private(folio); + if (blocks_per_folio <= BITS_PER_LONG) { + spin_lock_irqsave(&bfs->lock, flags); + value = bitmap_read(bfs->bitmaps, btrfs_bitmap_nr_dirty * blocks_per_folio, + blocks_per_folio); + spin_unlock_irqrestore(&bfs->lock, flags); + bitmap_copy(dst, &value, blocks_per_folio); + return; + } spin_lock_irqsave(&bfs->lock, flags); - GET_SUBPAGE_BITMAP(fs_info, folio, dirty, ret_bitmap); + bitmap_copy(dst, get_bitmap_pointer_dirty(fs_info, folio), + blocks_per_folio); spin_unlock_irqrestore(&bfs->lock, flags); } diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index d81a0ade559f..c6d7394e6418 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -14,15 +14,15 @@ struct folio; /* * Extra info for subpage bitmap. * - * For subpage we pack all uptodate/dirty/writeback/ordered bitmaps into + * For subpage we pack all uptodate/dirty/writeback bitmaps into * one larger bitmap. * * This structure records how they are organized in the bitmap: * - * /- uptodate /- dirty /- ordered + * /- uptodate /- dirty /- writeback * | | | * v v v - * |u|u|u|u|........|u|u|d|d|.......|d|d|o|o|.......|o|o| + * |u|u|u|u|........|u|u|d|d|.......|d|d|w|w|.......|w|w| * |< sectors_per_page >| * * Unlike regular macro-like enums, here we do not go upper-case names, as @@ -40,23 +40,6 @@ enum { */ btrfs_bitmap_nr_writeback, - /* - * The ordered and checked flags are for COW fixup, already marked - * deprecated, and will be removed eventually. - */ - btrfs_bitmap_nr_ordered, - btrfs_bitmap_nr_checked, - - /* - * The locked bit is for async delalloc range (compression), currently - * async extent is queued with the range locked, until the compression - * is done. - * So an async extent can unlock the range at any random timing. - * - * This will need a rework on the async extent lifespan (mark writeback - * and do compression) before deprecating this flag. - */ - btrfs_bitmap_nr_locked, btrfs_bitmap_nr_max }; @@ -133,7 +116,7 @@ void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len); void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, - struct folio *folio, unsigned long bitmap); + struct folio *folio, unsigned long *bitmap); /* * Template for subpage related operations. * @@ -181,8 +164,6 @@ bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffe DECLARE_BTRFS_SUBPAGE_OPS(uptodate); DECLARE_BTRFS_SUBPAGE_OPS(dirty); DECLARE_BTRFS_SUBPAGE_OPS(writeback); -DECLARE_BTRFS_SUBPAGE_OPS(ordered); -DECLARE_BTRFS_SUBPAGE_OPS(checked); /* * Helper for error cleanup, where a folio will have its dirty flag cleared, @@ -203,9 +184,9 @@ bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len); bool btrfs_meta_folio_clear_and_test_dirty(struct folio *folio, const struct extent_buffer *eb); -void btrfs_get_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, - struct folio *folio, - unsigned long *ret_bitmap); +void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + unsigned long *dst); void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index b26aa9169e83..a7d804219bec 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -60,6 +60,7 @@ #include "verity.h" #include "super.h" #include "extent-tree.h" +#include "tree-log.h" #define CREATE_TRACE_POINTS #include <trace/events/btrfs.h> @@ -1633,8 +1634,7 @@ static inline int btrfs_calc_avail_data_space(struct btrfs_fs_info *fs_info, } } - devices_info = kmalloc_array(nr_devices, sizeof(*devices_info), - GFP_KERNEL); + devices_info = kmalloc_objs(*devices_info, nr_devices); if (!devices_info) return -ENOMEM; @@ -1732,15 +1732,17 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) u64 total_free_data = 0; u64 total_free_meta = 0; u32 bits = fs_info->sectorsize_bits; - __be32 *fsid = (__be32 *)fs_info->fs_devices->fsid; + __be32 *fsid; unsigned factor = 1; struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; int ret; u64 thresh = 0; - int mixed = 0; + bool mixed = false; + __kernel_fsid_t f_fsid; list_for_each_entry(found, &fs_info->space_info, list) { - if (found->flags & BTRFS_BLOCK_GROUP_DATA) { + if (found->flags & BTRFS_BLOCK_GROUP_DATA && + found->subgroup_id != BTRFS_SUB_GROUP_DATA_RELOC) { int i; total_free_data += found->disk_total - found->disk_used; @@ -1759,7 +1761,7 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) */ if (!mixed && found->flags & BTRFS_BLOCK_GROUP_METADATA) { if (found->flags & BTRFS_BLOCK_GROUP_DATA) - mixed = 1; + mixed = true; else total_free_meta += found->disk_total - found->disk_used; @@ -1818,14 +1820,38 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) buf->f_bsize = fs_info->sectorsize; buf->f_namelen = BTRFS_NAME_LEN; - /* We treat it as constant endianness (it doesn't matter _which_) - because we want the fsid to come out the same whether mounted - on a big-endian or little-endian host */ - buf->f_fsid.val[0] = be32_to_cpu(fsid[0]) ^ be32_to_cpu(fsid[2]); - buf->f_fsid.val[1] = be32_to_cpu(fsid[1]) ^ be32_to_cpu(fsid[3]); + /* + * fs_devices->fsid is dynamically generated when temp_fsid is active + * to support cloned filesystems. Use the original on-disk fsid instead, + * as it remains consistent across mount cycles. + */ + if (fs_info->fs_devices->temp_fsid) + fsid = (__be32 *)fs_info->super_copy->fsid; + else + fsid = (__be32 *)fs_info->fs_devices->fsid; + + /* + * We treat it as constant endianness (it doesn't matter _which_) + * because we want the fsid to come out the same whether mounted + * on a big-endian or little-endian host. + */ + f_fsid.val[0] = be32_to_cpu(fsid[0]) ^ be32_to_cpu(fsid[2]); + f_fsid.val[1] = be32_to_cpu(fsid[1]) ^ be32_to_cpu(fsid[3]); + /* Mask in the root object ID too, to disambiguate subvols */ - buf->f_fsid.val[0] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root) >> 32; - buf->f_fsid.val[1] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root); + f_fsid.val[0] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root) >> 32; + f_fsid.val[1] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root); + + /* Hash dev_t to avoid f_fsid collision with cloned filesystems. */ + if (fs_info->fs_devices->total_devices == 1) { + __kernel_fsid_t dev_fsid = + u64_to_fsid(huge_encode_dev(fs_info->fs_devices->latest_dev->bdev->bd_dev)); + + f_fsid.val[0] ^= dev_fsid.val[1]; + f_fsid.val[1] ^= dev_fsid.val[0]; + } + + memcpy(&buf->f_fsid, &f_fsid, sizeof(f_fsid)); return 0; } @@ -1873,6 +1899,7 @@ static int btrfs_get_tree_super(struct fs_context *fc) fs_info->fs_devices = fs_devices; mutex_unlock(&uuid_mutex); + fc->sb_flags |= SB_NOSEC; sb = sget_fc(fc, btrfs_fc_test_super, set_anon_super_fc); if (IS_ERR(sb)) { @@ -2052,7 +2079,7 @@ static int btrfs_get_tree_subvol(struct fs_context *fc) * then open_ctree will properly initialize the file system specific * settings later. btrfs_init_fs_info initializes the static elements * of the fs_info (locks and such) to make cleanup easier if we find a - * superblock with our given fs_devices later on at sget() time. + * superblock with our given fs_devices later on at sget_fc() time. */ fs_info = kvzalloc_obj(struct btrfs_fs_info); if (!fs_info) @@ -2405,7 +2432,7 @@ static int btrfs_show_devname(struct seq_file *m, struct dentry *root) static long btrfs_nr_cached_objects(struct super_block *sb, struct shrink_control *sc) { struct btrfs_fs_info *fs_info = btrfs_sb(sb); - const s64 nr = percpu_counter_sum_positive(&fs_info->evictable_extent_maps); + const s64 nr = percpu_counter_read_positive(&fs_info->evictable_extent_maps); trace_btrfs_extent_map_shrinker_count(fs_info, nr); @@ -2605,6 +2632,9 @@ static const struct init_sequence mod_init_seq[] = { .init_func = btrfs_init_compress, .exit_func = btrfs_exit_compress, }, { + .init_func = btrfs_init_block_group, + .exit_func = btrfs_exit_block_group, + }, { .init_func = btrfs_init_cachep, .exit_func = btrfs_destroy_cachep, }, { diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c index 19c127ac6d10..6287d940323d 100644 --- a/fs/btrfs/tests/btrfs-tests.c +++ b/fs/btrfs/tests/btrfs-tests.c @@ -145,6 +145,7 @@ struct btrfs_fs_info *btrfs_alloc_dummy_fs_info(u32 nodesize, u32 sectorsize) fs_info->csum_size = 4; fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size; + fs_info->use_bitmap = btrfs_use_bitmap; set_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state); test_mnt->mnt_sb->s_fs_info = fs_info; diff --git a/fs/btrfs/tests/free-space-tests.c b/fs/btrfs/tests/free-space-tests.c index ebf68fcd2149..0425b3b68716 100644 --- a/fs/btrfs/tests/free-space-tests.c +++ b/fs/btrfs/tests/free-space-tests.c @@ -398,10 +398,8 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, int ret; u64 offset; u64 max_extent_size; - const struct btrfs_free_space_op test_free_space_ops = { - .use_bitmap = test_use_bitmap, - }; - const struct btrfs_free_space_op *orig_free_space_ops; + bool (*orig_use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); test_msg("running space stealing from bitmap to extent tests"); @@ -423,8 +421,8 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, * that forces use of bitmaps as soon as we have at least 1 * extent entry. */ - orig_free_space_ops = cache->free_space_ctl->op; - cache->free_space_ctl->op = &test_free_space_ops; + orig_use_bitmap = cache->fs_info->use_bitmap; + cache->fs_info->use_bitmap = test_use_bitmap; /* * Extent entry covering free space range [128Mb - 256Kb, 128Mb - 128Kb[ @@ -818,7 +816,7 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, if (ret) return ret; - cache->free_space_ctl->op = orig_free_space_ops; + cache->fs_info->use_bitmap = orig_use_bitmap; btrfs_remove_free_space_cache(cache); return 0; @@ -832,10 +830,8 @@ static bool bytes_index_use_bitmap(struct btrfs_free_space_ctl *ctl, static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) { - const struct btrfs_free_space_op test_free_space_ops = { - .use_bitmap = bytes_index_use_bitmap, - }; - const struct btrfs_free_space_op *orig_free_space_ops; + bool (*orig_use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; struct btrfs_free_space *entry; struct rb_node *node; @@ -892,8 +888,8 @@ static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) /* Now validate bitmaps with different ->max_extent_size. */ btrfs_remove_free_space_cache(cache); - orig_free_space_ops = cache->free_space_ctl->op; - cache->free_space_ctl->op = &test_free_space_ops; + orig_use_bitmap = cache->fs_info->use_bitmap; + cache->fs_info->use_bitmap = bytes_index_use_bitmap; ret = test_add_free_space_entry(cache, 0, sectorsize, 1); if (ret) { @@ -997,7 +993,7 @@ static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) return -EINVAL; } - cache->free_space_ctl->op = orig_free_space_ops; + cache->fs_info->use_bitmap = orig_use_bitmap; btrfs_remove_free_space_cache(cache); return 0; } diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 248adb785051..8f9419728100 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -394,6 +394,7 @@ loop: cur_trans->transid = fs_info->generation; fs_info->running_transaction = cur_trans; cur_trans->aborted = 0; + trace_btrfs_transaction_start(cur_trans); spin_unlock(&fs_info->trans_lock); return 0; @@ -630,7 +631,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, * the appropriate flushing if need be. */ if (num_items && root != fs_info->chunk_root) { - qgroup_reserved = num_items * fs_info->nodesize; + qgroup_reserved = (num_items << fs_info->nodesize_bits); /* * Use prealloc for now, as there might be a currently running * transaction that could free this reserved space prematurely @@ -1293,14 +1294,13 @@ static int btrfs_write_and_wait_transaction(struct btrfs_trans_handle *trans) blk_finish_plug(&plug); ret2 = btrfs_wait_extents(fs_info, dirty_pages); - btrfs_extent_io_tree_release(&trans->transaction->dirty_pages); - if (ret) return ret; - else if (ret2) + if (ret2) return ret2; - else - return 0; + + btrfs_extent_io_tree_release(&trans->transaction->dirty_pages); + return 0; } /* @@ -2115,7 +2115,7 @@ static void cleanup_transaction(struct btrfs_trans_handle *trans, int err) btrfs_put_transaction(cur_trans); btrfs_put_transaction(cur_trans); - trace_btrfs_transaction_commit(fs_info); + trace_btrfs_transaction_commit(trans); if (current->journal_info == trans) current->journal_info = NULL; @@ -2267,7 +2267,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) btrfs_create_pending_block_groups(trans); if (!test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &cur_trans->flags)) { - int run_it = 0; + bool run_it = false; /* this mutex is also taken before trying to set * block groups readonly. We need to make sure @@ -2285,7 +2285,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) mutex_lock(&fs_info->ro_block_group_mutex); if (!test_and_set_bit(BTRFS_TRANS_DIRTY_BG_RUN, &cur_trans->flags)) - run_it = 1; + run_it = true; mutex_unlock(&fs_info->ro_block_group_mutex); if (run_it) { @@ -2321,6 +2321,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) } cur_trans->state = TRANS_STATE_COMMIT_PREP; + trace_btrfs_transaction_commit(trans); wake_up(&fs_info->transaction_blocked_wait); btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_PREP); @@ -2359,6 +2360,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) } cur_trans->state = TRANS_STATE_COMMIT_START; + trace_btrfs_transaction_commit(trans); wake_up(&fs_info->transaction_blocked_wait); spin_unlock(&fs_info->trans_lock); @@ -2414,6 +2416,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) spin_lock(&fs_info->trans_lock); add_pending_snapshot(trans); cur_trans->state = TRANS_STATE_COMMIT_DOING; + trace_btrfs_transaction_commit(trans); spin_unlock(&fs_info->trans_lock); /* @@ -2562,6 +2565,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) spin_lock(&fs_info->trans_lock); cur_trans->state = TRANS_STATE_UNBLOCKED; + trace_btrfs_transaction_commit(trans); fs_info->running_transaction = NULL; spin_unlock(&fs_info->trans_lock); mutex_unlock(&fs_info->reloc_mutex); @@ -2604,6 +2608,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) * which can change it. */ cur_trans->state = TRANS_STATE_SUPER_COMMITTED; + trace_btrfs_transaction_commit(trans); wake_up(&cur_trans->commit_wait); btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); @@ -2620,6 +2625,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) * which can change it. */ cur_trans->state = TRANS_STATE_COMPLETED; + trace_btrfs_transaction_commit(trans); wake_up(&cur_trans->commit_wait); btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); @@ -2633,8 +2639,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) if (trans->type & __TRANS_FREEZABLE) sb_end_intwrite(fs_info->sb); - trace_btrfs_transaction_commit(fs_info); - btrfs_scrub_continue(fs_info); if (current->journal_info == trans) @@ -2723,17 +2727,33 @@ int btrfs_clean_one_deleted_snapshot(struct btrfs_fs_info *fs_info) * * We'll complete the cleanup in btrfs_end_transaction and * btrfs_commit_transaction. + * + * Note: the parameter @error encodes whether the transactin abort was first hit + * (setting the FS_ERROR state bit in btrfs_abort_transaction()) + * - positive number - first hit + * - negative number - abort after it was already done */ void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, const char *function, - unsigned int line, int error, bool first_hit) + unsigned int line, int error) { struct btrfs_fs_info *fs_info = trans->fs_info; + bool first_hit = false; + + if (error > 0) { + error = -error; + first_hit = true; + } WRITE_ONCE(trans->aborted, error); WRITE_ONCE(trans->transaction->aborted, error); - if (first_hit && error == -ENOSPC) - btrfs_dump_space_info_for_trans_abort(fs_info); + trace_btrfs_transaction_abort(trans); + if (first_hit) { + btrfs_err(fs_info, "Transaction %llu aborted (error %d)", + trans->transid, error); + if (error == -ENOSPC) + btrfs_dump_space_info_for_trans_abort(fs_info); + } /* Wake up anybody who may be waiting on this transaction */ wake_up(&fs_info->transaction_wait); wake_up(&fs_info->transaction_blocked_wait); diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 7d70fe486758..5e4b1106fd90 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -243,29 +243,47 @@ static inline bool btrfs_abort_should_print_stack(int error) } /* - * Call btrfs_abort_transaction as early as possible when an error condition is - * detected, that way the exact stack trace is reported for some errors. + * Compile-time and run-time verification of error passed to transaction abort. + * Direct constants will be caught at compile time, errors read from variables + * can be caught only at run-time and will warn under debugging config. + * + * How verification works: + * - accepted builtin constants are all -EIO and such + * - for compile-time check, invalid condition produces a negative-sized array + * type, valid zero-sized + * - when a variable is passed as error the first check is a no-op + * - with enabled debugging, the second array type size is constructed from the + * real variable value, valid condition produces array of size 1 + * - sizeof(type) does not generate any code + */ +#define VERIFY_NEGATIVE_ERROR(error) \ +do { \ + (void)sizeof(char[-!(__builtin_constant_p(error) ? (error) < 0 : 1)]); \ + if (IS_ENABLED(CONFIG_BTRFS_DEBUG)) { \ + if (sizeof(char[(error) < 0]) != 1) \ + DEBUG_WARN("error >= 0 passed to btrfs_abort_transaction()"); \ + } \ +} while(0) + +/* + * Call btrfs_abort_transaction() as early as possible when an error condition + * is detected, that way the exact stack trace is reported for some errors. + * + * Error number must be negative as it encodes wheather it's the first abort. */ #define btrfs_abort_transaction(trans, error) \ do { \ - bool __first = false; \ + int __error = (error); \ + \ + VERIFY_NEGATIVE_ERROR(error); \ /* Report first abort since mount */ \ if (!test_and_set_bit(BTRFS_FS_STATE_TRANS_ABORTED, \ &((trans)->fs_info->fs_state))) { \ - __first = true; \ - if (WARN(btrfs_abort_should_print_stack(error), \ - KERN_ERR \ - "BTRFS: Transaction aborted (error %d)\n", \ - (error))) { \ - /* Stack trace printed. */ \ - } else { \ - btrfs_err((trans)->fs_info, \ - "Transaction aborted (error %d)", \ - (error)); \ - } \ + WARN_ON(btrfs_abort_should_print_stack(__error)); \ + __error = -__error; \ } \ __btrfs_abort_transaction((trans), __func__, \ - __LINE__, (error), __first); \ + __LINE__, __error); \ } while (0) int btrfs_end_transaction(struct btrfs_trans_handle *trans); @@ -303,7 +321,7 @@ void btrfs_add_dropped_root(struct btrfs_trans_handle *trans, void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans); void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, const char *function, - unsigned int line, int error, bool first_hit); + unsigned int line, int error); int __init btrfs_transaction_init(void); void __cold btrfs_transaction_exit(void); diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 1f15d0793a9c..0ce91396b517 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -296,6 +296,33 @@ static int check_extent_data_item(struct extent_buffer *leaf, return 0; } + /* + * For the data reloc tree, file extent items are written by + * relocation's own paths. The data reloc inode is created with + * BTRFS_INODE_NOCOMPRESS, so insert_ordered_extent_file_extent() + * always leaves the compression field at 0. Encryption and + * other_encoding are reserved-and-zero in btrfs. A non-zero value + * for any of these means the leaf decoded from disk does not match + * what the kernel wrote, i.e. on-disk corruption. + * + * The file_extent_item's offset field is NOT a universal invariant + * here: partial-PREALLOC writebacks legitimately produce REG items + * with non-zero offset at non-boundary keys. The offset check is + * performed at the call site in get_new_location(), which only + * inspects cluster-boundary keys where offset is always 0. + */ + if (unlikely(btrfs_header_owner(leaf) == BTRFS_DATA_RELOC_TREE_OBJECTID && + (btrfs_file_extent_compression(leaf, fi) || + btrfs_file_extent_encryption(leaf, fi) || + btrfs_file_extent_other_encoding(leaf, fi)))) { + file_extent_err(leaf, slot, +"invalid encoding fields for data reloc tree, compression=%u encryption=%u other_encoding=%u", + btrfs_file_extent_compression(leaf, fi), + btrfs_file_extent_encryption(leaf, fi), + btrfs_file_extent_other_encoding(leaf, fi)); + return -EUCLEAN; + } + /* Regular or preallocated extent has fixed item size */ if (unlikely(item_size != sizeof(*fi))) { file_extent_err(leaf, slot, @@ -1371,6 +1398,37 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, return 0; } +static int check_root_ref(struct extent_buffer *leaf, struct btrfs_key *key, int slot) +{ + struct btrfs_root_ref *rref; + u32 item_size = btrfs_item_size(leaf, slot); + u32 name_len; + + if (unlikely(item_size <= sizeof(*rref))) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect > %zu", + key->type, item_size, sizeof(*rref)); + return -EUCLEAN; + } + + rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); + if (unlikely(name_len > BTRFS_NAME_LEN)) { + generic_err(leaf, slot, + "root ref name too long for key type %u, have %u max %u", + key->type, name_len, BTRFS_NAME_LEN); + return -EUCLEAN; + } + if (unlikely(item_size != sizeof(*rref) + name_len)) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect %zu", + key->type, item_size, sizeof(*rref) + name_len); + return -EUCLEAN; + } + + return 0; +} + __printf(3,4) __cold static void extent_err(const struct extent_buffer *eb, int slot, @@ -1865,6 +1923,12 @@ static int check_inode_ref(struct extent_buffer *leaf, iref = (struct btrfs_inode_ref *)ptr; namelen = btrfs_inode_ref_name_len(leaf, iref); + if (unlikely(namelen == 0 || namelen > BTRFS_NAME_LEN)) { + inode_ref_err(leaf, slot, + "invalid inode ref name length, has %u expect [1, %u]", + namelen, BTRFS_NAME_LEN); + return -EUCLEAN; + } if (unlikely(ptr + sizeof(*iref) + namelen > end)) { inode_ref_err(leaf, slot, "inode ref overflow, ptr %lu end %lu namelen %u", @@ -2071,6 +2135,7 @@ static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *k struct btrfs_fs_info *fs_info = leaf->fs_info; struct btrfs_free_space_info *fsi; const u32 blocksize = fs_info->sectorsize; + u64 end; u32 flags; if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { @@ -2085,6 +2150,12 @@ static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *k blocksize, BTRFS_KEY_FMT_VALUE(key)); return -EUCLEAN; } + if (unlikely(check_add_overflow(key->objectid, key->offset, &end))) { + generic_err(leaf, slot, + "free space info key overflows, has " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } if (unlikely(btrfs_item_size(leaf, slot) != sizeof(struct btrfs_free_space_info))) { generic_err(leaf, slot, @@ -2112,23 +2183,98 @@ static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *k return 0; } -static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key *key, int slot) +static int check_free_space_common_key(struct extent_buffer *leaf, struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) { struct btrfs_fs_info *fs_info = leaf->fs_info; const u32 blocksize = fs_info->sectorsize; + const char *type_str = (key->type == BTRFS_FREE_SPACE_EXTENT_KEY) ? "extent" : "bitmap"; + u64 end; if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { generic_err(leaf, slot, - "free space extent key objectid is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); + "free space %s key objectid is not aligned to %u, has " BTRFS_KEY_FMT, + type_str, blocksize, BTRFS_KEY_FMT_VALUE(key)); return -EUCLEAN; } if (unlikely(!IS_ALIGNED(key->offset, blocksize))) { generic_err(leaf, slot, - "free space extent key offset is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); + "free space %s key offset is not aligned to %u, has " BTRFS_KEY_FMT, + type_str, blocksize, BTRFS_KEY_FMT_VALUE(key)); return -EUCLEAN; } + if (unlikely(key->offset == 0)) { + generic_err(leaf, slot, "free space %s length is 0", type_str); + return -EUCLEAN; + } + if (unlikely(check_add_overflow(key->objectid, key->offset, &end))) { + generic_err(leaf, slot, + "free space %s end overflow, have objectid %llu offset %llu", + type_str, key->objectid, key->offset); + return -EUCLEAN; + } + if (slot == 0) + return 0; + + /* + * Make sure the current key is inside the block group, and matching + * the expected info type. + */ + if (prev_key->type == BTRFS_FREE_SPACE_INFO_KEY) { + struct btrfs_free_space_info *fsi; + u32 info_flags; + + if (unlikely(key->objectid < prev_key->objectid || + key->objectid + key->offset > prev_key->objectid + prev_key->offset)) { + generic_err(leaf, slot, +"free space %s is not inside the space info, prev key " BTRFS_KEY_FMT " current key " BTRFS_KEY_FMT, + type_str, BTRFS_KEY_FMT_VALUE(prev_key), + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + fsi = btrfs_item_ptr(leaf, slot - 1, struct btrfs_free_space_info); + info_flags = btrfs_free_space_flags(leaf, fsi); + if (unlikely((info_flags == BTRFS_FREE_SPACE_USING_BITMAPS && + key->type == BTRFS_FREE_SPACE_EXTENT_KEY) || + (info_flags != BTRFS_FREE_SPACE_USING_BITMAPS && + key->type == BTRFS_FREE_SPACE_BITMAP_KEY))) { + generic_err(leaf, slot, +"free space %s key type is not matching the type of space info, key type %u space info flags %u", + type_str, key->type, info_flags); + return -EUCLEAN; + } + return 0; + } + /* + * Previous key should be either FREE_SPACE_EXTENT or FREE_SPACE_BITMAP. + * Inside the same block group the key type should match each other, and + * no overlaps. + */ + if (unlikely(key->type != prev_key->type)) { + generic_err(leaf, slot, +"free space %s key type is not matching the type of previous key, key type %u prev key type %u", + type_str, key->type, prev_key->type); + return -EUCLEAN; + } + if (unlikely(prev_key->objectid + prev_key->offset > key->objectid)) { + generic_err(leaf, slot, +"free space %s key overlaps previous key, prev key " BTRFS_KEY_FMT " current key " BTRFS_KEY_FMT, + type_str, BTRFS_KEY_FMT_VALUE(prev_key), + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + return 0; +} + +static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + int ret; + + ret = check_free_space_common_key(leaf, key, slot, prev_key); + if (unlikely(ret < 0)) + return ret; + if (unlikely(btrfs_item_size(leaf, slot) != 0)) { generic_err(leaf, slot, "invalid item size for free space info, has %u expect 0", @@ -2139,28 +2285,17 @@ static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key } static int check_free_space_bitmap(struct extent_buffer *leaf, - struct btrfs_key *key, int slot) + struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) { struct btrfs_fs_info *fs_info = leaf->fs_info; - const u32 blocksize = fs_info->sectorsize; u32 expected_item_size; + int ret; + + ret = check_free_space_common_key(leaf, key, slot, prev_key); + if (unlikely(ret < 0)) + return ret; - if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { - generic_err(leaf, slot, - "free space bitmap key objectid is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); - return -EUCLEAN; - } - if (unlikely(!IS_ALIGNED(key->offset, blocksize))) { - generic_err(leaf, slot, - "free space bitmap key offset is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); - return -EUCLEAN; - } - if (unlikely(key->offset == 0)) { - generic_err(leaf, slot, "free space bitmap length is 0"); - return -EUCLEAN; - } /* * The item must hold exactly the right number of bitmap bytes for the * range described by key->offset. A mismatch means the item was @@ -2226,6 +2361,10 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf, case BTRFS_ROOT_ITEM_KEY: ret = check_root_item(leaf, key, slot); break; + case BTRFS_ROOT_REF_KEY: + case BTRFS_ROOT_BACKREF_KEY: + ret = check_root_ref(leaf, key, slot); + break; case BTRFS_EXTENT_ITEM_KEY: case BTRFS_METADATA_ITEM_KEY: ret = check_extent_item(leaf, key, slot, prev_key); @@ -2245,10 +2384,10 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf, ret = check_free_space_info(leaf, key, slot); break; case BTRFS_FREE_SPACE_EXTENT_KEY: - ret = check_free_space_extent(leaf, key, slot); + ret = check_free_space_extent(leaf, key, slot, prev_key); break; case BTRFS_FREE_SPACE_BITMAP_KEY: - ret = check_free_space_bitmap(leaf, key, slot); + ret = check_free_space_bitmap(leaf, key, slot, prev_key); break; case BTRFS_IDENTITY_REMAP_KEY: case BTRFS_REMAP_KEY: diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 9123adafa0d1..875e4ddc68ea 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -33,17 +33,6 @@ #define MAX_CONFLICT_INODES 10 -/* magic values for the inode_only field in btrfs_log_inode: - * - * LOG_INODE_ALL means to log everything - * LOG_INODE_EXISTS means to log just enough to recreate the inode - * during log replay - */ -enum { - LOG_INODE_ALL, - LOG_INODE_EXISTS, -}; - /* * directory trouble cases * @@ -227,7 +216,7 @@ static void do_abort_log_replay(struct walk_control *wc, const char *function, static int btrfs_log_inode(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, - int inode_only, + enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx); static int link_to_fixup_dir(struct walk_control *wc, u64 objectid); static noinline int replay_dir_deletes(struct walk_control *wc, @@ -502,7 +491,7 @@ static int overwrite_item(struct walk_control *wc) * the leaf before writing into the log tree. See the comments at * copy_items() for more details. */ - ASSERT(btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID, "root_id=%llu", btrfs_root_id(root)); + ASSERT(btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID); item_size = btrfs_item_size(wc->log_leaf, wc->log_slot); src_ptr = btrfs_item_ptr_offset(wc->log_leaf, wc->log_slot); @@ -3333,8 +3322,10 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, u64 log_root_level; mutex_lock(&root->log_mutex); + trace_btrfs_sync_log_enter(trans, root, ctx); log_transid = ctx->log_transid; if (root->log_transid_committed >= log_transid) { + trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); mutex_unlock(&root->log_mutex); return ctx->log_ret; } @@ -3342,6 +3333,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, index1 = log_transid % 2; if (atomic_read(&root->log_commit[index1])) { wait_log_commit(root, log_transid); + trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); mutex_unlock(&root->log_mutex); return ctx->log_ret; } @@ -3370,6 +3362,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, /* bail out if we need to do a full commit */ if (btrfs_need_log_full_commit(trans)) { ret = BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); mutex_unlock(&root->log_mutex); goto out; } @@ -3396,6 +3389,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) ret = 0; if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); blk_finish_plug(&plug); btrfs_set_log_full_commit(trans); mutex_unlock(&root->log_mutex); @@ -3433,6 +3427,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, if (!log_root_tree->node) { ret = btrfs_alloc_log_tree_node(trans, log_root_tree); if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); mutex_unlock(&fs_info->tree_root->log_mutex); blk_finish_plug(&plug); goto out; @@ -3456,6 +3451,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, */ ret = update_log_root(trans, log, &new_root_item); if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); list_del_init(&root_log_ctx.list); blk_finish_plug(&plug); btrfs_set_log_full_commit(trans); @@ -3473,6 +3469,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, list_del_init(&root_log_ctx.list); mutex_unlock(&log_root_tree->log_mutex); ret = root_log_ctx.log_ret; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); goto out; } @@ -3484,6 +3481,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, mutex_unlock(&log_root_tree->log_mutex); if (!ret) ret = root_log_ctx.log_ret; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); goto out; } ASSERT(root_log_ctx.log_transid == log_root_tree->log_transid, @@ -3505,6 +3503,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, btrfs_wait_tree_log_extents(log, mark); mutex_unlock(&log_root_tree->log_mutex); ret = BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); goto out_wake_log_root; } @@ -3518,11 +3517,13 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, * deadlock. Bail out to the full commit instead. */ if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); btrfs_wait_tree_log_extents(log, mark); mutex_unlock(&log_root_tree->log_mutex); goto out_wake_log_root; } else if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); mutex_unlock(&log_root_tree->log_mutex); goto out_wake_log_root; @@ -3532,6 +3533,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, ret = btrfs_wait_tree_log_extents(log_root_tree, EXTENT_DIRTY_LOG1 | EXTENT_DIRTY_LOG2); if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); mutex_unlock(&log_root_tree->log_mutex); goto out_wake_log_root; @@ -3568,6 +3570,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, */ if (unlikely(BTRFS_FS_ERROR(fs_info))) { ret = -EIO; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); btrfs_abort_transaction(trans, ret); mutex_unlock(&fs_info->tree_log_mutex); @@ -3579,6 +3582,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, ret = write_all_supers(trans); mutex_unlock(&fs_info->tree_log_mutex); if (unlikely(ret)) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); btrfs_abort_transaction(trans, ret); goto out_wake_log_root; @@ -4771,7 +4775,7 @@ static noinline int copy_items(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, struct btrfs_path *dst_path, struct btrfs_path *src_path, - int start_slot, int nr, int inode_only, + int start_slot, int nr, enum btrfs_log_mode log_mode, u64 logged_isize, struct btrfs_log_ctx *ctx) { struct btrfs_root *log = inode->root->log_root; @@ -4985,7 +4989,7 @@ copy_item: inode_item = btrfs_item_ptr(dst_path->nodes[0], dst_slot, struct btrfs_inode_item); fill_inode_item(trans, dst_path->nodes[0], inode_item, - inode, inode_only == LOG_INODE_EXISTS, + inode, log_mode == LOG_INODE_EXISTS, logged_isize); } else { copy_extent_buffer(dst_path->nodes[0], src, dst_offset, @@ -5913,9 +5917,13 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, struct btrfs_inode *curr_inode = start_inode; int ret = 0; + trace_btrfs_log_new_dir_dentries_enter(trans, start_inode); + path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } /* Pairs with btrfs_add_delayed_iput below. */ ihold(&curr_inode->vfs_inode); @@ -6034,6 +6042,8 @@ out: kfree(dir_elem); } + trace_btrfs_log_new_dir_dentries_exit(trans, start_inode, ret); + return ret; } @@ -6126,6 +6136,9 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, { struct btrfs_ino_list *ino_elem; struct btrfs_inode *inode; + int ret = 0; + + trace_btrfs_add_conflicting_inode_enter(trans, ctx, ino, parent); /* * It's rare to have a lot of conflicting inodes, in practice it is not @@ -6134,8 +6147,10 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, * LOG_INODE_EXISTS mode) and slow down other fsyncs or transaction * commits. */ - if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) - return BTRFS_LOG_FORCE_COMMIT; + if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } inode = btrfs_iget_logging(ino, root); /* @@ -6159,26 +6174,27 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, * some inode from it to some other directory). */ if (IS_ERR(inode)) { - int ret = PTR_ERR(inode); - + ret = PTR_ERR(inode); if (ret != -ENOENT) - return ret; + goto out; ret = conflicting_inode_is_dir(root, ino, path); /* Not a directory or we got an error. */ if (ret <= 0) - return ret; + goto out; /* Conflicting inode is a directory, so we'll log its parent. */ ino_elem = kmalloc_obj(*ino_elem, GFP_NOFS); - if (!ino_elem) - return -ENOMEM; + if (!ino_elem) { + ret = -ENOMEM; + goto out; + } ino_elem->ino = ino; ino_elem->parent = parent; list_add_tail(&ino_elem->list, &ctx->conflict_inodes); ctx->num_conflict_inodes++; - - return 0; + ret = 0; + goto out; } /* @@ -6218,25 +6234,31 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, */ if (!need_log_inode(trans, inode)) { btrfs_add_delayed_iput(inode); - return 0; + goto out; } if (!can_log_conflicting_inode(trans, inode)) { btrfs_add_delayed_iput(inode); - return BTRFS_LOG_FORCE_COMMIT; + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; } btrfs_add_delayed_iput(inode); ino_elem = kmalloc_obj(*ino_elem, GFP_NOFS); - if (!ino_elem) - return -ENOMEM; + if (!ino_elem) { + ret = -ENOMEM; + goto out; + } ino_elem->ino = ino; ino_elem->parent = parent; list_add_tail(&ino_elem->list, &ctx->conflict_inodes); ctx->num_conflict_inodes++; - return 0; +out: + trace_btrfs_add_conflicting_inode_exit(trans, ctx, ino, parent, ret); + + return ret; } static int log_conflicting_inodes(struct btrfs_trans_handle *trans, @@ -6254,7 +6276,15 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, if (ctx->logging_conflict_inodes) return 0; + /* + * Avoid any work if no conflicting inodes and emitting the trace event + * which only adds noise and it's useless if there are no inodes. + */ + if (list_empty(&ctx->conflict_inodes)) + return 0; + ctx->logging_conflict_inodes = true; + trace_btrfs_log_conflicting_inodes_enter(trans, ctx); /* * New conflicting inodes may be found and added to the list while we @@ -6348,6 +6378,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, ctx->logging_conflict_inodes = false; if (ret) free_conflicting_inodes(ctx); + trace_btrfs_log_conflicting_inodes_exit(trans, ctx, ret); return ret; } @@ -6359,7 +6390,7 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, struct btrfs_path *path, struct btrfs_path *dst_path, const u64 logged_isize, - const int inode_only, + const enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx, bool *need_log_inode_item) { @@ -6415,7 +6446,7 @@ again: } ret = copy_items(trans, inode, dst_path, path, ins_start_slot, ins_nr, - inode_only, logged_isize, ctx); + log_mode, logged_isize, ctx); if (ret < 0) return ret; ins_nr = 0; @@ -6434,7 +6465,7 @@ again: goto next_slot; ret = copy_items(trans, inode, dst_path, path, ins_start_slot, - ins_nr, inode_only, logged_isize, ctx); + ins_nr, log_mode, logged_isize, ctx); if (ret < 0) return ret; ins_nr = 0; @@ -6451,7 +6482,7 @@ again: } ret = copy_items(trans, inode, dst_path, path, ins_start_slot, - ins_nr, inode_only, logged_isize, ctx); + ins_nr, log_mode, logged_isize, ctx); if (ret < 0) return ret; ins_nr = 1; @@ -6465,7 +6496,7 @@ next_slot: } if (ins_nr) { ret = copy_items(trans, inode, dst_path, path, - ins_start_slot, ins_nr, inode_only, + ins_start_slot, ins_nr, log_mode, logged_isize, ctx); if (ret < 0) return ret; @@ -6491,12 +6522,12 @@ next_key: } if (ins_nr) { ret = copy_items(trans, inode, dst_path, path, ins_start_slot, - ins_nr, inode_only, logged_isize, ctx); + ins_nr, log_mode, logged_isize, ctx); if (ret) return ret; } - if (inode_only == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { + if (log_mode == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { /* * Release the path because otherwise we might attempt to double * lock the same leaf with btrfs_log_prealloc_extents() below. @@ -6827,8 +6858,16 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, */ lockdep_assert_not_held(&inode->log_mutex); - ASSERT(!ctx->logging_new_delayed_dentries, - "ctx->logging_new_delayed_dentries=%d", ctx->logging_new_delayed_dentries); + ASSERT(!ctx->logging_new_delayed_dentries); + + /* + * Return early if empty list, avoid emitting redundant trace events + * that generate noise only. + */ + if (list_empty(delayed_ins_list)) + return 0; + + trace_btrfs_log_new_delayed_dentries_enter(trans, inode); ctx->logging_new_delayed_dentries = true; list_for_each_entry(item, delayed_ins_list, log_list) { @@ -6871,6 +6910,7 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, ctx->log_new_dentries = orig_log_new_dentries; ctx->logging_new_delayed_dentries = false; + trace_btrfs_log_new_delayed_dentries_exit(trans, inode, ret); return ret; } @@ -6891,11 +6931,11 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, */ static int btrfs_log_inode(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, - int inode_only, + enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx) { struct btrfs_path *path; - struct btrfs_path *dst_path; + struct btrfs_path *dst_path = NULL; struct btrfs_key min_key; struct btrfs_key max_key; struct btrfs_root *log = inode->root->log_root; @@ -6911,13 +6951,17 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, LIST_HEAD(delayed_ins_list); LIST_HEAD(delayed_del_list); + trace_btrfs_log_inode_enter(trans, inode, ctx, log_mode); + path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } dst_path = btrfs_alloc_path(); if (!dst_path) { - btrfs_free_path(path); - return -ENOMEM; + ret = -ENOMEM; + goto out; } min_key.objectid = ino; @@ -6931,13 +6975,13 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, if (S_ISDIR(inode->vfs_inode.i_mode) || (!test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) && - inode_only >= LOG_INODE_EXISTS)) + log_mode >= LOG_INODE_EXISTS)) max_key.type = BTRFS_XATTR_ITEM_KEY; else max_key.type = (u8)-1; max_key.offset = (u64)-1; - if (S_ISDIR(inode->vfs_inode.i_mode) && inode_only == LOG_INODE_ALL) + if (S_ISDIR(inode->vfs_inode.i_mode) && log_mode == LOG_INODE_ALL) full_dir_logging = true; /* @@ -6988,7 +7032,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * for symlinks). */ if (S_ISLNK(inode->vfs_inode.i_mode)) - inode_only = LOG_INODE_ALL; + log_mode = LOG_INODE_ALL; /* * Before logging the inode item, cache the value returned by @@ -7023,7 +7067,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, ret = drop_inode_items(trans, log, path, inode, BTRFS_XATTR_ITEM_KEY); } else { - if (inode_only == LOG_INODE_EXISTS) { + if (log_mode == LOG_INODE_EXISTS) { /* * Make sure the new inode item we write to the log has * the same isize as the current one (if it exists). @@ -7043,7 +7087,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, } if (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags)) { - if (inode_only == LOG_INODE_EXISTS) { + if (log_mode == LOG_INODE_EXISTS) { max_key.type = BTRFS_XATTR_ITEM_KEY; if (ctx->logged_before) ret = drop_inode_items(trans, log, path, @@ -7059,15 +7103,15 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, } } else if (test_and_clear_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags) || - inode_only == LOG_INODE_EXISTS) { - if (inode_only == LOG_INODE_ALL) + log_mode == LOG_INODE_EXISTS) { + if (log_mode == LOG_INODE_ALL) fast_search = true; max_key.type = BTRFS_XATTR_ITEM_KEY; if (ctx->logged_before) ret = drop_inode_items(trans, log, path, inode, max_key.type); } else { - if (inode_only == LOG_INODE_ALL) + if (log_mode == LOG_INODE_ALL) fast_search = true; inode_item_dropped = false; goto log_extents; @@ -7102,8 +7146,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, ret = copy_inode_items_to_log(trans, inode, &min_key, &max_key, path, dst_path, logged_isize, - inode_only, ctx, - &need_log_inode_item); + log_mode, ctx, &need_log_inode_item); if (ret) goto out_unlock; @@ -7146,7 +7189,7 @@ log_extents: ret = btrfs_log_changed_extents(trans, inode, dst_path, ctx); if (ret) goto out_unlock; - } else if (inode_only == LOG_INODE_ALL) { + } else if (log_mode == LOG_INODE_ALL) { struct extent_map *em, *n; write_lock(&em_tree->lock); @@ -7202,7 +7245,7 @@ log_extents: * a power failure unless the log was synced as part of an fsync * against any other unrelated inode. */ - if (!ctx->logging_new_name && inode_only != LOG_INODE_EXISTS) + if (!ctx->logging_new_name && log_mode != LOG_INODE_EXISTS) inode->last_log_commit = inode->last_sub_trans; spin_unlock(&inode->lock); @@ -7210,7 +7253,7 @@ log_extents: * Reset the last_reflink_trans so that the next fsync does not need to * go through the slower path when logging extents and their checksums. */ - if (inode_only == LOG_INODE_ALL) + if (log_mode == LOG_INODE_ALL) inode->last_reflink_trans = 0; out_unlock: @@ -7233,6 +7276,8 @@ out: &delayed_del_list); } + trace_btrfs_log_inode_exit(trans, inode, ret); + return ret; } @@ -7246,9 +7291,13 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, struct btrfs_root *root = inode->root; const u64 ino = btrfs_ino(inode); + trace_btrfs_log_all_parents_enter(trans, inode); + path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } path->skip_locking = true; path->search_commit_root = true; @@ -7257,7 +7306,7 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, key.offset = 0; ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); if (ret < 0) - return ret; + goto out; while (true) { struct extent_buffer *leaf = path->nodes[0]; @@ -7269,9 +7318,11 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, if (slot >= btrfs_header_nritems(leaf)) { ret = btrfs_next_leaf(root, path); if (ret < 0) - return ret; - if (ret > 0) + goto out; + if (ret > 0) { + ret = 0; break; + } continue; } @@ -7324,8 +7375,10 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, * at both parents and the old parent B would still * exist. */ - if (IS_ERR(dir_inode)) - return PTR_ERR(dir_inode); + if (IS_ERR(dir_inode)) { + ret = PTR_ERR(dir_inode); + goto out; + } if (!need_log_inode(trans, dir_inode)) { btrfs_add_delayed_iput(dir_inode); @@ -7338,11 +7391,14 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, ret = log_new_dir_dentries(trans, dir_inode, ctx); btrfs_add_delayed_iput(dir_inode); if (ret) - return ret; + goto out; } path->slots[0]++; } - return 0; +out: + trace_btrfs_log_all_parents_exit(trans, inode, ret); + + return ret; } static int log_new_ancestors(struct btrfs_trans_handle *trans, @@ -7457,16 +7513,22 @@ static int log_all_new_ancestors(struct btrfs_trans_handle *trans, struct btrfs_key search_key; int ret; + trace_btrfs_log_all_new_ancestors_enter(trans, inode); + /* * For a single hard link case, go through a fast path that does not * need to iterate the fs/subvolume tree. */ - if (inode->vfs_inode.i_nlink < 2) - return log_new_ancestors_fast(trans, inode, parent, ctx); + if (inode->vfs_inode.i_nlink < 2) { + ret = log_new_ancestors_fast(trans, inode, parent, ctx); + goto out; + } path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } search_key.objectid = ino; search_key.type = BTRFS_INODE_REF_KEY; @@ -7474,7 +7536,7 @@ static int log_all_new_ancestors(struct btrfs_trans_handle *trans, again: ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0); if (ret < 0) - return ret; + goto out; if (ret == 0) path->slots[0]++; @@ -7486,9 +7548,11 @@ again: if (slot >= btrfs_header_nritems(leaf)) { ret = btrfs_next_leaf(root, path); if (ret < 0) - return ret; - if (ret > 0) + goto out; + if (ret > 0) { + ret = 0; break; + } continue; } @@ -7504,8 +7568,10 @@ again: * this loop, etc). So just return some error to fallback to * a transaction commit. */ - if (found_key.type == BTRFS_INODE_EXTREF_KEY) - return -EMLINK; + if (found_key.type == BTRFS_INODE_EXTREF_KEY) { + ret = -EMLINK; + goto out; + } /* * Logging ancestors needs to do more searches on the fs/subvol @@ -7517,11 +7583,13 @@ again: ret = log_new_ancestors(trans, root, path, ctx); if (ret) - return ret; + goto out; btrfs_release_path(path); goto again; } - return 0; +out: + trace_btrfs_log_all_new_ancestors_exit(trans, inode, ret); + return ret; } /* @@ -7533,7 +7601,7 @@ again: static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, struct dentry *parent, - int inode_only, + enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx) { struct btrfs_root *root = inode->root; @@ -7541,29 +7609,39 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, int ret = 0; bool log_dentries; - if (btrfs_test_opt(fs_info, NOTREELOG)) - return BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_log_inode_parent_enter(trans, inode); + + if (btrfs_test_opt(fs_info, NOTREELOG)) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } - if (btrfs_root_refs(&root->root_item) == 0) - return BTRFS_LOG_FORCE_COMMIT; + if (btrfs_root_refs(&root->root_item) == 0) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } /* * If we're logging an inode from a subvolume created in the current * transaction we must force a commit since the root is not persisted. */ - if (btrfs_root_generation(&root->root_item) == trans->transid) - return BTRFS_LOG_FORCE_COMMIT; + if (btrfs_root_generation(&root->root_item) == trans->transid) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } /* Skip already logged inodes and without new extents. */ if (btrfs_inode_in_log(inode, trans->transid) && - list_empty(&ctx->ordered_extents)) - return BTRFS_NO_LOG_SYNC; + list_empty(&ctx->ordered_extents)) { + ret = BTRFS_NO_LOG_SYNC; + goto out; + } ret = start_log_trans(trans, root, ctx); if (ret) - return ret; + goto out; - ret = btrfs_log_inode(trans, inode, inode_only, ctx); + ret = btrfs_log_inode(trans, inode, log_mode, ctx); if (ret) goto end_trans; @@ -7649,6 +7727,9 @@ end_trans: btrfs_remove_log_ctx(root, ctx); btrfs_end_log_trans(root); +out: + trace_btrfs_log_inode_parent_exit(trans, inode, ret); + return ret; } @@ -7872,6 +7953,8 @@ void btrfs_record_unlink_dir(struct btrfs_trans_handle *trans, struct btrfs_inode *dir, struct btrfs_inode *inode, bool for_rename) { + trace_btrfs_record_unlink_dir(trans, dir, inode, for_rename); + /* * when we're logging a file, if it hasn't been renamed * or unlinked, and its inode is fully committed on disk, @@ -7934,6 +8017,8 @@ void btrfs_record_unlink_dir(struct btrfs_trans_handle *trans, void btrfs_record_snapshot_destroy(struct btrfs_trans_handle *trans, struct btrfs_inode *dir) { + trace_btrfs_record_snapshot_destroy(trans, dir); + mutex_lock(&dir->log_mutex); dir->last_unlink_trans = trans->transid; mutex_unlock(&dir->log_mutex); @@ -7954,6 +8039,8 @@ void btrfs_record_snapshot_destroy(struct btrfs_trans_handle *trans, void btrfs_record_new_subvolume(const struct btrfs_trans_handle *trans, struct btrfs_inode *dir) { + trace_btrfs_record_new_subvolume(trans, dir); + mutex_lock(&dir->log_mutex); dir->last_unlink_trans = trans->transid; mutex_unlock(&dir->log_mutex); @@ -7986,6 +8073,8 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, bool log_pinned = false; int ret; + trace_btrfs_log_new_name_enter(trans, inode, old_dir, old_dir_index); + /* The inode has a new name (ref/extref), so make sure we log it. */ set_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); @@ -8008,7 +8097,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, goto out; } else if (ret == 0) { if (!old_dir) - return; + goto out; /* * If the inode was not logged and we are doing a rename (old_dir is not * NULL), check if old_dir was logged - if it was not we can return and @@ -8018,7 +8107,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, if (ret < 0) goto out; else if (ret == 0) - return; + goto out; } ret = 0; @@ -8117,6 +8206,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, btrfs_log_inode_parent(trans, inode, parent, LOG_INODE_EXISTS, &ctx); ASSERT(list_empty(&ctx.conflict_inodes)); out: + trace_btrfs_log_new_name_exit(trans, inode, old_dir, ret); /* * If an error happened mark the log for a full commit because it's not * consistent and up to date or we couldn't find out if one of the diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h index 4a626dc6a58b..81ab5eeeb974 100644 --- a/fs/btrfs/tree-log.h +++ b/fs/btrfs/tree-log.h @@ -11,6 +11,13 @@ #include <linux/fscrypt.h> #include "transaction.h" +enum btrfs_log_mode { + /* Log everything about an inode. */ + LOG_INODE_ALL, + /* Log just enough to recreate the inode during log replay. */ + LOG_INODE_EXISTS, +}; + struct inode; struct dentry; struct btrfs_ordered_extent; diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 0062b3a55781..983365a73541 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -458,7 +458,7 @@ static int rollback_verity(struct btrfs_inode *inode) if (ret) { btrfs_handle_fs_error(root->fs_info, ret, "failed to drop verity items in rollback %llu", - (u64)inode->vfs_inode.i_ino); + inode->vfs_inode.i_ino); goto out; } @@ -472,7 +472,7 @@ static int rollback_verity(struct btrfs_inode *inode) trans = NULL; btrfs_handle_fs_error(root->fs_info, ret, "failed to start transaction in verity rollback %llu", - (u64)inode->vfs_inode.i_ino); + inode->vfs_inode.i_ino); goto out; } inode->ro_flags &= ~BTRFS_INODE_RO_VERITY; diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index a88e68f90564..6eab4cc73ce4 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -963,6 +963,11 @@ static noinline struct btrfs_device *device_list_add(const char *path, devid, btrfs_dev_name(device), path, current->comm, task_pid_nr(current)); + } else { + btrfs_info(NULL, + "missing devid %llu re-appeared at %s scanned by %s (%d)", + devid, path, current->comm, + task_pid_nr(current)); } name = kstrdup(path, GFP_NOFS); @@ -1369,9 +1374,9 @@ struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev, (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT); } - filemap_invalidate_lock(mapping); + filemap_invalidate_lock_shared(mapping); page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); - filemap_invalidate_unlock(mapping); + filemap_invalidate_unlock_shared(mapping); if (IS_ERR(page)) return ERR_CAST(page); @@ -2286,6 +2291,38 @@ void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct btrfs_devic update_dev_time(rcu_dereference_raw(device->name)); } +int btrfs_remove_dev_stat_item(struct btrfs_trans_handle *trans, u64 devid) +{ + BTRFS_PATH_AUTO_RELEASE(path); + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *dev_root = fs_info->dev_root; + struct btrfs_key key; + int ret; + + key.objectid = BTRFS_DEV_STATS_OBJECTID; + key.type = BTRFS_PERSISTENT_ITEM_KEY; + key.offset = devid; + + ret = btrfs_search_slot(trans, dev_root, &key, &path, -1, 1); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while searching for dev_stats item for devid %llu", + ret, devid); + return ret; + } + /* The dev stats item does not exist, nothing to bother. */ + if (ret > 0) + return 0; + ret = btrfs_del_item(trans, dev_root, &path); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while deleting dev_stats item for devid %llu", + ret, devid); + return ret; + } + return 0; +} + int btrfs_rm_device(struct btrfs_fs_info *fs_info, struct btrfs_dev_lookup_args *args, struct file **bdev_file) @@ -2365,6 +2402,12 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, return ret; } + ret = btrfs_remove_dev_stat_item(trans, device->devid); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); btrfs_scrub_cancel_dev(device); @@ -2889,6 +2932,12 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path device->commit_total_bytes = device->total_bytes; set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + + /* + * Increase dev_stats_ccnt so that corresponding DEV_STATS item can be + * created at the next transaction commit. + */ + atomic_inc(&device->dev_stats_ccnt); device->dev_stats_valid = 1; set_blocksize(device->bdev_file, BTRFS_BDEV_BLOCKSIZE); @@ -3718,7 +3767,11 @@ static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info, u64 chunk_type; cache = btrfs_lookup_block_group(fs_info, chunk_offset); - ASSERT(cache); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } chunk_type = cache->flags; btrfs_put_block_group(cache); @@ -3957,16 +4010,21 @@ static bool chunk_profiles_filter(u64 chunk_type, struct btrfs_balance_args *bar return true; } -static bool chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, - struct btrfs_balance_args *bargs) +static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, + struct btrfs_balance_args *bargs) { struct btrfs_block_group *cache; u64 chunk_used; u64 user_thresh_min; u64 user_thresh_max; - bool ret = true; + int ret = 1; cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } chunk_used = cache->used; if (bargs->usage_min == 0) @@ -3982,20 +4040,25 @@ static bool chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_of user_thresh_max = mult_perc(cache->length, bargs->usage_max); if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max) - ret = false; + ret = 0; btrfs_put_block_group(cache); return ret; } -static bool chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, - struct btrfs_balance_args *bargs) +static int chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, + struct btrfs_balance_args *bargs) { struct btrfs_block_group *cache; u64 chunk_used, user_thresh; - bool ret = true; + int ret = 1; cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } chunk_used = cache->used; if (bargs->usage_min == 0) @@ -4006,7 +4069,7 @@ static bool chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, user_thresh = mult_perc(cache->length, bargs->usage); if (chunk_used < user_thresh) - ret = false; + ret = 0; btrfs_put_block_group(cache); return ret; @@ -4111,8 +4174,8 @@ static bool chunk_soft_convert_filter(u64 chunk_type, struct btrfs_balance_args return false; } -static bool should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk *chunk, - u64 chunk_offset) +static int should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk *chunk, + u64 chunk_offset) { struct btrfs_fs_info *fs_info = leaf->fs_info; struct btrfs_balance_control *bctl = fs_info->balance_ctl; @@ -4145,12 +4208,22 @@ static bool should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk } /* usage filter */ - if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) && - chunk_usage_filter(fs_info, chunk_offset, bargs)) { - return false; - } else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && - chunk_usage_range_filter(fs_info, chunk_offset, bargs)) { - return false; + if (bargs->flags & BTRFS_BALANCE_ARGS_USAGE) { + int ret2; + + ret2 = chunk_usage_filter(fs_info, chunk_offset, bargs); + if (ret2 < 0) + return ret2; + if (ret2) + return false; + } else if (bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) { + int ret2; + + ret2 = chunk_usage_range_filter(fs_info, chunk_offset, bargs); + if (ret2 < 0) + return ret2; + if (ret2) + return false; } /* devid filter */ @@ -4337,7 +4410,7 @@ static int __btrfs_balance(struct btrfs_fs_info *fs_info) u32 count_data = 0; u32 count_meta = 0; u32 count_sys = 0; - int chunk_reserved = 0; + bool chunk_reserved = false; struct remap_chunk_info *rci; unsigned int num_remap_chunks = 0; LIST_HEAD(remap_chunks); @@ -4430,6 +4503,10 @@ again: ret = should_balance_chunk(leaf, chunk, found_key.offset); btrfs_release_path(path); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto error; + } if (!ret) { mutex_unlock(&fs_info->reclaim_bgs_lock); goto loop; @@ -4502,7 +4579,7 @@ again: mutex_unlock(&fs_info->reclaim_bgs_lock); goto error; } else if (ret == 1) { - chunk_reserved = 1; + chunk_reserved = true; } } @@ -4763,7 +4840,7 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, { u64 meta_target, data_target; u64 allowed; - int mixed = 0; + bool mixed = false; int ret; u64 num_devices; unsigned seq; @@ -4780,7 +4857,7 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, allowed = btrfs_super_incompat_flags(fs_info->super_copy); if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) - mixed = 1; + mixed = true; /* * In case of mixed groups both data and meta should be picked, @@ -6053,7 +6130,7 @@ struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans, lockdep_assert_held(&info->chunk_mutex); - if (!alloc_profile_is_valid(type, 0)) { + if (unlikely(!alloc_profile_is_valid(type, 0))) { DEBUG_WARN("invalid alloc profile for type %llu", type); return ERR_PTR(-EINVAL); } @@ -6064,7 +6141,7 @@ struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans, return ERR_PTR(-ENOSPC); } - if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) { + if (unlikely(!(type & BTRFS_BLOCK_GROUP_TYPE_MASK))) { btrfs_err(info, "invalid chunk type 0x%llx requested", type); DEBUG_WARN(); return ERR_PTR(-EINVAL); @@ -6234,7 +6311,7 @@ static noinline int init_first_rw_device(struct btrfs_trans_handle *trans) alloc_profile = btrfs_metadata_alloc_profile(fs_info); meta_space_info = btrfs_find_space_info(fs_info, alloc_profile); - if (!meta_space_info) { + if (unlikely(!meta_space_info)) { DEBUG_WARN(); return -EINVAL; } @@ -6244,7 +6321,7 @@ static noinline int init_first_rw_device(struct btrfs_trans_handle *trans) alloc_profile = btrfs_system_alloc_profile(fs_info); sys_space_info = btrfs_find_space_info(fs_info, alloc_profile); - if (!sys_space_info) { + if (unlikely(!sys_space_info)) { DEBUG_WARN(); return -EINVAL; } @@ -8137,8 +8214,8 @@ static int btrfs_device_init_dev_stats(struct btrfs_device *device, for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) { if (item_size >= (1 + i) * sizeof(__le64)) - btrfs_dev_stat_set(device, i, - btrfs_dev_stats_value(eb, ptr, i)); + atomic_set(device->dev_stat_values + i, + btrfs_dev_stats_value(eb, ptr, i)); else btrfs_dev_stat_set(device, i, 0); } @@ -8179,6 +8256,37 @@ out: return ret; } +int btrfs_init_writeback_bio_size(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u32 writeback_bio_size = fs_info->sectorsize; + + mutex_lock(&fs_devices->device_list_mutex); + /* + * Let's take maximum over optimal request sizes for all devices. For + * RAID profiles writeback will submit stripe (64k) sized bios anyway + * so our value doesn't matter and for simple profiles this is a good + * approximation of sensible IO chunking. + */ + list_for_each_entry(device, &fs_devices->devices, dev_list) { + struct request_queue *queue; + unsigned int io_opt; + + if (!device->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) + continue; + queue = bdev_get_queue(device->bdev); + io_opt = queue_io_opt(queue) ? : + queue_max_sectors(queue) << SECTOR_SHIFT; + writeback_bio_size = max(writeback_bio_size, io_opt); + } + mutex_unlock(&fs_devices->device_list_mutex); + + fs_info->writeback_bio_size = writeback_bio_size; + + return 0; +} + static int update_dev_stat_item(struct btrfs_trans_handle *trans, struct btrfs_device *device) { diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 0082c166af91..63be45c3298c 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -784,6 +784,7 @@ int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info, struct btrfs_ioctl_get_dev_stats *stats); int btrfs_init_devices_late(struct btrfs_fs_info *fs_info); int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info); +int btrfs_init_writeback_bio_size(struct btrfs_fs_info *fs_info); int btrfs_run_dev_stats(struct btrfs_trans_handle *trans); void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev); void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev); @@ -932,6 +933,7 @@ bool btrfs_first_pending_extent(struct btrfs_device *device, u64 start, u64 len, u64 *pending_start, u64 *pending_end); bool btrfs_find_hole_in_pending_extents(struct btrfs_device *device, u64 *start, u64 *len, u64 min_hole_size); +int btrfs_remove_dev_stat_item(struct btrfs_trans_handle *trans, u64 devid); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 16dd87aa06f2..97f06dd01693 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -131,8 +131,10 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) - BTRFS_SUPER_INFO_SIZE; + filemap_invalidate_lock_shared(mapping); page[i] = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); + filemap_invalidate_unlock_shared(mapping); if (IS_ERR(page[i])) { if (i == 1) btrfs_release_disk_super(super[0]); @@ -354,12 +356,33 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) return ret; } +static int btrfs_get_max_active_zones(struct btrfs_device *device, + struct btrfs_zoned_device_info *zone_info) +{ + struct block_device *bdev = device->bdev; + int max_active_zones; + + if (unlikely(zone_info->nr_zones < BTRFS_MIN_ACTIVE_ZONES)) { + btrfs_err(device->fs_info, "zoned: not enough zones to mount filesystem: %u < %d", + zone_info->nr_zones, BTRFS_MIN_ACTIVE_ZONES); + return -EINVAL; + } + + max_active_zones = min_not_zero(bdev_max_active_zones(bdev), + bdev_max_open_zones(bdev)); + if (max_active_zones == 0) + max_active_zones = min(zone_info->nr_zones / 4, + BTRFS_DEFAULT_MAX_ACTIVE_ZONES); + + zone_info->max_active_zones = max(max_active_zones, BTRFS_MIN_ACTIVE_ZONES); + return 0; +} + int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) { struct btrfs_fs_info *fs_info = device->fs_info; struct btrfs_zoned_device_info *zone_info = NULL; struct block_device *bdev = device->bdev; - unsigned int max_active_zones; unsigned int nactive; sector_t nr_sectors; sector_t sector = 0; @@ -424,19 +447,9 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) if (!IS_ALIGNED(nr_sectors, zone_sectors)) zone_info->nr_zones++; - max_active_zones = min_not_zero(bdev_max_active_zones(bdev), - bdev_max_open_zones(bdev)); - if (!max_active_zones && zone_info->nr_zones > BTRFS_DEFAULT_MAX_ACTIVE_ZONES) - max_active_zones = BTRFS_DEFAULT_MAX_ACTIVE_ZONES; - if (max_active_zones && max_active_zones < BTRFS_MIN_ACTIVE_ZONES) { - btrfs_err(fs_info, -"zoned: %s: max active zones %u is too small, need at least %u active zones", - rcu_dereference(device->name), max_active_zones, - BTRFS_MIN_ACTIVE_ZONES); - ret = -EINVAL; + ret = btrfs_get_max_active_zones(device, zone_info); + if (ret) goto out; - } - zone_info->max_active_zones = max_active_zones; zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); if (!zone_info->seq_zones) { @@ -517,26 +530,29 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) goto out; } - if (max_active_zones) { - if (unlikely(nactive > max_active_zones)) { - if (bdev_max_active_zones(bdev) == 0) { - max_active_zones = 0; - zone_info->max_active_zones = 0; - goto validate; - } + if (unlikely(nactive > zone_info->max_active_zones)) { + if (bdev_max_active_zones(bdev) > 0) { btrfs_err(device->fs_info, - "zoned: %u active zones on %s exceeds max_active_zones %u", - nactive, rcu_dereference(device->name), - max_active_zones); + "zoned: %u active zones on %s exceeds max_active_zones %u", + nactive, rcu_dereference(device->name), + zone_info->max_active_zones); ret = -EIO; goto out; } + + /* + * This is for backward compatibility with old filesystems that + * have a lot of active zones because the device doesn't report + * a maximum number of zones and we previously didn't care for + * the limit. + */ + zone_info->max_active_zones = 0; + } else { atomic_set(&zone_info->active_zones_left, - max_active_zones - nactive); + zone_info->max_active_zones - nactive); set_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags); } -validate: /* Validate superblock log */ nr_zones = BTRFS_NR_SB_LOG_ZONES; for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { @@ -1311,7 +1327,7 @@ static int btrfs_load_zone_info(struct btrfs_fs_info *fs_info, int zone_idx, { struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; struct btrfs_device *device; - int dev_replace_is_ongoing = 0; + bool dev_replace_is_ongoing = false; unsigned int nofs_flag; struct blk_zone zone; int ret; @@ -2763,7 +2779,6 @@ void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info) struct btrfs_block_group *bg; struct list_head *bg_list; u64 alloc_flags; - bool first = true; bool did_chunk_alloc = false; int index; int ret; @@ -2780,17 +2795,12 @@ void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info) alloc_flags = btrfs_get_alloc_profile(fs_info, space_info->flags); index = btrfs_bg_flags_to_raid_index(alloc_flags); - /* Scan the data space_info to find empty block groups. Take the second one. */ again: bg_list = &space_info->block_groups[index]; list_for_each_entry(bg, bg_list, list) { - if (bg->alloc_offset != 0) - continue; - if (first) { - first = false; + if (bg->alloc_offset != 0) continue; - } if (space_info == data_sinfo) { /* Migrate the block group to the data relocation space_info. */ @@ -2803,8 +2813,6 @@ again: down_write(&space_info->groups_sem); list_del_init(&bg->list); - /* We can assume this as we choose the second empty one. */ - ASSERT(!list_empty(&space_info->block_groups[index])); up_write(&space_info->groups_sem); spin_lock(&space_info->lock); @@ -2849,7 +2857,6 @@ again: * We allocated a new block group in the data relocation space_info. We * can take that one. */ - first = false; did_chunk_alloc = true; goto again; } diff --git a/fs/buffer.c b/fs/buffer.c index b0b3792b1496..9af5f061a1f8 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -54,9 +54,6 @@ #include "internal.h" -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint hint, struct writeback_control *wbc); - #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) inline void touch_buffer(struct buffer_head *bh) @@ -74,9 +71,7 @@ EXPORT_SYMBOL(__lock_buffer); void unlock_buffer(struct buffer_head *bh) { - clear_bit_unlock(BH_Lock, &bh->b_state); - smp_mb__after_atomic(); - wake_up_bit(&bh->b_state, BH_Lock); + clear_and_wake_up_bit(BH_Lock, &bh->b_state); } EXPORT_SYMBOL(unlock_buffer); @@ -132,15 +127,43 @@ static void buffer_io_error(struct buffer_head *bh, char *msg) bh->b_bdev, (unsigned long long)bh->b_blocknr, msg); } -/* - * End-of-IO handler helper function which does not touch the bh after - * unlocking it. - * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but - * a race there is benign: unlock_buffer() only use the bh's address for - * hashing after unlocking the buffer, so it doesn't actually touch the bh - * itself. +/** + * bio_endio_bh - Discard the bio used to submit a buffer. + * @bio: The bio. + * @bhp: Where to return the buffer_head. + * + * Call this in your bio_end_io handler to retrieve the buffer_head + * submitted in bh_submit(). If you did not call bh_submit(), do not + * call this function; it will return garbage. + * + * This function consumes the bio refcount which will probably free the + * bio. + * + * Return: True if the I/O succeeded. */ -static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) +bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp) +{ + bool success = bio->bi_status == BLK_STS_OK; + struct buffer_head *bh = bio->bi_private; + + if (unlikely(bio_flagged(bio, BIO_QUIET))) + set_bit(BH_Quiet, &bh->b_state); + bio_put(bio); + + *bhp = bh; + return success; +} +EXPORT_SYMBOL(bio_endio_bh); + +/** + * end_buffer_read_sync - Handle buffer reads finishing + * @bh: The buffer. + * @uptodate: True if the read was successful. + * + * If a buffer is read through a mechanism that isn't bh_submit(), you + * can call this function to finish the read. + */ +void end_buffer_read_sync(struct buffer_head *bh, int uptodate) { if (uptodate) { set_buffer_uptodate(bh); @@ -150,21 +173,36 @@ static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) } unlock_buffer(bh); } +EXPORT_SYMBOL(end_buffer_read_sync); -/* - * Default synchronous end-of-IO handler.. Just mark it up-to-date and - * unlock the buffer. +/** + * bh_end_read - I/O end handler for reads + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're reading into the buffer, + * unless you need your own special I/O end handler. */ -void end_buffer_read_sync(struct buffer_head *bh, int uptodate) +void bh_end_read(struct bio *bio) { - put_bh(bh); - __end_buffer_read_notouch(bh, uptodate); + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + end_buffer_read_sync(bh, uptodate); } -EXPORT_SYMBOL(end_buffer_read_sync); +EXPORT_SYMBOL(bh_end_read); -void end_buffer_write_sync(struct buffer_head *bh, int uptodate) +/** + * bh_end_write - I/O end handler for writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're writing from the buffer, + * unless you need your own special I/O end handler. + */ +void bh_end_write(struct bio *bio) { - if (uptodate) { + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); + + if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost sync page write"); @@ -172,9 +210,8 @@ void end_buffer_write_sync(struct buffer_head *bh, int uptodate) clear_buffer_uptodate(bh); } unlock_buffer(bh); - put_bh(bh); } -EXPORT_SYMBOL(end_buffer_write_sync); +EXPORT_SYMBOL(bh_end_write); static struct buffer_head * __find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic) @@ -342,11 +379,13 @@ static void decrypt_bh(struct work_struct *work) } /* - * I/O completion handler for block_read_full_folio() - pages + * I/O completion handler for block_read_full_folio() - folios * which come unlocked at the end of I/O. */ -static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) +static void bh_end_async_read(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); struct inode *inode = bh->b_folio->mapping->host; bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode); struct fsverity_info *vi = NULL; @@ -371,17 +410,24 @@ static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) } return; } - uptodate = 0; + uptodate = false; } end_buffer_async_read(bh, uptodate); } -/* - * Completion handler for block_write_full_folio() - folios which are unlocked - * during I/O, and which have the writeback flag cleared upon I/O completion. +/** + * bh_end_async_write - I/O end handler for async folio writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're doing the equivalent of + * block_write_full_folio(). That is, the folio is unlocked, and will + * have its writeback flag cleared once all async write buffers have + * completed. */ -static void end_buffer_async_write(struct buffer_head *bh, int uptodate) +void bh_end_async_write(struct bio *bio) { + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); unsigned long flags; struct buffer_head *first; struct buffer_head *tmp; @@ -390,7 +436,7 @@ static void end_buffer_async_write(struct buffer_head *bh, int uptodate) BUG_ON(!buffer_async_write(bh)); folio = bh->b_folio; - if (uptodate) { + if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost async page write"); @@ -418,46 +464,7 @@ static void end_buffer_async_write(struct buffer_head *bh, int uptodate) still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags); } - -/* - * If a page's buffers are under async readin (end_buffer_async_read - * completion) then there is a possibility that another thread of - * control could lock one of the buffers after it has completed - * but while some of the other buffers have not completed. This - * locked buffer would confuse end_buffer_async_read() into not unlocking - * the page. So the absence of BH_Async_Read tells end_buffer_async_read() - * that this buffer is not under async I/O. - * - * The page comes unlocked when it has no locked buffer_async buffers - * left. - * - * PageLocked prevents anyone starting new async I/O reads any of - * the buffers. - * - * PageWriteback is used to prevent simultaneous writeout of the same - * page. - * - * PageLocked prevents anyone from starting writeback of a page which is - * under read I/O (PageWriteback is only ever set against a locked page). - */ -static void mark_buffer_async_read(struct buffer_head *bh) -{ - bh->b_end_io = end_buffer_async_read_io; - set_buffer_async_read(bh); -} - -static void mark_buffer_async_write_endio(struct buffer_head *bh, - bh_end_io_t *handler) -{ - bh->b_end_io = handler; - set_buffer_async_write(bh); -} - -void mark_buffer_async_write(struct buffer_head *bh) -{ - mark_buffer_async_write_endio(bh, end_buffer_async_write); -} -EXPORT_SYMBOL(mark_buffer_async_write); +EXPORT_SYMBOL(bh_end_async_write); /* @@ -916,7 +923,6 @@ static sector_t folio_init_buffers(struct folio *folio, do { if (!buffer_mapped(bh)) { - bh->b_end_io = NULL; bh->b_private = NULL; bh->b_bdev = bdev; bh->b_blocknr = block; @@ -1157,6 +1163,83 @@ void __bforget(struct buffer_head *bh) } EXPORT_SYMBOL(__bforget); +static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, + gfp_t gfp_mask) +{ + const struct address_space *mapping = folio_mapping(bh->b_folio); + + /* + * The ext4 journal (jbd2) can submit a buffer_head it directly created + * for a non-pagecache page. fscrypt doesn't care about these. + */ + if (!mapping) + return; + fscrypt_set_bio_crypt_ctx(bio, mapping->host, + folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask); +} + +static void __bh_submit(struct buffer_head *bh, blk_opf_t opf, + enum rw_hint write_hint, struct writeback_control *wbc, + bio_end_io_t end_bio) +{ + const enum req_op op = opf & REQ_OP_MASK; + struct bio *bio; + + BUG_ON(!buffer_locked(bh)); + BUG_ON(!buffer_mapped(bh)); + BUG_ON(buffer_delay(bh)); + BUG_ON(buffer_unwritten(bh)); + + /* + * Only clear out a write error when rewriting + */ + if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) + clear_buffer_write_io_error(bh); + + if (buffer_meta(bh)) + opf |= REQ_META; + if (buffer_prio(bh)) + opf |= REQ_PRIO; + + bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); + + if (IS_ENABLED(CONFIG_FS_ENCRYPTION)) + buffer_set_crypto_ctx(bio, bh, GFP_NOIO); + + bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); + bio->bi_write_hint = write_hint; + + bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); + + bio->bi_end_io = end_bio; + bio->bi_private = bh; + + /* Take care of bh's that straddle the end of the device */ + guard_bio_eod(bio); + + if (wbc) { + wbc_init_bio(wbc, bio); + wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); + } + + blk_crypto_submit_bio(bio); +} + +/** + * bh_submit - Start I/O against a buffer head + * @bh: The buffer head to perform I/O on. + * @opf: Operation and flags for bio. + * @end_io: The routine to call when I/O has completed. + * + * If you need to do I/O on an individual bh (instead of allowing the + * page cache to do I/O on the folio that it is in), call this function. + */ +void bh_submit(struct buffer_head *bh, blk_opf_t opf, bio_end_io_t end_io) +{ + __bh_submit(bh, opf, WRITE_LIFE_NOT_SET, NULL, end_io); +} +EXPORT_SYMBOL(bh_submit); + static struct buffer_head *__bread_slow(struct buffer_head *bh) { lock_buffer(bh); @@ -1164,9 +1247,7 @@ static struct buffer_head *__bread_slow(struct buffer_head *bh) unlock_buffer(bh); return bh; } else { - get_bh(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); wait_on_buffer(bh); if (buffer_uptodate(bh)) return bh; @@ -1716,15 +1797,15 @@ static struct buffer_head *folio_create_buffers(struct folio *folio, /* * While block_write_full_folio is writing back the dirty buffers under - * the page lock, whoever dirtied the buffers may decide to clean them + * the folio lock, whoever dirtied the buffers may decide to clean them * again at any time. We handle that by only looking at the buffer * state inside lock_buffer(). * * If block_write_full_folio() is called for regular writeback - * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a - * locked buffer. This only can happen if someone has written the buffer - * directly, with submit_bh(). At the address_space level PageWriteback - * prevents this contention from occurring. + * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a folio which + * has a locked buffer. This only can happen if someone has written + * the buffer directly, with bh_submit(). At the address_space level + * the folio writeback flag prevents this contention from occurring. * * If block_write_full_folio() is called with wbc->sync_mode == * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this @@ -1810,8 +1891,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, continue; } if (test_clear_buffer_dirty(bh)) { - mark_buffer_async_write_endio(bh, - end_buffer_async_write); + set_buffer_async_write(bh); } else { unlock_buffer(bh); } @@ -1827,8 +1907,9 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, - inode->i_write_hint, wbc); + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); nr_underway++; } bh = next; @@ -1841,7 +1922,7 @@ done: /* * The folio was marked dirty, but the buffers were * clean. Someone wrote them back by hand with - * write_dirty_buffer/submit_bh. A rare case. + * write_dirty_buffer/bh_submit. A rare case. */ folio_end_writeback(folio); @@ -1865,8 +1946,7 @@ recover: if (buffer_mapped(bh) && buffer_dirty(bh) && !buffer_delay(bh)) { lock_buffer(bh); - mark_buffer_async_write_endio(bh, - end_buffer_async_write); + set_buffer_async_write(bh); } else { /* * The buffer may have been set dirty during @@ -1882,8 +1962,9 @@ recover: struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { clear_buffer_dirty(bh); - submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, - inode->i_write_hint, wbc); + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); nr_underway++; } bh = next; @@ -2339,9 +2420,33 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) continue; } - mark_buffer_async_read(bh); + /* + * If a folio's buffers are under async readin + * (end_buffer_async_read completion) then there is a + * possibility that another thread of control could lock + * one of the buffers after it has completed but while + * some of the other buffers have not completed. This + * locked buffer would confuse end_buffer_async_read() + * into not unlocking the folio. So the absence of + * BH_Async_Read tells end_buffer_async_read() that this + * buffer is not under async I/O. + * + * The folio comes unlocked when it has no locked + * buffer_async buffers left. + * + * The folio lock prevents anyone starting new async + * I/O reads into any of the buffers. + * + * The writeback flag is used to prevent simultaneous + * writeout of the same folio. + * + * The folio lock prevents anyone from starting writeback + * of a folio which is under read I/O (the writeback + * flag is only ever set on a locked folio). + */ + set_buffer_async_read(bh); if (prev) - submit_bh(REQ_OP_READ, prev); + bh_submit(prev, REQ_OP_READ, bh_end_async_read); prev = bh; } while (iblock++, (bh = bh->b_this_page) != head); @@ -2355,7 +2460,7 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) * in this folio. */ if (prev) - submit_bh(REQ_OP_READ, prev); + bh_submit(prev, REQ_OP_READ, bh_end_async_read); else folio_end_read(folio, !page_error); @@ -2663,86 +2768,6 @@ sector_t generic_block_bmap(struct address_space *mapping, sector_t block, } EXPORT_SYMBOL(generic_block_bmap); -static void end_bio_bh_io_sync(struct bio *bio) -{ - struct buffer_head *bh = bio->bi_private; - - if (unlikely(bio_flagged(bio, BIO_QUIET))) - set_bit(BH_Quiet, &bh->b_state); - - bh->b_end_io(bh, !bio->bi_status); - bio_put(bio); -} - -static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, - gfp_t gfp_mask) -{ - const struct address_space *mapping = folio_mapping(bh->b_folio); - - /* - * The ext4 journal (jbd2) can submit a buffer_head it directly created - * for a non-pagecache page. fscrypt doesn't care about these. - */ - if (!mapping) - return; - fscrypt_set_bio_crypt_ctx(bio, mapping->host, - folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask); -} - -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint write_hint, - struct writeback_control *wbc) -{ - const enum req_op op = opf & REQ_OP_MASK; - struct bio *bio; - - BUG_ON(!buffer_locked(bh)); - BUG_ON(!buffer_mapped(bh)); - BUG_ON(!bh->b_end_io); - BUG_ON(buffer_delay(bh)); - BUG_ON(buffer_unwritten(bh)); - - /* - * Only clear out a write error when rewriting - */ - if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) - clear_buffer_write_io_error(bh); - - if (buffer_meta(bh)) - opf |= REQ_META; - if (buffer_prio(bh)) - opf |= REQ_PRIO; - - bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); - - if (IS_ENABLED(CONFIG_FS_ENCRYPTION)) - buffer_set_crypto_ctx(bio, bh, GFP_NOIO); - - bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); - bio->bi_write_hint = write_hint; - - bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); - - bio->bi_end_io = end_bio_bh_io_sync; - bio->bi_private = bh; - - /* Take care of bh's that straddle the end of the device */ - guard_bio_eod(bio); - - if (wbc) { - wbc_init_bio(wbc, bio); - wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); - } - - blk_crypto_submit_bio(bio); -} - -void submit_bh(blk_opf_t opf, struct buffer_head *bh) -{ - submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL); -} -EXPORT_SYMBOL(submit_bh); - void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) { lock_buffer(bh); @@ -2750,9 +2775,7 @@ void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) unlock_buffer(bh); return; } - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - submit_bh(REQ_OP_WRITE | op_flags, bh); + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); } EXPORT_SYMBOL(write_dirty_buffer); @@ -2775,9 +2798,7 @@ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) return -EIO; } - get_bh(bh); - bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | op_flags, bh); + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); wait_on_buffer(bh); if (!buffer_uptodate(bh)) return -EIO; @@ -3009,9 +3030,7 @@ int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait) BUG_ON(!buffer_locked(bh)); - get_bh(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ | op_flags, bh); + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); if (wait) { wait_on_buffer(bh); if (!buffer_uptodate(bh)) @@ -3053,9 +3072,7 @@ void __bh_read_batch(int nr, struct buffer_head *bhs[], continue; } - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ | op_flags, bh); + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); } } EXPORT_SYMBOL(__bh_read_batch); diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index 1b83ed0e0a63..8a9f6be15828 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -130,6 +130,8 @@ retry: ret = cachefiles_inject_write_error(); if (ret == 0) { subdir = vfs_mkdir(&nop_mnt_idmap, d_inode(dir), subdir, 0700, NULL); + if (IS_ERR(subdir)) + ret = PTR_ERR(subdir); } else { end_creating(subdir); subdir = ERR_PTR(ret); @@ -207,7 +209,6 @@ lookup_error: return ERR_PTR(ret); nomem_d_alloc: - inode_unlock(d_inode(dir)); _leave(" = -ENOMEM"); return ERR_PTR(-ENOMEM); } @@ -373,7 +374,7 @@ try_again: "Rename failed with error %d", ret); } - __cachefiles_unmark_inode_in_use(object, d_inode(rep)); + cachefiles_do_unmark_inode_in_use(object, d_inode(rep)); end_renaming(&rd); _leave(" = 0"); return 0; @@ -465,7 +466,6 @@ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) ret = -EINVAL; if (unlikely(!file->f_op->read_iter) || unlikely(!file->f_op->write_iter)) { - fput(file); pr_notice("Cache does not support read_iter and write_iter\n"); goto err_unuse; } diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 1454760332ff..61bd6d92ff25 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1336,6 +1336,7 @@ void ceph_process_folio_batch(struct address_space *mapping, ceph_wbc, folio); if (rc == -ENODATA) { folio_unlock(folio); + folio_put(folio); ceph_wbc->fbatch.folios[i] = NULL; continue; } else if (rc == -E2BIG) { @@ -1346,6 +1347,7 @@ void ceph_process_folio_batch(struct address_space *mapping, if (!folio_clear_dirty_for_io(folio)) { doutc(cl, "%p !folio_clear_dirty_for_io\n", folio); folio_unlock(folio); + folio_put(folio); ceph_wbc->fbatch.folios[i] = NULL; continue; } @@ -2559,7 +2561,8 @@ int ceph_pool_perm_check(struct inode *inode, int need) struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_string *pool_ns; s64 pool; - int ret, flags; + int ret; + unsigned long flags; /* Only need to do this for regular files */ if (!S_ISREG(inode->i_mode)) @@ -2601,20 +2604,19 @@ check: if (ret < 0) return ret; - flags = CEPH_I_POOL_PERM; - if (ret & POOL_READ) - flags |= CEPH_I_POOL_RD; - if (ret & POOL_WRITE) - flags |= CEPH_I_POOL_WR; - spin_lock(&ci->i_ceph_lock); if (pool == ci->i_layout.pool_id && pool_ns == rcu_dereference_raw(ci->i_layout.pool_ns)) { - ci->i_ceph_flags |= flags; - } else { + set_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags); + if (ret & POOL_READ) + set_bit(CEPH_I_POOL_RD_BIT, &ci->i_ceph_flags); + if (ret & POOL_WRITE) + set_bit(CEPH_I_POOL_WR_BIT, &ci->i_ceph_flags); + } else { pool = ci->i_layout.pool_id; - flags = ci->i_ceph_flags; } + /* Re-read flags under the lock so check: sees the updated bits. */ + flags = ci->i_ceph_flags; spin_unlock(&ci->i_ceph_lock); goto check; } diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index d51454e995a8..4b37d9ffdf7f 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -549,7 +549,7 @@ static void __cap_delay_requeue_front(struct ceph_mds_client *mdsc, doutc(mdsc->fsc->client, "%p %llx.%llx\n", inode, ceph_vinop(inode)); spin_lock(&mdsc->cap_delay_lock); - ci->i_ceph_flags |= CEPH_I_FLUSH; + set_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags); if (!list_empty(&ci->i_cap_delay_list)) list_del_init(&ci->i_cap_delay_list); list_add(&ci->i_cap_delay_list, &mdsc->cap_delay_list); @@ -1409,7 +1409,7 @@ static void __prep_cap(struct cap_msg_args *arg, struct ceph_cap *cap, ceph_cap_string(revoking)); BUG_ON((retain & CEPH_CAP_PIN) == 0); - ci->i_ceph_flags &= ~CEPH_I_FLUSH; + clear_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags); cap->issued &= retain; /* drop bits we don't want */ /* @@ -1648,6 +1648,7 @@ static void __ceph_flush_snaps(struct ceph_inode_info *ci, spin_lock(&mdsc->cap_dirty_lock); capsnap->cap_flush.tid = ++mdsc->last_cap_flush_tid; + capsnap->cap_flush.ci = ci; list_add_tail(&capsnap->cap_flush.g_list, &mdsc->cap_flush_list); if (oldest_flush_tid == 0) @@ -1666,7 +1667,7 @@ static void __ceph_flush_snaps(struct ceph_inode_info *ci, last_tid = capsnap->cap_flush.tid; } - ci->i_ceph_flags &= ~CEPH_I_FLUSH_SNAPS; + clear_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); while (first_tid <= last_tid) { struct ceph_cap *cap = ci->i_auth_cap; @@ -1846,6 +1847,7 @@ struct ceph_cap_flush *ceph_alloc_cap_flush(void) return NULL; cf->is_capsnap = false; + cf->ci = NULL; return cf; } @@ -1931,6 +1933,7 @@ static u64 __mark_caps_flushing(struct inode *inode, doutc(cl, "%p %llx.%llx now !dirty\n", inode, ceph_vinop(inode)); swap(cf, ci->i_prealloc_cap_flush); + cf->ci = ci; cf->caps = flushing; cf->wake = wake; @@ -2026,7 +2029,7 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags) spin_lock(&ci->i_ceph_lock); if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) { - ci->i_ceph_flags |= CEPH_I_ASYNC_CHECK_CAPS; + set_bit(CEPH_I_ASYNC_CHECK_CAPS_BIT, &ci->i_ceph_flags); /* Don't send messages until we get async create reply */ spin_unlock(&ci->i_ceph_lock); @@ -2577,7 +2580,7 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) return; - ci->i_ceph_flags &= ~CEPH_I_KICK_FLUSH; + clear_bit(CEPH_I_KICK_FLUSH_BIT, &ci->i_ceph_flags); list_for_each_entry_reverse(cf, &ci->i_cap_flush_list, i_list) { if (cf->is_capsnap) { @@ -2686,7 +2689,7 @@ void ceph_early_kick_flushing_caps(struct ceph_mds_client *mdsc, __kick_flushing_caps(mdsc, session, ci, oldest_flush_tid); } else { - ci->i_ceph_flags |= CEPH_I_KICK_FLUSH; + set_bit(CEPH_I_KICK_FLUSH_BIT, &ci->i_ceph_flags); } spin_unlock(&ci->i_ceph_lock); @@ -2829,7 +2832,7 @@ again: spin_lock(&ci->i_ceph_lock); if ((flags & CHECK_FILELOCK) && - (ci->i_ceph_flags & CEPH_I_ERROR_FILELOCK)) { + test_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags)) { doutc(cl, "%p %llx.%llx error filelock\n", inode, ceph_vinop(inode)); ret = -EIO; @@ -3207,7 +3210,7 @@ static int ceph_try_drop_cap_snap(struct ceph_inode_info *ci, BUG_ON(capsnap->cap_flush.tid > 0); ceph_put_snap_context(capsnap->context); if (!list_is_last(&capsnap->ci_item, &ci->i_cap_snaps)) - ci->i_ceph_flags |= CEPH_I_FLUSH_SNAPS; + set_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); list_del(&capsnap->ci_item); ceph_put_cap_snap(capsnap); @@ -3396,7 +3399,7 @@ void ceph_put_wrbuffer_cap_refs(struct ceph_inode_info *ci, int nr, if (ceph_try_drop_cap_snap(ci, capsnap)) { put++; } else { - ci->i_ceph_flags |= CEPH_I_FLUSH_SNAPS; + set_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); flush_snaps = true; } } @@ -3648,7 +3651,7 @@ static void handle_cap_grant(struct inode *inode, if (ci->i_layout.pool_id != old_pool || extra_info->pool_ns != old_ns) - ci->i_ceph_flags &= ~CEPH_I_POOL_PERM; + clear_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags); extra_info->pool_ns = old_ns; @@ -3826,6 +3829,13 @@ static void handle_cap_flush_ack(struct inode *inode, u64 flush_tid, bool wake_ci = false; bool wake_mdsc = false; + /* + * Flush tids are monotonically increasing and acks arrive in + * order under i_ceph_lock, so this is always the latest tid. + * Diagnostic readers use READ_ONCE() without holding the lock. + */ + WRITE_ONCE(ci->i_last_cap_flush_ack, flush_tid); + list_for_each_entry_safe(cf, tmp_cf, &ci->i_cap_flush_list, i_list) { /* Is this the one that was flushed? */ if (cf->tid == flush_tid) @@ -4815,7 +4825,7 @@ int ceph_drop_caps_for_unlink(struct inode *inode) doutc(mdsc->fsc->client, "%p %llx.%llx\n", inode, ceph_vinop(inode)); spin_lock(&mdsc->cap_delay_lock); - ci->i_ceph_flags |= CEPH_I_FLUSH; + set_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags); if (!list_empty(&ci->i_cap_delay_list)) list_del_init(&ci->i_cap_delay_list); list_add_tail(&ci->i_cap_delay_list, @@ -5080,7 +5090,7 @@ int ceph_purge_inode_cap(struct inode *inode, struct ceph_cap *cap, bool *invali if (atomic_read(&ci->i_filelock_ref) > 0) { /* make further file lock syscall return -EIO */ - ci->i_ceph_flags |= CEPH_I_ERROR_FILELOCK; + set_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags); pr_warn_ratelimited_client(cl, " dropping file locks for %p %llx.%llx\n", inode, ceph_vinop(inode)); diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c index e2463f93cf6b..18eb5da03411 100644 --- a/fs/ceph/debugfs.c +++ b/fs/ceph/debugfs.c @@ -9,6 +9,7 @@ #include <linux/seq_file.h> #include <linux/math64.h> #include <linux/ktime.h> +#include <linux/uaccess.h> #include <linux/atomic.h> #include <linux/ceph/libceph.h> @@ -392,6 +393,90 @@ static int status_show(struct seq_file *s, void *p) return 0; } +static int reset_status_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_client_reset_state *st; + u64 trigger = 0, success = 0, failure = 0; + unsigned long last_start = 0, last_finish = 0; + int last_errno = 0; + enum ceph_client_reset_phase phase = CEPH_CLIENT_RESET_IDLE; + bool drain_timed_out = false; + int sessions_reset = 0; + int blocked_requests = 0; + char reason[CEPH_CLIENT_RESET_REASON_LEN]; + + if (!mdsc) + return 0; + + st = &mdsc->reset_state; + + spin_lock(&st->lock); + trigger = st->trigger_count; + success = st->success_count; + failure = st->failure_count; + last_start = st->last_start; + last_finish = st->last_finish; + last_errno = st->last_errno; + phase = st->phase; + drain_timed_out = st->drain_timed_out; + sessions_reset = st->sessions_reset; + strscpy(reason, st->last_reason, sizeof(reason)); + spin_unlock(&st->lock); + + blocked_requests = atomic_read(&st->blocked_requests); + + seq_printf(s, "phase: %s\n", ceph_reset_phase_name(phase)); + seq_printf(s, "trigger_count: %llu\n", trigger); + seq_printf(s, "success_count: %llu\n", success); + seq_printf(s, "failure_count: %llu\n", failure); + if (last_start) + seq_printf(s, "last_start_ms_ago: %u\n", + jiffies_to_msecs(jiffies - last_start)); + else + seq_puts(s, "last_start_ms_ago: (never)\n"); + if (last_finish) + seq_printf(s, "last_finish_ms_ago: %u\n", + jiffies_to_msecs(jiffies - last_finish)); + else + seq_puts(s, "last_finish_ms_ago: (never)\n"); + seq_printf(s, "last_errno: %d\n", last_errno); + seq_printf(s, "last_reason: %s\n", + reason[0] ? reason : "(none)"); + seq_printf(s, "drain_timed_out: %s\n", + drain_timed_out ? "yes" : "no"); + seq_printf(s, "sessions_reset: %d\n", sessions_reset); + seq_printf(s, "blocked_requests: %d\n", blocked_requests); + + return 0; +} + +static ssize_t reset_trigger_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct ceph_fs_client *fsc = file->private_data; + struct ceph_mds_client *mdsc = fsc->mdsc; + char reason[CEPH_CLIENT_RESET_REASON_LEN]; + size_t copy; + int ret; + + if (!mdsc) + return -ENODEV; + + copy = min_t(size_t, len, sizeof(reason) - 1); + if (copy && copy_from_user(reason, buf, copy)) + return -EFAULT; + reason[copy] = '\0'; + strim(reason); + + ret = ceph_mdsc_schedule_reset(mdsc, reason); + if (ret) + return ret; + + return len; +} + static int subvolume_metrics_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; @@ -450,6 +535,7 @@ DEFINE_SHOW_ATTRIBUTE(mdsc); DEFINE_SHOW_ATTRIBUTE(caps); DEFINE_SHOW_ATTRIBUTE(mds_sessions); DEFINE_SHOW_ATTRIBUTE(status); +DEFINE_SHOW_ATTRIBUTE(reset_status); DEFINE_SHOW_ATTRIBUTE(metrics_file); DEFINE_SHOW_ATTRIBUTE(metrics_latency); DEFINE_SHOW_ATTRIBUTE(metrics_size); @@ -521,6 +607,13 @@ static int metric_features_show(struct seq_file *s, void *p) DEFINE_SHOW_ATTRIBUTE(metric_features); +static const struct file_operations ceph_reset_trigger_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = reset_trigger_write, + .llseek = noop_llseek, +}; + /* * debugfs */ @@ -554,6 +647,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc) debugfs_remove(fsc->debugfs_caps); debugfs_remove(fsc->debugfs_status); debugfs_remove(fsc->debugfs_mdsc); + debugfs_remove_recursive(fsc->debugfs_reset_dir); debugfs_remove(fsc->debugfs_subvolume_metrics); debugfs_remove_recursive(fsc->debugfs_metrics_dir); doutc(fsc->client, "done\n"); @@ -602,6 +696,15 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &caps_fops); + fsc->debugfs_reset_dir = debugfs_create_dir("reset", + fsc->client->debugfs_dir); + debugfs_create_file("trigger", 0200, + fsc->debugfs_reset_dir, fsc, + &ceph_reset_trigger_fops); + debugfs_create_file("status", 0400, + fsc->debugfs_reset_dir, fsc, + &reset_status_fops); + fsc->debugfs_status = debugfs_create_file("status", 0400, fsc->client->debugfs_dir, diff --git a/fs/ceph/file.c b/fs/ceph/file.c index d54d71669176..71161f2b2151 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -598,12 +598,12 @@ static void wake_async_create_waiters(struct inode *inode, spin_lock(&ci->i_ceph_lock); if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) { - clear_and_wake_up_bit(CEPH_ASYNC_CREATE_BIT, &ci->i_ceph_flags); + /* Serialized by i_ceph_lock; the two ops touch different bits. */ + clear_and_wake_up_bit(CEPH_I_ASYNC_CREATE_BIT, &ci->i_ceph_flags); - if (ci->i_ceph_flags & CEPH_I_ASYNC_CHECK_CAPS) { - ci->i_ceph_flags &= ~CEPH_I_ASYNC_CHECK_CAPS; + if (test_and_clear_bit(CEPH_I_ASYNC_CHECK_CAPS_BIT, + &ci->i_ceph_flags)) check_cap = true; - } } ceph_kick_flushing_inode_caps(session, ci); spin_unlock(&ci->i_ceph_lock); @@ -766,7 +766,8 @@ static int ceph_finish_async_create(struct inode *dir, struct inode *inode, * that point and don't worry about setting * CEPH_I_ASYNC_CREATE. */ - ceph_inode(inode)->i_ceph_flags = CEPH_I_ASYNC_CREATE; + set_bit(CEPH_I_ASYNC_CREATE_BIT, + &ceph_inode(inode)->i_ceph_flags); unlock_new_inode(inode); } if (d_in_lookup(dentry) || d_really_is_negative(dentry)) { @@ -996,6 +997,10 @@ retry: ceph_init_inode_acls(newino, &as_ctx); file->f_mode |= FMODE_CREATED; } + if ((flags & __O_REGULAR) && !d_is_reg(dentry)) { + err = -EFTYPE; + goto out_req; + } err = finish_open(file, dentry, ceph_open); } out_req: @@ -2482,7 +2487,7 @@ retry_snap: if ((got & (CEPH_CAP_FILE_BUFFER|CEPH_CAP_FILE_LAZYIO)) == 0 || (iocb->ki_flags & IOCB_DIRECT) || (fi->flags & CEPH_F_SYNC) || - (ci->i_ceph_flags & CEPH_I_ERROR_WRITE)) { + test_bit(CEPH_I_ERROR_WRITE_BIT, &ci->i_ceph_flags)) { struct ceph_snap_context *snapc; struct iov_iter data; diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 22c7da1ea61c..61d7c0b8161f 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -671,6 +671,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) INIT_LIST_HEAD(&ci->i_cap_snaps); ci->i_head_snapc = NULL; ci->i_snap_caps = 0; + ci->i_last_cap_flush_ack = 0; ci->i_last_rd = ci->i_last_wr = jiffies - 3600 * HZ; for (i = 0; i < CEPH_FILE_MODE_BITS; i++) @@ -1180,7 +1181,7 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page, rcu_assign_pointer(ci->i_layout.pool_ns, pool_ns); if (ci->i_layout.pool_id != old_pool || pool_ns != old_ns) - ci->i_ceph_flags &= ~CEPH_I_POOL_PERM; + clear_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags); pool_ns = old_ns; @@ -3240,7 +3241,7 @@ void ceph_inode_shutdown(struct inode *inode) bool invalidate = false; spin_lock(&ci->i_ceph_lock); - ci->i_ceph_flags |= CEPH_I_SHUTDOWN; + set_bit(CEPH_I_SHUTDOWN_BIT, &ci->i_ceph_flags); p = rb_first(&ci->i_caps); while (p) { struct ceph_cap *cap = rb_entry(p, struct ceph_cap, ci_node); diff --git a/fs/ceph/locks.c b/fs/ceph/locks.c index dd764f9c64b9..677221bd64e0 100644 --- a/fs/ceph/locks.c +++ b/fs/ceph/locks.c @@ -57,9 +57,7 @@ static void ceph_fl_release_lock(struct file_lock *fl) ci = ceph_inode(inode); if (atomic_dec_and_test(&ci->i_filelock_ref)) { /* clear error when all locks are released */ - spin_lock(&ci->i_ceph_lock); - ci->i_ceph_flags &= ~CEPH_I_ERROR_FILELOCK; - spin_unlock(&ci->i_ceph_lock); + clear_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags); } fl->fl_u.ceph.inode = NULL; iput(inode); @@ -251,6 +249,7 @@ int ceph_lock(struct file *file, int cmd, struct file_lock *fl) { struct inode *inode = file_inode(file); struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); struct ceph_client *cl = ceph_inode_to_client(inode); int err = 0; u16 op = CEPH_MDS_OP_SETFILELOCK; @@ -271,15 +270,17 @@ int ceph_lock(struct file *file, int cmd, struct file_lock *fl) else if (IS_SETLKW(cmd)) wait = 1; - spin_lock(&ci->i_ceph_lock); - if (ci->i_ceph_flags & CEPH_I_ERROR_FILELOCK) { - err = -EIO; - } - spin_unlock(&ci->i_ceph_lock); - if (err < 0) { + if (test_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags)) { if (op == CEPH_MDS_OP_SETFILELOCK && lock_is_unlock(fl)) posix_lock_file(file, fl, NULL); - return err; + return -EIO; + } + + /* Wait for reset to complete before acquiring new locks */ + if (op == CEPH_MDS_OP_SETFILELOCK && !lock_is_unlock(fl)) { + err = ceph_mdsc_wait_for_reset(mdsc); + if (err) + return err; } if (lock_is_read(fl)) @@ -318,6 +319,7 @@ int ceph_flock(struct file *file, int cmd, struct file_lock *fl) { struct inode *inode = file_inode(file); struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); struct ceph_client *cl = ceph_inode_to_client(inode); int err = 0; u8 wait = 0; @@ -331,15 +333,17 @@ int ceph_flock(struct file *file, int cmd, struct file_lock *fl) doutc(cl, "fl_file: %p\n", fl->c.flc_file); - spin_lock(&ci->i_ceph_lock); - if (ci->i_ceph_flags & CEPH_I_ERROR_FILELOCK) { - err = -EIO; - } - spin_unlock(&ci->i_ceph_lock); - if (err < 0) { + if (test_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags)) { if (lock_is_unlock(fl)) locks_lock_file_wait(file, fl); - return err; + return -EIO; + } + + /* Wait for reset to complete before acquiring new locks */ + if (!lock_is_unlock(fl)) { + err = ceph_mdsc_wait_for_reset(mdsc); + if (err) + return err; } if (IS_SETLKW(cmd)) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index ed17e0023705..853bf698b356 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -6,6 +6,7 @@ #include <linux/slab.h> #include <linux/gfp.h> #include <linux/sched.h> +#include <linux/delay.h> #include <linux/debugfs.h> #include <linux/seq_file.h> #include <linux/ratelimit.h> @@ -65,6 +66,7 @@ static void __wake_requests(struct ceph_mds_client *mdsc, struct list_head *head); static void ceph_cap_release_work(struct work_struct *work); static void ceph_cap_reclaim_work(struct work_struct *work); +static void ceph_mdsc_reset_workfn(struct work_struct *work); static const struct ceph_connection_operations mds_con_ops; @@ -2267,7 +2269,7 @@ static int trim_caps_cb(struct inode *inode, int mds, void *arg) int count; dput(dentry); d_prune_aliases(inode); - count = icount_read(inode); + count = icount_read_once(inode); if (count == 1) (*remaining)--; doutc(cl, "%p %llx.%llx cap %p pruned, count now %d\n", @@ -2330,19 +2332,112 @@ static int check_caps_flush(struct ceph_mds_client *mdsc, } /* - * flush all dirty inode data to disk. + * Snapshot of a single cap_flush entry for diagnostic dump. + * Collected under cap_dirty_lock, printed after releasing it. + */ +struct flush_dump_entry { + u64 ino; /* inode number */ + u64 snap; /* snap id */ + int caps; /* dirty cap bits */ + u64 tid; /* flush transaction id */ + u64 last_ack; /* most recent ack tid for this inode */ + bool wake; /* whether completion was requested */ + bool is_capsnap; /* true if this is a cap snap flush */ + bool ci_null; /* true if cf->ci was unexpectedly NULL */ +}; + +/* + * Dump pending cap flushes for diagnostic purposes. * - * returns true if we've flushed through want_flush_tid + * cf->ci is safe to dereference here: cap_flush entries hold a + * reference on the inode (via the cap), and entries are removed from + * cap_flush_list under cap_dirty_lock before the cap (and thus the + * inode reference) is released. Holding cap_dirty_lock therefore + * guarantees the inode remains valid for the lifetime of the scan. + */ + +static void dump_cap_flushes(struct ceph_mds_client *mdsc, u64 want_tid) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct flush_dump_entry entries[CEPH_CAP_FLUSH_MAX_DUMP_ENTRIES]; + struct ceph_cap_flush *cf; + int n = 0, remaining = 0; + int i; + + spin_lock(&mdsc->cap_dirty_lock); + list_for_each_entry(cf, &mdsc->cap_flush_list, g_list) { + if (cf->tid > want_tid) + break; + if (n < CEPH_CAP_FLUSH_MAX_DUMP_ENTRIES) { + struct flush_dump_entry *e = &entries[n++]; + + e->ci_null = WARN_ON_ONCE(!cf->ci); + if (!e->ci_null) { + e->ino = ceph_ino(&cf->ci->netfs.inode); + e->snap = ceph_snap(&cf->ci->netfs.inode); + e->last_ack = READ_ONCE(cf->ci->i_last_cap_flush_ack); + } + e->caps = cf->caps; + e->tid = cf->tid; + e->wake = cf->wake; + e->is_capsnap = cf->is_capsnap; + } else { + remaining++; + } + } + spin_unlock(&mdsc->cap_dirty_lock); + + pr_info_client(cl, "still waiting for cap flushes through %llu:\n", + want_tid); + for (i = 0; i < n; i++) { + struct flush_dump_entry *e = &entries[i]; + + if (e->ci_null) + pr_info_client(cl, + " (null ci) %s tid=%llu wake=%d%s\n", + ceph_cap_string(e->caps), e->tid, + e->wake, + e->is_capsnap ? " is_capsnap" : ""); + else + pr_info_client(cl, + " %llx.%llx %s tid=%llu last_ack=%llu wake=%d%s\n", + e->ino, e->snap, + ceph_cap_string(e->caps), e->tid, + e->last_ack, e->wake, + e->is_capsnap ? " is_capsnap" : ""); + } + if (remaining) + pr_info_client(cl, " ... and %d more pending flushes\n", + remaining); +} + +/* + * Wait for all cap flushes through @want_flush_tid to complete. + * Periodically dumps pending cap flush state for diagnostics. */ static void wait_caps_flush(struct ceph_mds_client *mdsc, u64 want_flush_tid) { struct ceph_client *cl = mdsc->fsc->client; + int i = 0; + long ret; doutc(cl, "want %llu\n", want_flush_tid); - wait_event(mdsc->cap_flushing_wq, - check_caps_flush(mdsc, want_flush_tid)); + do { + /* 60 * HZ fits in a long on all supported architectures. */ + ret = wait_event_timeout(mdsc->cap_flushing_wq, + check_caps_flush(mdsc, want_flush_tid), + CEPH_CAP_FLUSH_WAIT_TIMEOUT_SEC * HZ); + if (ret == 0) { + if (i < CEPH_CAP_FLUSH_MAX_DUMP_ITERS) + dump_cap_flushes(mdsc, want_flush_tid); + else if (i == CEPH_CAP_FLUSH_MAX_DUMP_ITERS) + pr_info_client(cl, + "still waiting for cap flushes; suppressing further dumps\n"); + i++; + } + } while (ret == 0); doutc(cl, "ok, flushed thru %llu\n", want_flush_tid); } @@ -3657,7 +3752,8 @@ static void __do_request(struct ceph_mds_client *mdsc, spin_lock(&ci->i_ceph_lock); cap = ci->i_auth_cap; - if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE && mds != cap->mds) { + if (test_bit(CEPH_I_ASYNC_CREATE_BIT, &ci->i_ceph_flags) && + mds != cap->mds) { doutc(cl, "session changed for auth cap %d -> %d\n", cap->session->s_mds, session->s_mds); @@ -3751,6 +3847,22 @@ int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, struct inode *dir, struct ceph_client *cl = mdsc->fsc->client; int err = 0; + /* + * If a reset is in progress, wait for it to complete. + * + * This is best-effort: a request can pass this check just + * before the phase leaves IDLE and proceed concurrently with + * reset. That is acceptable because (a) such requests will + * either complete normally or fail and be retried by the + * caller, and (b) adding lock serialization here would + * penalize every request for a rare manual operation. + */ + err = ceph_mdsc_wait_for_reset(mdsc); + if (err) { + doutc(cl, "wait_for_reset failed: %d\n", err); + return err; + } + /* take CAP_PIN refs for r_inode, r_parent, r_old_dentry */ if (req->r_inode) ceph_get_cap_refs(ceph_inode(req->r_inode), CEPH_CAP_PIN); @@ -4469,9 +4581,14 @@ skip_cap_auths: break; case CEPH_SESSION_REJECT: - WARN_ON(session->s_state != CEPH_MDS_SESSION_OPENING); - pr_info_client(cl, "mds%d rejected session\n", - session->s_mds); + WARN_ON(session->s_state != CEPH_MDS_SESSION_OPENING && + session->s_state != CEPH_MDS_SESSION_RECONNECTING); + if (session->s_state == CEPH_MDS_SESSION_RECONNECTING) + pr_info_client(cl, "mds%d reconnect rejected\n", + session->s_mds); + else + pr_info_client(cl, "mds%d rejected session\n", + session->s_mds); session->s_state = CEPH_MDS_SESSION_REJECTED; cleanup_session_requests(mdsc, session); remove_session_caps(session); @@ -4731,6 +4848,14 @@ static int reconnect_caps_cb(struct inode *inode, int mds, void *arg) cap->mseq = 0; /* and migrate_seq */ cap->cap_gen = atomic_read(&cap->session->s_cap_gen); + /* + * Note: CEPH_I_ERROR_FILELOCK is not set during reconnect. + * Instead, locks are submitted for best-effort MDS reclaim + * via the flock_len field below. If reclaim fails (e.g., + * another client grabbed a conflicting lock), future lock + * operations will fail and set the error flag at that point. + */ + /* These are lost when the session goes away */ if (S_ISDIR(inode->i_mode)) { if (cap->issued & CEPH_CAP_DIR_CREATE) { @@ -4746,8 +4871,9 @@ static int reconnect_caps_cb(struct inode *inode, int mds, void *arg) rec.v2.issued = cpu_to_le32(cap->issued); rec.v2.snaprealm = cpu_to_le64(ci->i_snap_realm->ino); rec.v2.pathbase = cpu_to_le64(path_info.vino.ino); - rec.v2.flock_len = (__force __le32) - ((ci->i_ceph_flags & CEPH_I_ERROR_FILELOCK) ? 0 : 1); + rec.v2.flock_len = cpu_to_le32( + test_bit(CEPH_I_ERROR_FILELOCK_BIT, + &ci->i_ceph_flags) ? 0 : 1); } else { struct timespec64 ts; @@ -4944,20 +5070,19 @@ fail: * * This is a relatively heavyweight operation, but it's rare. */ -static void send_mds_reconnect(struct ceph_mds_client *mdsc, - struct ceph_mds_session *session) +static int send_mds_reconnect(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) { struct ceph_client *cl = mdsc->fsc->client; struct ceph_msg *reply; int mds = session->s_mds; int err = -ENOMEM; + int old_state; struct ceph_reconnect_state recon_state = { .session = session, }; LIST_HEAD(dispose); - pr_info_client(cl, "mds%d reconnect start\n", mds); - recon_state.pagelist = ceph_pagelist_alloc(GFP_NOFS); if (!recon_state.pagelist) goto fail_nopagelist; @@ -4966,9 +5091,37 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, if (!reply) goto fail_nomsg; + mutex_lock(&session->s_mutex); + + /* Serialized by s_mutex against concurrent ceph_get_deleg_ino(). */ xa_destroy(&session->s_delegated_inos); + if (session->s_state == CEPH_MDS_SESSION_CLOSED || + session->s_state == CEPH_MDS_SESSION_REJECTED) { + pr_info_client(cl, "mds%d skipping reconnect, session %s\n", + mds, + ceph_session_state_name(session->s_state)); + mutex_unlock(&session->s_mutex); + ceph_msg_put(reply); + err = -ESTALE; + goto fail_return; + } - mutex_lock(&session->s_mutex); + /* s_mutex -> mdsc->mutex matches cleanup_session_requests() order. */ + mutex_lock(&mdsc->mutex); + if (mds >= mdsc->max_sessions || mdsc->sessions[mds] != session) { + mutex_unlock(&mdsc->mutex); + pr_info_client(cl, + "mds%d skipping reconnect, session unregistered\n", + mds); + mutex_unlock(&session->s_mutex); + ceph_msg_put(reply); + err = -ENOENT; + goto fail_return; + } + mutex_unlock(&mdsc->mutex); + + pr_info_client(cl, "mds%d reconnect start\n", mds); + old_state = session->s_state; session->s_state = CEPH_MDS_SESSION_RECONNECTING; session->s_seq = 0; @@ -5098,7 +5251,7 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, up_read(&mdsc->snap_rwsem); ceph_pagelist_release(recon_state.pagelist); - return; + return 0; fail_clear_cap_reconnect: spin_lock(&session->s_cap_lock); @@ -5107,13 +5260,504 @@ fail_clear_cap_reconnect: fail: ceph_msg_put(reply); up_read(&mdsc->snap_rwsem); + /* + * Restore prior session state so map-driven reconnect logic + * (check_new_map) can retry. Without this, a transient build + * failure strands the session in RECONNECTING indefinitely. + */ + session->s_state = old_state; mutex_unlock(&session->s_mutex); fail_nomsg: ceph_pagelist_release(recon_state.pagelist); fail_nopagelist: pr_err_client(cl, "error %d preparing reconnect for mds%d\n", err, mds); + return err; + +fail_return: + /* + * Early-exit path for expected concurrent-teardown races + * (-ESTALE for closed/rejected sessions, -ENOENT for + * unregistered sessions). Skip the pr_err_client diagnostic + * since these are not genuine reconnect build failures. + */ + ceph_pagelist_release(recon_state.pagelist); + return err; +} + +const char *ceph_reset_phase_name(enum ceph_client_reset_phase phase) +{ + switch (phase) { + case CEPH_CLIENT_RESET_IDLE: return "idle"; + case CEPH_CLIENT_RESET_QUIESCING: return "quiescing"; + case CEPH_CLIENT_RESET_DRAINING: return "draining"; + case CEPH_CLIENT_RESET_TEARDOWN: return "teardown"; + default: return "unknown"; + } +} + +/** + * ceph_mdsc_wait_for_reset - wait for an active reset to complete + * @mdsc: MDS client + * + * Returns 0 if reset completed successfully or no reset was active. + * Returns -EAGAIN if reset completed with an error, signalling the + * caller to retry. The internal error (e.g. -ENOMEM) is not propagated + * because callers like open() or flock() have no way to act on + * work-function internals. The detailed error is available via debugfs + * reset/status and tracepoints. + * Returns -ETIMEDOUT if we timed out waiting. + * Returns -ERESTARTSYS if interrupted by signal. + */ +int ceph_mdsc_wait_for_reset(struct ceph_mds_client *mdsc) +{ + struct ceph_client_reset_state *st = &mdsc->reset_state; + struct ceph_client *cl = mdsc->fsc->client; + unsigned long deadline = jiffies + CEPH_CLIENT_RESET_WAIT_TIMEOUT_SEC * HZ; + int blocked_count; + long remaining; + long wait_ret; + int ret; + + if (ceph_reset_is_idle(st)) + return 0; + + blocked_count = atomic_inc_return(&st->blocked_requests); + doutc(cl, "request blocked during reset, %d total blocked\n", + blocked_count); + trace_ceph_client_reset_blocked(mdsc, blocked_count); + +retry: + remaining = max_t(long, deadline - jiffies, 1); + wait_ret = wait_event_interruptible_timeout(st->blocked_wq, + ceph_reset_is_idle(st), + remaining); + + if (wait_ret == 0) { + atomic_dec(&st->blocked_requests); + pr_warn_client(cl, "timed out waiting for reset to complete\n"); + trace_ceph_client_reset_unblocked(mdsc, -ETIMEDOUT); + return -ETIMEDOUT; + } + if (wait_ret < 0) { + atomic_dec(&st->blocked_requests); + trace_ceph_client_reset_unblocked(mdsc, (int)wait_ret); + return (int)wait_ret; /* -ERESTARTSYS */ + } + + /* + * Verify phase is still IDLE under the lock. If another reset + * was scheduled between the wake-up and this check, loop back + * and wait for it to finish rather than returning a stale result. + */ + spin_lock(&st->lock); + if (st->phase != CEPH_CLIENT_RESET_IDLE) { + spin_unlock(&st->lock); + if (time_before(jiffies, deadline)) + goto retry; + atomic_dec(&st->blocked_requests); + trace_ceph_client_reset_unblocked(mdsc, -ETIMEDOUT); + return -ETIMEDOUT; + } + ret = st->last_errno; + spin_unlock(&st->lock); + + atomic_dec(&st->blocked_requests); + trace_ceph_client_reset_unblocked(mdsc, ret); + return ret ? -EAGAIN : 0; +} + +static void ceph_mdsc_reset_complete(struct ceph_mds_client *mdsc, int ret) +{ + struct ceph_client_reset_state *st = &mdsc->reset_state; + + spin_lock(&st->lock); + /* + * If destroy already marked us as shut down, it owns the + * final bookkeeping and waiter wakeup. Just bail so we + * don't overwrite its state. + */ + if (st->shutdown) { + spin_unlock(&st->lock); + return; + } + st->last_finish = jiffies; + st->last_errno = ret; + st->phase = CEPH_CLIENT_RESET_IDLE; + if (ret) + st->failure_count++; + else + st->success_count++; + spin_unlock(&st->lock); + + /* Wake up all requests that were blocked waiting for reset */ + wake_up_all(&st->blocked_wq); + + trace_ceph_client_reset_complete(mdsc, ret); +} + +static void ceph_mdsc_reset_workfn(struct work_struct *work) +{ + struct ceph_mds_client *mdsc = + container_of(work, struct ceph_mds_client, reset_work); + struct ceph_client_reset_state *st = &mdsc->reset_state; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_session **sessions = NULL; + char reason[CEPH_CLIENT_RESET_REASON_LEN]; + unsigned long drain_deadline; + int max_sessions, i, n = 0, torn_down = 0; + int ret = 0; + + spin_lock(&st->lock); + strscpy(reason, st->last_reason, sizeof(reason)); + spin_unlock(&st->lock); + + mutex_lock(&mdsc->mutex); + max_sessions = mdsc->max_sessions; + if (max_sessions <= 0) { + mutex_unlock(&mdsc->mutex); + goto out_complete; + } + + sessions = kcalloc(max_sessions, sizeof(*sessions), GFP_KERNEL); + if (!sessions) { + mutex_unlock(&mdsc->mutex); + ret = -ENOMEM; + pr_err_client(cl, + "manual session reset failed to allocate session array\n"); + ceph_mdsc_reset_complete(mdsc, ret); + return; + } + + for (i = 0; i < max_sessions; i++) { + struct ceph_mds_session *session = mdsc->sessions[i]; + + if (!session) + continue; + + /* + * Read session state without s_mutex to avoid nesting + * mdsc->mutex -> s_mutex, which would invert the + * s_mutex -> mdsc->mutex order used by + * cleanup_session_requests(). s_state is an int + * so loads are atomic; the teardown loop below + * handles races with concurrent state transitions. + */ + switch (READ_ONCE(session->s_state)) { + case CEPH_MDS_SESSION_OPEN: + case CEPH_MDS_SESSION_HUNG: + case CEPH_MDS_SESSION_OPENING: + case CEPH_MDS_SESSION_RESTARTING: + case CEPH_MDS_SESSION_RECONNECTING: + case CEPH_MDS_SESSION_CLOSING: + sessions[n++] = ceph_get_mds_session(session); + break; + default: + pr_info_client(cl, + "mds%d in state %s, skipping reset\n", + session->s_mds, + ceph_session_state_name(session->s_state)); + break; + } + } + mutex_unlock(&mdsc->mutex); + + pr_info_client(cl, + "manual session reset executing (sessions=%d, reason=\"%s\")\n", + n, reason); + + if (n == 0) { + kfree(sessions); + goto out_complete; + } + + spin_lock(&st->lock); + if (st->shutdown) { + spin_unlock(&st->lock); + goto out_sessions; + } + st->phase = CEPH_CLIENT_RESET_DRAINING; + spin_unlock(&st->lock); + + /* + * Best-effort drain: flush dirty state while sessions are still + * alive. New requests are blocked while phase != IDLE. + * The sessions are functional, so non-stuck state drains normally. + * Stuck state (the cause of the stalemate the operator is trying + * to break) will not drain -- that is expected, and we proceed to + * forced teardown after the timeout. + * + * Four things are drained: + * 1. MDS journal -- send_flush_mdlog asks each MDS to journal + * pending unsafe operations (creates, renames, setattrs). + * 2. Unsafe requests -- bounded wait for each unsafe write + * request to reach safe status via r_safe_completion. + * 3. Dirty caps -- ceph_flush_dirty_caps triggers cap flush on + * all sessions. Non-stuck caps flush in milliseconds. + * 4. Cap releases -- push pending cap release messages. + * + * The unsafe-request wait and cap-flush wait below provide + * the bounded drain window during which all categories can + * make progress. + */ + for (i = 0; i < n; i++) + send_flush_mdlog(sessions[i]); + + /* + * Both drain legs (unsafe requests and cap flushes) share a + * single deadline so the total drain time is bounded at + * CEPH_CLIENT_RESET_DRAIN_SEC. + */ + drain_deadline = jiffies + CEPH_CLIENT_RESET_DRAIN_SEC * HZ; + + /* + * Wait for unsafe write requests (creates, renames, setattrs) + * to reach safe status. Uses the same pattern as + * flush_mdlog_and_wait_mdsc_unsafe_requests() but bounded by + * the shared drain deadline. Requests that do not complete within + * the window are force-dropped during teardown. + */ + { + struct ceph_mds_request *req; + struct rb_node *rn; + u64 last_tid; + + mutex_lock(&mdsc->mutex); + last_tid = mdsc->last_tid; + mutex_unlock(&mdsc->mutex); + + mutex_lock(&mdsc->mutex); + rn = rb_first(&mdsc->request_tree); + while (rn) { + req = rb_entry(rn, struct ceph_mds_request, r_node); + if (req->r_tid > last_tid) + break; + if (req->r_op == CEPH_MDS_OP_SETFILELOCK || + !(req->r_op & CEPH_MDS_OP_WRITE)) { + rn = rb_next(rn); + continue; + } + ceph_mdsc_get_request(req); + mutex_unlock(&mdsc->mutex); + + wait_for_completion_timeout(&req->r_safe_completion, + max_t(long, drain_deadline - jiffies, 1)); + + mutex_lock(&mdsc->mutex); + ceph_mdsc_put_request(req); + if (time_after(jiffies, drain_deadline)) + break; + rn = rb_first(&mdsc->request_tree); + } + mutex_unlock(&mdsc->mutex); + + if (time_after_eq(jiffies, drain_deadline)) + WRITE_ONCE(st->drain_timed_out, true); + } + + ceph_flush_dirty_caps(mdsc); + ceph_flush_cap_releases(mdsc); + + spin_lock(&mdsc->cap_dirty_lock); + if (!list_empty(&mdsc->cap_flush_list)) { + struct ceph_cap_flush *cf = + list_last_entry(&mdsc->cap_flush_list, + struct ceph_cap_flush, g_list); + u64 want_flush = mdsc->last_cap_flush_tid; + long drain_ret; + + /* + * Setting wake on the last entry is sufficient: flush + * entries complete in order, so when this entry finishes + * all earlier ones are already done. + */ + cf->wake = true; + spin_unlock(&mdsc->cap_dirty_lock); + pr_info_client(cl, + "draining (want_flush=%llu, %d sessions)\n", + want_flush, n); + drain_ret = wait_event_timeout(mdsc->cap_flushing_wq, + check_caps_flush(mdsc, + want_flush), + max_t(long, + drain_deadline - jiffies, + 1)); + if (drain_ret == 0) { + pr_info_client(cl, + "drain timed out, proceeding with forced teardown\n"); + WRITE_ONCE(st->drain_timed_out, true); + } else { + pr_info_client(cl, "drain completed successfully\n"); + } + } else { + spin_unlock(&mdsc->cap_dirty_lock); + } + + spin_lock(&st->lock); + if (st->shutdown) { + spin_unlock(&st->lock); + goto out_sessions; + } + st->phase = CEPH_CLIENT_RESET_TEARDOWN; + spin_unlock(&st->lock); + + /* + * Ask each MDS to close the session before we tear it down + * locally. Without this the MDS sees only a connection drop and + * waits for the client to reconnect (up to session_autoclose + * seconds) before evicting the session and releasing locks. + * + * Reuse the normal close machinery so the session state/sequence + * snapshot is serialized under s_mutex and a racing s_seq bump + * retransmits REQUEST_CLOSE while the session remains CLOSING. + * We send all close requests first, then yield briefly to let the + * network stack transmit them before __unregister_session() + * closes the connections. + */ + for (i = 0; i < n; i++) { + int err; + + mutex_lock(&sessions[i]->s_mutex); + err = __close_session(mdsc, sessions[i]); + mutex_unlock(&sessions[i]->s_mutex); + if (err < 0) + pr_warn_client(cl, + "mds%d failed to queue close request before reset: %d\n", + sessions[i]->s_mds, err); + } + /* + * Best-effort grace period: yield briefly so the network stack + * can transmit the queued REQUEST_CLOSE messages before we tear + * down connections. Not a correctness requirement -- the MDS + * will still evict via session_autoclose if it never receives + * the close request. + * + * Event-based waiting is not viable here: there is no completion + * event for "message left the NIC," and waiting for the MDS + * SESSION_CLOSE response would re-create the stalemate that the + * reset is meant to break. + */ + if (n > 0) + msleep(CEPH_CLIENT_RESET_CLOSE_GRACE_MS); + + /* + * Tear down each session: close the connection, remove all + * caps, clean up requests, then kick pending requests so they + * re-open a fresh session on the next attempt. + * + * This is modeled on the check_new_map() forced-close path + * for stopped MDS ranks - a proven pattern for hard session + * teardown. We do NOT attempt send_mds_reconnect() because + * the MDS only accepts reconnects during its own RECONNECT + * phase (after MDS restart), not from an active client. + * + * Any state that did not drain (caps that didn't flush, unsafe + * requests that the MDS didn't journal) is force-dropped here. + * This is intentional: that state is stuck and is the reason + * the operator triggered the reset. + */ + for (i = 0; i < n; i++) { + int mds = sessions[i]->s_mds; + + pr_info_client(cl, "mds%d resetting session\n", mds); + + mutex_lock(&mdsc->mutex); + if (mds >= mdsc->max_sessions || + mdsc->sessions[mds] != sessions[i]) { + pr_info_client(cl, + "mds%d session already torn down, skipping\n", + mds); + mutex_unlock(&mdsc->mutex); + ceph_put_mds_session(sessions[i]); + sessions[i] = NULL; + continue; + } + sessions[i]->s_state = CEPH_MDS_SESSION_CLOSED; + __unregister_session(mdsc, sessions[i]); + __wake_requests(mdsc, &sessions[i]->s_waiting); + mutex_unlock(&mdsc->mutex); + + mutex_lock(&sessions[i]->s_mutex); + cleanup_session_requests(mdsc, sessions[i]); + remove_session_caps(sessions[i]); + mutex_unlock(&sessions[i]->s_mutex); + + wake_up_all(&mdsc->session_close_wq); + + ceph_put_mds_session(sessions[i]); + + mutex_lock(&mdsc->mutex); + kick_requests(mdsc, mds); + mutex_unlock(&mdsc->mutex); + + torn_down++; + pr_info_client(cl, "mds%d session reset complete\n", mds); + } + + kfree(sessions); + + spin_lock(&st->lock); + st->sessions_reset = torn_down; + spin_unlock(&st->lock); + +out_complete: + ceph_mdsc_reset_complete(mdsc, ret); return; + +out_sessions: + /* shutdown == true: ceph_mdsc_destroy() owns the final transition. */ + for (i = 0; i < n; i++) + ceph_put_mds_session(sessions[i]); + kfree(sessions); +} + +int ceph_mdsc_schedule_reset(struct ceph_mds_client *mdsc, + const char *reason) +{ + struct ceph_client_reset_state *st = &mdsc->reset_state; + struct ceph_fs_client *fsc = mdsc->fsc; + const char *msg = (reason && reason[0]) ? reason : "manual"; + int mount_state; + + mount_state = READ_ONCE(fsc->mount_state); + if (mount_state != CEPH_MOUNT_MOUNTED) { + pr_warn_client(fsc->client, + "reset rejected: mount_state=%d (not mounted)\n", + mount_state); + return -EINVAL; + } + + spin_lock(&st->lock); + if (st->phase != CEPH_CLIENT_RESET_IDLE) { + spin_unlock(&st->lock); + return -EBUSY; + } + + st->phase = CEPH_CLIENT_RESET_QUIESCING; + st->last_start = jiffies; + st->last_errno = 0; + st->drain_timed_out = false; + st->sessions_reset = 0; + st->trigger_count++; + strscpy(st->last_reason, msg, sizeof(st->last_reason)); + spin_unlock(&st->lock); + + if (WARN_ON_ONCE(!queue_work(system_unbound_wq, &mdsc->reset_work))) { + spin_lock(&st->lock); + st->phase = CEPH_CLIENT_RESET_IDLE; + st->last_errno = -EALREADY; + st->last_finish = jiffies; + st->failure_count++; + spin_unlock(&st->lock); + wake_up_all(&st->blocked_wq); + return -EALREADY; + } + + pr_info_client(mdsc->fsc->client, + "manual session reset scheduled (reason=\"%s\")\n", + msg); + trace_ceph_client_reset_schedule(mdsc, msg); + return 0; } @@ -5194,9 +5838,15 @@ static void check_new_map(struct ceph_mds_client *mdsc, */ if (s->s_state == CEPH_MDS_SESSION_RESTARTING && newstate >= CEPH_MDS_STATE_RECONNECT) { + int rc; + mutex_unlock(&mdsc->mutex); clear_bit(i, targets); - send_mds_reconnect(mdsc, s); + rc = send_mds_reconnect(mdsc, s); + if (rc) + pr_warn_client(cl, + "mds%d reconnect failed: %d\n", + i, rc); mutex_lock(&mdsc->mutex); } @@ -5260,7 +5910,11 @@ static void check_new_map(struct ceph_mds_client *mdsc, } doutc(cl, "send reconnect to export target mds.%d\n", i); mutex_unlock(&mdsc->mutex); - send_mds_reconnect(mdsc, s); + err = send_mds_reconnect(mdsc, s); + if (err) + pr_warn_client(cl, + "mds%d export target reconnect failed: %d\n", + i, err); ceph_put_mds_session(s); mutex_lock(&mdsc->mutex); } @@ -5651,6 +6305,11 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc) INIT_LIST_HEAD(&mdsc->dentry_leases); INIT_LIST_HEAD(&mdsc->dentry_dir_leases); + spin_lock_init(&mdsc->reset_state.lock); + init_waitqueue_head(&mdsc->reset_state.blocked_wq); + atomic_set(&mdsc->reset_state.blocked_requests, 0); + INIT_WORK(&mdsc->reset_work, ceph_mdsc_reset_workfn); + ceph_caps_init(mdsc); ceph_adjust_caps_max_min(mdsc, fsc->mount_options); @@ -6176,6 +6835,23 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc) /* flush out any connection work with references to us */ ceph_msgr_flush(); + /* + * Mark reset as failed and wake any blocked waiters before + * cancelling, so unmount doesn't stall on blocked_wq timeout + * if cancel_work_sync() prevents the work from running. + */ + spin_lock(&mdsc->reset_state.lock); + mdsc->reset_state.shutdown = true; + if (mdsc->reset_state.phase != CEPH_CLIENT_RESET_IDLE) { + mdsc->reset_state.phase = CEPH_CLIENT_RESET_IDLE; + mdsc->reset_state.last_errno = -ESHUTDOWN; + mdsc->reset_state.last_finish = jiffies; + mdsc->reset_state.failure_count++; + } + spin_unlock(&mdsc->reset_state.lock); + wake_up_all(&mdsc->reset_state.blocked_wq); + + cancel_work_sync(&mdsc->reset_work); ceph_mdsc_stop(mdsc); ceph_metric_destroy(&mdsc->metric); @@ -6348,12 +7024,92 @@ static void mds_peer_reset(struct ceph_connection *con) { struct ceph_mds_session *s = con->private; struct ceph_mds_client *mdsc = s->s_mdsc; + int session_state; pr_warn_client(mdsc->fsc->client, "mds%d closed our session\n", s->s_mds); - if (READ_ONCE(mdsc->fsc->mount_state) != CEPH_MOUNT_FENCE_IO && - ceph_mdsmap_get_state(mdsc->mdsmap, s->s_mds) >= CEPH_MDS_STATE_RECONNECT) - send_mds_reconnect(mdsc, s); + + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO || + ceph_mdsmap_get_state(mdsc->mdsmap, s->s_mds) < CEPH_MDS_STATE_RECONNECT) + return; + + /* + * Only reconnect if MDS is in its RECONNECT phase. An MDS past + * RECONNECT (REJOIN, CLIENTREPLAY, ACTIVE) will reject reconnect + * attempts, so those states fall through to session teardown below. + */ + if (ceph_mdsmap_get_state(mdsc->mdsmap, s->s_mds) == CEPH_MDS_STATE_RECONNECT) { + int rc = send_mds_reconnect(mdsc, s); + + if (rc) + pr_warn_client(mdsc->fsc->client, + "mds%d reconnect failed: %d\n", + s->s_mds, rc); + return; + } + + /* + * MDS is active (past RECONNECT). It will not accept a + * CLIENT_RECONNECT from us, so tear the session down locally + * and let new requests re-open a fresh session. + * + * Snapshot session state with READ_ONCE, then revalidate under + * mdsc->mutex before acting. The subsequent mdsc->mutex + * section rechecks s_state to catch concurrent transitions, so + * the lockless snapshot here is safe. s->s_mutex is taken + * separately for cleanup after unregistration, which avoids + * introducing a new s->s_mutex + mdsc->mutex nesting. + */ + session_state = READ_ONCE(s->s_state); + + switch (session_state) { + case CEPH_MDS_SESSION_RESTARTING: + case CEPH_MDS_SESSION_RECONNECTING: + case CEPH_MDS_SESSION_CLOSING: + case CEPH_MDS_SESSION_OPEN: + case CEPH_MDS_SESSION_HUNG: + case CEPH_MDS_SESSION_OPENING: + mutex_lock(&mdsc->mutex); + if (s->s_mds >= mdsc->max_sessions || + mdsc->sessions[s->s_mds] != s || + s->s_state != session_state) { + pr_info_client(mdsc->fsc->client, + "mds%d state changed to %s during peer reset\n", + s->s_mds, + ceph_session_state_name(s->s_state)); + mutex_unlock(&mdsc->mutex); + return; + } + + ceph_get_mds_session(s); + s->s_state = CEPH_MDS_SESSION_CLOSED; + __unregister_session(mdsc, s); + __wake_requests(mdsc, &s->s_waiting); + mutex_unlock(&mdsc->mutex); + + mutex_lock(&s->s_mutex); + cleanup_session_requests(mdsc, s); + remove_session_caps(s); + mutex_unlock(&s->s_mutex); + + wake_up_all(&mdsc->session_close_wq); + + mutex_lock(&mdsc->mutex); + kick_requests(mdsc, s->s_mds); + mutex_unlock(&mdsc->mutex); + + ceph_put_mds_session(s); + break; + case CEPH_MDS_SESSION_CLOSED: + case CEPH_MDS_SESSION_REJECTED: + break; + default: + pr_warn_client(mdsc->fsc->client, + "mds%d peer reset in unexpected state %s\n", + s->s_mds, + ceph_session_state_name(session_state)); + break; + } } static void mds_dispatch(struct ceph_connection *con, struct ceph_msg *msg) @@ -6365,6 +7121,8 @@ static void mds_dispatch(struct ceph_connection *con, struct ceph_msg *msg) mutex_lock(&mdsc->mutex); if (__verify_registered_session(mdsc, s) < 0) { + doutc(cl, "dropping tid %llu from unregistered session %d\n", + le64_to_cpu(msg->hdr.tid), s->s_mds); mutex_unlock(&mdsc->mutex); goto out; } diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 4e6c87f8414c..731d6ad04956 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -77,6 +77,50 @@ struct ceph_fs_client; struct ceph_cap; #define MDS_AUTH_UID_ANY -1 +#define CEPH_CAP_FLUSH_WAIT_TIMEOUT_SEC 60 +#define CEPH_CAP_FLUSH_MAX_DUMP_ENTRIES 5 +#define CEPH_CAP_FLUSH_MAX_DUMP_ITERS 5 +#define CEPH_CLIENT_RESET_REASON_LEN 64 +#define CEPH_CLIENT_RESET_DRAIN_SEC 30 +#define CEPH_CLIENT_RESET_CLOSE_GRACE_MS 100 +#define CEPH_CLIENT_RESET_WAIT_TIMEOUT_SEC 120 + +enum ceph_client_reset_phase { + CEPH_CLIENT_RESET_IDLE = 0, + /* + * QUIESCING is set synchronously by schedule_reset() before the + * workqueue item is dispatched. It gates new requests (any + * phase != IDLE blocks callers) during the window between + * scheduling and the work function's transition to DRAINING. + */ + CEPH_CLIENT_RESET_QUIESCING, + CEPH_CLIENT_RESET_DRAINING, + CEPH_CLIENT_RESET_TEARDOWN, +}; + +struct ceph_client_reset_state { + spinlock_t lock; /* protects all fields below */ + u64 trigger_count; /* number of resets triggered */ + u64 success_count; /* number of successful resets */ + u64 failure_count; /* number of failed resets */ + unsigned long last_start; /* jiffies when last reset started */ + unsigned long last_finish; /* jiffies when last reset finished */ + int last_errno; /* result of most recent reset */ + enum ceph_client_reset_phase phase; /* current reset phase */ + bool drain_timed_out; /* drain exceeded timeout */ + bool shutdown; /* destroy in progress */ + int sessions_reset; /* sessions torn down in last reset */ + char last_reason[CEPH_CLIENT_RESET_REASON_LEN]; /* operator-supplied reason */ + + /* Request blocking during reset */ + wait_queue_head_t blocked_wq; /* waitqueue for blocked callers */ + atomic_t blocked_requests; /* count of blocked callers */ +}; + +static inline bool ceph_reset_is_idle(struct ceph_client_reset_state *st) +{ + return READ_ONCE(st->phase) == CEPH_CLIENT_RESET_IDLE; +} struct ceph_mds_cap_match { s64 uid; /* default to MDS_AUTH_UID_ANY */ @@ -540,6 +584,8 @@ struct ceph_mds_client { struct list_head dentry_dir_leases; /* lru list */ struct ceph_client_metric metric; + struct work_struct reset_work; + struct ceph_client_reset_state reset_state; struct ceph_subvolume_metrics_tracker subvol_metrics; /* Subvolume metrics send tracking */ @@ -571,10 +617,14 @@ extern struct ceph_mds_session * __ceph_lookup_mds_session(struct ceph_mds_client *, int mds); extern const char *ceph_session_state_name(int s); +extern const char *ceph_reset_phase_name(enum ceph_client_reset_phase phase); extern struct ceph_mds_session * ceph_get_mds_session(struct ceph_mds_session *s); extern void ceph_put_mds_session(struct ceph_mds_session *s); +int ceph_mdsc_schedule_reset(struct ceph_mds_client *mdsc, + const char *reason); +int ceph_mdsc_wait_for_reset(struct ceph_mds_client *mdsc); extern int ceph_mdsc_init(struct ceph_fs_client *fsc); extern void ceph_mdsc_close_sessions(struct ceph_mds_client *mdsc); @@ -670,7 +720,7 @@ static inline int ceph_wait_on_async_create(struct inode *inode) { struct ceph_inode_info *ci = ceph_inode(inode); - return wait_on_bit(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT, + return wait_on_bit(&ci->i_ceph_flags, CEPH_I_ASYNC_CREATE_BIT, TASK_KILLABLE); } diff --git a/fs/ceph/quota.c b/fs/ceph/quota.c index 4dc9426643e8..053d5bf0c9f0 100644 --- a/fs/ceph/quota.c +++ b/fs/ceph/quota.c @@ -228,12 +228,19 @@ static int get_quota_realm(struct ceph_mds_client *mdsc, struct inode *inode, restart: realm = ceph_inode(inode)->i_snap_realm; - if (realm) + if (realm) { ceph_get_snap_realm(mdsc, realm); - else - pr_err_ratelimited_client(cl, - "%p %llx.%llx null i_snap_realm\n", - inode, ceph_vinop(inode)); + } else { + /* + * i_snap_realm is NULL when all caps have been released, e.g. + * after an MDS session rejection. This is a transient state; + * the realm will be restored once caps are re-granted. + * Treat it as "no quota realm found". + */ + doutc(cl, "%p %llx.%llx null i_snap_realm\n", + inode, ceph_vinop(inode)); + } + while (realm) { bool has_inode; @@ -340,12 +347,19 @@ static bool check_quota_exceeded(struct inode *inode, enum quota_check_op op, down_read(&mdsc->snap_rwsem); restart: realm = ceph_inode(inode)->i_snap_realm; - if (realm) + if (realm) { ceph_get_snap_realm(mdsc, realm); - else - pr_err_ratelimited_client(cl, - "%p %llx.%llx null i_snap_realm\n", - inode, ceph_vinop(inode)); + } else { + /* + * i_snap_realm is NULL when all caps have been released, e.g. + * after an MDS session rejection. This is a transient state; + * the realm will be restored once caps are re-granted. + * Treat it as "quota not exceeded". + */ + doutc(cl, "%p %llx.%llx null i_snap_realm\n", + inode, ceph_vinop(inode)); + } + while (realm) { bool has_inode; @@ -496,6 +510,9 @@ bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf) u64 total = 0, used, free; bool is_updated = false; + if (!ceph_has_realms_with_quotas(d_inode(fsc->sb->s_root))) + return false; + down_read(&mdsc->snap_rwsem); get_quota_realm(mdsc, d_inode(fsc->sb->s_root), QUOTA_GET_MAX_BYTES, &realm, true); diff --git a/fs/ceph/snap.c b/fs/ceph/snap.c index 52b4c2684f92..9b79a5eaca93 100644 --- a/fs/ceph/snap.c +++ b/fs/ceph/snap.c @@ -700,7 +700,7 @@ int __ceph_finish_cap_snap(struct ceph_inode_info *ci, return 0; } - ci->i_ceph_flags |= CEPH_I_FLUSH_SNAPS; + set_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); doutc(cl, "%p %llx.%llx cap_snap %p snapc %p %llu %s s=%llu\n", inode, ceph_vinop(inode), capsnap, capsnap->context, capsnap->context->seq, ceph_cap_string(capsnap->dirty), diff --git a/fs/ceph/super.h b/fs/ceph/super.h index afc89ce91804..1d6aab060780 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -179,6 +179,7 @@ struct ceph_fs_client { struct dentry *debugfs_status; struct dentry *debugfs_mds_sessions; struct dentry *debugfs_metrics_dir; + struct dentry *debugfs_reset_dir; struct dentry *debugfs_subvolume_metrics; #endif @@ -239,6 +240,7 @@ struct ceph_cap_flush { bool is_capsnap; /* true means capsnap */ struct list_head g_list; // global struct list_head i_list; // per inode + struct ceph_inode_info *ci; }; /* @@ -453,6 +455,11 @@ struct ceph_inode_info { struct ceph_snap_context *i_head_snapc; /* set if wr_buffer_head > 0 or dirty|flushing caps */ unsigned i_snap_caps; /* cap bits for snapped files */ + /* + * Written under i_ceph_lock, read via READ_ONCE() + * from diagnostic paths. + */ + u64 i_last_cap_flush_ack; unsigned long i_last_rd; unsigned long i_last_wr; @@ -665,23 +672,33 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, /* * Ceph inode. */ -#define CEPH_I_DIR_ORDERED (1 << 0) /* dentries in dir are ordered */ -#define CEPH_I_FLUSH (1 << 2) /* do not delay flush of dirty metadata */ -#define CEPH_I_POOL_PERM (1 << 3) /* pool rd/wr bits are valid */ -#define CEPH_I_POOL_RD (1 << 4) /* can read from pool */ -#define CEPH_I_POOL_WR (1 << 5) /* can write to pool */ -#define CEPH_I_SEC_INITED (1 << 6) /* security initialized */ -#define CEPH_I_KICK_FLUSH (1 << 7) /* kick flushing caps */ -#define CEPH_I_FLUSH_SNAPS (1 << 8) /* need flush snapss */ -#define CEPH_I_ERROR_WRITE (1 << 9) /* have seen write errors */ -#define CEPH_I_ERROR_FILELOCK (1 << 10) /* have seen file lock errors */ -#define CEPH_I_ODIRECT_BIT (11) /* inode in direct I/O mode */ -#define CEPH_I_ODIRECT (1 << CEPH_I_ODIRECT_BIT) -#define CEPH_ASYNC_CREATE_BIT (12) /* async create in flight for this */ -#define CEPH_I_ASYNC_CREATE (1 << CEPH_ASYNC_CREATE_BIT) -#define CEPH_I_SHUTDOWN (1 << 13) /* inode is no longer usable */ -#define CEPH_I_ASYNC_CHECK_CAPS (1 << 14) /* check caps immediately after async - creating finishes */ +#define CEPH_I_DIR_ORDERED_BIT (0) /* dentries in dir are ordered */ + /* bit 1 historically unused */ +#define CEPH_I_FLUSH_BIT (2) /* do not delay flush of dirty metadata */ +#define CEPH_I_POOL_PERM_BIT (3) /* pool rd/wr bits are valid */ +#define CEPH_I_POOL_RD_BIT (4) /* can read from pool */ +#define CEPH_I_POOL_WR_BIT (5) /* can write to pool */ +#define CEPH_I_SEC_INITED_BIT (6) /* security initialized */ +#define CEPH_I_KICK_FLUSH_BIT (7) /* kick flushing caps */ +#define CEPH_I_FLUSH_SNAPS_BIT (8) /* need flush snaps */ +#define CEPH_I_ERROR_WRITE_BIT (9) /* have seen write errors */ +#define CEPH_I_ERROR_FILELOCK_BIT (10) /* have seen file lock errors */ +#define CEPH_I_ODIRECT_BIT (11) /* inode in direct I/O mode */ +#define CEPH_I_ASYNC_CREATE_BIT (12) /* async create in flight for this */ +#define CEPH_I_SHUTDOWN_BIT (13) /* inode is no longer usable */ +#define CEPH_I_ASYNC_CHECK_CAPS_BIT (14) /* check caps after async creating finishes */ + +#define CEPH_I_DIR_ORDERED (1 << CEPH_I_DIR_ORDERED_BIT) +#define CEPH_I_FLUSH (1 << CEPH_I_FLUSH_BIT) +#define CEPH_I_POOL_PERM (1 << CEPH_I_POOL_PERM_BIT) +#define CEPH_I_POOL_RD (1 << CEPH_I_POOL_RD_BIT) +#define CEPH_I_POOL_WR (1 << CEPH_I_POOL_WR_BIT) +#define CEPH_I_SEC_INITED (1 << CEPH_I_SEC_INITED_BIT) +#define CEPH_I_KICK_FLUSH (1 << CEPH_I_KICK_FLUSH_BIT) +#define CEPH_I_FLUSH_SNAPS (1 << CEPH_I_FLUSH_SNAPS_BIT) +#define CEPH_I_ODIRECT (1 << CEPH_I_ODIRECT_BIT) +#define CEPH_I_ASYNC_CREATE (1 << CEPH_I_ASYNC_CREATE_BIT) +#define CEPH_I_SHUTDOWN (1 << CEPH_I_SHUTDOWN_BIT) /* * Masks of ceph inode work. @@ -694,27 +711,18 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, /* * We set the ERROR_WRITE bit when we start seeing write errors on an inode - * and then clear it when they start succeeding. Note that we do a lockless - * check first, and only take the lock if it looks like it needs to be changed. - * The write submission code just takes this as a hint, so we're not too - * worried if a few slip through in either direction. + * and then clear it when they start succeeding. The write submission code + * just takes this as a hint, so we're not too worried if a few slip through + * in either direction. */ static inline void ceph_set_error_write(struct ceph_inode_info *ci) { - if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ERROR_WRITE)) { - spin_lock(&ci->i_ceph_lock); - ci->i_ceph_flags |= CEPH_I_ERROR_WRITE; - spin_unlock(&ci->i_ceph_lock); - } + set_bit(CEPH_I_ERROR_WRITE_BIT, &ci->i_ceph_flags); } static inline void ceph_clear_error_write(struct ceph_inode_info *ci) { - if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ERROR_WRITE) { - spin_lock(&ci->i_ceph_lock); - ci->i_ceph_flags &= ~CEPH_I_ERROR_WRITE; - spin_unlock(&ci->i_ceph_lock); - } + clear_bit(CEPH_I_ERROR_WRITE_BIT, &ci->i_ceph_flags); } static inline void __ceph_dir_set_complete(struct ceph_inode_info *ci, diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index 5f87f62091a1..860fc8e1867d 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -1054,7 +1054,7 @@ handle_non_vxattrs: if (current->journal_info && !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN) && security_ismaclabel(name + XATTR_SECURITY_PREFIX_LEN)) - ci->i_ceph_flags |= CEPH_I_SEC_INITED; + set_bit(CEPH_I_SEC_INITED_BIT, &ci->i_ceph_flags); out: spin_unlock(&ci->i_ceph_lock); return err; @@ -1254,6 +1254,22 @@ retry: ceph_vinop(inode), name, ceph_cap_string(issued)); __build_xattrs(inode); + /* + * __build_xattrs() may have released and reacquired i_ceph_lock, + * during which handle_cap_grant() could have replaced i_xattrs.blob + * with a newer MDS-provided blob and bumped i_xattrs.version. If that + * caused __build_xattrs() to rebuild the rb-tree from the new blob, + * count/names_size/vals_size may now be larger than when + * required_blob_size was computed above. Recompute it here so the + * prealloc_blob size check below reflects the current tree state. + */ + required_blob_size = __get_required_blob_size(ci, name_len, val_len); + if (required_blob_size > mdsc->mdsmap->m_max_xattr_size) { + doutc(cl, "sync (size too large): %d > %llu\n", + required_blob_size, mdsc->mdsmap->m_max_xattr_size); + goto do_sync; + } + if (!ci->i_xattrs.prealloc_blob || required_blob_size > ci->i_xattrs.prealloc_blob->alloc_len) { struct ceph_buffer *blob; @@ -1294,6 +1310,7 @@ retry: do_sync: spin_unlock(&ci->i_ceph_lock); + ceph_buffer_put(old_blob); do_sync_unlocked: if (lock_snap_rwsem) up_read(&mdsc->snap_rwsem); diff --git a/fs/configfs/configfs_internal.h b/fs/configfs/configfs_internal.h index 0b969d0eb8ff..acdeea8e2d69 100644 --- a/fs/configfs/configfs_internal.h +++ b/fs/configfs/configfs_internal.h @@ -76,7 +76,6 @@ extern int configfs_make_dirent(struct configfs_dirent *, struct dentry *, extern int configfs_dirent_is_ready(struct configfs_dirent *); extern const unsigned char * configfs_get_name(struct configfs_dirent *sd); -extern void configfs_drop_dentry(struct configfs_dirent *sd, struct dentry *parent); extern int configfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *iattr); diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c index 362b6ff9b908..3c88f13f1ca2 100644 --- a/fs/configfs/dir.c +++ b/fs/configfs/dir.c @@ -235,15 +235,16 @@ static int configfs_dirent_exists(struct dentry *dentry) const unsigned char *new = dentry->d_name.name; struct configfs_dirent *sd; + spin_lock(&configfs_dirent_lock); list_for_each_entry(sd, &parent_sd->s_children, s_sibling) { if (sd->s_element) { - const unsigned char *existing = configfs_get_name(sd); - if (strcmp(existing, new)) - continue; - else + if (strcmp(configfs_get_name(sd), new) == 0) { + spin_unlock(&configfs_dirent_lock); return -EEXIST; + } } } + spin_unlock(&configfs_dirent_lock); return 0; } @@ -295,6 +296,7 @@ static int configfs_create_dir(struct config_item *item, struct dentry *dentry, int error; umode_t mode = S_IFDIR| S_IRWXU | S_IRUGO | S_IXUGO; struct dentry *p = dentry->d_parent; + struct inode *p_inode = d_inode(p); struct inode *inode; BUG_ON(!item); @@ -314,10 +316,9 @@ static int configfs_create_dir(struct config_item *item, struct dentry *dentry, inode->i_fop = &configfs_dir_operations; /* directory inodes start off with i_nlink == 2 (for "." entry) */ inc_nlink(inode); - d_instantiate(dentry, inode); - /* already hashed */ - dget(dentry); /* pin directory dentries in core */ - inc_nlink(d_inode(p)); + d_make_persistent(dentry, inode); + inc_nlink(p_inode); + inode_set_mtime_to_ts(p_inode, inode_set_ctime_current(p_inode)); item->ci_dentry = dentry; return 0; @@ -371,6 +372,7 @@ int configfs_create_link(struct configfs_dirent *target, struct dentry *parent, int err = 0; umode_t mode = S_IFLNK | S_IRWXUGO; struct configfs_dirent *p = parent->d_fsdata; + struct inode *p_inode = d_inode(parent); struct inode *inode; err = configfs_make_dirent(p, dentry, target, mode, CONFIGFS_ITEM_LINK, @@ -384,8 +386,8 @@ int configfs_create_link(struct configfs_dirent *target, struct dentry *parent, inode->i_link = body; inode->i_op = &configfs_symlink_inode_operations; - d_instantiate(dentry, inode); - dget(dentry); /* pin link dentries in core */ + d_make_persistent(dentry, inode); + inode_set_mtime_to_ts(p_inode, inode_set_ctime_current(p_inode)); return 0; out_remove: @@ -394,29 +396,9 @@ out_remove: return PTR_ERR(inode); } -static void remove_dir(struct dentry * d) -{ - struct dentry * parent = dget(d->d_parent); - - configfs_remove_dirent(d); - - if (d_really_is_positive(d)) { - if (likely(simple_empty(d))) { - __simple_rmdir(d_inode(parent),d); - dput(d); - } else { - pr_warn("remove_dir (%pd): attributes remain", d); - } - } - - pr_debug(" o %pd removing done (%d)\n", d, d_count(d)); - - dput(parent); -} - /** * configfs_remove_dir - remove an config_item's directory. - * @item: config_item we're removing. + * @d: dentry we're removing. * * The only thing special about this is that we remove any files in * the directory before we remove the directory, and we've inlined @@ -425,18 +407,20 @@ static void remove_dir(struct dentry * d) * Caller holds the mutex of the item's inode */ -static void configfs_remove_dir(struct config_item * item) +static void configfs_remove_dir(struct dentry *d) { - struct dentry * dentry = dget(item->ci_dentry); + struct dentry * parent = dget(d->d_parent); - if (!dentry) - return; + configfs_remove_dirent(d); - remove_dir(dentry); - /** - * Drop reference from dget() on entrance. - */ - dput(dentry); + if (d_really_is_positive(d)) { + if (unlikely(simple_rmdir(d_inode(parent), d))) + pr_warn("remove_dir (%pd): attributes remain", d); + } + + pr_debug(" o %pd removing done (%d)\n", d, d_count(d)); + + dput(parent); } static struct dentry * configfs_lookup(struct inode *dir, @@ -486,6 +470,9 @@ static struct dentry * configfs_lookup(struct inode *dir, inode = configfs_create(dentry, mode); if (IS_ERR(inode)) { + spin_lock(&configfs_dirent_lock); + sd->s_dentry = NULL; + spin_unlock(&configfs_dirent_lock); configfs_put(sd); return ERR_CAST(inode); } @@ -501,8 +488,7 @@ static struct dentry * configfs_lookup(struct inode *dir, } spin_unlock(&configfs_dirent_lock); done: - d_add(dentry, inode); - return NULL; + return d_splice_alias(inode, dentry); } /* @@ -513,9 +499,8 @@ done: * If there is an error, the caller will reset the flags via * configfs_detach_rollback(). */ -static int configfs_detach_prep(struct dentry *dentry, struct dentry **wait) +static int configfs_detach_prep(struct configfs_dirent *parent_sd, struct dentry **wait) { - struct configfs_dirent *parent_sd = dentry->d_fsdata; struct configfs_dirent *sd; int ret; @@ -543,7 +528,7 @@ static int configfs_detach_prep(struct dentry *dentry, struct dentry **wait) * Yup, recursive. If there's a problem, blame * deep nesting of default_groups */ - ret = configfs_detach_prep(sd->s_dentry, wait); + ret = configfs_detach_prep(sd, wait); if (!ret) continue; } else @@ -560,45 +545,77 @@ out: * Walk the tree, resetting CONFIGFS_USET_DROPPING wherever it was * set. */ -static void configfs_detach_rollback(struct dentry *dentry) +static void configfs_detach_rollback(struct configfs_dirent *parent_sd) { - struct configfs_dirent *parent_sd = dentry->d_fsdata; struct configfs_dirent *sd; parent_sd->s_type &= ~CONFIGFS_USET_DROPPING; list_for_each_entry(sd, &parent_sd->s_children, s_sibling) if (sd->s_type & CONFIGFS_USET_DEFAULT) - configfs_detach_rollback(sd->s_dentry); + configfs_detach_rollback(sd); } -static void detach_attrs(struct config_item * item) +/* + * Find the next non-cursor. configfs_dirent_lock held by caller. + */ +static struct configfs_dirent *next_dirent(struct configfs_dirent *parent, + struct configfs_dirent *last) { - struct dentry * dentry = dget(item->ci_dentry); - struct configfs_dirent * parent_sd; - struct configfs_dirent * sd, * tmp; + struct configfs_dirent *s; - if (!dentry) - return; + s = list_prepare_entry(last, &parent->s_children, s_sibling); + + list_for_each_entry_continue(s, &parent->s_children, s_sibling) { + if (s->s_element) + return s; + } + return NULL; +} - pr_debug("configfs %s: dropping attrs for dir\n", - dentry->d_name.name); +static void detach_attrs(struct dentry *dentry) +{ + struct configfs_dirent *parent_sd; + struct configfs_dirent *sd, *next; + + pr_debug("configfs %pd: dropping attrs for dir\n", dentry); parent_sd = dentry->d_fsdata; - list_for_each_entry_safe(sd, tmp, &parent_sd->s_children, s_sibling) { - if (!sd->s_element || !(sd->s_type & CONFIGFS_NOT_PINNED)) + + spin_lock(&configfs_dirent_lock); + for (sd = next_dirent(parent_sd, NULL); sd; sd = next) { + struct dentry *child; + + next = next_dirent(parent_sd, sd); + if (!(sd->s_type & CONFIGFS_NOT_PINNED)) continue; - spin_lock(&configfs_dirent_lock); list_del_init(&sd->s_sibling); + child = sd->s_dentry; + if (child) { + spin_lock(&child->d_lock); + if (simple_positive(child)) { + dget_dlock(child); + __d_drop(child); + spin_unlock(&child->d_lock); + } else { + spin_unlock(&child->d_lock); + child = NULL; + } + } spin_unlock(&configfs_dirent_lock); - configfs_drop_dentry(sd, dentry); + if (child) { + struct inode *inode = child->d_inode; + + inode_lock_nested(inode, I_MUTEX_NONDIR2); + inode_set_ctime_current(inode); + drop_nlink(inode); + inode_unlock(inode); + dput(child); + } configfs_put(sd); + spin_lock(&configfs_dirent_lock); } - - /** - * Drop reference from dget() on entrance. - */ - dput(dentry); + spin_unlock(&configfs_dirent_lock); } static int populate_attrs(struct config_item *item) @@ -620,54 +637,49 @@ static int populate_attrs(struct config_item *item) if (ops && ops->is_visible && !ops->is_visible(item, attr, i)) continue; - if ((error = configfs_create_file(item, attr))) - break; + error = configfs_create_file(item, attr); + if (error) + return error; } } - if (!error && t->ct_bin_attrs) { + if (t->ct_bin_attrs) { for (i = 0; (bin_attr = t->ct_bin_attrs[i]) != NULL; i++) { if (ops && ops->is_bin_visible && !ops->is_bin_visible(item, bin_attr, i)) continue; error = configfs_create_bin_file(item, bin_attr); if (error) - break; + return error; } } - if (error) - detach_attrs(item); - - return error; + return 0; } -static int configfs_attach_group(struct config_item *parent_item, - struct config_item *item, +static int configfs_attach_group(struct config_item *item, struct dentry *dentry, struct configfs_fragment *frag); -static void configfs_detach_group(struct config_item *item); +static void configfs_detach_group(struct dentry *dentry); -static void detach_groups(struct config_group *group) +static void detach_groups(struct dentry *dentry) { - struct dentry * dentry = dget(group->cg_item.ci_dentry); struct dentry *child; struct configfs_dirent *parent_sd; - struct configfs_dirent *sd, *tmp; - - if (!dentry) - return; + struct configfs_dirent *sd, *next; parent_sd = dentry->d_fsdata; - list_for_each_entry_safe(sd, tmp, &parent_sd->s_children, s_sibling) { - if (!sd->s_element || - !(sd->s_type & CONFIGFS_USET_DEFAULT)) + spin_lock(&configfs_dirent_lock); + for (sd = next_dirent(parent_sd, NULL); sd; sd = next) { + next = next_dirent(parent_sd, sd); + if (!(sd->s_type & CONFIGFS_USET_DEFAULT)) continue; - child = sd->s_dentry; + child = dget(sd->s_dentry); + spin_unlock(&configfs_dirent_lock); inode_lock(d_inode(child)); - configfs_detach_group(sd->s_element); + configfs_detach_group(child); d_inode(child)->i_flags |= S_DEAD; dont_mount(child); @@ -675,12 +687,9 @@ static void detach_groups(struct config_group *group) d_delete(child); dput(child); + spin_lock(&configfs_dirent_lock); } - - /** - * Drop reference from dget() on entrance. - */ - dput(dentry); + spin_unlock(&configfs_dirent_lock); } /* @@ -691,14 +700,14 @@ static void detach_groups(struct config_group *group) * We could, perhaps, tweak our parent's ->mkdir for a minute and * try using vfs_mkdir. Just a thought. */ -static int create_default_group(struct config_group *parent_group, +static int create_default_group(struct dentry *parent, struct config_group *group, struct configfs_fragment *frag) { int ret; struct configfs_dirent *sd; /* We trust the caller holds a reference to parent */ - struct dentry *child, *parent = parent_group->cg_item.ci_dentry; + struct dentry *child; if (!group->cg_item.ci_name) group->cg_item.ci_name = group->cg_item.ci_namebuf; @@ -708,16 +717,15 @@ static int create_default_group(struct config_group *parent_group, if (child) { d_add(child, NULL); - ret = configfs_attach_group(&parent_group->cg_item, - &group->cg_item, child, frag); + ret = configfs_attach_group(&group->cg_item, child, frag); if (!ret) { sd = child->d_fsdata; sd->s_type |= CONFIGFS_USET_DEFAULT; } else { BUG_ON(d_inode(child)); d_drop(child); - dput(child); } + dput(child); } return ret; @@ -726,18 +734,15 @@ static int create_default_group(struct config_group *parent_group, static int populate_groups(struct config_group *group, struct configfs_fragment *frag) { + struct dentry *parent = group->cg_item.ci_dentry; struct config_group *new_group; - int ret = 0; list_for_each_entry(new_group, &group->default_groups, group_entry) { - ret = create_default_group(group, new_group, frag); - if (ret) { - detach_groups(group); - break; - } + int ret = create_default_group(parent, new_group, frag); + if (ret) + return ret; } - - return ret; + return 0; } void configfs_remove_default_groups(struct config_group *group) @@ -827,6 +832,13 @@ static void link_group(struct config_group *parent_group, struct config_group *g link_group(group, new_group); } +/* Caller holds the mutex of the item's inode */ +static void configfs_detach_item(struct dentry *dentry) +{ + detach_attrs(dentry); + configfs_remove_dir(dentry); +} + /* * The goal is that configfs_attach_item() (and * configfs_attach_group()) can be called from either the VFS or this @@ -842,8 +854,7 @@ static void link_group(struct config_group *parent_group, struct config_group *g * clean up the configfs items, and they expect their callers will * handle the dcache bits. */ -static int configfs_attach_item(struct config_item *parent_item, - struct config_item *item, +static int configfs_attach_item(struct config_item *item, struct dentry *dentry, struct configfs_fragment *frag) { @@ -859,7 +870,7 @@ static int configfs_attach_item(struct config_item *parent_item, * we must lock them as rmdir() would. */ inode_lock(d_inode(dentry)); - configfs_remove_dir(item); + configfs_detach_item(dentry); d_inode(dentry)->i_flags |= S_DEAD; dont_mount(dentry); inode_unlock(d_inode(dentry)); @@ -870,22 +881,21 @@ static int configfs_attach_item(struct config_item *parent_item, return ret; } -/* Caller holds the mutex of the item's inode */ -static void configfs_detach_item(struct config_item *item) +/* Caller holds the mutex of the group's inode */ +static void configfs_detach_group(struct dentry *dentry) { - detach_attrs(item); - configfs_remove_dir(item); + detach_groups(dentry); + configfs_detach_item(dentry); } -static int configfs_attach_group(struct config_item *parent_item, - struct config_item *item, +static int configfs_attach_group(struct config_item *item, struct dentry *dentry, struct configfs_fragment *frag) { int ret; struct configfs_dirent *sd; - ret = configfs_attach_item(parent_item, item, dentry, frag); + ret = configfs_attach_item(item, dentry, frag); if (!ret) { sd = dentry->d_fsdata; sd->s_type |= CONFIGFS_USET_DIR; @@ -903,7 +913,7 @@ static int configfs_attach_group(struct config_item *parent_item, configfs_adjust_dir_dirent_depth_before_populate(sd); ret = populate_groups(to_config_group(item), frag); if (ret) { - configfs_detach_item(item); + configfs_detach_group(dentry); d_inode(dentry)->i_flags |= S_DEAD; dont_mount(dentry); } @@ -916,13 +926,6 @@ static int configfs_attach_group(struct config_item *parent_item, return ret; } -/* Caller holds the mutex of the group's inode */ -static void configfs_detach_group(struct config_item *item) -{ - detach_groups(to_config_group(item)); - configfs_detach_item(item); -} - /* * After the item has been detached from the filesystem view, we are * ready to tear it out of the hierarchy. Notify the client before @@ -1065,15 +1068,12 @@ static int configfs_dump(struct configfs_dirent *sd, int level) * much on the stack, though, so folks that need this function - be careful * about your stack! Patches will be accepted to make it iterative. */ -static int configfs_depend_prep(struct dentry *origin, +static int configfs_depend_prep(struct configfs_dirent *sd, struct config_item *target) { - struct configfs_dirent *child_sd, *sd; + struct configfs_dirent *child_sd; int ret = 0; - BUG_ON(!origin || !origin->d_fsdata); - sd = origin->d_fsdata; - if (sd->s_element == target) /* Boo-yah */ goto out; @@ -1081,8 +1081,7 @@ static int configfs_depend_prep(struct dentry *origin, if ((child_sd->s_type & CONFIGFS_DIR) && !(child_sd->s_type & CONFIGFS_USET_DROPPING) && !(child_sd->s_type & CONFIGFS_USET_CREATING)) { - ret = configfs_depend_prep(child_sd->s_dentry, - target); + ret = configfs_depend_prep(child_sd, target); if (!ret) goto out; /* Child path boo-yah */ } @@ -1095,7 +1094,7 @@ out: return ret; } -static int configfs_do_depend_item(struct dentry *subsys_dentry, +static int configfs_do_depend_item(struct configfs_dirent *subsys_sd, struct config_item *target) { struct configfs_dirent *p; @@ -1103,7 +1102,7 @@ static int configfs_do_depend_item(struct dentry *subsys_dentry, spin_lock(&configfs_dirent_lock); /* Scan the tree, return 0 if found */ - ret = configfs_depend_prep(subsys_dentry, target); + ret = configfs_depend_prep(subsys_sd, target); if (ret) goto out_unlock_dirent_lock; @@ -1127,6 +1126,7 @@ configfs_find_subsys_dentry(struct configfs_dirent *root_sd, struct configfs_dirent *p; struct configfs_dirent *ret = NULL; + spin_lock(&configfs_dirent_lock); list_for_each_entry(p, &root_sd->s_children, s_sibling) { if (p->s_type & CONFIGFS_DIR && p->s_element == subsys_item) { @@ -1134,6 +1134,7 @@ configfs_find_subsys_dentry(struct configfs_dirent *root_sd, break; } } + spin_unlock(&configfs_dirent_lock); return ret; } @@ -1169,7 +1170,7 @@ int configfs_depend_item(struct configfs_subsystem *subsys, } /* Ok, now we can trust subsys/s_item */ - ret = configfs_do_depend_item(subsys_sd->s_dentry, target); + ret = configfs_do_depend_item(subsys_sd, target); out_unlock_fs: inode_unlock(d_inode(root)); @@ -1271,7 +1272,7 @@ int configfs_depend_item_unlocked(struct configfs_subsystem *caller_subsys, } /* Now we can execute core of depend item */ - ret = configfs_do_depend_item(subsys_sd->s_dentry, target); + ret = configfs_do_depend_item(subsys_sd, target); if (target_subsys != caller_subsys) out_root_unlock: @@ -1298,7 +1299,11 @@ static struct dentry *configfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, const struct config_item_type *type; struct module *subsys_owner = NULL, *new_item_owner = NULL; struct configfs_fragment *frag; - char *name; + struct name_snapshot n; + const char *name; + + take_dentry_name_snapshot(&n, dentry); + name = n.name.name; sd = dentry->d_parent->d_fsdata; @@ -1350,14 +1355,6 @@ static struct dentry *configfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, goto out_put; } - name = kmalloc(dentry->d_name.len + 1, GFP_KERNEL); - if (!name) { - ret = -ENOMEM; - goto out_subsys_put; - } - - snprintf(name, dentry->d_name.len + 1, "%s", dentry->d_name.name); - mutex_lock(&subsys->su_mutex); if (type->ct_group_ops->make_group) { group = type->ct_group_ops->make_group(to_config_group(parent_item), name); @@ -1379,7 +1376,6 @@ static struct dentry *configfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, } mutex_unlock(&subsys->su_mutex); - kfree(name); if (ret) { /* * If ret != 0, then link_obj() was never called. @@ -1424,9 +1420,9 @@ static struct dentry *configfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, spin_unlock(&configfs_dirent_lock); if (group) - ret = configfs_attach_group(parent_item, item, dentry, frag); + ret = configfs_attach_group(item, dentry, frag); else - ret = configfs_attach_item(parent_item, item, dentry, frag); + ret = configfs_attach_item(item, dentry, frag); spin_lock(&configfs_dirent_lock); sd->s_type &= ~CONFIGFS_USET_IN_MKDIR; @@ -1466,6 +1462,7 @@ out_put: put_fragment(frag); out: + release_dentry_name_snapshot(&n); return ERR_PTR(ret); } @@ -1513,9 +1510,9 @@ static int configfs_rmdir(struct inode *dir, struct dentry *dentry) */ ret = sd->s_dependent_count ? -EBUSY : 0; if (!ret) { - ret = configfs_detach_prep(dentry, &wait); + ret = configfs_detach_prep(sd, &wait); if (ret) - configfs_detach_rollback(dentry); + configfs_detach_rollback(sd); } spin_unlock(&configfs_dirent_lock); mutex_unlock(&configfs_symlink_mutex); @@ -1536,7 +1533,7 @@ static int configfs_rmdir(struct inode *dir, struct dentry *dentry) frag = sd->s_frag; if (down_write_killable(&frag->frag_sem)) { spin_lock(&configfs_dirent_lock); - configfs_detach_rollback(dentry); + configfs_detach_rollback(sd); spin_unlock(&configfs_dirent_lock); config_item_put(parent_item); return -EINTR; @@ -1554,13 +1551,13 @@ static int configfs_rmdir(struct inode *dir, struct dentry *dentry) dead_item_owner = item->ci_type->ct_owner; if (sd->s_type & CONFIGFS_USET_DIR) { - configfs_detach_group(item); + configfs_detach_group(dentry); mutex_lock(&subsys->su_mutex); client_disconnect_notify(parent_item, item); unlink_group(to_config_group(item)); } else { - configfs_detach_item(item); + configfs_detach_item(dentry); mutex_lock(&subsys->su_mutex); client_disconnect_notify(parent_item, item); @@ -1770,7 +1767,7 @@ int configfs_register_group(struct config_group *parent_group, parent = parent_group->cg_item.ci_dentry; inode_lock_nested(d_inode(parent), I_MUTEX_PARENT); - ret = create_default_group(parent_group, group, frag); + ret = create_default_group(parent, group, frag); if (ret) goto err_out; @@ -1799,7 +1796,7 @@ EXPORT_SYMBOL(configfs_register_group); void configfs_unregister_group(struct config_group *group) { struct configfs_subsystem *subsys = group->cg_subsys; - struct dentry *dentry = group->cg_item.ci_dentry; + struct dentry *dentry = dget(group->cg_item.ci_dentry); struct dentry *parent = group->cg_item.ci_parent->ci_dentry; struct configfs_dirent *sd = dentry->d_fsdata; struct configfs_fragment *frag = sd->s_frag; @@ -1810,10 +1807,10 @@ void configfs_unregister_group(struct config_group *group) inode_lock_nested(d_inode(parent), I_MUTEX_PARENT); spin_lock(&configfs_dirent_lock); - configfs_detach_prep(dentry, NULL); + configfs_detach_prep(sd, NULL); spin_unlock(&configfs_dirent_lock); - configfs_detach_group(&group->cg_item); + configfs_detach_group(dentry); d_inode(dentry)->i_flags |= S_DEAD; dont_mount(dentry); d_drop(dentry); @@ -1908,18 +1905,17 @@ int configfs_register_subsystem(struct configfs_subsystem *subsys) err = configfs_dirent_exists(dentry); if (!err) - err = configfs_attach_group(sd->s_element, - &group->cg_item, + err = configfs_attach_group(&group->cg_item, dentry, frag); if (err) { BUG_ON(d_inode(dentry)); d_drop(dentry); - dput(dentry); } else { spin_lock(&configfs_dirent_lock); configfs_dir_set_ready(dentry->d_fsdata); spin_unlock(&configfs_dirent_lock); } + dput(dentry); } inode_unlock(d_inode(root)); @@ -1938,7 +1934,7 @@ int configfs_register_subsystem(struct configfs_subsystem *subsys) void configfs_unregister_subsystem(struct configfs_subsystem *subsys) { struct config_group *group = &subsys->su_group; - struct dentry *dentry = group->cg_item.ci_dentry; + struct dentry *dentry = dget(group->cg_item.ci_dentry); struct dentry *root = dentry->d_sb->s_root; struct configfs_dirent *sd = dentry->d_fsdata; struct configfs_fragment *frag = sd->s_frag; @@ -1957,12 +1953,12 @@ void configfs_unregister_subsystem(struct configfs_subsystem *subsys) inode_lock_nested(d_inode(dentry), I_MUTEX_CHILD); mutex_lock(&configfs_symlink_mutex); spin_lock(&configfs_dirent_lock); - if (configfs_detach_prep(dentry, NULL)) { + if (configfs_detach_prep(sd, NULL)) { pr_err("Tried to unregister non-empty subsystem!\n"); } spin_unlock(&configfs_dirent_lock); mutex_unlock(&configfs_symlink_mutex); - configfs_detach_group(&group->cg_item); + configfs_detach_group(dentry); d_inode(dentry)->i_flags |= S_DEAD; dont_mount(dentry); inode_unlock(d_inode(dentry)); diff --git a/fs/configfs/file.c b/fs/configfs/file.c index ef8c3cd10cc6..a48cece775a3 100644 --- a/fs/configfs/file.c +++ b/fs/configfs/file.c @@ -59,7 +59,7 @@ static int fill_read_buffer(struct file *file, struct configfs_buffer *buffer) ssize_t count = -ENOENT; if (!buffer->page) - buffer->page = (char *) get_zeroed_page(GFP_KERNEL); + buffer->page = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!buffer->page) return -ENOMEM; @@ -184,7 +184,7 @@ static int fill_write_buffer(struct configfs_buffer *buffer, int copied; if (!buffer->page) - buffer->page = (char *)__get_free_pages(GFP_KERNEL, 0); + buffer->page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!buffer->page) return -ENOMEM; @@ -381,8 +381,7 @@ static int configfs_release(struct inode *inode, struct file *filp) struct configfs_buffer *buffer = filp->private_data; module_put(buffer->owner); - if (buffer->page) - free_page((unsigned long)buffer->page); + kfree(buffer->page); mutex_destroy(&buffer->mutex); kfree(buffer); return 0; diff --git a/fs/configfs/inode.c b/fs/configfs/inode.c index bc507b720a01..68290fe0e374 100644 --- a/fs/configfs/inode.c +++ b/fs/configfs/inode.c @@ -157,7 +157,6 @@ struct inode *configfs_create(struct dentry *dentry, umode_t mode) { struct inode *inode = NULL; struct configfs_dirent *sd; - struct inode *p_inode; if (!dentry) return ERR_PTR(-ENOENT); @@ -170,8 +169,6 @@ struct inode *configfs_create(struct dentry *dentry, umode_t mode) if (!inode) return ERR_PTR(-ENOMEM); - p_inode = d_inode(dentry->d_parent); - inode_set_mtime_to_ts(p_inode, inode_set_ctime_current(p_inode)); configfs_set_inode_lock_class(sd, inode); return inode; } @@ -195,25 +192,3 @@ const unsigned char * configfs_get_name(struct configfs_dirent *sd) } return NULL; } - - -/* - * Unhashes the dentry corresponding to given configfs_dirent - * Called with parent inode's i_mutex held. - */ -void configfs_drop_dentry(struct configfs_dirent * sd, struct dentry * parent) -{ - struct dentry * dentry = sd->s_dentry; - - if (dentry) { - spin_lock(&dentry->d_lock); - if (simple_positive(dentry)) { - dget_dlock(dentry); - __d_drop(dentry); - spin_unlock(&dentry->d_lock); - __simple_unlink(d_inode(parent), dentry); - dput(dentry); - } else - spin_unlock(&dentry->d_lock); - } -} diff --git a/fs/configfs/symlink.c b/fs/configfs/symlink.c index f3f79c67add5..31eb28b27309 100644 --- a/fs/configfs/symlink.c +++ b/fs/configfs/symlink.c @@ -229,9 +229,8 @@ int configfs_unlink(struct inode *dir, struct dentry *dentry) spin_lock(&configfs_dirent_lock); list_del_init(&sd->s_sibling); spin_unlock(&configfs_dirent_lock); - configfs_drop_dentry(sd, dentry->d_parent); - dput(dentry); configfs_put(sd); + simple_unlink(dir, dentry); /* * drop_link() must be called before diff --git a/fs/coredump.c b/fs/coredump.c index bb6fdb1f458e..e68a76ff92a3 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -395,8 +395,7 @@ static bool coredump_parse(struct core_name *cn, struct coredump_params *cprm, cred->gid)); break; case 'd': - err = cn_printf(cn, "%d", - __get_dumpable(cprm->mm_flags)); + err = cn_printf(cn, "%d", cprm->dumpable); break; /* signal that caused the coredump */ case 's': @@ -869,11 +868,11 @@ static inline void coredump_sock_shutdown(struct file *file) { } static inline bool coredump_socket(struct core_name *cn, struct coredump_params *cprm) { return false; } #endif -/* cprm->mm_flags contains a stable snapshot of dumpability flags. */ +/* cprm->dumpable is the snapshot of task dumpability at dump start. */ static inline bool coredump_force_suid_safe(const struct coredump_params *cprm) { /* Require nonrelative corefile path and be extra careful. */ - return __get_dumpable(cprm->mm_flags) == SUID_DUMP_ROOT; + return cprm->dumpable == TASK_DUMPABLE_ROOT; } static bool coredump_file(struct core_name *cn, struct coredump_params *cprm, @@ -1085,7 +1084,7 @@ static inline bool coredump_skip(const struct coredump_params *cprm, return true; if (!binfmt->core_dump) return true; - if (!__get_dumpable(cprm->mm_flags)) + if (cprm->dumpable == TASK_DUMPABLE_OFF) return true; return false; } @@ -1170,14 +1169,9 @@ void vfs_coredump(const kernel_siginfo_t *siginfo) struct coredump_params cprm = { .siginfo = siginfo, .limit = rlimit(RLIMIT_CORE), - /* - * We must use the same mm->flags while dumping core to avoid - * inconsistency of bit flags, since this flag is not protected - * by any locks. - * - * Note that we only care about MMF_DUMP* flags. - */ - .mm_flags = __mm_flags_get_dumpable(mm), + /* Snapshot MMF_DUMP_FILTER_* (unlocked) and dumpable for the dump. */ + .mm_flags = __mm_flags_get_word(mm), + .dumpable = task_exec_state_get_dumpable(current), .vma_meta = NULL, .cpu = raw_smp_processor_id(), }; @@ -1419,7 +1413,7 @@ EXPORT_SYMBOL(dump_align); void validate_coredump_safety(void) { - if (suid_dumpable == SUID_DUMP_ROOT && + if (suid_dumpable == TASK_DUMPABLE_ROOT && core_pattern[0] != '/' && core_pattern[0] != '|' && core_pattern[0] != '@') { coredump_report_failure("Unsafe core_pattern used with fs.suid_dumpable=2: " @@ -1488,7 +1482,8 @@ static int proc_dostring_coredump(const struct ctl_table *table, int write, return -EINVAL; } - validate_coredump_safety(); + if (strncmp(old_core_pattern, core_pattern, CORENAME_MAX_SIZE)) + validate_coredump_safety(); return error; } diff --git a/fs/cramfs/README b/fs/cramfs/README index 778df5c4d70b..c0052a11aa28 100644 --- a/fs/cramfs/README +++ b/fs/cramfs/README @@ -104,94 +104,4 @@ Tools ----- The cramfs user-space tools, including mkcramfs and cramfsck, are -located at <http://sourceforge.net/projects/cramfs/>. - - -Future Development -================== - -Block Size ----------- - -(Block size in cramfs refers to the size of input data that is -compressed at a time. It's intended to be somewhere around -PAGE_SIZE for cramfs_read_folio's convenience.) - -The superblock ought to indicate the block size that the fs was -written for, since comments in <linux/pagemap.h> indicate that -PAGE_SIZE may grow in future (if I interpret the comment -correctly). - -Currently, mkcramfs #define's PAGE_SIZE as 4096 and uses that -for blksize, whereas Linux-2.3.39 uses its PAGE_SIZE, which in -turn is defined as PAGE_SIZE (which can be as large as 32KB on arm). -This discrepancy is a bug, though it's not clear which should be -changed. - -One option is to change mkcramfs to take its PAGE_SIZE from -<asm/page.h>. Personally I don't like this option, but it does -require the least amount of change: just change `#define -PAGE_SIZE (4096)' to `#include <asm/page.h>'. The disadvantage -is that the generated cramfs cannot always be shared between different -kernels, not even necessarily kernels of the same architecture if -PAGE_SIZE is subject to change between kernel versions -(currently possible with arm and ia64). - -The remaining options try to make cramfs more sharable. - -One part of that is addressing endianness. The two options here are -`always use little-endian' (like ext2fs) or `writer chooses -endianness; kernel adapts at runtime'. Little-endian wins because of -code simplicity and little CPU overhead even on big-endian machines. - -The cost of swabbing is changing the code to use the le32_to_cpu -etc. macros as used by ext2fs. We don't need to swab the compressed -data, only the superblock, inodes and block pointers. - - -The other part of making cramfs more sharable is choosing a block -size. The options are: - - 1. Always 4096 bytes. - - 2. Writer chooses blocksize; kernel adapts but rejects blocksize > - PAGE_SIZE. - - 3. Writer chooses blocksize; kernel adapts even to blocksize > - PAGE_SIZE. - -It's easy enough to change the kernel to use a smaller value than -PAGE_SIZE: just make cramfs_read_folio read multiple blocks. - -The cost of option 1 is that kernels with a larger PAGE_SIZE -value don't get as good compression as they can. - -The cost of option 2 relative to option 1 is that the code uses -variables instead of #define'd constants. The gain is that people -with kernels having larger PAGE_SIZE can make use of that if -they don't mind their cramfs being inaccessible to kernels with -smaller PAGE_SIZE values. - -Option 3 is easy to implement if we don't mind being CPU-inefficient: -e.g. get read_folio to decompress to a buffer of size MAX_BLKSIZE (which -must be no larger than 32KB) and discard what it doesn't need. -Getting read_folio to read into all the covered pages is harder. - -The main advantage of option 3 over 1, 2, is better compression. The -cost is greater complexity. Probably not worth it, but I hope someone -will disagree. (If it is implemented, then I'll re-use that code in -e2compr.) - - -Another cost of 2 and 3 over 1 is making mkcramfs use a different -block size, but that just means adding and parsing a -b option. - - -Inode Size ----------- - -Given that cramfs will probably be used for CDs etc. as well as just -silicon ROMs, it might make sense to expand the inode a little from -its current 12 bytes. Inodes other than the root inode are followed -by filename, so the expansion doesn't even have to be a multiple of 4 -bytes. +located at <https://github.com/npitre/cramfs-tools>. diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index 8d3c278a7591..0053b5c45412 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -236,7 +236,7 @@ struct fscrypt_symlink_data { * @tfm: crypto API transform object * @blk_key: key for blk-crypto * - * Normally only one of the fields will be non-NULL. + * Only one of the fields is non-NULL. */ struct fscrypt_prepared_key { struct crypto_sync_skcipher *tfm; @@ -245,6 +245,15 @@ struct fscrypt_prepared_key { #endif }; +/* An entry in the linked list ->mk_mode_keys */ +struct fscrypt_mode_key { + struct fscrypt_prepared_key key; + struct list_head link; + u8 hkdf_context; + u8 mode_num; + u8 data_unit_bits; +}; + /* * fscrypt_inode_info - the "encryption key" for an inode * @@ -430,20 +439,12 @@ int fscrypt_derive_sw_secret(struct super_block *sb, * @prep_key, depending on which encryption implementation the file will use. */ static inline bool -fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, +fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key, const struct fscrypt_inode_info *ci) { - /* - * The two smp_load_acquire()'s here pair with the smp_store_release()'s - * in fscrypt_prepare_inline_crypt_key() and fscrypt_prepare_key(). - * I.e., in some cases (namely, if this prep_key is a per-mode - * encryption key) another task can publish blk_key or tfm concurrently, - * executing a RELEASE barrier. We need to use smp_load_acquire() here - * to safely ACQUIRE the memory the other task published. - */ if (fscrypt_using_inline_encryption(ci)) - return smp_load_acquire(&prep_key->blk_key) != NULL; - return smp_load_acquire(&prep_key->tfm) != NULL; + return prep_key->blk_key != NULL; + return prep_key->tfm != NULL; } #else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ @@ -486,16 +487,29 @@ fscrypt_derive_sw_secret(struct super_block *sb, } static inline bool -fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, +fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key, const struct fscrypt_inode_info *ci) { - return smp_load_acquire(&prep_key->tfm) != NULL; + return prep_key->tfm != NULL; } #endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ /* keyring.c */ /* + * fscrypt_master_key_user - a user's claim to a master key + */ +struct fscrypt_master_key_user { + struct list_head link; + kuid_t uid; + /* + * This 'struct key' contains no secret. It exists solely to charge the + * appropriate user's key quota. + */ + struct key *quota_key; +}; + +/* * fscrypt_master_key_secret - secret key material of an in-use master key */ struct fscrypt_master_key_secret { @@ -577,8 +591,8 @@ struct fscrypt_master_key { /* * Active and structural reference counts. An active ref guarantees * that the struct continues to exist, continues to be in the keyring - * ->s_master_keys, and that any embedded subkeys (e.g. - * ->mk_direct_keys) that have been prepared continue to exist. + * ->s_master_keys, and that any non-file-scoped subkeys (e.g. + * ->mk_mode_keys) that have been prepared continue to exist. * A structural ref only guarantees that the struct continues to exist. * * There is one active ref associated with ->mk_present being true, and @@ -610,19 +624,18 @@ struct fscrypt_master_key { struct fscrypt_key_specifier mk_spec; /* - * Keyring which contains a key of type 'key_type_fscrypt_user' for each - * user who has added this key. Normally each key will be added by just - * one user, but it's possible that multiple users share a key, and in - * that case we need to keep track of those users so that one user can't - * remove the key before the others want it removed too. + * List of user claims to this key (struct fscrypt_master_key_user). + * Normally each key will be added by just one user, but it's possible + * that multiple users share a key, and in that case we need to keep + * track of those users so that one user can't remove the key before the + * others want it removed too. * - * This is NULL for v1 policy keys; those can only be added by root. + * Used only for v2 policy keys. v1 policy keys can be added only by + * root, so user tracking doesn't apply to them. * - * Locking: protected by ->mk_sem. (We don't just rely on the keyrings - * subsystem semaphore ->mk_users->sem, as we need support for atomic - * search+insert along with proper synchronization with other fields.) + * Locking: protected by ->mk_sem. */ - struct key *mk_users; + struct list_head mk_users; /* * List of inodes that were unlocked using this key. This allows the @@ -632,12 +645,21 @@ struct fscrypt_master_key { spinlock_t mk_decrypted_inodes_lock; /* - * Per-mode encryption keys for the various types of encryption policies - * that use them. Allocated and derived on-demand. + * A list of 'struct fscrypt_mode_key' for the (hkdf_context, mode_num, + * data_unit_bits, inlinecrypt) combinations that are in use for this + * master key, for hkdf_context in [HKDF_CONTEXT_DIRECT_KEY, + * HKDF_CONTEXT_IV_INO_LBLK_32_KEY, HKDF_CONTEXT_IV_INO_LBLK_64_KEY]. + * + * This is a linked list and not a hash table because in practice + * there's just a single encryption policy per master key, using + * _at most_ 2 nodes in this list. Per-file keys don't use this at all. + * + * This list is append-only until the master key is fully removed, at + * which time the list is cleared. Before then, + * fscrypt_mode_key_setup_mutex synchronizes appends, and searches use + * the RCU read lock together with ->mk_sem held for read. */ - struct fscrypt_prepared_key mk_direct_keys[FSCRYPT_MODE_MAX + 1]; - struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[FSCRYPT_MODE_MAX + 1]; - struct fscrypt_prepared_key mk_iv_ino_lblk_32_keys[FSCRYPT_MODE_MAX + 1]; + struct list_head mk_mode_keys; /* Hash key for inode numbers. Initialized only when needed. */ siphash_key_t mk_ino_hash_key; diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index 37d42d357925..47324062fee5 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -198,13 +198,7 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, goto fail; } - /* - * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared(). - * I.e., here we publish ->blk_key with a RELEASE barrier so that - * concurrent tasks can ACQUIRE it. Note that this concurrency is only - * possible for per-mode keys, not for per-file keys. - */ - smp_store_release(&prep_key->blk_key, blk_key); + prep_key->blk_key = blk_key; return 0; fail: diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index be8e6e8011f2..38b73e703073 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -65,36 +65,33 @@ static void fscrypt_free_master_key(struct rcu_head *head) kfree_sensitive(mk); } +static void clear_mk_users(struct fscrypt_master_key *mk); + void fscrypt_put_master_key(struct fscrypt_master_key *mk) { if (!refcount_dec_and_test(&mk->mk_struct_refs)) return; /* - * No structural references left, so free ->mk_users, and also free the + * No structural references left, so clear ->mk_users, and also free the * fscrypt_master_key struct itself after an RCU grace period ensures * that concurrent keyring lookups can no longer find it. */ WARN_ON_ONCE(refcount_read(&mk->mk_active_refs) != 0); - if (mk->mk_users) { - /* Clear the keyring so the quota gets released right away. */ - keyring_clear(mk->mk_users); - key_put(mk->mk_users); - mk->mk_users = NULL; - } + clear_mk_users(mk); call_rcu(&mk->mk_rcu_head, fscrypt_free_master_key); } void fscrypt_put_master_key_activeref(struct super_block *sb, struct fscrypt_master_key *mk) { - size_t i; + struct fscrypt_mode_key *node, *tmp; if (!refcount_dec_and_test(&mk->mk_active_refs)) return; /* * No active references left, so complete the full removal of this * fscrypt_master_key struct by removing it from the keyring and - * destroying any subkeys embedded in it. + * destroying any non-file-scoped subkeys. */ if (WARN_ON_ONCE(!sb->s_master_keys)) @@ -110,13 +107,16 @@ void fscrypt_put_master_key_activeref(struct super_block *sb, WARN_ON_ONCE(mk->mk_present); WARN_ON_ONCE(!list_empty(&mk->mk_decrypted_inodes)); - for (i = 0; i <= FSCRYPT_MODE_MAX; i++) { - fscrypt_destroy_prepared_key( - sb, &mk->mk_direct_keys[i]); - fscrypt_destroy_prepared_key( - sb, &mk->mk_iv_ino_lblk_64_keys[i]); - fscrypt_destroy_prepared_key( - sb, &mk->mk_iv_ino_lblk_32_keys[i]); + /* + * Destroy any non-file-scoped subkeys. Since ->mk_active_refs == 0, + * they're no longer referenced by any inodes. Nor can key setup run + * and use them again. So they're no longer needed. (This implies no + * concurrent readers, so we don't need list_del_rcu() for example.) + */ + list_for_each_entry_safe(node, tmp, &mk->mk_mode_keys, link) { + fscrypt_destroy_prepared_key(sb, &node->key); + list_del(&node->link); + kfree(node); } memzero_explicit(&mk->mk_ino_hash_key, sizeof(mk->mk_ino_hash_key)); @@ -162,8 +162,8 @@ static void fscrypt_user_key_describe(const struct key *key, struct seq_file *m) } /* - * Type of key in ->mk_users. Each key of this type represents a particular - * user who has added a particular master key. + * Type of fscrypt_master_key_user::quota_key. This contains no secret; it + * exists solely to charge a user's key quota. * * Note that the name of this key type really should be something like * ".fscrypt-user" instead of simply ".fscrypt". But the shorter name is chosen @@ -177,30 +177,9 @@ static struct key_type key_type_fscrypt_user = { .describe = fscrypt_user_key_describe, }; -#define FSCRYPT_MK_USERS_DESCRIPTION_SIZE \ - (CONST_STRLEN("fscrypt-") + 2 * FSCRYPT_KEY_IDENTIFIER_SIZE + \ - CONST_STRLEN("-users") + 1) - #define FSCRYPT_MK_USER_DESCRIPTION_SIZE \ (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + CONST_STRLEN(".uid.") + 10 + 1) -static void format_mk_users_keyring_description( - char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE], - const u8 mk_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) -{ - sprintf(description, "fscrypt-%*phN-users", - FSCRYPT_KEY_IDENTIFIER_SIZE, mk_identifier); -} - -static void format_mk_user_description( - char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE], - const u8 mk_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) -{ - - sprintf(description, "%*phN.uid.%u", FSCRYPT_KEY_IDENTIFIER_SIZE, - mk_identifier, __kuid_val(current_fsuid())); -} - /* Create ->s_master_keys if needed. Synchronized by fscrypt_add_key_mutex. */ static int allocate_filesystem_keyring(struct super_block *sb) { @@ -335,91 +314,94 @@ out: return mk; } -static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk) -{ - char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE]; - struct key *keyring; - - format_mk_users_keyring_description(description, - mk->mk_spec.u.identifier); - keyring = keyring_alloc(description, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, - current_cred(), KEY_POS_SEARCH | - KEY_USR_SEARCH | KEY_USR_READ | KEY_USR_VIEW, - KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); - if (IS_ERR(keyring)) - return PTR_ERR(keyring); - - mk->mk_users = keyring; - return 0; -} - -/* - * Find the current user's "key" in the master key's ->mk_users. - * Returns ERR_PTR(-ENOKEY) if not found. - */ -static struct key *find_master_key_user(struct fscrypt_master_key *mk) +/* Find the current user's claim in ->mk_users. ->mk_sem must be held. */ +static struct fscrypt_master_key_user * +find_master_key_user(struct fscrypt_master_key *mk) { - char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE]; - key_ref_t keyref; - - format_mk_user_description(description, mk->mk_spec.u.identifier); + struct fscrypt_master_key_user *mk_user; + kuid_t uid = current_fsuid(); - /* - * We need to mark the keyring reference as "possessed" so that we - * acquire permission to search it, via the KEY_POS_SEARCH permission. - */ - keyref = keyring_search(make_key_ref(mk->mk_users, true /*possessed*/), - &key_type_fscrypt_user, description, false); - if (IS_ERR(keyref)) { - if (PTR_ERR(keyref) == -EAGAIN || /* not found */ - PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */ - keyref = ERR_PTR(-ENOKEY); - return ERR_CAST(keyref); + list_for_each_entry(mk_user, &mk->mk_users, link) { + if (uid_eq(mk_user->uid, uid)) + return mk_user; } - return key_ref_to_ptr(keyref); + return NULL; } /* - * Give the current user a "key" in ->mk_users. This charges the user's quota + * Give the current user a claim in ->mk_users. This charges the user's quota * and marks the master key as added by the current user, so that it cannot be * removed by another user with the key. Either ->mk_sem must be held for * write, or the master key must be still undergoing initialization. */ static int add_master_key_user(struct fscrypt_master_key *mk) { + kuid_t uid = current_fsuid(); char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE]; - struct key *mk_user; + struct key *quota_key; + struct fscrypt_master_key_user *mk_user; int err; - format_mk_user_description(description, mk->mk_spec.u.identifier); - mk_user = key_alloc(&key_type_fscrypt_user, description, - current_fsuid(), current_gid(), current_cred(), - KEY_POS_SEARCH | KEY_USR_VIEW, 0, NULL); - if (IS_ERR(mk_user)) - return PTR_ERR(mk_user); + snprintf(description, sizeof(description), "%*phN.uid.%u", + FSCRYPT_KEY_IDENTIFIER_SIZE, mk->mk_spec.u.identifier, + __kuid_val(uid)); + quota_key = key_alloc(&key_type_fscrypt_user, description, uid, + current_gid(), current_cred(), + KEY_POS_SEARCH | KEY_USR_VIEW, 0, NULL); + if (IS_ERR(quota_key)) + return PTR_ERR(quota_key); + + err = key_instantiate_and_link(quota_key, NULL, 0, NULL, NULL); + if (err) { + key_put(quota_key); + return err; + } - err = key_instantiate_and_link(mk_user, NULL, 0, mk->mk_users, NULL); - key_put(mk_user); - return err; + mk_user = kzalloc_obj(*mk_user); + if (!mk_user) { + key_put(quota_key); + return -ENOMEM; + } + mk_user->uid = uid; + mk_user->quota_key = quota_key; + list_add(&mk_user->link, &mk->mk_users); + return 0; +} + +static void unlink_and_free_mk_user(struct fscrypt_master_key_user *mk_user) +{ + list_del(&mk_user->link); + key_put(mk_user->quota_key); + kfree(mk_user); } /* - * Remove the current user's "key" from ->mk_users. + * Remove the current user's claim from ->mk_users. * ->mk_sem must be held for write. * - * Returns 0 if removed, -ENOKEY if not found, or another -errno code. + * Returns 0 if removed or -ENOKEY if not found. */ static int remove_master_key_user(struct fscrypt_master_key *mk) { - struct key *mk_user; - int err; + struct fscrypt_master_key_user *mk_user; mk_user = find_master_key_user(mk); - if (IS_ERR(mk_user)) - return PTR_ERR(mk_user); - err = key_unlink(mk->mk_users, mk_user); - key_put(mk_user); - return err; + if (!mk_user) + return -ENOKEY; + unlink_and_free_mk_user(mk_user); + return 0; +} + +/* + * Clear ->mk_users. Either ->mk_sem must be held for write, or 'mk' must have + * no structural references left. + */ +static void clear_mk_users(struct fscrypt_master_key *mk) +{ + struct fscrypt_master_key_user *mk_user, *tmp; + + list_for_each_entry_safe(mk_user, tmp, &mk->mk_users, link) + unlink_and_free_mk_user(mk_user); } /* @@ -442,13 +424,14 @@ static int add_new_master_key(struct super_block *sb, refcount_set(&mk->mk_struct_refs, 1); mk->mk_spec = *mk_spec; + INIT_LIST_HEAD(&mk->mk_users); + INIT_LIST_HEAD(&mk->mk_decrypted_inodes); spin_lock_init(&mk->mk_decrypted_inodes_lock); + INIT_LIST_HEAD(&mk->mk_mode_keys); + if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { - err = allocate_master_key_users_keyring(mk); - if (err) - goto out_put; err = add_master_key_user(mk); if (err) goto out_put; @@ -477,19 +460,13 @@ static int add_existing_master_key(struct fscrypt_master_key *mk, int err; /* - * If the current user is already in ->mk_users, then there's nothing to - * do. Otherwise, we need to add the user to ->mk_users. (Neither is - * applicable for v1 policy keys, which have NULL ->mk_users.) + * For v2 policy keys (FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER): If the current + * user is already in ->mk_users, then there's nothing to do. + * Otherwise, add the user to ->mk_users. */ - if (mk->mk_users) { - struct key *mk_user = find_master_key_user(mk); - - if (mk_user != ERR_PTR(-ENOKEY)) { - if (IS_ERR(mk_user)) - return PTR_ERR(mk_user); - key_put(mk_user); + if (mk->mk_spec.type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + if (find_master_key_user(mk) != NULL) return 0; - } err = add_master_key_user(mk); if (err) return err; @@ -888,7 +865,6 @@ int fscrypt_verify_key_added(struct super_block *sb, { struct fscrypt_key_specifier mk_spec; struct fscrypt_master_key *mk; - struct key *mk_user; int err; mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER; @@ -900,13 +876,10 @@ int fscrypt_verify_key_added(struct super_block *sb, goto out; } down_read(&mk->mk_sem); - mk_user = find_master_key_user(mk); - if (IS_ERR(mk_user)) { - err = PTR_ERR(mk_user); - } else { - key_put(mk_user); + if (find_master_key_user(mk) != NULL) err = 0; - } + else + err = -ENOKEY; up_read(&mk->mk_sem); fscrypt_put_master_key(mk); out: @@ -1098,16 +1071,18 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) down_write(&mk->mk_sem); /* If relevant, remove current user's (or all users) claim to the key */ - if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) { - if (all_users) - err = keyring_clear(mk->mk_users); - else + if (!list_empty(&mk->mk_users)) { + if (all_users) { + clear_mk_users(mk); + err = 0; + } else { err = remove_master_key_user(mk); + } if (err) { up_write(&mk->mk_sem); goto out_put_key; } - if (mk->mk_users->keys.nr_leaves_on_tree != 0) { + if (!list_empty(&mk->mk_users)) { /* * Other users have still added the key too. We removed * the current user's claim to the key, but we still @@ -1193,6 +1168,8 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) struct super_block *sb = file_inode(filp)->i_sb; struct fscrypt_get_key_status_arg arg; struct fscrypt_master_key *mk; + kuid_t uid; + const struct fscrypt_master_key_user *mk_user; int err; if (copy_from_user(&arg, uarg, sizeof(arg))) @@ -1225,19 +1202,13 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) } arg.status = FSCRYPT_KEY_STATUS_PRESENT; - if (mk->mk_users) { - struct key *mk_user; - arg.user_count = mk->mk_users->keys.nr_leaves_on_tree; - mk_user = find_master_key_user(mk); - if (!IS_ERR(mk_user)) { + uid = current_fsuid(); + list_for_each_entry(mk_user, &mk->mk_users, link) { + arg.user_count++; + if (uid_eq(mk_user->uid, uid)) arg.status_flags |= FSCRYPT_KEY_STATUS_FLAG_ADDED_BY_SELF; - key_put(mk_user); - } else if (mk_user != ERR_PTR(-ENOKEY)) { - err = PTR_ERR(mk_user); - goto out_release_key; - } } err = 0; out_release_key: diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index ce327bfdada4..f905f9f94bdd 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -163,13 +163,7 @@ int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode); if (IS_ERR(tfm)) return PTR_ERR(tfm); - /* - * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared(). - * I.e., here we publish ->tfm with a RELEASE barrier so that - * concurrent tasks can ACQUIRE it. Note that this concurrency is only - * possible for per-mode keys, not for per-file keys. - */ - smp_store_release(&prep_key->tfm, tfm); + prep_key->tfm = tfm; return 0; } @@ -190,9 +184,37 @@ int fscrypt_set_per_file_enc_key(struct fscrypt_inode_info *ci, return fscrypt_prepare_key(&ci->ci_enc_key, raw_key, ci); } +/* + * Find the fscrypt_prepared_key (if any) for a particular (mk, hkdf_context, + * mode_num, data_unit_bits, inlinecrypt) combination. + * + * The caller must hold ->mk_sem for reading and ->mk_present must be true, + * ensuring that ->mk_mode_keys is still append-only. + */ +static struct fscrypt_prepared_key * +fscrypt_find_mode_key(struct fscrypt_master_key *mk, u8 hkdf_context, + u8 mode_num, const struct fscrypt_inode_info *ci) +{ + struct fscrypt_mode_key *node; + + /* + * The RCU read lock here is used only to synchronize with concurrent + * list_add_tail_rcu(). Concurrent deletions are impossible here, so + * returning a pointer to a node without taking any refcount is safe. + */ + guard(rcu)(); + list_for_each_entry_rcu(node, &mk->mk_mode_keys, link) { + if (node->hkdf_context == hkdf_context && + node->mode_num == mode_num && + node->data_unit_bits == ci->ci_data_unit_bits && + fscrypt_is_key_prepared(&node->key, ci)) + return &node->key; + } + return NULL; +} + static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci, struct fscrypt_master_key *mk, - struct fscrypt_prepared_key *keys, u8 hkdf_context, bool include_fs_uuid) { const struct inode *inode = ci->ci_inode; @@ -200,7 +222,8 @@ static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci, struct fscrypt_mode *mode = ci->ci_mode; const u8 mode_num = mode - fscrypt_modes; struct fscrypt_prepared_key *prep_key; - u8 mode_key[FSCRYPT_MAX_RAW_KEY_SIZE]; + struct fscrypt_mode_key *new_node; + u8 raw_mode_key[FSCRYPT_MAX_RAW_KEY_SIZE]; u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; unsigned int hkdf_infolen = 0; bool use_hw_wrapped_key = false; @@ -223,48 +246,56 @@ static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci, use_hw_wrapped_key = true; } - prep_key = &keys[mode_num]; - if (fscrypt_is_key_prepared(prep_key, ci)) { + prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci); + if (prep_key) { ci->ci_enc_key = *prep_key; return 0; } - mutex_lock(&fscrypt_mode_key_setup_mutex); + guard(mutex)(&fscrypt_mode_key_setup_mutex); - if (fscrypt_is_key_prepared(prep_key, ci)) - goto done_unlock; + prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci); + if (prep_key) { + ci->ci_enc_key = *prep_key; + return 0; + } + + new_node = kzalloc_obj(*new_node); + if (!new_node) + return -ENOMEM; + new_node->hkdf_context = hkdf_context; + new_node->mode_num = mode_num; + new_node->data_unit_bits = ci->ci_data_unit_bits; + prep_key = &new_node->key; if (use_hw_wrapped_key) { err = fscrypt_prepare_inline_crypt_key(prep_key, mk->mk_secret.bytes, mk->mk_secret.size, true, ci); - if (err) - goto out_unlock; - goto done_unlock; + } else { + static_assert(sizeof(mode_num) == 1); + static_assert(sizeof(sb->s_uuid) == 16); + static_assert(sizeof(hkdf_info) == 17); + hkdf_info[hkdf_infolen++] = mode_num; + if (include_fs_uuid) { + memcpy(&hkdf_info[hkdf_infolen], &sb->s_uuid, + sizeof(sb->s_uuid)); + hkdf_infolen += sizeof(sb->s_uuid); + } + fscrypt_hkdf_expand(&mk->mk_secret.hkdf, hkdf_context, + hkdf_info, hkdf_infolen, raw_mode_key, + mode->keysize); + err = fscrypt_prepare_key(prep_key, raw_mode_key, ci); + memzero_explicit(raw_mode_key, mode->keysize); } - - BUILD_BUG_ON(sizeof(mode_num) != 1); - BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); - BUILD_BUG_ON(sizeof(hkdf_info) != 17); - hkdf_info[hkdf_infolen++] = mode_num; - if (include_fs_uuid) { - memcpy(&hkdf_info[hkdf_infolen], &sb->s_uuid, - sizeof(sb->s_uuid)); - hkdf_infolen += sizeof(sb->s_uuid); + if (err) { + kfree(new_node); + return err; } - fscrypt_hkdf_expand(&mk->mk_secret.hkdf, hkdf_context, hkdf_info, - hkdf_infolen, mode_key, mode->keysize); - err = fscrypt_prepare_key(prep_key, mode_key, ci); - memzero_explicit(mode_key, mode->keysize); - if (err) - goto out_unlock; -done_unlock: + list_add_tail_rcu(&new_node->link, &mk->mk_mode_keys); ci->ci_enc_key = *prep_key; - err = 0; -out_unlock: - mutex_unlock(&fscrypt_mode_key_setup_mutex); - return err; + return 0; } /* @@ -311,8 +342,8 @@ static int fscrypt_setup_iv_ino_lblk_32_key(struct fscrypt_inode_info *ci, { int err; - err = setup_per_mode_enc_key(ci, mk, mk->mk_iv_ino_lblk_32_keys, - HKDF_CONTEXT_IV_INO_LBLK_32_KEY, true); + err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_IV_INO_LBLK_32_KEY, + true); if (err) return err; @@ -364,8 +395,8 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_inode_info *ci, * encryption key. This ensures that the master key is * consistently used only for HKDF, avoiding key reuse issues. */ - err = setup_per_mode_enc_key(ci, mk, mk->mk_direct_keys, - HKDF_CONTEXT_DIRECT_KEY, false); + err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_DIRECT_KEY, + false); } else if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { /* @@ -374,9 +405,8 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_inode_info *ci, * the IVs. This format is optimized for use with inline * encryption hardware compliant with the UFS standard. */ - err = setup_per_mode_enc_key(ci, mk, mk->mk_iv_ino_lblk_64_keys, - HKDF_CONTEXT_IV_INO_LBLK_64_KEY, - true); + err = setup_per_mode_enc_key( + ci, mk, HKDF_CONTEXT_IV_INO_LBLK_64_KEY, true); } else if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) { err = fscrypt_setup_iv_ino_lblk_32_key(ci, mk); diff --git a/fs/dcache.c b/fs/dcache.c index 2c61aeea41f4..3e9af9de7074 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -43,8 +43,8 @@ * - i_dentry, d_alias, d_inode of aliases * dcache_hash_bucket lock protects: * - the dcache hash table - * s_roots bl list spinlock protects: - * - the s_roots list (see __d_drop) + * s_roots_lock protects: + * - the s_roots list (see __d_move()/dentry_unlist()/d_obtain_root()) * dentry->d_sb->s_dentry_lru_lock protects: * - the dcache lru lists and counters * d_lock protects: @@ -426,9 +426,16 @@ static inline void __d_clear_type_and_inode(struct dentry *dentry) this_cpu_inc(nr_dentry_negative); } +#define DENTRY_WARN_ONCE(condition, dentry) \ + WARN_ONCE((condition), "dentry=%p d_flags=0x%x\n", (dentry), (dentry)->d_flags) +#define D_FLAG_VERIFY(dentry, x) \ + DENTRY_WARN_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x), (dentry)) + static void dentry_free(struct dentry *dentry) { - WARN_ON(d_really_is_positive(dentry)); + DENTRY_WARN_ONCE(d_really_is_positive(dentry), dentry); + DENTRY_WARN_ONCE(dentry->d_lockref.count >= 0, dentry); + D_FLAG_VERIFY(dentry, 0); if (unlikely(dname_external(dentry))) { struct external_name *p = external_name(dentry); if (likely(atomic_dec_and_test(&p->count))) { @@ -455,14 +462,10 @@ static void dentry_unlink_inode(struct dentry * dentry) raw_write_seqcount_begin(&dentry->d_seq); __d_clear_type_and_inode(dentry); - hlist_del_init(&dentry->d_alias); + __hlist_del(&dentry->d_alias); /* * dentry becomes negative, so the space occupied by ->d_alias - * belongs to ->waiters now; we could use __hlist_del() instead - * of hlist_del_init(), if not for the stunt pulled by nfs - * dummy root dentries - positive dentry *not* included into - * the alias list of its inode. Open-coding hlist_del_init() - * and removing zeroing would be too clumsy... + * belongs to ->waiters now. */ dentry->waiters = NULL; raw_write_seqcount_end(&dentry->d_seq); @@ -495,7 +498,6 @@ static void dentry_unlink_inode(struct dentry * dentry) * These helper functions make sure we always follow the * rules. d_lock must be held by the caller. */ -#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x)) static void d_lru_add(struct dentry *dentry) { D_FLAG_VERIFY(dentry, 0); @@ -562,16 +564,7 @@ static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry, static void ___d_drop(struct dentry *dentry) { - struct hlist_bl_head *b; - /* - * Hashed dentries are normally on the dentry hashtable, - * with the exception of those newly allocated by - * d_obtain_root, which are always IS_ROOT: - */ - if (unlikely(IS_ROOT(dentry))) - b = &dentry->d_sb->s_roots; - else - b = d_hash(dentry->d_name.hash); + struct hlist_bl_head *b = d_hash(dentry->d_name.hash); hlist_bl_lock(b); __hlist_bl_del(&dentry->d_hash); @@ -630,12 +623,14 @@ struct completion_list { * Use ->waiters for a single-linked list of struct completion_list of * waiters. */ -static inline void d_add_waiter(struct dentry *dentry, struct completion_list *p) +static inline bool d_add_waiter(struct dentry *dentry, struct completion_list *p) { - struct completion_list *v = dentry->waiters; + if (unlikely(dentry->d_flags & DCACHE_DENTRY_KILLED)) + return false; init_completion(&p->completion); - p->next = v; + p->next = dentry->waiters; dentry->waiters = p; + return true; } static inline void d_complete_waiters(struct dentry *dentry) @@ -652,6 +647,13 @@ static inline void d_complete_waiters(struct dentry *dentry) } } +static void unlink_secondary_root(struct dentry *dentry) +{ + spin_lock(&dentry->d_sb->s_roots_lock); + hlist_del_init(&dentry->d_sib); + spin_unlock(&dentry->d_sb->s_roots_lock); +} + static inline void dentry_unlist(struct dentry *dentry) { struct dentry *next; @@ -663,6 +665,10 @@ static inline void dentry_unlist(struct dentry *dentry) d_complete_waiters(dentry); if (unlikely(hlist_unhashed(&dentry->d_sib))) return; + if (unlikely(IS_ROOT(dentry))) { + unlink_secondary_root(dentry); // secondary root goes away + return; + } __hlist_del(&dentry->d_sib); /* * Cursors can move around the list of children. While we'd been @@ -691,11 +697,113 @@ static inline void dentry_unlist(struct dentry *dentry) } } -static struct dentry *__dentry_kill(struct dentry *dentry) +/* + * Prepare locking environment for killing a dentry. + * Called under dentry->d_lock. To proceed with eviction of a positive dentry + * we need to get ->i_lock of the inode of that dentry as well. + * However, ->i_lock nests outside of ->d_lock, so if trylock fails we might + * have to drop and regain the latter. Dentry state can change while its + * ->d_lock is not held - it might end up getting killed, becoming busy, + * negative, etc., so we need to be careful. + * + * For NORCU dentries memory safety relies upon having only one call of + * lock_for_kill() in the entire lifetime of dentry and dentry_free() being + * called only by the caller of lock_for_kill(). That this is NORCU-specific; + * the crucial part is that refcounts of NORCU dentries never grow once having + * dropped to zero. + * + * For normal dentries we can not assume that there won't be concurrent calls + * of dentry_free() - dentry might end up being evicted by another thread + * while we are dropping/retaking locks on the slow path. Memory safety is + * provided by keeping the RCU read-side critical area contiguous with + * an explicit rcu_read_lock() scope bridging over the break in spinlock scopes. + * + * If dentry is busy (or busy dying, or already dead), unlock dentry + * and return false. Otherwise, return true and have that dentry's + * inode (if any) locked in addition to dentry itself. + */ +static bool lock_for_kill(struct dentry *dentry) +{ + struct inode *inode = dentry->d_inode; + + if (unlikely(dentry->d_lockref.count)) { + spin_unlock(&dentry->d_lock); + return false; + } + + if (!inode || likely(spin_trylock(&inode->i_lock))) + return true; + + // Too bad - we need to drop ->d_lock and take locks in correct order. + // To avoid breaking RCU read-side critical area when we drop ->d_lock, + // take an explicit rcu_read_lock() while we are switching locks. + rcu_read_lock(); + do { + spin_unlock(&dentry->d_lock); + spin_lock(&inode->i_lock); + spin_lock(&dentry->d_lock); + // make sure we'd locked the right inode - ->d_inode might've + // changed while we were not holding ->d_lock + if (likely(inode == dentry->d_inode)) + break; + spin_unlock(&inode->i_lock); + inode = dentry->d_inode; + } while (inode); + rcu_read_unlock(); + if (likely(!dentry->d_lockref.count)) + return true; + if (inode) + spin_unlock(&inode->i_lock); + spin_unlock(&dentry->d_lock); + return false; +} + +/** + * dentry_kill - evict a dentry + * @dentry: dentry to be evicted + * + * All dentry evictions are done by this function. The reference we are + * passed does not contribute to the refcount; the caller had either + * already decremented the refcount or it had never held one in the + * first place. @dentry->d_lock is held by the caller and dropped + * by dentry_kill(@dentry). + * + * We are guaranteed that nobody had called dentry_free(@dentry) + * prior to the beginning of RCU read-side critical area we are in. + * + * Caller must not access @dentry after the call. + * + * If eviction of @dentry drops the last reference to its parent, + * the reference to parent is returned to caller. In that case + * it is guaranteed to satisfy the requirements for dentry_kill() + * argument - its ->d_lock is held and we are guaranteed that nobody + * had passed it to dentry_free() prior to acquisition of its ->d_lock. + * Otherwise %NULL is returned. + * + * If @dentry is idle and remains such after we assemble the full + * locking environment for eviction (see lock_for_kill() for details) + * we mark it doomed (->d_lockref.count < 0) and proceed to detaching + * it from any filesystem objects. Otherwise we drop ->d_lock and + * return %NULL. + * + * Once @dentry is detached from the filesystem objects, we complete + * detaching it from dentry tree. The parent, if any, gets locked + * and its refcount is decremented; dentry is carefully removed from + * the tree (see dentry_unlist() for details) and marked killed + * (%DCACHE_DENTRY_KILLED set in ->d_flags). At that point it's just + * an inert chunk of memory, accessible only via RCU references + * and possibly via a shrink list. If it is not on any shrink lists, + * we call dentry_free(), which schedules actual freeing of memory. + * Othewise freeing is left to the owner of the shrink list in question. + */ +static struct dentry *dentry_kill(struct dentry *dentry) { struct dentry *parent = NULL; bool can_free = true; + if (unlikely(!lock_for_kill(dentry))) + return NULL; + /* * The dentry is now unrecoverably dead to the world. */ @@ -743,43 +851,6 @@ static struct dentry *__dentry_kill(struct dentry *dentry) } /* - * Lock a dentry for feeding it to __dentry_kill(). - * Called under rcu_read_lock() and dentry->d_lock; the former - * guarantees that nothing we access will be freed under us. - * Note that dentry is *not* protected from concurrent dentry_kill(), - * d_delete(), etc. - * - * Return false if dentry is busy. Otherwise, return true and have - * that dentry's inode locked. - */ - -static bool lock_for_kill(struct dentry *dentry) -{ - struct inode *inode = dentry->d_inode; - - if (unlikely(dentry->d_lockref.count)) - return false; - - if (!inode || likely(spin_trylock(&inode->i_lock))) - return true; - - do { - spin_unlock(&dentry->d_lock); - spin_lock(&inode->i_lock); - spin_lock(&dentry->d_lock); - if (likely(inode == dentry->d_inode)) - break; - spin_unlock(&inode->i_lock); - inode = dentry->d_inode; - } while (inode); - if (likely(!dentry->d_lockref.count)) - return true; - if (inode) - spin_unlock(&inode->i_lock); - return false; -} - -/* * Decide if dentry is worth retaining. Usually this is called with dentry * locked; if not locked, we are more limited and might not be able to tell * without a lock. False in this case means "punt to locked path and recheck". @@ -854,17 +925,17 @@ EXPORT_SYMBOL(d_mark_dontcache); * If unsuccessful, we return false, having already taken the dentry lock. * In that case refcount is guaranteed to be zero and we have already * decided that it's not worth keeping around. - * - * The caller needs to hold the RCU read lock, so that the dentry is - * guaranteed to stay around even if the refcount goes down to zero! */ static inline bool fast_dput(struct dentry *dentry) { int ret; /* - * try to decrement the lockref optimistically. + * Try to decrement the lockref optimistically. + * RCU read lock held so that dentry is guaranteed to stay around + * even if the refcount goes down to zero. */ + rcu_read_lock(); ret = lockref_put_return(&dentry->d_lockref); /* @@ -874,6 +945,7 @@ static inline bool fast_dput(struct dentry *dentry) */ if (unlikely(ret < 0)) { spin_lock(&dentry->d_lock); + rcu_read_unlock(); if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) { spin_unlock(&dentry->d_lock); return true; @@ -885,8 +957,10 @@ static inline bool fast_dput(struct dentry *dentry) /* * If we weren't the last ref, we're done. */ - if (ret) + if (ret) { + rcu_read_unlock(); return true; + } /* * Can we decide that decrement of refcount is all we needed without @@ -894,8 +968,10 @@ static inline bool fast_dput(struct dentry *dentry) * dentry looks like it ought to be retained and there's nothing else * to do. */ - if (retain_dentry(dentry, false)) + if (retain_dentry(dentry, false)) { + rcu_read_unlock(); return true; + } /* * Either not worth retaining or we can't tell without the lock. @@ -903,6 +979,7 @@ static inline bool fast_dput(struct dentry *dentry) * but we'll need to re-check the situation after getting the lock. */ spin_lock(&dentry->d_lock); + rcu_read_unlock(); /* * Did somebody else grab a reference to it in the meantime, and @@ -920,21 +997,13 @@ locked: static void finish_dput(struct dentry *dentry) __releases(dentry->d_lock) - __releases(RCU) { - while (lock_for_kill(dentry)) { - rcu_read_unlock(); - dentry = __dentry_kill(dentry); - if (!dentry) - return; + while ((dentry = dentry_kill(dentry)) != NULL) { if (retain_dentry(dentry, true)) { spin_unlock(&dentry->d_lock); return; } - rcu_read_lock(); } - rcu_read_unlock(); - spin_unlock(&dentry->d_lock); } /* @@ -968,11 +1037,8 @@ void dput(struct dentry *dentry) if (!dentry) return; might_sleep(); - rcu_read_lock(); - if (likely(fast_dput(dentry))) { - rcu_read_unlock(); + if (likely(fast_dput(dentry))) return; - } finish_dput(dentry); } EXPORT_SYMBOL(dput); @@ -983,30 +1049,46 @@ void d_make_discardable(struct dentry *dentry) WARN_ON(!(dentry->d_flags & DCACHE_PERSISTENT)); dentry->d_flags &= ~DCACHE_PERSISTENT; dentry->d_lockref.count--; - rcu_read_lock(); finish_dput(dentry); } EXPORT_SYMBOL(d_make_discardable); -static void to_shrink_list(struct dentry *dentry, struct list_head *list) +/** + * __move_to_shrink_list - try to place a dentry into a shrink list + * @dentry: dentry to try putting into shrink list + * @list: the list to put @dentry into. + * Returns: true @dentry had been placed into @list, false otherwise + * + * If @dentry is idle and not already include into a shrink list, move + * it into @list and return %true; otherwise do nothing and return %false. + * + * Caller must be holding @dentry->d_lock. There must have been no calls of + * dentry_free(@dentry) prior to the beginning of the RCU read-side critical + * area in which __move_to_shrink_list(@dentry, @list) is called. + * + * @list should be thread-private and eventually emptied by passing it to + * shrink_dentry_list(). + */ + +bool __move_to_shrink_list(struct dentry *dentry, struct list_head *list) __must_hold(&dentry->d_lock) { - if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) { + if (likely(!dentry->d_lockref.count && + !(dentry->d_flags & DCACHE_SHRINK_LIST))) { if (dentry->d_flags & DCACHE_LRU_LIST) d_lru_del(dentry); d_shrink_add(dentry, list); + return true; } + return false; } +EXPORT_SYMBOL(__move_to_shrink_list); void dput_to_list(struct dentry *dentry, struct list_head *list) { - rcu_read_lock(); - if (likely(fast_dput(dentry))) { - rcu_read_unlock(); + if (likely(fast_dput(dentry))) return; - } - rcu_read_unlock(); - to_shrink_list(dentry, list); + __move_to_shrink_list(dentry, list); spin_unlock(&dentry->d_lock); } @@ -1052,7 +1134,10 @@ repeat: } EXPORT_SYMBOL(dget_parent); -static struct dentry * __d_find_any_alias(struct inode *inode) +/* + * inode is a directory, inode->i_lock is held by the caller + */ +static struct dentry * __d_find_dir_alias(struct inode *inode) { struct dentry *alias; @@ -1063,6 +1148,18 @@ static struct dentry * __d_find_any_alias(struct inode *inode) return alias; } +static struct dentry * __d_find_any_alias(struct inode *inode) +{ + struct dentry *alias; + + if (hlist_empty(&inode->i_dentry)) + return NULL; + for_each_alias(alias, inode) + if (dget_alias_ilocked(alias)) + return alias; + return NULL; +} + /** * d_find_any_alias - find any alias for a given inode * @inode: inode to find an alias for @@ -1086,7 +1183,7 @@ static struct dentry *__d_find_alias(struct inode *inode) struct dentry *alias; if (S_ISDIR(inode->i_mode)) - return __d_find_any_alias(inode); + return __d_find_dir_alias(inode); for_each_alias(alias, inode) { spin_lock(&alias->d_lock); @@ -1152,25 +1249,6 @@ struct dentry *d_find_alias_rcu(struct inode *inode) return de; } -/** - * d_dispose_if_unused - move unreferenced dentries to shrink list - * @dentry: dentry in question - * @dispose: head of shrink list - * - * If dentry has no external references, move it to shrink list. - * - * NOTE!!! The caller is responsible for preventing eviction of the dentry by - * holding dentry->d_inode->i_lock or equivalent. - */ -void d_dispose_if_unused(struct dentry *dentry, struct list_head *dispose) -{ - spin_lock(&dentry->d_lock); - if (!dentry->d_lockref.count) - to_shrink_list(dentry, dispose); - spin_unlock(&dentry->d_lock); -} -EXPORT_SYMBOL(d_dispose_if_unused); - /* * Try to kill dentries associated with this inode. * WARNING: you must own a reference to inode. @@ -1181,8 +1259,12 @@ void d_prune_aliases(struct inode *inode) struct dentry *dentry; spin_lock(&inode->i_lock); - for_each_alias(dentry, inode) - d_dispose_if_unused(dentry, &dispose); + for_each_alias(dentry, inode) { + spin_lock(&dentry->d_lock); + if (likely(!(dentry->d_flags & DCACHE_NORCU))) + __move_to_shrink_list(dentry, &dispose); + spin_unlock(&dentry->d_lock); + } spin_unlock(&inode->i_lock); shrink_dentry_list(&dispose); } @@ -1190,14 +1272,8 @@ EXPORT_SYMBOL(d_prune_aliases); static inline void shrink_kill(struct dentry *victim) { - do { - rcu_read_unlock(); - victim = __dentry_kill(victim); - rcu_read_lock(); - } while (victim && lock_for_kill(victim)); - rcu_read_unlock(); - if (victim) - spin_unlock(&victim->d_lock); + while ((victim = dentry_kill(victim)) != NULL) + ; } void shrink_dentry_list(struct list_head *list) @@ -1207,18 +1283,12 @@ void shrink_dentry_list(struct list_head *list) dentry = list_entry(list->prev, struct dentry, d_lru); spin_lock(&dentry->d_lock); - rcu_read_lock(); - if (!lock_for_kill(dentry)) { - bool can_free; - rcu_read_unlock(); - d_shrink_del(dentry); - can_free = dentry->d_flags & DCACHE_DENTRY_KILLED; + d_shrink_del(dentry); + if (unlikely(dentry->d_flags & DCACHE_DENTRY_KILLED)) { spin_unlock(&dentry->d_lock); - if (can_free) - dentry_free(dentry); + dentry_free(dentry); continue; } - d_shrink_del(dentry); shrink_kill(dentry); } } @@ -1379,6 +1449,8 @@ again: read_seqbegin_or_lock(&rename_lock, &seq); this_parent = parent; spin_lock(&this_parent->d_lock); + if (unlikely(this_parent->d_flags & DCACHE_DENTRY_CURSOR)) + goto out_unlock; ret = enter(data, this_parent); switch (ret) { @@ -1427,14 +1499,15 @@ resume: /* * All done at this level ... ascend and resume the search. */ - rcu_read_lock(); ascend: if (this_parent != parent) { dentry = this_parent; this_parent = dentry->d_parent; + rcu_read_lock(); spin_unlock(&dentry->d_lock); spin_lock(&this_parent->d_lock); + rcu_read_unlock(); /* might go back up the wrong parent if we have had a rename. */ if (need_seqretry(&rename_lock, seq)) @@ -1442,7 +1515,6 @@ ascend: /* go into the first sibling still alive */ hlist_for_each_entry_continue(dentry, d_sib) { if (likely(!(dentry->d_flags & DCACHE_DENTRY_KILLED))) { - rcu_read_unlock(); goto resume; } } @@ -1450,7 +1522,6 @@ ascend: } if (need_seqretry(&rename_lock, seq)) goto rename_retry; - rcu_read_unlock(); out_unlock: spin_unlock(&this_parent->d_lock); @@ -1459,7 +1530,6 @@ out_unlock: rename_retry: spin_unlock(&this_parent->d_lock); - rcu_read_unlock(); BUG_ON(seq & 1); if (!retry) return; @@ -1574,12 +1644,8 @@ static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) if (data->start == dentry) goto out; - if (dentry->d_flags & DCACHE_SHRINK_LIST) { - data->found++; - } else if (!dentry->d_lockref.count) { - to_shrink_list(dentry, &data->dispose); - data->found++; - } else if (dentry->d_lockref.count < 0) { + if (dentry->d_lockref.count <= 0) { + __move_to_shrink_list(dentry, &data->dispose); data->found++; } /* @@ -1610,17 +1676,21 @@ static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry) if (data->start == dentry) goto out; - if (!dentry->d_lockref.count) { - if (dentry->d_flags & DCACHE_SHRINK_LIST) { + if (dentry->d_lockref.count <= 0) { + if (!__move_to_shrink_list(dentry, &data->dispose)) { + /* + * We need an enter RCU read-side critical area that + * would extend past the return from d_walk() and + * we are in the scope of ->d_lock that will terminate + * before that, so we use rcu_read_lock() to bridge + * over to the scope of ->d_lock in d_walk() caller. + * The scope of rcu_read_lock() spans from here to + * paired rcu_read_unlock() in shrink_dcache_tree(). + */ rcu_read_lock(); data->victim = dentry; return D_WALK_QUIT; } - to_shrink_list(dentry, &data->dispose); - } else if (dentry->d_lockref.count < 0) { - rcu_read_lock(); - data->victim = dentry; - return D_WALK_QUIT; } /* * We can return to the caller if we have found some (this @@ -1643,7 +1713,9 @@ out: static void shrink_dcache_tree(struct dentry *parent, bool for_umount) { for (;;) { - struct select_data data = {.start = parent}; + struct completion_list wait; + bool need_wait = false; + struct select_data data = { .start = parent }; INIT_LIST_HEAD(&data.dispose); d_walk(parent, &data, @@ -1661,30 +1733,32 @@ static void shrink_dcache_tree(struct dentry *parent, bool for_umount) d_walk(parent, &data, select_collect2); if (data.victim) { struct dentry *v = data.victim; - + /* + * select_collect2() has picked a dentry that was + * either dying or on a shrink list and arranged + * for it to be returned to us. We are still in + * the RCU read-side critical area started there + * (rcu_read_lock() scope opened in select_collect2()), + * so dentry couldn't have been freed yet, but its + * state might've changed since we dropped ->d_lock + * on the way out. Switch over to ->d_lock scope + * and recheck the dentry state. + */ spin_lock(&v->d_lock); - if (v->d_lockref.count < 0 && - !(v->d_flags & DCACHE_DENTRY_KILLED)) { - struct completion_list wait; - // It's busy dying; have it notify us once - // it becomes invisible to d_walk(). - d_add_waiter(v, &wait); - spin_unlock(&v->d_lock); - rcu_read_unlock(); - if (!list_empty(&data.dispose)) - shrink_dentry_list(&data.dispose); - wait_for_completion(&wait.completion); - continue; - } - if (!lock_for_kill(v)) { + rcu_read_unlock(); + + if (unlikely(v->d_lockref.count < 0)) { + // It's doomed; if it isn't dead yet, notify us + // once it becomes invisible to d_walk(). + need_wait = d_add_waiter(v, &wait); spin_unlock(&v->d_lock); - rcu_read_unlock(); } else { shrink_kill(v); } } - if (!list_empty(&data.dispose)) - shrink_dentry_list(&data.dispose); + shrink_dentry_list(&data.dispose); + if (unlikely(need_wait)) + wait_for_completion(&wait.completion); } } @@ -1737,9 +1811,30 @@ void shrink_dcache_for_umount(struct super_block *sb) sb->s_root = NULL; do_one_tree(dentry); - while (!hlist_bl_empty(&sb->s_roots)) { - dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash)); - do_one_tree(dentry); + for (;;) { + spin_lock(&sb->s_roots_lock); + dentry = hlist_entry_safe(sb->s_roots.first, + struct dentry, d_sib); + if (!dentry) { + spin_unlock(&sb->s_roots_lock); + break; + } + rcu_read_lock(); + spin_unlock(&sb->s_roots_lock); + spin_lock(&dentry->d_lock); + rcu_read_unlock(); + if (unlikely(dentry->d_lockref.count < 0)) { + struct completion_list wait; + bool need_wait = d_add_waiter(dentry, &wait); + + spin_unlock(&dentry->d_lock); + if (need_wait) + wait_for_completion(&wait.completion); + } else { + dget_dlock(dentry); + spin_unlock(&dentry->d_lock); + do_one_tree(dentry); + } } } @@ -1820,10 +1915,10 @@ static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) name = &slash_name; dname = dentry->d_shortname.string; } else if (name->len > DNAME_INLINE_LEN-1) { - size_t size = offsetof(struct external_name, name[1]); - struct external_name *p = kmalloc(size + name->len, - GFP_KERNEL_ACCOUNT | - __GFP_RECLAIMABLE); + struct external_name *p; + + p = kmalloc_flex(*p, name, name->len + 1, + GFP_KERNEL_ACCOUNT | __GFP_RECLAIMABLE); if (!p) { kmem_cache_free(dentry_cache, dentry); return NULL; @@ -1909,7 +2004,7 @@ struct dentry *d_alloc_cursor(struct dentry * parent) { struct dentry *dentry = d_alloc_anon(parent->d_sb); if (dentry) { - dentry->d_flags |= DCACHE_DENTRY_CURSOR; + dentry->d_flags |= DCACHE_DENTRY_CURSOR | DCACHE_NORCU; dentry->d_parent = dget(parent); } return dentry; @@ -2100,6 +2195,10 @@ void d_instantiate_new(struct dentry *entry, struct inode *inode) __d_instantiate(entry, inode); spin_unlock(&entry->d_lock); WARN_ON(!(inode_state_read(inode) & I_NEW)); + /* + * Paired with igrab_from_hash() + */ + smp_wmb(); inode_state_clear(inode, I_NEW | I_CREATING); inode_wake_up_bit(inode, __I_NEW); spin_unlock(&inode->i_lock); @@ -2156,9 +2255,9 @@ static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected) __d_set_inode_and_type(new, inode, add_flags); hlist_add_head(&new->d_alias, &inode->i_dentry); if (!disconnected) { - hlist_bl_lock(&sb->s_roots); - hlist_bl_add_head(&new->d_hash, &sb->s_roots); - hlist_bl_unlock(&sb->s_roots); + spin_lock(&sb->s_roots_lock); + hlist_add_head(&new->d_sib, &sb->s_roots); + spin_unlock(&sb->s_roots_lock); } spin_unlock(&new->d_lock); spin_unlock(&inode->i_lock); @@ -2250,8 +2349,7 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, return found; } if (d_in_lookup(dentry)) { - found = d_alloc_parallel(dentry->d_parent, name, - dentry->d_wait); + found = d_alloc_parallel(dentry->d_parent, name); if (IS_ERR(found) || !d_in_lookup(found)) { iput(inode); return found; @@ -2638,32 +2736,24 @@ static inline unsigned start_dir_add(struct inode *dir) } } -static inline void end_dir_add(struct inode *dir, unsigned int n, - wait_queue_head_t *d_wait) +static inline void end_dir_add(struct inode *dir, unsigned int n) { smp_store_release(&dir->i_dir_seq, n + 2); preempt_enable_nested(); - if (wq_has_sleeper(d_wait)) - wake_up_all(d_wait); } static void d_wait_lookup(struct dentry *dentry) { - if (d_in_lookup(dentry)) { - DECLARE_WAITQUEUE(wait, current); - add_wait_queue(dentry->d_wait, &wait); - do { - set_current_state(TASK_UNINTERRUPTIBLE); - spin_unlock(&dentry->d_lock); - schedule(); - spin_lock(&dentry->d_lock); - } while (d_in_lookup(dentry)); + if (likely(d_in_lookup(dentry))) { + dentry->d_flags |= DCACHE_LOOKUP_WAITERS; + wait_var_event_spinlock(&dentry->d_flags, + !d_in_lookup(dentry), + &dentry->d_lock); } } struct dentry *d_alloc_parallel(struct dentry *parent, - const struct qstr *name, - wait_queue_head_t *wq) + const struct qstr *name) { unsigned int hash = name->hash; struct hlist_bl_head *b = in_lookup_hash(parent, hash); @@ -2684,38 +2774,33 @@ struct dentry *d_alloc_parallel(struct dentry *parent, spin_unlock(&parent->d_lock); retry: - rcu_read_lock(); seq = smp_load_acquire(&parent->d_inode->i_dir_seq); r_seq = read_seqbegin(&rename_lock); + rcu_read_lock(); dentry = __d_lookup_rcu(parent, name, &d_seq); if (unlikely(dentry)) { if (!lockref_get_not_dead(&dentry->d_lockref)) { rcu_read_unlock(); goto retry; } + rcu_read_unlock(); if (read_seqcount_retry(&dentry->d_seq, d_seq)) { - rcu_read_unlock(); dput(dentry); goto retry; } - rcu_read_unlock(); dput(new); return dentry; } - if (unlikely(read_seqretry(&rename_lock, r_seq))) { - rcu_read_unlock(); + rcu_read_unlock(); + if (unlikely(read_seqretry(&rename_lock, r_seq))) goto retry; - } - if (unlikely(seq & 1)) { - rcu_read_unlock(); + if (unlikely(seq & 1)) goto retry; - } hlist_bl_lock(b); if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) { hlist_bl_unlock(b); - rcu_read_unlock(); goto retry; } /* @@ -2732,19 +2817,20 @@ retry: continue; if (!d_same_name(dentry, parent, name)) continue; + rcu_read_lock(); hlist_bl_unlock(b); + spin_lock(&dentry->d_lock); + rcu_read_unlock(); /* now we can try to grab a reference */ - if (!lockref_get_not_dead(&dentry->d_lockref)) { - rcu_read_unlock(); + if (unlikely(dentry->d_lockref.count < 0)) { + spin_unlock(&dentry->d_lock); goto retry; } - - rcu_read_unlock(); /* * somebody is likely to be still doing lookup for it; - * wait for them to finish + * pin it and wait for them to finish */ - spin_lock(&dentry->d_lock); + dget_dlock(dentry); d_wait_lookup(dentry); /* * it's not in-lookup anymore; in principle we should repeat @@ -2765,8 +2851,6 @@ retry: dput(new); return dentry; } - rcu_read_unlock(); - new->d_wait = wq; hlist_bl_add_head(&new->d_in_lookup_hash, b); hlist_bl_unlock(b); return new; @@ -2778,13 +2862,26 @@ mismatch: EXPORT_SYMBOL(d_alloc_parallel); /* - * - Unhash the dentry - * - Retrieve and clear the waitqueue head in dentry - * - Return the waitqueue head + * Move dentry from in-lookup state to busy-negative one. + * + * From now on d_in_lookup(dentry) will return false and dentry is gone from + * in-lookup hash. + * + * Anyone who had been waiting on it in d_alloc_parallel() is free to + * proceed after that. Note that waking such waiters up is left to + * the callers; PREEMPT_RT kernels can't have that wakeup done while + * in write-side critical area for ->i_dir_seq, so it's done by calling + * __d_wake_in_lookup_waiters() once it's safe to do so. + * + * Both __d_lookup_unhash() and __d_wake_in_lookup_waiters() should + * be called within the same ->d_lock scope. PAR_LOOKUP is cleared + * here, while LOOKUP_WAITERS (set by somebody finding dentry in + * the in-lookup hash and setting down to wait) is checked and cleared + * in __d_wake_in_lookup_waiters(). Both are gone by the end of + * ->d_lock scope. */ -static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry) +static void __d_lookup_unhash(struct dentry *dentry) { - wait_queue_head_t *d_wait; struct hlist_bl_head *b; lockdep_assert_held(&dentry->d_lock); @@ -2793,18 +2890,23 @@ static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry) hlist_bl_lock(b); dentry->d_flags &= ~DCACHE_PAR_LOOKUP; __hlist_bl_del(&dentry->d_in_lookup_hash); - d_wait = dentry->d_wait; - dentry->d_wait = NULL; hlist_bl_unlock(b); dentry->waiters = NULL; - INIT_LIST_HEAD(&dentry->d_lru); - return d_wait; +} + +static inline void __d_wake_in_lookup_waiters(struct dentry *dentry) +{ + if (dentry->d_flags & DCACHE_LOOKUP_WAITERS) { + wake_up_var_locked(&dentry->d_flags, &dentry->d_lock); + dentry->d_flags &= ~DCACHE_LOOKUP_WAITERS; + } } void __d_lookup_unhash_wake(struct dentry *dentry) { spin_lock(&dentry->d_lock); - wake_up_all(__d_lookup_unhash(dentry)); + __d_lookup_unhash(dentry); + __d_wake_in_lookup_waiters(dentry); spin_unlock(&dentry->d_lock); } EXPORT_SYMBOL(__d_lookup_unhash_wake); @@ -2814,14 +2916,13 @@ EXPORT_SYMBOL(__d_lookup_unhash_wake); static inline void __d_add(struct dentry *dentry, struct inode *inode, const struct dentry_operations *ops) { - wait_queue_head_t *d_wait; struct inode *dir = NULL; unsigned n; spin_lock(&dentry->d_lock); if (unlikely(d_in_lookup(dentry))) { dir = dentry->d_parent->d_inode; n = start_dir_add(dir); - d_wait = __d_lookup_unhash(dentry); + __d_lookup_unhash(dentry); } if (unlikely(ops)) d_set_d_op(dentry, ops); @@ -2834,8 +2935,10 @@ static inline void __d_add(struct dentry *dentry, struct inode *inode, fsnotify_update_flags(dentry); } __d_rehash(dentry); - if (dir) - end_dir_add(dir, n, d_wait); + if (dir) { + end_dir_add(dir, n); + __d_wake_in_lookup_waiters(dentry); + } spin_unlock(&dentry->d_lock); if (inode) spin_unlock(&inode->i_lock); @@ -2948,7 +3051,6 @@ static void __d_move(struct dentry *dentry, struct dentry *target, bool exchange) { struct dentry *old_parent, *p; - wait_queue_head_t *d_wait; struct inode *dir = NULL; unsigned n; @@ -2979,7 +3081,7 @@ static void __d_move(struct dentry *dentry, struct dentry *target, if (unlikely(d_in_lookup(target))) { dir = target->d_parent->d_inode; n = start_dir_add(dir); - d_wait = __d_lookup_unhash(target); + __d_lookup_unhash(target); } write_seqcount_begin(&dentry->d_seq); @@ -3018,9 +3120,10 @@ static void __d_move(struct dentry *dentry, struct dentry *target, write_seqcount_end(&target->d_seq); write_seqcount_end(&dentry->d_seq); - if (dir) - end_dir_add(dir, n, d_wait); - + if (dir) { + end_dir_add(dir, n); + __d_wake_in_lookup_waiters(target); + } if (dentry->d_parent != old_parent) spin_unlock(&dentry->d_parent->d_lock); if (dentry != old_parent) @@ -3141,7 +3244,7 @@ struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry, security_d_instantiate(dentry, inode); spin_lock(&inode->i_lock); if (S_ISDIR(inode->i_mode)) { - struct dentry *new = __d_find_any_alias(inode); + struct dentry *new = __d_find_dir_alias(inode); if (unlikely(new)) { /* The reference to new ensures it remains an alias */ spin_unlock(&inode->i_lock); @@ -3166,6 +3269,12 @@ struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry, } dput(old_parent); } else { + if (unlikely(!hlist_unhashed(&new->d_sib))) { + // secondary root getting spliced + spin_lock(&new->d_lock); + unlink_secondary_root(new); + spin_unlock(&new->d_lock); + } __d_move(new, dentry, false); write_sequnlock(&rename_lock); } diff --git a/fs/direct-io.c b/fs/direct-io.c index 2267f5ae7f77..9e4976716985 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -502,12 +502,8 @@ static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio) const enum req_op dio_op = dio->opf & REQ_OP_MASK; bool should_dirty = dio_op == REQ_OP_READ && dio->should_dirty; - if (err) { - if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT)) - dio->io_error = -EAGAIN; - else - dio->io_error = -EIO; - } + if (err) + dio->io_error = -EIO; if (dio->is_async && should_dirty) { bio_check_pages_dirty(bio); /* transfers ownership */ @@ -1178,13 +1174,10 @@ ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, dio->is_async = true; dio->inode = inode; - if (iov_iter_rw(iter) == WRITE) { + if (iov_iter_rw(iter) == WRITE) dio->opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; - if (iocb->ki_flags & IOCB_NOWAIT) - dio->opf |= REQ_NOWAIT; - } else { + else dio->opf = REQ_OP_READ; - } /* * For AIO O_(D)SYNC writes we need to defer completions to a workqueue diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index 5b6142787919..2aff1c7c17de 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -271,7 +271,8 @@ static struct connection *__find_con(int nodeid, int r) { struct connection *con; - hlist_for_each_entry_rcu(con, &connection_hash[r], list) { + hlist_for_each_entry_srcu(con, &connection_hash[r], list, + srcu_read_lock_held(&connections_srcu)) { if (con->nodeid == nodeid) return con; } @@ -426,7 +427,8 @@ static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid, idx = srcu_read_lock(&connections_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(con, &connection_hash[i], list) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { WARN_ON_ONCE(!con->addr_count); spin_lock(&con->addrs_lock); @@ -1729,7 +1731,8 @@ void dlm_lowcomms_shutdown(void) idx = srcu_read_lock(&connections_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(con, &connection_hash[i], list) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { shutdown_connection(con, true); stop_connection_io(con); flush_workqueue(process_workqueue); @@ -1968,7 +1971,8 @@ void dlm_lowcomms_exit(void) idx = srcu_read_lock(&connections_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(con, &connection_hash[i], list) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { spin_lock(&connections_lock); hlist_del_rcu(&con->list); spin_unlock(&connections_lock); diff --git a/fs/dlm/memory.c b/fs/dlm/memory.c index 5c35cc67aca4..ee55994ce90d 100644 --- a/fs/dlm/memory.c +++ b/fs/dlm/memory.c @@ -48,8 +48,10 @@ int __init dlm_memory_init(void) if (!rsb_cache) goto rsb; - cb_cache = kmem_cache_create("dlm_cb", sizeof(struct dlm_callback), + cb_cache = kmem_cache_create_usercopy("dlm_cb", sizeof(struct dlm_callback), __alignof__(struct dlm_callback), 0, + offsetof(struct dlm_callback, lvbptr), + sizeof_field(struct dlm_callback, lvbptr), NULL); if (!cb_cache) goto cb; diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index d54bdd8fc4f2..8964164600d2 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -275,7 +275,8 @@ static struct midcomms_node *__find_node(int nodeid, int r) { struct midcomms_node *node; - hlist_for_each_entry_rcu(node, &node_hash[r], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[r], hlist, + srcu_read_lock_held(&nodes_srcu)) { if (node->nodeid == nodeid) return node; } @@ -355,6 +356,7 @@ int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr) if (!node) return -ENOMEM; + node->debugfs = dlm_create_debug_comms_file(nodeid, node); node->nodeid = nodeid; spin_lock_init(&node->state_lock); spin_lock_init(&node->send_queue_lock); @@ -368,7 +370,6 @@ int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr) hlist_add_head_rcu(&node->hlist, &node_hash[r]); spin_unlock_bh(&nodes_lock); - node->debugfs = dlm_create_debug_comms_file(nodeid, node); return 0; } @@ -968,10 +969,10 @@ static void midcomms_new_msg_cb(void *data) atomic_inc(&mh->node->send_queue_cnt); spin_lock_bh(&mh->node->send_queue_lock); + /* need to be locked with list_add_tail_rcu() because list is ordered */ + mh->seq = atomic_fetch_inc(&mh->node->seq_send); list_add_tail_rcu(&mh->list, &mh->node->send_queue); spin_unlock_bh(&mh->node->send_queue_lock); - - mh->seq = atomic_fetch_inc(&mh->node->seq_send); } static struct dlm_msg *dlm_midcomms_get_msg_3_2(struct dlm_mhandle *mh, int nodeid, @@ -1165,7 +1166,8 @@ void dlm_midcomms_exit(void) idx = srcu_read_lock(&nodes_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { dlm_delete_debug_comms_file(node->debugfs); spin_lock(&nodes_lock); @@ -1325,7 +1327,8 @@ void dlm_midcomms_version_wait(void) idx = srcu_read_lock(&nodes_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { ret = wait_event_timeout(node->shutdown_wait, node->version != DLM_VERSION_NOT_SET || node->state == DLM_CLOSED || @@ -1396,7 +1399,8 @@ void dlm_midcomms_shutdown(void) mutex_lock(&close_lock); idx = srcu_read_lock(&nodes_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { midcomms_shutdown(node); } } @@ -1404,7 +1408,8 @@ void dlm_midcomms_shutdown(void) dlm_lowcomms_shutdown(); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { midcomms_node_reset(node); } } diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index 5ef67b2674ee..74b02b55e3f6 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -49,25 +49,15 @@ void ecryptfs_from_hex(char *dst, char *src, int dst_size) } static int ecryptfs_crypto_api_algify_cipher_name(char **algified_name, - char *cipher_name, - char *chaining_modifier) + const char *cipher_name, + const char *chaining_modifier) { - int cipher_name_len = strlen(cipher_name); - int chaining_modifier_len = strlen(chaining_modifier); - int algified_name_len; - int rc; + (*algified_name) = kasprintf(GFP_KERNEL, "%s(%s)", chaining_modifier, + cipher_name); + if (!(*algified_name)) + return -ENOMEM; - algified_name_len = (chaining_modifier_len + cipher_name_len + 3); - (*algified_name) = kmalloc(algified_name_len, GFP_KERNEL); - if (!(*algified_name)) { - rc = -ENOMEM; - goto out; - } - snprintf((*algified_name), algified_name_len, "%s(%s)", - chaining_modifier, cipher_name); - rc = 0; -out: - return rc; + return 0; } /** diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index 546c1fe692c0..7aaf1913f9c6 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -350,11 +350,9 @@ static struct dentry *ecryptfs_lookup_interpose(struct dentry *dentry, */ lower_inode = READ_ONCE(lower_dentry->d_inode); - if (!lower_inode) { - /* We want to add because we couldn't find in lower */ - d_add(dentry, NULL); - return NULL; - } + if (!lower_inode) /* We want to add because we couldn't find in lower */ + return d_splice_alias(NULL, dentry); + inode = __ecryptfs_get_inode(lower_inode, dentry->d_sb); if (IS_ERR(inode)) { printk(KERN_ERR "%s: Error interposing; rc = [%ld]\n", diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c index 0be746493e56..ebebc9551f1f 100644 --- a/fs/ecryptfs/keystore.c +++ b/fs/ecryptfs/keystore.c @@ -1718,7 +1718,6 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, struct dentry *ecryptfs_dentry) { size_t i = 0; - size_t found_auth_tok; size_t next_packet_is_auth_tok_packet; LIST_HEAD(auth_tok_list); struct ecryptfs_auth_tok *matching_auth_tok; @@ -1822,7 +1821,6 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, * the metadata. There may be several potential matches, but * just one will be sufficient to decrypt to get the FEK. */ find_next_matching_auth_tok: - found_auth_tok = 0; list_for_each_entry(auth_tok_list_item, &auth_tok_list, list) { candidate_auth_tok = &auth_tok_list_item->auth_tok; if (unlikely(ecryptfs_verbosity > 0)) { @@ -1843,17 +1841,13 @@ find_next_matching_auth_tok: &matching_auth_tok, crypt_stat->mount_crypt_stat, candidate_auth_tok_sig); - if (!rc) { - found_auth_tok = 1; + if (!rc) goto found_matching_auth_tok; - } - } - if (!found_auth_tok) { - ecryptfs_printk(KERN_ERR, "Could not find a usable " - "authentication token\n"); - rc = -EIO; - goto out_wipe_list; } + ecryptfs_printk(KERN_ERR, + "Could not find a usable authentication token\n"); + rc = -EIO; + goto out_wipe_list; found_matching_auth_tok: if (candidate_auth_tok->token_type == ECRYPTFS_PRIVATE_KEY) { memcpy(&(candidate_auth_tok->token.private_key), diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c index 1c5224cf183e..733c19571f1c 100644 --- a/fs/efivarfs/super.c +++ b/fs/efivarfs/super.c @@ -191,13 +191,10 @@ static const struct dentry_operations efivarfs_d_ops = { static struct dentry *efivarfs_alloc_dentry(struct dentry *parent, char *name) { + struct qstr q = QSTR(name); struct dentry *d; - struct qstr q; int err; - q.name = name; - q.len = strlen(name); - err = efivarfs_d_hash(parent, &q); if (err) return ERR_PTR(err); diff --git a/fs/efs/file.c b/fs/efs/file.c index 9e641da6fab2..9153dfe79bbc 100644 --- a/fs/efs/file.c +++ b/fs/efs/file.c @@ -18,16 +18,9 @@ int efs_get_block(struct inode *inode, sector_t iblock, if (create) return error; - if (iblock >= inode->i_blocks) { -#ifdef DEBUG - /* - * i have no idea why this happens as often as it does - */ - pr_warn("%s(): block %d >= %ld (filesize %ld)\n", - __func__, block, inode->i_blocks, inode->i_size); -#endif + if (iblock >= inode->i_blocks) return 0; - } + phys = efs_map_block(inode, iblock); if (phys) map_bh(bh_result, inode->i_sb, phys); @@ -42,16 +35,8 @@ int efs_bmap(struct inode *inode, efs_block_t block) { } /* are we about to read past the end of a file ? */ - if (!(block < inode->i_blocks)) { -#ifdef DEBUG - /* - * i have no idea why this happens as often as it does - */ - pr_warn("%s(): block %d >= %ld (filesize %ld)\n", - __func__, block, inode->i_blocks, inode->i_size); -#endif + if (!(block < inode->i_blocks)) return 0; - } return efs_map_block(inode, block); } diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig index 97c48ebe8458..4789b1077d8c 100644 --- a/fs/erofs/Kconfig +++ b/fs/erofs/Kconfig @@ -3,13 +3,11 @@ config EROFS_FS tristate "EROFS filesystem support" depends on BLOCK - select CACHEFILES if EROFS_FS_ONDEMAND select CRC32 select CRYPTO if EROFS_FS_ZIP_ACCEL select CRYPTO_DEFLATE if EROFS_FS_ZIP_ACCEL select FS_IOMAP select LZ4_DECOMPRESS if EROFS_FS_ZIP - select NETFS_SUPPORT if EROFS_FS_ONDEMAND select XXHASH if EROFS_FS_XATTR select XZ_DEC if EROFS_FS_ZIP_LZMA select XZ_DEC_MICROLZMA if EROFS_FS_ZIP_LZMA @@ -109,9 +107,6 @@ config EROFS_FS_BACKED_BY_FILE be used to simplify error-prone lifetime management of unnecessary virtual block devices. - Note that this feature, along with ongoing fanotify pre-content - hooks, will eventually replace "EROFS over fscache." - If you don't want to enable this feature, say N. config EROFS_FS_ZIP @@ -172,20 +167,6 @@ config EROFS_FS_ZIP_ACCEL If unsure, say N. -config EROFS_FS_ONDEMAND - bool "EROFS fscache-based on-demand read support (deprecated)" - depends on EROFS_FS - select FSCACHE - select CACHEFILES_ONDEMAND - help - This permits EROFS to use fscache-backed data blobs with on-demand - read support. - - It is now deprecated and scheduled to be removed from the kernel - after fanotify pre-content hooks are landed. - - If unsure, say N. - config EROFS_FS_PCPU_KTHREAD bool "EROFS per-cpu decompression kthread workers" depends on EROFS_FS_ZIP @@ -207,7 +188,7 @@ config EROFS_FS_PCPU_KTHREAD_HIPRI config EROFS_FS_PAGE_CACHE_SHARE bool "EROFS page cache share support (experimental)" - depends on EROFS_FS && EROFS_FS_XATTR && !EROFS_FS_ONDEMAND + depends on EROFS_FS && EROFS_FS_XATTR help This enables page cache sharing among inodes with identical content fingerprints on the same machine. diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile index a80e1762b607..30423496786f 100644 --- a/fs/erofs/Makefile +++ b/fs/erofs/Makefile @@ -9,5 +9,4 @@ erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_ZIP_ACCEL) += decompressor_crypto.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o -erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o erofs-$(CONFIG_EROFS_FS_PAGE_CACHE_SHARE) += ishare.o diff --git a/fs/erofs/data.c b/fs/erofs/data.c index 44da21c9d777..9aa48c8d67d1 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -80,9 +80,7 @@ int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, if (erofs_is_fileio_mode(sbi)) { buf->file = sbi->dif0.file; /* some fs like FUSE needs it */ buf->mapping = buf->file->f_mapping; - } else if (erofs_is_fscache_mode(sb)) - buf->mapping = sbi->dif0.fscache->inode->i_mapping; - else + } else buf->mapping = sb->s_bdev->bd_mapping; return 0; } @@ -98,17 +96,73 @@ void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb, return erofs_bread(buf, offset, true); } -int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map) +static int erofs_map_chunks(struct inode *inode, struct erofs_map_blocks *map) { struct erofs_buf buf = __EROFS_BUF_INITIALIZER; struct super_block *sb = inode->i_sb; - unsigned int unit, blksz = sb->s_blocksize; struct erofs_inode *vi = EROFS_I(inode); struct erofs_inode_chunk_index *idx; - erofs_blk_t startblk, addrmask; - bool tailpacking; + unsigned int unit = vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES ? + sizeof(*idx) : EROFS_BLOCK_MAP_ENTRY_SIZE; + erofs_blk_t addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) ? + BIT_ULL(48) - 1 : BIT_ULL(32) - 1; + u64 nr = map->m_la >> vi->chunkbits, chunksize = 1ULL << vi->chunkbits; + erofs_off_t pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + + vi->xattr_isize, unit) + unit * nr; + /* m_llen will be clamped to EOF in the end */ + erofs_off_t endpos = round_up(pos + 1, sb->s_blocksize); + u64 last, addr; + + idx = erofs_read_metabuf(&buf, sb, pos, erofs_inode_in_metabox(inode)); + if (IS_ERR(idx)) + return PTR_ERR(idx); + + map->m_la = nr << vi->chunkbits; + map->m_llen = 0; + nr = 0; + do { + if (unit == EROFS_BLOCK_MAP_ENTRY_SIZE) { + addr = le32_to_cpu(((__le32 *)idx)[nr]); + if (addr == (u32)EROFS_NULL_ADDR) + addr = EROFS_NULL_ADDR; + } else { + addr = (((u64)le16_to_cpu(idx[nr].startblk_hi) << 32) | + le32_to_cpu(idx[nr].startblk_lo)) & addrmask; + if (addr ^ (EROFS_NULL_ADDR & addrmask)) + addr |= (u64)(le16_to_cpu(idx[nr].device_id) & + EROFS_SB(sb)->device_id_mask) << 48; + else + addr = EROFS_NULL_ADDR; + } + if (!nr) { + last = addr; + continue; + } + /* expand and account the prior chunk here */ + map->m_llen += chunksize; + if (last != EROFS_NULL_ADDR) + last += erofs_blknr(sb, chunksize); + } while (addr == last && pos + (++nr) * unit < endpos); + + if (last != EROFS_NULL_ADDR) { + map->m_pa = erofs_pos(sb, last & addrmask) - map->m_llen; + map->m_deviceid = last >> 48; + map->m_flags = EROFS_MAP_MAPPED; + } + if (addr == last) + map->m_llen += chunksize; + map->m_llen = min_t(erofs_off_t, map->m_llen, + round_up(inode->i_size - map->m_la, sb->s_blocksize)); + erofs_put_metabuf(&buf); + return 0; +} + +int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map) +{ + struct super_block *sb = inode->i_sb; + struct erofs_inode *vi = EROFS_I(inode); + bool tailinline = (vi->datalayout == EROFS_INODE_FLAT_INLINE); erofs_off_t pos; - u64 chunknr; int err = 0; trace_erofs_map_blocks_enter(inode, map, 0); @@ -116,13 +170,10 @@ int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map) map->m_flags = 0; if (map->m_la >= inode->i_size) goto out; - - if (vi->datalayout != EROFS_INODE_CHUNK_BASED) { - tailpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE); - if (!tailpacking && vi->startblk == EROFS_NULL_ADDR) - goto out; - pos = erofs_pos(sb, erofs_iblks(inode) - tailpacking); - + if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + err = erofs_map_chunks(inode, map); + } else if (tailinline || vi->startblk != EROFS_NULL_ADDR) { + pos = erofs_pos(sb, erofs_iblks(inode) - tailinline); map->m_flags = EROFS_MAP_MAPPED; if (map->m_la < pos) { map->m_pa = erofs_pos(sb, vi->startblk) + map->m_la; @@ -132,57 +183,15 @@ int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map) vi->xattr_isize + erofs_blkoff(sb, map->m_la); map->m_llen = inode->i_size - map->m_la; map->m_flags |= EROFS_MAP_META; - } - goto out; - } - - if (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) - unit = sizeof(*idx); /* chunk index */ - else - unit = EROFS_BLOCK_MAP_ENTRY_SIZE; /* block map */ - - chunknr = map->m_la >> vi->chunkbits; - pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + - vi->xattr_isize, unit) + unit * chunknr; - - idx = erofs_read_metabuf(&buf, sb, pos, erofs_inode_in_metabox(inode)); - if (IS_ERR(idx)) { - err = PTR_ERR(idx); - goto out; - } - map->m_la = chunknr << vi->chunkbits; - map->m_llen = min_t(erofs_off_t, 1UL << vi->chunkbits, - round_up(inode->i_size - map->m_la, blksz)); - if (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) { - addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) ? - BIT_ULL(48) - 1 : BIT_ULL(32) - 1; - startblk = (((u64)le16_to_cpu(idx->startblk_hi) << 32) | - le32_to_cpu(idx->startblk_lo)) & addrmask; - if ((startblk ^ EROFS_NULL_ADDR) & addrmask) { - map->m_deviceid = le16_to_cpu(idx->device_id) & - EROFS_SB(sb)->device_id_mask; - map->m_pa = erofs_pos(sb, startblk); - map->m_flags = EROFS_MAP_MAPPED; - } - } else { - startblk = le32_to_cpu(*(__le32 *)idx); - if (startblk != (u32)EROFS_NULL_ADDR) { - map->m_pa = erofs_pos(sb, startblk); - map->m_flags = EROFS_MAP_MAPPED; + if (erofs_blkoff(sb, map->m_pa) + map->m_llen > + sb->s_blocksize) { + erofs_err(sb, "inline data across blocks @ nid %llu", vi->nid); + return -EFSCORRUPTED; + } } } - erofs_put_metabuf(&buf); out: - if (!err) { - map->m_plen = map->m_llen; - /* inline data should be located in the same meta block */ - if ((map->m_flags & EROFS_MAP_META) && - erofs_blkoff(sb, map->m_pa) + map->m_plen > blksz) { - erofs_err(sb, "inline data across blocks @ nid %llu", vi->nid); - DBG_BUGON(1); - return -EFSCORRUPTED; - } - } + map->m_plen = err ? 0 : map->m_llen; trace_erofs_map_blocks_exit(inode, map, 0, err); return err; } diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index 7871b16c1d33..16ec4fd33ac6 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -396,6 +396,8 @@ enum { /* (noncompact only, HEAD) This pcluster refers to partial decompressed data */ #define Z_EROFS_LI_PARTIAL_REF (1 << 15) +/* (noncompact only, HEAD) This pcluster can also be regarded as a HOLE */ +#define Z_EROFS_LI_HOLE (1 << 14) /* Set on 1st non-head lcluster to store compressed block counti (in blocks) */ #define Z_EROFS_LI_D0_CBLKCNT (1 << 11) diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c deleted file mode 100644 index 685c68774379..000000000000 --- a/fs/erofs/fscache.c +++ /dev/null @@ -1,664 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2022, Alibaba Cloud - * Copyright (C) 2022, Bytedance Inc. All rights reserved. - */ -#include <linux/fscache.h> -#include "internal.h" - -static DEFINE_MUTEX(erofs_domain_list_lock); -static DEFINE_MUTEX(erofs_domain_cookies_lock); -static LIST_HEAD(erofs_domain_list); -static LIST_HEAD(erofs_domain_cookies_list); -static struct vfsmount *erofs_pseudo_mnt; - -struct erofs_fscache_io { - struct netfs_cache_resources cres; - struct iov_iter iter; - netfs_io_terminated_t end_io; - void *private; - refcount_t ref; -}; - -struct erofs_fscache_rq { - struct address_space *mapping; /* The mapping being accessed */ - loff_t start; /* Start position */ - size_t len; /* Length of the request */ - size_t submitted; /* Length of submitted */ - short error; /* 0 or error that occurred */ - refcount_t ref; -}; - -static bool erofs_fscache_io_put(struct erofs_fscache_io *io) -{ - if (!refcount_dec_and_test(&io->ref)) - return false; - if (io->cres.ops) - io->cres.ops->end_operation(&io->cres); - kfree(io); - return true; -} - -static void erofs_fscache_req_complete(struct erofs_fscache_rq *req) -{ - struct folio *folio; - bool failed = req->error; - pgoff_t start_page = req->start / PAGE_SIZE; - pgoff_t last_page = ((req->start + req->len) / PAGE_SIZE) - 1; - - XA_STATE(xas, &req->mapping->i_pages, start_page); - - rcu_read_lock(); - xas_for_each(&xas, folio, last_page) { - if (xas_retry(&xas, folio)) - continue; - if (!failed) - folio_mark_uptodate(folio); - folio_unlock(folio); - } - rcu_read_unlock(); -} - -static void erofs_fscache_req_put(struct erofs_fscache_rq *req) -{ - if (!refcount_dec_and_test(&req->ref)) - return; - erofs_fscache_req_complete(req); - kfree(req); -} - -static struct erofs_fscache_rq *erofs_fscache_req_alloc(struct address_space *mapping, - loff_t start, size_t len) -{ - struct erofs_fscache_rq *req = kzalloc_obj(*req); - - if (!req) - return NULL; - req->mapping = mapping; - req->start = start; - req->len = len; - refcount_set(&req->ref, 1); - return req; -} - -static void erofs_fscache_req_io_put(struct erofs_fscache_io *io) -{ - struct erofs_fscache_rq *req = io->private; - - if (erofs_fscache_io_put(io)) - erofs_fscache_req_put(req); -} - -static void erofs_fscache_req_end_io(void *priv, ssize_t transferred_or_error) -{ - struct erofs_fscache_io *io = priv; - struct erofs_fscache_rq *req = io->private; - - if (IS_ERR_VALUE(transferred_or_error)) - req->error = transferred_or_error; - erofs_fscache_req_io_put(io); -} - -static struct erofs_fscache_io *erofs_fscache_req_io_alloc(struct erofs_fscache_rq *req) -{ - struct erofs_fscache_io *io = kzalloc_obj(*io); - - if (!io) - return NULL; - io->end_io = erofs_fscache_req_end_io; - io->private = req; - refcount_inc(&req->ref); - refcount_set(&io->ref, 1); - return io; -} - -/* - * Read data from fscache described by cookie at pstart physical address - * offset, and fill the read data into buffer described by io->iter. - */ -static int erofs_fscache_read_io_async(struct fscache_cookie *cookie, - loff_t pstart, struct erofs_fscache_io *io) -{ - enum netfs_io_source source; - struct netfs_cache_resources *cres = &io->cres; - struct iov_iter *iter = &io->iter; - int ret; - - ret = fscache_begin_read_operation(cres, cookie); - if (ret) - return ret; - - while (iov_iter_count(iter)) { - size_t orig_count = iov_iter_count(iter), len = orig_count; - unsigned long flags = 1 << NETFS_SREQ_ONDEMAND; - - source = cres->ops->prepare_ondemand_read(cres, - pstart, &len, LLONG_MAX, &flags, 0); - if (WARN_ON(len == 0)) - source = NETFS_INVALID_READ; - if (source != NETFS_READ_FROM_CACHE) { - erofs_err(NULL, "prepare_ondemand_read failed (source %d)", source); - return -EIO; - } - - iov_iter_truncate(iter, len); - refcount_inc(&io->ref); - ret = fscache_read(cres, pstart, iter, NETFS_READ_HOLE_FAIL, - io->end_io, io); - if (ret == -EIOCBQUEUED) - ret = 0; - if (ret) { - erofs_err(NULL, "fscache_read failed (ret %d)", ret); - return ret; - } - if (WARN_ON(iov_iter_count(iter))) - return -EIO; - - iov_iter_reexpand(iter, orig_count - len); - pstart += len; - } - return 0; -} - -struct erofs_fscache_bio { - struct erofs_fscache_io io; - struct bio bio; /* w/o bdev to share bio_add_page/endio() */ - struct bio_vec bvecs[BIO_MAX_VECS]; -}; - -static void erofs_fscache_bio_endio(void *priv, ssize_t transferred_or_error) -{ - struct erofs_fscache_bio *io = priv; - - if (IS_ERR_VALUE(transferred_or_error)) - io->bio.bi_status = errno_to_blk_status(transferred_or_error); - bio_endio(&io->bio); - BUILD_BUG_ON(offsetof(struct erofs_fscache_bio, io) != 0); - erofs_fscache_io_put(&io->io); -} - -struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev) -{ - struct erofs_fscache_bio *io; - - io = kmalloc_obj(*io, GFP_KERNEL | __GFP_NOFAIL); - bio_init(&io->bio, NULL, io->bvecs, BIO_MAX_VECS, REQ_OP_READ); - io->io.private = mdev->m_dif->fscache->cookie; - io->io.end_io = erofs_fscache_bio_endio; - refcount_set(&io->io.ref, 1); - return &io->bio; -} - -void erofs_fscache_submit_bio(struct bio *bio) -{ - struct erofs_fscache_bio *io = container_of(bio, - struct erofs_fscache_bio, bio); - int ret; - - iov_iter_bvec(&io->io.iter, ITER_DEST, io->bvecs, bio->bi_vcnt, - bio->bi_iter.bi_size); - ret = erofs_fscache_read_io_async(io->io.private, - bio->bi_iter.bi_sector << 9, &io->io); - erofs_fscache_io_put(&io->io); - if (!ret) - return; - bio->bi_status = errno_to_blk_status(ret); - bio_endio(bio); -} - -static int erofs_fscache_meta_read_folio(struct file *data, struct folio *folio) -{ - struct erofs_fscache *ctx = folio->mapping->host->i_private; - int ret = -ENOMEM; - struct erofs_fscache_rq *req; - struct erofs_fscache_io *io; - - req = erofs_fscache_req_alloc(folio->mapping, - folio_pos(folio), folio_size(folio)); - if (!req) { - folio_unlock(folio); - return ret; - } - - io = erofs_fscache_req_io_alloc(req); - if (!io) { - req->error = ret; - goto out; - } - iov_iter_xarray(&io->iter, ITER_DEST, &folio->mapping->i_pages, - folio_pos(folio), folio_size(folio)); - - ret = erofs_fscache_read_io_async(ctx->cookie, folio_pos(folio), io); - if (ret) - req->error = ret; - - erofs_fscache_req_io_put(io); -out: - erofs_fscache_req_put(req); - return ret; -} - -static int erofs_fscache_data_read_slice(struct erofs_fscache_rq *req) -{ - struct address_space *mapping = req->mapping; - struct inode *inode = mapping->host; - struct super_block *sb = inode->i_sb; - struct erofs_fscache_io *io; - struct erofs_map_blocks map; - struct erofs_map_dev mdev; - loff_t pos = req->start + req->submitted; - size_t count; - int ret; - - map.m_la = pos; - ret = erofs_map_blocks(inode, &map); - if (ret) - return ret; - - if (map.m_flags & EROFS_MAP_META) { - struct erofs_buf buf = __EROFS_BUF_INITIALIZER; - struct iov_iter iter; - size_t size = map.m_llen; - void *src; - - src = erofs_read_metabuf(&buf, sb, map.m_pa, - erofs_inode_in_metabox(inode)); - if (IS_ERR(src)) - return PTR_ERR(src); - - iov_iter_xarray(&iter, ITER_DEST, &mapping->i_pages, pos, PAGE_SIZE); - if (copy_to_iter(src, size, &iter) != size) { - erofs_put_metabuf(&buf); - return -EFAULT; - } - iov_iter_zero(PAGE_SIZE - size, &iter); - erofs_put_metabuf(&buf); - req->submitted += PAGE_SIZE; - return 0; - } - - count = req->len - req->submitted; - if (!(map.m_flags & EROFS_MAP_MAPPED)) { - struct iov_iter iter; - - iov_iter_xarray(&iter, ITER_DEST, &mapping->i_pages, pos, count); - iov_iter_zero(count, &iter); - req->submitted += count; - return 0; - } - - count = min_t(size_t, map.m_llen - (pos - map.m_la), count); - DBG_BUGON(!count || count % PAGE_SIZE); - - mdev = (struct erofs_map_dev) { - .m_deviceid = map.m_deviceid, - .m_pa = map.m_pa, - }; - ret = erofs_map_dev(sb, &mdev); - if (ret) - return ret; - - io = erofs_fscache_req_io_alloc(req); - if (!io) - return -ENOMEM; - iov_iter_xarray(&io->iter, ITER_DEST, &mapping->i_pages, pos, count); - ret = erofs_fscache_read_io_async(mdev.m_dif->fscache->cookie, - mdev.m_pa + (pos - map.m_la), io); - erofs_fscache_req_io_put(io); - - req->submitted += count; - return ret; -} - -static int erofs_fscache_data_read(struct erofs_fscache_rq *req) -{ - int ret; - - do { - ret = erofs_fscache_data_read_slice(req); - if (ret) - req->error = ret; - } while (!ret && req->submitted < req->len); - return ret; -} - -static int erofs_fscache_read_folio(struct file *file, struct folio *folio) -{ - struct erofs_fscache_rq *req; - int ret; - - req = erofs_fscache_req_alloc(folio->mapping, - folio_pos(folio), folio_size(folio)); - if (!req) { - folio_unlock(folio); - return -ENOMEM; - } - - ret = erofs_fscache_data_read(req); - erofs_fscache_req_put(req); - return ret; -} - -static void erofs_fscache_readahead(struct readahead_control *rac) -{ - struct erofs_fscache_rq *req; - - if (!readahead_count(rac)) - return; - - req = erofs_fscache_req_alloc(rac->mapping, - readahead_pos(rac), readahead_length(rac)); - if (!req) - return; - - /* The request completion will drop refs on the folios. */ - while (readahead_folio(rac)) - ; - - erofs_fscache_data_read(req); - erofs_fscache_req_put(req); -} - -static const struct address_space_operations erofs_fscache_meta_aops = { - .read_folio = erofs_fscache_meta_read_folio, -}; - -const struct address_space_operations erofs_fscache_access_aops = { - .read_folio = erofs_fscache_read_folio, - .readahead = erofs_fscache_readahead, -}; - -static void erofs_fscache_domain_put(struct erofs_domain *domain) -{ - mutex_lock(&erofs_domain_list_lock); - if (refcount_dec_and_test(&domain->ref)) { - list_del(&domain->list); - if (list_empty(&erofs_domain_list)) { - kern_unmount(erofs_pseudo_mnt); - erofs_pseudo_mnt = NULL; - } - fscache_relinquish_volume(domain->volume, NULL, false); - mutex_unlock(&erofs_domain_list_lock); - kfree_sensitive(domain->domain_id); - kfree(domain); - return; - } - mutex_unlock(&erofs_domain_list_lock); -} - -static int erofs_fscache_register_volume(struct super_block *sb) -{ - struct erofs_sb_info *sbi = EROFS_SB(sb); - char *domain_id = sbi->domain_id; - struct fscache_volume *volume; - char *name; - int ret = 0; - - name = kasprintf(GFP_KERNEL, "erofs,%s", - domain_id ? domain_id : sbi->fsid); - if (!name) - return -ENOMEM; - - volume = fscache_acquire_volume(name, NULL, NULL, 0); - if (IS_ERR_OR_NULL(volume)) { - erofs_err(sb, "failed to register volume for %s", name); - ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP; - volume = NULL; - } - - sbi->volume = volume; - kfree(name); - return ret; -} - -static int erofs_fscache_init_domain(struct super_block *sb) -{ - int err; - struct erofs_domain *domain; - struct erofs_sb_info *sbi = EROFS_SB(sb); - - domain = kzalloc_obj(struct erofs_domain); - if (!domain) - return -ENOMEM; - - domain->domain_id = kstrdup(sbi->domain_id, GFP_KERNEL); - if (!domain->domain_id) { - kfree(domain); - return -ENOMEM; - } - - err = erofs_fscache_register_volume(sb); - if (err) - goto out; - - if (!erofs_pseudo_mnt) { - struct vfsmount *mnt = kern_mount(&erofs_anon_fs_type); - if (IS_ERR(mnt)) { - err = PTR_ERR(mnt); - goto out; - } - erofs_pseudo_mnt = mnt; - } - - domain->volume = sbi->volume; - refcount_set(&domain->ref, 1); - list_add(&domain->list, &erofs_domain_list); - sbi->domain = domain; - return 0; -out: - kfree_sensitive(domain->domain_id); - kfree(domain); - return err; -} - -static int erofs_fscache_register_domain(struct super_block *sb) -{ - int err; - struct erofs_domain *domain; - struct erofs_sb_info *sbi = EROFS_SB(sb); - - mutex_lock(&erofs_domain_list_lock); - list_for_each_entry(domain, &erofs_domain_list, list) { - if (!strcmp(domain->domain_id, sbi->domain_id)) { - sbi->domain = domain; - sbi->volume = domain->volume; - refcount_inc(&domain->ref); - mutex_unlock(&erofs_domain_list_lock); - return 0; - } - } - err = erofs_fscache_init_domain(sb); - mutex_unlock(&erofs_domain_list_lock); - return err; -} - -static struct erofs_fscache *erofs_fscache_acquire_cookie(struct super_block *sb, - char *name, unsigned int flags) -{ - struct fscache_volume *volume = EROFS_SB(sb)->volume; - struct erofs_fscache *ctx; - struct fscache_cookie *cookie; - struct super_block *isb; - struct inode *inode; - int ret; - - ctx = kzalloc_obj(*ctx); - if (!ctx) - return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&ctx->node); - refcount_set(&ctx->ref, 1); - - cookie = fscache_acquire_cookie(volume, FSCACHE_ADV_WANT_CACHE_SIZE, - name, strlen(name), NULL, 0, 0); - if (!cookie) { - erofs_err(sb, "failed to get cookie for %s", name); - ret = -EINVAL; - goto err; - } - fscache_use_cookie(cookie, false); - - /* - * Allocate anonymous inode in global pseudo mount for shareable blobs, - * so that they are accessible among erofs fs instances. - */ - isb = flags & EROFS_REG_COOKIE_SHARE ? erofs_pseudo_mnt->mnt_sb : sb; - inode = new_inode(isb); - if (!inode) { - erofs_err(sb, "failed to get anon inode for %s", name); - ret = -ENOMEM; - goto err_cookie; - } - - inode->i_size = OFFSET_MAX; - inode->i_mapping->a_ops = &erofs_fscache_meta_aops; - mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); - inode->i_blkbits = EROFS_SB(sb)->blkszbits; - inode->i_private = ctx; - - ctx->cookie = cookie; - ctx->inode = inode; - return ctx; - -err_cookie: - fscache_unuse_cookie(cookie, NULL, NULL); - fscache_relinquish_cookie(cookie, false); -err: - kfree(ctx); - return ERR_PTR(ret); -} - -static void erofs_fscache_relinquish_cookie(struct erofs_fscache *ctx) -{ - fscache_unuse_cookie(ctx->cookie, NULL, NULL); - fscache_relinquish_cookie(ctx->cookie, false); - iput(ctx->inode); - kfree(ctx->name); - kfree(ctx); -} - -static struct erofs_fscache *erofs_domain_init_cookie(struct super_block *sb, - char *name, unsigned int flags) -{ - struct erofs_fscache *ctx; - struct erofs_domain *domain = EROFS_SB(sb)->domain; - - ctx = erofs_fscache_acquire_cookie(sb, name, flags); - if (IS_ERR(ctx)) - return ctx; - - ctx->name = kstrdup(name, GFP_KERNEL); - if (!ctx->name) { - erofs_fscache_relinquish_cookie(ctx); - return ERR_PTR(-ENOMEM); - } - - refcount_inc(&domain->ref); - ctx->domain = domain; - list_add(&ctx->node, &erofs_domain_cookies_list); - return ctx; -} - -static struct erofs_fscache *erofs_domain_register_cookie(struct super_block *sb, - char *name, unsigned int flags) -{ - struct erofs_fscache *ctx; - struct erofs_domain *domain = EROFS_SB(sb)->domain; - - flags |= EROFS_REG_COOKIE_SHARE; - mutex_lock(&erofs_domain_cookies_lock); - list_for_each_entry(ctx, &erofs_domain_cookies_list, node) { - if (ctx->domain != domain || strcmp(ctx->name, name)) - continue; - if (!(flags & EROFS_REG_COOKIE_NEED_NOEXIST)) { - refcount_inc(&ctx->ref); - } else { - erofs_err(sb, "%s already exists in domain %s", name, - domain->domain_id); - ctx = ERR_PTR(-EEXIST); - } - mutex_unlock(&erofs_domain_cookies_lock); - return ctx; - } - ctx = erofs_domain_init_cookie(sb, name, flags); - mutex_unlock(&erofs_domain_cookies_lock); - return ctx; -} - -struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb, - char *name, - unsigned int flags) -{ - if (EROFS_SB(sb)->domain_id) - return erofs_domain_register_cookie(sb, name, flags); - return erofs_fscache_acquire_cookie(sb, name, flags); -} - -void erofs_fscache_unregister_cookie(struct erofs_fscache *ctx) -{ - struct erofs_domain *domain = NULL; - - if (!ctx) - return; - if (!ctx->domain) - return erofs_fscache_relinquish_cookie(ctx); - - mutex_lock(&erofs_domain_cookies_lock); - if (refcount_dec_and_test(&ctx->ref)) { - domain = ctx->domain; - list_del(&ctx->node); - erofs_fscache_relinquish_cookie(ctx); - } - mutex_unlock(&erofs_domain_cookies_lock); - if (domain) - erofs_fscache_domain_put(domain); -} - -int erofs_fscache_register_fs(struct super_block *sb) -{ - int ret; - struct erofs_sb_info *sbi = EROFS_SB(sb); - struct erofs_fscache *fscache; - unsigned int flags = 0; - - if (sbi->domain_id) - ret = erofs_fscache_register_domain(sb); - else - ret = erofs_fscache_register_volume(sb); - if (ret) - return ret; - - /* - * When shared domain is enabled, using NEED_NOEXIST to guarantee - * the primary data blob (aka fsid) is unique in the shared domain. - * - * For non-shared-domain case, fscache_acquire_volume() invoked by - * erofs_fscache_register_volume() has already guaranteed - * the uniqueness of primary data blob. - * - * Acquired domain/volume will be relinquished in kill_sb() on error. - */ - if (sbi->domain_id) - flags |= EROFS_REG_COOKIE_NEED_NOEXIST; - fscache = erofs_fscache_register_cookie(sb, sbi->fsid, flags); - if (IS_ERR(fscache)) - return PTR_ERR(fscache); - - sbi->dif0.fscache = fscache; - return 0; -} - -void erofs_fscache_unregister_fs(struct super_block *sb) -{ - struct erofs_sb_info *sbi = EROFS_SB(sb); - - erofs_fscache_unregister_cookie(sbi->dif0.fscache); - - if (sbi->domain) - erofs_fscache_domain_put(sbi->domain); - else - fscache_relinquish_volume(sbi->volume, NULL, false); - - sbi->dif0.fscache = NULL; - sbi->volume = NULL; - sbi->domain = NULL; -} diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index a188c570087a..45afe5c50de8 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -191,8 +191,9 @@ static int erofs_read_inode(struct inode *inode) err = -EFSCORRUPTED; goto err_out; } else { - inode->i_blocks = le32_to_cpu(copied.i_u.blocks_lo) << - (sb->s_blocksize_bits - 9); + inode->i_blocks = (le32_to_cpu(copied.i_u.blocks_lo) | + ((u64)le16_to_cpu(copied.i_nb.blocks_hi) << 32)) << + (sb->s_blocksize_bits - 9); } if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { @@ -255,7 +256,7 @@ static int erofs_fill_inode(struct inode *inode) } mapping_set_large_folios(inode->i_mapping); - aops = erofs_get_aops(inode, false); + aops = erofs_get_aops(inode); if (IS_ERR(aops)) return PTR_ERR(aops); inode->i_mapping->a_ops = aops; diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 4792490161ec..580f8d9f14e7 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -23,6 +23,8 @@ __printf(2, 3) void _erofs_printk(struct super_block *sb, const char *fmt, ...); #define erofs_err(sb, fmt, ...) \ _erofs_printk(sb, KERN_ERR fmt "\n", ##__VA_ARGS__) +#define erofs_warn(sb, fmt, ...) \ + _erofs_printk(sb, KERN_WARNING fmt "\n", ##__VA_ARGS__) #define erofs_info(sb, fmt, ...) \ _erofs_printk(sb, KERN_INFO fmt "\n", ##__VA_ARGS__) @@ -41,7 +43,6 @@ typedef u64 erofs_blk_t; struct erofs_device_info { char *path; - struct erofs_fscache *fscache; struct file *file; struct dax_device *dax_dev; u64 fsoff, dax_part_off; @@ -78,24 +79,6 @@ struct erofs_sb_lz4_info { u16 max_pclusterblks; }; -struct erofs_domain { - refcount_t ref; - struct list_head list; - struct fscache_volume *volume; - char *domain_id; -}; - -struct erofs_fscache { - struct fscache_cookie *cookie; - struct inode *inode; /* anonymous inode for the blob */ - - /* used for share domain mode */ - struct erofs_domain *domain; - struct list_head node; - refcount_t ref; - char *name; -}; - struct erofs_xattr_prefix_item { struct erofs_xattr_long_prefix *prefix; u8 infix_len; @@ -160,10 +143,6 @@ struct erofs_sb_info { struct completion s_kobj_unregister; erofs_off_t dir_ra_bytes; - /* fscache support */ - struct fscache_volume *volume; - struct erofs_domain *domain; - char *fsid; char *domain_id; }; @@ -189,12 +168,6 @@ static inline bool erofs_is_fileio_mode(struct erofs_sb_info *sbi) extern struct file_system_type erofs_anon_fs_type; -static inline bool erofs_is_fscache_mode(struct super_block *sb) -{ - return IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && - !erofs_is_fileio_mode(EROFS_SB(sb)) && !sb->s_bdev; -} - enum { EROFS_ZIP_CACHE_DISABLED, EROFS_ZIP_CACHE_READAHEAD, @@ -411,11 +384,9 @@ struct erofs_map_dev { }; extern const struct super_operations erofs_sops; - extern const struct address_space_operations erofs_aops; extern const struct address_space_operations erofs_fileio_aops; extern const struct address_space_operations z_erofs_aops; -extern const struct address_space_operations erofs_fscache_access_aops; extern const struct inode_operations erofs_generic_iops; extern const struct inode_operations erofs_symlink_iops; @@ -428,10 +399,6 @@ extern const struct file_operations erofs_ishare_fops; extern const struct iomap_ops z_erofs_iomap_report_ops; -/* flags for erofs_fscache_register_cookie() */ -#define EROFS_REG_COOKIE_SHARE 0x0001 -#define EROFS_REG_COOKIE_NEED_NOEXIST 0x0002 - void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf, erofs_off_t *offset, int *lengthp); void erofs_unmap_metabuf(struct erofs_buf *buf); @@ -471,7 +438,7 @@ static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count) } static inline const struct address_space_operations * -erofs_get_aops(struct inode *realinode, bool no_fscache) +erofs_get_aops(struct inode *realinode) { if (erofs_inode_is_data_compressed(EROFS_I(realinode)->datalayout)) { if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) @@ -481,9 +448,6 @@ erofs_get_aops(struct inode *realinode, bool no_fscache) "EXPERIMENTAL EROFS subpage compressed block support in use. Use at your own risk!"); return &z_erofs_aops; } - if (IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && !no_fscache && - erofs_is_fscache_mode(realinode->i_sb)) - return &erofs_fscache_access_aops; if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && erofs_is_fileio_mode(EROFS_SB(realinode->i_sb))) return &erofs_fileio_aops; @@ -546,36 +510,6 @@ static inline struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) { r static inline void erofs_fileio_submit_bio(struct bio *bio) {} #endif -#ifdef CONFIG_EROFS_FS_ONDEMAND -int erofs_fscache_register_fs(struct super_block *sb); -void erofs_fscache_unregister_fs(struct super_block *sb); - -struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb, - char *name, unsigned int flags); -void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache); -struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev); -void erofs_fscache_submit_bio(struct bio *bio); -#else -static inline int erofs_fscache_register_fs(struct super_block *sb) -{ - return -EOPNOTSUPP; -} -static inline void erofs_fscache_unregister_fs(struct super_block *sb) {} - -static inline -struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb, - char *name, unsigned int flags) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache) -{ -} -static inline struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev) { return NULL; } -static inline void erofs_fscache_submit_bio(struct bio *bio) {} -#endif - #ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE int __init erofs_init_ishare(void); void erofs_exit_ishare(void); diff --git a/fs/erofs/ishare.c b/fs/erofs/ishare.c index 6ed66b17359b..a1a20a5ba548 100644 --- a/fs/erofs/ishare.c +++ b/fs/erofs/ishare.c @@ -12,12 +12,6 @@ static struct vfsmount *erofs_ishare_mnt; -static inline bool erofs_is_ishare_inode(struct inode *inode) -{ - /* assumed FS_ONDEMAND is excluded with FS_PAGE_CACHE_SHARE feature */ - return inode->i_sb->s_type == &erofs_anon_fs_type; -} - static int erofs_ishare_iget5_eq(struct inode *inode, void *data) { struct erofs_inode_fingerprint *fp1 = &EROFS_I(inode)->fingerprint; @@ -40,49 +34,42 @@ static int erofs_ishare_iget5_set(struct inode *inode, void *data) bool erofs_ishare_fill_inode(struct inode *inode) { struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); - struct erofs_inode *vi = EROFS_I(inode); const struct address_space_operations *aops; + struct erofs_inode *vi = EROFS_I(inode); struct erofs_inode_fingerprint fp; - struct inode *sharedinode; - unsigned long hash; + struct inode *si; - aops = erofs_get_aops(inode, true); + aops = erofs_get_aops(inode); if (IS_ERR(aops)) return false; if (erofs_xattr_fill_inode_fingerprint(&fp, inode, sbi->domain_id)) return false; - hash = xxh32(fp.opaque, fp.size, 0); - sharedinode = iget5_locked(erofs_ishare_mnt->mnt_sb, hash, - erofs_ishare_iget5_eq, erofs_ishare_iget5_set, - &fp); - if (!sharedinode) { - kfree(fp.opaque); - return false; - } - if (inode_state_read_once(sharedinode) & I_NEW) { - sharedinode->i_mapping->a_ops = aops; - sharedinode->i_size = vi->vfs_inode.i_size; - unlock_new_inode(sharedinode); + si = iget5_locked(erofs_ishare_mnt->mnt_sb, + xxh32(fp.opaque, fp.size, 0), + erofs_ishare_iget5_eq, erofs_ishare_iget5_set, &fp); + if (si && (inode_state_read_once(si) & I_NEW)) { + si->i_mapping->a_ops = aops; + si->i_size = inode->i_size; + unlock_new_inode(si); } else { kfree(fp.opaque); - if (aops != sharedinode->i_mapping->a_ops) { - iput(sharedinode); + if (!si || aops != si->i_mapping->a_ops) { + iput(si); return false; } - if (sharedinode->i_size != vi->vfs_inode.i_size) { - _erofs_printk(inode->i_sb, KERN_WARNING - "size(%lld:%lld) not matches for the same fingerprint\n", - vi->vfs_inode.i_size, sharedinode->i_size); - iput(sharedinode); + if (si->i_size != inode->i_size) { + erofs_warn(inode->i_sb, "i_size mismatch (%lld != %lld) for the same fingerprint", + inode->i_size, si->i_size); + iput(si); return false; } } - vi->sharedinode = sharedinode; + vi->sharedinode = si; INIT_LIST_HEAD(&vi->ishare_list); - spin_lock(&EROFS_I(sharedinode)->ishare_lock); - list_add(&vi->ishare_list, &EROFS_I(sharedinode)->ishare_list); - spin_unlock(&EROFS_I(sharedinode)->ishare_lock); + spin_lock(&EROFS_I(si)->ishare_lock); + list_add(&vi->ishare_list, &EROFS_I(si)->ishare_list); + spin_unlock(&EROFS_I(si)->ishare_lock); return true; } @@ -186,7 +173,7 @@ struct inode *erofs_real_inode(struct inode *inode, bool *need_iput) struct inode *realinode; *need_iput = false; - if (!erofs_is_ishare_inode(inode)) + if (inode->i_sb != erofs_ishare_mnt->mnt_sb) return inode; vi_share = EROFS_I(inode); diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 802add6652fd..9d8f862f309f 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -126,7 +126,6 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, struct erofs_device_info *dif, erofs_off_t *pos) { struct erofs_sb_info *sbi = EROFS_SB(sb); - struct erofs_fscache *fscache; struct erofs_deviceslot *dis; struct file *file; bool _48bit; @@ -145,12 +144,7 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, return -ENOMEM; } - if (erofs_is_fscache_mode(sb)) { - fscache = erofs_fscache_register_cookie(sb, dif->path, 0); - if (IS_ERR(fscache)) - return PTR_ERR(fscache); - dif->fscache = fscache; - } else if (!sbi->devs->flatdev) { + if (!sbi->devs->flatdev) { file = erofs_is_fileio_mode(sbi) ? filp_open(dif->path, O_RDONLY | O_LARGEFILE, 0) : bdev_file_open_by_path(dif->path, @@ -216,7 +210,7 @@ static int erofs_scan_devices(struct super_block *sb, if (!ondisk_extradevs) return 0; - if (!sbi->devs->extra_devices && !erofs_is_fscache_mode(sb)) + if (!sbi->devs->extra_devices) sbi->devs->flatdev = true; sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1; @@ -372,8 +366,6 @@ static int erofs_read_superblock(struct super_block *sb) erofs_info(sb, "EXPERIMENTAL 48-bit layout support in use. Use at your own risk!"); if (erofs_sb_has_metabox(sbi)) erofs_info(sb, "EXPERIMENTAL metadata compression support in use. Use at your own risk!"); - if (erofs_is_fscache_mode(sb)) - erofs_info(sb, "[deprecated] fscache-based on-demand read feature in use. Use at your own risk!"); out: erofs_put_metabuf(&buf); return ret; @@ -393,8 +385,7 @@ static void erofs_default_options(struct erofs_sb_info *sbi) enum { Opt_user_xattr, Opt_acl, Opt_cache_strategy, Opt_dax, Opt_dax_enum, - Opt_device, Opt_fsid, Opt_domain_id, Opt_directio, Opt_fsoffset, - Opt_inode_share, + Opt_device, Opt_domain_id, Opt_directio, Opt_fsoffset, Opt_inode_share, }; static const struct constant_table erofs_param_cache_strategy[] = { @@ -418,7 +409,6 @@ static const struct fs_parameter_spec erofs_fs_parameters[] = { fsparam_flag("dax", Opt_dax), fsparam_enum("dax", Opt_dax_enum, erofs_dax_param_enums), fsparam_string("device", Opt_device), - fsparam_string("fsid", Opt_fsid), fsparam_string("domain_id", Opt_domain_id), fsparam_flag_no("directio", Opt_directio), fsparam_u64("fsoffset", Opt_fsoffset), @@ -509,25 +499,14 @@ static int erofs_fc_parse_param(struct fs_context *fc, } ++sbi->devs->extra_devices; break; -#ifdef CONFIG_EROFS_FS_ONDEMAND - case Opt_fsid: - kfree(sbi->fsid); - sbi->fsid = kstrdup(param->string, GFP_KERNEL); - if (!sbi->fsid) - return -ENOMEM; - break; -#endif -#if defined(CONFIG_EROFS_FS_ONDEMAND) || defined(CONFIG_EROFS_FS_PAGE_CACHE_SHARE) - case Opt_domain_id: - kfree_sensitive(sbi->domain_id); - sbi->domain_id = no_free_ptr(param->string); - break; -#else - case Opt_fsid: case Opt_domain_id: - errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name); + if (!IS_ENABLED(CONFIG_EROFS_FS_PAGE_CACHE_SHARE)) { + errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name); + } else { + kfree_sensitive(sbi->domain_id); + sbi->domain_id = no_free_ptr(param->string); + } break; -#endif case Opt_directio: if (!IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE)) errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name); @@ -616,22 +595,6 @@ static const struct export_operations erofs_export_ops = { .get_parent = erofs_get_parent, }; -static void erofs_set_sysfs_name(struct super_block *sb) -{ - struct erofs_sb_info *sbi = EROFS_SB(sb); - - if (sbi->domain_id && sbi->fsid) - super_set_sysfs_name_generic(sb, "%s,%s", sbi->domain_id, - sbi->fsid); - else if (sbi->fsid) - super_set_sysfs_name_generic(sb, "%s", sbi->fsid); - else if (erofs_is_fileio_mode(sbi)) - super_set_sysfs_name_generic(sb, "%s", - bdi_dev_name(sb->s_bdi)); - else - super_set_sysfs_name_id(sb); -} - static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) { struct inode *inode; @@ -680,20 +643,17 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_blocksize = PAGE_SIZE; sb->s_blocksize_bits = PAGE_SHIFT; - if (erofs_is_fscache_mode(sb)) { - err = erofs_fscache_register_fs(sb); - if (err) - return err; - } err = super_setup_bdi(sb); if (err) return err; + + snprintf(sb->s_id, sizeof(sb->s_id), + "%u:%u", MAJOR(sb->s_dev), MINOR(sb->s_dev)); } else { if (!sb_set_blocksize(sb, PAGE_SIZE)) { errorfc(fc, "failed to set initial blksize"); return -EINVAL; } - sbi->dif0.dax_dev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->dif0.dax_part_off, NULL, NULL); } @@ -703,11 +663,6 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) return err; if (sb->s_blocksize_bits != sbi->blkszbits) { - if (erofs_is_fscache_mode(sb)) { - errorfc(fc, "unsupported blksize for fscache mode"); - return -EINVAL; - } - if (erofs_is_fileio_mode(sbi)) { sb->s_blocksize = 1 << sbi->blkszbits; sb->s_blocksize_bits = sbi->blkszbits; @@ -716,14 +671,9 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) return -EINVAL; } } - - if (sbi->dif0.fsoff) { - if (sbi->dif0.fsoff & (sb->s_blocksize - 1)) - return invalfc(fc, "fsoffset %llu is not aligned to block size %lu", - sbi->dif0.fsoff, sb->s_blocksize); - if (erofs_is_fscache_mode(sb)) - return invalfc(fc, "cannot use fsoffset in fscache mode"); - } + if (sbi->dif0.fsoff & (sb->s_blocksize - 1)) + return invalfc(fc, "fsoffset %llu is not aligned to block size %lu", + sbi->dif0.fsoff, sb->s_blocksize); if (test_opt(&sbi->opt, DAX_ALWAYS) && sbi->blkszbits != PAGE_SHIFT) { erofs_info(sb, "unsupported blocksize for DAX"); @@ -781,7 +731,7 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) if (err) return err; - erofs_set_sysfs_name(sb); + super_set_sysfs_name_id(sb); err = erofs_register_sysfs(sb); if (err) return err; @@ -793,16 +743,13 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) static int erofs_fc_get_tree(struct fs_context *fc) { - struct erofs_sb_info *sbi = fc->s_fs_info; int ret; - if (IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && sbi->fsid) - return get_tree_nodev(fc, erofs_fc_fill_super); - ret = get_tree_bdev_flags(fc, erofs_fc_fill_super, IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) ? GET_TREE_BDEV_QUIET_LOOKUP : 0); if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && ret == -ENOTBLK) { + struct erofs_sb_info *sbi = fc->s_fs_info; struct file *file; if (!fc->source) @@ -827,8 +774,8 @@ static int erofs_fc_reconfigure(struct fs_context *fc) DBG_BUGON(!sb_rdonly(sb)); - if (new_sbi->fsid || new_sbi->domain_id) - erofs_info(sb, "ignoring reconfiguration for fsid|domain_id."); + if (new_sbi->domain_id) + erofs_info(sb, "ignoring reconfiguration for domain_id."); if (test_opt(&new_sbi->opt, POSIX_ACL)) fc->sb_flags |= SB_POSIXACL; @@ -848,8 +795,6 @@ static int erofs_release_device_info(int id, void *ptr, void *data) fs_put_dax(dif->dax_dev, NULL); if (dif->file) fput(dif->file); - erofs_fscache_unregister_cookie(dif->fscache); - dif->fscache = NULL; kfree(dif->path); kfree(dif); return 0; @@ -867,7 +812,6 @@ static void erofs_free_dev_context(struct erofs_dev_context *devs) static void erofs_sb_free(struct erofs_sb_info *sbi) { erofs_free_dev_context(sbi->devs); - kfree(sbi->fsid); kfree_sensitive(sbi->domain_id); if (sbi->dif0.file) fput(sbi->dif0.file); @@ -928,14 +872,12 @@ static void erofs_kill_sb(struct super_block *sb) { struct erofs_sb_info *sbi = EROFS_SB(sb); - if ((IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && sbi->fsid) || - sbi->dif0.file) + if (sbi->dif0.file) kill_anon_super(sb); else kill_block_super(sb); erofs_drop_internal_inodes(sbi); fs_put_dax(sbi->dif0.dax_dev, NULL); - erofs_fscache_unregister_fs(sb); erofs_sb_free(sbi); sb->s_fs_info = NULL; } @@ -950,7 +892,6 @@ static void erofs_put_super(struct super_block *sb) erofs_drop_internal_inodes(sbi); erofs_free_dev_context(sbi->devs); sbi->devs = NULL; - erofs_fscache_unregister_fs(sb); } static struct file_system_type erofs_fs_type = { @@ -962,14 +903,12 @@ static struct file_system_type erofs_fs_type = { }; MODULE_ALIAS_FS("erofs"); -#if defined(CONFIG_EROFS_FS_ONDEMAND) || defined(CONFIG_EROFS_FS_PAGE_CACHE_SHARE) +#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE static void erofs_free_anon_inode(struct inode *inode) { struct erofs_inode *vi = EROFS_I(inode); -#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE kfree(vi->fingerprint.opaque); -#endif kmem_cache_free(erofs_inode_cachep, vi); } @@ -1048,11 +987,11 @@ shrinker_err: static void __exit erofs_module_exit(void) { unregister_filesystem(&erofs_fs_type); + erofs_exit_ishare(); - /* Ensure all RCU free inodes / pclusters are safe to be destroyed. */ + /* ensure all delayed rcu free inodes & pclusters are flushed */ rcu_barrier(); - erofs_exit_ishare(); erofs_exit_sysfs(); z_erofs_exit_subsystem(); erofs_exit_shrinker(); @@ -1090,7 +1029,7 @@ static int erofs_show_options(struct seq_file *seq, struct dentry *root) ",user_xattr" : ",nouser_xattr"); if (IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL)) seq_puts(seq, test_opt(opt, POSIX_ACL) ? ",acl" : ",noacl"); - if (IS_ENABLED(CONFIG_EROFS_FS_ZIP)) + if (IS_ENABLED(CONFIG_EROFS_FS_ZIP) && sbi->available_compr_algs) seq_printf(seq, ",cache_strategy=%s", erofs_param_cache_strategy[opt->cache_strategy].name); if (test_opt(opt, DAX_ALWAYS)) @@ -1099,12 +1038,6 @@ static int erofs_show_options(struct seq_file *seq, struct dentry *root) seq_puts(seq, ",dax=never"); if (erofs_is_fileio_mode(sbi) && test_opt(opt, DIRECT_IO)) seq_puts(seq, ",directio"); - if (IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND)) { - if (sbi->fsid) - seq_printf(seq, ",fsid=%s", sbi->fsid); - if (sbi->domain_id) - seq_printf(seq, ",domain_id=%s", sbi->domain_id); - } if (sbi->dif0.fsoff) seq_printf(seq, ",fsoffset=%llu", sbi->dif0.fsoff); if (test_opt(opt, INODE_SHARE)) diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index 41e311019a25..df7ea019526d 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -89,13 +89,11 @@ static int erofs_init_inode_xattrs(struct inode *inode) vi->xattr_isize - sizeof(struct erofs_xattr_ibody_header)) { erofs_err(sb, "invalid h_shared_count %u @ nid %llu", vi->xattr_shared_count, vi->nid); - erofs_put_metabuf(&buf); ret = -EFSCORRUPTED; goto out_unlock; } vi->xattr_shared_xattrs = kmalloc_objs(uint, vi->xattr_shared_count); if (!vi->xattr_shared_xattrs) { - erofs_put_metabuf(&buf); ret = -ENOMEM; goto out_unlock; } @@ -112,12 +110,12 @@ static int erofs_init_inode_xattrs(struct inode *inode) } vi->xattr_shared_xattrs[i] = le32_to_cpu(*xattr_id); } - erofs_put_metabuf(&buf); /* paired with smp_mb() at the beginning of the function. */ smp_mb(); set_bit(EROFS_I_EA_INITED_BIT, &vi->flags); out_unlock: + erofs_put_metabuf(&buf); clear_and_wake_up_bit(EROFS_I_BL_XATTR_BIT, &vi->flags); return ret; } diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 43bb5a6a9924..74520e910259 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -806,6 +806,7 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) struct super_block *sb = fe->inode->i_sb; struct z_erofs_pcluster *pcl = NULL; void *ptr = NULL; + bool needretry; int ret; DBG_BUGON(fe->pcl); @@ -825,19 +826,16 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) } ptr = map->buf.page; } else { - while (1) { + do { rcu_read_lock(); pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa); - if (!pcl || z_erofs_get_pcluster(pcl)) { - DBG_BUGON(pcl && map->m_pa != pcl->pos); - rcu_read_unlock(); - break; - } + needretry = pcl && !z_erofs_get_pcluster(pcl); rcu_read_unlock(); - } + } while (needretry); } if (pcl) { + DBG_BUGON(map->m_pa != pcl->pos); fe->pcl = pcl; ret = -EEXIST; } else { @@ -1455,25 +1453,23 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, if (atomic_add_return(bios, &io->pending_bios)) return; if (z_erofs_in_atomic()) { + /* See `sync_decompress` in sysfs-fs-erofs for more details */ + if (sbi->sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) + sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD - struct kthread_worker *worker; + scoped_guard(rcu) { + struct kthread_worker *worker; - rcu_read_lock(); - worker = rcu_dereference( + worker = rcu_dereference( z_erofs_pcpu_workers[raw_smp_processor_id()]); - if (!worker) { - INIT_WORK(&io->u.work, z_erofs_decompressqueue_work); - queue_work(z_erofs_workqueue, &io->u.work); - } else { - kthread_queue_work(worker, &io->u.kthread_work); + if (worker) { + kthread_queue_work(worker, &io->u.kthread_work); + return; + } } - rcu_read_unlock(); -#else - queue_work(z_erofs_workqueue, &io->u.work); + INIT_WORK(&io->u.work, z_erofs_decompressqueue_work); #endif - /* See `sync_decompress` in sysfs-fs-erofs for more details */ - if (sbi->sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) - sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; + queue_work(z_erofs_workqueue, &io->u.work); return; } gfp_flag = memalloc_noio_save(); @@ -1509,8 +1505,15 @@ repeat: DBG_BUGON(z_erofs_is_shortlived_page(bvec->bv_page)); folio = page_folio(zbv.page); - /* For preallocated managed folios, add them to page cache here */ + /* + * Preallocated folios are added to the managed cache here rather than + * in z_erofs_bind_cache() in order to keep these folios locked in + * increasing (physical) address order. + * Clear folio->private before these folios become visible to others in + * the managed cache to avoid duplicate additions for unaligned extents. + */ if (folio->private == Z_EROFS_PREALLOCATED_FOLIO) { + folio->private = NULL; tocache = true; goto out_tocache; } @@ -1546,14 +1549,8 @@ repeat: } return; } - /* - * Already linked with another pcluster, which only appears in - * crafted images by fuzzers for now. But handle this anyway. - */ - tocache = false; /* use temporary short-lived pages */ } else { DBG_BUGON(1); /* referenced managed folios can't be truncated */ - tocache = true; } folio_unlock(folio); folio_put(folio); @@ -1713,8 +1710,6 @@ static void z_erofs_submit_queue(struct z_erofs_frontend *f, drain_io: if (erofs_is_fileio_mode(EROFS_SB(sb))) erofs_fileio_submit_bio(bio); - else if (erofs_is_fscache_mode(sb)) - erofs_fscache_submit_bio(bio); else submit_bio(bio); @@ -1743,8 +1738,6 @@ drain_io: if (!bio) { if (erofs_is_fileio_mode(EROFS_SB(sb))) bio = erofs_fileio_bio_alloc(&mdev); - else if (erofs_is_fscache_mode(sb)) - bio = erofs_fscache_bio_alloc(&mdev); else bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS, REQ_OP_READ, GFP_NOIO); @@ -1773,8 +1766,6 @@ drain_io: if (bio) { if (erofs_is_fileio_mode(EROFS_SB(sb))) erofs_fileio_submit_bio(bio); - else if (erofs_is_fscache_mode(sb)) - erofs_fscache_submit_bio(bio); else submit_bio(bio); } diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index a72db36096ca..5811556a7b71 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -15,8 +15,9 @@ struct z_erofs_maprecorder { u8 type, headtype; u16 clusterofs; u16 delta[2]; - erofs_blk_t pblk, compressedblks; + erofs_blk_t pblk; erofs_off_t nextpackoff; + int compressedblks; bool partialref, in_mbox; }; @@ -54,7 +55,12 @@ static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, u64 lcn) } else { m->partialref = !!(advise & Z_EROFS_LI_PARTIAL_REF); m->clusterofs = le16_to_cpu(di->di_clusterofs); - m->pblk = le32_to_cpu(di->di_u.blkaddr); + if (advise & Z_EROFS_LI_HOLE) { + m->compressedblks = 0; + m->pblk = EROFS_NULL_ADDR; + } else { + m->pblk = le32_to_cpu(di->di_u.blkaddr); + } } return 0; } @@ -309,9 +315,10 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) || (lcn << vi->z_lclusterbits) >= inode->i_size) - m->compressedblks = 1; + if (m->compressedblks < 0) + m->compressedblks = 1; - if (m->compressedblks) + if (m->compressedblks >= 0) goto out; err = z_erofs_load_lcluster_from_disk(m, lcn, false); @@ -329,19 +336,22 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, DBG_BUGON(lcn == initial_lcn && m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); - if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD && m->delta[0] != 1) { + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + /* + * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type + * rather than CBLKCNT, it's a 1 block-sized pcluster. + */ + if (m->compressedblks < 0) + m->compressedblks = 1; + } else if (m->delta[0] != 1 || m->compressedblks < 0) { erofs_err(sb, "bogus CBLKCNT @ lcn %llu of nid %llu", lcn, vi->nid); DBG_BUGON(1); return -EFSCORRUPTED; } - /* - * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type rather - * than CBLKCNT, it's a 1 block-sized pcluster. - */ - if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD || !m->compressedblks) - m->compressedblks = 1; out: + if (!m->compressedblks) + m->map->m_flags &= ~EROFS_MAP_MAPPED; m->map->m_plen = erofs_pos(sb, m->compressedblks); return 0; } @@ -395,6 +405,7 @@ static int z_erofs_map_blocks_fo(struct inode *inode, .inode = inode, .map = map, .in_mbox = erofs_inode_in_metabox(inode), + .compressedblks = -1, }; unsigned int endoff; unsigned long initial_lcn; @@ -716,12 +727,13 @@ static int z_erofs_map_sanity_check(struct inode *inode, } if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { - if (sbi->available_compr_algs ^ BIT(map->m_algorithmformat)) { + if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", map->m_algorithmformat, EROFS_I(inode)->nid); return -EFSCORRUPTED; } - if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen) { + if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen && + map->m_la + map->m_llen < inode->i_size) { erofs_err(inode->i_sb, "too much compressed data @ la %llu of nid %llu", map->m_la, EROFS_I(inode)->nid); return -EFSCORRUPTED; diff --git a/fs/eventpoll.c b/fs/eventpoll.c index a3090b446af1..0e65c7431dfc 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -38,48 +38,174 @@ #include <linux/compat.h> #include <linux/rculist.h> #include <linux/capability.h> +#include <linux/seqlock.h> #include <net/busy_poll.h> /* - * LOCKING: - * There are three level of locking required by epoll : - * - * 1) epnested_mutex (mutex) - * 2) ep->mtx (mutex) - * 3) ep->lock (spinlock) - * - * The acquire order is the one listed above, from 1 to 3. - * We need a spinlock (ep->lock) because we manipulate objects - * from inside the poll callback, that might be triggered from - * a wake_up() that in turn might be called from IRQ context. - * So we can't sleep inside the poll callback and hence we need - * a spinlock. During the event transfer loop (from kernel to - * user space) we could end up sleeping due a copy_to_user(), so - * we need a lock that will allow us to sleep. This lock is a - * mutex (ep->mtx). It is acquired during the event transfer loop, - * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file(). - * The epnested_mutex is acquired when inserting an epoll fd onto another - * epoll fd. We do this so that we walk the epoll tree and ensure that this - * insertion does not create a cycle of epoll file descriptors, which - * could lead to deadlock. We need a global mutex to prevent two - * simultaneous inserts (A into B and B into A) from racing and - * constructing a cycle without either insert observing that it is - * going to. - * It is necessary to acquire multiple "ep->mtx"es at once in the - * case when one epoll fd is added to another. In this case, we - * always acquire the locks in the order of nesting (i.e. after - * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired - * before e2->mtx). Since we disallow cycles of epoll file - * descriptors, this ensures that the mutexes are well-ordered. In - * order to communicate this nesting to lockdep, when walking a tree - * of epoll file descriptors, we use the current recursion depth as - * the lockdep subkey. - * It is possible to drop the "ep->mtx" and to use the global - * mutex "epnested_mutex" (together with "ep->lock") to have it working, - * but having "ep->mtx" will make the interface more scalable. - * Events that require holding "epnested_mutex" are very rare, while for - * normal operations the epoll private "ep->mtx" will guarantee - * a better scalability. + * fs/eventpoll.c - Efficient event polling ("epoll") kernel implementation. + * + * + * Overview + * -------- + * + * Each epoll_create(2) returns an anonymous [eventpoll] file whose + * ->private_data is a struct eventpoll. Each EPOLL_CTL_ADD installs + * a struct epitem linking one (watched file, fd) pair back to that + * eventpoll via the watched file's f_op->poll() wait queue(s). When + * the watched file signals readiness, ep_poll_callback() fires and + * marks the epitem ready. epoll_wait(2) drains the ready list under + * ep->mtx, re-queueing items in level-triggered mode. + * + * epoll instances can watch other epoll instances up to EP_MAX_NESTS + * deep; cycles are forbidden and detected at EPOLL_CTL_ADD time. + * + * + * Locking + * ------- + * + * Three levels, acquired from outer to inner: + * + * epnested_mutex (global; rare; taken only for EPOLL_CTL_ADD + * loop / path checks) + * > ep->mtx (per-eventpoll; sleepable; serializes most ops) + * > ep->lock (per-eventpoll; IRQ-safe spinlock) + * + * file->f_lock (per-file; NOT IRQ-safe; guards f_ep hlist ops; + * nested inside ep->mtx, outside ep->lock) + * + * Rationale: + * - ep->lock is a spinlock because ep_poll_callback() is called from + * wake_up() which may run in hard-IRQ context. All ep->lock + * critical sections use spin_lock_irqsave(). + * - ep->mtx is a sleepable mutex because the event delivery loop + * calls copy_to_user(), and ep_insert() may sleep in + * kmem_cache_alloc() and f_op->poll(). + * - epnested_mutex is global because cycle detection needs a global + * view of the epoll topology; a per-object scheme would let two + * concurrent inserts (A into B, B into A) construct a cycle + * without either observer seeing it. + * - Per-ep ep->mtx is preferred for scalability elsewhere. Events + * that require epnested_mutex are rare. + * + * When EPOLL_CTL_ADD nests one eventpoll inside another we acquire + * ep->mtx on both: outer first, target second. Since cycles are + * forbidden the set of live ep->mtx holds is always a strict chain, + * communicated to lockdep via mutex_lock_nested() subclasses derived + * from the current recursion depth. + * + * + * Field protection + * ---------------- + * + * struct eventpoll: + * mtx - self + * rbr - ep->mtx + * ovflist, rdllist - ep->lock (IRQ-safe) + * wq - ep->lock for queue mutation + * poll_wait - internal waitqueue spinlock + * refs - file->f_lock for adds; ep->mtx for removes; + * RCU for readers (hlist_del_rcu + kfree_rcu(ep)) + * ws - ep->mtx + * gen, loop_check_depth - epnested_mutex + * file, user - immutable after setup + * refcount - atomic (refcount_t) + * napi_* - READ_ONCE / WRITE_ONCE + * + * struct epitem: + * rbn / rcu union - rbn: ep->mtx (while epi is linked in ep->rbr). + * rcu: written only by kfree_rcu(epi) on the free + * path; otherwise untouched by epoll code. + * rdllink, next - ep->lock + * ffd, ep - immutable after ep_insert() + * pwqlist - ep->mtx for writes; POLLFREE clears pwq->whead + * via smp_store_release(), see below + * fllink - file->f_lock for mutation; hlist_del_rcu + + * kfree_rcu(epi) for safe RCU readers + * ws - RCU (rcu_assign_pointer / + * rcu_dereference_check(mtx)) + * event - ep->mtx for writes; lockless read in + * ep_poll_callback pairs with smp_mb() in + * ep_modify() + * + * + * Ready-list state machine + * ------------------------ + * + * Readiness is tracked in two lists under ep->lock: + * + * rdllist - doubly-linked FIFO; the "current" ready list. + * ovflist - singly-linked LIFO; used during a scan to catch + * events that arrive while rdllist is being iterated + * without ep->lock. + * + * Encoded in ep->ovflist: + * EP_UNACTIVE_PTR - no scan active; callback appends to rdllist. + * NULL - scan active, no spill yet. + * pointer to epi - scan active with spilled items (LIFO). + * + * Encoded in epi->ovflist_next: + * EP_UNACTIVE_PTR - epi is not on ovflist. + * otherwise - next epi on ovflist (NULL at tail). + * + * ep_start_scan() flips "not scanning" to "scanning" and splices + * rdllist into a caller-local scan_batch. ep_done_scan() drains ovflist + * back to rdllist (list_add head-insert reverses LIFO to FIFO), + * flips back to "not scanning", and re-splices any items the caller + * left in scan_batch (e.g., level-triggered re-queues). + * + * + * Removal paths + * ------------- + * + * Three paths dispose of epitems and/or eventpolls: + * + * A. ep_remove() - EPOLL_CTL_DEL and ep_insert() + * rollback. Caller holds ep->mtx. + * B. ep_clear_and_put() - close of the epoll fd itself + * (ep_eventpoll_release). + * C. eventpoll_release_file() - close of a watched file, invoked + * from __fput(). + * + * Coordination: + * A and C exclude each other via the watched file's refcount. + * A pins the file with epi_fget() before touching file->f_ep or + * file->f_lock; if the pin fails, __fput() is in flight and C + * will clean this epi up. See the epi_fget() block comment. + * A and B both hold ep->mtx serially. B walks the rbtree with + * rb_next() captured before ep_remove() erases the current node. + * B and C both take ep->mtx; the loser sees fewer entries or an + * empty file->f_ep. + * + * Within every path the internal order is strict: + * ep_unregister_pollwait() - drain pwqlist; synchronizes with any + * in-flight ep_poll_callback via the + * watched wait-queue head's lock. + * ep_remove_file() - hlist_del_rcu of epi->fllink and, + * if last watcher, clear file->f_ep, + * under file->f_lock. + * ep_remove_epi() - rb_erase, rdllist unlink (ep->lock), + * wakeup_source_unregister, + * kfree_rcu(epi). + * + * kfree_rcu(epi) defers the free past RCU readers in + * reverse_path_check_proc(); kfree_rcu(ep) defers past readers in + * ep_get_upwards_depth_proc(). + * + * + * POLLFREE handshake + * ------------------ + * + * When a subsystem tears down a wait-queue head that an epitem is + * registered on (binder, signalfd, ...), it wakes the callback with + * POLLFREE and must RCU-defer the head's free. The store/load pair: + * + * ep_poll_callback() POLLFREE branch: + * smp_store_release(&pwq->whead, NULL) + * + * ep_remove_wait_queue(): + * smp_load_acquire(&pwq->whead) + * + * See those sites for the full argument. */ /* Epoll private bits inside the event mask */ @@ -99,11 +225,6 @@ #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry)) -struct epoll_filefd { - struct file *file; - int fd; -} __packed; - /* Wait structure used by the poll hooks */ struct eppoll_entry { /* List header used to link this structure to the "struct epitem" */ @@ -136,17 +257,19 @@ struct epitem { struct rcu_head rcu; }; - /* List header used to link this structure to the eventpoll ready list */ + /* Link on the owning eventpoll's ready list (ep->rdllist). */ struct list_head rdllink; /* - * Works together "struct eventpoll"->ovflist in keeping the - * single linked chain of items. + * Link on the owning eventpoll's scan-overflow list (ep->ovflist), + * EP_UNACTIVE_PTR when not linked. See epi_on_ovflist() / + * epi_clear_ovflist() and the "Ready-list state machine" section + * in the top-of-file banner. */ - struct epitem *next; + struct epitem *ovflist_next; /* The file descriptor information this item refers to */ - struct epoll_filefd ffd; + struct epoll_key ffd; /* List containing poll wait queues */ struct eppoll_entry *pwqlist; @@ -190,6 +313,9 @@ struct eventpoll { /* Lock which protects rdllist and ovflist */ spinlock_t lock; + /* Protect switching between rdllist and ovflist */ + seqcount_spinlock_t seq; + /* RB tree root used to store monitored fd structs */ struct rb_root_cached rbr; @@ -247,13 +373,81 @@ struct ep_pqueue { /* Maximum number of epoll watched descriptors, per user */ static long max_user_watches __read_mostly; -/* Used for cycles detection */ +/* + * Cycle and path-length checks at EPOLL_CTL_ADD + * --------------------------------------------- + * + * When EPOLL_CTL_ADD creates a link that either targets an eventpoll + * file or extends an existing chain of eventpolls, two checks run: + * + * 1. no cycle is being formed -- ep_loop_check() walks downward + * from the candidate target, and ep_get_upwards_depth_proc() + * walks upward from the outer ep, both bounded by EP_MAX_NESTS. + * 2. no file accumulates more than path_limits[depth] wakeup paths + * of a given length -- reverse_path_check(). + * + * Both need a global view of the epoll topology and must be atomic + * with the insertion, so the check is serialized by epnested_mutex + * and carries its scratch state on a stack-allocated struct + * ep_ctl_ctx scoped to one do_epoll_ctl() call. Non-nested inserts + * skip this machinery entirely and take only ep->mtx. + * + * epnested_mutex Serializes the whole check. + * loop_check_gen Global monotonic stamp, bumped at the start of + * a check and again at the end. ep->gen caches + * the value under which ep was last visited by + * ep_loop_check_proc() or + * ep_get_upwards_depth_proc(); the post-check + * bump ensures those cached stamps can no longer + * equal loop_check_gen, so the + * "ep->gen == loop_check_gen" trigger in + * ep_ctl_lock() only fires while another check + * is in flight. + * + * struct ep_ctl_ctx carries the rest (inserting_into, tfile_check_list, + * path_count[]) through the walk; see its declaration below. + * + * Commits fdcfce93073d ("eventpoll: Fix integer overflow in + * ep_loop_check_proc()") and f2e467a48287 ("eventpoll: Fix + * semi-unbounded recursion") hardened the walk; any refactor must + * preserve both bail-outs. + */ static DEFINE_MUTEX(epnested_mutex); - static u64 loop_check_gen = 0; -/* Used to check for epoll file descriptor inclusion loops */ -static struct eventpoll *inserting_into; +#define PATH_ARR_SIZE 5 + +/* + * Per-do_epoll_ctl() scratch for the loop / path checks. Allocated on + * the caller's stack; populated by ep_ctl_lock() and the downward + * walk; consumed by reverse_path_check(); released by ep_ctl_unlock(). + * Only valid while the caller holds epnested_mutex. + */ +struct ep_ctl_ctx { + /* + * Outer eventpoll for one ep_loop_check(); if the downward walk + * reaches it the insert would form a cycle. + */ + struct eventpoll *inserting_into; + + /* + * Singly-linked list of epitems_head objects collected during + * ep_loop_check_proc(), then walked by reverse_path_check(). + * Terminated by EP_UNACTIVE_PTR, not NULL: epitems_head->next + * doubles as a membership flag (a NULL ->next means "not on this + * list", see ep_remove_file()), so the list uses a non-NULL + * sentinel to keep the tail head distinguishable from an unlisted + * one. + */ + struct epitems_head *tfile_check_list; + + /* + * Per-depth wakeup-path tally used by reverse_path_check_proc(); + * reinitialized to zero at the start of each reverse_path_check() + * iteration. + */ + int path_count[PATH_ARR_SIZE]; +}; /* Slab cache used to allocate "struct epitem" */ static struct kmem_cache *epi_cache __ro_after_init; @@ -262,14 +456,15 @@ static struct kmem_cache *epi_cache __ro_after_init; static struct kmem_cache *pwq_cache __ro_after_init; /* - * List of files with newly added links, where we may need to limit the number - * of emanating paths. Protected by the epnested_mutex. + * Wrapper anchor for file->f_ep when the watched file is not itself an + * eventpoll; for the epoll-watches-epoll case, file->f_ep points at + * &watched_ep->refs directly. The ->next field threads + * ctx->tfile_check_list during one EPOLL_CTL_ADD path check. */ struct epitems_head { struct hlist_head epitems; struct epitems_head *next; }; -static struct epitems_head *tfile_check_list = EP_UNACTIVE_PTR; static struct kmem_cache *ephead_cache __ro_after_init; @@ -279,14 +474,14 @@ static inline void free_ephead(struct epitems_head *head) kmem_cache_free(ephead_cache, head); } -static void list_file(struct file *file) +static void list_file(struct file *file, struct ep_ctl_ctx *ctx) { struct epitems_head *head; head = container_of(file->f_ep, struct epitems_head, epitems); if (!head->next) { - head->next = tfile_check_list; - tfile_check_list = head; + head->next = ctx->tfile_check_list; + ctx->tfile_check_list = head; } } @@ -334,29 +529,20 @@ static void __init epoll_sysctls_init(void) static const struct file_operations eventpoll_fops; -static inline int is_file_epoll(struct file *f) +bool is_file_epoll(struct file *f) { return f->f_op == &eventpoll_fops; } -/* Setup the structure that is used as key for the RB tree */ -static inline void ep_set_ffd(struct epoll_filefd *ffd, - struct file *file, int fd) -{ - ffd->file = file; - ffd->fd = fd; -} - /* Compare RB tree keys */ -static inline int ep_cmp_ffd(struct epoll_filefd *p1, - struct epoll_filefd *p2) +static inline int ep_cmp_ffd(struct epoll_key *p1, struct epoll_key *p2) { return (p1->file > p2->file ? +1: (p1->file < p2->file ? -1 : p1->fd - p2->fd)); } -/* Tells us if the item is currently linked */ -static inline int ep_is_linked(struct epitem *epi) +/* True iff @epi is on its owning ep's ready list. */ +static inline bool ep_is_linked(struct epitem *epi) { return !list_empty(&epi->rdllink); } @@ -372,18 +558,50 @@ static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p) return container_of(p, struct eppoll_entry, wait)->base; } -/** - * ep_events_available - Checks if ready events might be available. - * - * @ep: Pointer to the eventpoll context. - * - * Return: a value different than %zero if ready events are available, - * or %zero otherwise. +/* + * Ready-list / ovflist state (see "Ready-list state machine" in the + * top-of-file banner for the full state machine). EP_UNACTIVE_PTR is + * the sentinel; these wrappers name each transition and each test so + * call sites do not need to know the sentinel's value. */ -static inline int ep_events_available(struct eventpoll *ep) + +/* True iff @ep is between ep_enter_scan() and ep_exit_scan(). */ +static inline bool ep_is_scanning(struct eventpoll *ep) +{ + return READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; +} + +/* Called by ep_start_scan(): divert ep_poll_callback() to ovflist. */ +static inline void ep_enter_scan(struct eventpoll *ep) +{ + WRITE_ONCE(ep->ovflist, NULL); +} + +/* Called by ep_done_scan(): redirect ep_poll_callback() back to rdllist. */ +static inline void ep_exit_scan(struct eventpoll *ep) +{ + WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); +} + +/* True iff @epi is currently linked on its ep's ovflist. */ +static inline bool epi_on_ovflist(const struct epitem *epi) { - return !list_empty_careful(&ep->rdllist) || - READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; + return epi->ovflist_next != EP_UNACTIVE_PTR; +} + +/* Mark @epi as not on any ovflist (init and post-drain). */ +static inline void epi_clear_ovflist(struct epitem *epi) +{ + epi->ovflist_next = EP_UNACTIVE_PTR; +} + +/* True iff @ep has ready events that epoll_wait() might harvest. */ +static inline bool ep_events_available(struct eventpoll *ep) +{ + unsigned int seq = read_seqcount_begin(&ep->seq); + + return !list_empty_careful(&ep->rdllist) || ep_is_scanning(ep) || + read_seqcount_retry(&ep->seq, seq); } #ifdef CONFIG_NET_RX_BUSY_POLL @@ -659,10 +877,15 @@ static void ep_remove_wait_queue(struct eppoll_entry *pwq) rcu_read_lock(); /* - * If it is cleared by POLLFREE, it should be rcu-safe. - * If we read NULL we need a barrier paired with - * smp_store_release() in ep_poll_callback(), otherwise - * we rely on whead->lock. + * POLLFREE handshake, acquire side; see "POLLFREE handshake" + * at the top of this file. + * + * A NULL load is paired with the smp_store_release(&whead, NULL) + * in ep_poll_callback()'s POLLFREE branch: the teardown is + * complete and we must not touch whead again. On a non-NULL load + * rcu_read_lock() keeps the waitqueue memory alive (POLLFREE + * firers RCU-defer the free) and whead->lock inside + * remove_wait_queue() serializes us against the store side. */ whead = smp_load_acquire(&pwq->whead); if (whead) @@ -723,7 +946,7 @@ static inline void ep_pm_stay_awake_rcu(struct epitem *epi) * ep->mutex needs to be held because we could be hit by * eventpoll_release_file() and epoll_ctl(). */ -static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist) +static void ep_start_scan(struct eventpoll *ep, struct list_head *scan_batch) { /* * Steal the ready list, and re-init the original one to the @@ -735,13 +958,17 @@ static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist) */ lockdep_assert_irqs_enabled(); spin_lock_irq(&ep->lock); - list_splice_init(&ep->rdllist, txlist); - WRITE_ONCE(ep->ovflist, NULL); + write_seqcount_begin(&ep->seq); + + list_splice_init(&ep->rdllist, scan_batch); + ep_enter_scan(ep); + + write_seqcount_end(&ep->seq); spin_unlock_irq(&ep->lock); } static void ep_done_scan(struct eventpoll *ep, - struct list_head *txlist) + struct list_head *scan_batch) { struct epitem *epi, *nepi; @@ -751,34 +978,35 @@ static void ep_done_scan(struct eventpoll *ep, * other events might have been queued by the poll callback. * We re-insert them inside the main ready-list here. */ - for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; - nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { + for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; ) { + nepi = epi->ovflist_next; + epi_clear_ovflist(epi); /* - * We need to check if the item is already in the list. - * During the "sproc" callback execution time, items are - * queued into ->ovflist but the "txlist" might already - * contain them, and the list_splice() below takes care of them. + * Skip items that the caller already returned via @scan_batch + * -- the list_splice() below takes care of those. */ if (!ep_is_linked(epi)) { /* - * ->ovflist is LIFO, so we have to reverse it in order - * to keep in FIFO. + * ovflist is LIFO; list_add() head-insert here + * reverses the iteration order into FIFO. */ list_add(&epi->rdllink, &ep->rdllist); ep_pm_stay_awake(epi); } } - /* - * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after - * releasing the lock, events will be queued in the normal way inside - * ep->rdllist. - */ - WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); + + write_seqcount_begin(&ep->seq); + + /* Back out of scan mode; callbacks target ep->rdllist again. */ + ep_exit_scan(ep); /* - * Quickly re-inject items left on "txlist". + * Quickly re-inject items left on "scan_batch". */ - list_splice(txlist, &ep->rdllist); + list_splice(scan_batch, &ep->rdllist); + + write_seqcount_end(&ep->seq); + __pm_relax(ep->ws); if (!list_empty(&ep->rdllist)) { @@ -795,9 +1023,10 @@ static void ep_get(struct eventpoll *ep) } /* - * Returns true if the event poll can be disposed + * Drop a reference to @ep; returns true iff it was the last, in which + * case the caller is responsible for ep_free(). */ -static bool ep_refcount_dec_and_test(struct eventpoll *ep) +static bool ep_put(struct eventpoll *ep) { if (!refcount_dec_and_test(&ep->refcount)) return false; @@ -817,22 +1046,23 @@ static void ep_free(struct eventpoll *ep) } /* - * The ffd.file pointer may be in the process of being torn down due to - * being closed, but we may not have finished eventpoll_release() yet. - * - * Normally, even with the atomic_long_inc_not_zero, the file may have - * been free'd and then gotten re-allocated to something else (since - * files are not RCU-delayed, they are SLAB_TYPESAFE_BY_RCU). - * - * But for epoll, users hold the ep->mtx mutex, and as such any file in - * the process of being free'd will block in eventpoll_release_file() - * and thus the underlying file allocation will not be free'd, and the - * file re-use cannot happen. - * - * For the same reason we can avoid a rcu_read_lock() around the - * operation - 'ffd.file' cannot go away even if the refcount has - * reached zero (but we must still not call out to ->poll() functions - * etc). + * Pin @epi->ffd.file for operations that require both safe dereference + * and exclusion from __fput(). + * + * struct file uses SLAB_TYPESAFE_BY_RCU, so a freed slot can be + * reassigned at any time. The bare load of epi->ffd.file is safe here + * because the caller holds ep->mtx and eventpoll_release_file() blocks + * on that mutex while tearing down the epi, so the backing file + * allocation cannot be freed and reused under us. An rcu_read_lock() + * is therefore unnecessary for the load. + * + * A successful file_ref_get() additionally blocks __fput() from + * starting on this file: once the refcount has reached zero it cannot + * come back. ep_remove() relies on that to touch file->f_lock and + * file->f_ep without racing eventpoll_release_file() (see commit + * a6dc643c6931). A NULL return means __fput() is already in flight; + * the caller must bail without touching the file, and + * eventpoll_release_file() will clean the epi up from its side. */ static struct file *epi_fget(const struct epitem *epi) { @@ -858,7 +1088,13 @@ static void ep_remove_file(struct eventpoll *ep, struct epitem *epi, spin_lock(&file->f_lock); head = file->f_ep; if (hlist_is_singular_node(&epi->fllink, head)) { - /* See eventpoll_release() for details. */ + /* + * Last watcher: publish NULL so the eventpoll_release() + * fastpath in include/linux/eventpoll.h can skip the slow + * path on a future __fput(). Safe because every f_ep writer + * either holds a pin on @file via epi_fget() or is __fput() + * itself -- see the comment in eventpoll_release(). + */ WRITE_ONCE(file->f_ep, NULL); if (!is_file_epoll(file)) { struct epitems_head *v; @@ -919,47 +1155,82 @@ static void ep_remove(struct eventpoll *ep, struct epitem *epi) ep_remove_file(ep, epi, file); ep_remove_epi(ep, epi); - WARN_ON_ONCE(ep_refcount_dec_and_test(ep)); + WARN_ON_ONCE(ep_put(ep)); } -static void ep_clear_and_put(struct eventpoll *ep) +/* + * Pass 1 of ep_clear_and_put(): drain every epi's pwqlist. + * ep_unregister_pollwait() takes each watched wait-queue head's lock, + * which synchronizes with any in-flight ep_poll_callback(); after + * this returns no callback can still be about to dereference an epi + * on this ep. Must strictly precede ep_drain_tree() -- fusing the + * two walks would let a callback queued on epi_i still fire after + * epi_{i+k} had already been freed. + */ +static void ep_drain_pollwaits(struct eventpoll *ep) { - struct rb_node *rbp, *next; + struct rb_node *rbp; struct epitem *epi; - /* We need to release all tasks waiting for these file */ - if (waitqueue_active(&ep->poll_wait)) - ep_poll_safewake(ep, NULL, 0); - - mutex_lock(&ep->mtx); + lockdep_assert_held(&ep->mtx); - /* - * Walks through the whole tree by unregistering poll callbacks. - */ for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { epi = rb_entry(rbp, struct epitem, rbn); ep_unregister_pollwait(ep, epi); cond_resched(); } +} + +/* + * Pass 2 of ep_clear_and_put(): ep_remove() every epi. The per-epi + * pwqlist is already empty (ep_drain_pollwaits ran), but the rest of + * ep_remove() still runs: epi_fget() pin, f_ep clear under f_lock, + * rbtree erase, rdllist unlink, kfree_rcu(epi). rb_next() is captured + * before each erase so the iteration is stable. + * + * A concurrent eventpoll_release_file() (removal path C) on a watched + * file serializes with us via ep->mtx; ep_remove() transparently + * hands off any epi whose file is in __fput() by bailing when + * epi_fget() returns NULL, and path C will clean that epi up. + */ +static void ep_drain_tree(struct eventpoll *ep) +{ + struct rb_node *rbp, *next; + struct epitem *epi; + + lockdep_assert_held(&ep->mtx); - /* - * Walks through the whole tree and try to free each "struct epitem". - * Note that ep_remove() will not remove the epitem in case of a - * racing eventpoll_release_file(); the latter will do the removal. - * At this point we are sure no poll callbacks will be lingering around. - * Since we still own a reference to the eventpoll struct, the loop can't - * dispose it. - */ for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) { next = rb_next(rbp); epi = rb_entry(rbp, struct epitem, rbn); ep_remove(ep, epi); cond_resched(); } +} + +/* + * Removal path B (see "Removal paths" in the top-of-file banner): + * close of the epoll fd itself, reached via ep_eventpoll_release(). + * + * Two passes under ep->mtx: first ep_drain_pollwaits() quiesces + * in-flight callbacks, then ep_drain_tree() frees the epis. The + * ep->refcount is kept > 0 across the walk by the ep file's own + * share, which we drop below; ep_free() runs iff we were the last + * holder after the tree drained. + */ +static void ep_clear_and_put(struct eventpoll *ep) +{ + /* Release any threads blocked in poll-on-ep. */ + if (waitqueue_active(&ep->poll_wait)) + ep_poll_safewake(ep, NULL, 0); + mutex_lock(&ep->mtx); + ep_drain_pollwaits(ep); + ep_drain_tree(ep); mutex_unlock(&ep->mtx); - if (ep_refcount_dec_and_test(ep)) + + if (ep_put(ep)) ep_free(ep); } @@ -999,7 +1270,7 @@ static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth) { struct eventpoll *ep = file->private_data; - LIST_HEAD(txlist); + LIST_HEAD(scan_batch); struct epitem *epi, *tmp; poll_table pt; __poll_t res = 0; @@ -1014,8 +1285,8 @@ static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int dep * the ready list. */ mutex_lock_nested(&ep->mtx, depth); - ep_start_scan(ep, &txlist); - list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { + ep_start_scan(ep, &scan_batch); + list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) { if (ep_item_poll(epi, &pt, depth + 1)) { res = EPOLLIN | EPOLLRDNORM; break; @@ -1029,7 +1300,7 @@ static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int dep list_del_init(&epi->rdllink); } } - ep_done_scan(ep, &txlist); + ep_done_scan(ep, &scan_batch); mutex_unlock(&ep->mtx); return res; } @@ -1138,7 +1409,7 @@ again: mutex_unlock(&ep->mtx); - if (ep_refcount_dec_and_test(ep)) + if (ep_put(ep)) ep_free(ep); goto again; } @@ -1155,11 +1426,12 @@ static int ep_alloc(struct eventpoll **pep) mutex_init(&ep->mtx); spin_lock_init(&ep->lock); + seqcount_spinlock_init(&ep->seq, &ep->lock); init_waitqueue_head(&ep->wq); init_waitqueue_head(&ep->poll_wait); INIT_LIST_HEAD(&ep->rdllist); ep->rbr = RB_ROOT_CACHED; - ep->ovflist = EP_UNACTIVE_PTR; + ep->ovflist = EP_UNACTIVE_PTR; /* not scanning */ ep->user = get_current_user(); refcount_set(&ep->refcount, 1); @@ -1173,17 +1445,15 @@ static int ep_alloc(struct eventpoll **pep) * are protected by the "mtx" mutex, and ep_find() must be called with * "mtx" held. */ -static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd) +static struct epitem *ep_find(struct eventpoll *ep, struct epoll_key *tf) { int kcmp; struct rb_node *rbp; struct epitem *epi, *epir = NULL; - struct epoll_filefd ffd; - ep_set_ffd(&ffd, file, fd); for (rbp = ep->rbr.rb_root.rb_node; rbp; ) { epi = rb_entry(rbp, struct epitem, rbn); - kcmp = ep_cmp_ffd(&ffd, &epi->ffd); + kcmp = ep_cmp_ffd(tf, &epi->ffd); if (kcmp > 0) rbp = rbp->rb_right; else if (kcmp < 0) @@ -1197,50 +1467,6 @@ static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd) return epir; } -#ifdef CONFIG_KCMP -static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff) -{ - struct rb_node *rbp; - struct epitem *epi; - - for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { - epi = rb_entry(rbp, struct epitem, rbn); - if (epi->ffd.fd == tfd) { - if (toff == 0) - return epi; - else - toff--; - } - cond_resched(); - } - - return NULL; -} - -struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, - unsigned long toff) -{ - struct file *file_raw; - struct eventpoll *ep; - struct epitem *epi; - - if (!is_file_epoll(file)) - return ERR_PTR(-EINVAL); - - ep = file->private_data; - - mutex_lock(&ep->mtx); - epi = ep_find_tfd(ep, tfd, toff); - if (epi) - file_raw = epi->ffd.file; - else - file_raw = ERR_PTR(-ENOENT); - mutex_unlock(&ep->mtx); - - return file_raw; -} -#endif /* CONFIG_KCMP */ - /* * This is the callback that is passed to the wait queue wakeup * mechanism. It is called by the stored file descriptors when they @@ -1283,9 +1509,9 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v * semantics). All the events that happen during that period of time are * chained in ep->ovflist and requeued later on. */ - if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) { - if (epi->next == EP_UNACTIVE_PTR) { - epi->next = READ_ONCE(ep->ovflist); + if (ep_is_scanning(ep)) { + if (!epi_on_ovflist(epi)) { + epi->ovflist_next = READ_ONCE(ep->ovflist); WRITE_ONCE(ep->ovflist, epi); ep_pm_stay_awake_rcu(epi); } @@ -1336,17 +1562,24 @@ out_unlock: if (pollflags & POLLFREE) { /* - * If we race with ep_remove_wait_queue() it can miss - * ->whead = NULL and do another remove_wait_queue() after - * us, so we can't use __remove_wait_queue(). + * POLLFREE handshake, release side; see "POLLFREE handshake" + * at the top of this file. + * + * Unlink our wait entry with list_del_init rather than + * __remove_wait_queue: a concurrent ep_remove_wait_queue() + * that already loaded a non-NULL whead may still call + * remove_wait_queue() after us, and list_del_init() tolerates + * the second delete. + * + * smp_store_release(&whead, NULL) publishes the teardown to + * ep_remove_wait_queue()'s smp_load_acquire(). Before this + * store, a racing ep_clear_and_put() / ep_remove() reaches + * ep_remove_wait_queue() which sees whead != NULL and takes + * whead->lock -- the same lock held by our caller, so it + * serializes behind us. Once whead is zeroed, nothing else + * protects ep / epi / wait. */ list_del_init(&wait->entry); - /* - * ->whead != NULL protects us from the race with - * ep_clear_and_put() or ep_remove(), ep_remove_wait_queue() - * takes whead->lock held by the caller. Once we nullify it, - * nothing protects ep/epi or even wait. - */ smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL); } @@ -1407,41 +1640,40 @@ static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) -#define PATH_ARR_SIZE 5 /* - * These are the number paths of length 1 to 5, that we are allowing to emanate - * from a single file of interest. For example, we allow 1000 paths of length - * 1, to emanate from each file of interest. This essentially represents the - * potential wakeup paths, which need to be limited in order to avoid massive - * uncontrolled wakeup storms. The common use case should be a single ep which - * is connected to n file sources. In this case each file source has 1 path - * of length 1. Thus, the numbers below should be more than sufficient. These - * path limits are enforced during an EPOLL_CTL_ADD operation, since a modify - * and delete can't add additional paths. Protected by the epnested_mutex. + * Upper bound on wakeup paths emanating from any one watched file, + * indexed by path depth (1..PATH_ARR_SIZE). For example, we allow + * 1000 paths of length 1 from each watched file. These caps limit + * the wakeup amplification that can be built from epoll-watches- + * epoll topologies without rejecting reasonable usage. + * + * Enforced at EPOLL_CTL_ADD; CTL_MOD and CTL_DEL cannot add paths. + * The running tallies live in ctx->path_count[] and are protected by + * epnested_mutex. */ static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 }; -static int path_count[PATH_ARR_SIZE]; -static int path_count_inc(int nests) +static int path_count_inc(struct ep_ctl_ctx *ctx, int nests) { /* Allow an arbitrary number of depth 1 paths */ if (nests == 0) return 0; - if (++path_count[nests] > path_limits[nests]) + if (++ctx->path_count[nests] > path_limits[nests]) return -1; return 0; } -static void path_count_init(void) +static void path_count_init(struct ep_ctl_ctx *ctx) { int i; for (i = 0; i < PATH_ARR_SIZE; i++) - path_count[i] = 0; + ctx->path_count[i] = 0; } -static int reverse_path_check_proc(struct hlist_head *refs, int depth) +static int reverse_path_check_proc(struct ep_ctl_ctx *ctx, + struct hlist_head *refs, int depth) { int error = 0; struct epitem *epi; @@ -1453,9 +1685,9 @@ static int reverse_path_check_proc(struct hlist_head *refs, int depth) hlist_for_each_entry_rcu(epi, refs, fllink) { struct hlist_head *refs = &epi->ep->refs; if (hlist_empty(refs)) - error = path_count_inc(depth); + error = path_count_inc(ctx, depth); else - error = reverse_path_check_proc(refs, depth + 1); + error = reverse_path_check_proc(ctx, refs, depth + 1); if (error != 0) break; } @@ -1463,24 +1695,24 @@ static int reverse_path_check_proc(struct hlist_head *refs, int depth) } /** - * reverse_path_check - The tfile_check_list is list of epitem_head, which have - * links that are proposed to be newly added. We need to - * make sure that those added links don't add too many - * paths such that we will spend all our time waking up - * eventpoll objects. + * reverse_path_check - ctx->tfile_check_list is a list of epitems_head + * anchoring files with newly proposed links; make + * sure those links don't push any path-length bucket + * over its limit in path_limits[]. + * @ctx: Per-do_epoll_ctl() scratch for the loop / path checks. * * Return: %zero if the proposed links don't create too many paths, * %-1 otherwise. */ -static int reverse_path_check(void) +static int reverse_path_check(struct ep_ctl_ctx *ctx) { struct epitems_head *p; - for (p = tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) { + for (p = ctx->tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) { int error; - path_count_init(); + path_count_init(ctx); rcu_read_lock(); - error = reverse_path_check_proc(&p->epitems, 0); + error = reverse_path_check_proc(ctx, &p->epitems, 0); rcu_read_unlock(); if (error) return error; @@ -1526,7 +1758,7 @@ static noinline void ep_destroy_wakeup_source(struct epitem *epi) wakeup_source_unregister(ws); } -static int attach_epitem(struct file *file, struct epitem *epi) +static int ep_attach_file(struct file *file, struct epitem *epi) { struct epitems_head *to_free = NULL; struct hlist_head *head = NULL; @@ -1561,69 +1793,115 @@ allocate: } /* - * Must be called with "mtx" held. + * Charge the user's epoll_watches quota, allocate a fresh epitem for + * @tf, and initialize its fields. The returned item is not yet linked + * into any data structure; the caller must install it via + * ep_register_epitem() (which takes over on success) or kmem_cache_free() + * it and decrement epoll_watches on its own. + * + * Returns ERR_PTR(-ENOSPC) if the quota is exceeded, ERR_PTR(-ENOMEM) + * if the slab allocation fails. */ -static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, - struct file *tfile, int fd, int full_check) +static struct epitem *ep_alloc_epitem(struct eventpoll *ep, + const struct epoll_event *event, + struct epoll_key *tf) { - int error, pwake = 0; - __poll_t revents; struct epitem *epi; - struct ep_pqueue epq; - struct eventpoll *tep = NULL; - - if (is_file_epoll(tfile)) - tep = tfile->private_data; - - lockdep_assert_irqs_enabled(); if (unlikely(percpu_counter_compare(&ep->user->epoll_watches, max_user_watches) >= 0)) - return -ENOSPC; + return ERR_PTR(-ENOSPC); percpu_counter_inc(&ep->user->epoll_watches); - if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) { + epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL); + if (unlikely(!epi)) { percpu_counter_dec(&ep->user->epoll_watches); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } - /* Item initialization follow here ... */ INIT_LIST_HEAD(&epi->rdllink); epi->ep = ep; - ep_set_ffd(&epi->ffd, tfile, fd); + epi->ffd = *tf; epi->event = *event; - epi->next = EP_UNACTIVE_PTR; + epi_clear_ovflist(epi); + + return epi; +} + +/* + * Install @epi into its target file's f_ep hlist and into @ep's rbtree, + * taking one additional reference on @ep for the lifetime of the item. + * + * If @tep is non-NULL, the target file is itself an eventpoll; we hold + * tep->mtx at subclass 1 across the attach + rbtree insert to serialize + * with the target side. RB tree ops are protected by @ep->mtx, which + * the caller already holds. + * + * On failure the epi is freed and the epoll_watches counter decremented, + * matching ep_alloc_epitem()'s allocation. After this returns + * successfully, ep_insert()'s later error paths use ep_remove() for + * unwind; that cannot drop @ep's refcount to zero because the ep file + * itself still holds the original reference. + */ +static int ep_register_epitem(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + struct epitem *epi, struct eventpoll *tep, + int full_check) +{ + struct file *tfile = epi->ffd.file; + int error; if (tep) mutex_lock_nested(&tep->mtx, 1); - /* Add the current item to the list of active epoll hook for this file */ - if (unlikely(attach_epitem(tfile, epi) < 0)) { + + error = ep_attach_file(tfile, epi); + if (unlikely(error)) { if (tep) mutex_unlock(&tep->mtx); kmem_cache_free(epi_cache, epi); percpu_counter_dec(&ep->user->epoll_watches); - return -ENOMEM; + return error; } if (full_check && !tep) - list_file(tfile); + list_file(tfile, ctx); - /* - * Add the current item to the RB tree. All RB tree operations are - * protected by "mtx", and ep_insert() is called with "mtx" held. - */ ep_rbtree_insert(ep, epi); + if (tep) mutex_unlock(&tep->mtx); - /* - * ep_remove() calls in the later error paths can't lead to - * ep_free() as the ep file itself still holds an ep reference. - */ ep_get(ep); + return 0; +} + +/* + * Must be called with "mtx" held. + */ +static int ep_insert(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + const struct epoll_event *event, struct epoll_key *tf, + int full_check) +{ + int error, pwake = 0; + __poll_t revents; + struct epitem *epi; + struct ep_pqueue epq; + struct eventpoll *tep = NULL; + + if (is_file_epoll(tf->file)) + tep = tf->file->private_data; + + lockdep_assert_irqs_enabled(); + + epi = ep_alloc_epitem(ep, event, tf); + if (IS_ERR(epi)) + return PTR_ERR(epi); + + error = ep_register_epitem(ctx, ep, epi, tep, full_check); + if (error) + return error; - /* now check if we've created too many backpaths */ - if (unlikely(full_check && reverse_path_check())) { + /* Reject the insert if the new link would create too many back-paths. */ + if (unlikely(full_check && reverse_path_check(ctx))) { ep_remove(ep, epi); return -EINVAL; } @@ -1649,28 +1927,21 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, */ revents = ep_item_poll(epi, &epq.pt, 1); - /* - * We have to check if something went wrong during the poll wait queue - * install process. Namely an allocation for a wait queue failed due - * high memory pressure. - */ + /* ep_ptable_queue_proc() signals allocation failure by clearing epq.epi. */ if (unlikely(!epq.epi)) { ep_remove(ep, epi); return -ENOMEM; } - /* We have to drop the new item inside our item list to keep track of it */ + /* Drop the new item onto the ready list if it is already ready. */ spin_lock_irq(&ep->lock); - /* record NAPI ID of new item if present */ ep_set_busy_poll_napi_id(epi); - /* If the file is already "ready" we drop it inside the ready list */ if (revents && !ep_is_linked(epi)) { list_add_tail(&epi->rdllink, &ep->rdllist); ep_pm_stay_awake(epi); - /* Notify waiting tasks that events are available */ if (waitqueue_active(&ep->wq)) wake_up(&ep->wq); if (waitqueue_active(&ep->poll_wait)) @@ -1762,11 +2033,87 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi, return 0; } +/* + * Attempt to deliver one event for @epi into @*uevents. + * + * Returns 1 if an event was delivered (with *uevents advanced to the + * next slot), 0 if the re-poll reported no caller-requested events + * (@epi drops out of the ready list; a future callback will re-add + * it), or -EFAULT if copy_to_user() faulted (in which case @epi is + * re-inserted at the head of @scan_batch so ep_done_scan() merges it + * back to rdllist for the next attempt). + * + * PM bookkeeping and level-triggered re-queue are handled here. + * Caller holds ep->mtx and the scan is active. + */ +static int ep_deliver_event(struct eventpoll *ep, struct epitem *epi, + poll_table *pt, + struct epoll_event __user **uevents, + struct list_head *scan_batch) +{ + struct epoll_event __user *next; + struct wakeup_source *ws; + __poll_t revents; + + /* + * Activate ep->ws before deactivating epi->ws to prevent + * triggering auto-suspend here (in case we reactivate epi->ws + * below). Rearranging to delay the deactivation would let + * epi->ws drift out of sync with ep_is_linked(). + */ + ws = ep_wakeup_source(epi); + if (ws) { + if (ws->active) + __pm_stay_awake(ep->ws); + __pm_relax(ws); + } + + list_del_init(&epi->rdllink); + + /* + * Re-poll under ep->mtx so userspace cannot change the item + * out from under us. If no caller-requested events remain, + * @epi stays off the ready list; the poll callback will + * re-queue it when events next appear. + */ + revents = ep_item_poll(epi, pt, 1); + if (!revents) + return 0; + + next = epoll_put_uevent(revents, epi->event.data, *uevents); + if (!next) { + /* + * copy_to_user() faulted: put the item back so + * ep_done_scan() splices it onto rdllist for the next + * attempt. + */ + list_add(&epi->rdllink, scan_batch); + ep_pm_stay_awake(epi); + return -EFAULT; + } + *uevents = next; + + if (epi->event.events & EPOLLONESHOT) { + epi->event.events &= EP_PRIVATE_BITS; + } else if (!(epi->event.events & EPOLLET)) { + /* + * Level-triggered: re-queue so the next epoll_wait() + * rechecks availability. We are the sole writer to + * rdllist here -- epoll_ctl() callers are locked out + * by ep->mtx, and the poll callback queues to ovflist + * during scans. + */ + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake(epi); + } + return 1; +} + static int ep_send_events(struct eventpoll *ep, struct epoll_event __user *events, int maxevents) { struct epitem *epi, *tmp; - LIST_HEAD(txlist); + LIST_HEAD(scan_batch); poll_table pt; int res = 0; @@ -1781,74 +2128,28 @@ static int ep_send_events(struct eventpoll *ep, init_poll_funcptr(&pt, NULL); mutex_lock(&ep->mtx); - ep_start_scan(ep, &txlist); + ep_start_scan(ep, &scan_batch); /* - * We can loop without lock because we are passed a task private list. - * Items cannot vanish during the loop we are holding ep->mtx. + * We can loop without lock because we are passed a task-private + * scan_batch; items cannot vanish while we hold ep->mtx. */ - list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { - struct wakeup_source *ws; - __poll_t revents; + list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) { + int delivered; if (res >= maxevents) break; - /* - * Activate ep->ws before deactivating epi->ws to prevent - * triggering auto-suspend here (in case we reactive epi->ws - * below). - * - * This could be rearranged to delay the deactivation of epi->ws - * instead, but then epi->ws would temporarily be out of sync - * with ep_is_linked(). - */ - ws = ep_wakeup_source(epi); - if (ws) { - if (ws->active) - __pm_stay_awake(ep->ws); - __pm_relax(ws); - } - - list_del_init(&epi->rdllink); - - /* - * If the event mask intersect the caller-requested one, - * deliver the event to userspace. Again, we are holding ep->mtx, - * so no operations coming from userspace can change the item. - */ - revents = ep_item_poll(epi, &pt, 1); - if (!revents) - continue; - - events = epoll_put_uevent(revents, epi->event.data, events); - if (!events) { - list_add(&epi->rdllink, &txlist); - ep_pm_stay_awake(epi); + delivered = ep_deliver_event(ep, epi, &pt, &events, &scan_batch); + if (delivered < 0) { if (!res) - res = -EFAULT; + res = delivered; break; } - res++; - if (epi->event.events & EPOLLONESHOT) - epi->event.events &= EP_PRIVATE_BITS; - else if (!(epi->event.events & EPOLLET)) { - /* - * If this file has been added with Level - * Trigger mode, we need to insert back inside - * the ready list, so that the next call to - * epoll_wait() will check again the events - * availability. At this point, no one can insert - * into ep->rdllist besides us. The epoll_ctl() - * callers are locked out by - * ep_send_events() holding "mtx" and the - * poll callback will queue them in ep->ovflist. - */ - list_add_tail(&epi->rdllink, &ep->rdllist); - ep_pm_stay_awake(epi); - } + res += delivered; } - ep_done_scan(ep, &txlist); + + ep_done_scan(ep, &scan_batch); mutex_unlock(&ep->mtx); return res; @@ -1938,7 +2239,8 @@ static int ep_schedule_timeout(ktime_t *to) static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, int maxevents, struct timespec64 *timeout) { - int res, eavail, timed_out = 0; + int res, timed_out = 0; + bool eavail; u64 slack = 0; wait_queue_entry_t wait; ktime_t expires, *to = NULL; @@ -2036,7 +2338,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, * If timed out and still on the wait queue, recheck eavail * carefully under lock, below. */ - eavail = 1; + eavail = true; if (!list_empty_careful(&wait.entry)) { spin_lock_irq(&ep->lock); @@ -2060,13 +2362,15 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, * epoll file does not create closed loops, and * determine the depth of the subtree starting at @ep * + * @ctx: Per-do_epoll_ctl() scratch for the loop / path checks. * @ep: the &struct eventpoll to be currently checked. * @depth: Current depth of the path being checked. * * Return: depth of the subtree, or a value bigger than EP_MAX_NESTS if we found * a loop or went too deep. */ -static int ep_loop_check_proc(struct eventpoll *ep, int depth) +static int ep_loop_check_proc(struct ep_ctl_ctx *ctx, + struct eventpoll *ep, int depth) { int result = 0; struct rb_node *rbp; @@ -2082,22 +2386,23 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth) if (unlikely(is_file_epoll(epi->ffd.file))) { struct eventpoll *ep_tovisit; ep_tovisit = epi->ffd.file->private_data; - if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS) + if (ep_tovisit == ctx->inserting_into || + depth > EP_MAX_NESTS) result = EP_MAX_NESTS+1; else - result = max(result, ep_loop_check_proc(ep_tovisit, depth + 1) + 1); + result = max(result, + ep_loop_check_proc(ctx, ep_tovisit, + depth + 1) + 1); if (result > EP_MAX_NESTS) break; } else { /* - * If we've reached a file that is not associated with - * an ep, then we need to check if the newly added - * links are going to add too many wakeup paths. We do - * this by adding it to the tfile_check_list, if it's - * not already there, and calling reverse_path_check() - * during ep_insert(). + * A non-epoll leaf. Queue it for the companion + * reverse_path_check() that runs after this walk so + * any new links we propose don't add too many wakeup + * paths. */ - list_file(epi->ffd.file); + list_file(epi->ffd.file, ctx); } } ep->loop_check_depth = result; @@ -2126,22 +2431,24 @@ static int ep_get_upwards_depth_proc(struct eventpoll *ep, int depth) * into another epoll file (represented by @ep) does not create * closed loops or too deep chains. * - * @ep: Pointer to the epoll we are inserting into. - * @to: Pointer to the epoll to be inserted. + * @ctx: Per-CTL_ADD scratch context. + * @ep: Pointer to the epoll we are inserting into. + * @to: Pointer to the epoll to be inserted. * * Return: %zero if adding the epoll @to inside the epoll @from * does not violate the constraints, or %-1 otherwise. */ -static int ep_loop_check(struct eventpoll *ep, struct eventpoll *to) +static int ep_loop_check(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + struct eventpoll *to) { int depth, upwards_depth; - inserting_into = ep; + ctx->inserting_into = ep; /* * Check how deep down we can get from @to, and whether it is possible * to loop up to @ep. */ - depth = ep_loop_check_proc(to, 0); + depth = ep_loop_check_proc(ctx, to, 0); if (depth > EP_MAX_NESTS) return -1; /* Check how far up we can go from @ep. */ @@ -2152,12 +2459,12 @@ static int ep_loop_check(struct eventpoll *ep, struct eventpoll *to) return (depth+1+upwards_depth > EP_MAX_NESTS) ? -1 : 0; } -static void clear_tfile_check_list(void) +static void clear_tfile_check_list(struct ep_ctl_ctx *ctx) { rcu_read_lock(); - while (tfile_check_list != EP_UNACTIVE_PTR) { - struct epitems_head *head = tfile_check_list; - tfile_check_list = head->next; + while (ctx->tfile_check_list != EP_UNACTIVE_PTR) { + struct epitems_head *head = ctx->tfile_check_list; + ctx->tfile_check_list = head->next; unlist_file(head); } rcu_read_unlock(); @@ -2223,38 +2530,107 @@ static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev) } #endif -static inline int epoll_mutex_lock(struct mutex *mutex, int depth, - bool nonblock) +static inline int epoll_mutex_lock(struct mutex *mutex, bool nonblock) { if (!nonblock) { - mutex_lock_nested(mutex, depth); + mutex_lock(mutex); return 0; } - if (mutex_trylock(mutex)) + return mutex_trylock(mutex) ? 0 : -EAGAIN; +} + +/* + * Acquire the locks required for do_epoll_ctl() on @ep for @op. + * + * Always takes ep->mtx. For EPOLL_CTL_ADD, additionally runs the + * loop / path check under epnested_mutex when the topology can + * change: @ep is already watched (epfile->f_ep non-NULL), @ep was + * recently loop-checked (ep->gen == loop_check_gen), or @tfile is + * itself an eventpoll. + * + * Return value encodes both outcome and lock state: + * + * 0 success; ep->mtx held. + * 1 success; ep->mtx held AND the full check ran under + * epnested_mutex (which is also still held). The value + * doubles as the @full_check argument to ep_insert(). + * -errno failure; no locks held. + * + * The caller releases what was taken with ep_ctl_unlock(ep, ret). + * + * Holding epnested_mutex on add is what prevents two racing + * EPOLL_CTL_ADDs on different eps from building a cycle without + * either walker observing it. + */ +static int ep_ctl_lock(struct ep_ctl_ctx *ctx, struct eventpoll *ep, int op, + struct file *epfile, struct file *tfile, bool nonblock) +{ + struct eventpoll *tep; + int error; + + error = epoll_mutex_lock(&ep->mtx, nonblock); + if (error) + return error; + + if (op != EPOLL_CTL_ADD) + return 0; + if (!READ_ONCE(epfile->f_ep) && ep->gen != loop_check_gen && + !is_file_epoll(tfile)) return 0; - return -EAGAIN; + + /* Full check needed: drop ep->mtx so we can take epnested_mutex. */ + mutex_unlock(&ep->mtx); + error = epoll_mutex_lock(&epnested_mutex, nonblock); + if (error) + return error; + + loop_check_gen++; + + if (is_file_epoll(tfile)) { + tep = tfile->private_data; + if (ep_loop_check(ctx, ep, tep) != 0) { + error = -ELOOP; + goto err_unlock_nested; + } + } + + error = epoll_mutex_lock(&ep->mtx, nonblock); + if (error) + goto err_unlock_nested; + + return 1; + +err_unlock_nested: + clear_tfile_check_list(ctx); + loop_check_gen++; + mutex_unlock(&epnested_mutex); + return error; } -int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, - bool nonblock) +static void ep_ctl_unlock(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + int full_check) +{ + mutex_unlock(&ep->mtx); + if (full_check) { + clear_tfile_check_list(ctx); + loop_check_gen++; + mutex_unlock(&epnested_mutex); + } +} + +int do_epoll_ctl_file(struct file *f, int op, struct epoll_key *tf, + struct epoll_event *epds, bool nonblock) { int error; - int full_check = 0; + int full_check; struct eventpoll *ep; struct epitem *epi; - struct eventpoll *tep = NULL; - - CLASS(fd, f)(epfd); - if (fd_empty(f)) - return -EBADF; - - /* Get the "struct file *" for the target file */ - CLASS(fd, tf)(fd); - if (fd_empty(tf)) - return -EBADF; + struct ep_ctl_ctx ctx = { + .tfile_check_list = EP_UNACTIVE_PTR, + }; /* The target file descriptor must support poll */ - if (!file_can_poll(fd_file(tf))) + if (!file_can_poll(tf->file)) return -EPERM; /* Check if EPOLLWAKEUP is allowed */ @@ -2262,85 +2638,43 @@ int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, ep_take_care_of_epollwakeup(epds); /* - * We have to check that the file structure underneath the file descriptor - * the user passed to us _is_ an eventpoll file. And also we do not permit + * The @f file must itself be an eventpoll, and we do not permit * adding an epoll file descriptor inside itself. */ - error = -EINVAL; - if (fd_file(f) == fd_file(tf) || !is_file_epoll(fd_file(f))) - goto error_tgt_fput; + if (f == tf->file || !is_file_epoll(f)) + return -EINVAL; /* * epoll adds to the wakeup queue at EPOLL_CTL_ADD time only, * so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation. - * Also, we do not currently supported nested exclusive wakeups. + * Also, nested exclusive wakeups are not supported. */ if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) { if (op == EPOLL_CTL_MOD) - goto error_tgt_fput; - if (op == EPOLL_CTL_ADD && (is_file_epoll(fd_file(tf)) || + return -EINVAL; + if (op == EPOLL_CTL_ADD && (is_file_epoll(tf->file) || (epds->events & ~EPOLLEXCLUSIVE_OK_BITS))) - goto error_tgt_fput; + return -EINVAL; } - /* - * At this point it is safe to assume that the "private_data" contains - * our own data structure. - */ - ep = fd_file(f)->private_data; + ep = f->private_data; - /* - * When we insert an epoll file descriptor inside another epoll file - * descriptor, there is the chance of creating closed loops, which are - * better be handled here, than in more critical paths. While we are - * checking for loops we also determine the list of files reachable - * and hang them on the tfile_check_list, so we can check that we - * haven't created too many possible wakeup paths. - * - * We do not need to take the global 'epumutex' on EPOLL_CTL_ADD when - * the epoll file descriptor is attaching directly to a wakeup source, - * unless the epoll file descriptor is nested. The purpose of taking the - * 'epnested_mutex' on add is to prevent complex toplogies such as loops and - * deep wakeup paths from forming in parallel through multiple - * EPOLL_CTL_ADD operations. - */ - error = epoll_mutex_lock(&ep->mtx, 0, nonblock); - if (error) - goto error_tgt_fput; - if (op == EPOLL_CTL_ADD) { - if (READ_ONCE(fd_file(f)->f_ep) || ep->gen == loop_check_gen || - is_file_epoll(fd_file(tf))) { - mutex_unlock(&ep->mtx); - error = epoll_mutex_lock(&epnested_mutex, 0, nonblock); - if (error) - goto error_tgt_fput; - loop_check_gen++; - full_check = 1; - if (is_file_epoll(fd_file(tf))) { - tep = fd_file(tf)->private_data; - error = -ELOOP; - if (ep_loop_check(ep, tep) != 0) - goto error_tgt_fput; - } - error = epoll_mutex_lock(&ep->mtx, 0, nonblock); - if (error) - goto error_tgt_fput; - } - } + full_check = ep_ctl_lock(&ctx, ep, op, f, tf->file, nonblock); + if (full_check < 0) + return full_check; /* - * Try to lookup the file inside our RB tree. Since we grabbed "mtx" - * above, we can be sure to be able to use the item looked up by - * ep_find() till we release the mutex. + * Look the target up in ep's RB tree. We hold ep->mtx, so the + * item stays valid until we release. */ - epi = ep_find(ep, fd_file(tf), fd); + epi = ep_find(ep, tf); error = -EINVAL; switch (op) { case EPOLL_CTL_ADD: if (!epi) { epds->events |= EPOLLERR | EPOLLHUP; - error = ep_insert(ep, epds, fd_file(tf), fd, full_check); + error = ep_insert(&ctx, ep, epds, tf, full_check); } else error = -EEXIST; break; @@ -2366,17 +2700,30 @@ int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, error = -ENOENT; break; } - mutex_unlock(&ep->mtx); -error_tgt_fput: - if (full_check) { - clear_tfile_check_list(); - loop_check_gen++; - mutex_unlock(&epnested_mutex); - } + ep_ctl_unlock(&ctx, ep, full_check); return error; } +int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, + bool nonblock) +{ + struct epoll_key efd; + + CLASS(fd, f)(epfd); + if (fd_empty(f)) + return -EBADF; + + /* Get the "struct file *" for the target file */ + CLASS(fd, tf)(fd); + if (fd_empty(tf)) + return -EBADF; + + efd.file = fd_file(tf); + efd.fd = fd; + return do_epoll_ctl_file(fd_file(f), op, &efd, epds, nonblock); +} + /* * The following function implements the controller interface for * the eventpoll file that enables the insertion/removal/change of @@ -2527,6 +2874,50 @@ SYSCALL_DEFINE6(epoll_pwait2, int, epfd, struct epoll_event __user *, events, sigmask, sigsetsize); } +#ifdef CONFIG_KCMP +static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff) +{ + struct rb_node *rbp; + struct epitem *epi; + + for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { + epi = rb_entry(rbp, struct epitem, rbn); + if (epi->ffd.fd == tfd) { + if (toff == 0) + return epi; + else + toff--; + } + cond_resched(); + } + + return NULL; +} + +struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, + unsigned long toff) +{ + struct file *file_raw; + struct eventpoll *ep; + struct epitem *epi; + + if (!is_file_epoll(file)) + return ERR_PTR(-EINVAL); + + ep = file->private_data; + + mutex_lock(&ep->mtx); + epi = ep_find_tfd(ep, tfd, toff); + if (epi) + file_raw = epi->ffd.file; + else + file_raw = ERR_PTR(-ENOENT); + mutex_unlock(&ep->mtx); + + return file_raw; +} +#endif /* CONFIG_KCMP */ + #ifdef CONFIG_COMPAT static int do_compat_epoll_pwait(int epfd, struct epoll_event __user *events, int maxevents, struct timespec64 *timeout, diff --git a/fs/exec.c b/fs/exec.c index ba12b4c466f6..d5993cedc829 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -35,6 +35,7 @@ #include <linux/init.h> #include <linux/sched/mm.h> #include <linux/sched/coredump.h> +#include <linux/sched/exec_state.h> #include <linux/sched/signal.h> #include <linux/sched/numa_balancing.h> #include <linux/sched/task.h> @@ -263,6 +264,9 @@ static int bprm_mm_init(struct linux_binprm *bprm) if (!mm) goto err; + /* Staged for would_dump() narrowing; consumed by begin_new_exec(). */ + bprm->user_ns = get_user_ns(current_user_ns()); + /* Save current stack limit for all calculations made during exec. */ task_lock(current->group_leader); bprm->rlim_stack = current->signal->rlim[RLIMIT_STACK]; @@ -832,14 +836,21 @@ EXPORT_SYMBOL(read_code); /* * Maps the mm_struct mm into the current task struct. * On success, this function returns with exec_update_lock - * held for writing. + * held for writing. The replaced address space is stashed in + * bprm->old_mm for setup_new_exec() to release outside the lock. */ -static int exec_mmap(struct mm_struct *mm) +static int exec_mmap(struct linux_binprm *bprm) { + struct task_exec_state *exec_state __free(put_task_exec_state) = NULL; + struct mm_struct *mm = bprm->mm; struct task_struct *tsk; struct mm_struct *old_mm, *active_mm; int ret; + exec_state = alloc_task_exec_state(bprm->user_ns); + if (!exec_state) + return -ENOMEM; + /* Notify parent that we're no longer interested in the old VM */ tsk = current; old_mm = current->mm; @@ -870,6 +881,7 @@ static int exec_mmap(struct mm_struct *mm) tsk->active_mm = mm; tsk->mm = mm; mm_init_cid(mm, tsk); + exec_state = task_exec_state_replace(tsk, exec_state); /* * This prevents preemption while active_mm is being loaded and * it and mm are being updated, which could cause problems for @@ -888,15 +900,22 @@ static int exec_mmap(struct mm_struct *mm) if (old_mm) { mmap_read_unlock(old_mm); BUG_ON(active_mm != old_mm); - setmax_mm_hiwater_rss(&tsk->signal->maxrss, old_mm); - mm_update_next_owner(old_mm); - mmput(old_mm); + /* Defer teardown to setup_new_exec(), outside the exec locks. */ + bprm->old_mm = old_mm; return 0; } mmdrop_lazy_tlb(active_mm); return 0; } +/* Release the address space replaced by exec, outside the exec locks. */ +static void exec_mm_put_old(struct mm_struct *old_mm) +{ + setmax_mm_hiwater_rss(¤t->signal->maxrss, old_mm); + mm_update_next_owner(old_mm); + mmput(old_mm); +} + static int de_thread(struct task_struct *tsk) { struct signal_struct *sig = tsk->signal; @@ -1145,7 +1164,7 @@ int begin_new_exec(struct linux_binprm * bprm) * Release all of the old mmap stuff */ acct_arg_size(bprm, 0); - retval = exec_mmap(bprm->mm); + retval = exec_mmap(bprm); if (retval) goto out; @@ -1210,9 +1229,9 @@ int begin_new_exec(struct linux_binprm * bprm) if (bprm->interp_flags & BINPRM_FLAGS_ENFORCE_NONDUMP || !(uid_eq(current_euid(), current_uid()) && gid_eq(current_egid(), current_gid()))) - set_dumpable(current->mm, suid_dumpable); + task_exec_state_set_dumpable(suid_dumpable); else - set_dumpable(current->mm, SUID_DUMP_USER); + task_exec_state_set_dumpable(TASK_DUMPABLE_OWNER); perf_event_exec(); @@ -1261,7 +1280,7 @@ int begin_new_exec(struct linux_binprm * bprm) * wait until new credentials are committed * by commit_creds() above */ - if (get_dumpable(me->mm) != SUID_DUMP_USER) + if (task_exec_state_get_dumpable(me) != TASK_DUMPABLE_OWNER) perf_event_exit_task(me); /* * cred_guard_mutex must be held at least to this point to prevent @@ -1298,14 +1317,14 @@ void would_dump(struct linux_binprm *bprm, struct file *file) struct user_namespace *old, *user_ns; bprm->interp_flags |= BINPRM_FLAGS_ENFORCE_NONDUMP; - /* Ensure mm->user_ns contains the executable */ - user_ns = old = bprm->mm->user_ns; + /* Ensure bprm->user_ns contains the executable. */ + user_ns = old = bprm->user_ns; while ((user_ns != &init_user_ns) && !privileged_wrt_inode_uidgid(user_ns, idmap, inode)) user_ns = user_ns->parent; if (old != user_ns) { - bprm->mm->user_ns = get_user_ns(user_ns); + bprm->user_ns = get_user_ns(user_ns); put_user_ns(old); } } @@ -1328,6 +1347,12 @@ void setup_new_exec(struct linux_binprm * bprm) me->mm->task_size = TASK_SIZE; up_write(&me->signal->exec_update_lock); mutex_unlock(&me->signal->cred_guard_mutex); + + /* The exec locks are dropped: release the old address space now. */ + if (bprm->old_mm) { + exec_mm_put_old(bprm->old_mm); + bprm->old_mm = NULL; + } } EXPORT_SYMBOL(setup_new_exec); @@ -1375,6 +1400,8 @@ static void free_bprm(struct linux_binprm *bprm) acct_arg_size(bprm, 0); mmput(bprm->mm); } + if (bprm->user_ns) + put_user_ns(bprm->user_ns); free_arg_pages(bprm); if (bprm->cred) { /* in case exec fails before de_thread() succeeds */ @@ -1382,6 +1409,9 @@ static void free_bprm(struct linux_binprm *bprm) mutex_unlock(¤t->signal->cred_guard_mutex); abort_creds(bprm->cred); } + /* exec swapped the mm but failed before setup_new_exec() freed it */ + if (bprm->old_mm) + exec_mm_put_old(bprm->old_mm); do_close_execat(bprm->file); if (bprm->executable) fput(bprm->executable); @@ -1687,7 +1717,7 @@ static int exec_binprm(struct linux_binprm *bprm) old_vpid = task_pid_nr_ns(current, task_active_pid_ns(current->parent)); rcu_read_unlock(); - /* This allows 4 levels of binfmt rewrites before failing hard. */ + /* This allows 5 levels of binfmt rewrites before failing hard. */ for (depth = 0;; depth++) { struct file *exec; if (depth > 5) @@ -1905,17 +1935,6 @@ void set_binfmt(struct linux_binfmt *new) } EXPORT_SYMBOL(set_binfmt); -/* - * set_dumpable stores three-value SUID_DUMP_* into mm->flags. - */ -void set_dumpable(struct mm_struct *mm, int value) -{ - if (WARN_ON((unsigned)value > SUID_DUMP_ROOT)) - return; - - __mm_flags_set_mask_dumpable(mm, value); -} - static inline struct user_arg_ptr native_arg(const char __user *const __user *p) { return (struct user_arg_ptr){.ptr.native = p}; @@ -1975,9 +1994,11 @@ COMPAT_SYSCALL_DEFINE5(execveat, int, fd, static int proc_dointvec_minmax_coredump(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { - int error = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + int error, old = READ_ONCE(suid_dumpable); + + error = proc_dointvec_minmax(table, write, buffer, lenp, ppos); - if (!error && write) + if (!error && write && (old != READ_ONCE(suid_dumpable))) validate_coredump_safety(); return error; } diff --git a/fs/exfat/Kconfig b/fs/exfat/Kconfig index cbeca8e44d9b..1fcb10c8d7bc 100644 --- a/fs/exfat/Kconfig +++ b/fs/exfat/Kconfig @@ -4,7 +4,7 @@ config EXFAT_FS tristate "exFAT filesystem support" select BUFFER_HEAD select NLS - select LEGACY_DIRECT_IO + select FS_IOMAP help This allows you to mount devices formatted with the exFAT file system. exFAT is typically used on SD-Cards or USB sticks. diff --git a/fs/exfat/Makefile b/fs/exfat/Makefile index ed51926a4971..e06bf85870ae 100644 --- a/fs/exfat/Makefile +++ b/fs/exfat/Makefile @@ -5,4 +5,4 @@ obj-$(CONFIG_EXFAT_FS) += exfat.o exfat-y := inode.o namei.o dir.o super.o fatent.o cache.o nls.o misc.o \ - file.o balloc.o + file.o balloc.o iomap.o diff --git a/fs/exfat/balloc.c b/fs/exfat/balloc.c index 625f2f14d4fe..e66ebf899778 100644 --- a/fs/exfat/balloc.c +++ b/fs/exfat/balloc.c @@ -112,7 +112,7 @@ static int exfat_allocate_bitmap(struct super_block *sb, } if (exfat_test_bitmap_range(sb, sbi->map_clu, - EXFAT_B_TO_CLU_ROUND_UP(map_size, sbi)) == false) + exfat_bytes_to_cluster_round_up(sbi, map_size)) == false) goto err_out; return 0; diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c index ac008ccaa97d..4fd4ec52b8c0 100644 --- a/fs/exfat/dir.c +++ b/fs/exfat/dir.c @@ -76,7 +76,7 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); struct exfat_inode_info *ei = EXFAT_I(inode); - unsigned int dentry = EXFAT_B_TO_DEN(*cpos) & 0xFFFFFFFF; + unsigned int dentry = exfat_bytes_to_dentries(*cpos) & 0xFFFFFFFF; struct buffer_head *bh; /* check if the given file ID is opened */ @@ -84,13 +84,13 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent return -EPERM; exfat_chain_set(&dir, ei->start_clu, - EXFAT_B_TO_CLU(i_size_read(inode), sbi), ei->flags); + exfat_bytes_to_cluster(sbi, i_size_read(inode)), ei->flags); dentries_per_clu = sbi->dentries_per_clu; - max_dentries = (unsigned int)min_t(u64, MAX_EXFAT_DENTRIES, - (u64)EXFAT_CLU_TO_DEN(sbi->num_clusters, sbi)); + max_dentries = min(MAX_EXFAT_DENTRIES, + exfat_cluster_to_dentries(sbi, sbi->num_clusters)); - clu_offset = EXFAT_DEN_TO_CLU(dentry, sbi); + clu_offset = exfat_dentries_to_cluster(sbi, dentry); exfat_chain_dup(&clu, &dir); if (clu.flags == ALLOC_FAT_CHAIN) { @@ -147,10 +147,10 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent dir_entry->dir = clu; brelse(bh); - ei->hint_bmap.off = EXFAT_DEN_TO_CLU(dentry, sbi); + ei->hint_bmap.off = exfat_dentries_to_cluster(sbi, dentry); ei->hint_bmap.clu = clu.dir; - *cpos = EXFAT_DEN_TO_B(dentry + 1 + num_ext); + *cpos = exfat_dentries_to_bytes(dentry + 1 + num_ext); return 0; } @@ -160,7 +160,7 @@ static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_ent out: dir_entry->namebuf.lfn[0] = '\0'; - *cpos = EXFAT_DEN_TO_B(dentry); + *cpos = exfat_dentries_to_bytes(dentry); return 0; } @@ -295,7 +295,7 @@ int exfat_alloc_new_dir(struct inode *inode, struct exfat_chain *clu) exfat_chain_set(clu, EXFAT_EOF_CLUSTER, 0, ALLOC_NO_FAT_CHAIN); - ret = exfat_alloc_cluster(inode, 1, clu, IS_DIRSYNC(inode)); + ret = exfat_alloc_cluster(inode, 1, clu, IS_DIRSYNC(inode), false); if (ret) return ret; @@ -465,37 +465,75 @@ static void exfat_free_benign_secondary_clusters(struct inode *inode, return; exfat_chain_set(&dir, start_clu, - EXFAT_B_TO_CLU_ROUND_UP(size, EXFAT_SB(sb)), + exfat_bytes_to_cluster_round_up(EXFAT_SB(sb), size), flags); exfat_free_cluster(inode, &dir); } +/* + * exfat_init_ext_entry - initialize extension entries in a directory entry set + * @es: target entry set + * @num_entries: number of entries excluding benign secondary entries + * @p_uniname: filename to store + * @old_es: optional source entry set with benign secondary entries, or NULL + * @num_extra: number of benign secondary entries to copy from @old_es + * + * Set up the file, stream extension, and filename entries in @es, optionally + * preserving @num_extra benign secondary entries from @old_es. @es and @old_es + * may refer to the same entry set; excess entries are marked as deleted. + */ void exfat_init_ext_entry(struct exfat_entry_set_cache *es, int num_entries, - struct exfat_uni_name *p_uniname) + struct exfat_uni_name *p_uniname, + struct exfat_entry_set_cache *old_es, int num_extra) { - int i; + int i, src_start = 0, old_num; unsigned short *uniname = p_uniname->name; struct exfat_dentry *ep; - es->num_entries = num_entries; + if (WARN_ON(num_extra < 0 || (num_extra && (!old_es || + old_es->num_entries < ES_IDX_FIRST_FILENAME + num_extra)))) + num_extra = 0; + + /* + * Save old entry count and source position before modifying + * es->num_entries, since old_es and es may point to the same + * entry set. + */ + old_num = es->num_entries; + if (old_es && num_extra > 0) + src_start = old_es->num_entries - num_extra; + + es->num_entries = num_entries + num_extra; ep = exfat_get_dentry_cached(es, ES_IDX_FILE); - ep->dentry.file.num_ext = (unsigned char)(num_entries - 1); + ep->dentry.file.num_ext = (unsigned char)(num_entries - 1 + num_extra); ep = exfat_get_dentry_cached(es, ES_IDX_STREAM); ep->dentry.stream.name_len = p_uniname->name_len; ep->dentry.stream.name_hash = cpu_to_le16(p_uniname->name_hash); + if (old_es && num_extra > 0) { + for (i = 0; i < num_extra; i++) + *exfat_get_dentry_cached(es, num_entries + i) = + *exfat_get_dentry_cached(old_es, src_start + i); + } + for (i = ES_IDX_FIRST_FILENAME; i < num_entries; i++) { ep = exfat_get_dentry_cached(es, i); exfat_init_name_entry(ep, uniname); uniname += EXFAT_FILE_NAME_LEN; } + /* Mark excess old entries as deleted (in-place shrink) */ + for (i = num_entries + num_extra; i < old_num; i++) { + ep = exfat_get_dentry_cached(es, i); + exfat_set_entry_type(ep, TYPE_DELETED); + } + exfat_update_dir_chksum(es); } void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, - int order) + int order, bool free_benign) { int i; struct exfat_dentry *ep; @@ -503,7 +541,7 @@ void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, for (i = order; i < es->num_entries; i++) { ep = exfat_get_dentry_cached(es, i); - if (exfat_get_entry_type(ep) & TYPE_BENIGN_SEC) + if (free_benign && (exfat_get_entry_type(ep) & TYPE_BENIGN_SEC)) exfat_free_benign_secondary_clusters(inode, ep); exfat_set_entry_type(ep, TYPE_DELETED); @@ -556,10 +594,11 @@ static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir unsigned int off, clu = 0; struct exfat_sb_info *sbi = EXFAT_SB(sb); - off = EXFAT_DEN_TO_B(entry); + off = exfat_dentries_to_bytes(entry); clu = p_dir->dir; - ret = exfat_cluster_walk(sb, &clu, EXFAT_B_TO_CLU(off, sbi), p_dir->flags); + ret = exfat_cluster_walk(sb, &clu, exfat_bytes_to_cluster(sbi, off), + p_dir->flags); if (ret) return ret; @@ -567,7 +606,7 @@ static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir exfat_fs_error(sb, "unexpected early break in cluster chain (clu : %u, len : %d)", p_dir->dir, - EXFAT_B_TO_CLU(off, sbi)); + exfat_bytes_to_cluster(sbi, off)); return -EIO; } @@ -577,13 +616,13 @@ static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir } /* byte offset in cluster */ - off = EXFAT_CLU_OFFSET(off, sbi); + off = exfat_cluster_offset(sbi, off); /* byte offset in sector */ - *offset = EXFAT_BLK_OFFSET(off, sb); + *offset = exfat_block_offset(sb, off); /* sector offset in cluster */ - *sector = EXFAT_B_TO_BLK(off, sb); + *sector = exfat_bytes_to_block(sb, off); *sector += exfat_cluster_to_sector(sbi, clu); return 0; } @@ -593,7 +632,7 @@ struct exfat_dentry *exfat_get_dentry(struct super_block *sb, { struct exfat_sb_info *sbi = EXFAT_SB(sb); unsigned int sect_per_clus = sbi->sect_per_clus; - unsigned int dentries_per_page = EXFAT_B_TO_DEN(PAGE_SIZE); + unsigned int dentries_per_page = exfat_bytes_to_dentries(PAGE_SIZE); int off; sector_t sec; @@ -672,8 +711,8 @@ struct exfat_dentry *exfat_get_dentry_cached( struct exfat_entry_set_cache *es, int num) { int off = es->start_off + num * DENTRY_SIZE; - struct buffer_head *bh = es->bh[EXFAT_B_TO_BLK(off, es->sb)]; - char *p = bh->b_data + EXFAT_BLK_OFFSET(off, es->sb); + struct buffer_head *bh = es->bh[exfat_bytes_to_block(es->sb, off)]; + char *p = bh->b_data + exfat_block_offset(es->sb, off); return (struct exfat_dentry *)p; } @@ -741,7 +780,7 @@ static int __exfat_get_dentry_set(struct exfat_entry_set_cache *es, es->num_entries = num_entries; - num_bh = EXFAT_B_TO_BLK_ROUND_UP(off + num_entries * DENTRY_SIZE, sb); + num_bh = exfat_bytes_to_block_round_up(sb, off + num_entries * DENTRY_SIZE); if (num_bh > ARRAY_SIZE(es->__bh)) { es->bh = kmalloc_objs(*es->bh, num_bh, GFP_NOFS); if (!es->bh) { @@ -830,7 +869,7 @@ static int exfat_validate_empty_dentry_set(struct exfat_entry_set_cache *es) err_used_follow_unused: off = es->start_off + (i << DENTRY_SIZE_BITS); - bh = es->bh[EXFAT_B_TO_BLK(off, es->sb)]; + bh = es->bh[exfat_bytes_to_block(es->sb, off)]; exfat_fs_error(es->sb, "in sector %lld, dentry %d should be unused, but 0x%x", @@ -839,7 +878,8 @@ err_used_follow_unused: return -EIO; count_skip_entries: - es->num_entries = EXFAT_B_TO_DEN(EXFAT_BLK_TO_B(es->num_bh, es->sb) - es->start_off); + es->num_entries = + exfat_bytes_to_dentries(exfat_block_to_bytes(es->sb, es->num_bh) - es->start_off); for (; i < es->num_entries; i++) { ep = exfat_get_dentry_cached(es, i); if (IS_EXFAT_DELETED(ep->type)) @@ -892,7 +932,7 @@ static inline void exfat_set_empty_hint(struct exfat_inode_info *ei, { if (ei->hint_femp.eidx == EXFAT_HINT_NONE || ei->hint_femp.eidx > dentry) { - int total_entries = EXFAT_B_TO_DEN(i_size_read(&ei->vfs_inode)); + int total_entries = exfat_bytes_to_dentries(i_size_read(&ei->vfs_inode)); if (candi_empty->count == 0) { candi_empty->cur = *clu; @@ -1027,22 +1067,26 @@ rewind: continue; } - brelse(bh); if (entry_type == TYPE_EXTEND) { unsigned short entry_uniname[16], unichar; + unsigned int offset; if (step != DIRENT_STEP_NAME || name_len >= MAX_NAME_LENGTH) { + brelse(bh); step = DIRENT_STEP_FILE; continue; } - if (++order == 2) - uniname = p_uniname->name; - else - uniname += EXFAT_FILE_NAME_LEN; - + offset = (++order - 2) * EXFAT_FILE_NAME_LEN; len = exfat_extract_uni_name(ep, entry_uniname); + brelse(bh); + if (offset > MAX_NAME_LENGTH || + len > MAX_NAME_LENGTH - offset) { + step = DIRENT_STEP_FILE; + continue; + } + uniname = p_uniname->name + offset; name_len += len; unichar = *(uniname+len); @@ -1061,6 +1105,7 @@ rewind: continue; } + brelse(bh); if (entry_type & (TYPE_CRITICAL_SEC | TYPE_BENIGN_SEC)) { if (step == DIRENT_STEP_SECD) { @@ -1215,7 +1260,7 @@ static int exfat_get_volume_label_dentry(struct super_block *sb, es->bh = es->__bh; es->bh[0] = bh; es->num_bh = 1; - es->start_off = EXFAT_DEN_TO_B(i) % sb->s_blocksize; + es->start_off = exfat_dentries_to_bytes(i) % sb->s_blocksize; return 0; } diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h index 89ef5368277f..9be50949ce34 100644 --- a/fs/exfat/exfat_fs.h +++ b/fs/exfat/exfat_fs.h @@ -12,6 +12,7 @@ #include <linux/blkdev.h> #include <linux/backing-dev.h> #include <uapi/linux/exfat.h> +#include <linux/buffer_head.h> #define EXFAT_ROOT_INO 1 @@ -85,38 +86,6 @@ enum { << (PAGE_SHIFT - (sb)->s_blocksize_bits)) /* - * helpers for cluster size to byte conversion. - */ -#define EXFAT_CLU_TO_B(b, sbi) ((b) << (sbi)->cluster_size_bits) -#define EXFAT_B_TO_CLU(b, sbi) ((b) >> (sbi)->cluster_size_bits) -#define EXFAT_B_TO_CLU_ROUND_UP(b, sbi) \ - (((b - 1) >> (sbi)->cluster_size_bits) + 1) -#define EXFAT_CLU_OFFSET(off, sbi) ((off) & ((sbi)->cluster_size - 1)) - -/* - * helpers for block size to byte conversion. - */ -#define EXFAT_BLK_TO_B(b, sb) ((b) << (sb)->s_blocksize_bits) -#define EXFAT_B_TO_BLK(b, sb) ((b) >> (sb)->s_blocksize_bits) -#define EXFAT_B_TO_BLK_ROUND_UP(b, sb) \ - (((b - 1) >> (sb)->s_blocksize_bits) + 1) -#define EXFAT_BLK_OFFSET(off, sb) ((off) & ((sb)->s_blocksize - 1)) - -/* - * helpers for block size to dentry size conversion. - */ -#define EXFAT_B_TO_DEN(b) ((b) >> DENTRY_SIZE_BITS) -#define EXFAT_DEN_TO_B(b) ((b) << DENTRY_SIZE_BITS) - -/* - * helpers for cluster size to dentry size conversion. - */ -#define EXFAT_CLU_TO_DEN(clu, sbi) \ - ((clu) << ((sbi)->cluster_size_bits - DENTRY_SIZE_BITS)) -#define EXFAT_DEN_TO_CLU(dentry, sbi) \ - ((dentry) >> ((sbi)->cluster_size_bits - DENTRY_SIZE_BITS)) - -/* * helpers for fat entry. */ #define FAT_ENT_SIZE (4) @@ -149,7 +118,7 @@ enum { * The 608 bytes are in 3 sectors at most (even 512 Byte sector). */ #define DIR_CACHE_SIZE \ - (DIV_ROUND_UP(EXFAT_DEN_TO_B(ES_MAX_ENTRY_NUM), SECTOR_SIZE) + 1) + (DIV_ROUND_UP(ES_MAX_ENTRY_NUM << DENTRY_SIZE_BITS, SECTOR_SIZE) + 1) /* Superblock flags */ #define EXFAT_FLAGS_SHUTDOWN 1 @@ -259,6 +228,7 @@ struct exfat_sb_info { unsigned long long FAT1_start_sector; /* FAT1 start sector */ unsigned long long FAT2_start_sector; /* FAT2 start sector */ unsigned long long data_start_sector; /* data area start sector */ + unsigned long long data_start_bytes; unsigned int num_FAT_sectors; /* num of FAT sectors */ unsigned int root_dir; /* root dir cluster */ unsigned int dentries_per_clu; /* num of dentries per cluster */ @@ -324,6 +294,8 @@ struct exfat_inode_info { /* on-disk position of directory entry or 0 */ loff_t i_pos; loff_t valid_size; + /* page-aligned size that has been zeroed out for mmap */ + loff_t zeroed_size; /* hash by i_location */ struct hlist_node i_hash_fat; /* protect bmap against truncate */ @@ -432,6 +404,101 @@ static inline loff_t exfat_ondisk_size(const struct inode *inode) return ((loff_t)inode->i_blocks) << 9; } +static inline loff_t exfat_cluster_to_phys_bytes(struct exfat_sb_info *sbi, + unsigned int clus) +{ + return ((loff_t)(clus - EXFAT_RESERVED_CLUSTERS) << sbi->cluster_size_bits) + + sbi->data_start_bytes; +} + +/* + * helpers for cluster size to byte conversion. + */ +static inline loff_t exfat_cluster_to_bytes(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return (loff_t)nr_clusters << sbi->cluster_size_bits; +} + +static inline blkcnt_t exfat_cluster_to_sectors(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return (blkcnt_t)nr_clusters << (sbi->cluster_size_bits - 9); +} + +static inline u32 exfat_bytes_to_cluster(struct exfat_sb_info *sbi, loff_t size) +{ + return (u32)(size >> sbi->cluster_size_bits); +} + +static inline u32 exfat_bytes_to_cluster_round_up(struct exfat_sb_info *sbi, + loff_t size) +{ + if (size <= 0) + return 0; + return (u32)((size - 1) >> sbi->cluster_size_bits) + 1; +} + +static inline u32 exfat_cluster_offset(struct exfat_sb_info *sbi, loff_t off) +{ + return off & (sbi->cluster_size - 1); +} + +/* + * helpers for block size to byte conversion. + */ +static inline loff_t exfat_block_to_bytes(struct super_block *sb, + sector_t block) +{ + return (loff_t)block << sb->s_blocksize_bits; +} + +static inline sector_t exfat_bytes_to_block(struct super_block *sb, loff_t size) +{ + return (sector_t)(size >> sb->s_blocksize_bits); +} + +static inline sector_t exfat_bytes_to_block_round_up(struct super_block *sb, + loff_t size) +{ + if (size <= 0) + return 0; + return (sector_t)(((size - 1) >> sb->s_blocksize_bits) + 1); +} + +static inline u32 exfat_block_offset(struct super_block *sb, loff_t off) +{ + return (u32)(off & (sb->s_blocksize - 1)); +} + +/* + * helpers for block size to dentry size conversion. + */ +static inline u32 exfat_bytes_to_dentries(loff_t b) +{ + return (u32)(b >> DENTRY_SIZE_BITS); +} + +static inline u32 exfat_dentries_to_bytes(u32 dentry) +{ + return dentry << DENTRY_SIZE_BITS; +} + +/* + * helpers for cluster size to dentry size conversion. + */ +static inline u32 exfat_cluster_to_dentries(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return nr_clusters << (sbi->cluster_size_bits - DENTRY_SIZE_BITS); +} + +static inline u32 exfat_dentries_to_cluster(struct exfat_sb_info *sbi, + u32 dentry) +{ + return dentry >> (sbi->cluster_size_bits - DENTRY_SIZE_BITS); +} + /* super.c */ int exfat_set_volume_dirty(struct super_block *sb); int exfat_clear_volume_dirty(struct super_block *sb); @@ -441,7 +508,7 @@ int exfat_clear_volume_dirty(struct super_block *sb); exfat_cluster_walk(sb, (pclu), 1, ALLOC_FAT_CHAIN) int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc, - struct exfat_chain *p_chain, bool sync_bmap); + struct exfat_chain *p_chain, bool sync_bmap, bool contig); int exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain); int exfat_ent_get(struct super_block *sb, unsigned int loc, unsigned int *content, struct buffer_head **last); @@ -490,12 +557,13 @@ int exfat_trim_fs(struct inode *inode, struct fstrim_range *range); /* file.c */ extern const struct file_operations exfat_file_operations; int __exfat_truncate(struct inode *inode); -void exfat_truncate(struct inode *inode); int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); int exfat_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, unsigned int request_mask, unsigned int query_flags); +struct file_kattr; +int exfat_fileattr_get(struct dentry *dentry, struct file_kattr *fa); int exfat_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); long exfat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); long exfat_compat_ioctl(struct file *filp, unsigned int cmd, @@ -524,9 +592,10 @@ void exfat_init_dir_entry(struct exfat_entry_set_cache *es, unsigned int type, unsigned int start_clu, unsigned long long size, struct timespec64 *ts); void exfat_init_ext_entry(struct exfat_entry_set_cache *es, int num_entries, - struct exfat_uni_name *p_uniname); + struct exfat_uni_name *p_uniname, + struct exfat_entry_set_cache *old_es, int num_extra); void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, - int order); + int order, bool free_benign); void exfat_update_dir_chksum(struct exfat_entry_set_cache *es); int exfat_calc_num_entries(struct exfat_uni_name *p_uniname); int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, @@ -584,7 +653,9 @@ struct inode *exfat_iget(struct super_block *sb, loff_t i_pos); int __exfat_write_inode(struct inode *inode, int sync); int exfat_write_inode(struct inode *inode, struct writeback_control *wbc); void exfat_evict_inode(struct inode *inode); -int exfat_block_truncate_page(struct inode *inode, loff_t from); +int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, + unsigned int *clu, unsigned int *count, int create, + bool *balloc); /* exfat/nls.c */ unsigned short exfat_toupper(struct super_block *sb, unsigned short a); diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c index dce0955e689a..a8b11e2ce43f 100644 --- a/fs/exfat/fatent.c +++ b/fs/exfat/fatent.c @@ -412,14 +412,14 @@ int exfat_zeroed_cluster(struct inode *dir, unsigned int clu) if (IS_DIRSYNC(dir)) return sync_blockdev_range(sb->s_bdev, - EXFAT_BLK_TO_B(blknr, sb), - EXFAT_BLK_TO_B(last_blknr, sb) - 1); + exfat_block_to_bytes(sb, blknr), + exfat_block_to_bytes(sb, last_blknr) - 1); return 0; } int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc, - struct exfat_chain *p_chain, bool sync_bmap) + struct exfat_chain *p_chain, bool sync_bmap, bool contig) { int ret = -ENOSPC; unsigned int total_cnt; @@ -470,14 +470,20 @@ int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc, while ((new_clu = exfat_find_free_bitmap(sb, hint_clu)) != EXFAT_EOF_CLUSTER) { - if (new_clu != hint_clu && - p_chain->flags == ALLOC_NO_FAT_CHAIN) { - if (exfat_chain_cont_cluster(sb, p_chain->dir, - p_chain->size)) { - ret = -EIO; - goto free_cluster; + if (new_clu != hint_clu) { + if (p_chain->flags == ALLOC_NO_FAT_CHAIN) { + if (exfat_chain_cont_cluster(sb, p_chain->dir, + p_chain->size)) { + ret = -EIO; + goto free_cluster; + } + p_chain->flags = ALLOC_FAT_CHAIN; + } + + if (contig && p_chain->size > 0) { + hint_clu = last_clu; + goto done; } - p_chain->flags = ALLOC_FAT_CHAIN; } /* update allocation bitmap */ @@ -507,9 +513,9 @@ int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc, last_clu = new_clu; if (p_chain->size == num_alloc) { +done: sbi->clu_srch_ptr = hint_clu; - sbi->used_clusters += num_alloc; - + sbi->used_clusters += p_chain->size; mutex_unlock(&sbi->bitmap_lock); return 0; } diff --git a/fs/exfat/file.c b/fs/exfat/file.c index 354bdcfe4abc..5fc13378d35f 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -14,9 +14,12 @@ #include <linux/writeback.h> #include <linux/filelock.h> #include <linux/falloc.h> +#include <linux/fileattr.h> +#include <linux/iomap.h> #include "exfat_raw.h" #include "exfat_fs.h" +#include "iomap.h" static int exfat_cont_expand(struct inode *inode, loff_t size) { @@ -26,16 +29,17 @@ static int exfat_cont_expand(struct inode *inode, loff_t size) struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); struct exfat_chain clu; + loff_t oldsize = i_size_read(inode); - truncate_pagecache(inode, i_size_read(inode)); + truncate_pagecache(inode, oldsize); ret = inode_newsize_ok(inode, size); if (ret) return ret; - num_clusters = EXFAT_B_TO_CLU(exfat_ondisk_size(inode), sbi); + num_clusters = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); /* integer overflow is already checked in inode_newsize_ok(). */ - new_num_clusters = EXFAT_B_TO_CLU_ROUND_UP(size, sbi); + new_num_clusters = exfat_bytes_to_cluster_round_up(sbi, size); if (new_num_clusters == num_clusters) goto out; @@ -56,7 +60,7 @@ static int exfat_cont_expand(struct inode *inode, loff_t size) clu.flags = ei->flags; ret = exfat_alloc_cluster(inode, new_num_clusters - num_clusters, - &clu, inode_needs_sync(inode)); + &clu, inode_needs_sync(inode), false); if (ret) return ret; @@ -78,6 +82,13 @@ out: inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); /* Expanded range not zeroed, do not update valid_size */ i_size_write(inode, size); + /* + * When extending file size, call truncate_pagecache() first, + * then update i_size, and call pagecache_isize_extended() + * to ensures the straddling folio is properly marked RO so + * page_mkwrite() is called and post-EOF area is zeroed. + */ + pagecache_isize_extended(inode, oldsize, inode->i_size); inode->i_blocks = round_up(size, sbi->cluster_size) >> 9; mark_inode_dirty(inode); @@ -200,8 +211,8 @@ int __exfat_truncate(struct inode *inode) exfat_set_volume_dirty(sb); - num_clusters_new = EXFAT_B_TO_CLU_ROUND_UP(i_size_read(inode), sbi); - num_clusters_phys = EXFAT_B_TO_CLU(exfat_ondisk_size(inode), sbi); + num_clusters_new = exfat_bytes_to_cluster_round_up(sbi, i_size_read(inode)); + num_clusters_phys = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); exfat_chain_set(&clu, ei->start_clu, num_clusters_phys, ei->flags); @@ -236,7 +247,7 @@ int __exfat_truncate(struct inode *inode) } if (i_size_read(inode) < ei->valid_size) - ei->valid_size = i_size_read(inode); + ei->valid_size = ei->zeroed_size = i_size_read(inode); if (ei->type == TYPE_FILE) ei->attr |= EXFAT_ATTR_ARCHIVE; @@ -282,7 +293,7 @@ int __exfat_truncate(struct inode *inode) return 0; } -void exfat_truncate(struct inode *inode) +static void exfat_truncate(struct inode *inode) { struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); @@ -323,6 +334,18 @@ int exfat_getattr(struct mnt_idmap *idmap, const struct path *path, return 0; } +int exfat_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + /* + * exFAT compares filenames through an upcase table, so lookup + * is always case-insensitive. Long names are stored in UTF-16 + * with case intact; CASENONPRESERVING stays clear. + */ + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + return 0; +} + int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -383,10 +406,12 @@ int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry, exfat_truncate_inode_atime(inode); if (attr->ia_valid & ATTR_SIZE) { - error = exfat_block_truncate_page(inode, attr->ia_size); - if (error) - goto out; - + /* + * Wait for any in-flight DIO to finish before truncating to + * prevent a concurrent DIO from writing to clusters that are + * about to be freed. + */ + inode_dio_wait(inode); down_write(&EXFAT_I(inode)->truncate_lock); truncate_setsize(inode, attr->ia_size); @@ -631,42 +656,76 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size) { - int err; - loff_t pos; struct exfat_inode_info *ei = EXFAT_I(inode); - struct address_space *mapping = inode->i_mapping; - const struct address_space_operations *ops = mapping->a_ops; + loff_t old_valid_size = ei->valid_size; + int ret = 0; - pos = ei->valid_size; - while (pos < new_valid_size) { - u32 len; - struct folio *folio; - unsigned long off; + if (old_valid_size < new_valid_size) { + if (i_size_read(inode) < new_valid_size) { + i_size_write(inode, new_valid_size); + mark_inode_dirty(inode); + } - len = PAGE_SIZE - (pos & (PAGE_SIZE - 1)); - if (pos + len > new_valid_size) - len = new_valid_size - pos; + ret = iomap_zero_range(inode, old_valid_size, + new_valid_size - old_valid_size, NULL, + &exfat_write_iomap_ops, NULL, NULL); + if (ret) { + truncate_setsize(inode, old_valid_size); + exfat_truncate(inode); + } + } - err = ops->write_begin(NULL, mapping, pos, len, &folio, NULL); - if (err) - goto out; + return ret; +} - off = offset_in_folio(folio, pos); - folio_zero_new_buffers(folio, off, off + len); +static ssize_t exfat_fallback_buffered_write(struct kiocb *iocb, + struct iov_iter *from) +{ + loff_t offset = iocb->ki_pos, end; + ssize_t written; + int ret; - err = ops->write_end(NULL, mapping, pos, len, len, folio, NULL); - if (err < 0) - goto out; - pos += len; + iocb->ki_flags &= ~IOCB_DIRECT; - balance_dirty_pages_ratelimited(mapping); - cond_resched(); - } + written = iomap_file_buffered_write(iocb, from, &exfat_write_iomap_ops, + NULL, NULL); + if (written < 0) + return written; - return 0; + end = iocb->ki_pos + written - 1; + ret = filemap_write_and_wait_range(iocb->ki_filp->f_mapping, + offset, end); + if (ret) + return -EIO; -out: - return err; + invalidate_mapping_pages(iocb->ki_filp->f_mapping, + offset >> PAGE_SHIFT, + end >> PAGE_SHIFT); + + return written; +} + +static ssize_t exfat_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t ret; + + ret = iomap_dio_rw(iocb, from, &exfat_write_iomap_ops, + &exfat_write_dio_ops, 0, NULL, 0); + if (ret == -ENOTBLK) + ret = 0; + else if (ret < 0) + return ret; + + if (iov_iter_count(from)) { + ssize_t written; + + written = exfat_fallback_buffered_write(iocb, from); + if (written < 0) + return written; + ret += written; + } + + return ret; } static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) @@ -677,6 +736,7 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) struct exfat_inode_info *ei = EXFAT_I(inode); loff_t pos = iocb->ki_pos; loff_t valid_size; + int err; if (unlikely(exfat_forced_shutdown(inode->i_sb))) return -EIO; @@ -692,14 +752,10 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) if (ret <= 0) goto unlock; - if (iocb->ki_flags & IOCB_DIRECT) { - unsigned long align = pos | iov_iter_alignment(iter); - - if (!IS_ALIGNED(align, i_blocksize(inode)) && - !IS_ALIGNED(align, bdev_logical_block_size(inode->i_sb->s_bdev))) { - ret = -EINVAL; - goto unlock; - } + err = file_modified(iocb->ki_filp); + if (err) { + ret = err; + goto unlock; } if (pos > valid_size) { @@ -713,7 +769,11 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) goto unlock; } - ret = __generic_file_write_iter(iocb, iter); + if (iocb->ki_flags & IOCB_DIRECT) + ret = exfat_dio_write_iter(iocb, iter); + else + ret = iomap_file_buffered_write(iocb, iter, + &exfat_write_iomap_ops, NULL, NULL); if (ret < 0) goto unlock; @@ -740,37 +800,78 @@ unlock: static ssize_t exfat_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) { struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; if (unlikely(exfat_forced_shutdown(inode->i_sb))) return -EIO; - return generic_file_read_iter(iocb, iter); + inode_lock_shared(inode); + + if (iocb->ki_flags & IOCB_DIRECT) { + file_accessed(iocb->ki_filp); + ret = iomap_dio_rw(iocb, iter, &exfat_iomap_ops, NULL, 0, + NULL, 0); + } else { + ret = generic_file_read_iter(iocb, iter); + } + + inode_unlock_shared(inode); + + return ret; } static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) { - int err; struct inode *inode = file_inode(vmf->vma->vm_file); struct exfat_inode_info *ei = EXFAT_I(inode); - loff_t new_valid_size; + vm_fault_t ret; + loff_t new_valid_size, mmap_valid_size; if (!inode_trylock(inode)) return VM_FAULT_RETRY; - new_valid_size = ((loff_t)vmf->pgoff + 1) << PAGE_SHIFT; - new_valid_size = min(new_valid_size, i_size_read(inode)); + mmap_valid_size = ((loff_t)vmf->pgoff + 1) << PAGE_SHIFT; + new_valid_size = min(mmap_valid_size, i_size_read(inode)); if (ei->valid_size < new_valid_size) { - err = exfat_extend_valid_size(inode, new_valid_size); - if (err < 0) { - inode_unlock(inode); - return vmf_fs_error(err); + if (ei->zeroed_size < mmap_valid_size) { + int err; + + /* + * Only zero the range that hasn't been zeroed yet for + * this mmap write path. zeroed_size tracks the largest + * page-aligned offset that has already been zeroed. + * + * This prevents unnecessarily zeroing out the entire + * tail page on every page fault when userspace writes + * data byte-by-byte through mmap (after a small + * fallocate). It fixes data corruption in the tail page + * while preserving the existing valid_size semantics. + */ + err = iomap_zero_range(inode, ei->zeroed_size, + mmap_valid_size - ei->zeroed_size, NULL, + &exfat_iomap_ops, NULL, NULL); + if (err < 0) { + inode_unlock(inode); + return vmf_fs_error(err); + } + ei->zeroed_size = mmap_valid_size; } + + ei->valid_size = new_valid_size; + mark_inode_dirty(inode); } + sb_start_pagefault(inode->i_sb); + file_update_time(vmf->vma->vm_file); + + filemap_invalidate_lock_shared(inode->i_mapping); + ret = iomap_page_mkwrite(vmf, &exfat_write_iomap_ops, NULL); + filemap_invalidate_unlock_shared(inode->i_mapping); + sb_end_pagefault(inode->i_sb); inode_unlock(inode); - return filemap_page_mkwrite(vmf); + return ret; } static const struct vm_operations_struct exfat_file_vm_ops = { @@ -786,6 +887,21 @@ static int exfat_file_mmap_prepare(struct vm_area_desc *desc) if (unlikely(exfat_forced_shutdown(file_inode(desc->file)->i_sb))) return -EIO; + if (vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT)) { + struct inode *inode = file_inode(file); + loff_t from, to; + int err; + + from = ((loff_t)desc->pgoff << PAGE_SHIFT); + to = min_t(loff_t, i_size_read(inode), + from + vma_desc_size(desc)); + if (EXFAT_I(inode)->valid_size < to) { + err = exfat_extend_valid_size(inode, to); + if (err) + return err; + } + } + file_accessed(file); desc->vm_ops = &exfat_file_vm_ops; return 0; @@ -800,8 +916,48 @@ static ssize_t exfat_splice_read(struct file *in, loff_t *ppos, return filemap_splice_read(in, ppos, pipe, len, flags); } +static int exfat_file_open(struct inode *inode, struct file *filp) +{ + int err; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + err = generic_file_open(inode, filp); + if (err) + return err; + + filp->f_mode |= FMODE_CAN_ODIRECT; + + return 0; +} + +static loff_t exfat_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + + switch (whence) { + case SEEK_HOLE: + inode_lock_shared(inode); + offset = iomap_seek_hole(inode, offset, &exfat_iomap_ops); + inode_unlock_shared(inode); + break; + case SEEK_DATA: + inode_lock_shared(inode); + offset = iomap_seek_data(inode, offset, &exfat_iomap_ops); + inode_unlock_shared(inode); + break; + default: + return generic_file_llseek(file, offset, whence); + } + if (offset < 0) + return offset; + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); +} + const struct file_operations exfat_file_operations = { - .llseek = generic_file_llseek, + .open = exfat_file_open, + .llseek = exfat_file_llseek, .read_iter = exfat_file_read_iter, .write_iter = exfat_file_write_iter, .unlocked_ioctl = exfat_ioctl, @@ -817,6 +973,7 @@ const struct file_operations exfat_file_operations = { }; const struct inode_operations exfat_file_inode_operations = { - .setattr = exfat_setattr, - .getattr = exfat_getattr, + .setattr = exfat_setattr, + .getattr = exfat_getattr, + .fileattr_get = exfat_fileattr_get, }; diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c index 1ea4c740fef9..89826aea5e1e 100644 --- a/fs/exfat/inode.c +++ b/fs/exfat/inode.c @@ -13,13 +13,16 @@ #include <linux/uio.h> #include <linux/random.h> #include <linux/iversion.h> +#include <linux/iomap.h> #include "exfat_raw.h" #include "exfat_fs.h" +#include "iomap.h" int __exfat_write_inode(struct inode *inode, int sync) { unsigned long long on_disk_size; + unsigned long long on_disk_valid_size; struct exfat_dentry *ep, *ep2; struct exfat_entry_set_cache es; struct super_block *sb = inode->i_sb; @@ -69,22 +72,30 @@ int __exfat_write_inode(struct inode *inode, int sync) &ep->dentry.file.access_date, NULL); - /* File size should be zero if there is no cluster allocated */ - on_disk_size = i_size_read(inode); + /* + * During a DIO write, valid_size is updated eagerly in iomap_end (so + * that concurrent buffered reads see IOMAP_MAPPED) while i_size is + * updated asynchronously in end_io. The FAT chain was already + * extended to cover ceil(valid_size/cluster_size) clusters. Use the + * maximum so the on-disk size field always covers the FAT chain, + * preventing fsck from reporting "more clusters are allocated". + */ + on_disk_size = max_t(unsigned long long, i_size_read(inode), + ei->valid_size); if (ei->start_clu == EXFAT_EOF_CLUSTER) on_disk_size = 0; - - ep2->dentry.stream.size = cpu_to_le64(on_disk_size); /* - * mmap write does not use exfat_write_end(), valid_size may be - * extended to the sector-aligned length in exfat_get_block(). - * So we need to fixup valid_size to the writren length. + * valid_size on disk must reflect only confirmed data (up to i_size) + * and must not exceed on_disk_size. */ - if (on_disk_size < ei->valid_size) - ep2->dentry.stream.valid_size = ep2->dentry.stream.size; - else - ep2->dentry.stream.valid_size = cpu_to_le64(ei->valid_size); + on_disk_valid_size = min_t(unsigned long long, ei->valid_size, + i_size_read(inode)); + if (ei->start_clu == EXFAT_EOF_CLUSTER) + on_disk_valid_size = 0; + + ep2->dentry.stream.size = cpu_to_le64(on_disk_size); + ep2->dentry.stream.valid_size = cpu_to_le64(on_disk_valid_size); if (on_disk_size) { ep2->dentry.stream.flags = ei->flags; @@ -123,8 +134,9 @@ void exfat_sync_inode(struct inode *inode) * Output: errcode, cluster number * *clu = (~0), if it's unable to allocate a new cluster */ -static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, - unsigned int *clu, unsigned int *count, int create) +int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, + unsigned int *clu, unsigned int *count, int create, + bool *balloc) { int ret; unsigned int last_clu; @@ -135,10 +147,10 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, unsigned int local_clu_offset = clu_offset; unsigned int num_to_be_allocated = 0, num_clusters; - num_clusters = EXFAT_B_TO_CLU(exfat_ondisk_size(inode), sbi); - - if (clu_offset >= num_clusters) - num_to_be_allocated = clu_offset - num_clusters + 1; + num_clusters = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); + if (clu_offset > num_clusters || + *count > num_clusters - clu_offset) + num_to_be_allocated = clu_offset + *count - num_clusters; if (!create && (num_to_be_allocated > 0)) { *clu = EXFAT_EOF_CLUSTER; @@ -181,7 +193,7 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, } ret = exfat_alloc_cluster(inode, num_to_be_allocated, &new_clu, - inode_needs_sync(inode)); + inode_needs_sync(inode), true); if (ret) return ret; @@ -215,19 +227,11 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, } *clu = new_clu.dir; + *count = new_clu.size; - inode->i_blocks += EXFAT_CLU_TO_B(num_to_be_allocated, sbi) >> 9; - - /* - * Move *clu pointer along FAT chains (hole care) because the - * caller of this function expect *clu to be the last cluster. - * This only works when num_to_be_allocated >= 2, - * *clu = (the first cluster of the allocated chain) => - * (the last cluster of ...) - */ - if (exfat_cluster_walk(sb, clu, num_to_be_allocated - 1, ei->flags)) - return -EIO; - *count = 1; + inode->i_blocks += exfat_cluster_to_sectors(sbi, new_clu.size); + if (balloc) + *balloc = true; } /* hint information */ @@ -237,154 +241,15 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, return 0; } -static int exfat_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh_result, int create) -{ - struct exfat_inode_info *ei = EXFAT_I(inode); - struct super_block *sb = inode->i_sb; - struct exfat_sb_info *sbi = EXFAT_SB(sb); - unsigned long max_blocks = bh_result->b_size >> inode->i_blkbits; - int err = 0; - unsigned long mapped_blocks = 0; - unsigned int cluster, sec_offset, count; - sector_t last_block; - sector_t phys = 0; - sector_t valid_blks; - loff_t i_size; - - mutex_lock(&sbi->s_lock); - i_size = i_size_read(inode); - last_block = EXFAT_B_TO_BLK_ROUND_UP(i_size, sb); - if (iblock >= last_block && !create) - goto done; - - /* Is this block already allocated? */ - count = EXFAT_B_TO_CLU_ROUND_UP(bh_result->b_size, sbi); - err = exfat_map_cluster(inode, iblock >> sbi->sect_per_clus_bits, - &cluster, &count, create); - if (err) { - if (err != -ENOSPC) - exfat_fs_error_ratelimit(sb, - "failed to bmap (inode : %p iblock : %llu, err : %d)", - inode, (unsigned long long)iblock, err); - goto unlock_ret; - } - - if (cluster == EXFAT_EOF_CLUSTER) - goto done; - - /* sector offset in cluster */ - sec_offset = iblock & (sbi->sect_per_clus - 1); - - phys = exfat_cluster_to_sector(sbi, cluster) + sec_offset; - mapped_blocks = ((unsigned long)count << sbi->sect_per_clus_bits) - sec_offset; - max_blocks = min(mapped_blocks, max_blocks); - - map_bh(bh_result, sb, phys); - if (buffer_delay(bh_result)) - clear_buffer_delay(bh_result); - - /* - * In most cases, we just need to set bh_result to mapped, unmapped - * or new status as follows: - * 1. i_size == valid_size - * 2. write case (create == 1) - * 3. direct_read (!bh_result->b_folio) - * -> the unwritten part will be zeroed in exfat_direct_IO() - * - * Otherwise, in the case of buffered read, it is necessary to take - * care the last nested block if valid_size is not equal to i_size. - */ - if (i_size == ei->valid_size || create || !bh_result->b_folio) - valid_blks = EXFAT_B_TO_BLK_ROUND_UP(ei->valid_size, sb); - else - valid_blks = EXFAT_B_TO_BLK(ei->valid_size, sb); - - /* The range has been fully written, map it */ - if (iblock + max_blocks < valid_blks) - goto done; - - /* The range has been partially written, map the written part */ - if (iblock < valid_blks) { - max_blocks = valid_blks - iblock; - goto done; - } - - /* The area has not been written, map and mark as new for create case */ - if (create) { - set_buffer_new(bh_result); - ei->valid_size = EXFAT_BLK_TO_B(iblock + max_blocks, sb); - mark_inode_dirty(inode); - goto done; - } - - /* - * The area has just one block partially written. - * In that case, we should read and fill the unwritten part of - * a block with zero. - */ - if (bh_result->b_folio && iblock == valid_blks && - (ei->valid_size & (sb->s_blocksize - 1))) { - loff_t size, pos; - void *addr; - - max_blocks = 1; - - /* - * No buffer_head is allocated. - * (1) bmap: It's enough to set blocknr without I/O. - * (2) read: The unwritten part should be filled with zero. - * If a folio does not have any buffers, - * let's returns -EAGAIN to fallback to - * block_read_full_folio() for per-bh IO. - */ - if (!folio_buffers(bh_result->b_folio)) { - err = -EAGAIN; - goto done; - } - - pos = EXFAT_BLK_TO_B(iblock, sb); - size = ei->valid_size - pos; - addr = folio_address(bh_result->b_folio) + - offset_in_folio(bh_result->b_folio, pos); - - /* Check if bh->b_data points to proper addr in folio */ - if (bh_result->b_data != addr) { - exfat_fs_error_ratelimit(sb, - "b_data(%p) != folio_addr(%p)", - bh_result->b_data, addr); - err = -EINVAL; - goto done; - } - - /* Read a block */ - err = bh_read(bh_result, 0); - if (err < 0) - goto done; - - /* Zero unwritten part of a block */ - memset(bh_result->b_data + size, 0, bh_result->b_size - size); - err = 0; - goto done; - } - - /* - * The area has not been written, clear mapped for read/bmap cases. - * If so, it will be filled with zero without reading from disk. - */ - clear_buffer_mapped(bh_result); -done: - bh_result->b_size = EXFAT_BLK_TO_B(max_blocks, sb); - if (err < 0) - clear_buffer_mapped(bh_result); -unlock_ret: - mutex_unlock(&sbi->s_lock); - return err; -} - static int exfat_read_folio(struct file *file, struct folio *folio) { - return mpage_read_folio(folio, exfat_get_block); + struct iomap_read_folio_ctx ctx = { + .cur_folio = folio, + .ops = &exfat_iomap_bio_read_ops, + }; + + iomap_read_folio(&exfat_iomap_ops, &ctx, NULL); + return 0; } static void exfat_readahead(struct readahead_control *rac) @@ -393,6 +258,10 @@ static void exfat_readahead(struct readahead_control *rac) struct inode *inode = mapping->host; struct exfat_inode_info *ei = EXFAT_I(inode); loff_t pos = readahead_pos(rac); + struct iomap_read_folio_ctx ctx = { + .ops = &exfat_iomap_bio_read_ops, + .rac = rac, + }; /* Range cross valid_size, read it page by page. */ if (ei->valid_size < i_size_read(inode) && @@ -400,115 +269,22 @@ static void exfat_readahead(struct readahead_control *rac) ei->valid_size < pos + readahead_length(rac)) return; - mpage_readahead(rac, exfat_get_block); + iomap_readahead(&exfat_iomap_ops, &ctx, NULL); } static int exfat_writepages(struct address_space *mapping, struct writeback_control *wbc) { - if (unlikely(exfat_forced_shutdown(mapping->host->i_sb))) - return -EIO; - - return mpage_writepages(mapping, wbc, exfat_get_block); -} - -static void exfat_write_failed(struct address_space *mapping, loff_t to) -{ - struct inode *inode = mapping->host; - - if (to > i_size_read(inode)) { - truncate_pagecache(inode, i_size_read(inode)); - inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - exfat_truncate(inode); - } -} - -static int exfat_write_begin(const struct kiocb *iocb, - struct address_space *mapping, - loff_t pos, unsigned int len, - struct folio **foliop, void **fsdata) -{ - int ret; + struct iomap_writepage_ctx wpc = { + .inode = mapping->host, + .wbc = wbc, + .ops = &exfat_writeback_ops, + }; if (unlikely(exfat_forced_shutdown(mapping->host->i_sb))) return -EIO; - ret = block_write_begin(mapping, pos, len, foliop, exfat_get_block); - - if (ret < 0) - exfat_write_failed(mapping, pos+len); - - return ret; -} - -static int exfat_write_end(const struct kiocb *iocb, - struct address_space *mapping, - loff_t pos, unsigned int len, unsigned int copied, - struct folio *folio, void *fsdata) -{ - struct inode *inode = mapping->host; - struct exfat_inode_info *ei = EXFAT_I(inode); - int err; - - err = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); - if (err < len) - exfat_write_failed(mapping, pos+len); - - if (!(err < 0) && pos + err > ei->valid_size) { - ei->valid_size = pos + err; - mark_inode_dirty(inode); - } - - if (!(err < 0) && !(ei->attr & EXFAT_ATTR_ARCHIVE)) { - inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - ei->attr |= EXFAT_ATTR_ARCHIVE; - mark_inode_dirty(inode); - } - - return err; -} - -static ssize_t exfat_direct_IO(struct kiocb *iocb, struct iov_iter *iter) -{ - struct address_space *mapping = iocb->ki_filp->f_mapping; - struct inode *inode = mapping->host; - struct exfat_inode_info *ei = EXFAT_I(inode); - loff_t pos = iocb->ki_pos; - loff_t size = pos + iov_iter_count(iter); - int rw = iov_iter_rw(iter); - ssize_t ret; - - /* - * Need to use the DIO_LOCKING for avoiding the race - * condition of exfat_get_block() and ->truncate(). - */ - ret = blockdev_direct_IO(iocb, inode, iter, exfat_get_block); - if (ret < 0) { - if (rw == WRITE && ret != -EIOCBQUEUED) - exfat_write_failed(mapping, size); - - return ret; - } - - size = pos + ret; - - if (rw == WRITE) { - /* - * If the block had been partially written before this write, - * ->valid_size will not be updated in exfat_get_block(), - * update it here. - */ - if (ei->valid_size < size) { - ei->valid_size = size; - mark_inode_dirty(inode); - } - } else if (pos < ei->valid_size && ei->valid_size < size) { - /* zero the unwritten part in the partially written block */ - iov_iter_revert(iter, size - ei->valid_size); - iov_iter_zero(size - ei->valid_size, iter); - } - - return ret; + return iomap_writepages(&wpc); } static sector_t exfat_aop_bmap(struct address_space *mapping, sector_t block) @@ -517,34 +293,23 @@ static sector_t exfat_aop_bmap(struct address_space *mapping, sector_t block) /* exfat_get_cluster() assumes the requested blocknr isn't truncated. */ down_read(&EXFAT_I(mapping->host)->truncate_lock); - blocknr = generic_block_bmap(mapping, block, exfat_get_block); + blocknr = iomap_bmap(mapping, block, &exfat_iomap_ops); up_read(&EXFAT_I(mapping->host)->truncate_lock); return blocknr; } -/* - * exfat_block_truncate_page() zeroes out a mapping from file offset `from' - * up to the end of the block which corresponds to `from'. - * This is required during truncate to physically zeroout the tail end - * of that block so it doesn't yield old data if the file is later grown. - * Also, avoid causing failure from fsx for cases of "data past EOF" - */ -int exfat_block_truncate_page(struct inode *inode, loff_t from) -{ - return block_truncate_page(inode->i_mapping, from, exfat_get_block); -} - static const struct address_space_operations exfat_aops = { - .dirty_folio = block_dirty_folio, - .invalidate_folio = block_invalidate_folio, - .read_folio = exfat_read_folio, - .readahead = exfat_readahead, - .writepages = exfat_writepages, - .write_begin = exfat_write_begin, - .write_end = exfat_write_end, - .direct_IO = exfat_direct_IO, - .bmap = exfat_aop_bmap, - .migrate_folio = buffer_migrate_folio, + .read_folio = exfat_read_folio, + .readahead = exfat_readahead, + .writepages = exfat_writepages, + .dirty_folio = iomap_dirty_folio, + .bmap = exfat_aop_bmap, + .migrate_folio = filemap_migrate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .swap_activate = exfat_iomap_swap_activate, }; static inline unsigned long exfat_hash(loff_t i_pos) @@ -608,6 +373,7 @@ static int exfat_fill_inode(struct inode *inode, struct exfat_dir_entry *info) ei->flags = info->flags; ei->type = info->type; ei->valid_size = info->valid_size; + ei->zeroed_size = info->valid_size; ei->version = 0; ei->hint_stat.eidx = 0; diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c new file mode 100644 index 000000000000..190fc6471f84 --- /dev/null +++ b/fs/exfat/iomap.c @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * iomap callack functions + * + * Copyright (C) 2026 Namjae Jeon <linkinjeon@kernel.org> + */ + +#include <linux/iomap.h> +#include <linux/pagemap.h> + +#include "exfat_raw.h" +#include "exfat_fs.h" +#include "iomap.h" + +/* + * exfat_file_write_dio_end_io - Direct I/O write completion handler + * + * Updates i_size if the write extended the file. Called from the dio layer + * after I/O completion. + */ +static int exfat_file_write_dio_end_io(struct kiocb *iocb, ssize_t size, + int error, unsigned int flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + if (error) + return error; + + if (size && i_size_read(inode) < iocb->ki_pos + size) { + i_size_write(inode, iocb->ki_pos + size); + mark_inode_dirty(inode); + } + + return 0; +} + +const struct iomap_dio_ops exfat_write_dio_ops = { + .end_io = exfat_file_write_dio_end_io, +}; + +static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, bool may_alloc) +{ + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + unsigned int cluster, num_clusters; + loff_t cluster_offset, cluster_length; + int err; + bool balloc = false; + + if (!may_alloc) { + /* Completely beyond EOF. Treat as hole */ + if (i_size_read(inode) <= offset) { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + iomap->offset = offset; + iomap->length = length; + return 0; + } + + /* Clamp length if the requested range goes beyond i_size */ + if (offset + length > i_size_read(inode)) + length = round_up(i_size_read(inode), + i_blocksize(inode)) - offset; + } + + num_clusters = exfat_bytes_to_cluster_round_up(sbi, + offset + length) - exfat_bytes_to_cluster(sbi, offset); + + mutex_lock(&sbi->s_lock); + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + + err = exfat_map_cluster(inode, exfat_bytes_to_cluster(sbi, offset), + &cluster, &num_clusters, may_alloc, &balloc); + if (err) + goto out; + + cluster_offset = exfat_cluster_offset(sbi, offset); + cluster_length = exfat_cluster_to_bytes(sbi, num_clusters); + + iomap->length = min_t(loff_t, length, cluster_length - cluster_offset); + iomap->addr = exfat_cluster_to_phys_bytes(sbi, cluster) + cluster_offset; + iomap->type = IOMAP_MAPPED; + iomap->flags = IOMAP_F_MERGED; + + if (may_alloc || flags & IOMAP_ZERO) { + if (balloc) + iomap->flags |= IOMAP_F_NEW; + else if (iomap->offset + iomap->length >= ei->valid_size) { + /* + * This is a write that starts at or extends beyond + * the current valid_size. The region between the old + * valid_size and the end of this write needs to be + * zeroed in the page cache to prevent stale data + * exposure (see IOMAP_F_ZERO_TAIL handling in + * __iomap_write_begin()). + */ + iomap->flags |= IOMAP_F_ZERO_TAIL; + } + } else { + /* + * valid_size is tracked in byte granularity and + * marks the exact boundary between valid data and + * holes (or unwritten space). + * + * When IOMAP_REPORT is set (used by lseek(SEEK_HOLE) + * and SEEK_DATA), we return IOMAP_HOLE. This allows + * iomap_seek_hole_iter() to directly return the + * precise byte position. + * + * For normal I/O paths (without IOMAP_REPORT) we + * return IOMAP_UNWRITTEN so the write path can + * distinguish it from a real hole. + */ + if (offset >= ei->valid_size) { + iomap->type = flags & IOMAP_REPORT ? + IOMAP_HOLE : IOMAP_UNWRITTEN; + } else if (offset + iomap->length > ei->valid_size) { + if (flags & IOMAP_REPORT) { + /* + * For SEEK_HOLE/SEEK_DATA, clip the length + * to the exact byte boundary (valid_size). + * This ensures the caller gets the precise + * hole position in byte units. + */ + iomap->length = ei->valid_size - iomap->offset; + } else + iomap->length = round_up(ei->valid_size, + i_blocksize(inode)) - + iomap->offset; + } + } + + iomap->flags |= IOMAP_F_MERGED; +out: + mutex_unlock(&sbi->s_lock); + return err; +} + +static int exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap) +{ + return __exfat_iomap_begin(inode, offset, length, flags, iomap, false); +} + +static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap) +{ + return __exfat_iomap_begin(inode, offset, length, flags, iomap, true); +} + +const struct iomap_ops exfat_iomap_ops = { + .iomap_begin = exfat_iomap_begin, +}; + +/* + * exfat_write_iomap_end - Update the state after write + * + * Extends ->valid_size to cover the newly written range. + * Marks the inode dirty if metadata was changed. + */ +static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, struct iomap *iomap) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + bool dirtied = false; + loff_t end; + + if (!written) + return 0; + + end = pos + written; + + if (ei->valid_size < end) { + ei->valid_size = end; + if (ei->zeroed_size < end) + ei->zeroed_size = end; + dirtied = true; + } + + if (dirtied || iomap->flags & IOMAP_F_SIZE_CHANGED) + mark_inode_dirty(inode); + + return written; +} + +const struct iomap_ops exfat_write_iomap_ops = { + .iomap_begin = exfat_write_iomap_begin, + .iomap_end = exfat_write_iomap_end, +}; + +/* + * exfat_writeback_range - Map folio during writeback + * + * Called for each folio during writeback. If the folio falls outside the + * current iomap, remaps by calling read_iomap_begin. + */ +static ssize_t exfat_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, unsigned int len, u64 end_pos) +{ + if (offset < wpc->iomap.offset || + offset >= wpc->iomap.offset + wpc->iomap.length) { + int error; + + error = __exfat_iomap_begin(wpc->inode, offset, len, + 0, &wpc->iomap, false); + if (error) + return error; + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); +} + +const struct iomap_writeback_ops exfat_writeback_ops = { + .writeback_range = exfat_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, +}; + +/** + * exfat_iomap_read_end_io - iomap read bio completion handler for exFAT + * @bio: bio that has completed reading + * + * exfat_iomap_begin() rounds up MAPPED extents to the block boundary of + * valid_size. This ensures that any subsequent blocks are treated as + * IOMAP_UNWRITTEN, but it also causes the "straddle block" containing + * valid_size to be read from disk. The disk data beyond valid_size in + * this block is stale and must be zeroed to prevent data leakage. + */ +static void exfat_iomap_read_end_io(struct bio *bio) +{ + int error = blk_status_to_errno(bio->bi_status); + struct folio_iter iter; + + bio_for_each_folio_all(iter, bio) { + struct folio *folio = iter.folio; + struct exfat_inode_info *ei = EXFAT_I(folio->mapping->host); + s64 valid_size; + loff_t pos = folio_pos(folio); + + valid_size = ei->valid_size; + if (pos + iter.offset < valid_size && + pos + iter.offset + iter.length > valid_size) + folio_zero_segment(folio, offset_in_folio(folio, valid_size), + iter.offset + iter.length); + + iomap_finish_folio_read(folio, iter.offset, iter.length, error); + } + bio_put(bio); +} + +static void exfat_iomap_bio_submit_read(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + iomap_bio_submit_read_endio(iter, ctx, exfat_iomap_read_end_io); +} + +const struct iomap_read_ops exfat_iomap_bio_read_ops = { + .read_folio_range = iomap_bio_read_folio_range, + .submit_read = exfat_iomap_bio_submit_read, +}; + +int exfat_iomap_swap_activate(struct swap_info_struct *sis, + struct file *file, sector_t *span) +{ + return iomap_swapfile_activate(sis, file, span, &exfat_iomap_ops); +} diff --git a/fs/exfat/iomap.h b/fs/exfat/iomap.h new file mode 100644 index 000000000000..fd8a913f7794 --- /dev/null +++ b/fs/exfat/iomap.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2026 Namjae Jeon <linkinjeon@kernel.org> + */ + +#ifndef _LINUX_EXFAT_IOMAP_H +#define _LINUX_EXFAT_IOMAP_H + +extern const struct iomap_dio_ops exfat_write_dio_ops; +extern const struct iomap_ops exfat_iomap_ops; +extern const struct iomap_ops exfat_write_iomap_ops; +extern const struct iomap_writeback_ops exfat_writeback_ops; +extern const struct iomap_read_ops exfat_iomap_bio_read_ops; + +int exfat_iomap_swap_activate(struct swap_info_struct *sis, + struct file *file, sector_t *span); + +#endif /* _LINUX_EXFAT_IOMAP_H */ diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c index 2c5636634b4a..b7d5e44ad38e 100644 --- a/fs/exfat/namei.c +++ b/fs/exfat/namei.c @@ -208,7 +208,7 @@ static int exfat_search_empty_slot(struct super_block *sb, int dentries_per_clu; struct exfat_chain clu; struct exfat_sb_info *sbi = EXFAT_SB(sb); - int total_entries = EXFAT_CLU_TO_DEN(p_dir->size, sbi); + unsigned int total_entries = exfat_cluster_to_dentries(sbi, p_dir->size); dentries_per_clu = sbi->dentries_per_clu; @@ -266,7 +266,7 @@ static int exfat_search_empty_slot(struct super_block *sb, static int exfat_check_max_dentries(struct inode *inode) { - if (EXFAT_B_TO_DEN(i_size_read(inode)) >= MAX_EXFAT_DENTRIES) { + if (exfat_bytes_to_dentries(i_size_read(inode)) >= MAX_EXFAT_DENTRIES) { /* * exFAT spec allows a dir to grow up to 8388608(256MB) * dentries @@ -314,7 +314,8 @@ int exfat_find_empty_entry(struct inode *inode, } exfat_chain_set(p_dir, ei->start_clu, - EXFAT_B_TO_CLU(i_size_read(inode), sbi), ei->flags); + exfat_bytes_to_cluster(sbi, i_size_read(inode)), + ei->flags); while ((dentry = exfat_search_empty_slot(sb, &hint_femp, p_dir, num_entries, es)) < 0) { @@ -340,7 +341,7 @@ int exfat_find_empty_entry(struct inode *inode, } /* allocate a cluster */ - ret = exfat_alloc_cluster(inode, 1, &clu, IS_DIRSYNC(inode)); + ret = exfat_alloc_cluster(inode, 1, &clu, IS_DIRSYNC(inode), false); if (ret) return ret; @@ -375,7 +376,7 @@ int exfat_find_empty_entry(struct inode *inode, hint_femp.cur.size++; p_dir->size++; - size = EXFAT_CLU_TO_B(p_dir->size, sbi); + size = exfat_cluster_to_bytes(sbi, p_dir->size); /* directory inode should be updated in here */ i_size_write(inode, size); @@ -503,7 +504,7 @@ static int exfat_add_entry(struct inode *inode, const char *path, * the first cluster is not determined yet. (0) */ exfat_init_dir_entry(&es, type, start_clu, clu_size, &ts); - exfat_init_ext_entry(&es, num_entries, &uniname); + exfat_init_ext_entry(&es, num_entries, &uniname, NULL, 0); ret = exfat_put_dentry_set(&es, IS_DIRSYNC(inode)); if (ret) @@ -604,7 +605,7 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, return ret; exfat_chain_set(&cdir, ei->start_clu, - EXFAT_B_TO_CLU(i_size_read(dir), sbi), ei->flags); + exfat_bytes_to_cluster(sbi, i_size_read(dir)), ei->flags); /* check the validation of hint_stat and initialize it if required */ if (ei->version != (inode_peek_iversion_raw(dir) & 0xffffffff)) { @@ -681,7 +682,7 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, return -EIO; } - if (unlikely(EXFAT_B_TO_CLU_ROUND_UP(info->size, sbi) > sbi->used_clusters)) { + if (unlikely(exfat_bytes_to_cluster_round_up(sbi, info->size) > sbi->used_clusters)) { exfat_fs_error(sb, "data size is invalid(%lld)", info->size); return -EIO; } @@ -695,7 +696,8 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, if (info->type == TYPE_DIR) { exfat_chain_set(&cdir, info->start_clu, - EXFAT_B_TO_CLU(info->size, sbi), info->flags); + exfat_bytes_to_cluster(sbi, info->size), + info->flags); count = exfat_count_dir_entries(sb, &cdir); if (count < 0) return -EIO; @@ -705,71 +707,44 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, return 0; } -static int exfat_d_anon_disconn(struct dentry *dentry) -{ - return IS_ROOT(dentry) && (dentry->d_flags & DCACHE_DISCONNECTED); -} - static struct dentry *exfat_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct super_block *sb = dir->i_sb; - struct inode *inode; + struct inode *inode = NULL; struct dentry *alias; struct exfat_dir_entry info; int err; loff_t i_pos; - mode_t i_mode; mutex_lock(&EXFAT_SB(sb)->s_lock); err = exfat_find(dir, &dentry->d_name, &info); if (err) { - if (err == -ENOENT) { - inode = NULL; - goto out; - } - goto unlock; + if (unlikely(err != -ENOENT)) + inode = ERR_PTR(err); + goto out; } i_pos = exfat_make_i_pos(&info); inode = exfat_build_inode(sb, &info, i_pos); - err = PTR_ERR_OR_ZERO(inode); - if (err) - goto unlock; + if (IS_ERR(inode) || S_ISDIR(inode->i_mode)) + goto out; - i_mode = inode->i_mode; alias = d_find_alias(inode); /* * Checking "alias->d_parent == dentry->d_parent" to make sure * FS is not corrupted (especially double linked dir). */ - if (alias && alias->d_parent == dentry->d_parent && - !exfat_d_anon_disconn(alias)) { - + if (alias && alias->d_parent == dentry->d_parent) { /* - * Unhashed alias is able to exist because of revalidate() - * called by lookup_fast. You can easily make this status - * by calling create and lookup concurrently - * In such case, we reuse an alias instead of new dentry + * This inode has a hashed alias dentry with different + * name. This means, the user did ->lookup() by an + * another name (longname vs 8.3 alias of it) in past. + * + * Switch to new one for reason of locality if possible. */ - if (d_unhashed(alias)) { - WARN_ON(alias->d_name.hash_len != - dentry->d_name.hash_len); - exfat_info(sb, "rehashed a dentry(%p) in read lookup", - alias); - d_drop(dentry); - d_rehash(alias); - } else if (!S_ISDIR(i_mode)) { - /* - * This inode has non anonymous-DCACHE_DISCONNECTED - * dentry. This means, the user did ->lookup() by an - * another name (longname vs 8.3 alias of it) in past. - * - * Switch to new one for reason of locality if possible. - */ - d_move(alias, dentry); - } + d_move(alias, dentry); iput(inode); mutex_unlock(&EXFAT_SB(sb)->s_lock); return alias; @@ -781,9 +756,6 @@ out: exfat_d_version_set(dentry, inode_query_iversion(dir)); return d_splice_alias(inode, dentry); -unlock: - mutex_unlock(&EXFAT_SB(sb)->s_lock); - return ERR_PTR(err); } /* remove an entry, BUT don't truncate */ @@ -814,7 +786,7 @@ static int exfat_unlink(struct inode *dir, struct dentry *dentry) exfat_set_volume_dirty(sb); /* update the directory entry */ - exfat_remove_entries(inode, &es, ES_IDX_FILE); + exfat_remove_entries(inode, &es, ES_IDX_FILE, true); err = exfat_put_dentry_set(&es, IS_DIRSYNC(inode)); if (err) @@ -951,7 +923,7 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) } exfat_chain_set(&clu_to_free, ei->start_clu, - EXFAT_B_TO_CLU_ROUND_UP(i_size_read(inode), sbi), ei->flags); + exfat_bytes_to_cluster_round_up(sbi, i_size_read(inode)), ei->flags); err = exfat_check_dir_empty(sb, &clu_to_free); if (err) { @@ -969,7 +941,7 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) exfat_set_volume_dirty(sb); - exfat_remove_entries(inode, &es, ES_IDX_FILE); + exfat_remove_entries(inode, &es, ES_IDX_FILE, true); err = exfat_put_dentry_set(&es, IS_DIRSYNC(dir)); if (err) @@ -996,6 +968,23 @@ unlock: return err; } +/* + * Count benign secondary entries beyond the filename entries. + * Returns the count, or -EIO if the entry set is inconsistent. + */ +static int exfat_count_extra_entries(struct exfat_entry_set_cache *es) +{ + struct exfat_dentry *stream; + unsigned int name_entries; + int extra; + + stream = exfat_get_dentry_cached(es, ES_IDX_STREAM); + name_entries = EXFAT_FILENAME_ENTRY_NUM(stream->dentry.stream.name_len); + extra = es->num_entries - (ES_IDX_FIRST_FILENAME + name_entries); + + return extra >= 0 ? extra : -EIO; +} + static int exfat_rename_file(struct inode *parent_inode, struct exfat_uni_name *p_uniname, struct exfat_inode_info *ei) { @@ -1004,6 +993,7 @@ static int exfat_rename_file(struct inode *parent_inode, struct super_block *sb = parent_inode->i_sb; struct exfat_entry_set_cache old_es, new_es; int sync = IS_DIRSYNC(parent_inode); + unsigned int num_extra_entries, num_total_entries; if (unlikely(exfat_forced_shutdown(sb))) return -EIO; @@ -1013,19 +1003,23 @@ static int exfat_rename_file(struct inode *parent_inode, return num_new_entries; ret = exfat_get_dentry_set_by_ei(&old_es, sb, ei); - if (ret) { - ret = -EIO; - return ret; - } + if (ret) + return -EIO; epold = exfat_get_dentry_cached(&old_es, ES_IDX_FILE); - if (old_es.num_entries < num_new_entries) { + ret = exfat_count_extra_entries(&old_es); + if (ret < 0) + goto put_old_es; + num_extra_entries = ret; + num_total_entries = num_new_entries + num_extra_entries; + + if (old_es.num_entries < num_total_entries) { int newentry; struct exfat_chain dir; newentry = exfat_find_empty_entry(parent_inode, &dir, - num_new_entries, &new_es); + num_total_entries, &new_es); if (newentry < 0) { ret = newentry; /* -EIO or -ENOSPC */ goto put_old_es; @@ -1042,13 +1036,23 @@ static int exfat_rename_file(struct inode *parent_inode, epnew = exfat_get_dentry_cached(&new_es, ES_IDX_STREAM); *epnew = *epold; - exfat_init_ext_entry(&new_es, num_new_entries, p_uniname); + exfat_init_ext_entry(&new_es, num_new_entries, p_uniname, + &old_es, num_extra_entries); ret = exfat_put_dentry_set(&new_es, sync); - if (ret) + if (ret) { + /* Best-effort delete to avoid duplicate entries */ + if (!exfat_get_dentry_set(&new_es, sb, + &dir, newentry, + ES_ALL_ENTRIES)) { + exfat_remove_entries(parent_inode, &new_es, + ES_IDX_FILE, false); + exfat_put_dentry_set(&new_es, false); + } goto put_old_es; + } - exfat_remove_entries(parent_inode, &old_es, ES_IDX_FILE); + exfat_remove_entries(parent_inode, &old_es, ES_IDX_FILE, false); ei->dir = dir; ei->entry = newentry; } else { @@ -1057,8 +1061,8 @@ static int exfat_rename_file(struct inode *parent_inode, ei->attr |= EXFAT_ATTR_ARCHIVE; } - exfat_remove_entries(parent_inode, &old_es, ES_IDX_FIRST_FILENAME + 1); - exfat_init_ext_entry(&old_es, num_new_entries, p_uniname); + exfat_init_ext_entry(&old_es, num_new_entries, p_uniname, + &old_es, num_extra_entries); } return exfat_put_dentry_set(&old_es, sync); @@ -1074,6 +1078,7 @@ static int exfat_move_file(struct inode *parent_inode, struct exfat_dentry *epmov, *epnew; struct exfat_entry_set_cache mov_es, new_es; struct exfat_chain newdir; + unsigned int num_extra_entries, num_total_entries; num_new_entries = exfat_calc_num_entries(p_uniname); if (num_new_entries < 0) @@ -1083,8 +1088,14 @@ static int exfat_move_file(struct inode *parent_inode, if (ret) return -EIO; + ret = exfat_count_extra_entries(&mov_es); + if (ret < 0) + goto put_mov_es; + num_extra_entries = ret; + num_total_entries = num_new_entries + num_extra_entries; + newentry = exfat_find_empty_entry(parent_inode, &newdir, - num_new_entries, &new_es); + num_total_entries, &new_es); if (newentry < 0) { ret = newentry; /* -EIO or -ENOSPC */ goto put_mov_es; @@ -1102,21 +1113,31 @@ static int exfat_move_file(struct inode *parent_inode, epnew = exfat_get_dentry_cached(&new_es, ES_IDX_STREAM); *epnew = *epmov; - exfat_init_ext_entry(&new_es, num_new_entries, p_uniname); - exfat_remove_entries(parent_inode, &mov_es, ES_IDX_FILE); + exfat_init_ext_entry(&new_es, num_new_entries, p_uniname, + &mov_es, num_extra_entries); + + exfat_remove_entries(parent_inode, &mov_es, ES_IDX_FILE, false); ei->dir = newdir; ei->entry = newentry; ret = exfat_put_dentry_set(&new_es, IS_DIRSYNC(parent_inode)); - if (ret) + if (ret) { + /* Best-effort delete to avoid duplicate entries */ + if (!exfat_get_dentry_set(&new_es, parent_inode->i_sb, + &newdir, newentry, + ES_ALL_ENTRIES)) { + exfat_remove_entries(parent_inode, &new_es, + ES_IDX_FILE, false); + exfat_put_dentry_set(&new_es, false); + } goto put_mov_es; + } return exfat_put_dentry_set(&mov_es, IS_DIRSYNC(parent_inode)); put_mov_es: exfat_put_dentry_set(&mov_es, false); - return ret; } @@ -1158,8 +1179,8 @@ static int __exfat_rename(struct inode *old_parent_inode, new_clu.dir = new_ei->start_clu; new_clu.size = - EXFAT_B_TO_CLU_ROUND_UP(i_size_read(new_inode), - sbi); + exfat_bytes_to_cluster_round_up(sbi, + i_size_read(new_inode)); new_clu.flags = new_ei->flags; ret = exfat_check_dir_empty(sb, &new_clu); @@ -1190,7 +1211,7 @@ static int __exfat_rename(struct inode *old_parent_inode, goto del_out; } - exfat_remove_entries(new_inode, &es, ES_IDX_FILE); + exfat_remove_entries(new_inode, &es, ES_IDX_FILE, true); ret = exfat_put_dentry_set(&es, IS_DIRSYNC(new_inode)); if (ret) @@ -1203,8 +1224,8 @@ static int __exfat_rename(struct inode *old_parent_inode, struct exfat_chain new_clu_to_free; exfat_chain_set(&new_clu_to_free, new_ei->start_clu, - EXFAT_B_TO_CLU_ROUND_UP(i_size_read(new_inode), - sbi), new_ei->flags); + exfat_bytes_to_cluster_round_up(sbi, i_size_read(new_inode)), + new_ei->flags); if (exfat_free_cluster(new_inode, &new_clu_to_free)) { /* just set I/O error only */ @@ -1311,4 +1332,5 @@ const struct inode_operations exfat_dir_inode_operations = { .rename = exfat_rename, .setattr = exfat_setattr, .getattr = exfat_getattr, + .fileattr_get = exfat_fileattr_get, }; diff --git a/fs/exfat/nls.c b/fs/exfat/nls.c index 57db08a5271c..055447edcf9a 100644 --- a/fs/exfat/nls.c +++ b/fs/exfat/nls.c @@ -769,13 +769,18 @@ int exfat_create_upcase_table(struct super_block *sb) tbl_clu = le32_to_cpu(ep->dentry.upcase.start_clu); tbl_size = le64_to_cpu(ep->dentry.upcase.size); - - sector = exfat_cluster_to_sector(sbi, tbl_clu); - num_sectors = ((tbl_size - 1) >> blksize_bits) + 1; - ret = exfat_load_upcase_table(sb, sector, num_sectors, - le32_to_cpu(ep->dentry.upcase.checksum)); - + if (tbl_size) { + sector = exfat_cluster_to_sector(sbi, tbl_clu); + num_sectors = ((tbl_size - 1) >> blksize_bits) + 1; + ret = exfat_load_upcase_table(sb, sector, num_sectors, + le32_to_cpu(ep->dentry.upcase.checksum)); + } else { + exfat_fs_error(sb, + "bad upcase table size (0 bytes). Please run fsck"); + ret = -EINVAL; + } brelse(bh); + if (ret && ret != -EIO) { /* free memory from exfat_load_upcase_table call */ exfat_free_upcase_table(sbi); @@ -790,6 +795,8 @@ int exfat_create_upcase_table(struct super_block *sb) return -EIO; } + exfat_fs_error(sb, "no upcase table entry. Please run fsck"); + load_default: /* load default upcase table */ return exfat_load_default_upcase_table(sb); diff --git a/fs/exfat/super.c b/fs/exfat/super.c index 95d87e2d7717..388db271c6bf 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -369,7 +369,7 @@ static int exfat_read_root(struct inode *inode, struct exfat_chain *root_clu) ei->hint_stat.clu = sbi->root_dir; ei->hint_femp.eidx = EXFAT_HINT_NONE; - i_size_write(inode, EXFAT_CLU_TO_B(root_clu->size, sbi)); + i_size_write(inode, exfat_cluster_to_bytes(sbi, root_clu->size)); num_subdirs = exfat_count_dir_entries(sb, root_clu); if (num_subdirs < 0) @@ -499,6 +499,7 @@ static int exfat_read_boot_sector(struct super_block *sb) if (p_boot->num_fats == 2) sbi->FAT2_start_sector += sbi->num_FAT_sectors; sbi->data_start_sector = le32_to_cpu(p_boot->clu_offset); + sbi->data_start_bytes = sbi->data_start_sector << p_boot->sect_size_bits; sbi->num_sectors = le64_to_cpu(p_boot->vol_length); /* because the cluster index starts with 2 */ sbi->num_clusters = le32_to_cpu(p_boot->clu_count) + @@ -538,7 +539,7 @@ static int exfat_read_boot_sector(struct super_block *sb) * machines. */ sb->s_maxbytes = min(MAX_LFS_FILESIZE, - EXFAT_CLU_TO_B((loff_t)EXFAT_MAX_NUM_CLUSTER, sbi)); + exfat_cluster_to_bytes(sbi, (loff_t)EXFAT_MAX_NUM_CLUSTER)); /* check logical sector size */ if (exfat_calibrate_blocksize(sb, 1 << p_boot->sect_size_bits)) diff --git a/fs/exportfs/expfs.c b/fs/exportfs/expfs.c index fbd45e7ae706..eafd99507afe 100644 --- a/fs/exportfs/expfs.c +++ b/fs/exportfs/expfs.c @@ -53,10 +53,10 @@ find_acceptable_alias(struct dentry *result, inode = result->d_inode; spin_lock(&inode->i_lock); for_each_alias(dentry, inode) { - dget(dentry); + if (!dget_alias_ilocked(dentry)) + continue; spin_unlock(&inode->i_lock); - if (toput) - dput(toput); + dput(toput); if (dentry != result && acceptable(context, dentry)) { dput(result); return dentry; @@ -66,8 +66,7 @@ find_acceptable_alias(struct dentry *result, } spin_unlock(&inode->i_lock); - if (toput) - dput(toput); + dput(toput); return NULL; } diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 3eb1f342645c..79f7b395258c 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -114,8 +114,6 @@ struct ext2_sb_info { */ spinlock_t s_lock; struct mb_cache *s_ea_block_cache; - struct dax_device *s_daxdev; - u64 s_dax_part_off; }; static inline spinlock_t * @@ -373,11 +371,9 @@ struct ext2_inode { #define EXT2_MOUNT_NO_UID32 0x000200 /* Disable 32-bit UIDs */ #define EXT2_MOUNT_XATTR_USER 0x004000 /* Extended user attributes */ #define EXT2_MOUNT_POSIX_ACL 0x008000 /* POSIX Access Control Lists */ -#define EXT2_MOUNT_XIP 0x010000 /* Obsolete, use DAX */ #define EXT2_MOUNT_USRQUOTA 0x020000 /* user quota */ #define EXT2_MOUNT_GRPQUOTA 0x040000 /* group quota */ #define EXT2_MOUNT_RESERVATION 0x080000 /* Preallocation */ -#define EXT2_MOUNT_DAX 0x100000 /* Direct Access */ #define clear_opt(o, opt) o &= ~EXT2_MOUNT_##opt diff --git a/fs/ext2/file.c b/fs/ext2/file.c index d9b1eb34694a..8dca9ec4cacd 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -21,7 +21,6 @@ #include <linux/time.h> #include <linux/pagemap.h> -#include <linux/dax.h> #include <linux/filelock.h> #include <linux/quotaops.h> #include <linux/iomap.h> @@ -32,111 +31,6 @@ #include "acl.h" #include "trace.h" -#ifdef CONFIG_FS_DAX -static ssize_t ext2_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) -{ - struct inode *inode = iocb->ki_filp->f_mapping->host; - ssize_t ret; - - if (!iov_iter_count(to)) - return 0; /* skip atime */ - - inode_lock_shared(inode); - ret = dax_iomap_rw(iocb, to, &ext2_iomap_ops); - inode_unlock_shared(inode); - - file_accessed(iocb->ki_filp); - return ret; -} - -static ssize_t ext2_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) -{ - struct file *file = iocb->ki_filp; - struct inode *inode = file->f_mapping->host; - ssize_t ret; - - inode_lock(inode); - ret = generic_write_checks(iocb, from); - if (ret <= 0) - goto out_unlock; - ret = file_remove_privs(file); - if (ret) - goto out_unlock; - ret = file_update_time(file); - if (ret) - goto out_unlock; - - ret = dax_iomap_rw(iocb, from, &ext2_iomap_ops); - if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { - i_size_write(inode, iocb->ki_pos); - mark_inode_dirty(inode); - } - -out_unlock: - inode_unlock(inode); - if (ret > 0) - ret = generic_write_sync(iocb, ret); - return ret; -} - -/* - * The lock ordering for ext2 DAX fault paths is: - * - * mmap_lock (MM) - * sb_start_pagefault (vfs, freeze) - * address_space->invalidate_lock - * address_space->i_mmap_rwsem or page_lock (mutually exclusive in DAX) - * ext2_inode_info->truncate_mutex - * - * The default page_lock and i_size verification done by non-DAX fault paths - * is sufficient because ext2 doesn't support hole punching. - */ -static vm_fault_t ext2_dax_fault(struct vm_fault *vmf) -{ - struct inode *inode = file_inode(vmf->vma->vm_file); - vm_fault_t ret; - bool write = (vmf->flags & FAULT_FLAG_WRITE) && - (vmf->vma->vm_flags & VM_SHARED); - - if (write) { - sb_start_pagefault(inode->i_sb); - file_update_time(vmf->vma->vm_file); - } - filemap_invalidate_lock_shared(inode->i_mapping); - - ret = dax_iomap_fault(vmf, 0, NULL, NULL, &ext2_iomap_ops); - - filemap_invalidate_unlock_shared(inode->i_mapping); - if (write) - sb_end_pagefault(inode->i_sb); - return ret; -} - -static const struct vm_operations_struct ext2_dax_vm_ops = { - .fault = ext2_dax_fault, - /* - * .huge_fault is not supported for DAX because allocation in ext2 - * cannot be reliably aligned to huge page sizes and so pmd faults - * will always fail and fail back to regular faults. - */ - .page_mkwrite = ext2_dax_fault, - .pfn_mkwrite = ext2_dax_fault, -}; - -static int ext2_file_mmap_prepare(struct vm_area_desc *desc) -{ - struct file *file = desc->file; - - if (!IS_DAX(file_inode(file))) - return generic_file_mmap_prepare(desc); - - file_accessed(file); - desc->vm_ops = &ext2_dax_vm_ops; - return 0; -} -#else -#define ext2_file_mmap_prepare generic_file_mmap_prepare -#endif /* * Called when filp is released. This happens when all file descriptors @@ -267,12 +161,15 @@ static ssize_t ext2_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) endbyte = pos + status - 1; ret2 = filemap_write_and_wait_range(inode->i_mapping, pos, endbyte); - if (!ret2) + if (!ret2) { invalidate_mapping_pages(inode->i_mapping, pos >> PAGE_SHIFT, endbyte >> PAGE_SHIFT); - if (ret > 0) - generic_write_sync(iocb, ret); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + } else { + ret = ret2; + } } out_unlock: @@ -285,10 +182,6 @@ out_unlock: static ssize_t ext2_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { -#ifdef CONFIG_FS_DAX - if (IS_DAX(iocb->ki_filp->f_mapping->host)) - return ext2_dax_read_iter(iocb, to); -#endif if (iocb->ki_flags & IOCB_DIRECT) return ext2_dio_read_iter(iocb, to); @@ -297,10 +190,6 @@ static ssize_t ext2_file_read_iter(struct kiocb *iocb, struct iov_iter *to) static ssize_t ext2_file_write_iter(struct kiocb *iocb, struct iov_iter *from) { -#ifdef CONFIG_FS_DAX - if (IS_DAX(iocb->ki_filp->f_mapping->host)) - return ext2_dax_write_iter(iocb, from); -#endif if (iocb->ki_flags & IOCB_DIRECT) return ext2_dio_write_iter(iocb, from); @@ -321,7 +210,7 @@ const struct file_operations ext2_file_operations = { #ifdef CONFIG_COMPAT .compat_ioctl = ext2_compat_ioctl, #endif - .mmap_prepare = ext2_file_mmap_prepare, + .mmap_prepare = generic_file_mmap_prepare, .open = ext2_file_open, .release = ext2_release_file, .fsync = ext2_fsync, diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 74aca5eb572d..29808629cce5 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -26,7 +26,6 @@ #include <linux/time.h> #include <linux/highuid.h> #include <linux/pagemap.h> -#include <linux/dax.h> #include <linux/blkdev.h> #include <linux/quotaops.h> #include <linux/writeback.h> @@ -741,27 +740,6 @@ static int ext2_get_blocks(struct inode *inode, goto cleanup; } - if (IS_DAX(inode)) { - /* - * We must unmap blocks before zeroing so that writeback cannot - * overwrite zeros with stale data from block device page cache. - */ - clean_bdev_aliases(inode->i_sb->s_bdev, - le32_to_cpu(chain[depth-1].key), - count); - /* - * block must be initialised before we put it in the tree - * so that it's not found by another thread before it's - * initialised - */ - err = sb_issue_zeroout(inode->i_sb, - le32_to_cpu(chain[depth-1].key), count, - GFP_KERNEL); - if (err) { - mutex_unlock(&ei->truncate_mutex); - goto cleanup; - } - } *new = true; ext2_splice_branch(inode, iblock, partial, indirect_blks, count); @@ -811,7 +789,6 @@ static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int blkbits = inode->i_blkbits; unsigned long first_block = offset >> blkbits; unsigned long max_blocks = (length + (1 << blkbits) - 1) >> blkbits; - struct ext2_sb_info *sbi = EXT2_SB(inode->i_sb); bool new = false, boundary = false; u32 bno; int ret; @@ -841,10 +818,7 @@ static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length, iomap->flags = 0; iomap->offset = (u64)first_block << blkbits; - if (flags & IOMAP_DAX) - iomap->dax_dev = sbi->s_daxdev; - else - iomap->bdev = inode->i_sb->s_bdev; + iomap->bdev = inode->i_sb->s_bdev; if (ret == 0) { /* @@ -859,8 +833,6 @@ static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length, } else { iomap->type = IOMAP_MAPPED; iomap->addr = (u64)bno << blkbits; - if (flags & IOMAP_DAX) - iomap->addr += sbi->s_dax_part_off; iomap->length = (u64)ret << blkbits; iomap->flags |= IOMAP_F_MERGED; } @@ -962,13 +934,6 @@ ext2_writepages(struct address_space *mapping, struct writeback_control *wbc) return mpage_writepages(mapping, wbc, ext2_get_block); } -static int -ext2_dax_writepages(struct address_space *mapping, struct writeback_control *wbc) -{ - struct ext2_sb_info *sbi = EXT2_SB(mapping->host->i_sb); - - return dax_writeback_mapping_range(mapping, sbi->s_daxdev, wbc); -} const struct address_space_operations ext2_aops = { .dirty_folio = block_dirty_folio, @@ -984,10 +949,6 @@ const struct address_space_operations ext2_aops = { .error_remove_folio = generic_error_remove_folio, }; -static const struct address_space_operations ext2_dax_aops = { - .writepages = ext2_dax_writepages, - .dirty_folio = noop_dirty_folio, -}; /* * Probably it should be a library function... search for first non-zero word @@ -1186,9 +1147,6 @@ static void __ext2_truncate_blocks(struct inode *inode, loff_t offset) blocksize = inode->i_sb->s_blocksize; iblock = (offset + blocksize-1) >> EXT2_BLOCK_SIZE_BITS(inode->i_sb); -#ifdef CONFIG_FS_DAX - WARN_ON(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)); -#endif n = ext2_block_to_path(inode, iblock, offsets, NULL); if (n == 0) @@ -1290,12 +1248,7 @@ static int ext2_setsize(struct inode *inode, loff_t newsize) inode_dio_wait(inode); - if (IS_DAX(inode)) - error = dax_truncate_page(inode, newsize, NULL, - &ext2_iomap_ops); - else - error = block_truncate_page(inode->i_mapping, - newsize, ext2_get_block); + error = block_truncate_page(inode->i_mapping, newsize, ext2_get_block); if (error) return error; @@ -1363,7 +1316,7 @@ void ext2_set_inode_flags(struct inode *inode) unsigned int flags = EXT2_I(inode)->i_flags; inode->i_flags &= ~(S_SYNC | S_APPEND | S_IMMUTABLE | S_NOATIME | - S_DIRSYNC | S_DAX); + S_DIRSYNC); if (flags & EXT2_SYNC_FL) inode->i_flags |= S_SYNC; if (flags & EXT2_APPEND_FL) @@ -1374,18 +1327,13 @@ void ext2_set_inode_flags(struct inode *inode) inode->i_flags |= S_NOATIME; if (flags & EXT2_DIRSYNC_FL) inode->i_flags |= S_DIRSYNC; - if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode)) - inode->i_flags |= S_DAX; } void ext2_set_file_ops(struct inode *inode) { inode->i_op = &ext2_file_inode_operations; inode->i_fop = &ext2_file_operations; - if (IS_DAX(inode)) - inode->i_mapping->a_ops = &ext2_dax_aops; - else - inode->i_mapping->a_ops = &ext2_aops; + inode->i_mapping->a_ops = &ext2_aops; } struct inode *ext2_iget (struct super_block *sb, unsigned long ino) diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 19f76d8cb473..3999f8f3b156 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -34,7 +34,6 @@ #include <linux/log2.h> #include <linux/quotaops.h> #include <linux/uaccess.h> -#include <linux/dax.h> #include <linux/iversion.h> #include "ext2.h" #include "xattr.h" @@ -198,7 +197,6 @@ static void ext2_put_super (struct super_block * sb) brelse (sbi->s_sbh); sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); - fs_put_dax(sbi->s_daxdev, NULL); kfree(sbi); } @@ -329,11 +327,7 @@ static int ext2_show_options(struct seq_file *seq, struct dentry *root) if (test_opt(sb, GRPQUOTA)) seq_puts(seq, ",grpquota"); - if (test_opt(sb, XIP)) - seq_puts(seq, ",xip"); - if (test_opt(sb, DAX)) - seq_puts(seq, ",dax"); if (!test_opt(sb, RESERVATION)) seq_puts(seq, ",noreservation"); @@ -596,18 +590,10 @@ static int ext2_parse_param(struct fs_context *fc, struct fs_parameter *param) break; #endif case Opt_xip: - ext2_msg_fc(fc, KERN_INFO, "use dax instead of xip"); - ctx_set_mount_opt(ctx, EXT2_MOUNT_XIP); - fallthrough; + ext2_msg_fc(fc, KERN_WARNING, "DAX support has been removed. xip option ignored."); + break; case Opt_dax: -#ifdef CONFIG_FS_DAX - ext2_msg_fc(fc, KERN_WARNING, - "DAX enabled. Warning: DAX support in ext2 driver is deprecated" - " and will be removed at the end of 2025. Please use ext4 driver instead."); - ctx_set_mount_opt(ctx, EXT2_MOUNT_DAX); -#else - ext2_msg_fc(fc, KERN_INFO, "dax option not supported"); -#endif + ext2_msg_fc(fc, KERN_WARNING, "DAX support has been removed. dax option ignored."); break; #if defined(CONFIG_QUOTA) @@ -906,8 +892,6 @@ static int ext2_fill_super(struct super_block *sb, struct fs_context *fc) } sb->s_fs_info = sbi; sbi->s_sb_block = sb_block; - sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off, - NULL, NULL); spin_lock_init(&sbi->s_lock); ret = -EINVAL; @@ -992,17 +976,6 @@ static int ext2_fill_super(struct super_block *sb, struct fs_context *fc) } blocksize = BLOCK_SIZE << le32_to_cpu(sbi->s_es->s_log_block_size); - if (test_opt(sb, DAX)) { - if (!sbi->s_daxdev) { - ext2_msg(sb, KERN_ERR, - "DAX unsupported by block device. Turning off DAX."); - clear_opt(sbi->s_mount_opt, DAX); - } else if (blocksize != PAGE_SIZE) { - ext2_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); - clear_opt(sbi->s_mount_opt, DAX); - } - } - /* If the blocksize doesn't match, re-read the thing.. */ if (sb->s_blocksize != blocksize) { brelse(bh); @@ -1252,7 +1225,6 @@ failed_mount_group_desc: failed_mount: brelse(bh); failed_sbi: - fs_put_dax(sbi->s_daxdev, NULL); sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); kfree(sbi); @@ -1379,11 +1351,6 @@ static int ext2_reconfigure(struct fs_context *fc) spin_lock(&sbi->s_lock); es = sbi->s_es; - if ((sbi->s_mount_opt ^ new_opts.s_mount_opt) & EXT2_MOUNT_DAX) { - ext2_msg(sb, KERN_WARNING, "warning: refusing change of " - "dax flag with busy inodes while remounting"); - new_opts.s_mount_opt ^= EXT2_MOUNT_DAX; - } if ((bool)(flags & SB_RDONLY) == sb_rdonly(sb)) goto out_set; if (flags & SB_RDONLY) { diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile index 3baee4e7c1cf..3f9fc0eb8eca 100644 --- a/fs/ext4/Makefile +++ b/fs/ext4/Makefile @@ -15,7 +15,7 @@ ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \ ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o ext4-test-objs += inode-test.o mballoc-test.o \ - extents-test.o + extents-test.o hash-test.o obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-test.o ext4-$(CONFIG_FS_VERITY) += verity.o ext4-$(CONFIG_FS_ENCRYPTION) += crypto.o diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 94283a991e5c..b37c136ea3ab 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1015,14 +1015,32 @@ do { \ * than the first * I_DATA_SEM_QUOTA - Used for quota inodes only * I_DATA_SEM_EA - Used for ea_inodes only + * I_DATA_SEM_JOURNAL - Used for journal inode only */ enum { I_DATA_SEM_NORMAL = 0, I_DATA_SEM_OTHER, I_DATA_SEM_QUOTA, - I_DATA_SEM_EA + I_DATA_SEM_EA, + I_DATA_SEM_JOURNAL }; +struct ext4_fc_inode_snap; + +/* + * Snapshot failure reasons for ext4_fc_lock_updates tracepoint. + * Keep these stable for tooling. + */ +enum ext4_fc_snap_err { + EXT4_FC_SNAP_ERR_NONE = 0, + EXT4_FC_SNAP_ERR_ES_MISS, + EXT4_FC_SNAP_ERR_ES_DELAYED, + EXT4_FC_SNAP_ERR_ES_OTHER, + EXT4_FC_SNAP_ERR_INODES_CAP, + EXT4_FC_SNAP_ERR_RANGES_CAP, + EXT4_FC_SNAP_ERR_NOMEM, + EXT4_FC_SNAP_ERR_INODE_LOC, +}; /* * fourth extended file system inode data in memory @@ -1079,6 +1097,22 @@ struct ext4_inode_info { /* End of lblk range that needs to be committed in this fast commit */ ext4_lblk_t i_fc_lblk_len; + /* + * Commit-time fast commit snapshots. + * + * i_fc_snap is installed and freed under sbi->s_fc_lock. The fast + * commit log writing path reads the snapshot under sbi->s_fc_lock while + * serializing fast commit TLVs. + * + * The snapshot lifetime is bounded by EXT4_STATE_FC_COMMITTING and the + * corresponding cleanup / eviction paths. + * + * i_fc_snap points to per-inode snapshot data for fast commit: + * - a raw inode snapshot for EXT4_FC_TAG_INODE + * - data range records for EXT4_FC_TAG_{ADD,DEL}_RANGE + */ + struct ext4_fc_inode_snap *i_fc_snap; + spinlock_t i_raw_lock; /* protects updates to the raw inode */ /* @@ -1517,6 +1551,36 @@ struct ext4_orphan_info { }; /* + * Ext4 fast commit snapshot statistics. + * + * These are best-effort counters intended for debugging / performance + * introspection; they are not exact under concurrent updates. + */ +struct ext4_fc_snap_stats { + atomic64_t lock_updates_ns_total; + atomic64_t lock_updates_ns_max; + atomic64_t lock_updates_samples; + + atomic64_t snap_inodes; + atomic64_t snap_ranges; + + atomic64_t snap_fail_es_miss; + atomic64_t snap_fail_es_delayed; + atomic64_t snap_fail_es_other; + + atomic64_t snap_fail_inodes_cap; + atomic64_t snap_fail_ranges_cap; + atomic64_t snap_fail_nomem; + atomic64_t snap_fail_inode_loc; + + /* + * Missing inode snapshots during log writing should never happen. + * Keep this counter to help catch unexpected regressions. + */ + atomic64_t snap_fail_no_snap; +}; + +/* * fourth extended-fs super-block data in memory */ struct ext4_sb_info { @@ -1790,6 +1854,7 @@ struct ext4_sb_info { struct mutex s_fc_lock; struct buffer_head *s_fc_bh; struct ext4_fc_stats s_fc_stats; + struct ext4_fc_snap_stats s_fc_snap_stats; tid_t s_fc_ineligible_tid; #ifdef CONFIG_EXT4_DEBUG int s_fc_debug_max_replay; @@ -1972,6 +2037,7 @@ enum { EXT4_STATE_FC_COMMITTING, /* Fast commit ongoing */ EXT4_STATE_FC_FLUSHING_DATA, /* Fast commit flushing data */ EXT4_STATE_ORPHAN_FILE, /* Inode orphaned in orphan file */ + EXT4_STATE_FC_REQUEUE, /* Inode modified during fast commit */ }; #define EXT4_INODE_BIT_FNS(name, field, offset) \ @@ -2000,6 +2066,8 @@ EXT4_INODE_BIT_FNS(flag, flags, 0) static inline int ext4_test_inode_state(struct inode *inode, int bit); static inline void ext4_set_inode_state(struct inode *inode, int bit); static inline void ext4_clear_inode_state(struct inode *inode, int bit); +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode); +static inline int ext4_inode_state_wait_bit(int bit); #if (BITS_PER_LONG < 64) EXT4_INODE_BIT_FNS(state, state_flags, 0) @@ -2015,6 +2083,24 @@ static inline void ext4_clear_state_flags(struct ext4_inode_info *ei) /* We depend on the fact that callers will set i_flags */ } #endif + +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode) +{ +#if (BITS_PER_LONG < 64) + return &EXT4_I(inode)->i_state_flags; +#else + return &EXT4_I(inode)->i_flags; +#endif +} + +static inline int ext4_inode_state_wait_bit(int bit) +{ +#if (BITS_PER_LONG < 64) + return bit; +#else + return bit + 32; +#endif +} #else /* Assume that user mode programs are passing in an ext4fs superblock, not * a kernel struct super_block. This will allow us to call the feature-test @@ -2959,7 +3045,7 @@ extern unsigned long ext4_count_dirs(struct super_block *); extern void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap); extern int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, int barrier); -extern void ext4_end_bitmap_read(struct buffer_head *bh, int uptodate); +void ext4_end_bitmap_read(struct bio *bio); /* fast_commit.c */ int ext4_fc_info_show(struct seq_file *seq, void *v); @@ -3080,8 +3166,9 @@ extern int ext4_file_getattr(struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int); extern void ext4_dirty_inode(struct inode *, int); extern int ext4_change_inode_journal_flag(struct inode *, int); -extern int ext4_get_inode_loc(struct inode *, struct ext4_iloc *); -extern int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, +int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc); +int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc); +int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, struct ext4_iloc *iloc); extern int ext4_inode_attach_jinode(struct inode *inode); extern int ext4_can_truncate(struct inode *inode); @@ -3184,10 +3271,10 @@ extern struct buffer_head *ext4_sb_bread_unmovable(struct super_block *sb, sector_t block); extern struct buffer_head *ext4_sb_bread_nofail(struct super_block *sb, sector_t block); -extern void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail); -extern int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail); +void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); +int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); extern int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait); extern void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block); extern int ext4_seq_options_show(struct seq_file *seq, void *offset); diff --git a/fs/ext4/extents-test.c b/fs/ext4/extents-test.c index 6b53a3f39fcd..bd7795a82607 100644 --- a/fs/ext4/extents-test.c +++ b/fs/ext4/extents-test.c @@ -37,6 +37,7 @@ #include <kunit/test.h> #include <kunit/static_stub.h> +#include <linux/fs_context.h> #include <linux/gfp_types.h> #include <linux/stddef.h> @@ -130,14 +131,20 @@ static void ext_kill_sb(struct super_block *sb) generic_shutdown_super(sb); } -static int ext_set(struct super_block *sb, void *data) +static int ext_init_fs_context(struct fs_context *fc) +{ + return 0; +} + +static int ext_set(struct super_block *sb, struct fs_context *fc) { return 0; } static struct file_system_type ext_fs_type = { - .name = "extents test", - .kill_sb = ext_kill_sb, + .name = "extents test", + .init_fs_context = ext_init_fs_context, + .kill_sb = ext_kill_sb, }; static void extents_kunit_exit(struct kunit *test) @@ -223,6 +230,7 @@ static int extents_kunit_init(struct kunit *test) struct ext4_inode_info *ei; struct inode *inode; struct super_block *sb; + struct fs_context *fc; struct ext4_sb_info *sbi = NULL; struct kunit_ext_test_param *param = (struct kunit_ext_test_param *)(test->param_value); @@ -232,7 +240,13 @@ static int extents_kunit_init(struct kunit *test) if (sbi == NULL) return -ENOMEM; - sb = sget(&ext_fs_type, NULL, ext_set, 0, NULL); + fc = fs_context_for_mount(&ext_fs_type, 0); + if (IS_ERR(fc)) { + kfree(sbi); + return PTR_ERR(fc); + } + sb = sget_fc(fc, NULL, ext_set); + put_fs_context(fc); if (IS_ERR(sb)) { kfree(sbi); return PTR_ERR(sb); diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 125f628e738a..91c97af64b31 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -3268,8 +3268,8 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, */ path = ext4_find_extent(inode, ee_block, NULL, flags | EXT4_EX_NOFAIL); if (IS_ERR(path)) { - EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld", - split, PTR_ERR(path)); + EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %pe", + split, path); goto out_path; } diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index b3c22636251d..8e2259799614 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -56,21 +56,22 @@ * deleted while it is being flushed. * [2] Flush data buffers to disk and clear "EXT4_STATE_FC_FLUSHING_DATA" * state. - * [3] Lock the journal by calling jbd2_journal_lock_updates. This ensures that - * all the exsiting handles finish and no new handles can start. - * [4] Mark all the fast commit eligible inodes as undergoing fast commit - * by setting "EXT4_STATE_FC_COMMITTING" state. - * [5] Unlock the journal by calling jbd2_journal_unlock_updates. This allows - * starting of new handles. If new handles try to start an update on - * any of the inodes that are being committed, ext4_fc_track_inode() - * will block until those inodes have finished the fast commit. + * [3] Lock the journal by calling jbd2_journal_lock_updates(). This ensures + * that all the existing handles finish and no new handles can start. + * [4] Mark all the fast commit eligible inodes as undergoing fast commit by + * setting "EXT4_STATE_FC_COMMITTING" state, and snapshot the inode state + * needed for log writing. + * [5] Unlock the journal by calling jbd2_journal_unlock_updates(). This allows + * starting of new handles. Updates to inodes being fast committed are + * tracked for requeue rather than blocking. * [6] Commit all the directory entry updates in the fast commit space. - * [7] Commit all the changed inodes in the fast commit space and clear - * "EXT4_STATE_FC_COMMITTING" for these inodes. + * [7] Commit all the changed inodes in the fast commit space. * [8] Write tail tag (this tag ensures the atomicity, please read the following * section for more details). + * [9] Clear "EXT4_STATE_FC_COMMITTING" and wake up waiters in + * ext4_fc_cleanup(). * - * All the inode updates must be enclosed within jbd2_jounrnal_start() + * All the inode updates must be enclosed within jbd2_journal_start() * and jbd2_journal_stop() similar to JBD2 journaling. * * Fast Commit Ineligibility @@ -183,9 +184,27 @@ #include <trace/events/ext4.h> static struct kmem_cache *ext4_fc_dentry_cachep; +static struct kmem_cache *ext4_fc_range_cachep; -static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate) +/* + * Avoid spending unbounded time/memory snapshotting highly fragmented files + * under jbd2_journal_lock_updates(). If we exceed this limit, fall back to + * full commit. + */ +#define EXT4_FC_SNAPSHOT_MAX_INODES 1024 +#define EXT4_FC_SNAPSHOT_MAX_RANGES 2048 + +static inline void ext4_fc_set_snap_err(int *snap_err, int err) +{ + if (snap_err && *snap_err == EXT4_FC_SNAP_ERR_NONE) + *snap_err = err; +} + +static void ext4_end_buffer_io_sync(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + BUFFER_TRACE(bh, ""); if (uptodate) { ext4_debug("%s: Block %lld up-to-date", @@ -200,6 +219,8 @@ static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate) unlock_buffer(bh); } +static void ext4_fc_free_inode_snap(struct inode *inode); + static inline void ext4_fc_reset_inode(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); @@ -214,8 +235,10 @@ void ext4_fc_init_inode(struct inode *inode) ext4_fc_reset_inode(inode); ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING); + ext4_clear_inode_state(inode, EXT4_STATE_FC_REQUEUE); INIT_LIST_HEAD(&ei->i_fc_list); INIT_LIST_HEAD(&ei->i_fc_dilist); + ei->i_fc_snap = NULL; } static bool ext4_fc_disabled(struct super_block *sb) @@ -231,6 +254,50 @@ static bool ext4_fc_eligible(struct super_block *sb) } /* + * Wait for an inode fast-commit state bit to clear while dropping the + * fast-commit lock around schedule(). + */ +static void ext4_fc_wait_inode_state(struct inode *inode, int bit, + int *alloc_ctx) +{ + wait_queue_head_t *wq; + unsigned long *wait_word = ext4_inode_state_wait_word(inode); + int wait_bit = ext4_inode_state_wait_bit(bit); + + while (ext4_test_inode_state(inode, bit)) { + DEFINE_WAIT_BIT(wait, wait_word, wait_bit); + + wq = bit_waitqueue(wait_word, wait_bit); + prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); + if (ext4_test_inode_state(inode, bit)) { + ext4_fc_unlock(inode->i_sb, *alloc_ctx); + schedule(); + *alloc_ctx = ext4_fc_lock(inode->i_sb); + } + finish_wait(wq, &wait.wq_entry); + } +} + +static inline void ext4_fc_wake_inode_state(struct inode *inode, int bit) +{ + wake_up_bit(ext4_inode_state_wait_word(inode), + ext4_inode_state_wait_bit(bit)); +} + +static void ext4_fc_snap_stats_update_max(atomic64_t *stat, u64 value) +{ + u64 old = atomic64_read(stat); + + while (value > old) { + u64 prev = atomic64_cmpxchg(stat, old, value); + + if (prev == old) + break; + old = prev; + } +} + +/* * Remove inode from fast commit list. If the inode is being committed * we wait until inode commit is done. */ @@ -238,7 +305,6 @@ void ext4_fc_del(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_dentry_update *fc_dentry; - wait_queue_head_t *wq; int alloc_ctx; if (ext4_fc_disabled(inode->i_sb)) @@ -246,59 +312,43 @@ void ext4_fc_del(struct inode *inode) alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { + ext4_fc_free_inode_snap(inode); ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } /* - * Since ext4_fc_del is called from ext4_evict_inode while having a - * handle open, there is no need for us to wait here even if a fast - * commit is going on. That is because, if this inode is being - * committed, ext4_mark_inode_dirty would have waited for inode commit - * operation to finish before we come here. So, by the time we come - * here, inode's EXT4_STATE_FC_COMMITTING would have been cleared. So, - * we shouldn't see EXT4_STATE_FC_COMMITTING to be set on this inode - * here. - * - * We may come here without any handles open in the "no_delete" case of - * ext4_evict_inode as well. However, if that happens, we first mark the - * file system as fast commit ineligible anyway. So, even in that case, - * it is okay to remove the inode from the fc list. + * Wait for ongoing fast commit to finish. We cannot remove the inode + * from fast commit lists while it is being committed. If we wake from + * FC_FLUSHING_DATA, re-check FC_COMMITTING before deleting because the + * commit thread sets FC_COMMITTING only after clearing FLUSHING_DATA. */ - WARN_ON(ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING) - && !ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)); - while (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { -#if (BITS_PER_LONG < 64) - DEFINE_WAIT_BIT(wait, &ei->i_state_flags, - EXT4_STATE_FC_FLUSHING_DATA); - wq = bit_waitqueue(&ei->i_state_flags, - EXT4_STATE_FC_FLUSHING_DATA); -#else - DEFINE_WAIT_BIT(wait, &ei->i_flags, - EXT4_STATE_FC_FLUSHING_DATA); - wq = bit_waitqueue(&ei->i_flags, - EXT4_STATE_FC_FLUSHING_DATA); -#endif - prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); - if (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { - ext4_fc_unlock(inode->i_sb, alloc_ctx); - schedule(); - alloc_ctx = ext4_fc_lock(inode->i_sb); - } - finish_wait(wq, &wait.wq_entry); + for (;;) { + ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_COMMITTING, + &alloc_ctx); + + if (!ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) + break; + + ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA, + &alloc_ctx); } + + ext4_fc_free_inode_snap(inode); list_del_init(&ei->i_fc_list); /* - * Since this inode is getting removed, let's also remove all FC - * dentry create references, since it is not needed to log it anyways. + * Since this inode is getting removed, let's also remove all FC dentry + * create references, since it is not needed to log it anyways. */ if (list_empty(&ei->i_fc_dilist)) { ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } - fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist); + fc_dentry = list_first_entry(&ei->i_fc_dilist, + struct ext4_fc_dentry_update, + fcd_dilist); WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT); list_del_init(&fc_dentry->fcd_list); list_del_init(&fc_dentry->fcd_dilist); @@ -370,6 +420,8 @@ static int ext4_fc_track_template( tid = handle->h_transaction->t_tid; spin_lock(&ei->i_fc_lock); + if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) + ext4_set_inode_state(inode, EXT4_STATE_FC_REQUEUE); if (tid == ei->i_sync_tid) { update = true; } else { @@ -540,8 +592,6 @@ static int __track_inode(handle_t *handle, struct inode *inode, void *arg, void ext4_fc_track_inode(handle_t *handle, struct inode *inode) { - struct ext4_inode_info *ei = EXT4_I(inode); - wait_queue_head_t *wq; int ret; if (S_ISDIR(inode->i_mode)) @@ -557,29 +607,11 @@ void ext4_fc_track_inode(handle_t *handle, struct inode *inode) return; /* - * If we come here, we may sleep while waiting for the inode to - * commit. We shouldn't be holding i_data_sem when we go to sleep since - * the commit path needs to grab the lock while committing the inode. + * Fast commit snapshots inode state at commit time, so there's no need + * to wait for EXT4_STATE_FC_COMMITTING here. If the inode is already + * on the commit queue, ext4_fc_cleanup() will requeue it for the new + * transaction once the current commit finishes. */ - lockdep_assert_not_held(&ei->i_data_sem); - - while (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) { -#if (BITS_PER_LONG < 64) - DEFINE_WAIT_BIT(wait, &ei->i_state_flags, - EXT4_STATE_FC_COMMITTING); - wq = bit_waitqueue(&ei->i_state_flags, - EXT4_STATE_FC_COMMITTING); -#else - DEFINE_WAIT_BIT(wait, &ei->i_flags, - EXT4_STATE_FC_COMMITTING); - wq = bit_waitqueue(&ei->i_flags, - EXT4_STATE_FC_COMMITTING); -#endif - prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); - if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) - schedule(); - finish_wait(wq, &wait.wq_entry); - } /* * From this point on, this inode will not be committed either @@ -659,8 +691,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) lock_buffer(bh); set_buffer_dirty(bh); set_buffer_uptodate(bh); - bh->b_end_io = ext4_end_buffer_io_sync; - submit_bh(REQ_OP_WRITE | write_flags, bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, ext4_end_buffer_io_sync); EXT4_SB(sb)->s_fc_bh = NULL; } @@ -829,6 +860,21 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, return true; } +struct ext4_fc_range { + struct list_head list; + u16 tag; + ext4_lblk_t lblk; + ext4_lblk_t len; + ext4_fsblk_t pblk; + bool unwritten; +}; + +struct ext4_fc_inode_snap { + struct list_head data_list; + unsigned int inode_len; + u8 inode_buf[]; +}; + /* * Writes inode in the fast commit space under TLV with tag @tag. * Returns 0 on success, error on failure. @@ -836,21 +882,27 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, static int ext4_fc_write_inode(struct inode *inode, u32 *crc) { struct ext4_inode_info *ei = EXT4_I(inode); - int inode_len = EXT4_GOOD_OLD_INODE_SIZE; - int ret; - struct ext4_iloc iloc; + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; struct ext4_fc_inode fc_inode; struct ext4_fc_tl tl; u8 *dst; + u8 *src; + int inode_len; + int ret; - ret = ext4_get_inode_loc(inode, &iloc); - if (ret) - return ret; + if (!snap) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } - if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) - inode_len = EXT4_INODE_SIZE(inode->i_sb); - else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) - inode_len += ei->i_extra_isize; + src = snap->inode_buf; + inode_len = snap->inode_len; + if (!src || inode_len == 0) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } fc_inode.fc_ino = cpu_to_le32(inode->i_ino); tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); @@ -866,10 +918,9 @@ static int ext4_fc_write_inode(struct inode *inode, u32 *crc) dst += EXT4_FC_TAG_BASE_LEN; memcpy(dst, &fc_inode, sizeof(fc_inode)); dst += sizeof(fc_inode); - memcpy(dst, (u8 *)ext4_raw_inode(&iloc), inode_len); + memcpy(dst, src, inode_len); ret = 0; err: - brelse(iloc.bh); return ret; } @@ -879,76 +930,244 @@ err: */ static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc) { - ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size; struct ext4_inode_info *ei = EXT4_I(inode); - struct ext4_map_blocks map; + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; struct ext4_fc_add_range fc_ext; struct ext4_fc_del_range lrange; struct ext4_extent *ex; - int ret; + struct ext4_fc_range *range; + + if (!snap) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } + + list_for_each_entry(range, &snap->data_list, list) { + if (range->tag == EXT4_FC_TAG_DEL_RANGE) { + lrange.fc_ino = cpu_to_le32(inode->i_ino); + lrange.fc_lblk = cpu_to_le32(range->lblk); + lrange.fc_len = cpu_to_le32(range->len); + if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, + sizeof(lrange), (u8 *)&lrange, crc)) + return -ENOSPC; + continue; + } + + fc_ext.fc_ino = cpu_to_le32(inode->i_ino); + ex = (struct ext4_extent *)&fc_ext.fc_ex; + ex->ee_block = cpu_to_le32(range->lblk); + ex->ee_len = cpu_to_le16(range->len); + ext4_ext_store_pblock(ex, range->pblk); + if (range->unwritten) + ext4_ext_mark_unwritten(ex); + else + ext4_ext_mark_initialized(ex); + + if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, + sizeof(fc_ext), (u8 *)&fc_ext, crc)) + return -ENOSPC; + } + + return 0; +} + +static void ext4_fc_free_ranges(struct list_head *head) +{ + struct ext4_fc_range *range, *range_n; + + list_for_each_entry_safe(range, range_n, head, list) { + list_del(&range->list); + kmem_cache_free(ext4_fc_range_cachep, range); + } +} + +static void ext4_fc_free_inode_snap(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + + if (!snap) + return; + + ext4_fc_free_ranges(&snap->data_list); + kfree(snap); + ei->i_fc_snap = NULL; +} + +static int ext4_fc_snapshot_inode_data(struct inode *inode, + struct list_head *ranges, + unsigned int nr_ranges_total, + unsigned int *nr_rangesp, + int *snap_err) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + ext4_lblk_t start_lblk, end_lblk, cur_lblk; + unsigned int nr_ranges = 0; spin_lock(&ei->i_fc_lock); if (ei->i_fc_lblk_len == 0) { spin_unlock(&ei->i_fc_lock); + if (nr_rangesp) + *nr_rangesp = 0; return 0; } - old_blk_size = ei->i_fc_lblk_start; - new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; + start_lblk = ei->i_fc_lblk_start; + end_lblk = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; ei->i_fc_lblk_len = 0; spin_unlock(&ei->i_fc_lock); - cur_lblk_off = old_blk_size; - ext4_debug("will try writing %d to %d for inode %llu\n", - cur_lblk_off, new_blk_size, inode->i_ino); + cur_lblk = start_lblk; + ext4_debug("snapshot data ranges %u-%u for inode %llu\n", + start_lblk, end_lblk, + (unsigned long long)inode->i_ino); + + while (cur_lblk <= end_lblk) { + struct extent_status es; + struct ext4_fc_range *range; + ext4_lblk_t len; + u64 remaining = (u64)end_lblk - cur_lblk + 1; + + if (!ext4_es_lookup_extent(inode, cur_lblk, NULL, &es, NULL)) { + atomic64_inc(&stats->snap_fail_es_miss); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_MISS); + return -EAGAIN; + } - while (cur_lblk_off <= new_blk_size) { - map.m_lblk = cur_lblk_off; - map.m_len = new_blk_size - cur_lblk_off + 1; - ret = ext4_map_blocks(NULL, inode, &map, - EXT4_GET_BLOCKS_IO_SUBMIT | - EXT4_EX_NOCACHE); - if (ret < 0) - return -ECANCELED; + if (ext4_es_is_delayed(&es)) { + atomic64_inc(&stats->snap_fail_es_delayed); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_ES_DELAYED); + return -EAGAIN; + } - if (map.m_len == 0) { - cur_lblk_off++; + len = es.es_len - (cur_lblk - es.es_lblk); + if (len > remaining) + len = remaining; + if (len == 0) { + cur_lblk++; continue; } - if (ret == 0) { - lrange.fc_ino = cpu_to_le32(inode->i_ino); - lrange.fc_lblk = cpu_to_le32(map.m_lblk); - lrange.fc_len = cpu_to_le32(map.m_len); - if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, - sizeof(lrange), (u8 *)&lrange, crc)) - return -ENOSPC; + if (nr_ranges_total + nr_ranges >= EXT4_FC_SNAPSHOT_MAX_RANGES) { + atomic64_inc(&stats->snap_fail_ranges_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_RANGES_CAP); + return -E2BIG; + } + + range = kmem_cache_alloc(ext4_fc_range_cachep, GFP_NOFS); + if (!range) { + atomic64_inc(&stats->snap_fail_nomem); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); + return -ENOMEM; + } + nr_ranges++; + + range->lblk = cur_lblk; + range->len = len; + range->pblk = 0; + range->unwritten = false; + + if (ext4_es_is_hole(&es)) { + range->tag = EXT4_FC_TAG_DEL_RANGE; + } else if (ext4_es_is_written(&es) || + ext4_es_is_unwritten(&es)) { + unsigned int max; + + range->tag = EXT4_FC_TAG_ADD_RANGE; + range->pblk = ext4_es_pblock(&es) + + (cur_lblk - es.es_lblk); + range->unwritten = ext4_es_is_unwritten(&es); + + max = range->unwritten ? EXT_UNWRITTEN_MAX_LEN : + EXT_INIT_MAX_LEN; + if (range->len > max) + range->len = max; } else { - unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ? - EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN; - - /* Limit the number of blocks in one extent */ - map.m_len = min(max, map.m_len); - - fc_ext.fc_ino = cpu_to_le32(inode->i_ino); - ex = (struct ext4_extent *)&fc_ext.fc_ex; - ex->ee_block = cpu_to_le32(map.m_lblk); - ex->ee_len = cpu_to_le16(map.m_len); - ext4_ext_store_pblock(ex, map.m_pblk); - if (map.m_flags & EXT4_MAP_UNWRITTEN) - ext4_ext_mark_unwritten(ex); - else - ext4_ext_mark_initialized(ex); - if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, - sizeof(fc_ext), (u8 *)&fc_ext, crc)) - return -ENOSPC; + kmem_cache_free(ext4_fc_range_cachep, range); + atomic64_inc(&stats->snap_fail_es_other); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_OTHER); + return -EAGAIN; } - cur_lblk_off += map.m_len; + INIT_LIST_HEAD(&range->list); + list_add_tail(&range->list, ranges); + + if ((u64)range->len > (u64)end_lblk - cur_lblk) + break; + + cur_lblk += range->len; } + if (nr_rangesp) + *nr_rangesp = nr_ranges; return 0; } +static int ext4_fc_snapshot_inode(struct inode *inode, + unsigned int nr_ranges_total, + unsigned int *nr_rangesp, int *snap_err) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + struct ext4_fc_inode_snap *snap; + int inode_len = EXT4_GOOD_OLD_INODE_SIZE; + struct ext4_iloc iloc; + LIST_HEAD(ranges); + unsigned int nr_ranges = 0; + int ret; + int alloc_ctx; + + ret = ext4_get_inode_loc_noio(inode, &iloc); + if (ret) { + atomic64_inc(&stats->snap_fail_inode_loc); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_INODE_LOC); + return ret; + } + + if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) + inode_len = EXT4_INODE_SIZE(inode->i_sb); + else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) + inode_len += ei->i_extra_isize; + + snap = kmalloc(struct_size(snap, inode_buf, inode_len), GFP_NOFS); + if (!snap) { + atomic64_inc(&stats->snap_fail_nomem); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); + brelse(iloc.bh); + return -ENOMEM; + } + INIT_LIST_HEAD(&snap->data_list); + snap->inode_len = inode_len; + + memcpy(snap->inode_buf, (u8 *)ext4_raw_inode(&iloc), inode_len); + brelse(iloc.bh); + + ret = ext4_fc_snapshot_inode_data(inode, &ranges, nr_ranges_total, + &nr_ranges, snap_err); + if (ret) { + kfree(snap); + ext4_fc_free_ranges(&ranges); + return ret; + } + + alloc_ctx = ext4_fc_lock(inode->i_sb); + ext4_fc_free_inode_snap(inode); + ei->i_fc_snap = snap; + list_splice_tail_init(&ranges, &snap->data_list); + ext4_fc_unlock(inode->i_sb, alloc_ctx); + + atomic64_inc(&stats->snap_inodes); + atomic64_add(nr_ranges, &stats->snap_ranges); + if (nr_rangesp) + *nr_rangesp = nr_ranges; + return 0; +} /* Flushes data of all the inodes in the commit queue. */ static int ext4_fc_flush_data(journal_t *journal) @@ -999,6 +1218,11 @@ static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc) */ if (list_empty(&fc_dentry->fcd_dilist)) continue; + /* + * For EXT4_FC_TAG_CREAT, fcd_dilist is linked on the created + * inode's i_fc_dilist list (kept singular), so we can recover the + * inode through it. + */ ei = list_first_entry(&fc_dentry->fcd_dilist, struct ext4_inode_info, i_fc_dilist); inode = &ei->vfs_inode; @@ -1023,17 +1247,114 @@ static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc) return 0; } -static int ext4_fc_perform_commit(journal_t *journal) +static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, + struct inode ***inodesp, + unsigned int *nr_inodesp); + +static int ext4_fc_snapshot_inodes(journal_t *journal, struct inode **inodes, + unsigned int inodes_size, + unsigned int *nr_inodesp, + unsigned int *nr_rangesp, + int *snap_err) { struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *iter; + struct ext4_fc_dentry_update *fc_dentry; + unsigned int i = 0; + unsigned int idx; + unsigned int nr_ranges = 0; + int ret = 0; + int alloc_ctx; + + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + if (i >= inodes_size) { + atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_INODES_CAP); + ret = -E2BIG; + goto unlock; + } + inodes[i++] = &iter->vfs_inode; + } + + list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { + struct ext4_inode_info *ei; + struct inode *inode; + + if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) + continue; + if (list_empty(&fc_dentry->fcd_dilist)) + continue; + + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, i_fc_dilist); + inode = &ei->vfs_inode; + if (!list_empty(&ei->i_fc_list)) + continue; + + if (i >= inodes_size) { + atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_INODES_CAP); + ret = -E2BIG; + goto unlock; + } + /* + * Create-only inodes may only be referenced via fcd_dilist and + * not appear on s_fc_q[MAIN]. They may hit the last iput while + * we are snapshotting, but inode eviction calls ext4_fc_del(), + * which waits for FC_COMMITTING to clear. Mark them FC_COMMITTING + * so the inode stays pinned and the snapshot stays valid until + * ext4_fc_cleanup(). + */ + ext4_set_inode_state(inode, EXT4_STATE_FC_COMMITTING); + inodes[i++] = inode; + } +unlock: + ext4_fc_unlock(sb, alloc_ctx); + + if (ret) + return ret; + + for (idx = 0; idx < i; idx++) { + unsigned int inode_ranges = 0; + + ret = ext4_fc_snapshot_inode(inodes[idx], nr_ranges, + &inode_ranges, snap_err); + if (ret) + break; + nr_ranges += inode_ranges; + } + + if (nr_inodesp) + *nr_inodesp = idx; + if (nr_rangesp) + *nr_rangesp = nr_ranges; + return ret; +} + +static int ext4_fc_perform_commit(journal_t *journal, tid_t commit_tid) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; + struct ext4_inode_info *iter; struct ext4_fc_head head; struct inode *inode; + struct inode **inodes; + unsigned int inodes_size; + unsigned int snap_inodes = 0; + unsigned int snap_ranges = 0; + int snap_err = EXT4_FC_SNAP_ERR_NONE; struct blk_plug plug; int ret = 0; u32 crc = 0; int alloc_ctx; + ktime_t lock_start; + u64 locked_ns; /* * Step 1: Mark all inodes on s_fc_q[MAIN] with @@ -1059,11 +1380,8 @@ static int ext4_fc_perform_commit(journal_t *journal) list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); -#if (BITS_PER_LONG < 64) - wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_FLUSHING_DATA); -#else - wake_up_bit(&iter->i_flags, EXT4_STATE_FC_FLUSHING_DATA); -#endif + ext4_fc_wake_inode_state(&iter->vfs_inode, + EXT4_STATE_FC_FLUSHING_DATA); } /* @@ -1081,13 +1399,23 @@ static int ext4_fc_perform_commit(journal_t *journal) if (ret) return ret; + ret = ext4_fc_alloc_snapshot_inodes(sb, &inodes, &inodes_size); + if (ret) { + if (ret == -E2BIG) + atomic64_inc(&snap_stats->snap_fail_inodes_cap); + else if (ret == -ENOMEM) + atomic64_inc(&snap_stats->snap_fail_nomem); + return ret; + } /* Step 4: Mark all inodes as being committed. */ jbd2_journal_lock_updates(journal); + lock_start = ktime_get(); /* * The journal is now locked. No more handles can start and all the - * previous handles are now drained. We now mark the inodes on the - * commit queue as being committed. + * previous handles are now drained. Snapshotting happens in this + * window so log writing can consume only stable snapshots without + * doing logical-to-physical mapping. */ alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { @@ -1095,7 +1423,22 @@ static int ext4_fc_perform_commit(journal_t *journal) EXT4_STATE_FC_COMMITTING); } ext4_fc_unlock(sb, alloc_ctx); + + ret = ext4_fc_snapshot_inodes(journal, inodes, inodes_size, + &snap_inodes, &snap_ranges, &snap_err); jbd2_journal_unlock_updates(journal); + locked_ns = ktime_to_ns(ktime_sub(ktime_get(), lock_start)); + atomic64_add(locked_ns, &snap_stats->lock_updates_ns_total); + atomic64_inc(&snap_stats->lock_updates_samples); + ext4_fc_snap_stats_update_max(&snap_stats->lock_updates_ns_max, + locked_ns); + if (trace_ext4_fc_lock_updates_enabled()) + trace_call__ext4_fc_lock_updates(sb, commit_tid, locked_ns, + snap_inodes, snap_ranges, + ret, snap_err); + kvfree(inodes); + if (ret) + return ret; /* * Step 5: If file system device is different from journal device, @@ -1149,6 +1492,64 @@ out: return ret; } +static unsigned int ext4_fc_count_snapshot_inodes(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_inode_info *iter; + struct ext4_fc_dentry_update *fc_dentry; + unsigned int nr_inodes = 0; + int alloc_ctx; + + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) + nr_inodes++; + + list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { + struct ext4_inode_info *ei; + + if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) + continue; + if (list_empty(&fc_dentry->fcd_dilist)) + continue; + + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, i_fc_dilist); + if (!list_empty(&ei->i_fc_list)) + continue; + + nr_inodes++; + } + ext4_fc_unlock(sb, alloc_ctx); + + return nr_inodes; +} + +static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, + struct inode ***inodesp, + unsigned int *nr_inodesp) +{ + unsigned int nr_inodes = ext4_fc_count_snapshot_inodes(sb); + struct inode **inodes; + + *inodesp = NULL; + *nr_inodesp = 0; + + if (!nr_inodes) + return 0; + + if (nr_inodes > EXT4_FC_SNAPSHOT_MAX_INODES) + return -E2BIG; + + inodes = kvcalloc(nr_inodes, sizeof(*inodes), GFP_NOFS); + if (!inodes) + return -ENOMEM; + + *inodesp = inodes; + *nr_inodesp = nr_inodes; + return 0; +} + static void ext4_fc_update_stats(struct super_block *sb, int status, u64 commit_time, int nblks, tid_t commit_tid) { @@ -1239,9 +1640,12 @@ restart_fc: journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO; set_task_ioprio(current, journal_ioprio); fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; - ret = ext4_fc_perform_commit(journal); + ret = ext4_fc_perform_commit(journal, commit_tid); if (ret < 0) { - status = EXT4_FC_STATUS_FAILED; + if (ret == -EAGAIN || ret == -E2BIG || ret == -ECANCELED) + status = EXT4_FC_STATUS_INELIGIBLE; + else + status = EXT4_FC_STATUS_FAILED; goto fallback; } nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; @@ -1288,45 +1692,66 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) alloc_ctx = ext4_fc_lock(sb); while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { + bool requeue; + ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], struct ext4_inode_info, i_fc_list); list_del_init(&ei->i_fc_list); + ext4_fc_free_inode_snap(&ei->vfs_inode); + spin_lock(&ei->i_fc_lock); + if (full) + requeue = !tid_geq(tid, ei->i_sync_tid); + else + requeue = ext4_test_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_REQUEUE); + if (!requeue) + ext4_fc_reset_inode(&ei->vfs_inode); + ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_REQUEUE); ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); - if (tid_geq(tid, ei->i_sync_tid)) { - ext4_fc_reset_inode(&ei->vfs_inode); - } else if (full) { - /* - * We are called after a full commit, inode has been - * modified while the commit was running. Re-enqueue - * the inode into STAGING, which will then be splice - * back into MAIN. This cannot happen during - * fastcommit because the journal is locked all the - * time in that case (and tid doesn't increase so - * tid check above isn't reliable). - */ + spin_unlock(&ei->i_fc_lock); + if (requeue) list_add_tail(&ei->i_fc_list, &sbi->s_fc_q[FC_Q_STAGING]); - } /* * Make sure clearing of EXT4_STATE_FC_COMMITTING is * visible before we send the wakeup. Pairs with implicit - * barrier in prepare_to_wait() in ext4_fc_track_inode(). + * barrier in prepare_to_wait() in ext4_fc_del(). */ smp_mb(); -#if (BITS_PER_LONG < 64) - wake_up_bit(&ei->i_state_flags, EXT4_STATE_FC_COMMITTING); -#else - wake_up_bit(&ei->i_flags, EXT4_STATE_FC_COMMITTING); -#endif + ext4_fc_wake_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); } while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], - struct ext4_fc_dentry_update, - fcd_list); + struct ext4_fc_dentry_update, + fcd_list); list_del_init(&fc_dentry->fcd_list); + if (fc_dentry->fcd_op == EXT4_FC_TAG_CREAT && + !list_empty(&fc_dentry->fcd_dilist)) { + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, + i_fc_dilist); + ext4_fc_free_inode_snap(&ei->vfs_inode); + spin_lock(&ei->i_fc_lock); + ext4_clear_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_REQUEUE); + ext4_clear_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + spin_unlock(&ei->i_fc_lock); + /* + * Make sure clearing of EXT4_STATE_FC_COMMITTING is + * visible before we send the wakeup. Pairs with + * implicit barrier in prepare_to_wait() in + * ext4_fc_del(). + */ + smp_mb(); + ext4_fc_wake_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + } list_del_init(&fc_dentry->fcd_dilist); release_dentry_name_snapshot(&fc_dentry->fcd_name); @@ -2278,11 +2703,26 @@ int ext4_fc_info_show(struct seq_file *seq, void *v) { struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private); struct ext4_fc_stats *stats = &sbi->s_fc_stats; + struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; + u64 lock_avg_ns = 0; + u64 lock_updates_samples; + u64 lock_updates_ns_total; + u64 lock_updates_ns_max; int i; if (v != SEQ_START_TOKEN) return 0; + lock_updates_samples = + atomic64_read(&snap_stats->lock_updates_samples); + lock_updates_ns_total = + atomic64_read(&snap_stats->lock_updates_ns_total); + lock_updates_ns_max = + atomic64_read(&snap_stats->lock_updates_ns_max); + if (lock_updates_samples) + lock_avg_ns = div64_u64(lock_updates_ns_total, + lock_updates_samples); + seq_printf(seq, "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n", stats->fc_num_commits, stats->fc_ineligible_commits, @@ -2293,6 +2733,23 @@ int ext4_fc_info_show(struct seq_file *seq, void *v) seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i], stats->fc_ineligible_reason_count[i]); + seq_printf(seq, + "Snapshot stats:\n%llu inodes\n%llu ranges\n%lluus lock_updates_avg\n%lluus lock_updates_max\n", + atomic64_read(&snap_stats->snap_inodes), + atomic64_read(&snap_stats->snap_ranges), + div_u64(lock_avg_ns, 1000), + div_u64(lock_updates_ns_max, 1000)); + seq_printf(seq, + "Snapshot failures:\n%llu es_miss\n%llu es_delayed\n%llu es_other\n%llu inodes_cap\n%llu ranges_cap\n%llu nomem\n%llu inode_loc\n%llu no_snap\n", + atomic64_read(&snap_stats->snap_fail_es_miss), + atomic64_read(&snap_stats->snap_fail_es_delayed), + atomic64_read(&snap_stats->snap_fail_es_other), + atomic64_read(&snap_stats->snap_fail_inodes_cap), + atomic64_read(&snap_stats->snap_fail_ranges_cap), + atomic64_read(&snap_stats->snap_fail_nomem), + atomic64_read(&snap_stats->snap_fail_inode_loc), + atomic64_read(&snap_stats->snap_fail_no_snap)); + return 0; } @@ -2301,13 +2758,20 @@ int __init ext4_fc_init_dentry_cache(void) ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update, SLAB_RECLAIM_ACCOUNT); - if (ext4_fc_dentry_cachep == NULL) + if (!ext4_fc_dentry_cachep) return -ENOMEM; + ext4_fc_range_cachep = KMEM_CACHE(ext4_fc_range, SLAB_RECLAIM_ACCOUNT); + if (!ext4_fc_range_cachep) { + kmem_cache_destroy(ext4_fc_dentry_cachep); + return -ENOMEM; + } + return 0; } void ext4_fc_destroy_dentry_cache(void) { + kmem_cache_destroy(ext4_fc_range_cachep); kmem_cache_destroy(ext4_fc_dentry_cachep); } diff --git a/fs/ext4/hash-test.c b/fs/ext4/hash-test.c new file mode 100644 index 000000000000..49b0d874c833 --- /dev/null +++ b/fs/ext4/hash-test.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for ext4 directory hash computation. + */ + +#include <kunit/test.h> +#include <kunit/resource.h> +#include <linux/fs.h> +#include <linux/stddef.h> +#include <linux/string.h> +#include <linux/unicode.h> +#include "ext4.h" + +static void ext4_hash_init_fake_dir(struct inode *dir, struct super_block *sb) +{ + memset(sb, 0, sizeof(*sb)); + memset(dir, 0, sizeof(*dir)); + dir->i_sb = sb; + strscpy(sb->s_id, "kunit-ext4", sizeof(sb->s_id)); +} + +static void ext4_hash_init_fake_dir_with_sbi(struct inode *dir, + struct super_block *sb, + struct ext4_sb_info *sbi) +{ + ext4_hash_init_fake_dir(dir, sb); + memset(sbi, 0, sizeof(*sbi)); + sb->s_fs_info = sbi; + sbi->s_sb = sb; +} + +#ifdef CONFIG_FS_ENCRYPTION +static const struct fscrypt_operations ext4_hash_test_cryptops = { + .inode_info_offs = + (int)offsetof(struct ext4_inode_info, i_crypt_info) - + (int)offsetof(struct ext4_inode_info, vfs_inode), +}; +#endif + +static void ext4_hash_init_fake_ext4_dir(struct ext4_inode_info *ei, + struct super_block *sb, + struct ext4_sb_info *sbi) +{ + struct inode *dir = &ei->vfs_inode; + + memset(sb, 0, sizeof(*sb)); + memset(ei, 0, sizeof(*ei)); + memset(sbi, 0, sizeof(*sbi)); + + strscpy(sb->s_id, "kunit-ext4", sizeof(sb->s_id)); + sb->s_fs_info = sbi; + sbi->s_sb = sb; + + dir->i_sb = sb; + dir->i_mode = S_IFDIR; + +#ifdef CONFIG_FS_ENCRYPTION + fscrypt_set_ops(sb, &ext4_hash_test_cryptops); +#endif +} + +struct ext4_dirhash_test_case { + const char *name; + u32 hash_version; + const char *input; + int len; + u32 seed[4]; + bool use_seed; + u32 expected_hash; + u32 expected_minor_hash; +}; + +static const struct ext4_dirhash_test_case ext4_dirhash_test_cases[] = { + { + .name = "legacy_abc", + .hash_version = DX_HASH_LEGACY, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0x75afd992, + .expected_minor_hash = 0x00000000, + }, + { + .name = "legacy_unsigned_abc", + .hash_version = DX_HASH_LEGACY_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0x75afd992, + .expected_minor_hash = 0x00000000, + }, + { + .name = "half_md4_abc", + .hash_version = DX_HASH_HALF_MD4, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xd196a868, + .expected_minor_hash = 0xc420eb28, + }, + { + .name = "half_md4_unsigned_abc", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xd196a868, + .expected_minor_hash = 0xc420eb28, + }, + { + .name = "tea_abc", + .hash_version = DX_HASH_TEA, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xb1435ec4, + .expected_minor_hash = 0x3f7eaa0e, + }, + { + .name = "tea_unsigned_abc", + .hash_version = DX_HASH_TEA_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xb1435ec4, + .expected_minor_hash = 0x3f7eaa0e, + }, + { + .name = "empty_half_md4", + .hash_version = DX_HASH_HALF_MD4, + .input = "", + .len = 0, + .use_seed = false, + .expected_hash = 0xefcdab88, + .expected_minor_hash = 0x98badcfe, + }, + { + .name = "half_md4_31bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "1234567890123456789012345678901", + .len = 31, + .use_seed = false, + .expected_hash = 0xc4db1f78, + .expected_minor_hash = 0xea23921b, + }, + { + .name = "half_md4_32bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "12345678901234567890123456789012", + .len = 32, + .use_seed = false, + .expected_hash = 0xfa6cc63e, + .expected_minor_hash = 0x2f77bd1c, + }, + { + .name = "half_md4_33bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "123456789012345678901234567890123", + .len = 33, + .use_seed = false, + .expected_hash = 0xdc0c2dec, + .expected_minor_hash = 0x5ca23365, + }, + { + .name = "half_md4_unsigned_31bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "1234567890123456789012345678901", + .len = 31, + .use_seed = false, + .expected_hash = 0xc4db1f78, + .expected_minor_hash = 0xea23921b, + }, + { + .name = "half_md4_unsigned_32bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "12345678901234567890123456789012", + .len = 32, + .use_seed = false, + .expected_hash = 0xfa6cc63e, + .expected_minor_hash = 0x2f77bd1c, + }, + { + .name = "half_md4_unsigned_33bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "123456789012345678901234567890123", + .len = 33, + .use_seed = false, + .expected_hash = 0xdc0c2dec, + .expected_minor_hash = 0x5ca23365, + }, + { + .name = "tea_15bytes", + .hash_version = DX_HASH_TEA, + .input = "123456789abcdef", + .len = 15, + .use_seed = false, + .expected_hash = 0xa562903a, + .expected_minor_hash = 0x6174a00f, + }, + { + .name = "tea_16bytes", + .hash_version = DX_HASH_TEA, + .input = "1234567890abcdef", + .len = 16, + .use_seed = false, + .expected_hash = 0x8449f258, + .expected_minor_hash = 0x49a16d46, + }, + { + .name = "tea_17bytes", + .hash_version = DX_HASH_TEA, + .input = "123456789abcdefgh", + .len = 17, + .use_seed = false, + .expected_hash = 0xf32ec10c, + .expected_minor_hash = 0x58ceae61, + }, + { + .name = "half_md4_seeded", + .hash_version = DX_HASH_HALF_MD4, + .input = "same-name", + .len = 9, + .seed = { 0x11111111, 0x22222222, 0x33333333, 0x44444444 }, + .use_seed = true, + .expected_hash = 0x8aebf604, + .expected_minor_hash = 0x66ce48fe, + }, + { + .name = "half_md4_non_ascii_signed", + .hash_version = DX_HASH_HALF_MD4, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x8bab0498, + .expected_minor_hash = 0xc326632d, + }, + { + .name = "half_md4_non_ascii_unsigned", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0xbc48596e, + .expected_minor_hash = 0xde0fad41, + }, + { + .name = "tea_non_ascii_signed", + .hash_version = DX_HASH_TEA, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x21e3a154, + .expected_minor_hash = 0x90112c3d, + }, + { + .name = "tea_non_ascii_unsigned", + .hash_version = DX_HASH_TEA_UNSIGNED, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x9b648616, + .expected_minor_hash = 0x011dd507, + }, +}; + +static void test_ext4fs_dirhash_vectors(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + int i; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + for (i = 0; i < ARRAY_SIZE(ext4_dirhash_test_cases); i++) { + const struct ext4_dirhash_test_case *tc = + &ext4_dirhash_test_cases[i]; + struct dx_hash_info hinfo; + int ret; + + memset(&hinfo, 0, sizeof(hinfo)); + hinfo.hash_version = tc->hash_version; + hinfo.seed = tc->use_seed ? (u32 *)tc->seed : NULL; + + ret = ext4fs_dirhash(dir, tc->input, tc->len, &hinfo); + + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "case=%s", tc->name); + KUNIT_EXPECT_EQ_MSG(test, hinfo.hash, tc->expected_hash, + "case=%s", tc->name); + KUNIT_EXPECT_EQ_MSG(test, hinfo.minor_hash, + tc->expected_minor_hash, + "case=%s", tc->name); + } +} + +static void test_ext4fs_dirhash_seed_changes_result(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + u32 seed[4] = { 0x11111111, 0x22222222, 0x33333333, 0x44444444 }; + struct dx_hash_info plain = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info seeded = { + .hash_version = DX_HASH_HALF_MD4, + .seed = seed, + }; + int ret_plain, ret_seeded; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + ret_plain = ext4fs_dirhash(dir, "same-name", 9, &plain); + ret_seeded = ext4fs_dirhash(dir, "same-name", 9, &seeded); + + KUNIT_ASSERT_EQ(test, ret_plain, 0); + KUNIT_ASSERT_EQ(test, ret_seeded, 0); + + KUNIT_EXPECT_TRUE(test, + plain.hash != seeded.hash || + plain.minor_hash != seeded.minor_hash); +} + +static void test_ext4fs_dirhash_invalid_version_returns_einval(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + struct ext4_sb_info *sbi; + struct dx_hash_info hinfo = { + .hash = 0xdeadbeef, + .minor_hash = 0xcafebabe, + .hash_version = DX_HASH_LAST + 1, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + ext4_hash_init_fake_dir_with_sbi(dir, sb, sbi); + + ret = ext4fs_dirhash(dir, "abc", 3, &hinfo); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, hinfo.hash, 0); + KUNIT_EXPECT_EQ(test, hinfo.minor_hash, 0); +} + +static void test_ext4fs_dirhash_siphash_without_key_returns_einval(struct kunit *test) +{ + struct super_block *sb; + struct ext4_inode_info *ei; + struct inode *dir; + struct ext4_sb_info *sbi; + struct dx_hash_info hinfo = { + .hash_version = DX_HASH_SIPHASH, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + ext4_hash_init_fake_ext4_dir(ei, sb, sbi); + dir = &ei->vfs_inode; + + ret = ext4fs_dirhash(dir, "name", strlen("name"), &hinfo); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_ext4fs_dirhash_signed_unsigned_differ_on_nonascii(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + static const char input[] = "\x80\xff\x81\xfe\101bc"; + struct dx_hash_info legacy_signed = { + .hash_version = DX_HASH_LEGACY, + }; + struct dx_hash_info legacy_unsigned = { + .hash_version = DX_HASH_LEGACY_UNSIGNED, + }; + struct dx_hash_info md4_signed = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info md4_unsigned = { + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + }; + struct dx_hash_info tea_signed = { + .hash_version = DX_HASH_TEA, + }; + struct dx_hash_info tea_unsigned = { + .hash_version = DX_HASH_TEA_UNSIGNED, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &legacy_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &legacy_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NE(test, legacy_signed.hash, legacy_unsigned.hash); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &md4_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &md4_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, + md4_signed.hash != md4_unsigned.hash || + md4_signed.minor_hash != md4_unsigned.minor_hash); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &tea_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &tea_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, + tea_signed.hash != tea_unsigned.hash || + tea_signed.minor_hash != tea_unsigned.minor_hash); +} + +#if IS_ENABLED(CONFIG_UNICODE) +KUNIT_DEFINE_ACTION_WRAPPER(utf8_unload_action, utf8_unload, + struct unicode_map *); +static void test_ext4fs_dirhash_casefolded_names_hash_consistently(struct kunit *test) +{ + struct super_block *sb; + struct ext4_inode_info *ei; + struct ext4_sb_info *sbi; + struct unicode_map *um; + struct dx_hash_info h1 = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info h2 = { + .hash_version = DX_HASH_HALF_MD4, + }; + int ret, ret1, ret2; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + um = utf8_load(UTF8_LATEST); + if (IS_ERR(um)) { + kunit_skip(test, "utf8_load(UTF8_LATEST) failed: %pe", + um); + return; + } + + ret = kunit_add_action_or_reset(test, utf8_unload_action, um); + KUNIT_ASSERT_EQ(test, ret, 0); + + ext4_hash_init_fake_ext4_dir(ei, sb, sbi); + sb->s_encoding = um; + ei->vfs_inode.i_flags |= S_CASEFOLD; + + KUNIT_ASSERT_TRUE(test, IS_CASEFOLDED(&ei->vfs_inode)); + + ret1 = ext4fs_dirhash(&ei->vfs_inode, "Alpha", 5, &h1); + ret2 = ext4fs_dirhash(&ei->vfs_inode, "aLPHa", 5, &h2); + + KUNIT_ASSERT_EQ(test, ret1, 0); + KUNIT_ASSERT_EQ(test, ret2, 0); + KUNIT_EXPECT_EQ(test, h1.hash, h2.hash); + KUNIT_EXPECT_EQ(test, h1.minor_hash, h2.minor_hash); +} + +static void test_ext4fs_dirhash_casefold_fallback(struct kunit *test) +{ + struct super_block *sb_cf, *sb_plain; + struct ext4_inode_info *ei; + struct ext4_sb_info *sbi; + struct inode *plain_dir; + struct unicode_map *um; + static const char invalid_utf8[] = "\xc3\x28"; + struct dx_hash_info folded_dir = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info plain = { + .hash_version = DX_HASH_HALF_MD4, + }; + int ret, ret_cf, ret_plain; + + sb_cf = kunit_kzalloc(test, sizeof(*sb_cf), GFP_KERNEL); + sb_plain = kunit_kzalloc(test, sizeof(*sb_plain), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + plain_dir = kunit_kzalloc(test, sizeof(*plain_dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb_cf); + KUNIT_ASSERT_NOT_NULL(test, sb_plain); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + KUNIT_ASSERT_NOT_NULL(test, plain_dir); + + um = utf8_load(UTF8_LATEST); + if (IS_ERR(um)) { + kunit_skip(test, "utf8_load(UTF8_LATEST) failed: %pe", + um); + return; + } + + ret = kunit_add_action_or_reset(test, utf8_unload_action, um); + KUNIT_ASSERT_EQ(test, ret, 0); + + ext4_hash_init_fake_ext4_dir(ei, sb_cf, sbi); + sb_cf->s_encoding = um; + ei->vfs_inode.i_flags |= S_CASEFOLD; + + KUNIT_ASSERT_TRUE(test, IS_CASEFOLDED(&ei->vfs_inode)); + + ext4_hash_init_fake_dir(plain_dir, sb_plain); + + ret_cf = ext4fs_dirhash(&ei->vfs_inode, invalid_utf8, + sizeof(invalid_utf8) - 1, &folded_dir); + ret_plain = ext4fs_dirhash(plain_dir, invalid_utf8, + sizeof(invalid_utf8) - 1, &plain); + + KUNIT_ASSERT_EQ(test, ret_cf, 0); + KUNIT_ASSERT_EQ(test, ret_plain, 0); + KUNIT_EXPECT_EQ(test, folded_dir.hash, plain.hash); + KUNIT_EXPECT_EQ(test, folded_dir.minor_hash, plain.minor_hash); +} +#endif + +static struct kunit_case ext4_hash_test_cases[] = { + KUNIT_CASE(test_ext4fs_dirhash_vectors), + KUNIT_CASE(test_ext4fs_dirhash_seed_changes_result), + KUNIT_CASE(test_ext4fs_dirhash_invalid_version_returns_einval), + KUNIT_CASE(test_ext4fs_dirhash_siphash_without_key_returns_einval), + KUNIT_CASE(test_ext4fs_dirhash_signed_unsigned_differ_on_nonascii), +#if IS_ENABLED(CONFIG_UNICODE) + KUNIT_CASE(test_ext4fs_dirhash_casefolded_names_hash_consistently), + KUNIT_CASE(test_ext4fs_dirhash_casefold_fallback), +#endif + {} +}; + +static struct kunit_suite ext4_hash_test_suite = { + .name = "ext4_hash", + .test_cases = ext4_hash_test_cases, +}; + +kunit_test_suites(&ext4_hash_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/fs/ext4/hash.c b/fs/ext4/hash.c index 48483cd015d3..978bd92da0ad 100644 --- a/fs/ext4/hash.c +++ b/fs/ext4/hash.c @@ -9,6 +9,7 @@ #include <linux/unicode.h> #include <linux/compiler.h> #include <linux/bitops.h> +#include <linux/unaligned.h> #include "ext4.h" #define DELTA 0x9E3779B9 @@ -141,21 +142,28 @@ static void str2hashbuf_signed(const char *msg, int len, __u32 *buf, int num) pad = (__u32)len | ((__u32)len << 8); pad |= pad << 16; - val = pad; if (len > num*4) len = num * 4; - for (i = 0; i < len; i++) { - val = ((int) scp[i]) + (val << 8); - if ((i % 4) == 3) { - *buf++ = val; - val = pad; - num--; - } + + while (len >= 4) { + val = ((__u32)scp[0] << 24) + ((__u32)scp[1] << 16) + ((__u32)scp[2] << 8) + scp[3]; + *buf++ = val; + scp += 4; + len -= 4; + num--; } + + val = pad; + + for (i = 0; i < len; i++) + val = scp[i] + (val << 8); + if (--num >= 0) *buf++ = val; + while (--num >= 0) *buf++ = pad; + } static void str2hashbuf_unsigned(const char *msg, int len, __u32 *buf, int num) @@ -167,21 +175,28 @@ static void str2hashbuf_unsigned(const char *msg, int len, __u32 *buf, int num) pad = (__u32)len | ((__u32)len << 8); pad |= pad << 16; - val = pad; if (len > num*4) len = num * 4; - for (i = 0; i < len; i++) { - val = ((int) ucp[i]) + (val << 8); - if ((i % 4) == 3) { - *buf++ = val; - val = pad; - num--; - } + + while (len >= 4) { + val = get_unaligned_be32(ucp); + *buf++ = val; + ucp += 4; + len -= 4; + num--; } + + val = pad; + + for (i = 0; i < len; i++) + val = ucp[i] + (val << 8); + if (--num >= 0) *buf++ = val; + while (--num >= 0) *buf++ = pad; + } /* @@ -205,8 +220,7 @@ static int __ext4fs_dirhash(const struct inode *dir, const char *name, int len, const char *p; int i; __u32 in[8], buf[4]; - void (*str2hashbuf)(const char *, int, __u32 *, int) = - str2hashbuf_signed; + bool use_unsigned = false; /* Initialize the default seed for the hash checksum functions */ buf[0] = 0x67452301; @@ -232,12 +246,15 @@ static int __ext4fs_dirhash(const struct inode *dir, const char *name, int len, hash = dx_hack_hash_signed(name, len); break; case DX_HASH_HALF_MD4_UNSIGNED: - str2hashbuf = str2hashbuf_unsigned; + use_unsigned = true; fallthrough; case DX_HASH_HALF_MD4: p = name; while (len > 0) { - (*str2hashbuf)(p, len, in, 8); + if (use_unsigned) + str2hashbuf_unsigned(p, len, in, 8); + else + str2hashbuf_signed(p, len, in, 8); half_md4_transform(buf, in); len -= 32; p += 32; @@ -246,12 +263,15 @@ static int __ext4fs_dirhash(const struct inode *dir, const char *name, int len, hash = buf[1]; break; case DX_HASH_TEA_UNSIGNED: - str2hashbuf = str2hashbuf_unsigned; + use_unsigned = true; fallthrough; case DX_HASH_TEA: p = name; while (len > 0) { - (*str2hashbuf)(p, len, in, 4); + if (use_unsigned) + str2hashbuf_unsigned(p, len, in, 4); + else + str2hashbuf_signed(p, len, in, 4); TEA_transform(buf, in); len -= 16; p += 16; @@ -321,3 +341,7 @@ opaque_seq: #endif return __ext4fs_dirhash(dir, name, len, hinfo); } + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4fs_dirhash); +#endif diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index 3fd8f0099852..a40cb27f8116 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -66,14 +66,16 @@ void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap) memset(bitmap + (i >> 3), 0xff, (end_bit - i) >> 3); } -void ext4_end_bitmap_read(struct buffer_head *bh, int uptodate) +void ext4_end_bitmap_read(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + if (uptodate) { set_buffer_uptodate(bh); set_bitmap_uptodate(bh); } unlock_buffer(bh); - put_bh(bh); } static int ext4_validate_inode_bitmap(struct super_block *sb, @@ -252,10 +254,10 @@ void ext4_free_inode(handle_t *handle, struct inode *inode) "nonexistent device\n", __func__, __LINE__); return; } - if (icount_read(inode) > 1) { + if (icount_read_once(inode) > 1) { ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: count=%d", __func__, __LINE__, inode->i_ino, - icount_read(inode)); + icount_read_once(inode)); return; } if (inode->i_nlink) { diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index c2c2d6ac7f3d..ce99807c5f5b 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1560,7 +1560,8 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode)) + if (ext4_has_inline_data(inode) && + ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -5025,6 +5026,57 @@ int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) return ret; } +/* + * ext4_get_inode_loc_noio() is a best-effort variant of ext4_get_inode_loc(). + * It looks up the inode table block in the buffer cache and returns -EAGAIN if + * the block is not present or not uptodate, without starting any I/O. + */ +int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc) +{ + struct super_block *sb = inode->i_sb; + struct ext4_group_desc *gdp; + struct buffer_head *bh; + ext4_fsblk_t block; + int inodes_per_block, inode_offset; + unsigned long ino = inode->i_ino; + + iloc->bh = NULL; + if (ino < EXT4_ROOT_INO || + ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)) + return -EFSCORRUPTED; + + iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); + if (!gdp) + return -EIO; + + /* Figure out the offset within the block group inode table. */ + inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; + inode_offset = ((ino - 1) % EXT4_INODES_PER_GROUP(sb)); + iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); + + block = ext4_inode_table(sb, gdp); + if (block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block) || + block >= ext4_blocks_count(EXT4_SB(sb)->s_es)) { + ext4_error(sb, + "Invalid inode table block %llu in block_group %u", + block, iloc->block_group); + return -EFSCORRUPTED; + } + block += inode_offset / inodes_per_block; + + bh = sb_find_get_block(sb, block); + if (!bh) + return -EAGAIN; + if (!ext4_buffer_uptodate(bh)) { + brelse(bh); + return -EAGAIN; + } + + iloc->bh = bh; + return 0; +} + int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, struct ext4_iloc *iloc) diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 1d0c3d4bdf47..c8387e6a2c6e 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -830,11 +830,17 @@ int ext4_force_shutdown(struct super_block *sb, u32 flags) bdev_thaw(sb->s_bdev); break; case EXT4_GOING_FLAGS_LOGFLUSH: + /* + * Call ext4_force_commit() before setting EXT4_FLAGS_SHUTDOWN. + * This is because in data=ordered mode, journal commit + * triggers data writeback which fails if shutdown is already + * set, causing the journal to be aborted prematurely before + * the commit succeeds. + */ + (void) ext4_force_commit(sb); set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); - if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) { - (void) ext4_force_commit(sb); + if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) jbd2_journal_abort(sbi->s_journal, -ESHUTDOWN); - } break; case EXT4_GOING_FLAGS_NOLOGFLUSH: set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); @@ -1650,6 +1656,9 @@ group_extend_out: if (!(fd_file(donor)->f_mode & FMODE_WRITE)) return -EBADF; + if (file_inode(filp)->i_sb != file_inode(fd_file(donor))->i_sb) + return -EXDEV; + err = mnt_want_write_file(filp); if (err) return err; diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index 90ed505fa4b1..0424b8b0b4c3 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -5,6 +5,7 @@ #include <kunit/test.h> #include <kunit/static_stub.h> +#include <linux/fs_context.h> #include <linux/random.h> #include "ext4.h" @@ -63,8 +64,14 @@ static void mbt_kill_sb(struct super_block *sb) generic_shutdown_super(sb); } +static int mbt_init_fs_context(struct fs_context *fc) +{ + return 0; +} + static struct file_system_type mbt_fs_type = { .name = "mballoc test", + .init_fs_context = mbt_init_fs_context, .kill_sb = mbt_kill_sb, }; @@ -127,7 +134,7 @@ static void mbt_mb_release(struct super_block *sb) kfree(sb->s_bdev); } -static int mbt_set(struct super_block *sb, void *data) +static int mbt_set(struct super_block *sb, struct fs_context *fc) { return 0; } @@ -136,13 +143,19 @@ static struct super_block *mbt_ext4_alloc_super_block(void) { struct mbt_ext4_super_block *fsb; struct super_block *sb; + struct fs_context *fc; struct ext4_sb_info *sbi; fsb = kzalloc_obj(*fsb); if (fsb == NULL) return NULL; - sb = sget(&mbt_fs_type, NULL, mbt_set, 0, NULL); + fc = fs_context_for_mount(&mbt_fs_type, 0); + if (IS_ERR(fc)) + goto out; + + sb = sget_fc(fc, NULL, mbt_set); + put_fs_context(fc); if (IS_ERR(sb)) goto out; @@ -714,8 +727,7 @@ do_test_generate_buddy(struct kunit *test, struct super_block *sb, void *bitmap, ext4_mb_generate_buddy_test(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, ext4_grp); - KUNIT_ASSERT_EQ(test, memcmp(mbt_buddy, ext4_buddy, sb->s_blocksize), - 0); + KUNIT_ASSERT_MEMEQ(test, mbt_buddy, ext4_buddy, sb->s_blocksize); mbt_validate_group_info(test, mbt_grp, ext4_grp); } @@ -776,8 +788,7 @@ test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, grp->bb_counters[i] = 0; ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); - KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), - 0); + KUNIT_ASSERT_MEMEQ(test, buddy, e4b->bd_buddy, sb->s_blocksize); mbt_validate_group_info(test, grp, e4b->bd_info); } @@ -841,8 +852,7 @@ test_mb_free_blocks_range(struct kunit *test, struct ext4_buddy *e4b, grp->bb_counters[i] = 0; ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); - KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), - 0); + KUNIT_ASSERT_MEMEQ(test, buddy, e4b->bd_buddy, sb->s_blocksize); mbt_validate_group_info(test, grp, e4b->bd_info); } diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c index 6f57c181ff77..7ce361484b38 100644 --- a/fs/ext4/mmp.c +++ b/fs/ext4/mmp.c @@ -46,9 +46,8 @@ static int write_mmp_block_thawed(struct super_block *sb, ext4_mmp_csum_set(sb, mmp); lock_buffer(bh); - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, bh); + bh_submit(bh, REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, + bh_end_write); wait_on_buffer(bh); if (unlikely(!buffer_uptodate(bh))) return -EIO; diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 4a47fbd8dd30..cc49ae04a6f6 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -145,9 +145,9 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode, if (IS_ERR(bh)) { __ext4_warning(inode->i_sb, func, line, "inode #%llu: lblock %lu: comm %s: " - "error %ld reading directory block", + "error %pe reading directory block", inode->i_ino, (unsigned long)block, - current->comm, PTR_ERR(bh)); + current->comm, bh); return bh; } @@ -3054,7 +3054,7 @@ out_stop: out_retry: if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) goto retry; - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; } /* diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index dc82e7b57e75..bc674aa4a656 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -168,7 +168,7 @@ static void ext4_release_io_end(ext4_io_end_t *io_end) * written. On IO failure, check if journal abort is needed. Note that * we are protected from truncate touching same part of extent tree by the * fact that truncate code waits for all DIO to finish (thus exclusion from - * direct IO is achieved) and also waits for PageWriteback bits. Thus we + * direct IO is achieved) and also waits for writeback to complete. Thus we * cannot get to ext4_ext_truncate() before all IOs overlapping that range are * completed (happens from ext4_free_ioend()). */ diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 6a77db4d3124..245f67d10ded 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -161,7 +161,7 @@ MODULE_ALIAS("ext3"); static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { if (simu_fail) { clear_buffer_uptodate(bh); @@ -176,13 +176,13 @@ static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, */ clear_buffer_verified(bh); - bh->b_end_io = end_io ? end_io : end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ | op_flags, bh); + if (!end_io) + end_io = bh_end_read; + bh_submit(bh, REQ_OP_READ | op_flags, end_io); } void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { BUG_ON(!buffer_locked(bh)); @@ -194,7 +194,7 @@ void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, } int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { BUG_ON(!buffer_locked(bh)); @@ -1431,6 +1431,9 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) ext4_fc_init_inode(&ei->vfs_inode); spin_lock_init(&ei->i_fc_lock); mmb_init(&ei->i_metadata_bhs, &ei->vfs_inode.i_data); +#ifdef CONFIG_LOCKDEP + lockdep_set_subclass(&ei->i_data_sem, I_DATA_SEM_NORMAL); +#endif return &ei->vfs_inode; } @@ -4541,6 +4544,7 @@ static void ext4_fast_commit_init(struct super_block *sb) sbi->s_fc_ineligible_tid = 0; mutex_init(&sbi->s_fc_lock); memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats)); + memset(&sbi->s_fc_snap_stats, 0, sizeof(sbi->s_fc_snap_stats)); sbi->s_fc_replay_state.fc_regions = NULL; sbi->s_fc_replay_state.fc_regions_size = 0; sbi->s_fc_replay_state.fc_regions_used = 0; @@ -5910,6 +5914,11 @@ static struct inode *ext4_get_journal_inode(struct super_block *sb, return ERR_PTR(-EFSCORRUPTED); } +#ifdef CONFIG_LOCKDEP + lockdep_set_subclass(&EXT4_I(journal_inode)->i_data_sem, + I_DATA_SEM_JOURNAL); +#endif + ext4_debug("Journal inode found at %p: %lld bytes\n", journal_inode, journal_inode->i_size); return journal_inode; @@ -5977,8 +5986,8 @@ static struct file *ext4_get_journal_blkdev(struct super_block *sb, sb, &fs_holder_ops); if (IS_ERR(bdev_file)) { ext4_msg(sb, KERN_ERR, - "failed to open journal device unknown-block(%u,%u) %ld", - MAJOR(j_dev), MINOR(j_dev), PTR_ERR(bdev_file)); + "failed to open journal device unknown-block(%u,%u) %pe", + MAJOR(j_dev), MINOR(j_dev), bdev_file); return bdev_file; } @@ -6316,12 +6325,10 @@ static int ext4_commit_super(struct super_block *sb) clear_buffer_write_io_error(sbh); set_buffer_uptodate(sbh); } - get_bh(sbh); /* Clear potential dirty bit if it was journalled update */ clear_buffer_dirty(sbh); - sbh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | REQ_SYNC | - (test_opt(sb, BARRIER) ? REQ_FUA : 0), sbh); + bh_submit(sbh, REQ_OP_WRITE | REQ_SYNC | + (test_opt(sb, BARRIER) ? REQ_FUA : 0), bh_end_write); wait_on_buffer(sbh); if (buffer_write_io_error(sbh)) { ext4_msg(sb, KERN_ERR, "I/O error while writing " diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index fa8d81a30fb9..d3253549173e 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -47,6 +47,7 @@ static inline int f2fs_acl_count(size_t size) static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) { int i, count; + int err = -EINVAL; struct posix_acl *acl; struct f2fs_acl_header *hdr = (struct f2fs_acl_header *)value; struct f2fs_acl_entry *entry = (struct f2fs_acl_entry *)(hdr + 1); @@ -70,8 +71,11 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) for (i = 0; i < count; i++) { - if ((char *)entry > end) + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry_short) > end)) { + err = -EFSCORRUPTED; goto fail; + } acl->a_entries[i].e_tag = le16_to_cpu(entry->e_tag); acl->a_entries[i].e_perm = le16_to_cpu(entry->e_perm); @@ -86,6 +90,11 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) break; case ACL_USER: + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry) > end)) { + err = -EFSCORRUPTED; + goto fail; + } acl->a_entries[i].e_uid = make_kuid(&init_user_ns, le32_to_cpu(entry->e_id)); @@ -93,6 +102,11 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) sizeof(struct f2fs_acl_entry)); break; case ACL_GROUP: + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry) > end)) { + err = -EFSCORRUPTED; + goto fail; + } acl->a_entries[i].e_gid = make_kgid(&init_user_ns, le32_to_cpu(entry->e_id)); @@ -108,7 +122,7 @@ static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) return acl; fail: posix_acl_release(acl); - return ERR_PTR(-EINVAL); + return ERR_PTR(err); } static void *f2fs_acl_to_disk(struct f2fs_sb_info *sbi, diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index c00a6b6ebcbd..064f5b537423 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -943,6 +943,7 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi) for (i = 0; i < orphan_blocks; i++) { struct folio *folio; struct f2fs_orphan_block *orphan_blk; + unsigned int entry_count; folio = f2fs_get_meta_folio(sbi, start_blk + i); if (IS_ERR(folio)) { @@ -951,7 +952,18 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi) } orphan_blk = folio_address(folio); - for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) { + entry_count = le32_to_cpu(orphan_blk->entry_count); + if (entry_count > F2FS_ORPHANS_PER_BLOCK) { + f2fs_err(sbi, "invalid orphan inode entry count %u", + entry_count); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_ORPHAN); + err = -EFSCORRUPTED; + f2fs_folio_put(folio, true); + goto out; + } + + for (j = 0; j < entry_count; j++) { nid_t ino = le32_to_cpu(orphan_blk->ino[j]); err = recover_orphan_inode(sbi, ino); diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 881e76158b96..91855d91bbdd 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -14,6 +14,7 @@ #include <linux/lz4.h> #include <linux/zstd.h> #include <linux/folio_batch.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" @@ -760,6 +761,7 @@ void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) /* Avoid f2fs_commit_super in irq context */ f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); + fserror_report_file_metadata(dic->inode, ret, GFP_NOFS); goto out_release; } @@ -1453,6 +1455,9 @@ unlock_continue: out_destroy_crypt: page_array_free(sbi, cic->rpages, cc->cluster_size); + if (!fio.encrypted) + goto out_put_cic; + for (--i; i >= 0; i--) { if (!cc->cpages[i]) continue; @@ -1482,8 +1487,7 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) struct page *page = &folio->page; struct f2fs_sb_info *sbi = bio->bi_private; struct compress_io_ctx *cic = folio->private; - enum count_type type = WB_DATA_TYPE(folio, - f2fs_is_compressed_page(folio)); + enum count_type type = WB_DATA_TYPE(folio, true); int i; if (unlikely(bio->bi_status != BLK_STS_OK)) @@ -1807,7 +1811,7 @@ static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task) f2fs_free_dic(dic, false); } else { INIT_WORK(&dic->free_work, f2fs_late_free_dic); - queue_work(dic->sbi->post_read_wq, &dic->free_work); + queue_work(dic->sbi->wq, &dic->free_work); } } } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 8d4f1e75dee3..a765fda71536 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -20,6 +20,7 @@ #include <linux/sched/signal.h> #include <linux/fiemap.h> #include <linux/iomap.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" @@ -40,12 +41,17 @@ struct f2fs_folio_state { unsigned int read_pages_pending; }; +struct f2fs_bio { + struct work_struct work; + struct bio bio; +}; + #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE int __init f2fs_init_bioset(void) { return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, - 0, BIOSET_NEED_BVECS); + offsetof(struct f2fs_bio, bio), BIOSET_NEED_BVECS); } void f2fs_destroy_bioset(void) @@ -336,7 +342,7 @@ static void f2fs_read_end_io(struct bio *bio) f2fs_handle_step_decompress(ctx, intask); } else if (enabled_steps) { INIT_WORK(&ctx->work, f2fs_post_read_work); - queue_work(ctx->sbi->post_read_wq, &ctx->work); + queue_work(ctx->sbi->wq, &ctx->work); return; } } @@ -344,14 +350,11 @@ static void f2fs_read_end_io(struct bio *bio) f2fs_verify_and_finish_bio(bio, intask); } -static void f2fs_write_end_io(struct bio *bio) +static void f2fs_write_end_bio(struct bio *bio) { - struct f2fs_sb_info *sbi; + struct f2fs_sb_info *sbi = bio->bi_private; struct folio_iter fi; - iostat_update_and_unbind_ctx(bio); - sbi = bio->bi_private; - if (time_to_inject(sbi, FAULT_WRITE_IO)) bio->bi_status = BLK_STS_IOERR; @@ -377,9 +380,10 @@ static void f2fs_write_end_io(struct bio *bio) if (unlikely(bio->bi_status != BLK_STS_OK)) { mapping_set_error(folio->mapping, -EIO); - if (type == F2FS_WB_CP_DATA) + if (type == F2FS_WB_CP_DATA) { f2fs_stop_checkpoint(sbi, true, STOP_CP_REASON_WRITE_FAIL); + } } if (is_node_folio(folio)) { @@ -407,6 +411,13 @@ static void f2fs_write_end_io(struct bio *bio) bio_put(bio); } +static void f2fs_write_end_io(struct bio *bio) +{ + iostat_update_and_unbind_ctx(bio); + + f2fs_write_end_bio(bio); +} + #ifdef CONFIG_BLK_DEV_ZONED static void f2fs_zone_write_end_io(struct bio *bio) { @@ -509,6 +520,8 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) bio->bi_private = sbi; bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, fio->type, fio->temp); + bio->bi_write_stream = f2fs_io_type_to_write_stream(bdev, fio->type, + fio->temp); } iostat_alloc_and_bind_ctx(sbi, bio, NULL); @@ -943,6 +956,35 @@ void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, } } +void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi) +{ + struct bio_entry *be, *tmp; + struct f2fs_bio_info *io; + enum temp_type temp; + + for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { + LIST_HEAD(list); + + io = sbi->write_io[DATA] + temp; + + /* A lockless list_empty() check is safe here: any bios from + * other kworkers that we miss will be submitted by those + * kworkers accordingly. + */ + if (list_empty(&io->bio_list)) + continue; + + f2fs_down_write(&io->bio_list_lock); + list_splice_init(&io->bio_list, &list); + f2fs_up_write(&io->bio_list_lock); + + list_for_each_entry_safe(be, tmp, &list, list) { + f2fs_submit_write_bio(sbi, be->bio, DATA); + del_bio_entry(be); + } + } +} + int f2fs_merge_page_bio(struct f2fs_io_info *fio) { struct bio *bio = *fio->bio; @@ -1748,6 +1790,7 @@ next_block: err = -EFSCORRUPTED; f2fs_handle_error(sbi, ERROR_CORRUPTED_CLUSTER); + fserror_report_file_metadata(inode, err, GFP_NOFS); goto sync_out; } @@ -2495,7 +2538,7 @@ static int f2fs_read_data_large_folio(struct inode *inode, unsigned nrpages; struct f2fs_folio_state *ffs; int ret = 0; - bool folio_in_bio; + bool folio_in_bio = false; if (!IS_IMMUTABLE(inode) || f2fs_compressed_file(inode)) { if (folio) @@ -2611,18 +2654,17 @@ submit_and_realloc: } trace_f2fs_read_folio(folio, DATA); err_out: - if (!folio_in_bio) { + if (!folio_in_bio) folio_end_read(folio, !ret); - if (ret) - return ret; - } + if (ret) + goto out; if (rac) { folio = readahead_folio(rac); goto next_folio; } out: f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); - if (ret) { + if (ret && folio_in_bio) { /* Wait bios and clear uptodate. */ folio_lock(folio); folio_clear_uptodate(folio); @@ -3686,6 +3728,11 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, int flag = F2FS_GET_BLOCK_PRE_AIO; int err = 0; + if (!f2fs_has_inline_data(inode) && !f2fs_compressed_file(inode) && + (pos & PAGE_MASK) < i_size_read(inode) && + f2fs_lookup_read_extent_cache_block(inode, index, blk_addr)) + return 0; + /* * If a whole page is being written and we already preallocated all the * blocks, then there is no need to get a block address now. @@ -3822,13 +3869,14 @@ unlock_out: static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, struct folio *folio, loff_t pos, unsigned int len, - block_t *blk_addr, bool *node_changed, bool *use_cow) + block_t *blk_addr, bool *node_changed) { struct inode *inode = folio->mapping->host; struct inode *cow_inode = F2FS_I(inode)->cow_inode; pgoff_t index = folio->index; int err = 0; block_t ori_blk_addr = NULL_ADDR; + bool cow_has_reserved_block = false; /* If pos is beyond the end of file, reserve a new block in COW inode */ if ((pos & PAGE_MASK) >= i_size_read(inode)) @@ -3836,12 +3884,14 @@ static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, /* Look for the block in COW inode first */ err = __find_data_block(cow_inode, index, blk_addr); - if (err) { + if (err) return err; - } else if (*blk_addr != NULL_ADDR) { - *use_cow = true; + + if (__is_valid_data_blkaddr(*blk_addr)) return 0; - } + + if (*blk_addr == NEW_ADDR) + cow_has_reserved_block = true; if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE)) goto reserve_block; @@ -3853,10 +3903,13 @@ static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, reserve_block: /* Finally, we should reserve a new block in COW inode for the update */ - err = __reserve_data_block(cow_inode, index, blk_addr, node_changed); - if (err) - return err; - inc_atomic_write_cnt(inode); + if (!cow_has_reserved_block) { + err = __reserve_data_block(cow_inode, index, blk_addr, + node_changed); + if (err) + return err; + inc_atomic_write_cnt(inode); + } if (ori_blk_addr != NULL_ADDR) *blk_addr = ori_blk_addr; @@ -3873,7 +3926,6 @@ static int f2fs_write_begin(const struct kiocb *iocb, struct folio *folio; pgoff_t index = pos >> PAGE_SHIFT; bool need_balance = false; - bool use_cow = false; block_t blkaddr = NULL_ADDR; int err = 0; @@ -3936,7 +3988,7 @@ repeat: if (f2fs_is_atomic_file(inode)) err = prepare_atomic_write_begin(sbi, folio, pos, len, - &blkaddr, &need_balance, &use_cow); + &blkaddr, &need_balance); else err = prepare_write_begin(sbi, folio, pos, len, &blkaddr, &need_balance); @@ -3976,8 +4028,15 @@ repeat: err = -EFSCORRUPTED; goto put_folio; } - f2fs_submit_page_read(use_cow ? F2FS_I(inode)->cow_inode : - inode, + /* + * Although the block may be stored in the COW inode, the folio + * belongs to @inode and its data was encrypted (or not) using + * @inode's context (see f2fs_encrypt_one_page()). Read with + * @inode so the post-read decryption decision matches the + * folio's owner; otherwise an unencrypted @inode whose COW inode + * is encrypted hits a NULL ->i_crypt_info on decryption. + */ + f2fs_submit_page_read(inode, NULL, /* can't write to fsverity files */ folio, blkaddr, 0, true); @@ -4472,23 +4531,17 @@ void f2fs_destroy_post_read_processing(void) kmem_cache_destroy(bio_post_read_ctx_cache); } -int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi) +int f2fs_init_wq(struct f2fs_sb_info *sbi) { - if (!f2fs_sb_has_encrypt(sbi) && - !f2fs_sb_has_verity(sbi) && - !f2fs_sb_has_compression(sbi)) - return 0; - - sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq", - WQ_UNBOUND | WQ_HIGHPRI, - num_online_cpus()); - return sbi->post_read_wq ? 0 : -ENOMEM; + sbi->wq = alloc_workqueue("f2fs_wq", WQ_UNBOUND | WQ_HIGHPRI, + num_online_cpus()); + return sbi->wq ? 0 : -ENOMEM; } -void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi) +void f2fs_destroy_wq(struct f2fs_sb_info *sbi) { - if (sbi->post_read_wq) - destroy_workqueue(sbi->post_read_wq); + if (sbi->wq) + destroy_workqueue(sbi->wq); } int __init f2fs_init_bio_entry_cache(void) diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index 38802ee2e40d..a9563f7fcd88 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -11,6 +11,7 @@ #include <linux/filelock.h> #include <linux/sched/signal.h> #include <linux/unicode.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" #include "acl.h" @@ -249,6 +250,11 @@ struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d, continue; } + if (unlikely(le16_to_cpu(de->name_len) > F2FS_NAME_LEN || + bit_pos + GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)) > + d->max)) + return ERR_PTR(-EFSCORRUPTED); + if (!use_hash || de->hash_code == fname->hash) { res = f2fs_match_name(d->inode, fname, d->filename[bit_pos], @@ -1020,6 +1026,7 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, set_sbi_flag(sbi, SBI_NEED_FSCK); err = -EFSCORRUPTED; f2fs_handle_error(sbi, ERROR_CORRUPTED_DIRENT); + fserror_report_file_metadata(d->inode, err, GFP_NOFS); goto out; } diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index d2e006420f04..61f6b9714366 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -119,10 +119,9 @@ static bool __may_extent_tree(struct inode *inode, enum extent_type type) if (!__init_may_extent_tree(inode, type)) return false; - if (is_inode_flag_set(inode, FI_NO_EXTENT)) - return false; - if (type == EX_READ) { + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return false; if (is_inode_flag_set(inode, FI_COMPRESSED_FILE) && !f2fs_sb_has_readonly(F2FS_I_SB(inode))) return false; @@ -645,14 +644,10 @@ static unsigned int __destroy_extent_node(struct inode *inode, while (atomic_read(&et->node_cnt)) { write_lock(&et->lock); - if (!is_inode_flag_set(inode, FI_NO_EXTENT)) - set_inode_flag(inode, FI_NO_EXTENT); node_cnt += __free_extent_tree(sbi, et, nr_shrink); write_unlock(&et->lock); } - f2fs_bug_on(sbi, atomic_read(&et->node_cnt)); - return node_cnt; } @@ -691,12 +686,12 @@ static void __update_extent_tree_range(struct inode *inode, write_lock(&et->lock); - if (is_inode_flag_set(inode, FI_NO_EXTENT)) { - write_unlock(&et->lock); - return; - } - if (type == EX_READ) { + if (is_inode_flag_set(inode, FI_NO_EXTENT)) { + write_unlock(&et->lock); + return; + } + prev = et->largest; dei.len = 0; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 91f506e7c9cf..8f3e632f315c 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -96,6 +96,15 @@ extern const char *f2fs_fault_name[FAULT_MAX]; #define DEFAULT_FAILURE_RETRY_COUNT 1 #endif +enum { + REPORT_FAULT_NEED_FSCK, + REPORT_FAULT_STOP_CP, + REPORT_FAULT_MAX, +}; + +void f2fs_fault_report(struct super_block *sb, unsigned int err_code, + const char *func, unsigned int data); + /* * For mount options */ @@ -1583,6 +1592,7 @@ enum node_type { NODE_TYPE_INODE, NODE_TYPE_XATTR, NODE_TYPE_NON_INODE, + NODE_TYPE_NON_IXNODE, /* non inode and xnode */ }; /* a threshold of maximum elapsed time in critical region to print tracepoint */ @@ -1969,7 +1979,7 @@ struct f2fs_sb_info { /* Precomputed FS UUID checksum for seeding other checksums */ __u32 s_chksum_seed; - struct workqueue_struct *post_read_wq; /* post read workqueue */ + struct workqueue_struct *wq; /* bio completion workqueue */ /* * If we are in irq context, let's update error information into @@ -1980,6 +1990,7 @@ struct f2fs_sb_info { unsigned char stop_reason[MAX_STOP_REASON]; /* stop reason */ spinlock_t error_lock; /* protect errors/stop_reason array */ bool error_dirty; /* errors of sb is dirty */ + bool stop_reason_dirty; /* stop reason of sb is dirty */ /* For reclaimed segs statistics per each GC mode */ unsigned int gc_segment_mode; /* GC state for reclaimed segments */ @@ -2125,12 +2136,12 @@ static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type) { unsigned long now = jiffies; - sbi->last_time[type] = now; + WRITE_ONCE(sbi->last_time[type], now); /* DISCARD_TIME and GC_TIME are based on REQ_TIME */ if (type == REQ_TIME) { - sbi->last_time[DISCARD_TIME] = now; - sbi->last_time[GC_TIME] = now; + WRITE_ONCE(sbi->last_time[DISCARD_TIME], now); + WRITE_ONCE(sbi->last_time[GC_TIME], now); } } @@ -2138,7 +2149,7 @@ static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type) { unsigned long interval = sbi->interval_time[type] * HZ; - return time_after(jiffies, sbi->last_time[type] + interval); + return time_after(jiffies, READ_ONCE(sbi->last_time[type]) + interval); } static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi, @@ -2148,7 +2159,7 @@ static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi, unsigned int wait_ms = 0; long delta; - delta = (sbi->last_time[type] + interval) - jiffies; + delta = (READ_ONCE(sbi->last_time[type]) + interval) - jiffies; if (delta > 0) wait_ms = jiffies_to_msecs(delta); @@ -2279,11 +2290,18 @@ static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type) return test_bit(type, &sbi->s_flag); } -static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type) +static inline void __set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type) { set_bit(type, &sbi->s_flag); } +#define set_sbi_flag(sbi, type) \ +do { \ + __set_sbi_flag(sbi, type); \ + if ((type) == SBI_NEED_FSCK) \ + f2fs_fault_report(sbi->sb, REPORT_FAULT_NEED_FSCK, __func__, __LINE__); \ +} while (0) + static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type) { clear_bit(type, &sbi->s_flag); @@ -4061,6 +4079,8 @@ void f2fs_destroy_segment_manager_caches(void); int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint); enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, enum page_type type, enum temp_type temp); +u8 f2fs_io_type_to_write_stream(struct block_device *bdev, + enum page_type type, enum temp_type temp); unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi); unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, unsigned int segno); @@ -4153,6 +4173,7 @@ void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi, struct folio *folio, enum page_type type); void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, struct bio **bio, struct folio *folio); +void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi); void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi); int f2fs_submit_page_bio(struct f2fs_io_info *fio); int f2fs_merge_page_bio(struct f2fs_io_info *fio); @@ -4193,8 +4214,8 @@ bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len); void f2fs_clear_page_cache_dirty_tag(struct folio *folio); int f2fs_init_post_read_processing(void); void f2fs_destroy_post_read_processing(void); -int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi); -void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi); +int f2fs_init_wq(struct f2fs_sb_info *sbi); +void f2fs_destroy_wq(struct f2fs_sb_info *sbi); extern const struct iomap_ops f2fs_iomap_ops; /* diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index fb12c5c9affd..4b52c56d71f0 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1098,6 +1098,8 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, return -EPERM; if ((attr->ia_valid & ATTR_SIZE)) { + if (mapping_large_folio_support(inode->i_mapping)) + return -EOPNOTSUPP; if (!f2fs_is_compress_backend_ready(inode) || IS_DEVICE_ALIASING(inode)) return -EOPNOTSUPP; @@ -1914,8 +1916,15 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, if (f2fs_is_pinned_file(inode)) { block_t sec_blks = CAP_BLKS_PER_SEC(sbi); - block_t sec_len = roundup(map.m_len, sec_blks); + block_t sec_len; + if (map.m_lblk % sec_blks) { + map.m_lblk = rounddown(map.m_lblk, sec_blks); + map.m_len = pg_end - map.m_lblk; + if (off_end) + map.m_len++; + } + sec_len = roundup(map.m_len, sec_blks); map.m_len = sec_blks; next_alloc: f2fs_down_write(&sbi->pin_sem); @@ -4784,6 +4793,33 @@ static bool f2fs_should_use_dio(struct inode *inode, struct kiocb *iocb, return true; } +#ifdef CONFIG_F2FS_IOSTAT +static void f2fs_dio_end_bio(struct bio *bio) +{ + struct bio_iostat_ctx *iostat_ctx = bio->bi_private; + void *orig_bi_private = iostat_ctx->post_read_ctx; + + iostat_update_and_unbind_ctx(bio); + bio->bi_private = orig_bi_private; + iomap_dio_bio_end_io(bio); +} + +static void f2fs_dio_iostat_start(struct f2fs_sb_info *sbi, struct bio *bio) +{ + void *bi_private = bio->bi_private; + + if (!sbi->iostat_enable) + return; + + iostat_alloc_and_bind_ctx(sbi, bio, bi_private); + iostat_update_submit_ctx(bio, DATA); + bio->bi_end_io = f2fs_dio_end_bio; +} +#else +static inline void f2fs_dio_iostat_start(struct f2fs_sb_info *sbi, + struct bio *bio) {} +#endif + static int f2fs_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error, unsigned int flags) { @@ -4796,8 +4832,18 @@ static int f2fs_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error, return 0; } +static void f2fs_dio_read_submit_io(const struct iomap_iter *iter, + struct bio *bio, loff_t file_offset) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(iter->inode); + + f2fs_dio_iostat_start(sbi, bio); + blk_crypto_submit_bio(bio); +} + static const struct iomap_dio_ops f2fs_iomap_dio_read_ops = { .end_io = f2fs_dio_read_end_io, + .submit_io = f2fs_dio_read_submit_io, }; static ssize_t f2fs_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) @@ -5067,15 +5113,35 @@ static int f2fs_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error, return 0; } +static bool f2fs_valid_write_stream(struct f2fs_sb_info *sbi, u8 write_stream) +{ + int i; + + if (!write_stream) + return true; + if (!f2fs_is_multi_device(sbi)) + return write_stream <= bdev_max_write_streams(sbi->sb->s_bdev); + + for (i = 0; i < sbi->s_ndevs; i++) + if (write_stream > bdev_max_write_streams(FDEV(i).bdev)) + return false; + return true; +} + static void f2fs_dio_write_submit_io(const struct iomap_iter *iter, struct bio *bio, loff_t file_offset) { struct inode *inode = iter->inode; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct kiocb *iocb = iter->private; enum log_type type = f2fs_rw_hint_to_seg_type(sbi, inode->i_write_hint); enum temp_type temp = f2fs_get_segment_temp(sbi, type); bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, DATA, temp); + bio->bi_write_stream = + iocb->ki_write_stream ? iocb->ki_write_stream : + f2fs_io_type_to_write_stream(bio->bi_bdev, DATA, temp); + f2fs_dio_iostat_start(sbi, bio); blk_crypto_submit_bio(bio); } @@ -5113,6 +5179,11 @@ static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from, trace_f2fs_direct_IO_enter(inode, iocb, count, WRITE); + if (!f2fs_valid_write_stream(sbi, iocb->ki_write_stream)) { + ret = -EINVAL; + goto out; + } + if (iocb->ki_flags & IOCB_NOWAIT) { /* f2fs_convert_inline_inode() and block allocation can block */ if (f2fs_has_inline_data(inode) || @@ -5150,7 +5221,7 @@ static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from, if (pos + count > inode->i_size) dio_flags |= IOMAP_DIO_FORCE_WAIT; dio = __iomap_dio_rw(iocb, from, &f2fs_iomap_ops, - &f2fs_iomap_dio_write_ops, dio_flags, NULL, 0); + &f2fs_iomap_dio_write_ops, dio_flags, iocb, 0); if (IS_ERR_OR_NULL(dio)) { ret = PTR_ERR_OR_ZERO(dio); if (ret == -ENOTBLK) diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index ba93010924c0..ffaa7ba76a1b 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1220,8 +1220,8 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, static int ra_data_block(struct inode *inode, pgoff_t index) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct address_space *mapping = f2fs_is_cow_file(inode) ? - F2FS_I(inode)->atomic_inode->i_mapping : inode->i_mapping; + struct address_space *mapping = inode->i_mapping; + struct inode *atomic_inode = NULL; struct dnode_of_data dn; struct folio *folio, *efolio; struct f2fs_io_info fio = { @@ -1236,9 +1236,22 @@ static int ra_data_block(struct inode *inode, pgoff_t index) }; int err = 0; + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (f2fs_is_cow_file(inode)) { + atomic_inode = igrab(F2FS_I(inode)->atomic_inode); + if (!atomic_inode) { + f2fs_up_read(&F2FS_I(inode)->i_sem); + return -EBUSY; + } + mapping = atomic_inode->i_mapping; + } + f2fs_up_read(&F2FS_I(inode)->i_sem); + folio = f2fs_grab_cache_folio(mapping, index, true); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out_iput; + } if (f2fs_lookup_read_extent_cache_block(inode, index, &dn.data_blkaddr)) { @@ -1299,11 +1312,16 @@ got_it: f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE); + if (atomic_inode) + iput(atomic_inode); return 0; put_encrypted_page: f2fs_put_page(fio.encrypted_page, true); put_folio: f2fs_folio_put(folio, true); +out_iput: + if (atomic_inode) + iput(atomic_inode); return err; } @@ -1314,8 +1332,8 @@ put_folio: static int move_data_block(struct inode *inode, block_t bidx, int gc_type, unsigned int segno, int off) { - struct address_space *mapping = f2fs_is_cow_file(inode) ? - F2FS_I(inode)->atomic_inode->i_mapping : inode->i_mapping; + struct address_space *mapping = inode->i_mapping; + struct inode *atomic_inode = NULL; struct f2fs_io_info fio = { .sbi = F2FS_I_SB(inode), .ino = inode->i_ino, @@ -1337,10 +1355,23 @@ static int move_data_block(struct inode *inode, block_t bidx, (fio.sbi->gc_mode != GC_URGENT_HIGH) ? CURSEG_ALL_DATA_ATGC : CURSEG_COLD_DATA; + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (f2fs_is_cow_file(inode)) { + atomic_inode = igrab(F2FS_I(inode)->atomic_inode); + if (!atomic_inode) { + f2fs_up_read(&F2FS_I(inode)->i_sem); + return -EBUSY; + } + mapping = atomic_inode->i_mapping; + } + f2fs_up_read(&F2FS_I(inode)->i_sem); + /* do not read out */ folio = f2fs_grab_cache_folio(mapping, bidx, false); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out_iput; + } if (!check_valid_map(F2FS_I_SB(inode), segno, off)) { err = -ENOENT; @@ -1468,11 +1499,10 @@ up_out: put_out: f2fs_put_dnode(&dn); out: - if (!folio_test_uptodate(folio)) - __folio_set_dropbehind(folio); - folio_unlock(folio); - folio_end_dropbehind(folio); - folio_put(folio); + f2fs_folio_put(folio, true); +out_iput: + if (atomic_inode) + iput(atomic_inode); return err; } @@ -1754,9 +1784,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, unsigned int end_segno = start_segno + SEGS_PER_SEC(sbi); unsigned int sec_end_segno; int seg_freed = 0, migrated = 0; - unsigned char type = IS_DATASEG(get_seg_entry(sbi, segno)->type) ? - SUM_TYPE_DATA : SUM_TYPE_NODE; - unsigned char data_type = (type == SUM_TYPE_DATA) ? DATA : NODE; + unsigned char type; + unsigned char data_type; int submitted = 0, sum_blk_cnt; if (__is_large_section(sbi)) { @@ -1855,10 +1884,16 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, migrated >= sbi->migration_granularity) continue; + if (migrated == 0) { + type = IS_DATASEG(get_seg_entry(sbi, cur_segno)->type) ? + SUM_TYPE_DATA : SUM_TYPE_NODE; + data_type = (type == SUM_TYPE_DATA) ? DATA : NODE; + } + sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno); if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) { f2fs_err(sbi, "Inconsistent segment (%u) type " - "[%d, %d] in SSA and SIT", + "[%d, %d] in SIT and SSA", cur_segno, type, GET_SUM_TYPE( sum_footer(sbi, sum))); diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 7aabfc9b43cb..e2f7bedf1552 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -9,6 +9,7 @@ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/fiemap.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" @@ -179,6 +180,7 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio) f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%llu, i_addr[0]:0x%x, run fsck to fix.", __func__, dn->inode->i_ino, dn->data_blkaddr); f2fs_handle_error(fio.sbi, ERROR_INVALID_BLKADDR); + fserror_report_file_metadata(dn->inode, -EFSCORRUPTED, GFP_NOFS); return -EFSCORRUPTED; } @@ -435,6 +437,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio, __func__, dir->i_ino, dn.data_blkaddr); f2fs_handle_error(F2FS_F_SB(folio), ERROR_INVALID_BLKADDR); err = -EFSCORRUPTED; + fserror_report_file_metadata(dn.inode, err, GFP_NOFS); goto out; } @@ -507,6 +510,12 @@ static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry) bit_pos++; continue; } + if (unlikely(le16_to_cpu(de->name_len) > F2FS_NAME_LEN || + bit_pos + GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)) > + d.max)) { + err = -EFSCORRUPTED; + goto punch_dentry_pages; + } /* * We only need the disk_name and hash to move the dentry. @@ -527,6 +536,7 @@ static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry) bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); } return 0; + punch_dentry_pages: truncate_inode_pages(&dir->i_data, 0); f2fs_truncate_blocks(dir, 0, false); diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index c6dcda447882..c95e0b126da4 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -11,6 +11,7 @@ #include <linux/sched/mm.h> #include <linux/lz4.h> #include <linux/zstd.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" @@ -324,9 +325,9 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio) } if (f2fs_sb_has_flexible_inline_xattr(sbi) && - f2fs_has_inline_xattr(inode) && - (fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE || - fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE)) { + (fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE || + (f2fs_has_inline_xattr(inode) && + fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE))) { f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %lu", __func__, inode->i_ino, fi->i_inline_xattr_size, MIN_INLINE_XATTR_SIZE, MAX_INLINE_XATTR_SIZE); @@ -480,6 +481,7 @@ static int do_read_inode(struct inode *inode) f2fs_folio_put(node_folio, true); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); return -EFSCORRUPTED; } @@ -541,6 +543,7 @@ static int do_read_inode(struct inode *inode) if (!sanity_check_extent_cache(inode, node_folio)) { f2fs_folio_put(node_folio, true); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); return -EFSCORRUPTED; } @@ -561,8 +564,13 @@ static int do_read_inode(struct inode *inode) static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino) { - return ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi) || - ino == F2FS_COMPRESS_INO(sbi); + if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi)) + return true; +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (test_opt(sbi, COMPRESS_CACHE) && ino == F2FS_COMPRESS_INO(sbi)) + return true; +#endif + return false; } struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) @@ -583,6 +591,7 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) trace_f2fs_iget_exit(inode, ret); iput(inode); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + fserror_report_file_metadata(inode, ret, GFP_NOFS); return ERR_PTR(ret); } @@ -787,6 +796,7 @@ retry: if (err == -ENOMEM || ++count <= DEFAULT_RETRY_IO_COUNT) goto retry; stop_checkpoint: + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_UPDATE_INODE); return; } @@ -858,10 +868,15 @@ void f2fs_evict_inode(struct inode *inode) f2fs_abort_atomic_write(inode, true); if (fi->cow_inode && f2fs_is_cow_file(fi->cow_inode)) { - clear_inode_flag(fi->cow_inode, FI_COW_FILE); - F2FS_I(fi->cow_inode)->atomic_inode = NULL; - iput(fi->cow_inode); + struct inode *cow_inode = fi->cow_inode; + + f2fs_down_write(&F2FS_I(cow_inode)->i_sem); + clear_inode_flag(cow_inode, FI_COW_FILE); + F2FS_I(cow_inode)->atomic_inode = NULL; fi->cow_inode = NULL; + f2fs_up_write(&F2FS_I(cow_inode)->i_sem); + + iput(cow_inode); } trace_f2fs_evict_inode(inode); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 4e5bd9e4cfc3..38917e4a7319 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -12,6 +12,7 @@ #include <linux/blkdev.h> #include <linux/folio_batch.h> #include <linux/swap.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" @@ -72,7 +73,11 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) sizeof(struct free_nid)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == NAT_ENTRIES) { - mem_size = (nm_i->nat_cnt[TOTAL_NAT] * + /* + * nat_cnt[] is heuristic accounting. Sample it locklessly here + * to avoid taking nat_tree_lock in the balance path. + */ + mem_size = (data_race(READ_ONCE(nm_i->nat_cnt[TOTAL_NAT])) * sizeof(struct nat_entry)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); if (excess_cached_nats(sbi)) @@ -1265,6 +1270,8 @@ skip_partial: if (err == -ENOENT) { set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); + fserror_report_file_metadata(dn.inode, -EFSCORRUPTED, + GFP_NOFS); f2fs_err_ratelimited(sbi, "truncate node fail, ino:%llu, nid:%u, " "offset[0]:%d, offset[1]:%d, nofs:%d", @@ -1541,6 +1548,10 @@ int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, if (is_inode) goto out_err; break; + case NODE_TYPE_NON_IXNODE: + if (is_inode || is_xnode) + goto out_err; + break; default: break; } @@ -1556,6 +1567,8 @@ out_err: next_blkaddr_of_node(folio)); f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); + fserror_report_file_metadata(folio->mapping->host, + -EFSCORRUPTED, in_irq ? GFP_NOWAIT : GFP_NOFS); return -EFSCORRUPTED; } @@ -1634,7 +1647,7 @@ static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start) struct f2fs_sb_info *sbi = F2FS_F_SB(parent); nid_t nid = get_nid(parent, start, false); - return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_REGULAR); + return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE); } static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) @@ -1778,6 +1791,7 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, if (f2fs_sanity_check_node_footer(sbi, folio, nid, NODE_TYPE_REGULAR, false)) { + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); goto redirty_out; } @@ -1875,7 +1889,7 @@ int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, } if (!__write_node_folio(node_folio, false, false, NULL, - &wbc, false, FS_GC_NODE_IO, NULL)) + &wbc, false, io_type, NULL)) err = -EAGAIN; goto release_folio; out_folio: @@ -2703,6 +2717,8 @@ retry: spin_unlock(&nm_i->nid_list_lock); f2fs_err(sbi, "Corrupted nid %u in free_nid_list", i->nid); + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, + GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); return false; diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index bcf2034e4263..5e114f352099 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -129,13 +129,17 @@ static inline void raw_nat_from_node_info(struct f2fs_nat_entry *raw_ne, static inline bool excess_dirty_nats(struct f2fs_sb_info *sbi) { - return NM_I(sbi)->nat_cnt[DIRTY_NAT] >= NM_I(sbi)->max_nid * + /* nat_cnt[] is heuristic accounting sampled locklessly here. */ + return data_race(READ_ONCE(NM_I(sbi)->nat_cnt[DIRTY_NAT])) >= + NM_I(sbi)->max_nid * NM_I(sbi)->dirty_nats_ratio / 100; } static inline bool excess_cached_nats(struct f2fs_sb_info *sbi) { - return NM_I(sbi)->nat_cnt[TOTAL_NAT] >= DEF_NAT_CACHE_THRESHOLD; + /* nat_cnt[] is heuristic accounting sampled locklessly here. */ + return data_race(READ_ONCE(NM_I(sbi)->nat_cnt[TOTAL_NAT])) >= + DEF_NAT_CACHE_THRESHOLD; } enum mem_type { diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 3d3dacec9482..89af8407b667 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -9,6 +9,7 @@ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/sched/mm.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" #include "segment.h" @@ -679,6 +680,7 @@ retry_dn: ofs_of_node(folio)); err = -EFSCORRUPTED; f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); + fserror_report_file_metadata(dn.inode, err, GFP_NOFS); goto err; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 788f8b050249..d71ddb3ee918 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -17,6 +17,7 @@ #include <linux/freezer.h> #include <linux/sched/signal.h> #include <linux/random.h> +#include <linux/fserror.h> #include "f2fs.h" #include "segment.h" @@ -444,6 +445,13 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) if (has_enough_free_secs(sbi, 0, 0)) return; + /* + * Submit all cached OPU/IPU DATA bios before triggering + * foreground GC to avoid potential deadlocks. + */ + f2fs_submit_merged_write(sbi, DATA); + f2fs_submit_all_merged_ipu_writes(sbi); + if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && sbi->gc_thread->f2fs_gc_task) { DEFINE_WAIT(wait); @@ -462,6 +470,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) .should_migrate_blocks = false, .err_gc_skipped = false, .nr_free_secs = 1 }; + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); stat_inc_gc_call_count(sbi, FOREGROUND); f2fs_gc(sbi, &gc_control); @@ -2446,9 +2455,6 @@ static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_ent unsigned int segno, block_t blkaddr, unsigned int offset, int del) { bool exist; -#ifdef CONFIG_F2FS_CHECK_FS - bool mir_exist; -#endif int i; int del_count = -del; @@ -2456,15 +2462,6 @@ static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_ent for (i = 0; i < del_count; i++) { exist = f2fs_test_and_clear_bit(offset + i, se->cur_valid_map); -#ifdef CONFIG_F2FS_CHECK_FS - mir_exist = f2fs_test_and_clear_bit(offset + i, - se->cur_valid_map_mir); - if (unlikely(exist != mir_exist)) { - f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d", - blkaddr + i, exist); - f2fs_bug_on(sbi, 1); - } -#endif if (unlikely(!exist)) { f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", blkaddr + i); f2fs_bug_on(sbi, 1); @@ -2505,20 +2502,8 @@ static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry unsigned int segno, block_t blkaddr, unsigned int offset, int del) { bool exist; -#ifdef CONFIG_F2FS_CHECK_FS - bool mir_exist; -#endif exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); -#ifdef CONFIG_F2FS_CHECK_FS - mir_exist = f2fs_test_and_set_bit(offset, - se->cur_valid_map_mir); - if (unlikely(exist != mir_exist)) { - f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d", - blkaddr, exist); - f2fs_bug_on(sbi, 1); - } -#endif if (unlikely(exist)) { f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr); f2fs_bug_on(sbi, 1); @@ -2896,6 +2881,7 @@ got_it: /* set it as dirty segment in free segmap */ if (test_bit(segno, free_i->free_segmap)) { ret = -EFSCORRUPTED; + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_FREE_BITMAP); goto out_unlock; } @@ -3636,6 +3622,19 @@ enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, } } +u8 f2fs_io_type_to_write_stream(struct block_device *bdev, + enum page_type type, enum temp_type temp) +{ + unsigned short nr = bdev_max_write_streams(bdev); + + if (type != DATA || !nr) + return 0; + if (nr < NR_TEMP_TYPE) + return temp == COLD ? nr : HOT + 1; + + return temp + 1; +} + static int __get_segment_type_2(struct f2fs_io_info *fio) { if (fio->type == DATA) @@ -4748,11 +4747,6 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) int offset, sit_offset; se = get_seg_entry(sbi, segno); -#ifdef CONFIG_F2FS_CHECK_FS - if (memcmp(se->cur_valid_map, se->cur_valid_map_mir, - SIT_VBLOCK_MAP_SIZE)) - f2fs_bug_on(sbi, 1); -#endif /* add discard candidates */ if (!(cpc->reason & CP_DISCARD)) { @@ -4779,10 +4773,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) } /* update ckpt_valid_block */ - if (__is_large_section(sbi)) { + if (__is_large_section(sbi)) set_ckpt_valid_blocks(sbi, segno); - sanity_check_valid_blocks(sbi, segno); - } __clear_bit(segno, bitmap); sit_i->dirty_sentries--; @@ -4843,11 +4835,7 @@ static int build_sit_info(struct f2fs_sb_info *sbi) if (!sit_i->dirty_sentries_bitmap) return -ENOMEM; -#ifdef CONFIG_F2FS_CHECK_FS - bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map); -#else bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map); -#endif sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); if (!sit_i->bitmap) return -ENOMEM; @@ -4861,11 +4849,6 @@ static int build_sit_info(struct f2fs_sb_info *sbi) sit_i->sentries[start].ckpt_valid_map = bitmap; bitmap += SIT_VBLOCK_MAP_SIZE; -#ifdef CONFIG_F2FS_CHECK_FS - sit_i->sentries[start].cur_valid_map_mir = bitmap; - bitmap += SIT_VBLOCK_MAP_SIZE; -#endif - if (discard_map) { sit_i->sentries[start].discard_map = bitmap; bitmap += SIT_VBLOCK_MAP_SIZE; @@ -4897,11 +4880,6 @@ static int build_sit_info(struct f2fs_sb_info *sbi) return -ENOMEM; #ifdef CONFIG_F2FS_CHECK_FS - sit_i->sit_bitmap_mir = kmemdup(src_bitmap, - sit_bitmap_size, GFP_KERNEL); - if (!sit_i->sit_bitmap_mir) - return -ENOMEM; - sit_i->invalid_segmap = f2fs_kvzalloc(sbi, main_bitmap_size, GFP_KERNEL); if (!sit_i->invalid_segmap) @@ -5110,10 +5088,8 @@ init_discard_map_done: if (__is_large_section(sbi)) { unsigned int segno; - for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { + for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) set_ckpt_valid_blocks(sbi, segno); - sanity_check_valid_blocks(sbi, segno); - } } if (err) @@ -5864,7 +5840,6 @@ static void destroy_sit_info(struct f2fs_sb_info *sbi) SM_I(sbi)->sit_info = NULL; kfree(sit_i->sit_bitmap); #ifdef CONFIG_F2FS_CHECK_FS - kfree(sit_i->sit_bitmap_mir); kvfree(sit_i->invalid_segmap); #endif kfree(sit_i); diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 08735a165433..b0c06b3580b4 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -177,9 +177,6 @@ struct seg_entry { unsigned int ckpt_valid_blocks:10; /* # of valid blocks last cp */ unsigned int padding:6; /* padding */ unsigned char *cur_valid_map; /* validity bitmap of blocks */ -#ifdef CONFIG_F2FS_CHECK_FS - unsigned char *cur_valid_map_mir; /* mirror of current valid bitmap */ -#endif /* * # of valid blocks and the validity bitmap stored in the last * checkpoint pack. This information is used by the SSR mode. @@ -209,8 +206,6 @@ struct sit_info { char *bitmap; /* all bitmaps pointer */ char *sit_bitmap; /* SIT bitmap pointer */ #ifdef CONFIG_F2FS_CHECK_FS - char *sit_bitmap_mir; /* SIT bitmap mirror */ - /* bitmap of segments to be ignored by GC in case of errors */ unsigned long *invalid_segmap; #endif @@ -408,9 +403,6 @@ static inline void seg_info_from_raw_sit(struct seg_entry *se, se->ckpt_valid_blocks = GET_SIT_VBLOCKS(rs); memcpy(se->cur_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); -#ifdef CONFIG_F2FS_CHECK_FS - memcpy(se->cur_valid_map_mir, rs->valid_map, SIT_VBLOCK_MAP_SIZE); -#endif se->type = GET_SIT_TYPE(rs); se->mtime = le64_to_cpu(rs->mtime); } @@ -555,11 +547,6 @@ static inline void get_sit_bitmap(struct f2fs_sb_info *sbi, { struct sit_info *sit_i = SIT_I(sbi); -#ifdef CONFIG_F2FS_CHECK_FS - if (memcmp(sit_i->sit_bitmap, sit_i->sit_bitmap_mir, - sit_i->bitmap_size)) - f2fs_bug_on(sbi, 1); -#endif memcpy(dst_addr, sit_i->sit_bitmap, sit_i->bitmap_size); } @@ -900,12 +887,6 @@ static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, !valid_main_segno(sbi, start)); -#ifdef CONFIG_F2FS_CHECK_FS - if (f2fs_test_bit(offset, sit_i->sit_bitmap) != - f2fs_test_bit(offset, sit_i->sit_bitmap_mir)) - f2fs_bug_on(sbi, 1); -#endif - /* calculate sit block address */ if (f2fs_test_bit(offset, sit_i->sit_bitmap)) blk_addr += sit_i->sit_blocks; @@ -931,9 +912,6 @@ static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start) unsigned int block_off = SIT_BLOCK_OFFSET(start); f2fs_change_bit(block_off, sit_i->sit_bitmap); -#ifdef CONFIG_F2FS_CHECK_FS - f2fs_change_bit(block_off, sit_i->sit_bitmap_mir); -#endif } static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi, diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index ccf806b676f5..2b8d96411156 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -29,6 +29,7 @@ #include <linux/lz4.h> #include <linux/ctype.h> #include <linux/fs_parser.h> +#include <linux/fserror.h> #include "f2fs.h" #include "node.h" @@ -2074,7 +2075,7 @@ static void f2fs_put_super(struct super_block *sb) /* flush s_error_work before sbi destroy */ flush_work(&sbi->s_error_work); - f2fs_destroy_post_read_wq(sbi); + f2fs_destroy_wq(sbi); kvfree(sbi->ckpt); @@ -2632,6 +2633,9 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) /* check if we need more GC first */ unusable = f2fs_get_unusable_blocks(sbi); + + f2fs_info(sbi, "%s starts, unusable: %u", __func__, unusable); + if (!f2fs_disable_cp_again(sbi, unusable)) goto skip_gc; @@ -2639,6 +2643,8 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) sbi->gc_mode = GC_URGENT_HIGH; + f2fs_info(sbi, "%s: run f2fs_gc() to migrate blocks", __func__); + while (!f2fs_time_over(sbi, DISABLE_TIME)) { struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, @@ -2659,6 +2665,12 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) break; } + f2fs_info(sbi, "%s: call sync_filesystem() to persist meta: %lld, node: %lld, data: %lld", + __func__, + get_pages(sbi, F2FS_DIRTY_META), + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DATA)); + ret = sync_filesystem(sbi->sb); if (ret || err) { err = ret ? ret : err; @@ -2672,6 +2684,12 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) } skip_gc: + f2fs_info(sbi, "%s: call f2fs_write_checkpoint(), meta: %lld, node: %lld, data: %lld", + __func__, + get_pages(sbi, F2FS_DIRTY_META), + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DATA)); + f2fs_down_write_trace(&sbi->gc_lock, &lc); cpc.reason = CP_PAUSE; set_sbi_flag(sbi, SBI_CP_DISABLED); @@ -2689,7 +2707,7 @@ out_unlock: restore_flag: sbi->gc_mode = gc_mode; sbi->sb->s_flags = s_flags; /* Restore SB_RDONLY status */ - f2fs_info(sbi, "f2fs_disable_checkpoint() finish, err:%d", err); + f2fs_info(sbi, "%s finishes, err:%d", __func__, err); return err; } @@ -4608,6 +4626,7 @@ static void save_stop_reason(struct f2fs_sb_info *sbi, unsigned char reason) spin_lock_irqsave(&sbi->error_lock, flags); if (sbi->stop_reason[reason] < GENMASK(BITS_PER_BYTE - 1, 0)) sbi->stop_reason[reason]++; + sbi->stop_reason_dirty = true; spin_unlock_irqrestore(&sbi->error_lock, flags); } @@ -4615,17 +4634,21 @@ static void f2fs_record_stop_reason(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); unsigned long flags; + bool report_shutdown = false; int err; f2fs_down_write(&sbi->sb_lock); spin_lock_irqsave(&sbi->error_lock, flags); if (sbi->error_dirty) { - memcpy(F2FS_RAW_SUPER(sbi)->s_errors, sbi->errors, - MAX_F2FS_ERRORS); + memcpy(raw_super->s_errors, sbi->errors, MAX_F2FS_ERRORS); sbi->error_dirty = false; } memcpy(raw_super->s_stop_reason, sbi->stop_reason, MAX_STOP_REASON); + if (sbi->stop_reason_dirty) { + report_shutdown = true; + sbi->stop_reason_dirty = false; + } spin_unlock_irqrestore(&sbi->error_lock, flags); err = f2fs_commit_super(sbi, false); @@ -4635,6 +4658,9 @@ static void f2fs_record_stop_reason(struct f2fs_sb_info *sbi) f2fs_err_ratelimited(sbi, "f2fs_commit_super fails to record stop_reason, err:%d", err); + + if (report_shutdown) + fserror_report_shutdown(sbi->sb, GFP_NOFS); } void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag) @@ -4649,6 +4675,27 @@ void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag) spin_unlock_irqrestore(&sbi->error_lock, flags); } +static void f2fs_report_fserror(struct f2fs_sb_info *sbi, unsigned char error) +{ + switch (error) { + case ERROR_INVALID_BLKADDR: + case ERROR_CORRUPTED_INODE: + case ERROR_INCONSISTENT_SUMMARY: + case ERROR_INCONSISTENT_SUM_TYPE: + case ERROR_CORRUPTED_JOURNAL: + case ERROR_INCONSISTENT_NODE_COUNT: + case ERROR_INCONSISTENT_BLOCK_COUNT: + case ERROR_INVALID_CURSEG: + case ERROR_INCONSISTENT_SIT: + case ERROR_INVALID_NODE_REFERENCE: + case ERROR_INCONSISTENT_NAT: + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); + break; + default: + return; + } +} + void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error) { f2fs_save_errors(sbi, error); @@ -4658,6 +4705,8 @@ void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error) if (!test_bit(error, (unsigned long *)sbi->errors)) return; schedule_work(&sbi->s_error_work); + + f2fs_report_fserror(sbi, error); } static bool system_going_down(void) @@ -4724,9 +4773,18 @@ static void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, */ } +void f2fs_fault_report(struct super_block *sb, unsigned int err_code, + const char *func, unsigned int data) +{ + trace_f2fs_fault_report(sb, err_code, func, data); +} + void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, unsigned char reason) { + if (reason != STOP_CP_REASON_SHUTDOWN) + f2fs_fault_report(sbi->sb, REPORT_FAULT_STOP_CP, __func__, reason); + f2fs_build_fault_attr(sbi, 0, 0, FAULT_ALL); if (!end_io) f2fs_flush_merged_writes(sbi); @@ -5130,9 +5188,9 @@ try_onemore: goto free_devices; } - err = f2fs_init_post_read_wq(sbi); + err = f2fs_init_wq(sbi); if (err) { - f2fs_err(sbi, "Failed to initialize post read workqueue"); + f2fs_err(sbi, "Failed to create workqueue"); goto free_devices; } @@ -5419,7 +5477,7 @@ stop_ckpt_thread: f2fs_stop_ckpt_thread(sbi); /* flush s_error_work before sbi destroy */ flush_work(&sbi->s_error_work); - f2fs_destroy_post_read_wq(sbi); + f2fs_destroy_wq(sbi); free_devices: destroy_device_list(sbi); kvfree(sbi->ckpt); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 352e96ad5c3a..665687244c93 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -1399,6 +1399,7 @@ F2FS_FEATURE_RO_ATTR(pin_file); F2FS_FEATURE_RO_ATTR(linear_lookup); #endif F2FS_FEATURE_RO_ATTR(packed_ssa); +F2FS_FEATURE_RO_ATTR(fserror); #define ATTR_LIST(name) (&f2fs_attr_##name.attr) static struct attribute *f2fs_attrs[] = { @@ -1566,6 +1567,7 @@ static struct attribute *f2fs_feat_attrs[] = { BASE_ATTR_LIST(linear_lookup), #endif BASE_ATTR_LIST(packed_ssa), + BASE_ATTR_LIST(fserror), NULL, }; ATTRIBUTE_GROUPS(f2fs_feat); diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c index 92ebcc19cab0..39f482515445 100644 --- a/fs/f2fs/verity.c +++ b/fs/f2fs/verity.c @@ -25,6 +25,7 @@ */ #include <linux/f2fs_fs.h> +#include <linux/fserror.h> #include "f2fs.h" #include "xattr.h" @@ -243,6 +244,7 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, f2fs_warn(F2FS_I_SB(inode), "invalid verity xattr"); f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_VERITY_XATTR); + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); return -EFSCORRUPTED; } if (buf_size) { diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 610d5810074d..ed33e5110f2a 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -19,6 +19,7 @@ #include <linux/f2fs_fs.h> #include <linux/security.h> #include <linux/posix_acl_xattr.h> +#include <linux/fserror.h> #include "f2fs.h" #include "xattr.h" #include "segment.h" @@ -371,6 +372,7 @@ static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio, err = -ENODATA; f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_XATTR); + fserror_report_file_metadata(inode, err, GFP_NOFS); goto out; } check: @@ -581,8 +583,6 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) size_t prefix_len; size_t size; - prefix = f2fs_xattr_prefix(entry->e_name_index, dentry); - if ((void *)(entry) + sizeof(__u32) > last_base_addr || (void *)XATTR_NEXT_ENTRY(entry) > last_base_addr) { f2fs_err(F2FS_I_SB(inode), "list inode (%llu) has corrupted xattr", @@ -590,9 +590,13 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_XATTR); - break; + fserror_report_file_metadata(inode, + -EFSCORRUPTED, GFP_NOFS); + error = -EFSCORRUPTED; + goto cleanup; } + prefix = f2fs_xattr_prefix(entry->e_name_index, dentry); if (!prefix) continue; @@ -677,6 +681,7 @@ retry: error = -EFSCORRUPTED; f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_XATTR); + fserror_report_file_metadata(inode, error, GFP_NOFS); goto exit; } @@ -705,6 +710,7 @@ retry: error = -EFSCORRUPTED; f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_XATTR); + fserror_report_file_metadata(inode, error, GFP_NOFS); goto exit; } last = XATTR_NEXT_ENTRY(last); diff --git a/fs/fat/dir.c b/fs/fat/dir.c index 4f6f42f33613..c6cca5d00ffd 100644 --- a/fs/fat/dir.c +++ b/fs/fat/dir.c @@ -131,6 +131,31 @@ static inline int fat_get_entry(struct inode *dir, loff_t *pos, } /* + * Like fat_get_entry(), but honour the FAT end-of-directory marker: + * a dirent whose first name byte is NUL terminates iteration per the + * spec, which also guarantees that every following slot is zeroed. + * Skip straight to the end of the directory so the next call returns + * -1 from fat_bmap() without re-reading the trailing zero slots, and + * so callers that persist *pos across invocations (e.g. readdir's + * ctx->pos) keep reporting EOD. Release *bh and set it to NULL to + * match fat_get_entry()'s contract that *bh is NULL on the -1 return. + */ +static int fat_get_entry_eod(struct inode *dir, loff_t *pos, + struct buffer_head **bh, + struct msdos_dir_entry **de) +{ + int err = fat_get_entry(dir, pos, bh, de); + + if (err == 0 && (*de)->name[0] == 0) { + brelse(*bh); + *bh = NULL; + *pos = dir->i_size; + return -1; + } + return err; +} + +/* * Convert Unicode 16 to UTF-8, translated Unicode, or ASCII. * If uni_xlate is enabled and we can't get a 1:1 conversion, use a * colon as an escape character since it is normally invalid on the vfat @@ -327,7 +352,7 @@ parse_long: if (ds->id & 0x40) (*unicode)[offset + 13] = 0; - if (fat_get_entry(dir, pos, bh, de) < 0) + if (fat_get_entry_eod(dir, pos, bh, de) < 0) return PARSE_EOF; if (slot == 0) break; @@ -489,7 +514,7 @@ int fat_search_long(struct inode *inode, const unsigned char *name, err = -ENOENT; while (1) { - if (fat_get_entry(inode, &cpos, &bh, &de) == -1) + if (fat_get_entry_eod(inode, &cpos, &bh, &de) == -1) goto end_of_dir; parse_record: nr_slots = 0; @@ -601,7 +626,7 @@ static int __fat_readdir(struct inode *inode, struct file *file, bh = NULL; get_new: - if (fat_get_entry(inode, &cpos, &bh, &de) == -1) + if (fat_get_entry_eod(inode, &cpos, &bh, &de) == -1) goto end_of_dir; parse_record: nr_slots = 0; @@ -885,7 +910,7 @@ static int fat_get_short_entry(struct inode *dir, loff_t *pos, struct buffer_head **bh, struct msdos_dir_entry **de) { - while (fat_get_entry(dir, pos, bh, de) >= 0) { + while (fat_get_entry_eod(dir, pos, bh, de) >= 0) { /* free entry or long name entry or volume label */ if (!IS_FREE((*de)->name) && !((*de)->attr & ATTR_VOLUME)) return 0; @@ -1302,6 +1327,7 @@ int fat_add_entries(struct inode *dir, void *slots, int nr_slots, struct msdos_dir_entry *de; int err, free_slots, i, nr_bhs; loff_t pos; + bool saw_eod; sinfo->nr_slots = nr_slots; @@ -1310,12 +1336,15 @@ int fat_add_entries(struct inode *dir, void *slots, int nr_slots, bh = prev = NULL; pos = 0; err = -ENOSPC; + saw_eod = false; while (fat_get_entry(dir, &pos, &bh, &de) > -1) { /* check the maximum size of directory */ if (pos >= FAT_MAX_DIR_SIZE) goto error; if (IS_FREE(de->name)) { + if (de->name[0] == 0) + saw_eod = true; if (prev != bh) { get_bh(bh); bhs[nr_bhs] = prev = bh; @@ -1325,6 +1354,13 @@ int fat_add_entries(struct inode *dir, void *slots, int nr_slots, if (free_slots == nr_slots) goto found; } else { + if (saw_eod) { + fat_fs_error_ratelimit(sb, + "allocated dir entry found after end-of-directory marker (i_pos %lld)", + MSDOS_I(dir)->i_pos); + err = -EIO; + goto error; + } for (i = 0; i < nr_bhs; i++) brelse(bhs[i]); prev = NULL; diff --git a/fs/fat/fat.h b/fs/fat/fat.h index 5a58f0bf8ce8..2772675bd35a 100644 --- a/fs/fat/fat.h +++ b/fs/fat/fat.h @@ -10,6 +10,8 @@ #include <linux/fs_context.h> #include <linux/fs_parser.h> +struct file_kattr; + /* * vfat shortname flags */ @@ -247,13 +249,13 @@ static inline unsigned char fat_checksum(const __u8 *name) return s; } -static inline sector_t fat_clus_to_blknr(struct msdos_sb_info *sbi, int clus) +static inline sector_t fat_clus_to_blknr(const struct msdos_sb_info *sbi, int clus) { return ((sector_t)clus - FAT_START_ENT) * sbi->sec_per_clus + sbi->data_start; } -static inline void fat_get_blknr_offset(struct msdos_sb_info *sbi, +static inline void fat_get_blknr_offset(const struct msdos_sb_info *sbi, loff_t i_pos, sector_t *blknr, int *offset) { *blknr = i_pos >> sbi->dir_per_block_bits; @@ -408,6 +410,7 @@ extern void fat_truncate_blocks(struct inode *inode, loff_t offset); extern int fat_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags); +int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa); extern int fat_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); diff --git a/fs/fat/fat_test.c b/fs/fat/fat_test.c index 886bf044a9f1..9583ce66dca3 100644 --- a/fs/fat/fat_test.c +++ b/fs/fat/fat_test.c @@ -20,6 +20,37 @@ static void fat_checksum_test(struct kunit *test) KUNIT_EXPECT_EQ(test, fat_checksum("ABCDEFGHA "), (u8)98); } +static void fat_clus_to_blknr_test(struct kunit *test) +{ + static const struct msdos_sb_info sbi = { + .sec_per_clus = 4, + .data_start = 100, + }; + + KUNIT_EXPECT_EQ(test, (sector_t)100, + fat_clus_to_blknr(&sbi, FAT_START_ENT)); + KUNIT_EXPECT_EQ(test, (sector_t)112, fat_clus_to_blknr(&sbi, 5)); +} + +static void fat_get_blknr_offset_test(struct kunit *test) +{ + static const struct msdos_sb_info sbi = { + .dir_per_block = 16, + .dir_per_block_bits = 4, + }; + + sector_t blknr; + int offset; + + fat_get_blknr_offset(&sbi, 0, &blknr, &offset); + KUNIT_EXPECT_EQ(test, (sector_t)0, blknr); + KUNIT_EXPECT_EQ(test, 0, offset); + + fat_get_blknr_offset(&sbi, (10 << 4) | 7, &blknr, &offset); + KUNIT_EXPECT_EQ(test, (sector_t)10, blknr); + KUNIT_EXPECT_EQ(test, 7, offset); +} + struct fat_timestamp_testcase { const char *name; struct timespec64 ts; @@ -341,6 +372,8 @@ static void fat_truncate_atime_test(struct kunit *test) static struct kunit_case fat_test_cases[] = { KUNIT_CASE(fat_checksum_test), + KUNIT_CASE(fat_clus_to_blknr_test), + KUNIT_CASE(fat_get_blknr_offset_test), KUNIT_CASE_PARAM(fat_time_fat2unix_test, fat_time_gen_params), KUNIT_CASE_PARAM(fat_time_unix2fat_test, fat_time_gen_params), KUNIT_CASE_PARAM(fat_time_unix2fat_clamp_test, diff --git a/fs/fat/file.c b/fs/fat/file.c index becccdd2e501..37e7049b4c8c 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -17,6 +17,7 @@ #include <linux/fsnotify.h> #include <linux/security.h> #include <linux/falloc.h> +#include <linux/fileattr.h> #include "fat.h" static long fat_fallocate(struct file *file, int mode, @@ -398,6 +399,40 @@ void fat_truncate_blocks(struct inode *inode, loff_t offset) fat_flush_inodes(inode->i_sb, inode, NULL); } +int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct msdos_sb_info *sbi = MSDOS_SB(dentry->d_sb); + bool case_sensitive; + + /* + * FAT filesystems are case-insensitive by default. VFAT + * becomes case-sensitive when mounted with 'check=strict', + * which installs vfat_dentry_ops. MSDOS has no such option; + * its 'nocase' mount option selects case-sensitive matching. + * + * VFAT long filename entries preserve case. Without VFAT, only + * uppercased 8.3 short names are stored. MSDOS with 'nocase' + * also preserves case. + */ + if (sbi->options.isvfat) + case_sensitive = sbi->options.name_check == 's'; + else + case_sensitive = sbi->options.nocase; + + if (!case_sensitive) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + if (!sbi->options.isvfat) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + } + if (d_inode(dentry)->i_flags & S_IMMUTABLE) { + fa->fsx_xflags |= FS_XFLAG_IMMUTABLE; + fa->flags |= FS_IMMUTABLE_FL; + } + return 0; +} +EXPORT_SYMBOL_GPL(fat_fileattr_get); + int fat_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { @@ -575,5 +610,6 @@ EXPORT_SYMBOL_GPL(fat_setattr); const struct inode_operations fat_file_inode_operations = { .setattr = fat_setattr, .getattr = fat_getattr, + .fileattr_get = fat_fileattr_get, .update_time = fat_update_time, }; diff --git a/fs/fat/inode.c b/fs/fat/inode.c index 28f78df086ef..3aa52481ad5c 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -1738,6 +1738,14 @@ int fat_fill_super(struct super_block *sb, struct fs_context *fc, if (total_sectors == 0) total_sectors = bpb.fat_total_sect; + if (total_sectors < sbi->data_start) { + if (!silent) + fat_msg(sb, KERN_ERR, + "data area starts beyond volume (%lu > %u)", + sbi->data_start, total_sectors); + goto out_invalid; + } + total_clusters = (total_sectors - sbi->data_start) / sbi->sec_per_clus; if (!is_fat32(sbi)) @@ -1786,7 +1794,7 @@ int fat_fill_super(struct super_block *sb, struct fs_context *fc, */ error = -EINVAL; - sprintf(buf, "cp%d", sbi->options.codepage); + scnprintf(buf, sizeof(buf), "cp%d", sbi->options.codepage); sbi->nls_disk = load_nls(buf); if (!sbi->nls_disk) { fat_msg(sb, KERN_ERR, "codepage %s not found", buf); diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c index 4cc65f330fb7..0fd2971ad4b1 100644 --- a/fs/fat/namei_msdos.c +++ b/fs/fat/namei_msdos.c @@ -644,6 +644,7 @@ static const struct inode_operations msdos_dir_inode_operations = { .rename = msdos_rename, .setattr = fat_setattr, .getattr = fat_getattr, + .fileattr_get = fat_fileattr_get, .update_time = fat_update_time, }; diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c index 918b3756674c..e909447873e3 100644 --- a/fs/fat/namei_vfat.c +++ b/fs/fat/namei_vfat.c @@ -1185,6 +1185,7 @@ static const struct inode_operations vfat_dir_inode_operations = { .rename = vfat_rename2, .setattr = fat_setattr, .getattr = fat_getattr, + .fileattr_get = fat_fileattr_get, .update_time = fat_update_time, }; diff --git a/fs/fcntl.c b/fs/fcntl.c index beab8080badf..c158f082f1da 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -929,11 +929,11 @@ void send_sigio(struct fown_struct *fown, int fd, int band) send_sigio_to_task(p, fown, fd, band, type); rcu_read_unlock(); } else { - read_lock(&tasklist_lock); + rcu_read_lock(); do_each_pid_task(pid, type, p) { send_sigio_to_task(p, fown, fd, band, type); } while_each_pid_task(pid, type, p); - read_unlock(&tasklist_lock); + rcu_read_unlock(); } out_unlock_fown: read_unlock_irqrestore(&fown->lock, flags); @@ -975,11 +975,11 @@ int send_sigurg(struct file *file) send_sigurg_to_task(p, fown, type); rcu_read_unlock(); } else { - read_lock(&tasklist_lock); + rcu_read_lock(); do_each_pid_task(pid, type, p) { send_sigurg_to_task(p, fown, type); } while_each_pid_task(pid, type, p); - read_unlock(&tasklist_lock); + rcu_read_unlock(); } out_unlock_fown: read_unlock_irqrestore(&fown->lock, flags); @@ -1169,10 +1169,10 @@ static int __init fcntl_init(void) * Exceptions: O_NONBLOCK is a two bit define on parisc; O_NDELAY * is defined as O_NONBLOCK on some platforms and not on others. */ - BUILD_BUG_ON(20 - 1 /* for O_RDONLY being 0 */ != + BUILD_BUG_ON(22 - 1 /* for O_RDONLY being 0 */ != HWEIGHT32( (VALID_OPEN_FLAGS & ~(O_NONBLOCK | O_NDELAY)) | - __FMODE_EXEC)); + __FMODE_EXEC | __O_REGULAR)); fasync_cache = kmem_cache_create("fasync_cache", sizeof(struct fasync_struct), 0, diff --git a/fs/fhandle.c b/fs/fhandle.c index 642e3d569497..f8829231e3d7 100644 --- a/fs/fhandle.c +++ b/fs/fhandle.c @@ -285,6 +285,19 @@ static int do_handle_to_path(struct file_handle *handle, struct path *path, return 0; } +static bool capable_wrt_mount(struct mount *mount) +{ + struct mnt_namespace *mnt_ns; + + /* + * For ->mnt_ns access. + * The following READ_ONCE() is semantically rcu_dereference(). + */ + guard(rcu)(); + mnt_ns = READ_ONCE(mount->mnt_ns); + return mnt_ns && ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN); +} + static inline int may_decode_fh(struct handle_to_path_ctx *ctx, unsigned int o_flags) { @@ -320,8 +333,7 @@ static inline int may_decode_fh(struct handle_to_path_ctx *ctx, if (ns_capable(root->mnt->mnt_sb->s_user_ns, CAP_SYS_ADMIN)) ctx->flags = HANDLE_CHECK_PERMS; else if (is_mounted(root->mnt) && - ns_capable(real_mount(root->mnt)->mnt_ns->user_ns, - CAP_SYS_ADMIN) && + capable_wrt_mount(real_mount(root->mnt)) && !has_locked_children(real_mount(root->mnt), root->dentry)) ctx->flags = HANDLE_CHECK_PERMS | HANDLE_CHECK_SUBTREE; else diff --git a/fs/file.c b/fs/file.c index 2c81c0b162d0..628ca07dc4b1 100644 --- a/fs/file.c +++ b/fs/file.c @@ -544,24 +544,23 @@ struct files_struct init_files = { static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start) { unsigned int maxfd = fdt->max_fds; /* always multiple of BITS_PER_LONG */ - unsigned int maxbit = maxfd / BITS_PER_LONG; - unsigned int bitbit = start / BITS_PER_LONG; + unsigned int max_fds_words = maxfd / BITS_PER_LONG; + unsigned int fds_word_idx = start / BITS_PER_LONG; unsigned int bit; /* * Try to avoid looking at the second level bitmap */ - bit = find_next_zero_bit(&fdt->open_fds[bitbit], BITS_PER_LONG, + bit = find_next_zero_bit(&fdt->open_fds[fds_word_idx], BITS_PER_LONG, start & (BITS_PER_LONG - 1)); if (bit < BITS_PER_LONG) - return bit + bitbit * BITS_PER_LONG; + return bit + (fds_word_idx * BITS_PER_LONG); - bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG; - if (bitbit >= maxfd) + bit = BITS_PER_LONG * + find_next_zero_bit(fdt->full_fds_bits, max_fds_words, fds_word_idx + 1); + if (bit >= maxfd) return maxfd; - if (bitbit > start) - start = bitbit; - return find_next_zero_bit(fdt->open_fds, maxfd, start); + return find_next_zero_bit(fdt->open_fds, maxfd, bit); } /* @@ -1134,7 +1133,6 @@ struct file *fget_task(struct task_struct *task, unsigned int fd) struct file *fget_task_next(struct task_struct *task, unsigned int *ret_fd) { - /* Must be called with rcu_read_lock held */ struct files_struct *files; unsigned int fd = *ret_fd; struct file *file = NULL; diff --git a/fs/file_attr.c b/fs/file_attr.c index da983e105d70..bfb00d256dd5 100644 --- a/fs/file_attr.c +++ b/fs/file_attr.c @@ -15,12 +15,10 @@ * @fa: fileattr pointer * @xflags: FS_XFLAG_* flags * - * Set ->fsx_xflags, ->fsx_valid and ->flags (translated xflags). All - * other fields are zeroed. + * Set ->fsx_xflags, ->fsx_valid and ->flags (translated xflags). */ void fileattr_fill_xflags(struct file_kattr *fa, u32 xflags) { - memset(fa, 0, sizeof(*fa)); fa->fsx_valid = true; fa->fsx_xflags = xflags; if (fa->fsx_xflags & FS_XFLAG_IMMUTABLE) @@ -39,6 +37,8 @@ void fileattr_fill_xflags(struct file_kattr *fa, u32 xflags) fa->flags |= FS_PROJINHERIT_FL; if (fa->fsx_xflags & FS_XFLAG_VERITY) fa->flags |= FS_VERITY_FL; + if (fa->fsx_xflags & FS_XFLAG_CASEFOLD) + fa->flags |= FS_CASEFOLD_FL; } EXPORT_SYMBOL(fileattr_fill_xflags); @@ -48,11 +48,9 @@ EXPORT_SYMBOL(fileattr_fill_xflags); * @flags: FS_*_FL flags * * Set ->flags, ->flags_valid and ->fsx_xflags (translated flags). - * All other fields are zeroed. */ void fileattr_fill_flags(struct file_kattr *fa, u32 flags) { - memset(fa, 0, sizeof(*fa)); fa->flags_valid = true; fa->flags = flags; if (fa->flags & FS_SYNC_FL) @@ -71,6 +69,8 @@ void fileattr_fill_flags(struct file_kattr *fa, u32 flags) fa->fsx_xflags |= FS_XFLAG_PROJINHERIT; if (fa->flags & FS_VERITY_FL) fa->fsx_xflags |= FS_XFLAG_VERITY; + if (fa->flags & FS_CASEFOLD_FL) + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; } EXPORT_SYMBOL(fileattr_fill_flags); @@ -325,7 +325,7 @@ int ioctl_setflags(struct file *file, unsigned int __user *argp) { struct mnt_idmap *idmap = file_mnt_idmap(file); struct dentry *dentry = file->f_path.dentry; - struct file_kattr fa; + struct file_kattr fa = {}; unsigned int flags; int err; @@ -357,7 +357,7 @@ int ioctl_fssetxattr(struct file *file, void __user *argp) { struct mnt_idmap *idmap = file_mnt_idmap(file); struct dentry *dentry = file->f_path.dentry; - struct file_kattr fa; + struct file_kattr fa = {}; int err; err = copy_fsxattr_from_user(&fa, argp); @@ -431,7 +431,7 @@ SYSCALL_DEFINE5(file_setattr, int, dfd, const char __user *, filename, struct path filepath __free(path_put) = {}; unsigned int lookup_flags = 0; struct file_attr fattr; - struct file_kattr fa; + struct file_kattr fa = {}; int error; BUILD_BUG_ON(sizeof(struct file_attr) < FILE_ATTR_SIZE_VER0); diff --git a/fs/file_table.c b/fs/file_table.c index 16e52e7fc2ac..c68b8c0a4097 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -231,13 +231,13 @@ static int init_file(struct file *f, int flags, const struct cred *cred) } /* Find an unused file structure and return a pointer to it. - * Returns an error pointer if some error happend e.g. we over file + * Returns an error pointer if some error happened, e.g., we exceed the file * structures limit, run out of memory or operation is not permitted. * * Be very careful using this. You are responsible for * getting write access to any mount that you might assign * to this filp, if it is opened for write. If this is not - * done, you will imbalance int the mount's writer count + * done, the mount's writer count will be wrong * and a warning at __fput() time. */ struct file *alloc_empty_file(int flags, const struct cred *cred) @@ -402,6 +402,8 @@ static struct file *alloc_file(const struct path *path, int flags, static inline int alloc_path_pseudo(const char *name, struct inode *inode, struct vfsmount *mnt, struct path *path) { + if (WARN_ON_ONCE(S_ISDIR(inode->i_mode))) + return -EINVAL; path->dentry = d_alloc_pseudo(mnt->mnt_sb, &QSTR(name)); if (!path->dentry) return -ENOMEM; diff --git a/fs/filesystems.c b/fs/filesystems.c index 0c7d2b7ac26c..673a03b5f32b 100644 --- a/fs/filesystems.c +++ b/fs/filesystems.c @@ -17,22 +17,49 @@ #include <linux/slab.h> #include <linux/uaccess.h> #include <linux/fs_parser.h> +#include <linux/rculist.h> /* - * Handling of filesystem drivers list. - * Rules: - * Inclusion to/removals from/scanning of list are protected by spinlock. - * During the unload module must call unregister_filesystem(). - * We can access the fields of list element if: - * 1) spinlock is held or - * 2) we hold the reference to the module. - * The latter can be guaranteed by call of try_module_get(); if it - * returned 0 we must skip the element, otherwise we got the reference. - * Once the reference is obtained we can drop the spinlock. + * Read-mostly filesystem drivers list. + * + * Readers walk under rcu_read_lock(); writers take file_systems_lock + * and publish via _rcu hlist primitives. unregister_filesystem() + * synchronize_rcu()s after unlock so the embedded file_system_type + * can't go away under a reader. To keep using a filesystem after + * the RCU section ends, take a module reference via try_module_get(). + */ +static HLIST_HEAD(file_systems); +static DEFINE_SPINLOCK(file_systems_lock); + +#ifdef CONFIG_PROC_FS +/* + * Cache a stringified version of the filesystem list. + * + * The fs list gets queried a lot by userspace because of libselinux, including + * rather surprising programs (would you guess *sed* is on the list?). In order + * to reduce the overhead we cache the resulting string, which normally hangs + * around below 512 bytes in size. + * + * As the list almost never changes, its creation is not particularly optimized + * to keep things simple. + * + * We sort it out on read in order to not introduce a failure point for fs + * registration (in principle we may be unable to alloc memory for the list). */ +struct file_systems_string { + struct rcu_head rcu; + unsigned long gen; + size_t len; + char string[]; +}; + +static unsigned long file_systems_gen; +static struct file_systems_string __read_mostly __rcu *file_systems_string; -static struct file_system_type *file_systems; -static DEFINE_RWLOCK(file_systems_lock); +static void invalidate_filesystems_string(void); +#else +static inline void invalidate_filesystems_string(void) { } +#endif /* WARNING: This can be used only if we _already_ own a reference */ struct file_system_type *get_filesystem(struct file_system_type *fs) @@ -46,14 +73,15 @@ void put_filesystem(struct file_system_type *fs) module_put(fs->owner); } -static struct file_system_type **find_filesystem(const char *name, unsigned len) +static struct file_system_type *find_filesystem(const char *name, unsigned len) { - struct file_system_type **p; - for (p = &file_systems; *p; p = &(*p)->next) - if (strncmp((*p)->name, name, len) == 0 && - !(*p)->name[len]) - break; - return p; + struct file_system_type *fs; + + hlist_for_each_entry_rcu(fs, &file_systems, list, + lockdep_is_held(&file_systems_lock)) + if (strncmp(fs->name, name, len) == 0 && !fs->name[len]) + return fs; + return NULL; } /** @@ -64,33 +92,27 @@ static struct file_system_type **find_filesystem(const char *name, unsigned len) * is aware of for mount and other syscalls. Returns 0 on success, * or a negative errno code on an error. * - * The &struct file_system_type that is passed is linked into the kernel + * The &struct file_system_type that is passed is linked into the kernel * structures and must not be freed until the file system has been * unregistered. */ - -int register_filesystem(struct file_system_type * fs) +int register_filesystem(struct file_system_type *fs) { - int res = 0; - struct file_system_type ** p; - if (fs->parameters && !fs_validate_description(fs->name, fs->parameters)) return -EINVAL; BUG_ON(strchr(fs->name, '.')); - if (fs->next) + if (!hlist_unhashed_lockless(&fs->list)) return -EBUSY; - write_lock(&file_systems_lock); - p = find_filesystem(fs->name, strlen(fs->name)); - if (*p) - res = -EBUSY; - else - *p = fs; - write_unlock(&file_systems_lock); - return res; -} + guard(spinlock)(&file_systems_lock); + if (find_filesystem(fs->name, strlen(fs->name))) + return -EBUSY; + hlist_add_tail_rcu(&fs->list, &file_systems); + invalidate_filesystems_string(); + return 0; +} EXPORT_SYMBOL(register_filesystem); /** @@ -100,94 +122,79 @@ EXPORT_SYMBOL(register_filesystem); * Remove a file system that was previously successfully registered * with the kernel. An error is returned if the file system is not found. * Zero is returned on a success. - * + * * Once this function has returned the &struct file_system_type structure * may be freed or reused. */ - -int unregister_filesystem(struct file_system_type * fs) +int unregister_filesystem(struct file_system_type *fs) { - struct file_system_type ** tmp; - - write_lock(&file_systems_lock); - tmp = &file_systems; - while (*tmp) { - if (fs == *tmp) { - *tmp = fs->next; - fs->next = NULL; - write_unlock(&file_systems_lock); - synchronize_rcu(); - return 0; - } - tmp = &(*tmp)->next; + scoped_guard(spinlock, &file_systems_lock) { + if (hlist_unhashed(&fs->list)) + return -EINVAL; + hlist_del_init_rcu(&fs->list); + invalidate_filesystems_string(); } - write_unlock(&file_systems_lock); - - return -EINVAL; + synchronize_rcu(); + return 0; } - EXPORT_SYMBOL(unregister_filesystem); #ifdef CONFIG_SYSFS_SYSCALL -static int fs_index(const char __user * __name) +static int fs_index(const char __user *__name) { - struct file_system_type * tmp; + struct file_system_type *p; char *name __free(kfree) = strndup_user(__name, PATH_MAX); - int err, index; + int index = 0; if (IS_ERR(name)) return PTR_ERR(name); - err = -EINVAL; - read_lock(&file_systems_lock); - for (tmp=file_systems, index=0 ; tmp ; tmp=tmp->next, index++) { - if (strcmp(tmp->name, name) == 0) { - err = index; - break; - } + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) { + if (strcmp(p->name, name) == 0) + return index; + index++; } - read_unlock(&file_systems_lock); - return err; + return -EINVAL; } -static int fs_name(unsigned int index, char __user * buf) +static int fs_name(unsigned int index, char __user *buf) { - struct file_system_type * tmp; - int len, res = -EINVAL; - - read_lock(&file_systems_lock); - for (tmp = file_systems; tmp; tmp = tmp->next, index--) { - if (index == 0) { - if (try_module_get(tmp->owner)) - res = 0; + struct file_system_type *p, *found = NULL; + int len, res; + + scoped_guard(rcu) { + hlist_for_each_entry_rcu(p, &file_systems, list) { + if (index--) + continue; + if (try_module_get(p->owner)) + found = p; break; } } - read_unlock(&file_systems_lock); - if (res) - return res; + if (!found) + return -EINVAL; /* OK, we got the reference, so we can safely block */ - len = strlen(tmp->name) + 1; - res = copy_to_user(buf, tmp->name, len) ? -EFAULT : 0; - put_filesystem(tmp); + len = strlen(found->name) + 1; + res = copy_to_user(buf, found->name, len) ? -EFAULT : 0; + put_filesystem(found); return res; } static int fs_maxindex(void) { - struct file_system_type * tmp; - int index; + struct file_system_type *p; + int index = 0; - read_lock(&file_systems_lock); - for (tmp = file_systems, index = 0 ; tmp ; tmp = tmp->next, index++) - ; - read_unlock(&file_systems_lock); + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) + index++; return index; } /* - * Whee.. Weird sysv syscall. + * Whee.. Weird sysv syscall. */ SYSCALL_DEFINE3(sysfs, int, option, unsigned long, arg1, unsigned long, arg2) { @@ -216,8 +223,8 @@ int __init list_bdev_fs_names(char *buf, size_t size) size_t len; int count = 0; - read_lock(&file_systems_lock); - for (p = file_systems; p; p = p->next) { + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) { if (!(p->fs_flags & FS_REQUIRES_DEV)) continue; len = strlen(p->name) + 1; @@ -230,30 +237,143 @@ int __init list_bdev_fs_names(char *buf, size_t size) size -= len; count++; } - read_unlock(&file_systems_lock); return count; } #ifdef CONFIG_PROC_FS -static int filesystems_proc_show(struct seq_file *m, void *v) +static void invalidate_filesystems_string(void) +{ + struct file_systems_string *old; + + lockdep_assert_held_write(&file_systems_lock); + file_systems_gen++; + old = rcu_replace_pointer(file_systems_string, NULL, + lockdep_is_held(&file_systems_lock)); + if (old) + kfree_rcu(old, rcu); +} + +static __cold noinline int regen_filesystems_string(void) +{ + struct file_system_type *p; + struct file_systems_string *old, *new; + size_t newlen, usedlen; + unsigned long gen; + +retry: + newlen = 0; + + /* pre-calc space for each fs */ + spin_lock(&file_systems_lock); + gen = file_systems_gen; + hlist_for_each_entry_rcu(p, &file_systems, list) { + if (!(p->fs_flags & FS_REQUIRES_DEV)) + newlen += strlen("nodev"); + newlen += strlen("\t") + strlen(p->name) + strlen("\n"); + } + spin_unlock(&file_systems_lock); + + new = kmalloc(offsetof(struct file_systems_string, string) + newlen + 1, + GFP_KERNEL); + if (!new) + return -ENOMEM; + + new->gen = gen; + new->len = newlen; + new->string[newlen] = '\0'; + + spin_lock(&file_systems_lock); + old = file_systems_string; + + /* + * Did someone beat us to it? + */ + if (old && old->gen == file_systems_gen) { + spin_unlock(&file_systems_lock); + kfree(new); + return 0; + } + + /* + * Did the list change in the meantime? + */ + if (gen != file_systems_gen) { + spin_unlock(&file_systems_lock); + kfree(new); + goto retry; + } + + /* + * Populate the string. + * + * We know we have just enough space because we calculated the right + * size the previous time we had the lock and confirmed the list has + * not changed after reacquiring it. + */ + usedlen = 0; + hlist_for_each_entry_rcu(p, &file_systems, list) { + usedlen += sprintf(&new->string[usedlen], "%s\t%s\n", + (p->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev", + p->name); + } + + if (WARN_ON_ONCE(new->len != strlen(new->string))) { + /* + * Should never happen of course, keep this in case someone changes string + * generation above and messes it up. + */ + spin_unlock(&file_systems_lock); + kfree(new); + return -EINVAL; + } + + rcu_assign_pointer(file_systems_string, new); + spin_unlock(&file_systems_lock); + if (old) + kfree_rcu(old, rcu); + return 0; +} + +static __cold noinline int filesystems_proc_show_fallback(struct seq_file *m, void *v) { - struct file_system_type * tmp; + struct file_system_type *p; - read_lock(&file_systems_lock); - tmp = file_systems; - while (tmp) { + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) { seq_printf(m, "%s\t%s\n", - (tmp->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev", - tmp->name); - tmp = tmp->next; + (p->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev", + p->name); } - read_unlock(&file_systems_lock); return 0; } +static int filesystems_proc_show(struct seq_file *m, void *v) +{ + struct file_systems_string *fss; + + for (;;) { + scoped_guard(rcu) { + fss = rcu_dereference(file_systems_string); + if (likely(fss)) { + seq_write(m, fss->string, fss->len); + return 0; + } + } + + int err = regen_filesystems_string(); + if (unlikely(err)) + return filesystems_proc_show_fallback(m, v); + } +} + static int __init proc_filesystems_init(void) { - proc_create_single("filesystems", 0, NULL, filesystems_proc_show); + struct proc_dir_entry *pde; + + pde = proc_create_single("filesystems", 0, NULL, filesystems_proc_show); + if (!pde) + return -ENOMEM; + proc_make_permanent(pde); return 0; } module_init(proc_filesystems_init); @@ -263,11 +383,10 @@ static struct file_system_type *__get_fs_type(const char *name, int len) { struct file_system_type *fs; - read_lock(&file_systems_lock); - fs = *(find_filesystem(name, len)); + guard(rcu)(); + fs = find_filesystem(name, len); if (fs && !try_module_get(fs->owner)) fs = NULL; - read_unlock(&file_systems_lock); return fs; } @@ -291,5 +410,4 @@ struct file_system_type *get_fs_type(const char *name) } return fs; } - EXPORT_SYMBOL(get_fs_type); diff --git a/fs/freevxfs/vxfs_bmap.c b/fs/freevxfs/vxfs_bmap.c index e85222892038..1b8216eb1d90 100644 --- a/fs/freevxfs/vxfs_bmap.c +++ b/fs/freevxfs/vxfs_bmap.c @@ -227,7 +227,8 @@ vxfs_bmap_typed(struct inode *ip, long iblock) return 0; } default: - BUG(); + WARN_ON_ONCE(1); + return 0; } } diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index a65694cbfe68..fdb8766d275a 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -432,6 +432,10 @@ static bool inode_do_switch_wbs(struct inode *inode, long nr = folio_nr_pages(folio); wb_stat_mod(old_wb, WB_RECLAIMABLE, -nr); wb_stat_mod(new_wb, WB_RECLAIMABLE, nr); + if (folio_test_dropbehind(folio)) { + wb_stat_mod(old_wb, WB_DONTCACHE_DIRTY, -nr); + wb_stat_mod(new_wb, WB_DONTCACHE_DIRTY, nr); + } } } @@ -497,6 +501,23 @@ skip_switch: return switched; } +static inline void cgroup_writeback_pin(struct super_block *sb) +{ + atomic_inc(&sb->s_isw_nr_in_flight); +} + +static inline void cgroup_writeback_unpin(struct super_block *sb) +{ + if (atomic_dec_and_test(&sb->s_isw_nr_in_flight)) + wake_up_var(&sb->s_isw_nr_in_flight); +} + +static inline void cgroup_writeback_drain(struct super_block *sb) +{ + wait_var_event(&sb->s_isw_nr_in_flight, + !atomic_read(&sb->s_isw_nr_in_flight)); +} + static void process_inode_switch_wbs(struct bdi_writeback *new_wb, struct inode_switch_wbs_context *isw) { @@ -554,8 +575,12 @@ relock: wb_put_many(old_wb, nr_switched); } - for (inodep = isw->inodes; *inodep; inodep++) + for (inodep = isw->inodes; *inodep; inodep++) { + struct super_block *sb = (*inodep)->i_sb; + iput(*inodep); + cgroup_writeback_unpin(sb); + } wb_put(new_wb); kfree(isw); atomic_dec(&isw_nr_in_flight); @@ -598,16 +623,19 @@ void inode_switch_wbs_work_fn(struct work_struct *work) static bool inode_prepare_wbs_switch(struct inode *inode, struct bdi_writeback *new_wb) { + /* Avoid the atomic_inc/smp_mb dance once SB_ACTIVE is gone. */ + if (!(inode->i_sb->s_flags & SB_ACTIVE)) + return false; + /* - * Paired with smp_mb() in cgroup_writeback_umount(). - * isw_nr_in_flight must be increased before checking SB_ACTIVE and - * grabbing an inode, otherwise isw_nr_in_flight can be observed as 0 - * in cgroup_writeback_umount() and the isw_wq will be not flushed. + * Pairs with smp_mb() in cgroup_writeback_umount(): the umounter either + * sees a non-zero counter and waits, or we see SB_ACTIVE clear below. */ + cgroup_writeback_pin(inode->i_sb); smp_mb(); if (IS_DAX(inode)) - return false; + goto out_unpin; /* while holding I_WB_SWITCH, no one else can update the association */ spin_lock(&inode->i_lock); @@ -615,13 +643,17 @@ static bool inode_prepare_wbs_switch(struct inode *inode, inode_state_read(inode) & (I_WB_SWITCH | I_FREEING | I_WILL_FREE) || inode_to_wb(inode) == new_wb) { spin_unlock(&inode->i_lock); - return false; + goto out_unpin; } inode_state_set(inode, I_WB_SWITCH); __iget(inode); spin_unlock(&inode->i_lock); return true; + +out_unpin: + cgroup_writeback_unpin(inode->i_sb); + return false; } static void wb_queue_isw(struct bdi_writeback *wb, @@ -1198,36 +1230,27 @@ out_bdi_put: } /** - * cgroup_writeback_umount - flush inode wb switches for umount + * cgroup_writeback_umount - wait for in-flight inode wb switches on @sb * @sb: target super_block * - * This function is called when a super_block is about to be destroyed and - * flushes in-flight inode wb switches. An inode wb switch goes through - * RCU and then workqueue, so the two need to be flushed in order to ensure - * that all previously scheduled switches are finished. As wb switches are - * rare occurrences and synchronize_rcu() can take a while, perform - * flushing iff wb switches are in flight. + * Wait until every inode wb switch that already passed the SB_ACTIVE + * check on this superblock has been completed by the worker. Since + * SB_ACTIVE is cleared before this is called, no new switches can start + * for @sb, so s_isw_nr_in_flight will monotonically drop to zero. */ void cgroup_writeback_umount(struct super_block *sb) { - if (!(sb->s_bdi->capabilities & BDI_CAP_WRITEBACK)) return; /* - * SB_ACTIVE should be reliably cleared before checking - * isw_nr_in_flight, see generic_shutdown_super(). + * Pairs with smp_mb() in inode_prepare_wbs_switch(): we either observe + * a non-zero counter and wait, or the switcher sees SB_ACTIVE clear + * (cleared by generic_shutdown_super()) and bails before grabbing the + * inode. */ smp_mb(); - - if (atomic_read(&isw_nr_in_flight)) { - /* - * Use rcu_barrier() to wait for all pending callbacks to - * ensure that all in-flight wb switches are in the workqueue. - */ - rcu_barrier(); - flush_workqueue(isw_wq); - } + cgroup_writeback_drain(sb); } static int __init cgroup_writeback_init(void) @@ -2373,6 +2396,27 @@ static long wb_check_start_all(struct bdi_writeback *wb) return nr_pages; } +static long wb_check_start_dontcache(struct bdi_writeback *wb) +{ + long nr_pages; + + if (!test_and_clear_bit(WB_start_dontcache, &wb->state)) + return 0; + + nr_pages = wb_stat_sum(wb, WB_DONTCACHE_DIRTY); + if (nr_pages) { + struct wb_writeback_work work = { + .nr_pages = nr_pages, + .sync_mode = WB_SYNC_NONE, + .range_cyclic = 1, + .reason = WB_REASON_DONTCACHE, + }; + + nr_pages = wb_writeback(wb, &work); + } + + return nr_pages; +} /* * Retrieve work items and do the writeback they describe @@ -2395,6 +2439,11 @@ static long wb_do_writeback(struct bdi_writeback *wb) wrote += wb_check_start_all(wb); /* + * Check for dontcache writeback request + */ + wrote += wb_check_start_dontcache(wb); + + /* * Check for periodic writeback, kupdated() style */ wrote += wb_check_old_data_flush(wb); @@ -2468,6 +2517,43 @@ void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi, rcu_read_unlock(); } +/** + * filemap_dontcache_kick_writeback - kick flusher for IOCB_DONTCACHE writes + * @mapping: address_space that was just written to + * + * Kick the writeback flusher thread to expedite writeback of dontcache dirty + * pages. Queue writeback for the inode's wb for as many pages as there are + * dontcache pages, but don't restrict writeback to dontcache pages only. + * + * This significantly improves performance over either writing all wb's pages + * or writing only dontcache pages. Although it doesn't guarantee quick + * writeback and reclaim of dontcache pages, it keeps the amount of dirty pages + * in check. Over longer term dontcache pages get written and reclaimed by + * background writeback even with this rough heuristic. + */ +void filemap_dontcache_kick_writeback(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + struct bdi_writeback *wb; + struct wb_lock_cookie cookie = {}; + bool need_wakeup = false; + + wb = unlocked_inode_to_wb_begin(inode, &cookie); + if (wb_has_dirty_io(wb) && + !test_bit(WB_start_dontcache, &wb->state) && + !test_and_set_bit(WB_start_dontcache, &wb->state)) { + wb_get(wb); + need_wakeup = true; + } + unlocked_inode_to_wb_end(inode, &cookie); + + if (need_wakeup) { + wb_wakeup(wb); + wb_put(wb); + } +} +EXPORT_SYMBOL_GPL(filemap_dontcache_kick_writeback); + /* * Wakeup the flusher threads to start writeback of all currently dirty pages */ diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index 22ad9538dfc4..245e67852b03 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -11,7 +11,8 @@ obj-$(CONFIG_CUSE) += cuse.o obj-$(CONFIG_VIRTIO_FS) += virtiofs.o fuse-y := trace.o # put trace.o first so we see ftrace errors sooner -fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o +fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o +fuse-y += poll.o notify.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o diff --git a/fs/fuse/acl.c b/fs/fuse/acl.c index cbde6ac1add3..31fb50e16aed 100644 --- a/fs/fuse/acl.c +++ b/fs/fuse/acl.c @@ -1,9 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * FUSE: Filesystem in Userspace * Copyright (C) 2016 Canonical Ltd. <seth.forshee@canonical.com> - * - * This program can be distributed under the terms of the GNU GPL. - * See the file COPYING. */ #include "fuse_i.h" diff --git a/fs/fuse/args.h b/fs/fuse/args.h new file mode 100644 index 000000000000..ecfe51a192af --- /dev/null +++ b/fs/fuse/args.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FS_FUSE_ARGS_H +#define _FS_FUSE_ARGS_H + +#include <linux/types.h> + +struct fuse_mount; + +/** One input argument of a request */ +struct fuse_in_arg { + unsigned size; + const void *value; +}; + +/** One output argument of a request */ +struct fuse_arg { + unsigned size; + void *value; +}; + +struct fuse_args { + u64 nodeid; + u32 opcode; + u32 uid; + u32 gid; + u32 pid; + u8 in_numargs; + u8 out_numargs; + u8 ext_idx; + bool force:1; + bool noreply:1; + bool nocreds:1; + bool in_pages:1; + bool out_pages:1; + bool user_pages:1; + bool out_argvar:1; + bool page_zeroing:1; + bool page_replace:1; + bool may_block:1; + bool is_ext:1; + bool is_pinned:1; + bool invalidate_vmap:1; + bool abort_on_kill:1; + struct fuse_in_arg in_args[4]; + struct fuse_arg out_args[2]; + void (*end)(struct fuse_args *args, int error); + /* Used for kvec iter backed by vmalloc address */ + void *vmap_base; +}; + +/** FUSE folio descriptor */ +struct fuse_folio_desc { + unsigned int length; + unsigned int offset; +}; + +struct fuse_args_pages { + struct fuse_args args; + struct folio **folios; + struct fuse_folio_desc *descs; + unsigned int num_folios; +}; + +#endif /* _FS_FUSE_ARGS_H */ diff --git a/fs/fuse/backing.c b/fs/fuse/backing.c index d95dfa48483f..472b6afa7dff 100644 --- a/fs/fuse/backing.c +++ b/fs/fuse/backing.c @@ -5,6 +5,7 @@ * Copyright (c) 2023 CTERA Networks. */ +#include "dev.h" #include "fuse_i.h" #include <linux/file.h> @@ -118,7 +119,7 @@ int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map) goto out_fput; fb->file = file; - fb->cred = prepare_creds(); + fb->cred = get_current_cred(); refcount_set(&fb->count, 1); res = fuse_backing_id_alloc(fc, fb); diff --git a/fs/fuse/control.c b/fs/fuse/control.c index f902a7fb4630..21ffde596d61 100644 --- a/fs/fuse/control.c +++ b/fs/fuse/control.c @@ -1,12 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ #include "fuse_i.h" +#include "dev.h" #include <linux/init.h> #include <linux/module.h> @@ -37,9 +36,7 @@ static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf, { struct fuse_conn *fc = fuse_ctl_file_conn_get(file); if (fc) { - if (fc->abort_err) - fc->aborted = true; - fuse_abort_conn(fc); + fuse_chan_abort(fc->chan, fc->abort_err); fuse_conn_put(fc); } return count; @@ -57,7 +54,7 @@ static ssize_t fuse_conn_waiting_read(struct file *file, char __user *buf, if (!fc) return 0; - value = atomic_read(&fc->num_waiting); + value = fuse_chan_num_waiting(fc->chan); file->private_data = (void *)value; fuse_conn_put(fc); } @@ -111,7 +108,7 @@ static ssize_t fuse_conn_max_background_read(struct file *file, if (!fc) return 0; - val = READ_ONCE(fc->max_background); + val = fuse_chan_max_background(fc->chan); fuse_conn_put(fc); return fuse_conn_limit_read(file, buf, len, ppos, val); @@ -129,12 +126,7 @@ static ssize_t fuse_conn_max_background_write(struct file *file, if (ret > 0) { struct fuse_conn *fc = fuse_ctl_file_conn_get(file); if (fc) { - spin_lock(&fc->bg_lock); - fc->max_background = val; - fc->blocked = fc->num_background >= fc->max_background; - if (!fc->blocked) - wake_up(&fc->blocked_waitq); - spin_unlock(&fc->bg_lock); + fuse_chan_max_background_set(fc->chan, val); fuse_conn_put(fc); } } diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 174333633471..3c15b5ba16d7 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -51,6 +51,7 @@ #include <linux/uio.h> #include <linux/user_namespace.h> +#include "dev.h" #include "fuse_i.h" #include "fuse_dev_i.h" @@ -306,6 +307,7 @@ struct cuse_init_args { struct cuse_init_out out; struct folio *folio; struct fuse_folio_desc desc; + struct fuse_conn *fc; }; /** @@ -319,11 +321,10 @@ struct cuse_init_args { * required data structures for it. Please read the comment at the * top of this file for high level overview. */ -static void cuse_process_init_reply(struct fuse_mount *fm, - struct fuse_args *args, int error) +static void cuse_process_init_reply(struct fuse_args *args, int error) { - struct fuse_conn *fc = fm->fc; struct cuse_init_args *ia = container_of(args, typeof(*ia), ap.args); + struct fuse_conn *fc = ia->fc; struct fuse_args_pages *ap = &ia->ap; struct cuse_conn *cc = fc_to_cc(fc), *pos; struct cuse_init_out *arg = &ia->out; @@ -391,7 +392,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, rc = -ENOMEM; cdev = cdev_alloc(); if (!cdev) - goto err_unlock; + goto err_dev; cdev->owner = THIS_MODULE; cdev->ops = &cuse_frontend_fops; @@ -417,13 +418,15 @@ out: err_cdev: cdev_del(cdev); +err_dev: + device_del(dev); err_unlock: mutex_unlock(&cuse_lock); put_device(dev); err_region: unregister_chrdev_region(devt, 1); err: - fuse_abort_conn(fc); + fuse_chan_abort(fc->chan, false); goto out; } @@ -466,6 +469,7 @@ static int cuse_send_init(struct cuse_conn *cc) ap->descs = &ia->desc; ia->folio = folio; ia->desc.length = ap->args.out_args[1].size; + ia->fc = &cc->fc; ap->args.end = cuse_process_init_reply; rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); @@ -502,8 +506,12 @@ static int cuse_channel_open(struct inode *inode, struct file *file) { struct fuse_dev *fud; struct cuse_conn *cc; + struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new(); int rc; + if (!fch) + return -ENOMEM; + /* set up cuse_conn */ cc = kzalloc_obj(*cc); if (!cc) @@ -513,18 +521,16 @@ static int cuse_channel_open(struct inode *inode, struct file *file) * Limit the cuse channel to requests that can * be represented in file->f_cred->user_ns. */ - fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns, - &fuse_dev_fiq_ops, NULL); - + fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns, no_free_ptr(fch)); cc->fc.release = cuse_fc_release; - fud = fuse_dev_alloc_install(&cc->fc); + fud = fuse_dev_alloc_install(cc->fc.chan); fuse_conn_put(&cc->fc); if (!fud) return -ENOMEM; INIT_LIST_HEAD(&cc->list); - cc->fc.initialized = 1; + cc->fc.chan->initialized = 1; rc = cuse_send_init(cc); if (rc) { fuse_dev_put(fud); @@ -549,7 +555,7 @@ static int cuse_channel_open(struct inode *inode, struct file *file) static int cuse_channel_release(struct inode *inode, struct file *file) { struct fuse_dev *fud = __fuse_get_dev(file); - struct cuse_conn *cc = fc_to_cc(fud->fc); + struct cuse_conn *cc = fc_to_cc(fud->chan->conn); /* remove from the conntbl, no more access from this point on */ mutex_lock(&cuse_lock); @@ -581,7 +587,7 @@ static ssize_t cuse_class_waiting_show(struct device *dev, { struct cuse_conn *cc = dev_get_drvdata(dev); - return sprintf(buf, "%d\n", atomic_read(&cc->fc.num_waiting)); + return sprintf(buf, "%d\n", atomic_read(&cc->fc.chan->num_waiting)); } static DEVICE_ATTR(waiting, 0400, cuse_class_waiting_show, NULL); @@ -591,7 +597,7 @@ static ssize_t cuse_class_abort_store(struct device *dev, { struct cuse_conn *cc = dev_get_drvdata(dev); - fuse_abort_conn(&cc->fc); + fuse_chan_abort(cc->fc.chan, false); return count; } static DEVICE_ATTR(abort, 0200, NULL, cuse_class_abort_store); diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c index f6cf00a8938c..8b53625ac7ab 100644 --- a/fs/fuse/dax.c +++ b/fs/fuse/dax.c @@ -113,7 +113,7 @@ __kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms) free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100, 1); if (fcd->nr_free_ranges < free_threshold) - queue_delayed_work(system_long_wq, &fcd->free_work, + queue_delayed_work(system_dfl_long_wq, &fcd->free_work, msecs_to_jiffies(delay_ms)); } diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 5dda7080f4a9..5763a7cd3b37 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -1,14 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ +#include "dev.h" +#include "args.h" #include "dev_uring_i.h" -#include "fuse_i.h" -#include "fuse_dev_i.h" #include <linux/init.h> #include <linux/module.h> @@ -30,124 +28,33 @@ MODULE_ALIAS_MISCDEV(FUSE_MINOR); MODULE_ALIAS("devname:fuse"); -static struct kmem_cache *fuse_req_cachep; - -const unsigned long fuse_timeout_timer_freq = - secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ); - -bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list) -{ - struct fuse_req *req; - - req = list_first_entry_or_null(list, struct fuse_req, list); - if (!req) - return false; - return time_is_before_jiffies(req->create_time + fc->timeout.req_timeout); -} - -static bool fuse_fpq_processing_expired(struct fuse_conn *fc, struct list_head *processing) -{ - int i; - - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - if (fuse_request_expired(fc, &processing[i])) - return true; - - return false; -} - -/* - * Check if any requests aren't being completed by the time the request timeout - * elapses. To do so, we: - * - check the fiq pending list - * - check the bg queue - * - check the fpq io and processing lists - * - * To make this fast, we only check against the head request on each list since - * these are generally queued in order of creation time (eg newer requests get - * queued to the tail). We might miss a few edge cases (eg requests transitioning - * between lists, re-sent requests at the head of the pending list having a - * later creation time than other requests on that list, etc.) but that is fine - * since if the request never gets fulfilled, it will eventually be caught. - */ -void fuse_check_timeout(struct work_struct *work) -{ - struct delayed_work *dwork = to_delayed_work(work); - struct fuse_conn *fc = container_of(dwork, struct fuse_conn, - timeout.work); - struct fuse_iqueue *fiq = &fc->iq; - struct fuse_dev *fud; - struct fuse_pqueue *fpq; - bool expired = false; - - if (!atomic_read(&fc->num_waiting)) - goto out; +static DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq); - spin_lock(&fiq->lock); - expired = fuse_request_expired(fc, &fiq->pending); - spin_unlock(&fiq->lock); - if (expired) - goto abort_conn; - - spin_lock(&fc->bg_lock); - expired = fuse_request_expired(fc, &fc->bg_queue); - spin_unlock(&fc->bg_lock); - if (expired) - goto abort_conn; - - spin_lock(&fc->lock); - if (!fc->connected) { - spin_unlock(&fc->lock); - return; - } - list_for_each_entry(fud, &fc->devices, entry) { - fpq = &fud->pq; - spin_lock(&fpq->lock); - if (fuse_request_expired(fc, &fpq->io) || - fuse_fpq_processing_expired(fc, fpq->processing)) { - spin_unlock(&fpq->lock); - spin_unlock(&fc->lock); - goto abort_conn; - } - - spin_unlock(&fpq->lock); - } - spin_unlock(&fc->lock); - - if (fuse_uring_request_expired(fc)) - goto abort_conn; - -out: - queue_delayed_work(system_percpu_wq, &fc->timeout.work, - fuse_timeout_timer_freq); - return; - -abort_conn: - fuse_abort_conn(fc); -} +static struct kmem_cache *fuse_req_cachep; -static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req) +static void fuse_request_init(struct fuse_chan *fch, struct fuse_req *req) { INIT_LIST_HEAD(&req->list); INIT_LIST_HEAD(&req->intr_entry); init_waitqueue_head(&req->waitq); refcount_set(&req->count, 1); __set_bit(FR_PENDING, &req->flags); - req->fm = fm; + req->chan = fch; req->create_time = jiffies; } -static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags) +static struct fuse_req *fuse_request_alloc(struct fuse_chan *fch, gfp_t flags) { struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags); if (req) - fuse_request_init(fm, req); + fuse_request_init(fch, req); return req; } static void fuse_request_free(struct fuse_req *req) { + WARN_ON(!list_empty(&req->intr_entry)); kmem_cache_free(fuse_req_cachep, req); } @@ -162,110 +69,86 @@ static void __fuse_put_request(struct fuse_req *req) refcount_dec(&req->count); } -void fuse_set_initialized(struct fuse_conn *fc) +void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param) { + if (param) { + fch->minor = param->minor; + fch->max_write = param->max_write; + fch->max_pages = param->max_pages; + } + /* Make sure stores before this are seen on another CPU */ smp_wmb(); - fc->initialized = 1; + fch->initialized = 1; + wake_up_all(&fch->blocked_waitq); } -static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background) +static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - return !fc->initialized || (for_background && fc->blocked) || - (fc->io_uring && fc->connected && !fuse_uring_ready(fc)); + return !fch->initialized || (for_background && fch->blocked) || + (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } -static void fuse_drop_waiting(struct fuse_conn *fc) +static void fuse_drop_waiting(struct fuse_chan *fch) { /* - * lockess check of fc->connected is okay, because atomic_dec_and_test() - * provides a memory barrier matched with the one in fuse_wait_aborted() + * lockess check of fch->connected is okay, because atomic_dec_and_test() + * provides a memory barrier matched with the one in fuse_chan_wait_aborted() * to ensure no wake-up is missed. */ - if (atomic_dec_and_test(&fc->num_waiting) && - !READ_ONCE(fc->connected)) { + if (atomic_dec_and_test(&fch->num_waiting) && + !READ_ONCE(fch->connected)) { /* wake up aborters */ - wake_up_all(&fc->blocked_waitq); + wake_up_all(&fch->blocked_waitq); } } static void fuse_put_request(struct fuse_req *req); -static struct fuse_req *fuse_get_req(struct mnt_idmap *idmap, - struct fuse_mount *fm, - bool for_background) +static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background) { - struct fuse_conn *fc = fm->fc; struct fuse_req *req; - bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP); - kuid_t fsuid; - kgid_t fsgid; int err; - atomic_inc(&fc->num_waiting); + atomic_inc(&fch->num_waiting); - if (fuse_block_alloc(fc, for_background)) { + if (fuse_block_alloc(fch, for_background)) { err = -EINTR; - if (wait_event_state_exclusive(fc->blocked_waitq, - !fuse_block_alloc(fc, for_background), + if (wait_event_state_exclusive(fch->blocked_waitq, + !fuse_block_alloc(fch, for_background), (TASK_KILLABLE | TASK_FREEZABLE))) goto out; } - /* Matches smp_wmb() in fuse_set_initialized() */ + + /* Matches smp_wmb() in fuse_chan_set_initialized() */ smp_rmb(); err = -ENOTCONN; - if (!fc->connected) + if (!fch->connected) goto out; - err = -ECONNREFUSED; - if (fc->conn_error) - goto out; - - req = fuse_request_alloc(fm, GFP_KERNEL); + req = fuse_request_alloc(fch, GFP_KERNEL); err = -ENOMEM; if (!req) { if (for_background) - wake_up(&fc->blocked_waitq); + wake_up(&fch->blocked_waitq); goto out; } - req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); - __set_bit(FR_WAITING, &req->flags); if (for_background) __set_bit(FR_BACKGROUND, &req->flags); - /* - * Keep the old behavior when idmappings support was not - * declared by a FUSE server. - * - * For those FUSE servers who support idmapped mounts, - * we send UID/GID only along with "inode creation" - * fuse requests, otherwise idmap == &invalid_mnt_idmap and - * req->in.h.{u,g}id will be equal to FUSE_INVALID_UIDGID. - */ - fsuid = no_idmap ? current_fsuid() : mapped_fsuid(idmap, fc->user_ns); - fsgid = no_idmap ? current_fsgid() : mapped_fsgid(idmap, fc->user_ns); - req->in.h.uid = from_kuid(fc->user_ns, fsuid); - req->in.h.gid = from_kgid(fc->user_ns, fsgid); - - if (no_idmap && unlikely(req->in.h.uid == ((uid_t)-1) || - req->in.h.gid == ((gid_t)-1))) { - fuse_put_request(req); - return ERR_PTR(-EOVERFLOW); - } - return req; out: - fuse_drop_waiting(fc); + fuse_drop_waiting(fch); return ERR_PTR(err); } static void fuse_put_request(struct fuse_req *req) { - struct fuse_conn *fc = req->fm->fc; + struct fuse_chan *fch = req->chan; if (refcount_dec_and_test(&req->count)) { if (test_bit(FR_BACKGROUND, &req->flags)) { @@ -273,15 +156,15 @@ static void fuse_put_request(struct fuse_req *req) * We get here in the unlikely case that a background * request was allocated but not sent */ - spin_lock(&fc->bg_lock); - if (!fc->blocked) - wake_up(&fc->blocked_waitq); - spin_unlock(&fc->bg_lock); + spin_lock(&fch->bg_lock); + if (!fch->blocked) + wake_up(&fch->blocked_waitq); + spin_unlock(&fch->bg_lock); } if (test_bit(FR_WAITING, &req->flags)) { __clear_bit(FR_WAITING, &req->flags); - fuse_drop_waiting(fc); + fuse_drop_waiting(fch); } fuse_request_free(req); @@ -335,6 +218,11 @@ __releases(fiq->lock) spin_unlock(&fiq->lock); } +struct fuse_forget_link *fuse_alloc_forget(void) +{ + return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT); +} + void fuse_dev_queue_forget(struct fuse_iqueue *fiq, struct fuse_forget_link *forget) { @@ -406,12 +294,271 @@ static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req) } } -const struct fuse_iqueue_ops fuse_dev_fiq_ops = { +static const struct fuse_iqueue_ops fuse_dev_fiq_ops = { .send_forget = fuse_dev_queue_forget, .send_interrupt = fuse_dev_queue_interrupt, .send_req = fuse_dev_queue_req, }; -EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops); + +void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv) +{ + spin_lock_init(&fiq->lock); + init_waitqueue_head(&fiq->waitq); + INIT_LIST_HEAD(&fiq->pending); + INIT_LIST_HEAD(&fiq->interrupts); + fiq->forget_list_tail = &fiq->forget_list_head; + fiq->connected = 1; + fiq->ops = ops; + fiq->priv = priv; +} +EXPORT_SYMBOL_GPL(fuse_iqueue_init); + +void fuse_chan_release(struct fuse_chan *fch) +{ + struct fuse_iqueue *fiq = &fch->iq; + + if (fiq->ops->release) + fiq->ops->release(fiq); + + if (fch->timeout.req_timeout) + cancel_delayed_work_sync(&fch->timeout.work); +} + +void fuse_chan_free(struct fuse_chan *fch) +{ + WARN_ON(!list_empty(&fch->devices)); + kfree(fch->pq_prealloc); + kfree(fch); +} +EXPORT_SYMBOL_GPL(fuse_chan_free); + +struct fuse_chan *fuse_chan_new(void) +{ + struct fuse_chan *fch = kzalloc_obj(struct fuse_chan); + if (!fch) + return NULL; + + spin_lock_init(&fch->lock); + INIT_LIST_HEAD(&fch->devices); + spin_lock_init(&fch->bg_lock); + INIT_LIST_HEAD(&fch->bg_queue); + init_waitqueue_head(&fch->blocked_waitq); + atomic_set(&fch->num_waiting, 0); + fch->max_background = FUSE_DEFAULT_MAX_BACKGROUND; + fch->initialized = 0; + fch->blocked = 0; + fch->connected = 1; + fch->timeout.req_timeout = 0; + + return fch; +} +EXPORT_SYMBOL_GPL(fuse_chan_new); + +struct list_head *fuse_pqueue_alloc(void) +{ + struct list_head *pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE); + + if (pq) { + for (int i = 0; i < FUSE_PQ_HASH_SIZE; i++) + INIT_LIST_HEAD(&pq[i]); + } + return pq; +} + +struct fuse_chan *fuse_dev_chan_new(void) +{ + struct fuse_chan *fch __free(kfree) = fuse_chan_new(); + if (!fch) + return NULL; + + fch->pq_prealloc = fuse_pqueue_alloc(); + if (!fch->pq_prealloc) + return NULL; + + fuse_iqueue_init(&fch->iq, &fuse_dev_fiq_ops, NULL); + + return no_free_ptr(fch); +} +EXPORT_SYMBOL_GPL(fuse_dev_chan_new); + +unsigned int fuse_chan_num_background(struct fuse_chan *fch) +{ + return READ_ONCE(fch->num_background); +} + +unsigned int fuse_chan_max_background(struct fuse_chan *fch) +{ + return READ_ONCE(fch->max_background); +} + +void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val) +{ + spin_lock(&fch->bg_lock); + fch->max_background = val; + fch->blocked = fch->num_background >= fch->max_background; + if (!fch->blocked) + wake_up(&fch->blocked_waitq); + spin_unlock(&fch->bg_lock); +} + +unsigned int fuse_chan_num_waiting(struct fuse_chan *fch) +{ + return atomic_read(&fch->num_waiting); +} + +void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc) +{ + fch->conn = fc; +} + +void fuse_chan_io_uring_enable(struct fuse_chan *fch) +{ + fch->io_uring = 1; +} + +void fuse_pqueue_init(struct fuse_pqueue *fpq) +{ + spin_lock_init(&fpq->lock); + INIT_LIST_HEAD(&fpq->io); + fpq->connected = 1; + fpq->processing = NULL; +} + +static struct fuse_dev *fuse_dev_alloc_no_pq(void) +{ + struct fuse_dev *fud; + + fud = kzalloc_obj(struct fuse_dev); + if (!fud) + return NULL; + + refcount_set(&fud->ref, 1); + fuse_pqueue_init(&fud->pq); + + return fud; +} + +struct fuse_dev *fuse_dev_alloc(void) +{ + struct fuse_dev *fud __free(kfree) = fuse_dev_alloc_no_pq(); + if (!fud) + return NULL; + + fud->pq.processing = fuse_pqueue_alloc(); + if (!fud->pq.processing) + return NULL; + + return no_free_ptr(fud); +} +EXPORT_SYMBOL_GPL(fuse_dev_alloc); + +/* + * Installs @fch into @fud, return true on success. "Consumes" @pq in either case. + */ +static bool fuse_dev_install_with_pq(struct fuse_dev *fud, struct fuse_chan *fch, + struct list_head *pq) +{ + struct fuse_chan *old_fch; + + guard(spinlock)(&fch->lock); + /* + * Pairs with: + * - xchg() in fuse_dev_release() + * - smp_load_acquire() in fuse_dev_fc_get() + */ + old_fch = cmpxchg(&fud->chan, NULL, fch); + if (old_fch) { + /* + * failed to set fud->chan because + * - it was already set to a different fc + * - it was set to disconneted + */ + kfree(pq); + return false; + } + if (pq) { + WARN_ON(fud->pq.processing); + fud->pq.processing = pq; + } + list_add_tail(&fud->entry, &fch->devices); + fuse_conn_get(fch->conn); + wake_up_all(&fuse_dev_waitq); + return true; +} + +void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch) +{ + struct list_head *pq = fch->pq_prealloc; + + fch->pq_prealloc = NULL; + if (!fuse_dev_install_with_pq(fud, fch, pq)) { + /* Channel is not usable without a dev */ + fuse_chan_abort(fch, false); + } +} +EXPORT_SYMBOL_GPL(fuse_dev_install); + +struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch) +{ + struct fuse_dev *fud; + + fud = fuse_dev_alloc_no_pq(); + if (!fud) + return NULL; + + fuse_dev_install(fud, fch); + return fud; +} +EXPORT_SYMBOL_GPL(fuse_dev_alloc_install); + +void fuse_dev_put(struct fuse_dev *fud) +{ + struct fuse_chan *fch; + + if (!refcount_dec_and_test(&fud->ref)) + return; + + fch = fuse_dev_chan_get(fud); + if (fch && fch != FUSE_DEV_CHAN_DISCONNECTED) { + /* This is the virtiofs case (fuse_dev_release() not called) */ + spin_lock(&fch->lock); + list_del(&fud->entry); + spin_unlock(&fch->lock); + + fuse_conn_put(fch->conn); + } + kfree(fud->pq.processing); + kfree(fud); +} +EXPORT_SYMBOL_GPL(fuse_dev_put); + +bool fuse_dev_is_installed(struct fuse_dev *fud) +{ + struct fuse_chan *fch = fuse_dev_chan_get(fud); + + return fch != NULL && fch != FUSE_DEV_CHAN_DISCONNECTED; +} + +/* + * Checks if @fc matches the one installed in @fud + */ +bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch) +{ + return fuse_dev_chan_get(fud) == fch; +} + +bool fuse_dev_is_sync_init(struct fuse_dev *fud) +{ + return fud->sync_init; +} + +struct fuse_dev *fuse_dev_grab(struct file *file) +{ + struct fuse_dev *fud = fuse_file_to_fud(file); + + refcount_inc(&fud->ref); + return fud; +} static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req) { @@ -421,10 +568,10 @@ static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req) fiq->ops->send_req(fiq, req); } -void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, - u64 nodeid, u64 nlookup) +void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget, + u64 nodeid, u64 nlookup) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fch->iq; forget->forget_one.nodeid = nodeid; forget->forget_one.nlookup = nlookup; @@ -432,21 +579,44 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, fiq->ops->send_forget(fiq, forget); } -static void flush_bg_queue(struct fuse_conn *fc) +static void flush_bg_queue(struct fuse_chan *fch) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fch->iq; - while (fc->active_background < fc->max_background && - !list_empty(&fc->bg_queue)) { + while (fch->active_background < fch->max_background && + !list_empty(&fch->bg_queue)) { struct fuse_req *req; - req = list_first_entry(&fc->bg_queue, struct fuse_req, list); + req = list_first_entry(&fch->bg_queue, struct fuse_req, list); list_del(&req->list); - fc->active_background++; + fch->active_background++; fuse_send_one(fiq, req); } } +void fuse_request_bg_finish(struct fuse_chan *fch, struct fuse_req *req) +{ + lockdep_assert_held(&fch->bg_lock); + + clear_bit(FR_BACKGROUND, &req->flags); + if (fch->num_background == fch->max_background) { + fch->blocked = 0; + wake_up(&fch->blocked_waitq); + } else if (!fch->blocked) { + /* + * Wake up next waiter, if any. It's okay to use + * waitqueue_active(), as we've already synced up + * fch->blocked with waiters with the wake_up() call + * above. + */ + if (waitqueue_active(&fch->blocked_waitq)) + wake_up(&fch->blocked_waitq); + } + + fch->num_background--; + fch->active_background--; +} + /* * This function is called when a request is finished. Either a reply * has arrived or it was aborted (and not yet sent) or some error @@ -457,9 +627,8 @@ static void flush_bg_queue(struct fuse_conn *fc) */ void fuse_request_end(struct fuse_req *req) { - struct fuse_mount *fm = req->fm; - struct fuse_conn *fc = fm->fc; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_chan *fch = req->chan; + struct fuse_iqueue *fiq = &fch->iq; if (test_and_set_bit(FR_FINISHED, &req->flags)) goto put_request; @@ -478,33 +647,17 @@ void fuse_request_end(struct fuse_req *req) WARN_ON(test_bit(FR_PENDING, &req->flags)); WARN_ON(test_bit(FR_SENT, &req->flags)); if (test_bit(FR_BACKGROUND, &req->flags)) { - spin_lock(&fc->bg_lock); - clear_bit(FR_BACKGROUND, &req->flags); - if (fc->num_background == fc->max_background) { - fc->blocked = 0; - wake_up(&fc->blocked_waitq); - } else if (!fc->blocked) { - /* - * Wake up next waiter, if any. It's okay to use - * waitqueue_active(), as we've already synced up - * fc->blocked with waiters with the wake_up() call - * above. - */ - if (waitqueue_active(&fc->blocked_waitq)) - wake_up(&fc->blocked_waitq); - } - - fc->num_background--; - fc->active_background--; - flush_bg_queue(fc); - spin_unlock(&fc->bg_lock); + spin_lock(&fch->bg_lock); + fuse_request_bg_finish(fch, req); + flush_bg_queue(fch); + spin_unlock(&fch->bg_lock); } else { /* Wake up waiter sleeping in request_wait_answer() */ wake_up(&req->waitq); } if (test_bit(FR_ASYNC, &req->flags)) - req->args->end(fm, req->args, req->out.h.error); + req->args->end(req->args, req->out.h.error); put_request: fuse_put_request(req); } @@ -512,7 +665,7 @@ EXPORT_SYMBOL_GPL(fuse_request_end); static int queue_interrupt(struct fuse_req *req) { - struct fuse_iqueue *fiq = &req->fm->fc->iq; + struct fuse_iqueue *fiq = &req->chan->iq; /* Check for we've sent request to interrupt this req */ if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) @@ -543,11 +696,11 @@ bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock) static void request_wait_answer(struct fuse_req *req) { - struct fuse_conn *fc = req->fm->fc; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_chan *fch = req->chan; + struct fuse_iqueue *fiq = &fch->iq; int err; - if (!fc->no_interrupt) { + if (!fch->no_interrupt) { /* Any signal may interrupt this */ err = wait_event_interruptible(req->waitq, test_bit(FR_FINISHED, &req->flags)); @@ -571,7 +724,7 @@ static void request_wait_answer(struct fuse_req *req) return; if (req->args->abort_on_kill) { - fuse_abort_conn(fc); + fuse_chan_abort(fch, false); return; } @@ -592,7 +745,7 @@ static void request_wait_answer(struct fuse_req *req) static void __fuse_request_send(struct fuse_req *req) { - struct fuse_iqueue *fiq = &req->fm->fc->iq; + struct fuse_iqueue *fiq = &req->chan->iq; BUG_ON(test_bit(FR_BACKGROUND, &req->flags)); @@ -606,12 +759,12 @@ static void __fuse_request_send(struct fuse_req *req) smp_rmb(); } -static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args) +static void fuse_adjust_compat(struct fuse_chan *fch, struct fuse_args *args) { - if (fc->minor < 4 && args->opcode == FUSE_STATFS) + if (fch->minor < 4 && args->opcode == FUSE_STATFS) args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE; - if (fc->minor < 9) { + if (fch->minor < 9) { switch (args->opcode) { case FUSE_LOOKUP: case FUSE_CREATE: @@ -627,7 +780,7 @@ static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args) break; } } - if (fc->minor < 12) { + if (fch->minor < 12) { switch (args->opcode) { case FUSE_CREATE: args->in_args[0].size = sizeof(struct fuse_open_in); @@ -639,25 +792,13 @@ static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args) } } -static void fuse_force_creds(struct fuse_req *req) -{ - struct fuse_conn *fc = req->fm->fc; - - if (!req->fm->sb || req->fm->sb->s_iflags & SB_I_NOIDMAP) { - req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid()); - req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid()); - } else { - req->in.h.uid = FUSE_INVALID_UIDGID; - req->in.h.gid = FUSE_INVALID_UIDGID; - } - - req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); -} - static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) { req->in.h.opcode = args->opcode; req->in.h.nodeid = args->nodeid; + req->in.h.uid = args->uid; + req->in.h.gid = args->gid; + req->in.h.pid = args->pid; req->args = args; if (args->is_ext) req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; @@ -665,33 +806,26 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) __set_bit(FR_ASYNC, &req->flags); } -ssize_t __fuse_simple_request(struct mnt_idmap *idmap, - struct fuse_mount *fm, - struct fuse_args *args) +ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args) { - struct fuse_conn *fc = fm->fc; struct fuse_req *req; ssize_t ret; if (args->force) { - atomic_inc(&fc->num_waiting); - req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL); - - if (!args->nocreds) - fuse_force_creds(req); + atomic_inc(&fch->num_waiting); + req = fuse_request_alloc(fch, GFP_KERNEL | __GFP_NOFAIL); __set_bit(FR_WAITING, &req->flags); if (!args->abort_on_kill) __set_bit(FR_FORCE, &req->flags); } else { - WARN_ON(args->nocreds); - req = fuse_get_req(idmap, fm, false); + req = fuse_get_req(fch, false); if (IS_ERR(req)) return PTR_ERR(req); } - /* Needs to be done after fuse_get_req() so that fc->minor is valid */ - fuse_adjust_compat(fc, args); + /* Needs to be done after fuse_get_req() so that fch->minor is valid */ + fuse_adjust_compat(fch, args); fuse_args_to_req(req, args); if (!args->noreply) @@ -708,10 +842,9 @@ ssize_t __fuse_simple_request(struct mnt_idmap *idmap, } #ifdef CONFIG_FUSE_IO_URING -static bool fuse_request_queue_background_uring(struct fuse_conn *fc, - struct fuse_req *req) +static bool fuse_request_queue_background_uring(struct fuse_req *req) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &req->chan->iq; req->in.h.len = sizeof(struct fuse_in_header) + fuse_len_args(req->args->in_numargs, @@ -727,50 +860,46 @@ static bool fuse_request_queue_background_uring(struct fuse_conn *fc, */ static int fuse_request_queue_background(struct fuse_req *req) { - struct fuse_mount *fm = req->fm; - struct fuse_conn *fc = fm->fc; + struct fuse_chan *fch = req->chan; bool queued = false; WARN_ON(!test_bit(FR_BACKGROUND, &req->flags)); if (!test_bit(FR_WAITING, &req->flags)) { __set_bit(FR_WAITING, &req->flags); - atomic_inc(&fc->num_waiting); + atomic_inc(&fch->num_waiting); } __set_bit(FR_ISREPLY, &req->flags); #ifdef CONFIG_FUSE_IO_URING - if (fuse_uring_ready(fc)) - return fuse_request_queue_background_uring(fc, req); + if (fuse_uring_ready(fch)) + return fuse_request_queue_background_uring(req); #endif - spin_lock(&fc->bg_lock); - if (likely(fc->connected)) { - fc->num_background++; - if (fc->num_background == fc->max_background) - fc->blocked = 1; - list_add_tail(&req->list, &fc->bg_queue); - flush_bg_queue(fc); + spin_lock(&fch->bg_lock); + if (likely(fch->connected)) { + fch->num_background++; + if (fch->num_background == fch->max_background) + fch->blocked = 1; + list_add_tail(&req->list, &fch->bg_queue); + flush_bg_queue(fch); queued = true; } - spin_unlock(&fc->bg_lock); + spin_unlock(&fch->bg_lock); return queued; } -int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, - gfp_t gfp_flags) +int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags) { struct fuse_req *req; if (args->force) { - WARN_ON(!args->nocreds); - req = fuse_request_alloc(fm, gfp_flags); + req = fuse_request_alloc(fch, gfp_flags); if (!req) return -ENOMEM; __set_bit(FR_BACKGROUND, &req->flags); } else { - WARN_ON(args->nocreds); - req = fuse_get_req(&invalid_mnt_idmap, fm, true); + req = fuse_get_req(fch, true); if (IS_ERR(req)) return PTR_ERR(req); } @@ -784,15 +913,13 @@ int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, return 0; } -EXPORT_SYMBOL_GPL(fuse_simple_background); -static int fuse_simple_notify_reply(struct fuse_mount *fm, - struct fuse_args *args, u64 unique) +int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique) { struct fuse_req *req; - struct fuse_iqueue *fiq = &fm->fc->iq; + struct fuse_iqueue *fiq = &fch->iq; - req = fuse_get_req(&invalid_mnt_idmap, fm, false); + req = fuse_get_req(fch, false); if (IS_ERR(req)) return PTR_ERR(req); @@ -1037,6 +1164,10 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop if (WARN_ON(folio_test_mlocked(oldfolio))) goto out_fallback_unlock; + err = lock_request(cs->req); + if (err) + goto out_fallback_unlock; + replace_page_cache_folio(oldfolio, newfolio); folio_get(newfolio); @@ -1050,20 +1181,7 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop */ pipe_buf_release(cs->pipe, buf); - err = 0; - spin_lock(&cs->req->waitq.lock); - if (test_bit(FR_ABORTED, &cs->req->flags)) - err = -ENOENT; - else - *foliop = newfolio; - spin_unlock(&cs->req->waitq.lock); - - if (err) { - folio_unlock(newfolio); - folio_put(newfolio); - goto out_put_old; - } - + *foliop = newfolio; folio_unlock(oldfolio); /* Drop ref for ap->pages[] array */ folio_put(oldfolio); @@ -1115,15 +1233,15 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio, cs->nr_segs++; cs->len = 0; - return 0; + return lock_request(cs->req); } /* * Copy a folio in the request to/from the userspace buffer. Must be * done atomically */ -static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, - unsigned offset, unsigned count, int zeroing) +int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, + unsigned offset, unsigned count, int zeroing) { int err; struct folio *folio = *foliop; @@ -1204,7 +1322,7 @@ static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes, } /* Copy a single argument in the request to/from userspace buffer */ -static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) +int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) { while (size) { if (!cs->len) { @@ -1254,34 +1372,29 @@ static int request_pending(struct fuse_iqueue *fiq) * * Called with fiq->lock held, releases it */ -static int fuse_read_interrupt(struct fuse_iqueue *fiq, - struct fuse_copy_state *cs, - size_t nbytes, struct fuse_req *req) +static int fuse_read_interrupt(struct fuse_iqueue *fiq, struct fuse_copy_state *cs) __releases(fiq->lock) { - struct fuse_in_header ih; - struct fuse_interrupt_in arg; - unsigned reqsize = sizeof(ih) + sizeof(arg); + struct fuse_req *req = list_first_entry(&fiq->interrupts, struct fuse_req, intr_entry); + struct fuse_interrupt_in arg = { + .unique = req->in.h.unique, + }; + struct fuse_in_header ih = { + .opcode = FUSE_INTERRUPT, + .unique = (req->in.h.unique | FUSE_INT_REQ_BIT), + .len = sizeof(ih) + sizeof(arg), + }; int err; list_del_init(&req->intr_entry); - memset(&ih, 0, sizeof(ih)); - memset(&arg, 0, sizeof(arg)); - ih.len = reqsize; - ih.opcode = FUSE_INTERRUPT; - ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT); - arg.unique = req->in.h.unique; - spin_unlock(&fiq->lock); - if (nbytes < reqsize) - return -EINVAL; err = fuse_copy_one(cs, &ih, sizeof(ih)); if (!err) err = fuse_copy_one(cs, &arg, sizeof(arg)); fuse_copy_finish(cs); - return err ? err : reqsize; + return err ? err : ih.len; } static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq, @@ -1307,8 +1420,7 @@ static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq, } static int fuse_read_single_forget(struct fuse_iqueue *fiq, - struct fuse_copy_state *cs, - size_t nbytes) + struct fuse_copy_state *cs) __releases(fiq->lock) { int err; @@ -1325,8 +1437,6 @@ __releases(fiq->lock) spin_unlock(&fiq->lock); kfree(forget); - if (nbytes < ih.len) - return -EINVAL; err = fuse_copy_one(cs, &ih, sizeof(ih)); if (!err) @@ -1354,11 +1464,6 @@ __releases(fiq->lock) .len = sizeof(ih) + sizeof(arg), }; - if (nbytes < ih.len) { - spin_unlock(&fiq->lock); - return -EINVAL; - } - max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one); head = fuse_dequeue_forget(fiq, max_forgets, &count); spin_unlock(&fiq->lock); @@ -1388,13 +1493,13 @@ __releases(fiq->lock) return ih.len; } -static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq, +static int fuse_read_forget(struct fuse_chan *fch, struct fuse_iqueue *fiq, struct fuse_copy_state *cs, size_t nbytes) __releases(fiq->lock) { - if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL) - return fuse_read_single_forget(fiq, cs, nbytes); + if (fch->minor < 16 || fiq->forget_list_head.next->next == NULL) + return fuse_read_single_forget(fiq, cs); else return fuse_read_batch_forget(fiq, cs, nbytes); } @@ -1412,8 +1517,8 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, struct fuse_copy_state *cs, size_t nbytes) { ssize_t err; - struct fuse_conn *fc = fud->fc; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_chan *fch = fud->chan; + struct fuse_iqueue *fiq = &fch->iq; struct fuse_pqueue *fpq = &fud->pq; struct fuse_req *req; struct fuse_args *args; @@ -1435,7 +1540,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER, sizeof(struct fuse_in_header) + sizeof(struct fuse_write_in) + - fc->max_write)) + fch->max_write)) return -EINVAL; restart: @@ -1454,19 +1559,16 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, } if (!fiq->connected) { - err = fc->aborted ? -ECONNABORTED : -ENODEV; + err = fch->abort_with_err ? -ECONNABORTED : -ENODEV; goto err_unlock; } - if (!list_empty(&fiq->interrupts)) { - req = list_entry(fiq->interrupts.next, struct fuse_req, - intr_entry); - return fuse_read_interrupt(fiq, cs, nbytes, req); - } + if (!list_empty(&fiq->interrupts)) + return fuse_read_interrupt(fiq, cs); if (forget_pending(fiq)) { if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0) - return fuse_read_forget(fc, fiq, cs, nbytes); + return fuse_read_forget(fch, fiq, cs, nbytes); if (fiq->forget_batch <= -8) fiq->forget_batch = 16; @@ -1492,12 +1594,11 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, spin_lock(&fpq->lock); /* * Must not put request on fpq->io queue after having been shut down by - * fuse_abort_conn() + * fuse_chan_abort() */ if (!fpq->connected) { req->out.h.error = err = -ECONNABORTED; goto out_end; - } list_add(&req->list, &fpq->io); spin_unlock(&fpq->lock); @@ -1510,7 +1611,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, spin_lock(&fpq->lock); clear_bit(FR_LOCKED, &req->flags); if (!fpq->connected) { - err = fc->aborted ? -ECONNABORTED : -ENODEV; + err = fch->abort_with_err ? -ECONNABORTED : -ENODEV; goto out_end; } if (err) { @@ -1525,6 +1626,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, list_move_tail(&req->list, &fpq->processing[hash]); __fuse_get_request(req); set_bit(FR_SENT, &req->flags); + trace_fuse_request_sent(req); spin_unlock(&fpq->lock); /* matches barrier in request_wait_answer() */ smp_mb__after_atomic(); @@ -1548,7 +1650,7 @@ out_end: static int fuse_dev_open(struct inode *inode, struct file *file) { - struct fuse_dev *fud = fuse_dev_alloc(); + struct fuse_dev *fud = fuse_dev_alloc_no_pq(); if (!fud) return -ENOMEM; @@ -1562,9 +1664,15 @@ struct fuse_dev *fuse_get_dev(struct file *file) struct fuse_dev *fud = fuse_file_to_fud(file); int err; - err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_fc_get(fud) != NULL); - if (err) - return ERR_PTR(err); + if (unlikely(!fuse_dev_chan_get(fud))) { + /* only block waiting for mount if sync init was requested */ + if (!fud->sync_init) + return ERR_PTR(-EPERM); + + err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_chan_get(fud) != NULL); + if (err) + return ERR_PTR(err); + } return fud; } @@ -1636,344 +1744,6 @@ out: return ret; } -static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_poll_wakeup_out outarg; - int err; - - if (size != sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - fuse_copy_finish(cs); - return fuse_notify_poll_wakeup(fc, &outarg); -} - -static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_inval_inode_out outarg; - int err; - - if (size != sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - fuse_copy_finish(cs); - - down_read(&fc->killsb); - err = fuse_reverse_inval_inode(fc, outarg.ino, - outarg.off, outarg.len); - up_read(&fc->killsb); - return err; -} - -static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_inval_entry_out outarg; - int err; - char *buf; - struct qstr name; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (outarg.namelen > fc->name_max) - return -ENAMETOOLONG; - - err = -EINVAL; - if (size != sizeof(outarg) + outarg.namelen + 1) - return -EINVAL; - - buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - name.name = buf; - name.len = outarg.namelen; - err = fuse_copy_one(cs, buf, outarg.namelen + 1); - if (err) - goto err; - fuse_copy_finish(cs); - buf[outarg.namelen] = 0; - - down_read(&fc->killsb); - err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags); - up_read(&fc->killsb); -err: - kfree(buf); - return err; -} - -static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_delete_out outarg; - int err; - char *buf; - struct qstr name; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (outarg.namelen > fc->name_max) - return -ENAMETOOLONG; - - if (size != sizeof(outarg) + outarg.namelen + 1) - return -EINVAL; - - buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - name.name = buf; - name.len = outarg.namelen; - err = fuse_copy_one(cs, buf, outarg.namelen + 1); - if (err) - goto err; - fuse_copy_finish(cs); - buf[outarg.namelen] = 0; - - down_read(&fc->killsb); - err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0); - up_read(&fc->killsb); -err: - kfree(buf); - return err; -} - -static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_store_out outarg; - struct inode *inode; - struct address_space *mapping; - u64 nodeid; - int err; - unsigned int num; - loff_t file_size; - loff_t pos; - loff_t end; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (size - sizeof(outarg) != outarg.size) - return -EINVAL; - - if (outarg.offset >= MAX_LFS_FILESIZE) - return -EINVAL; - - nodeid = outarg.nodeid; - pos = outarg.offset; - num = min(outarg.size, MAX_LFS_FILESIZE - pos); - - down_read(&fc->killsb); - - err = -ENOENT; - inode = fuse_ilookup(fc, nodeid, NULL); - if (!inode) - goto out_up_killsb; - - mapping = inode->i_mapping; - file_size = i_size_read(inode); - end = pos + num; - if (end > file_size) { - file_size = end; - fuse_write_update_attr(inode, file_size, num); - } - - while (num) { - struct folio *folio; - unsigned int folio_offset; - unsigned int nr_bytes; - pgoff_t index = pos >> PAGE_SHIFT; - - folio = filemap_grab_folio(mapping, index); - err = PTR_ERR(folio); - if (IS_ERR(folio)) - goto out_iput; - - folio_offset = offset_in_folio(folio, pos); - nr_bytes = min(num, folio_size(folio) - folio_offset); - - err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0); - if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && - (nr_bytes == folio_size(folio) || file_size == end)) { - folio_zero_segment(folio, nr_bytes, folio_size(folio)); - folio_mark_uptodate(folio); - } - folio_unlock(folio); - folio_put(folio); - - if (err) - goto out_iput; - - pos += nr_bytes; - num -= nr_bytes; - } - - err = 0; - -out_iput: - iput(inode); -out_up_killsb: - up_read(&fc->killsb); - return err; -} - -struct fuse_retrieve_args { - struct fuse_args_pages ap; - struct fuse_notify_retrieve_in inarg; -}; - -static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, - int error) -{ - struct fuse_retrieve_args *ra = - container_of(args, typeof(*ra), ap.args); - - release_pages(ra->ap.folios, ra->ap.num_folios); - kfree(ra); -} - -static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, - struct fuse_notify_retrieve_out *outarg) -{ - int err; - struct address_space *mapping = inode->i_mapping; - loff_t file_size; - unsigned int num; - unsigned int offset; - size_t total_len = 0; - unsigned int num_pages; - struct fuse_conn *fc = fm->fc; - struct fuse_retrieve_args *ra; - size_t args_size = sizeof(*ra); - struct fuse_args_pages *ap; - struct fuse_args *args; - loff_t pos = outarg->offset; - - offset = offset_in_page(pos); - file_size = i_size_read(inode); - - num = min(outarg->size, fc->max_write); - if (pos > file_size) - num = 0; - else if (num > file_size - pos) - num = file_size - pos; - - num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE); - num_pages = min(num_pages, fc->max_pages); - num = min(num, num_pages << PAGE_SHIFT); - - args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); - - ra = kzalloc(args_size, GFP_KERNEL); - if (!ra) - return -ENOMEM; - - ap = &ra->ap; - ap->folios = (void *) (ra + 1); - ap->descs = (void *) (ap->folios + num_pages); - - args = &ap->args; - args->nodeid = outarg->nodeid; - args->opcode = FUSE_NOTIFY_REPLY; - args->in_numargs = 3; - args->in_pages = true; - args->end = fuse_retrieve_end; - - while (num && ap->num_folios < num_pages) { - struct folio *folio; - unsigned int folio_offset; - unsigned int nr_bytes; - pgoff_t index = pos >> PAGE_SHIFT; - - folio = filemap_get_folio(mapping, index); - if (IS_ERR(folio)) - break; - - folio_offset = offset_in_folio(folio, pos); - nr_bytes = min(folio_size(folio) - folio_offset, num); - - ap->folios[ap->num_folios] = folio; - ap->descs[ap->num_folios].offset = folio_offset; - ap->descs[ap->num_folios].length = nr_bytes; - ap->num_folios++; - - pos += nr_bytes; - num -= nr_bytes; - total_len += nr_bytes; - } - ra->inarg.offset = outarg->offset; - ra->inarg.size = total_len; - fuse_set_zero_arg0(args); - args->in_args[1].size = sizeof(ra->inarg); - args->in_args[1].value = &ra->inarg; - args->in_args[2].size = total_len; - - err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); - if (err) - fuse_retrieve_end(fm, args, err); - - return err; -} - -static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_retrieve_out outarg; - struct fuse_mount *fm; - struct inode *inode; - u64 nodeid; - int err; - - if (size != sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - fuse_copy_finish(cs); - - if (outarg.offset >= MAX_LFS_FILESIZE) - return -EINVAL; - - down_read(&fc->killsb); - err = -ENOENT; - nodeid = outarg.nodeid; - - inode = fuse_ilookup(fc, nodeid, &fm); - if (inode) { - err = fuse_retrieve(fm, inode, &outarg); - iput(inode); - } - up_read(&fc->killsb); - - return err; -} - /* * Resending all processing queue requests. * @@ -1987,21 +1757,21 @@ static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, * if the FUSE daemon takes careful measures to avoid processing duplicated * non-idempotent requests. */ -static void fuse_resend(struct fuse_conn *fc) +void fuse_chan_resend(struct fuse_chan *fch) { struct fuse_dev *fud; struct fuse_req *req, *next; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fch->iq; LIST_HEAD(to_queue); unsigned int i; - spin_lock(&fc->lock); - if (!fc->connected) { - spin_unlock(&fc->lock); + spin_lock(&fch->lock); + if (!fch->connected) { + spin_unlock(&fch->lock); return; } - list_for_each_entry(fud, &fc->devices, entry) { + list_for_each_entry(fud, &fch->devices, entry) { struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); @@ -2009,7 +1779,7 @@ static void fuse_resend(struct fuse_conn *fc) list_splice_tail_init(&fpq->processing[i], &to_queue); spin_unlock(&fpq->lock); } - spin_unlock(&fc->lock); + spin_unlock(&fch->lock); list_for_each_entry_safe(req, next, &to_queue, list) { set_bit(FR_PENDING, &req->flags); @@ -2026,111 +1796,17 @@ static void fuse_resend(struct fuse_conn *fc) fuse_dev_end_requests(&to_queue); return; } - /* iq and pq requests are both oldest to newest */ - list_splice(&to_queue, &fiq->pending); - fuse_dev_wake_and_unlock(fiq); -} - -static int fuse_notify_resend(struct fuse_conn *fc) -{ - fuse_resend(fc); - return 0; -} - -/* - * Increments the fuse connection epoch. This will result of dentries from - * previous epochs to be invalidated. Additionally, if inval_wq is set, a work - * queue is scheduled to trigger the invalidation. - */ -static int fuse_notify_inc_epoch(struct fuse_conn *fc) -{ - atomic_inc(&fc->epoch); - if (inval_wq) - schedule_work(&fc->epoch_work); - - return 0; -} - -static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size, - struct fuse_copy_state *cs) -{ - struct fuse_notify_prune_out outarg; - const unsigned int batch = 512; - u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL); - unsigned int num, i; - int err; - - if (!nodeids) - return -ENOMEM; - - if (size < sizeof(outarg)) - return -EINVAL; - - err = fuse_copy_one(cs, &outarg, sizeof(outarg)); - if (err) - return err; - - if (size - sizeof(outarg) != outarg.count * sizeof(u64)) - return -EINVAL; - - for (; outarg.count; outarg.count -= num) { - num = min(batch, outarg.count); - err = fuse_copy_one(cs, nodeids, num * sizeof(u64)); - if (err) - return err; - - scoped_guard(rwsem_read, &fc->killsb) { - for (i = 0; i < num; i++) - fuse_try_prune_one_inode(fc, nodeids[i]); - } - } - return 0; -} - -static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, - unsigned int size, struct fuse_copy_state *cs) -{ /* - * Only allow notifications during while the connection is in an - * initialized and connected state + * Remove interrupt entries for resent requests to prevent stale + * intr_entry on fiq->interrupts after the request is re-queued. */ - if (!fc->initialized || !fc->connected) - return -EINVAL; - - /* Don't try to move folios (yet) */ - cs->move_folios = false; - - switch (code) { - case FUSE_NOTIFY_POLL: - return fuse_notify_poll(fc, size, cs); - - case FUSE_NOTIFY_INVAL_INODE: - return fuse_notify_inval_inode(fc, size, cs); - - case FUSE_NOTIFY_INVAL_ENTRY: - return fuse_notify_inval_entry(fc, size, cs); - - case FUSE_NOTIFY_STORE: - return fuse_notify_store(fc, size, cs); - - case FUSE_NOTIFY_RETRIEVE: - return fuse_notify_retrieve(fc, size, cs); - - case FUSE_NOTIFY_DELETE: - return fuse_notify_delete(fc, size, cs); - - case FUSE_NOTIFY_RESEND: - return fuse_notify_resend(fc); - - case FUSE_NOTIFY_INC_EPOCH: - return fuse_notify_inc_epoch(fc); - - case FUSE_NOTIFY_PRUNE: - return fuse_notify_prune(fc, size, cs); - - default: - return -EINVAL; + list_for_each_entry(req, &to_queue, list) { + if (test_bit(FR_INTERRUPTED, &req->flags)) + list_del_init(&req->intr_entry); } + /* iq and pq requests are both oldest to newest */ + list_splice(&to_queue, &fiq->pending); + fuse_dev_wake_and_unlock(fiq); } /* Look up request on processing list by unique ID */ @@ -2185,7 +1861,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, struct fuse_copy_state *cs, size_t nbytes) { int err; - struct fuse_conn *fc = fud->fc; + struct fuse_chan *fch = fud->chan; struct fuse_pqueue *fpq = &fud->pq; struct fuse_req *req; struct fuse_out_header oh; @@ -2207,7 +1883,18 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, * and error contains notification code. */ if (!oh.unique) { - err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); + /* + * Only allow notifications during while the connection is in an + * initialized and connected state + */ + err = -EINVAL; + if (!fch->initialized || !fch->connected) + goto copy_finish; + + /* Don't try to move folios (yet) */ + cs->move_folios = false; + + err = fuse_notify(fch->conn, oh.error, nbytes - sizeof(oh), cs); goto copy_finish; } @@ -2235,7 +1922,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, if (nbytes != sizeof(struct fuse_out_header)) err = -EINVAL; else if (oh.error == -ENOSYS) - fc->no_interrupt = 1; + fch->no_interrupt = 1; else if (oh.error == -EAGAIN) err = queue_interrupt(req); @@ -2395,7 +2082,7 @@ static __poll_t fuse_dev_poll(struct file *file, poll_table *wait) if (IS_ERR(fud)) return EPOLLERR; - fiq = &fud->fc->iq; + fiq = &fud->chan->iq; poll_wait(file, &fiq->waitq, wait); spin_lock(&fiq->lock); @@ -2421,21 +2108,6 @@ void fuse_dev_end_requests(struct list_head *head) } } -static void end_polls(struct fuse_conn *fc) -{ - struct rb_node *p; - - p = rb_first(&fc->polled_files); - - while (p) { - struct fuse_file *ff; - ff = rb_entry(p, struct fuse_file, polled_node); - wake_up_interruptible_all(&ff->poll_wait); - - p = rb_next(p); - } -} - /* * Abort all requests. * @@ -2454,27 +2126,29 @@ static void end_polls(struct fuse_conn *fc) * is OK, the request will in that case be removed from the list before we touch * it. */ -void fuse_abort_conn(struct fuse_conn *fc) +void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fch->iq; + + fch->abort_with_err = abort_with_err; - spin_lock(&fc->lock); - if (fc->connected) { + spin_lock(&fch->lock); + if (fch->connected) { struct fuse_dev *fud; struct fuse_req *req, *next; LIST_HEAD(to_end); unsigned int i; - if (fc->timeout.req_timeout) - cancel_delayed_work(&fc->timeout.work); + if (fch->timeout.req_timeout) + cancel_delayed_work(&fch->timeout.work); - /* Background queuing checks fc->connected under bg_lock */ - spin_lock(&fc->bg_lock); - fc->connected = 0; - spin_unlock(&fc->bg_lock); + /* Background queuing checks fch->connected under bg_lock */ + spin_lock(&fch->bg_lock); + fch->connected = 0; + spin_unlock(&fch->bg_lock); - fuse_set_initialized(fc); - list_for_each_entry(fud, &fc->devices, entry) { + fuse_chan_set_initialized(fch, NULL); + list_for_each_entry(fud, &fch->devices, entry) { struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); @@ -2495,11 +2169,11 @@ void fuse_abort_conn(struct fuse_conn *fc) &to_end); spin_unlock(&fpq->lock); } - spin_lock(&fc->bg_lock); - fc->blocked = 0; - fc->max_background = UINT_MAX; - flush_bg_queue(fc); - spin_unlock(&fc->bg_lock); + spin_lock(&fch->bg_lock); + fch->blocked = 0; + fch->max_background = UINT_MAX; + flush_bg_queue(fch); + spin_unlock(&fch->bg_lock); spin_lock(&fiq->lock); fiq->connected = 0; @@ -2511,63 +2185,64 @@ void fuse_abort_conn(struct fuse_conn *fc) wake_up_all(&fiq->waitq); spin_unlock(&fiq->lock); kill_fasync(&fiq->fasync, SIGIO, POLL_IN); - end_polls(fc); - wake_up_all(&fc->blocked_waitq); - spin_unlock(&fc->lock); + fuse_end_polls(fch->conn); + wake_up_all(&fch->blocked_waitq); + spin_unlock(&fch->lock); fuse_dev_end_requests(&to_end); /* - * fc->lock must not be taken to avoid conflicts with io-uring + * fch->lock must not be taken to avoid conflicts with io-uring * locks */ - fuse_uring_abort(fc); + fuse_uring_abort(fch); } else { - spin_unlock(&fc->lock); + spin_unlock(&fch->lock); } } -EXPORT_SYMBOL_GPL(fuse_abort_conn); +EXPORT_SYMBOL_GPL(fuse_chan_abort); -void fuse_wait_aborted(struct fuse_conn *fc) +void fuse_chan_wait_aborted(struct fuse_chan *fch) { /* matches implicit memory barrier in fuse_drop_waiting() */ smp_mb(); - wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0); + wait_event(fch->blocked_waitq, fuse_chan_num_waiting(fch) == 0); - fuse_uring_wait_stopped_queues(fc); + fuse_uring_wait_stopped_queues(fch); } int fuse_dev_release(struct inode *inode, struct file *file) { struct fuse_dev *fud = fuse_file_to_fud(file); /* Pairs with cmpxchg() in fuse_dev_install() */ - struct fuse_conn *fc = xchg(&fud->fc, FUSE_DEV_FC_DISCONNECTED); + struct fuse_chan *fch = xchg(&fud->chan, FUSE_DEV_CHAN_DISCONNECTED); - if (fc) { + if (fch) { struct fuse_pqueue *fpq = &fud->pq; LIST_HEAD(to_end); unsigned int i; bool last; + /* Make sure fuse_dev_install_with_pq() has finished */ + spin_lock(&fch->lock); spin_lock(&fpq->lock); WARN_ON(!list_empty(&fpq->io)); for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) list_splice_init(&fpq->processing[i], &to_end); spin_unlock(&fpq->lock); - fuse_dev_end_requests(&to_end); - - spin_lock(&fc->lock); list_del(&fud->entry); /* Are we the last open device? */ - last = list_empty(&fc->devices); - spin_unlock(&fc->lock); + last = list_empty(&fch->devices); + spin_unlock(&fch->lock); + + fuse_dev_end_requests(&to_end); if (last) { - WARN_ON(fc->iq.fasync != NULL); - fuse_abort_conn(fc); + WARN_ON(fch->iq.fasync != NULL); + fuse_chan_abort(fch, false); } - fuse_conn_put(fc); + fuse_conn_put(fch->conn); } fuse_dev_put(fud); return 0; @@ -2582,13 +2257,14 @@ static int fuse_dev_fasync(int fd, struct file *file, int on) return PTR_ERR(fud); /* No locking - fasync_helper does its own locking */ - return fasync_helper(fd, file, on, &fud->fc->iq.fasync); + return fasync_helper(fd, file, on, &fud->chan->iq.fasync); } static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) { int oldfd; struct fuse_dev *fud, *new_fud; + struct list_head *pq; if (get_user(oldfd, argp)) return -EFAULT; @@ -2608,12 +2284,14 @@ static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) if (IS_ERR(fud)) return PTR_ERR(fud); + pq = fuse_pqueue_alloc(); + if (!pq) + return -ENOMEM; + new_fud = fuse_file_to_fud(file); - if (fuse_dev_fc_get(new_fud)) + if (!fuse_dev_install_with_pq(new_fud, fud->chan, pq)) return -EINVAL; - fuse_dev_install(new_fud, fud->fc); - return 0; } @@ -2632,7 +2310,7 @@ static long fuse_dev_ioctl_backing_open(struct file *file, if (copy_from_user(&map, argp, sizeof(map))) return -EFAULT; - return fuse_backing_open(fud->fc, &map); + return fuse_backing_open(fud->chan->conn, &map); } static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp) @@ -2649,21 +2327,18 @@ static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp) if (get_user(backing_id, argp)) return -EFAULT; - return fuse_backing_close(fud->fc, backing_id); + return fuse_backing_close(fud->chan->conn, backing_id); } static long fuse_dev_ioctl_sync_init(struct file *file) { - int err = -EINVAL; struct fuse_dev *fud = fuse_file_to_fud(file); - mutex_lock(&fuse_mutex); - if (!fuse_dev_fc_get(fud)) { - fud->sync_init = true; - err = 0; - } - mutex_unlock(&fuse_mutex); - return err; + if (fuse_dev_chan_get(fud)) + return -EINVAL; + + fud->sync_init = true; + return 0; } static long fuse_dev_ioctl(struct file *file, unsigned int cmd, @@ -2696,7 +2371,7 @@ static void fuse_dev_show_fdinfo(struct seq_file *seq, struct file *file) if (!fud) return; - seq_printf(seq, "fuse_connection:\t%u\n", fud->fc->dev); + seq_printf(seq, "fuse_connection:\t%u\n", fuse_conn_get_id(fud->chan->conn)); } #endif diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h new file mode 100644 index 000000000000..aed69fd14c41 --- /dev/null +++ b/fs/fuse/dev.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FS_FUSE_DEV_H +#define _FS_FUSE_DEV_H + +#include <linux/cleanup.h> + +/** Maximum number of outstanding background requests */ +#define FUSE_DEFAULT_MAX_BACKGROUND 12 + +struct fuse_conn; +struct fuse_chan; +struct fuse_dev; +struct fuse_args; +struct fuse_copy_state; +struct fuse_backing_map; +struct file; +struct folio; +enum fuse_notify_code; + +struct fuse_chan_param { + unsigned int minor; + unsigned int max_write; + unsigned int max_pages; +}; + +struct fuse_chan *fuse_chan_new(void); +struct fuse_chan *fuse_dev_chan_new(void); +void fuse_chan_release(struct fuse_chan *fch); +void fuse_chan_free(struct fuse_chan *fch); +unsigned int fuse_chan_num_background(struct fuse_chan *fch); +unsigned int fuse_chan_max_background(struct fuse_chan *fch); +void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val); +unsigned int fuse_chan_num_waiting(struct fuse_chan *fch); +void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc); +void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param); +void fuse_chan_io_uring_enable(struct fuse_chan *fch); +ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args); +int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags); +int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique); +void fuse_chan_resend(struct fuse_chan *fch); + +struct fuse_forget_link *fuse_alloc_forget(void); +void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget, + u64 nodeid, u64 nlookup); + +DEFINE_FREE(fuse_chan_free, struct fuse_chan *, if (_T) fuse_chan_free(_T)) + +/** + * Initialize the client device + */ +int fuse_dev_init(void); + +/** + * Cleanup the client device + */ +void fuse_dev_cleanup(void); + +void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch); +bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch); +void fuse_dev_put(struct fuse_dev *fud); +bool fuse_dev_is_installed(struct fuse_dev *fud); +bool fuse_dev_is_sync_init(struct fuse_dev *fud); +struct fuse_dev *fuse_dev_grab(struct file *file); + +void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout); + +/* Abort all requests */ +void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err); +void fuse_chan_wait_aborted(struct fuse_chan *fch); + +/** + * Acquire reference to fuse_conn + */ +struct fuse_conn *fuse_conn_get(struct fuse_conn *fc); + +/** + * Release reference to fuse_conn + */ +void fuse_conn_put(struct fuse_conn *fc); + +dev_t fuse_conn_get_id(struct fuse_conn *fc); + +void fuse_end_polls(struct fuse_conn *fc); +int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, + unsigned int size, struct fuse_copy_state *cs); + +int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map); +int fuse_backing_close(struct fuse_conn *fc, int backing_id); + +int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size); +int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, + unsigned offset, unsigned count, int zeroing); +void fuse_copy_finish(struct fuse_copy_state *cs); + +#ifdef CONFIG_FUSE_IO_URING +bool fuse_uring_enabled(void); +void fuse_uring_destruct(struct fuse_chan *fch); +#else /* CONFIG_FUSE_IO_URING */ +static inline bool fuse_uring_enabled(void) +{ + return false; +} + +static inline void fuse_uring_destruct(struct fuse_chan *fch) +{ +} +#endif /* CONFIG_FUSE_IO_URING */ + +#endif /* _FS_FUSE_DEV_H */ diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 7b9822e8837b..77c8cec43d9c 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -4,9 +4,9 @@ * Copyright (c) 2023-2024 DataDirect Networks. */ -#include "fuse_i.h" +#include "dev.h" +#include "args.h" #include "dev_uring_i.h" -#include "fuse_dev_i.h" #include "fuse_trace.h" #include <linux/fs.h> @@ -18,7 +18,8 @@ MODULE_PARM_DESC(enable_uring, "Enable userspace communication through io-uring"); #define FUSE_URING_IOV_SEGS 2 /* header and payload */ - +#define FUSE_URING_IOV_HEADERS 0 +#define FUSE_URING_IOV_PAYLOAD 1 bool fuse_uring_enabled(void) { @@ -31,6 +32,15 @@ struct fuse_uring_pdu { static const struct fuse_iqueue_ops fuse_io_uring_ops; +enum fuse_uring_header_type { + /* struct fuse_in_header / struct fuse_out_header */ + FUSE_URING_HEADER_IN_OUT, + /* per op code header */ + FUSE_URING_HEADER_OP, + /* struct fuse_uring_ent_in_out header */ + FUSE_URING_HEADER_RING_ENT, +}; + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -51,10 +61,10 @@ static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd) static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) { struct fuse_ring *ring = queue->ring; - struct fuse_conn *fc = ring->fc; + struct fuse_chan *fch = ring->chan; lockdep_assert_held(&queue->lock); - lockdep_assert_held(&fc->bg_lock); + lockdep_assert_held(&fch->bg_lock); /* * Allow one bg request per queue, ignoring global fc limits. @@ -62,14 +72,14 @@ static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) * eliminates the need for remote queue wake-ups when global * limits are met but this queue has no more waiting requests. */ - while ((fc->active_background < fc->max_background || + while ((fch->active_background < fch->max_background || !queue->active_background) && (!list_empty(&queue->fuse_req_bg_queue))) { struct fuse_req *req; req = list_first_entry(&queue->fuse_req_bg_queue, struct fuse_req, list); - fc->active_background++; + fch->active_background++; queue->active_background++; list_move_tail(&req->list, &queue->fuse_req_queue); @@ -81,7 +91,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; - struct fuse_conn *fc = ring->fc; + struct fuse_chan *fch = ring->chan; lockdep_assert_not_held(&queue->lock); spin_lock(&queue->lock); @@ -89,9 +99,10 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, list_del_init(&req->list); if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; - spin_lock(&fc->bg_lock); + spin_lock(&fch->bg_lock); + fuse_request_bg_finish(fch, req); fuse_uring_flush_bg(queue); - spin_unlock(&fc->bg_lock); + spin_unlock(&fch->bg_lock); } spin_unlock(&queue->lock); @@ -123,26 +134,25 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring) { int qid; struct fuse_ring_queue *queue; - struct fuse_conn *fc = ring->fc; + struct fuse_chan *fch = ring->chan; for (qid = 0; qid < ring->nr_queues; qid++) { queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; - queue->stopped = true; - - WARN_ON_ONCE(ring->fc->max_background != UINT_MAX); + WARN_ON_ONCE(fch->max_background != UINT_MAX); spin_lock(&queue->lock); - spin_lock(&fc->bg_lock); + queue->stopped = true; + spin_lock(&fch->bg_lock); fuse_uring_flush_bg(queue); - spin_unlock(&fc->bg_lock); + spin_unlock(&fch->bg_lock); spin_unlock(&queue->lock); fuse_uring_abort_end_queue_requests(queue); } } -static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list) +static bool ent_list_request_expired(struct fuse_chan *fch, struct list_head *list) { struct fuse_ring_ent *ent; struct fuse_req *req; @@ -154,12 +164,12 @@ static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *lis req = ent->fuse_req; return time_is_before_jiffies(req->create_time + - fc->timeout.req_timeout); + fch->timeout.req_timeout); } -bool fuse_uring_request_expired(struct fuse_conn *fc) +bool fuse_uring_request_expired(struct fuse_chan *fch) { - struct fuse_ring *ring = fc->ring; + struct fuse_ring *ring = fch->ring; struct fuse_ring_queue *queue; int qid; @@ -172,10 +182,10 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) continue; spin_lock(&queue->lock); - if (fuse_request_expired(fc, &queue->fuse_req_queue) || - fuse_request_expired(fc, &queue->fuse_req_bg_queue) || - ent_list_request_expired(fc, &queue->ent_w_req_queue) || - ent_list_request_expired(fc, &queue->ent_in_userspace)) { + if (fuse_request_expired(fch, &queue->fuse_req_queue) || + fuse_request_expired(fch, &queue->fuse_req_bg_queue) || + ent_list_request_expired(fch, &queue->ent_w_req_queue) || + ent_list_request_expired(fch, &queue->ent_in_userspace)) { spin_unlock(&queue->lock); return true; } @@ -185,9 +195,9 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) return false; } -void fuse_uring_destruct(struct fuse_conn *fc) +void fuse_uring_destruct(struct fuse_chan *fch) { - struct fuse_ring *ring = fc->ring; + struct fuse_ring *ring = fch->ring; int qid; if (!ring) @@ -218,20 +228,20 @@ void fuse_uring_destruct(struct fuse_conn *fc) kfree(ring->queues); kfree(ring); - fc->ring = NULL; + fch->ring = NULL; } /* * Basic ring setup for this connection based on the provided configuration */ -static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) +static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) { struct fuse_ring *ring; size_t nr_queues = num_possible_cpus(); struct fuse_ring *res = NULL; size_t max_payload_size; - ring = kzalloc_obj(*fc->ring, GFP_KERNEL_ACCOUNT); + ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT); if (!ring) return NULL; @@ -240,25 +250,29 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) if (!ring->queues) goto out_err; - max_payload_size = max(FUSE_MIN_READ_BUFFER, fc->max_write); - max_payload_size = max(max_payload_size, fc->max_pages * PAGE_SIZE); + max_payload_size = max(FUSE_MIN_READ_BUFFER, fch->max_write); + max_payload_size = max(max_payload_size, fch->max_pages * PAGE_SIZE); - spin_lock(&fc->lock); - if (fc->ring) { + spin_lock(&fch->lock); + if (!fch->connected) { + spin_unlock(&fch->lock); + goto out_err; + } + if (fch->ring) { /* race, another thread created the ring in the meantime */ - spin_unlock(&fc->lock); - res = fc->ring; + spin_unlock(&fch->lock); + res = fch->ring; goto out_err; } init_waitqueue_head(&ring->stop_waitq); ring->nr_queues = nr_queues; - ring->fc = fc; + ring->chan = fch; ring->max_payload_sz = max_payload_size; - smp_store_release(&fc->ring, ring); + smp_store_release(&fch->ring, ring); - spin_unlock(&fc->lock); + spin_unlock(&fch->lock); return ring; out_err: @@ -270,14 +284,14 @@ out_err: static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, int qid) { - struct fuse_conn *fc = ring->fc; + struct fuse_chan *fch = ring->chan; struct fuse_ring_queue *queue; struct list_head *pq; queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT); if (!queue) return NULL; - pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE); + pq = fuse_pqueue_alloc(); if (!pq) { kfree(queue); return NULL; @@ -295,12 +309,12 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->fuse_req_bg_queue); INIT_LIST_HEAD(&queue->ent_released); - queue->fpq.processing = pq; fuse_pqueue_init(&queue->fpq); + queue->fpq.processing = pq; - spin_lock(&fc->lock); + spin_lock(&fch->lock); if (ring->queues[qid]) { - spin_unlock(&fc->lock); + spin_unlock(&fch->lock); kfree(queue->fpq.processing); kfree(queue); return ring->queues[qid]; @@ -310,7 +324,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, * write_once and lock as the caller mostly doesn't take the lock at all */ WRITE_ONCE(ring->queues[qid], queue); - spin_unlock(&fc->lock); + spin_unlock(&fch->lock); return queue; } @@ -466,6 +480,7 @@ static void fuse_uring_async_stop_queues(struct work_struct *work) FUSE_URING_TEARDOWN_INTERVAL); } else { wake_up_all(&ring->stop_waitq); + fuse_conn_put(ring->chan->conn); } } @@ -477,6 +492,7 @@ void fuse_uring_stop_queues(struct fuse_ring *ring) fuse_uring_teardown_all_queues(ring); if (atomic_read(&ring->queue_refs) > 0) { + fuse_conn_get(ring->chan->conn); ring->teardown_time = jiffies; INIT_DELAYED_WORK(&ring->async_teardown_work, fuse_uring_async_stop_queues); @@ -507,8 +523,7 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd, queue = ent->queue; spin_lock(&queue->lock); if (ent->state == FRRS_AVAILABLE) { - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); + list_del_init(&ent->list); need_cmd_done = true; ent->cmd = NULL; } @@ -517,6 +532,9 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd, if (need_cmd_done) { /* no queue lock to avoid lock order issues */ io_uring_cmd_done(cmd, -ENOTCONN, issue_flags); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); } } @@ -531,8 +549,7 @@ static void fuse_uring_prepare_cancel(struct io_uring_cmd *cmd, int issue_flags, * Checks for errors and stores it into the request */ static int fuse_uring_out_header_has_err(struct fuse_out_header *oh, - struct fuse_req *req, - struct fuse_conn *fc) + struct fuse_req *req) { int err; @@ -571,6 +588,82 @@ err: return err; } +static int ring_header_type_offset(enum fuse_uring_header_type type) +{ + switch (type) { + case FUSE_URING_HEADER_IN_OUT: + return 0; + case FUSE_URING_HEADER_OP: + return offsetof(struct fuse_uring_req_header, op_in); + case FUSE_URING_HEADER_RING_ENT: + return offsetof(struct fuse_uring_req_header, ring_ent_in_out); + default: + WARN_ONCE(1, "Invalid header type: %d\n", type); + return -EINVAL; + } +} + +static int copy_header_to_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, + const void *header, size_t header_size) +{ + int offset = ring_header_type_offset(type); + void __user *ring; + + if (offset < 0) + return offset; + + ring = (void __user *)ent->headers + offset; + + if (copy_to_user(ring, header, header_size)) { + pr_info_ratelimited("Copying header to ring failed.\n"); + return -EFAULT; + } + + return 0; +} + +static int copy_header_from_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, void *header, + size_t header_size) +{ + int offset = ring_header_type_offset(type); + const void __user *ring; + + if (offset < 0) + return offset; + + ring = (void __user *)ent->headers + offset; + + if (copy_from_user(header, ring, header_size)) { + pr_info_ratelimited("Copying header from ring failed.\n"); + return -EFAULT; + } + + return 0; +} + +static int setup_fuse_copy_state(struct fuse_copy_state *cs, + struct fuse_ring *ring, struct fuse_req *req, + struct fuse_ring_ent *ent, int dir, + struct iov_iter *iter) +{ + int err; + + err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter); + if (err) { + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + return err; + } + + fuse_copy_init(cs, dir == ITER_DEST, iter); + + cs->is_uring = true; + cs->req = req; + + return 0; +} + static int fuse_uring_copy_from_ring(struct fuse_ring *ring, struct fuse_req *req, struct fuse_ring_ent *ent) @@ -581,20 +674,15 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, int err; struct fuse_uring_ent_in_out ring_in_out; - err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out, - sizeof(ring_in_out)); + err = copy_header_from_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ring_in_out, sizeof(ring_in_out)); if (err) - return -EFAULT; + return err; - err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz, - &iter); + err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter); if (err) return err; - fuse_copy_init(&cs, false, &iter); - cs.is_uring = true; - cs.req = req; - err = fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); fuse_copy_finish(&cs); return err; @@ -617,15 +705,9 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter); - if (err) { - pr_info_ratelimited("fuse: Import of user buffer failed\n"); + err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter); + if (err) return err; - } - - fuse_copy_init(&cs, true, &iter); - cs.is_uring = true; - cs.req = req; if (num_args > 0) { /* @@ -633,13 +715,11 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, * Some op code have that as zero size. */ if (args->in_args[0].size > 0) { - err = copy_to_user(&ent->headers->op_in, in_args->value, - in_args->size); - if (err) { - pr_info_ratelimited( - "Copying the header failed.\n"); - return -EFAULT; - } + err = copy_header_to_ring(ent, FUSE_URING_HEADER_OP, + in_args->value, + in_args->size); + if (err) + return err; } in_args++; num_args--; @@ -655,9 +735,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; - err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out, - sizeof(ent_in_out)); - return err ? -EFAULT : 0; + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ent_in_out, sizeof(ent_in_out)); } static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, @@ -686,14 +765,8 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, } /* copy fuse_in_header */ - err = copy_to_user(&ent->headers->in_out, &req->in.h, - sizeof(req->in.h)); - if (err) { - err = -EFAULT; - return err; - } - - return 0; + return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h, + sizeof(req->in.h)); } static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, @@ -702,40 +775,36 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, int err; err = fuse_uring_copy_to_ring(ent, req); - if (!err) + if (!err) { set_bit(FR_SENT, &req->flags); - else + trace_fuse_request_sent(req); + } else { + /* + * Copying the request failed. Remove the entry from the + * ent_w_req_queue list and terminate the request + */ + spin_lock(&ent->queue->lock); + list_del_init(&ent->list); + ent->state = FRRS_INVALID; + spin_unlock(&ent->queue->lock); + fuse_uring_req_end(ent, req, err); + } return err; } -/* - * Write data to the ring buffer and send the request to userspace, - * userspace will read it - * This is comparable with classical read(/dev/fuse) - */ -static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent, - struct fuse_req *req, - unsigned int issue_flags) +/* Used to find the request on SQE commit */ +static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent) { struct fuse_ring_queue *queue = ent->queue; - int err; - struct io_uring_cmd *cmd; - - err = fuse_uring_prepare_send(ent, req); - if (err) - return err; - - spin_lock(&queue->lock); - cmd = ent->cmd; - ent->cmd = NULL; - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - spin_unlock(&queue->lock); + struct fuse_pqueue *fpq = &queue->fpq; + unsigned int hash; + struct fuse_req *req = ent->fuse_req; - io_uring_cmd_done(cmd, 0, issue_flags); - return 0; + req->ring_entry = ent; + hash = fuse_req_hash(req->in.h.unique); + list_move_tail(&req->list, &fpq->processing[hash]); } /* @@ -749,19 +818,6 @@ static void fuse_uring_ent_avail(struct fuse_ring_ent *ent, ent->state = FRRS_AVAILABLE; } -/* Used to find the request on SQE commit */ -static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent, - struct fuse_req *req) -{ - struct fuse_ring_queue *queue = ent->queue; - struct fuse_pqueue *fpq = &queue->fpq; - unsigned int hash; - - req->ring_entry = ent; - hash = fuse_req_hash(req->in.h.unique); - list_move_tail(&req->list, &fpq->processing[hash]); -} - /* * Assign a fuse queue entry to the given entry */ @@ -779,10 +835,13 @@ static void fuse_uring_add_req_to_ring_ent(struct fuse_ring_ent *ent, } clear_bit(FR_PENDING, &req->flags); + + /* Until fuse_uring_add_to_pq() the req is not attached to any list */ + list_del_init(&req->list); + ent->fuse_req = req; ent->state = FRRS_FUSE_REQ; list_move_tail(&ent->list, &queue->ent_w_req_queue); - fuse_uring_add_to_pq(ent, req); } /* Fetch the next fuse request if available */ @@ -812,17 +871,13 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { struct fuse_ring *ring = ent->queue->ring; - struct fuse_conn *fc = ring->fc; - ssize_t err = 0; + ssize_t err = -EFAULT; - err = copy_from_user(&req->out.h, &ent->headers->in_out, - sizeof(req->out.h)); - if (err) { - req->out.h.error = -EFAULT; + if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, + sizeof(req->out.h))) goto out; - } - err = fuse_uring_out_header_has_err(&req->out.h, req, fc); + err = fuse_uring_out_header_has_err(&req->out.h, req); if (err) { /* req->out.h.error already set */ goto out; @@ -834,11 +889,13 @@ out: } /* - * Get the next fuse req and send it + * Get the next fuse req. + * + * Returns true if the next fuse request has been assigned to the ent. + * Else, there is no next fuse request and this returns false. */ -static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, - struct fuse_ring_queue *queue, - unsigned int issue_flags) +static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent, + struct fuse_ring_queue *queue) { int err; struct fuse_req *req; @@ -850,10 +907,12 @@ retry: spin_unlock(&queue->lock); if (req) { - err = fuse_uring_send_next_to_ring(ent, req, issue_flags); + err = fuse_uring_prepare_send(ent, req); if (err) goto retry; } + + return req != NULL; } static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) @@ -871,15 +930,30 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } +static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, + ssize_t ret, unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + ent->state = FRRS_USERSPACE; + list_move_tail(&ent->list, &queue->ent_in_userspace); + ent->cmd = NULL; + fuse_uring_add_to_pq(ent); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, ret, issue_flags); +} + /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, - struct fuse_conn *fc) + struct fuse_chan *fch) { const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); struct fuse_ring_ent *ent; int err; - struct fuse_ring *ring = fc->ring; + struct fuse_ring *ring = fch->ring; struct fuse_ring_queue *queue; uint64_t commit_id = READ_ONCE(cmd_req->commit_id); unsigned int qid = READ_ONCE(cmd_req->qid); @@ -898,10 +972,15 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, return err; fpq = &queue->fpq; - if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped)) + if (!READ_ONCE(fch->connected)) return err; spin_lock(&queue->lock); + if (unlikely(queue->stopped)) { + spin_unlock(&queue->lock); + return err; + } + /* Find a request based on the unique ID of the fuse request * This should get revised, as it needs a hash calculation and list * search. And full struct fuse_pqueue is needed (memory overhead). @@ -924,9 +1003,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); spin_unlock(&queue->lock); - req->out.h.error = err; - clear_bit(FR_SENT, &req->flags); - fuse_request_end(req); + fuse_uring_req_end(ent, req, err); return err; } @@ -943,7 +1020,8 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * and fetching is done in one step vs legacy fuse, which has separated * read (fetch request) and write (commit result). */ - fuse_uring_next_fuse_req(ent, queue, issue_flags); + if (fuse_uring_get_next_fuse_req(ent, queue)) + fuse_uring_send(ent, cmd, 0, issue_flags); return 0; } @@ -975,14 +1053,25 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) /* * fuse_uring_req_fetch command handling */ -static void fuse_uring_do_register(struct fuse_ring_ent *ent, - struct io_uring_cmd *cmd, - unsigned int issue_flags) +static int fuse_uring_do_register(struct fuse_ring_ent *ent, + struct io_uring_cmd *cmd, + unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; - struct fuse_conn *fc = ring->fc; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_chan *fch = ring->chan; + struct fuse_iqueue *fiq = &fch->iq; + + spin_lock(&fch->lock); + /* abort teardown path is running or has run */ + if (!fch->connected) { + spin_unlock(&fch->lock); + if (atomic_dec_and_test(&ring->queue_refs)) + wake_up_all(&ring->stop_waitq); + kfree(ent); + return -ECONNABORTED; + } + spin_unlock(&fch->lock); fuse_uring_prepare_cancel(cmd, issue_flags, ent); @@ -991,20 +1080,21 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent, fuse_uring_ent_avail(ent, queue); spin_unlock(&queue->lock); - if (!ring->ready) { + if (!READ_ONCE(ring->ready)) { bool ready = is_ring_ready(ring, queue->qid); if (ready) { WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); - WRITE_ONCE(ring->ready, true); - wake_up_all(&fc->blocked_waitq); + smp_store_release(&ring->ready, true); + wake_up_all(&fch->blocked_waitq); } } + return 0; } /* - * sqe->addr is a ptr to an iovec array, iov[0] has the headers, iov[1] - * the payload + * sqe->addr is a ptr to an iovec array, iov[FUSE_URING_IOV_HEADERS] has the + * headers, iov[FUSE_URING_IOV_PAYLOAD] the payload */ static int fuse_uring_get_iovec_from_sqe(const struct io_uring_sqe *sqe, struct iovec iov[FUSE_URING_IOV_SEGS]) @@ -1034,8 +1124,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, { struct fuse_ring *ring = queue->ring; struct fuse_ring_ent *ent; - size_t payload_size; struct iovec iov[FUSE_URING_IOV_SEGS]; + struct iovec *headers, *payload; int err; err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); @@ -1046,15 +1136,16 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } err = -EINVAL; - if (iov[0].iov_len < sizeof(struct fuse_uring_req_header)) { - pr_info_ratelimited("Invalid header len %zu\n", iov[0].iov_len); + headers = &iov[FUSE_URING_IOV_HEADERS]; + if (headers->iov_len < sizeof(struct fuse_uring_req_header)) { + pr_info_ratelimited("Invalid header len %zu\n", headers->iov_len); return ERR_PTR(err); } - payload_size = iov[1].iov_len; - if (payload_size < ring->max_payload_sz) { + payload = &iov[FUSE_URING_IOV_PAYLOAD]; + if (payload->iov_len < ring->max_payload_sz) { pr_info_ratelimited("Invalid req payload len %zu\n", - payload_size); + payload->iov_len); return ERR_PTR(err); } @@ -1066,8 +1157,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, INIT_LIST_HEAD(&ent->list); ent->queue = queue; - ent->headers = iov[0].iov_base; - ent->payload = iov[1].iov_base; + ent->headers = headers->iov_base; + ent->payload = payload->iov_base; atomic_inc(&ring->queue_refs); return ent; @@ -1078,11 +1169,11 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, * entry as "ready to get fuse requests" on the queue */ static int fuse_uring_register(struct io_uring_cmd *cmd, - unsigned int issue_flags, struct fuse_conn *fc) + unsigned int issue_flags, struct fuse_chan *fch) { const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); - struct fuse_ring *ring = smp_load_acquire(&fc->ring); + struct fuse_ring *ring = smp_load_acquire(&fch->ring); struct fuse_ring_queue *queue; struct fuse_ring_ent *ent; int err; @@ -1090,7 +1181,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, err = -ENOMEM; if (!ring) { - ring = fuse_uring_create(fc); + ring = fuse_uring_create(fch); if (!ring) return err; } @@ -1116,9 +1207,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, if (IS_ERR(ent)) return PTR_ERR(ent); - fuse_uring_do_register(ent, cmd, issue_flags); - - return 0; + return fuse_uring_do_register(ent, cmd, issue_flags); } /* @@ -1128,7 +1217,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { struct fuse_dev *fud; - struct fuse_conn *fc; + struct fuse_chan *fch; u32 cmd_op = cmd->cmd_op; int err; @@ -1146,39 +1235,39 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) pr_info_ratelimited("No fuse device found\n"); return PTR_ERR(fud); } - fc = fud->fc; + fch = fud->chan; /* Once a connection has io-uring enabled on it, it can't be disabled */ - if (!enable_uring && !fc->io_uring) { + if (!enable_uring && !fch->io_uring) { pr_info_ratelimited("fuse-io-uring is disabled\n"); return -EOPNOTSUPP; } - if (fc->aborted) + if (fch->abort_with_err) return -ECONNABORTED; - if (!fc->connected) + if (!fch->connected) return -ENOTCONN; /* * fuse_uring_register() needs the ring to be initialized, * we need to know the max payload size */ - if (!fc->initialized) + if (!fch->initialized) return -EAGAIN; switch (cmd_op) { case FUSE_IO_URING_CMD_REGISTER: - err = fuse_uring_register(cmd, issue_flags, fc); + err = fuse_uring_register(cmd, issue_flags, fch); if (err) { pr_info_once("FUSE_IO_URING_CMD_REGISTER failed err=%d\n", err); - fc->io_uring = 0; - wake_up_all(&fc->blocked_waitq); + fch->io_uring = 0; + wake_up_all(&fch->blocked_waitq); return err; } break; case FUSE_IO_URING_CMD_COMMIT_AND_FETCH: - err = fuse_uring_commit_fetch(cmd, issue_flags, fc); + err = fuse_uring_commit_fetch(cmd, issue_flags, fch); if (err) { pr_info_once("FUSE_IO_URING_COMMIT_AND_FETCH failed err=%d\n", err); @@ -1192,20 +1281,6 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) return -EIOCBQUEUED; } -static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, - ssize_t ret, unsigned int issue_flags) -{ - struct fuse_ring_queue *queue = ent->queue; - - spin_lock(&queue->lock); - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - ent->cmd = NULL; - spin_unlock(&queue->lock); - - io_uring_cmd_done(cmd, ret, issue_flags); -} - /* * This prepares and sends the ring request in fuse-uring task context. * User buffers are not mapped yet - the application does not have permission @@ -1222,14 +1297,25 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) if (!tw.cancel) { err = fuse_uring_prepare_send(ent, ent->fuse_req); if (err) { - fuse_uring_next_fuse_req(ent, queue, issue_flags); - return; + if (!fuse_uring_get_next_fuse_req(ent, queue)) + return; + err = 0; } + fuse_uring_send(ent, cmd, err, issue_flags); } else { err = -ECANCELED; - } - fuse_uring_send(ent, cmd, err, issue_flags); + spin_lock(&queue->lock); + list_del_init(&ent->list); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, err, issue_flags); + + fuse_uring_req_end(ent, ent->fuse_req, err); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); + } } static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) @@ -1261,8 +1347,7 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent) /* queue a fuse request and send it if a ring entry is available */ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) { - struct fuse_conn *fc = req->fm->fc; - struct fuse_ring *ring = fc->ring; + struct fuse_ring *ring = req->chan->ring; struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; int err; @@ -1304,8 +1389,8 @@ err: bool fuse_uring_queue_bq_req(struct fuse_req *req) { - struct fuse_conn *fc = req->fm->fc; - struct fuse_ring *ring = fc->ring; + struct fuse_chan *fch = req->chan; + struct fuse_ring *ring = fch->ring; struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; @@ -1325,12 +1410,12 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) ent = list_first_entry_or_null(&queue->ent_avail_queue, struct fuse_ring_ent, list); - spin_lock(&fc->bg_lock); - fc->num_background++; - if (fc->num_background == fc->max_background) - fc->blocked = 1; + spin_lock(&fch->bg_lock); + fch->num_background++; + if (fch->num_background == fch->max_background) + fch->blocked = 1; fuse_uring_flush_bg(queue); - spin_unlock(&fc->bg_lock); + spin_unlock(&fch->bg_lock); /* * Due to bg_queue flush limits there might be other bg requests diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 51a563922ce1..55f8d04e4b0b 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -7,7 +7,7 @@ #ifndef _FS_FUSE_DEV_URING_I_H #define _FS_FUSE_DEV_URING_I_H -#include "fuse_i.h" +#include "fuse_dev_i.h" #ifdef CONFIG_FUSE_IO_URING @@ -101,13 +101,13 @@ struct fuse_ring_queue { bool stopped; }; -/** +/* * Describes if uring is for communication and holds alls the data needed * for uring communication */ struct fuse_ring { /* back pointer */ - struct fuse_conn *fc; + struct fuse_chan *chan; /* number of ring queues */ size_t nr_queues; @@ -135,63 +135,54 @@ struct fuse_ring { bool ready; }; -bool fuse_uring_enabled(void); -void fuse_uring_destruct(struct fuse_conn *fc); void fuse_uring_stop_queues(struct fuse_ring *ring); void fuse_uring_abort_end_requests(struct fuse_ring *ring); int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req); bool fuse_uring_queue_bq_req(struct fuse_req *req); bool fuse_uring_remove_pending_req(struct fuse_req *req); -bool fuse_uring_request_expired(struct fuse_conn *fc); +bool fuse_uring_request_expired(struct fuse_chan *fch); -static inline void fuse_uring_abort(struct fuse_conn *fc) +static inline void fuse_uring_abort(struct fuse_chan *fch) { - struct fuse_ring *ring = fc->ring; + struct fuse_ring *ring = fch->ring; if (ring == NULL) return; - if (atomic_read(&ring->queue_refs) > 0) { - fuse_uring_abort_end_requests(ring); + fuse_uring_abort_end_requests(ring); + + if (atomic_read(&ring->queue_refs) > 0) fuse_uring_stop_queues(ring); - } } -static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc) +static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch) { - struct fuse_ring *ring = fc->ring; + struct fuse_ring *ring = fch->ring; if (ring) wait_event(ring->stop_waitq, atomic_read(&ring->queue_refs) == 0); } -static inline bool fuse_uring_ready(struct fuse_conn *fc) +static inline bool fuse_uring_ready(struct fuse_chan *fch) { - return fc->ring && fc->ring->ready; -} + struct fuse_ring *ring = READ_ONCE(fch->ring); -#else /* CONFIG_FUSE_IO_URING */ - -static inline void fuse_uring_destruct(struct fuse_conn *fc) -{ + return ring && smp_load_acquire(&ring->ready); } -static inline bool fuse_uring_enabled(void) -{ - return false; -} +#else /* CONFIG_FUSE_IO_URING */ -static inline void fuse_uring_abort(struct fuse_conn *fc) +static inline void fuse_uring_abort(struct fuse_chan *fch) { } -static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc) +static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch) { } -static inline bool fuse_uring_ready(struct fuse_conn *fc) +static inline bool fuse_uring_ready(struct fuse_chan *fch) { return false; } @@ -201,7 +192,7 @@ static inline bool fuse_uring_remove_pending_req(struct fuse_req *req) return false; } -static inline bool fuse_uring_request_expired(struct fuse_conn *fc) +static inline bool fuse_uring_request_expired(struct fuse_chan *fch) { return false; } diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index b658b6baf72f..0e2a1039fa43 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ +#include "dev.h" #include "fuse_i.h" #include <linux/pagemap.h> @@ -177,8 +176,8 @@ static void fuse_dentry_tree_work(struct work_struct *work) spin_lock(&fd->dentry->d_lock); /* If dentry is still referenced, let next dput release it */ fd->dentry->d_flags |= DCACHE_OP_DELETE; + __move_to_shrink_list(fd->dentry, &dispose); spin_unlock(&fd->dentry->d_lock); - d_dispose_if_unused(fd->dentry, &dispose); if (need_resched()) { spin_unlock(&dentry_hash[i].lock); cond_resched(); @@ -317,7 +316,7 @@ void fuse_invalidate_attr(struct inode *inode) static void fuse_dir_changed(struct inode *dir) { - fuse_invalidate_attr(dir); + fuse_invalidate_attr_mask(dir, FUSE_STATX_MODDIR); inode_maybe_inc_iversion(dir, false); } @@ -430,7 +429,7 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, fi = get_fuse_inode(inode); if (outarg.nodeid != get_node_id(inode) || (bool) IS_AUTOMOUNT(inode) != (bool) (outarg.attr.flags & FUSE_ATTR_SUBMOUNT)) { - fuse_queue_forget(fm->fc, forget, + fuse_chan_queue_forget(fm->fc->chan, forget, outarg.nodeid, 1); goto invalid; } @@ -593,7 +592,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name attr_version, evict_ctr); err = -ENOMEM; if (!*inode) { - fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1); + fuse_chan_queue_forget(fm->fc->chan, forget, outarg->nodeid, 1); goto out; } err = 0; @@ -837,7 +836,6 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, if (!forget) goto out_err; - err = -ENOMEM; ff = fuse_file_alloc(fm, true); if (!ff) goto out_put_forget_req; @@ -894,7 +892,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, if (!inode) { flags &= ~(O_CREAT | O_EXCL | O_TRUNC); fuse_sync_release(NULL, ff, flags); - fuse_queue_forget(fm->fc, forget, outentry.nodeid, 1); + fuse_chan_queue_forget(fm->fc->chan, forget, outentry.nodeid, 1); err = -ENOMEM; goto out_err; } @@ -1019,7 +1017,7 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0); if (!inode) { - fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1); + fuse_chan_queue_forget(fm->fc->chan, forget, outarg.nodeid, 1); return ERR_PTR(-ENOMEM); } kfree(forget); @@ -1587,40 +1585,34 @@ int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, { int err = -ENOTDIR; struct inode *parent; - struct dentry *dir = NULL; - struct dentry *entry = NULL; + struct dentry *dir; + struct dentry *entry; parent = fuse_ilookup(fc, parent_nodeid, NULL); if (!parent) return -ENOENT; + inode_lock_nested(parent, I_MUTEX_PARENT); if (!S_ISDIR(parent->i_mode)) - goto put_parent; + goto unlock; err = -ENOENT; dir = d_find_alias(parent); if (!dir) - goto put_parent; - while (!entry) { - struct dentry *child = try_lookup_noperm(name, dir); - if (!child || IS_ERR(child)) - goto put_parent; - entry = start_removing_dentry(dir, child); - dput(child); - if (IS_ERR(entry)) - goto put_parent; - if (!d_same_name(entry, dir, name)) { - end_removing(entry); - entry = NULL; - } - } + goto unlock; + + name->hash = full_name_hash(dir, name->name, name->len); + entry = d_lookup(dir, name); + dput(dir); + if (!entry) + goto unlock; fuse_dir_changed(parent); if (!(flags & FUSE_EXPIRE_ONLY)) d_invalidate(entry); fuse_invalidate_entry_cache(entry); - if (child_nodeid != 0) { + if (child_nodeid != 0 && d_really_is_positive(entry)) { inode_lock(d_inode(entry)); if (get_node_id(d_inode(entry)) != child_nodeid) { err = -ENOENT; @@ -1648,10 +1640,10 @@ int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, } else { err = 0; } + dput(entry); - end_removing(entry); - put_parent: - dput(dir); + unlock: + inode_unlock(parent); iput(parent); return err; } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index c59452d60b8d..ceada75310b8 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1,12 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ #include "fuse_i.h" +#include "dev.h" #include <linux/pagemap.h> #include <linux/slab.h> @@ -91,8 +90,7 @@ static struct fuse_file *fuse_file_get(struct fuse_file *ff) return ff; } -static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args, - int error) +static void fuse_release_end(struct fuse_args *args, int error) { struct fuse_release_args *ra = container_of(args, typeof(*ra), args); @@ -112,10 +110,10 @@ static void fuse_file_put(struct fuse_file *ff, bool sync) if (!args) { /* Do nothing when server does not implement 'opendir' */ } else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) { - fuse_release_end(ff->fm, args, 0); + fuse_release_end(args, 0); } else if (sync) { fuse_simple_request(ff->fm, args); - fuse_release_end(ff->fm, args, 0); + fuse_release_end(args, 0); } else { /* * DAX inodes may need to issue a number of synchronous @@ -126,7 +124,7 @@ static void fuse_file_put(struct fuse_file *ff, bool sync) args->end = fuse_release_end; if (fuse_simple_background(ff->fm, args, GFP_KERNEL | __GFP_NOFAIL)) - fuse_release_end(ff->fm, args, -ENOTCONN); + fuse_release_end(args, -ENOTCONN); } kfree(ff); } @@ -380,8 +378,14 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * aio and closes the fd before the aio completes. Since aio takes its * own ref to the file, the IO completion has to drop the ref, which is * how the fuse server can end up closing its clients' files. + * + * Exception is virtio-fs, which is not affected by the above (server is + * on host, cannot close open files in guest). Virtio-fs needs sync + * release, because the num_waiting mechanism to wait for all requests + * before commencing with fs shutdown doesn't work if submounts are + * used. */ - fuse_file_put(ff, false); + fuse_file_put(ff, ff->fm->fc->auto_submounts); } void fuse_release_common(struct file *file, bool isdir) @@ -635,6 +639,19 @@ static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io) return io->bytes < 0 ? io->size : io->bytes; } +static void fuse_aio_invalidate_worker(struct work_struct *work) +{ + struct fuse_io_priv *io = container_of(work, struct fuse_io_priv, work); + struct address_space *mapping = io->iocb->ki_filp->f_mapping; + ssize_t res = fuse_get_res_by_io(io); + pgoff_t start = io->offset >> PAGE_SHIFT; + pgoff_t end = (io->offset + res - 1) >> PAGE_SHIFT; + + invalidate_inode_pages2_range(mapping, start, end); + io->iocb->ki_complete(io->iocb, res); + kref_put(&io->refcnt, fuse_io_release); +} + /* * In case of short read, the caller sets 'pos' to the position of * actual end of fuse request in IO request. Otherwise, if bytes_requested @@ -667,10 +684,11 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) spin_unlock(&io->lock); if (!left && !io->blocking) { + struct inode *inode = file_inode(io->iocb->ki_filp); + struct address_space *mapping = io->iocb->ki_filp->f_mapping; ssize_t res = fuse_get_res_by_io(io); if (res >= 0) { - struct inode *inode = file_inode(io->iocb->ki_filp); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); @@ -679,6 +697,17 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) spin_unlock(&fi->lock); } + if (io->write && res > 0 && mapping->nrpages) { + /* + * As in generic_file_direct_write(), invalidate after the + * write, to invalidate read-ahead cache that may have competed + * with the write. + */ + INIT_WORK(&io->work, fuse_aio_invalidate_worker); + queue_work(inode->i_sb->s_dio_done_wq, &io->work); + return; + } + io->iocb->ki_complete(io->iocb, res); } @@ -709,8 +738,7 @@ static void fuse_io_free(struct fuse_io_args *ia) kfree(ia); } -static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, - int err) +static void fuse_aio_complete_req(struct fuse_args *args, int err) { struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); struct fuse_io_priv *io = ia->io; @@ -758,7 +786,7 @@ static ssize_t fuse_async_req_send(struct fuse_mount *fm, ia->ap.args.may_block = io->should_dirty; err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL); if (err) - fuse_aio_complete_req(fm, &ia->ap.args, err); + fuse_aio_complete_req(&ia->ap.args, err); return num_bytes; } @@ -902,7 +930,7 @@ static int fuse_handle_readahead(struct folio *folio, ia = NULL; } if (!ia) { - if (fc->num_background >= fc->congestion_threshold && + if (fuse_chan_num_background(fc->chan) >= fc->congestion_threshold && rac->ra->async_size >= readahead_count(rac)) /* * Congested and only async pages left, so skip the @@ -953,19 +981,8 @@ static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter, return ret; } -static void fuse_iomap_submit_read(const struct iomap_iter *iter, - struct iomap_read_folio_ctx *ctx) -{ - struct fuse_fill_read_data *data = ctx->read_ctx; - - if (data->ia) - fuse_send_readpages(data->ia, data->file, data->nr_bytes, - data->fc->async_read); -} - static const struct iomap_read_ops fuse_iomap_read_ops = { .read_folio_range = fuse_iomap_read_folio_range_async, - .submit_read = fuse_iomap_submit_read, }; static int fuse_read_folio(struct file *file, struct folio *folio) @@ -1001,8 +1018,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, return fuse_do_readfolio(file, folio, off, len); } -static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, - int err) +static void fuse_readpages_end(struct fuse_args *args, int err) { int i; struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); @@ -1068,7 +1084,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } - fuse_readpages_end(fm, &ap->args, err); + fuse_readpages_end(&ap->args, err); } static void fuse_readahead(struct readahead_control *rac) @@ -1089,6 +1105,9 @@ static void fuse_readahead(struct readahead_control *rac) return; iomap_readahead(&fuse_iomap_ops, &ctx, NULL); + if (data.ia) + fuse_send_readpages(data.ia, data.file, data.nr_bytes, + fc->async_read); } static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) @@ -1586,7 +1605,7 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, * manually extract pages using iov_iter_extract_pages() and then * copy that to a folios array. */ - struct page **pages = kzalloc(max_pages * sizeof(struct page *), + struct page **pages = kcalloc(max_pages, sizeof(struct page *), GFP_KERNEL); if (!pages) { ret = -ENOMEM; @@ -1743,15 +1762,6 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (res > 0) *ppos = pos; - if (res > 0 && write && fopen_direct_io) { - /* - * As in generic_file_direct_write(), invalidate after the - * write, to invalidate read-ahead cache that may have competed - * with the write. - */ - invalidate_inode_pages2_range(mapping, idx_from, idx_to); - } - return res > 0 ? res : err; } EXPORT_SYMBOL_GPL(fuse_direct_io); @@ -1790,6 +1800,8 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); + struct address_space *mapping = inode->i_mapping; + loff_t pos = iocb->ki_pos; ssize_t res; bool exclusive; @@ -1806,6 +1818,16 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) FUSE_DIO_WRITE); fuse_write_update_attr(inode, iocb->ki_pos, res); } + if (res > 0 && mapping->nrpages) { + /* + * As in generic_file_direct_write(), invalidate after + * write, to invalidate read-ahead cache that may have + * with the write. + */ + invalidate_inode_pages2_range(mapping, + pos >> PAGE_SHIFT, + (pos + res - 1) >> PAGE_SHIFT); + } } fuse_dio_unlock(iocb, exclusive); @@ -1985,8 +2007,7 @@ __acquires(fi->lock) } } -static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, - int error) +static void fuse_writepage_end(struct fuse_args *args, int error) { struct fuse_writepage_args *wpa = container_of(args, typeof(*wpa), ia.ap.args); @@ -2176,7 +2197,10 @@ static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, WARN_ON(!ap->num_folios); - /* Reached max pages */ + /* Reached max pages or max folio slots */ + if (ap->num_folios >= fc->max_pages) + return true; + if (DIV_ROUND_UP(bytes, PAGE_SIZE) > fc->max_pages) return true; @@ -2294,7 +2318,7 @@ static int fuse_writepages(struct address_space *mapping, return -EIO; if (wbc->sync_mode == WB_SYNC_NONE && - fc->num_background >= fc->congestion_threshold) + fuse_chan_num_background(fc->chan) >= fc->congestion_threshold) return 0; return iomap_writepages(&wpc); @@ -2568,8 +2592,9 @@ static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) struct fuse_file *ff = file->private_data; /* emulate flock with POSIX locks */ - ff->flock = true; err = fuse_setlk(file, fl, 1); + if (!err) + ff->flock = true; } return err; @@ -2680,125 +2705,6 @@ static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence) return retval; } -/* - * All files which have been polled are linked to RB tree - * fuse_conn->polled_files which is indexed by kh. Walk the tree and - * find the matching one. - */ -static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh, - struct rb_node **parent_out) -{ - struct rb_node **link = &fc->polled_files.rb_node; - struct rb_node *last = NULL; - - while (*link) { - struct fuse_file *ff; - - last = *link; - ff = rb_entry(last, struct fuse_file, polled_node); - - if (kh < ff->kh) - link = &last->rb_left; - else if (kh > ff->kh) - link = &last->rb_right; - else - return link; - } - - if (parent_out) - *parent_out = last; - return link; -} - -/* - * The file is about to be polled. Make sure it's on the polled_files - * RB tree. Note that files once added to the polled_files tree are - * not removed before the file is released. This is because a file - * polled once is likely to be polled again. - */ -static void fuse_register_polled_file(struct fuse_conn *fc, - struct fuse_file *ff) -{ - spin_lock(&fc->lock); - if (RB_EMPTY_NODE(&ff->polled_node)) { - struct rb_node **link, *parent; - - link = fuse_find_polled_node(fc, ff->kh, &parent); - BUG_ON(*link); - rb_link_node(&ff->polled_node, parent, link); - rb_insert_color(&ff->polled_node, &fc->polled_files); - } - spin_unlock(&fc->lock); -} - -__poll_t fuse_file_poll(struct file *file, poll_table *wait) -{ - struct fuse_file *ff = file->private_data; - struct fuse_mount *fm = ff->fm; - struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; - struct fuse_poll_out outarg; - FUSE_ARGS(args); - int err; - - if (fm->fc->no_poll) - return DEFAULT_POLLMASK; - - poll_wait(file, &ff->poll_wait, wait); - inarg.events = mangle_poll(poll_requested_events(wait)); - - /* - * Ask for notification iff there's someone waiting for it. - * The client may ignore the flag and always notify. - */ - if (waitqueue_active(&ff->poll_wait)) { - inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; - fuse_register_polled_file(fm->fc, ff); - } - - args.opcode = FUSE_POLL; - args.nodeid = ff->nodeid; - args.in_numargs = 1; - args.in_args[0].size = sizeof(inarg); - args.in_args[0].value = &inarg; - args.out_numargs = 1; - args.out_args[0].size = sizeof(outarg); - args.out_args[0].value = &outarg; - err = fuse_simple_request(fm, &args); - - if (!err) - return demangle_poll(outarg.revents); - if (err == -ENOSYS) { - fm->fc->no_poll = 1; - return DEFAULT_POLLMASK; - } - return EPOLLERR; -} -EXPORT_SYMBOL_GPL(fuse_file_poll); - -/* - * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and - * wakes up the poll waiters. - */ -int fuse_notify_poll_wakeup(struct fuse_conn *fc, - struct fuse_notify_poll_wakeup_out *outarg) -{ - u64 kh = outarg->kh; - struct rb_node **link; - - spin_lock(&fc->lock); - - link = fuse_find_polled_node(fc, kh, NULL); - if (*link) { - struct fuse_file *ff; - - ff = rb_entry(*link, struct fuse_file, polled_node); - wake_up_interruptible_sync(&ff->poll_wait); - } - - spin_unlock(&fc->lock); - return 0; -} - static void fuse_do_truncate(struct file *file) { struct inode *inode = file->f_mapping->host; @@ -2831,6 +2737,7 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) size_t count = iov_iter_count(iter), shortened = 0; loff_t offset = iocb->ki_pos; struct fuse_io_priv *io; + bool async = ff->fm->fc->async_dio; pos = offset; inode = file->f_mapping->host; @@ -2839,6 +2746,12 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if ((iov_iter_rw(iter) == READ) && (offset >= i_size)) return 0; + if ((iov_iter_rw(iter) == WRITE) && async && !inode->i_sb->s_dio_done_wq) { + ret = sb_init_dio_done_wq(inode->i_sb); + if (ret < 0) + return ret; + } + io = kmalloc_obj(struct fuse_io_priv); if (!io) return -ENOMEM; @@ -2854,7 +2767,7 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) * By default, we want to optimize all I/Os with async request * submission to the client filesystem if supported. */ - io->async = ff->fm->fc->async_dio; + io->async = async; io->iocb = iocb; io->blocking = is_sync_kiocb(iocb); diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 910f883cd090..668c8391d61c 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -6,20 +6,311 @@ #ifndef _FS_FUSE_DEV_I_H #define _FS_FUSE_DEV_I_H +#include <linux/fuse.h> #include <linux/types.h> +#include <linux/refcount.h> +#include <linux/wait.h> +#include <linux/workqueue.h> +#include <linux/fs.h> /* Ordinary requests have even IDs, while interrupts IDs are odd */ #define FUSE_INT_REQ_BIT (1ULL << 0) #define FUSE_REQ_ID_STEP (1ULL << 1) -extern struct wait_queue_head fuse_dev_waitq; - struct fuse_arg; struct fuse_args; struct fuse_pqueue; -struct fuse_req; struct fuse_iqueue; -struct fuse_forget_link; + +/** + * enum fuse_req_flag - Request flags + * + * @FR_ISREPLY: set if the request has reply + * @FR_FORCE: force sending of the request even if interrupted + * @FR_BACKGROUND: request is sent in the background + * @FR_WAITING: request is counted as "waiting" + * @FR_ABORTED: the request was aborted + * @FR_INTERRUPTED: the request has been interrupted + * @FR_LOCKED: data is being copied to/from the request + * @FR_PENDING: request is not yet in userspace + * @FR_SENT: request is in userspace, waiting for an answer + * @FR_FINISHED: request is finished + * @FR_PRIVATE: request is on private list + * @FR_ASYNC: request is asynchronous + * @FR_URING: request is handled through fuse-io-uring + */ +enum fuse_req_flag { + FR_ISREPLY, + FR_FORCE, + FR_BACKGROUND, + FR_WAITING, + FR_ABORTED, + FR_INTERRUPTED, + FR_LOCKED, + FR_PENDING, + FR_SENT, + FR_FINISHED, + FR_PRIVATE, + FR_ASYNC, + FR_URING, +}; + +/** + * struct fuse_req - A request to the client + * + * .waitq.lock protects the following fields: + * - FR_ABORTED + * - FR_LOCKED (may also be modified under fpq->lock, tested under both) + */ +struct fuse_req { + /** + * @list: This can be on either pending processing or io lists in + * fuse_conn + */ + struct list_head list; + + /** @intr_entry: Entry on the interrupts list */ + struct list_head intr_entry; + + /** @args: Input/output arguments */ + struct fuse_args *args; + + /** @count: refcount */ + refcount_t count; + + /** @flags: Request flags, updated with test/set/clear_bit() */ + unsigned long flags; + + /** @in: The request input header */ + struct { + /** @in.h: The request input header */ + struct fuse_in_header h; + } in; + + /** @out: The request output header */ + struct { + /** @out.h: The request output header */ + struct fuse_out_header h; + } out; + + /** @waitq: Used to wake up the task waiting for completion of request */ + wait_queue_head_t waitq; + +#if IS_ENABLED(CONFIG_VIRTIO_FS) + /** + * @argbuf: virtio-fs's physically contiguous buffer for in and out + * args + */ + void *argbuf; +#endif + + /** @chan: fuse_chan this request belongs to */ + struct fuse_chan *chan; + +#ifdef CONFIG_FUSE_IO_URING + void *ring_entry; + void *ring_queue; +#endif + /** @create_time: When (in jiffies) the request was created */ + unsigned long create_time; +}; + +/* One forget request */ +struct fuse_forget_link { + struct fuse_forget_one forget_one; + struct fuse_forget_link *next; +}; + +/** + * struct fuse_iqueue_ops - Input queue callbacks + * + * Input queue signalling is device-specific. For example, the /dev/fuse file + * uses fiq->waitq and fasync to wake processes that are waiting on queue + * readiness. These callbacks allow other device types to respond to input + * queue activity. + */ +struct fuse_iqueue_ops { + /** + * @send_forget: Send one forget + */ + void (*send_forget)(struct fuse_iqueue *fiq, struct fuse_forget_link *link); + + /** + * @send_interrupt: Send interrupt for request + */ + void (*send_interrupt)(struct fuse_iqueue *fiq, struct fuse_req *req); + + /** + * @send_req: Send one request + */ + void (*send_req)(struct fuse_iqueue *fiq, struct fuse_req *req); + + /** + * @release: Clean up when fuse_iqueue is destroyed + */ + void (*release)(struct fuse_iqueue *fiq); +}; + +struct fuse_iqueue { + /** Connection established */ + unsigned connected; + + /** Lock protecting accesses to members of this structure */ + spinlock_t lock; + + /** Readers of the connection are waiting on this */ + wait_queue_head_t waitq; + + /** The next unique request id */ + u64 reqctr; + + /** The list of pending requests */ + struct list_head pending; + + /** Pending interrupts */ + struct list_head interrupts; + + /** Queue of pending forgets */ + struct fuse_forget_link forget_list_head; + struct fuse_forget_link *forget_list_tail; + + /** Batching of FORGET requests (positive indicates FORGET batch) */ + int forget_batch; + + /** O_ASYNC requests */ + struct fasync_struct *fasync; + + /** Device-specific callbacks */ + const struct fuse_iqueue_ops *ops; + + /** Device-specific state */ + void *priv; +}; + +struct fuse_chan { + /** Lock protecting: + - devices + - connected + - ring + - ring->queues[qid] + */ + spinlock_t lock; + + /* back pointer: fc->chan->conn == fc */ + struct fuse_conn *conn; + + /** Input queue */ + struct fuse_iqueue iq; + + /** List of device instances belonging to this connection */ + struct list_head devices; + + /** Maximum number of outstanding background requests */ + unsigned max_background; + + /** Number of requests currently in the background */ + unsigned num_background; + + /** Number of background requests currently queued for userspace */ + unsigned active_background; + + /** The list of background requests set aside for later queuing */ + struct list_head bg_queue; + + /** Protects: max_background, num_background, active_background, bg_queue, blocked */ + spinlock_t bg_lock; + + /** Flag indicating that INIT reply has been received. Allocating + * any fuse request will be suspended until the flag is set */ + int initialized; + + /** Flag indicating if connection is blocked. This will be + the case before the INIT reply is received, and if there + are too many outstading backgrounds requests */ + int blocked; + + /** waitq for blocked connection */ + wait_queue_head_t blocked_waitq; + + /** Connection established, cleared on umount, connection + abort and device release */ + unsigned connected; + + /** The number of requests waiting for completion */ + atomic_t num_waiting; + + /** Is interrupt not implemented by fs? */ + bool no_interrupt; + + /* Use io_uring for communication */ + unsigned int io_uring; + + /* Negotiated minor version */ + unsigned int minor; + + /* Maximum write size */ + unsigned int max_write; + + /* Maximum number of pages that can be used in a single request */ + unsigned int max_pages; + + /* Before being installed into fud, contains the preallocated pq array*/ + struct list_head *pq_prealloc; + + /** Connection aborted via sysfs, respond with ECONNABORTED on device I/O */ + bool abort_with_err; + +#ifdef CONFIG_FUSE_IO_URING + /** uring connection information*/ + struct fuse_ring *ring; +#endif + + /** Only used if the connection opts into request timeouts */ + struct { + /* Worker for checking if any requests have timed out */ + struct delayed_work work; + + /* Request timeout (in jiffies). 0 = no timeout */ + unsigned int req_timeout; + } timeout; +}; + +#define FUSE_PQ_HASH_BITS 8 +#define FUSE_PQ_HASH_SIZE (1 << FUSE_PQ_HASH_BITS) + +struct fuse_pqueue { + /** Connection established */ + unsigned connected; + + /** Lock protecting accessess to members of this structure */ + spinlock_t lock; + + /** Hash table of requests being processed */ + struct list_head *processing; + + /** The list of requests under I/O */ + struct list_head io; +}; + +/** + * struct fuse_dev - Fuse device instance + */ +struct fuse_dev { + /** @ref: Reference count of this object */ + refcount_t ref; + + /** @sync_init: Issue FUSE_INIT synchronously */ + bool sync_init; + + /** @chan: Fuse channel for this device */ + struct fuse_chan *chan; + + /** @pq: Processing queue */ + struct fuse_pqueue pq; + + /** @entry: list entry on fch->devices */ + struct list_head entry; +}; struct fuse_copy_state { struct fuse_req *req; @@ -39,21 +330,21 @@ struct fuse_copy_state { } ring; }; -/* fud->fc gets assigned to this value when /dev/fuse is closed */ -#define FUSE_DEV_FC_DISCONNECTED ((struct fuse_conn *) 1) +/* fud->chan gets assigned to this value when /dev/fuse is closed */ +#define FUSE_DEV_CHAN_DISCONNECTED ((struct fuse_chan *) 1) /* - * Lockless access is OK, because fud->fc is set once during mount and is valid + * Lockless access is OK, because fud->chan is set once during mount and is valid * until the file is released. * - * fud->fc is set to FUSE_DEV_FC_DISCONNECTED only after the containing file is + * fud->chan is set to FUSE_DEV_CHAN_DISCONNECTED only after the containing file is * released, so result is safe to dereference in most cases. Exceptions are: * fuse_dev_put() and fuse_fill_super_common(). */ -static inline struct fuse_conn *fuse_dev_fc_get(struct fuse_dev *fud) +static inline struct fuse_chan *fuse_dev_chan_get(struct fuse_dev *fud) { /* Pairs with xchg() in fuse_dev_install() */ - return smp_load_acquire(&fud->fc); + return smp_load_acquire(&fud->chan); } static inline struct fuse_dev *fuse_file_to_fud(struct file *file) @@ -65,22 +356,29 @@ static inline struct fuse_dev *__fuse_get_dev(struct file *file) { struct fuse_dev *fud = fuse_file_to_fud(file); - if (!fuse_dev_fc_get(fud)) + if (!fuse_dev_chan_get(fud)) return NULL; return fud; } +void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv); + struct fuse_dev *fuse_get_dev(struct file *file); unsigned int fuse_req_hash(u64 unique); struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique); void fuse_dev_end_requests(struct list_head *head); +void fuse_request_bg_finish(struct fuse_chan *fch, struct fuse_req *req); void fuse_copy_init(struct fuse_copy_state *cs, bool write, struct iov_iter *iter); -void fuse_copy_finish(struct fuse_copy_state *cs); +/* + * Return the number of bytes in an arguments list + */ +unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args); + int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs, unsigned int argpages, struct fuse_arg *args, int zeroing); @@ -91,7 +389,34 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req); bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock); -bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list); +bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list); + +/* + * Assign a unique id to a fuse request + */ +void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req); + +/* + * Get the next unique ID for a request + */ +u64 fuse_get_unique(struct fuse_iqueue *fiq); + +struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch); +struct fuse_dev *fuse_dev_alloc(void); + +int fuse_dev_release(struct inode *inode, struct file *file); + +struct list_head *fuse_pqueue_alloc(void); + +/* + * Initialize the fuse processing queue + */ +void fuse_pqueue_init(struct fuse_pqueue *fpq); + +/* + * End a finished request + */ +void fuse_request_end(struct fuse_req *req); #endif diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 17423d4e3cfa..85f738c53122 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1,9 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ #ifndef _FS_FUSE_I_H @@ -13,6 +11,7 @@ # define pr_fmt(fmt) "fuse: " fmt #endif +#include "args.h" #include <linux/fuse.h> #include <linux/fs.h> #include <linux/mount.h> @@ -48,12 +47,6 @@ /** Number of dentries for each connection in the control filesystem */ #define FUSE_CTL_NUM_DENTRIES 5 -/* Frequency (in seconds) of request timeout checks, if opted into */ -#define FUSE_TIMEOUT_TIMER_FREQ 15 - -/** Frequency (in jiffies) of request timeout checks, if opted into */ -extern const unsigned long fuse_timeout_timer_freq; - /* * Dentries invalidation workqueue period, in seconds. The value of this * parameter shall be >= FUSE_DENTRY_INVAL_FREQ_MIN seconds, or 0 (zero), in @@ -63,16 +56,6 @@ extern unsigned inval_wq __read_mostly; /** Maximum of max_pages received in init_out */ extern unsigned int fuse_max_pages_limit; -/* - * Default timeout (in seconds) for the server to reply to a request - * before the connection is aborted, if no timeout was specified on mount. - */ -extern unsigned int fuse_default_req_timeout; -/* - * Max timeout (in seconds) for the server to reply to a request before - * the connection is aborted. - */ -extern unsigned int fuse_max_req_timeout; /** List of active connections */ extern struct list_head fuse_conn_list; @@ -84,143 +67,173 @@ extern struct mutex fuse_mutex; extern unsigned int max_user_bgreq; extern unsigned int max_user_congthresh; -/* One forget request */ -struct fuse_forget_link { - struct fuse_forget_one forget_one; - struct fuse_forget_link *next; -}; +struct fuse_forget_link; -/* Submount lookup tracking */ +/** + * struct fuse_submount_lookup - Submount lookup tracking + */ struct fuse_submount_lookup { - /** Refcount */ + /** @count: Refcount */ refcount_t count; - /** Unique ID, which identifies the inode between userspace - * and kernel */ + /** + * @nodeid: Unique ID, which identifies the inode between userspace + * and kernel + */ u64 nodeid; - /** The request used for sending the FORGET message */ + /** @forget: The request used for sending the FORGET message */ struct fuse_forget_link *forget; }; -/** Container for data related to mapping to backing file */ +/* Container for data related to mapping to backing file */ struct fuse_backing { struct file *file; - struct cred *cred; + const struct cred *cred; - /** refcount */ + /* refcount */ refcount_t count; struct rcu_head rcu; }; -/** FUSE inode */ +/** + * struct fuse_inode - FUSE inode + */ struct fuse_inode { - /** Inode data */ + /** @inode: Inode data */ struct inode inode; - /** Unique ID, which identifies the inode between userspace - * and kernel */ + /** + * @nodeid: Unique ID, which identifies the inode between userspace + * and kernel + */ u64 nodeid; - /** Number of lookups on this inode */ + /** @nlookup: Number of lookups on this inode */ u64 nlookup; - /** The request used for sending the FORGET message */ + /** @forget: The request used for sending the FORGET message */ struct fuse_forget_link *forget; - /** Time in jiffies until the file attributes are valid */ + /** @i_time: Time in jiffies until the file attributes are valid */ u64 i_time; - /* Which attributes are invalid */ + /** @inval_mask: Which attributes are invalid */ u32 inval_mask; - /** The sticky bit in inode->i_mode may have been removed, so - preserve the original mode */ + /** + * @orig_i_mode: The sticky bit in inode->i_mode may have been removed, + * so preserve the original mode + */ umode_t orig_i_mode; - /* Cache birthtime */ + /** @i_btime: Cache birthtime */ struct timespec64 i_btime; - /** 64 bit inode number */ + /** @orig_ino: 64-bit inode number */ u64 orig_ino; - /** Version of last attribute change */ + /** @attr_version: Version of last attribute change */ u64 attr_version; union { /* read/write io cache (regular file only) */ struct { - /* Files usable in writepage. Protected by fi->lock */ + /** + * @write_files: Files usable in writepage. + * Protected by fi->lock + */ struct list_head write_files; - /* Writepages pending on truncate or fsync */ + /** + * @queued_writes: Writepages pending on truncate or + * fsync + */ struct list_head queued_writes; - /* Number of sent writes, a negative bias - * (FUSE_NOWRITE) means more writes are blocked */ + /** + * @writectr: Number of sent writes, a negative bias + * (FUSE_NOWRITE) means more writes are blocked + */ int writectr; - /** Number of files/maps using page cache */ + /** @iocachectr: Number of files/maps using page cache */ int iocachectr; - /* Waitq for writepage completion */ + /** @page_waitq: Waitq for writepage completion */ wait_queue_head_t page_waitq; - /* waitq for direct-io completion */ + /** @direct_io_waitq: waitq for direct-io completion */ wait_queue_head_t direct_io_waitq; }; - /* readdir cache (directory only) */ + /** @rdc: readdir cache (directory only) */ struct { - /* true if fully cached */ + /** @cached: true if fully cached */ bool cached; - /* size of cache */ + /** @size: size of cache */ loff_t size; - /* position at end of cache (position of next entry) */ + /** + * @pos: position at end of cache (position of next + * entry) + */ loff_t pos; - /* version of the cache */ + /** @version: version of the cache */ u64 version; - /* modification time of directory when cache was - * started */ + /** + * @mtime: modification time of directory when cache was + * started + */ struct timespec64 mtime; - /* iversion of directory when cache was started */ + /** + * @epoch: epoch of fc when cache was started + */ + int epoch; + + /** + * @iversion: iversion of directory when cache was + * started + */ u64 iversion; - /* protects above fields */ + /** @lock: protects above fields */ spinlock_t lock; } rdc; }; - /** Miscellaneous bits describing inode state */ + /** @state: Miscellaneous bits describing inode state */ unsigned long state; - /** Lock for serializing lookup and readdir for back compatibility*/ + /** + * @mutex: Lock for serializing lookup and readdir for back + * compatibility + */ struct mutex mutex; - /** Lock to protect write related fields */ + /** @lock: Lock to protect write-related fields */ spinlock_t lock; #ifdef CONFIG_FUSE_DAX - /* - * Dax specific inode data + /** + * @dax: Dax specific inode data */ struct fuse_inode_dax *dax; #endif - /** Submount specific lookup tracking */ + /** @submount_lookup: Submount specific lookup tracking */ struct fuse_submount_lookup *submount_lookup; #ifdef CONFIG_FUSE_PASSTHROUGH - /** Reference to backing file in passthrough mode */ + /** @fb: Reference to backing file in passthrough mode */ struct fuse_backing *fb; #endif - /* - * The underlying inode->i_blkbits value will not be modified, - * so preserve the blocksize specified by the server. + /** + * @cached_i_blkbits: The underlying inode->i_blkbits value will not + * be modified, so preserve the blocksize specified by the server. */ u8 cached_i_blkbits; }; @@ -250,116 +263,67 @@ struct fuse_conn; struct fuse_mount; union fuse_file_args; -/** FUSE specific file data */ +/** + * struct fuse_file - FUSE-specific file data + */ struct fuse_file { - /** Fuse connection for this file */ + /** @fm: Fuse connection for this file */ struct fuse_mount *fm; - /* Argument space reserved for open/release */ + /** @args: Argument space reserved for open/release */ union fuse_file_args *args; - /** Kernel file handle guaranteed to be unique */ + /** @kh: Kernel file handle guaranteed to be unique */ u64 kh; - /** File handle used by userspace */ + /** @fh: File handle used by userspace */ u64 fh; - /** Node id of this file */ + /** @nodeid: Node id of this file */ u64 nodeid; - /** Refcount */ + /** @count: Refcount */ refcount_t count; - /** FOPEN_* flags returned by open */ + /** @open_flags: FOPEN_* flags returned by open */ u32 open_flags; - /** Entry on inode's write_files list */ + /** @write_entry: Entry on inode's write_files list */ struct list_head write_entry; - /* Readdir related */ + /** @readdir: Readdir-related */ struct { - /* Dir stream position */ + /** @pos: Dir stream position */ loff_t pos; - /* Offset in cache */ + /** @cache_off: Offset in cache */ loff_t cache_off; - /* Version of cache we are reading */ + /** @version: Version of cache we are reading */ u64 version; } readdir; - /** RB node to be linked on fuse_conn->polled_files */ + /** @polled_node: RB node to be linked on fuse_conn->polled_files */ struct rb_node polled_node; - /** Wait queue head for poll */ + /** @poll_wait: Wait queue head for poll */ wait_queue_head_t poll_wait; - /** Does file hold a fi->iocachectr refcount? */ + /** @iomode: Does file hold a fi->iocachectr refcount? */ enum { IOM_NONE, IOM_CACHED, IOM_UNCACHED } iomode; #ifdef CONFIG_FUSE_PASSTHROUGH - /** Reference to backing file in passthrough mode */ + /** @passthrough: Reference to backing file in passthrough mode */ struct file *passthrough; + /** @cred: passthrough file credentials */ const struct cred *cred; #endif - /** Has flock been performed on this file? */ + /** @flock: Has flock been performed on this file? */ bool flock:1; }; -/** One input argument of a request */ -struct fuse_in_arg { - unsigned size; - const void *value; -}; - -/** One output argument of a request */ -struct fuse_arg { - unsigned size; - void *value; -}; - -/** FUSE folio descriptor */ -struct fuse_folio_desc { - unsigned int length; - unsigned int offset; -}; - -struct fuse_args { - uint64_t nodeid; - uint32_t opcode; - uint8_t in_numargs; - uint8_t out_numargs; - uint8_t ext_idx; - bool force:1; - bool noreply:1; - bool nocreds:1; - bool in_pages:1; - bool out_pages:1; - bool user_pages:1; - bool out_argvar:1; - bool page_zeroing:1; - bool page_replace:1; - bool may_block:1; - bool is_ext:1; - bool is_pinned:1; - bool invalidate_vmap:1; - bool abort_on_kill:1; - struct fuse_in_arg in_args[4]; - struct fuse_arg out_args[2]; - void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); - /* Used for kvec iter backed by vmalloc address */ - void *vmap_base; -}; - -struct fuse_args_pages { - struct fuse_args args; - struct folio **folios; - struct fuse_folio_desc *descs; - unsigned int num_folios; -}; - struct fuse_release_args { struct fuse_args args; struct fuse_release_in inarg; @@ -378,6 +342,7 @@ union fuse_file_args { /** The request IO state (for asynchronous processing) */ struct fuse_io_priv { struct kref refcnt; + struct work_struct work; int async; spinlock_t lock; unsigned reqs; @@ -399,200 +364,6 @@ struct fuse_io_priv { .iocb = i, \ } -/** - * Request flags - * - * FR_ISREPLY: set if the request has reply - * FR_FORCE: force sending of the request even if interrupted - * FR_BACKGROUND: request is sent in the background - * FR_WAITING: request is counted as "waiting" - * FR_ABORTED: the request was aborted - * FR_INTERRUPTED: the request has been interrupted - * FR_LOCKED: data is being copied to/from the request - * FR_PENDING: request is not yet in userspace - * FR_SENT: request is in userspace, waiting for an answer - * FR_FINISHED: request is finished - * FR_PRIVATE: request is on private list - * FR_ASYNC: request is asynchronous - * FR_URING: request is handled through fuse-io-uring - */ -enum fuse_req_flag { - FR_ISREPLY, - FR_FORCE, - FR_BACKGROUND, - FR_WAITING, - FR_ABORTED, - FR_INTERRUPTED, - FR_LOCKED, - FR_PENDING, - FR_SENT, - FR_FINISHED, - FR_PRIVATE, - FR_ASYNC, - FR_URING, -}; - -/** - * A request to the client - * - * .waitq.lock protects the following fields: - * - FR_ABORTED - * - FR_LOCKED (may also be modified under fc->lock, tested under both) - */ -struct fuse_req { - /** This can be on either pending processing or io lists in - fuse_conn */ - struct list_head list; - - /** Entry on the interrupts list */ - struct list_head intr_entry; - - /* Input/output arguments */ - struct fuse_args *args; - - /** refcount */ - refcount_t count; - - /* Request flags, updated with test/set/clear_bit() */ - unsigned long flags; - - /* The request input header */ - struct { - struct fuse_in_header h; - } in; - - /* The request output header */ - struct { - struct fuse_out_header h; - } out; - - /** Used to wake up the task waiting for completion of request*/ - wait_queue_head_t waitq; - -#if IS_ENABLED(CONFIG_VIRTIO_FS) - /** virtio-fs's physically contiguous buffer for in and out args */ - void *argbuf; -#endif - - /** fuse_mount this request belongs to */ - struct fuse_mount *fm; - -#ifdef CONFIG_FUSE_IO_URING - void *ring_entry; - void *ring_queue; -#endif - /** When (in jiffies) the request was created */ - unsigned long create_time; -}; - -struct fuse_iqueue; - -/** - * Input queue callbacks - * - * Input queue signalling is device-specific. For example, the /dev/fuse file - * uses fiq->waitq and fasync to wake processes that are waiting on queue - * readiness. These callbacks allow other device types to respond to input - * queue activity. - */ -struct fuse_iqueue_ops { - /** - * Send one forget - */ - void (*send_forget)(struct fuse_iqueue *fiq, struct fuse_forget_link *link); - - /** - * Send interrupt for request - */ - void (*send_interrupt)(struct fuse_iqueue *fiq, struct fuse_req *req); - - /** - * Send one request - */ - void (*send_req)(struct fuse_iqueue *fiq, struct fuse_req *req); - - /** - * Clean up when fuse_iqueue is destroyed - */ - void (*release)(struct fuse_iqueue *fiq); -}; - -/** /dev/fuse input queue operations */ -extern const struct fuse_iqueue_ops fuse_dev_fiq_ops; - -struct fuse_iqueue { - /** Connection established */ - unsigned connected; - - /** Lock protecting accesses to members of this structure */ - spinlock_t lock; - - /** Readers of the connection are waiting on this */ - wait_queue_head_t waitq; - - /** The next unique request id */ - u64 reqctr; - - /** The list of pending requests */ - struct list_head pending; - - /** Pending interrupts */ - struct list_head interrupts; - - /** Queue of pending forgets */ - struct fuse_forget_link forget_list_head; - struct fuse_forget_link *forget_list_tail; - - /** Batching of FORGET requests (positive indicates FORGET batch) */ - int forget_batch; - - /** O_ASYNC requests */ - struct fasync_struct *fasync; - - /** Device-specific callbacks */ - const struct fuse_iqueue_ops *ops; - - /** Device-specific state */ - void *priv; -}; - -#define FUSE_PQ_HASH_BITS 8 -#define FUSE_PQ_HASH_SIZE (1 << FUSE_PQ_HASH_BITS) - -struct fuse_pqueue { - /** Connection established */ - unsigned connected; - - /** Lock protecting accessess to members of this structure */ - spinlock_t lock; - - /** Hash table of requests being processed */ - struct list_head *processing; - - /** The list of requests under I/O */ - struct list_head io; -}; - -/** - * Fuse device instance - */ -struct fuse_dev { - /** Reference count of this object */ - refcount_t ref; - - /** Issue FUSE_INIT synchronously */ - bool sync_init; - - /** Fuse connection for this device */ - struct fuse_conn *fc; - - /** Processing queue */ - struct fuse_pqueue pq; - - /** list entry on fc->devices */ - struct list_head entry; -}; - enum fuse_dax_mode { FUSE_DAX_INODE_DEFAULT, /* default */ FUSE_DAX_ALWAYS, /* "-o dax=always" */ @@ -637,133 +408,135 @@ struct fuse_sync_bucket { }; /** - * A Fuse connection. + * struct fuse_conn - A Fuse connection. * * This structure is created, when the root filesystem is mounted, and * is destroyed, when the client device is closed and the last * fuse_mount is destroyed. */ struct fuse_conn { - /** Lock protecting accessess to members of this structure */ + /** + * @lock: Lock protecting: + * - polled_files + * - backing_files_map + * - curr_bucket + */ spinlock_t lock; - /** Refcount */ + /** @count: Refcount */ refcount_t count; - /** Current epoch for up-to-date dentries */ + /** @epoch: Current epoch for up-to-date dentries */ atomic_t epoch; + /** @epoch_work: Used to invalidate dentries from old epochs */ struct work_struct epoch_work; + /** @rcu: Used to delay freeing fuse_conn, making it safe */ struct rcu_head rcu; - /** The user id for this mount */ + /** @user_id: The user id for this mount */ kuid_t user_id; - /** The group id for this mount */ + /** @group_id: The group id for this mount */ kgid_t group_id; - /** The pid namespace for this mount */ + /** @pid_ns: The pid namespace for this mount */ struct pid_namespace *pid_ns; - /** The user namespace for this mount */ + /** @user_ns: The user namespace for this mount */ struct user_namespace *user_ns; - /** Maximum read size */ + /** @max_read: Maximum read size */ unsigned max_read; - /** Maximum write size */ + /** @max_write: Maximum write size */ unsigned max_write; - /** Maximum number of pages that can be used in a single request */ + /** + * @max_pages: Maximum number of pages that can be used in a + * single request + */ unsigned int max_pages; - /** Constrain ->max_pages to this value during feature negotiation */ + /** + * @max_pages_limit: Constrain ->max_pages to this value during + * feature negotiation + */ unsigned int max_pages_limit; - /** Input queue */ - struct fuse_iqueue iq; + /** @chan: transport layer object */ + struct fuse_chan *chan; - /** The next unique kernel file handle */ + /** @khctr: The next unique kernel file handle */ atomic64_t khctr; - /** rbtree of fuse_files waiting for poll events indexed by ph */ + /** + * @polled_files: rbtree of fuse_files waiting for poll events + * indexed by ph + */ struct rb_root polled_files; - /** Maximum number of outstanding background requests */ - unsigned max_background; - - /** Number of background requests at which congestion starts */ + /** + * @congestion_threshold: Number of background requests at which + * congestion starts + */ unsigned congestion_threshold; - /** Number of requests currently in the background */ - unsigned num_background; - - /** Number of background requests currently queued for userspace */ - unsigned active_background; - - /** The list of background requests set aside for later queuing */ - struct list_head bg_queue; - - /** Protects: max_background, congestion_threshold, num_background, - * active_background, bg_queue, blocked */ - spinlock_t bg_lock; - - /** Flag indicating that INIT reply has been received. Allocating - * any fuse request will be suspended until the flag is set */ - int initialized; - - /** Flag indicating if connection is blocked. This will be - the case before the INIT reply is received, and if there - are too many outstading backgrounds requests */ - int blocked; - - /** waitq for blocked connection */ - wait_queue_head_t blocked_waitq; - - /** Connection established, cleared on umount, connection - abort and device release */ - unsigned connected; - - /** Connection aborted via sysfs */ - bool aborted; - - /** Connection failed (version mismatch). Cannot race with - setting other bitfields since it is only set once in INIT - reply, before any other request, and never cleared */ + /** + * @conn_error: Connection failed (version mismatch). Cannot race with + * setting other bitfields since it is only set once in INIT + * reply, before any other request, and never cleared + */ unsigned conn_error:1; - /** Connection successful. Only set in INIT */ + /** @conn_init: Connection successful. Only set in INIT */ unsigned conn_init:1; - /** Do readahead asynchronously? Only set in INIT */ + /** @async_read: Do readahead asynchronously? Only set in INIT */ unsigned async_read:1; - /** Return an unique read error after abort. Only set in INIT */ + /** + * @abort_err: Return an unique read error after abort. + * Only set in INIT + */ unsigned abort_err:1; - /** Do not send separate SETATTR request before open(O_TRUNC) */ + /** + * @atomic_o_trunc: Do not send separate SETATTR request before + * open(O_TRUNC) + */ unsigned atomic_o_trunc:1; - /** Filesystem supports NFS exporting. Only set in INIT */ + /** + * @export_support: Filesystem supports NFS exporting. + * Only set in INIT + */ unsigned export_support:1; - /** write-back cache policy (default is write-through) */ + /** @writeback_cache: write-back cache policy (default is write-through) */ unsigned writeback_cache:1; - /** allow parallel lookups and readdir (default is serialized) */ + /** + * @parallel_dirops: allow parallel lookups and readdir (default is + * serialized) + */ unsigned parallel_dirops:1; - /** handle fs handles killing suid/sgid/cap on write/chown/trunc */ + /** + * @handle_killpriv: handle fs handles killing suid/sgid/cap on + * write/chown/trunc + */ unsigned handle_killpriv:1; - /** cache READLINK responses in page cache */ + /** @cache_symlinks: cache READLINK responses in page cache */ unsigned cache_symlinks:1; - /* show legacy mount options */ + /** @legacy_opts_show: show legacy mount options */ unsigned int legacy_opts_show:1; - /* + /** + * @handle_killpriv_v2: * fs kills suid/sgid/cap on write/chown/trunc. suid is killed on * write/trunc only if caller did not have CAP_FSETID. sgid is killed * on write/truncate only if caller did not have CAP_FSETID as well as @@ -776,224 +549,219 @@ struct fuse_conn { * and hence races in setting them will not cause malfunction */ - /** Is open/release not implemented by fs? */ + /** @no_open: Is open/release not implemented by fs? */ unsigned no_open:1; - /** Is opendir/releasedir not implemented by fs? */ + /** @no_opendir: Is opendir/releasedir not implemented by fs? */ unsigned no_opendir:1; - /** Is fsync not implemented by fs? */ + /** @no_fsync: Is fsync not implemented by fs? */ unsigned no_fsync:1; - /** Is fsyncdir not implemented by fs? */ + /** @no_fsyncdir: Is fsyncdir not implemented by fs? */ unsigned no_fsyncdir:1; - /** Is flush not implemented by fs? */ + /** @no_flush: Is flush not implemented by fs? */ unsigned no_flush:1; - /** Is setxattr not implemented by fs? */ + /** @no_setxattr: Is setxattr not implemented by fs? */ unsigned no_setxattr:1; - /** Does file server support extended setxattr */ + /** @setxattr_ext: Does file server support extended setxattr */ unsigned setxattr_ext:1; - /** Is getxattr not implemented by fs? */ + /** @no_getxattr: Is getxattr not implemented by fs? */ unsigned no_getxattr:1; - /** Is listxattr not implemented by fs? */ + /** @no_listxattr: Is listxattr not implemented by fs? */ unsigned no_listxattr:1; - /** Is removexattr not implemented by fs? */ + /** @no_removexattr: Is removexattr not implemented by fs? */ unsigned no_removexattr:1; - /** Are posix file locking primitives not implemented by fs? */ + /** @no_lock: Are posix file locking primitives not implemented by fs? */ unsigned no_lock:1; - /** Is access not implemented by fs? */ + /** @no_access: Is access not implemented by fs? */ unsigned no_access:1; - /** Is create not implemented by fs? */ + /** @no_create: Is create not implemented by fs? */ unsigned no_create:1; - /** Is interrupt not implemented by fs? */ - unsigned no_interrupt:1; - - /** Is bmap not implemented by fs? */ + /** @no_bmap: Is bmap not implemented by fs? */ unsigned no_bmap:1; - /** Is poll not implemented by fs? */ + /** @no_poll: Is poll not implemented by fs? */ unsigned no_poll:1; - /** Do multi-page cached writes */ + /** @big_writes: Do multi-page cached writes */ unsigned big_writes:1; - /** Don't apply umask to creation modes */ + /** @dont_mask: Don't apply umask to creation modes */ unsigned dont_mask:1; - /** Are BSD file locking primitives not implemented by fs? */ + /** @no_flock: Are BSD file locking primitives not implemented by fs? */ unsigned no_flock:1; - /** Is fallocate not implemented by fs? */ + /** @no_fallocate: Is fallocate not implemented by fs? */ unsigned no_fallocate:1; - /** Is rename with flags implemented by fs? */ + /** @no_rename2: Is rename with flags implemented by fs? */ unsigned no_rename2:1; - /** Use enhanced/automatic page cache invalidation. */ + /** @auto_inval_data: Use enhanced/automatic page cache invalidation. */ unsigned auto_inval_data:1; - /** Filesystem is fully responsible for page cache invalidation. */ + /** + * @explicit_inval_data: Filesystem is fully responsible for page cache + * invalidation. + */ unsigned explicit_inval_data:1; - /** Does the filesystem support readdirplus? */ + /** @do_readdirplus: Does the filesystem support readdirplus? */ unsigned do_readdirplus:1; - /** Does the filesystem want adaptive readdirplus? */ + /** @readdirplus_auto: Does the filesystem want adaptive readdirplus? */ unsigned readdirplus_auto:1; - /** Does the filesystem support asynchronous direct-IO submission? */ + /** + * @async_dio: Does the filesystem support asynchronous direct-IO + * submission? + */ unsigned async_dio:1; - /** Is lseek not implemented by fs? */ + /** @no_lseek: Is lseek not implemented by fs? */ unsigned no_lseek:1; - /** Does the filesystem support posix acls? */ + /** @posix_acl: Does the filesystem support posix acls? */ unsigned posix_acl:1; - /** Check permissions based on the file mode or not? */ + /** + * @default_permissions: Check permissions based on the file mode + * or not? + */ unsigned default_permissions:1; - /** Allow other than the mounter user to access the filesystem ? */ + /** + * @allow_other: Allow other than the mounter user to access the + * filesystem ? + */ unsigned allow_other:1; - /** Does the filesystem support copy_file_range? */ + /** @no_copy_file_range: Does the filesystem support copy_file_range? */ unsigned no_copy_file_range:1; - /** Does the filesystem support copy_file_range_64? */ + /** + * @no_copy_file_range_64: Does the filesystem support + * copy_file_range_64? + */ unsigned no_copy_file_range_64:1; - /* Send DESTROY request */ + /** @destroy: Send DESTROY request */ unsigned int destroy:1; - /* Delete dentries that have gone stale */ + /** @delete_stale: Delete dentries that have gone stale */ unsigned int delete_stale:1; - /** Do not create entry in fusectl fs */ + /** @no_control: Do not create entry in fusectl fs */ unsigned int no_control:1; - /** Do not allow MNT_FORCE umount */ + /** @no_force_umount: Do not allow MNT_FORCE umount */ unsigned int no_force_umount:1; - /* Auto-mount submounts announced by the server */ + /** @auto_submounts: Auto-mount submounts announced by the server */ unsigned int auto_submounts:1; - /* Propagate syncfs() to server */ + /** @sync_fs: Propagate syncfs() to server */ unsigned int sync_fs:1; - /* Initialize security xattrs when creating a new inode */ + /** @init_security: Initialize security xattrs when creating a new inode */ unsigned int init_security:1; - /* Add supplementary group info when creating a new inode */ + /** + * @create_supp_group: Add supplementary group info when creating + * a new inode + */ unsigned int create_supp_group:1; - /* Does the filesystem support per inode DAX? */ + /** @inode_dax: Does the filesystem support per inode DAX? */ unsigned int inode_dax:1; - /* Is tmpfile not implemented by fs? */ + /** @no_tmpfile: Is tmpfile not implemented by fs? */ unsigned int no_tmpfile:1; - /* Relax restrictions to allow shared mmap in FOPEN_DIRECT_IO mode */ + /** + * @direct_io_allow_mmap: Relax restrictions to allow shared mmap + * in FOPEN_DIRECT_IO mode + */ unsigned int direct_io_allow_mmap:1; - /* Is statx not implemented by fs? */ + /** @no_statx: Is statx not implemented by fs? */ unsigned int no_statx:1; - /** Passthrough support for read/write IO */ + /** @passthrough: Passthrough support for read/write IO */ unsigned int passthrough:1; - /* Use pages instead of pointer for kernel I/O */ + /** @use_pages_for_kvec_io: Use pages instead of pointer for kernel I/O */ unsigned int use_pages_for_kvec_io:1; - /* Is link not implemented by fs? */ + /** @no_link: Is link not implemented by fs? */ unsigned int no_link:1; - /* Is synchronous FUSE_INIT allowed? */ + /** @sync_init: Is synchronous FUSE_INIT allowed? */ unsigned int sync_init:1; - /* Use io_uring for communication */ - unsigned int io_uring; - - /** Maximum stack depth for passthrough backing files */ + /** @max_stack_depth: Maximum stack depth for passthrough backing files */ int max_stack_depth; - /** The number of requests waiting for completion */ - atomic_t num_waiting; - - /** Negotiated minor version */ + /** @minor: Negotiated minor version */ unsigned minor; - /** Entry on the fuse_conn_list */ + /** @entry: Entry on the fuse_conn_list */ struct list_head entry; - /** Device ID from the root super block */ + /** @dev: Device ID from the root super block */ dev_t dev; - /** Key for lock owner ID scrambling */ + /** @scramble_key: Key for lock owner ID scrambling */ u32 scramble_key[4]; - /** Version counter for attribute changes */ + /** @attr_version: Version counter for attribute changes */ atomic64_t attr_version; - /** Version counter for evict inode */ + /** @evict_ctr: Version counter for evict inode */ atomic64_t evict_ctr; - /* maximum file name length */ + /** @name_max: maximum file name length */ u32 name_max; - /** Called on final put */ + /** @release: Called on final put */ void (*release)(struct fuse_conn *); /** - * Read/write semaphore to hold when accessing the sb of any + * @killsb: Read/write semaphore to hold when accessing the sb of any * fuse_mount belonging to this connection */ struct rw_semaphore killsb; - /** List of device instances belonging to this connection */ - struct list_head devices; - #ifdef CONFIG_FUSE_DAX - /* Dax mode */ + /** @dax_mode: Dax mode */ enum fuse_dax_mode dax_mode; - /* Dax specific conn data, non-NULL if DAX is enabled */ + /** @dax: Dax specific conn data, non-NULL if DAX is enabled */ struct fuse_conn_dax *dax; #endif - /** List of filesystems using this connection */ + /** @mounts: List of filesystems using this connection */ struct list_head mounts; - /* New writepages go into this bucket */ + /** @curr_bucket: New writepages go into this bucket */ struct fuse_sync_bucket __rcu *curr_bucket; #ifdef CONFIG_FUSE_PASSTHROUGH - /** IDR for backing files ids */ + /** @backing_files_map: IDR for backing files ids */ struct idr backing_files_map; #endif - -#ifdef CONFIG_FUSE_IO_URING - /** uring connection information*/ - struct fuse_ring *ring; -#endif - - /** Only used if the connection opts into request timeouts */ - struct { - /* Worker for checking if any requests have timed out */ - struct delayed_work work; - - /* Request timeout (in jiffies). 0 = no timeout */ - unsigned int req_timeout; - } timeout; }; /* @@ -1136,7 +904,7 @@ extern const struct file_operations fuse_dev_operations; extern const struct dentry_operations fuse_dentry_operations; -/** +/* * Get a filled in inode */ struct inode *fuse_iget(struct super_block *sb, u64 nodeid, @@ -1147,14 +915,6 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid, int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, struct fuse_entry_out *outarg, struct inode **inode); -/** - * Send FORGET command - */ -void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, - u64 nodeid, u64 nlookup); - -struct fuse_forget_link *fuse_alloc_forget(void); - /* * Initialize READ or READDIR request */ @@ -1186,44 +946,44 @@ int fuse_finish_open(struct inode *inode, struct file *file); void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, unsigned int flags); -/** +/* * Send RELEASE or RELEASEDIR request */ void fuse_release_common(struct file *file, bool isdir); -/** +/* * Send FSYNC or FSYNCDIR request */ int fuse_fsync_common(struct file *file, loff_t start, loff_t end, int datasync, int opcode); -/** +/* * Notify poll wakeup */ int fuse_notify_poll_wakeup(struct fuse_conn *fc, struct fuse_notify_poll_wakeup_out *outarg); -/** +/* * Initialize file operations on a regular file */ void fuse_init_file_inode(struct inode *inode, unsigned int flags); -/** +/* * Initialize inode operations on regular files and special files */ void fuse_init_common(struct inode *inode); -/** +/* * Initialize inode and file operations on a directory */ void fuse_init_dir(struct inode *inode); -/** +/* * Initialize inode operations on a symlink */ void fuse_init_symlink(struct inode *inode); -/** +/* * Change attributes of an inode */ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, @@ -1237,20 +997,10 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, u32 fuse_get_cache_mask(struct inode *inode); -/** - * Initialize the client device - */ -int fuse_dev_init(void); - -/** - * Cleanup the client device - */ -void fuse_dev_cleanup(void); - int fuse_ctl_init(void); void __exit fuse_ctl_cleanup(void); -/** +/* * Simple request sending that does request allocation and freeing */ ssize_t __fuse_simple_request(struct mnt_idmap *idmap, @@ -1271,30 +1021,14 @@ static inline ssize_t fuse_simple_idmap_request(struct mnt_idmap *idmap, int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags); - -/** - * Assign a unique id to a fuse request - */ -void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req); - -/** - * End a finished request - */ -void fuse_request_end(struct fuse_req *req); - -/* Abort all requests */ -void fuse_abort_conn(struct fuse_conn *fc); -void fuse_wait_aborted(struct fuse_conn *fc); - -/* Check if any requests timed out */ -void fuse_check_timeout(struct work_struct *work); +int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique); void fuse_dentry_tree_init(void); void fuse_dentry_tree_cleanup(void); void fuse_epoch_work(struct work_struct *work); -/** +/* * Invalidate inode attributes */ @@ -1304,6 +1038,9 @@ void fuse_epoch_work(struct work_struct *work); /* Attributes possibly changed on data and/or size modification */ #define FUSE_STATX_MODSIZE (FUSE_STATX_MODIFY | STATX_SIZE) +/* Attributes possibly changed on directory modification */ +#define FUSE_STATX_MODDIR (FUSE_STATX_MODSIZE | STATX_NLINK) + void fuse_invalidate_attr(struct inode *inode); void fuse_invalidate_attr_mask(struct inode *inode, u32 mask); @@ -1317,45 +1054,26 @@ u64 fuse_time_to_jiffies(u64 sec, u32 nsec); void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); -/** - * Acquire reference to fuse_conn - */ -struct fuse_conn *fuse_conn_get(struct fuse_conn *fc); - -/** - * Initialize the fuse processing queue - */ -void fuse_pqueue_init(struct fuse_pqueue *fpq); - -/** +/* * Initialize fuse_conn */ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, - struct user_namespace *user_ns, - const struct fuse_iqueue_ops *fiq_ops, void *fiq_priv); + struct user_namespace *user_ns, struct fuse_chan *fch); -/** - * Release reference to fuse_conn - */ -void fuse_conn_put(struct fuse_conn *fc); - -struct fuse_dev *fuse_dev_alloc_install(struct fuse_conn *fc); -struct fuse_dev *fuse_dev_alloc(void); -void fuse_dev_install(struct fuse_dev *fud, struct fuse_conn *fc); -void fuse_dev_put(struct fuse_dev *fud); int fuse_send_init(struct fuse_mount *fm); /** - * Fill in superblock and initialize fuse connection + * fuse_fill_super_common - Fill in superblock and initialize fuse connection * @sb: partially-initialized superblock to fill in * @ctx: mount context */ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx); -/* - * Remove the mount from the connection +/** + * fuse_mount_remove - Remove the mount from the connection + * @fm: fuse_mount to remove * - * Returns whether this was the last mount + * Returns: whether this was the last mount */ bool fuse_mount_remove(struct fuse_mount *fm); @@ -1373,23 +1091,25 @@ void fuse_conn_destroy(struct fuse_mount *fm); void fuse_mount_destroy(struct fuse_mount *fm); /** - * Add connection to control filesystem + * fuse_ctl_add_conn - Add connection to control filesystem + * @fc: Fuse connection to add */ int fuse_ctl_add_conn(struct fuse_conn *fc); /** - * Remove connection from control filesystem + * fuse_ctl_remove_conn - Remove connection from control filesystem + * @fc: Fuse connection to remove */ void fuse_ctl_remove_conn(struct fuse_conn *fc); -/** +/* * Is file type valid? */ int fuse_valid_type(int m); bool fuse_invalid_attr(struct fuse_attr *attr); -/** +/* * Is current process allowed to perform filesystem operation? */ bool fuse_allow_current_process(struct fuse_conn *fc); @@ -1406,7 +1126,7 @@ void fuse_flush_writepages(struct inode *inode); void fuse_set_nowrite(struct inode *inode); void fuse_release_nowrite(struct inode *inode); -/** +/* * Scan all fuse_mounts belonging to fc to find the first where * ilookup5() returns a result. Return that result and the * respective fuse_mount in *fm (unless fm is NULL). @@ -1416,13 +1136,13 @@ void fuse_release_nowrite(struct inode *inode); struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid, struct fuse_mount **fm); -/** +/* * File-system tells the kernel to invalidate cache for the given node id. */ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len); -/** +/* * File-system tells the kernel to invalidate parent attributes and * the dentry matching parent/name. * @@ -1444,7 +1164,7 @@ void fuse_try_prune_one_inode(struct fuse_conn *fc, u64 nodeid); int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, bool isdir); -/** +/* * fuse_direct_io() flags */ @@ -1461,7 +1181,6 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, long fuse_ioctl_common(struct file *file, unsigned int cmd, unsigned long arg, unsigned int flags); __poll_t fuse_file_poll(struct file *file, poll_table *wait); -int fuse_dev_release(struct inode *inode, struct file *file); bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written); @@ -1471,8 +1190,6 @@ int fuse_write_inode(struct inode *inode, struct writeback_control *wbc); int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr, struct file *file); -void fuse_set_initialized(struct fuse_conn *fc); - void fuse_unlock_inode(struct inode *inode, bool locked); bool fuse_lock_inode(struct inode *inode); @@ -1494,15 +1211,6 @@ int fuse_set_acl(struct mnt_idmap *, struct dentry *dentry, /* readdir.c */ int fuse_readdir(struct file *file, struct dir_context *ctx); -/** - * Return the number of bytes in an arguments list - */ -unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args); - -/** - * Get the next unique ID for a request - */ -u64 fuse_get_unique(struct fuse_iqueue *fiq); void fuse_free_conn(struct fuse_conn *fc); /* dax.c */ @@ -1570,8 +1278,6 @@ static inline struct fuse_backing *fuse_backing_lookup(struct fuse_conn *fc, void fuse_backing_files_init(struct fuse_conn *fc); void fuse_backing_files_free(struct fuse_conn *fc); -int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map); -int fuse_backing_close(struct fuse_conn *fc, int backing_id); /* passthrough.c */ static inline struct fuse_backing *fuse_inode_backing(struct fuse_inode *fi) diff --git a/fs/fuse/fuse_trace.h b/fs/fuse/fuse_trace.h index bbe9ddd8c716..60baa10bbcb9 100644 --- a/fs/fuse/fuse_trace.h +++ b/fs/fuse/fuse_trace.h @@ -90,7 +90,7 @@ TRACE_EVENT(fuse_request_send, ), TP_fast_assign( - __entry->connection = req->fm->fc->dev; + __entry->connection = req->chan->conn->dev; __entry->unique = req->in.h.unique; __entry->opcode = req->in.h.opcode; __entry->len = req->in.h.len; @@ -101,6 +101,28 @@ TRACE_EVENT(fuse_request_send, __print_symbolic(__entry->opcode, OPCODES), __entry->len) ); +TRACE_EVENT(fuse_request_sent, + TP_PROTO(const struct fuse_req *req), + + TP_ARGS(req), + + TP_STRUCT__entry( + __field(dev_t, connection) + __field(uint64_t, unique) + __field(enum fuse_opcode, opcode) + ), + + TP_fast_assign( + __entry->connection = req->chan->conn->dev; + __entry->unique = req->in.h.unique; + __entry->opcode = req->in.h.opcode; + ), + + TP_printk("connection %u req %llu opcode %u (%s)", + __entry->connection, __entry->unique, __entry->opcode, + __print_symbolic(__entry->opcode, OPCODES)) +); + TRACE_EVENT(fuse_request_end, TP_PROTO(const struct fuse_req *req), @@ -114,7 +136,7 @@ TRACE_EVENT(fuse_request_end, ), TP_fast_assign( - __entry->connection = req->fm->fc->dev; + __entry->connection = req->chan->conn->dev; __entry->unique = req->in.h.unique; __entry->len = req->out.h.len; __entry->error = req->out.h.error; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index deddfffb037f..d975073c6029 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1,14 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ +#include "dev.h" #include "fuse_i.h" -#include "fuse_dev_i.h" -#include "dev_uring_i.h" #include <linux/dax.h> #include <linux/pagemap.h> @@ -35,14 +32,11 @@ MODULE_LICENSE("GPL"); static struct kmem_cache *fuse_inode_cachep; struct list_head fuse_conn_list; DEFINE_MUTEX(fuse_mutex); -DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq); static int set_global_limit(const char *val, const struct kernel_param *kp); unsigned int fuse_max_pages_limit = 256; /* default is no timeout */ -unsigned int fuse_default_req_timeout; -unsigned int fuse_max_req_timeout; unsigned int max_user_bgreq; module_param_call(max_user_bgreq, set_global_limit, param_get_uint, @@ -62,9 +56,6 @@ MODULE_PARM_DESC(max_user_congthresh, #define FUSE_DEFAULT_BLKSIZE 512 -/** Maximum number of outstanding background requests */ -#define FUSE_DEFAULT_MAX_BACKGROUND 12 - /** Congestion starts at 75% of maximum */ #define FUSE_DEFAULT_CONGESTION_THRESHOLD (FUSE_DEFAULT_MAX_BACKGROUND * 3 / 4) @@ -72,11 +63,6 @@ MODULE_PARM_DESC(max_user_congthresh, static struct file_system_type fuseblk_fs_type; #endif -struct fuse_forget_link *fuse_alloc_forget(void) -{ - return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT); -} - static struct fuse_submount_lookup *fuse_alloc_submount_lookup(void) { struct fuse_submount_lookup *sl; @@ -150,7 +136,7 @@ static void fuse_cleanup_submount_lookup(struct fuse_conn *fc, if (!refcount_dec_and_test(&sl->count)) return; - fuse_queue_forget(fc, sl->forget, sl->nodeid, 1); + fuse_chan_queue_forget(fc->chan, sl->forget, sl->nodeid, 1); sl->forget = NULL; kfree(sl); } @@ -173,8 +159,8 @@ static void fuse_evict_inode(struct inode *inode) if (FUSE_IS_DAX(inode)) fuse_dax_inode_cleanup(inode); if (fi->nlookup) { - fuse_queue_forget(fc, fi->forget, fi->nodeid, - fi->nlookup); + fuse_chan_queue_forget(fc->chan, fi->forget, fi->nodeid, + fi->nlookup); fi->forget = NULL; } @@ -624,7 +610,7 @@ static void fuse_umount_begin(struct super_block *sb) if (fc->no_force_umount) return; - fuse_abort_conn(fc); + fuse_chan_abort(fc->chan, false); // Only retire block-device-based superblocks. if (sb->s_bdev != NULL) @@ -688,11 +674,9 @@ static struct fuse_sync_bucket *fuse_sync_bucket_alloc(void) struct fuse_sync_bucket *bucket; bucket = kzalloc_obj(*bucket, GFP_KERNEL | __GFP_NOFAIL); - if (bucket) { - init_waitqueue_head(&bucket->waitq); - /* Initial active count */ - atomic_set(&bucket->count, 1); - } + init_waitqueue_head(&bucket->waitq); + /* Initial active count */ + atomic_set(&bucket->count, 1); return bucket; } @@ -816,8 +800,7 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file) if (file->f_cred->user_ns != fsc->user_ns) return invalfc(fsc, "wrong user namespace for fuse device"); - ctx->fud = file->private_data; - refcount_inc(&ctx->fud->ref); + ctx->fud = fuse_dev_grab(file); return 0; } @@ -970,56 +953,19 @@ static int fuse_show_options(struct seq_file *m, struct dentry *root) return 0; } -static void fuse_iqueue_init(struct fuse_iqueue *fiq, - const struct fuse_iqueue_ops *ops, - void *priv) -{ - memset(fiq, 0, sizeof(struct fuse_iqueue)); - spin_lock_init(&fiq->lock); - init_waitqueue_head(&fiq->waitq); - INIT_LIST_HEAD(&fiq->pending); - INIT_LIST_HEAD(&fiq->interrupts); - fiq->forget_list_tail = &fiq->forget_list_head; - fiq->connected = 1; - fiq->ops = ops; - fiq->priv = priv; -} - -void fuse_pqueue_init(struct fuse_pqueue *fpq) -{ - unsigned int i; - - spin_lock_init(&fpq->lock); - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - INIT_LIST_HEAD(&fpq->processing[i]); - INIT_LIST_HEAD(&fpq->io); - fpq->connected = 1; -} - void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, - struct user_namespace *user_ns, - const struct fuse_iqueue_ops *fiq_ops, void *fiq_priv) + struct user_namespace *user_ns, struct fuse_chan *fch) { memset(fc, 0, sizeof(*fc)); spin_lock_init(&fc->lock); - spin_lock_init(&fc->bg_lock); init_rwsem(&fc->killsb); refcount_set(&fc->count, 1); atomic_set(&fc->epoch, 1); INIT_WORK(&fc->epoch_work, fuse_epoch_work); - init_waitqueue_head(&fc->blocked_waitq); - fuse_iqueue_init(&fc->iq, fiq_ops, fiq_priv); - INIT_LIST_HEAD(&fc->bg_queue); INIT_LIST_HEAD(&fc->entry); - INIT_LIST_HEAD(&fc->devices); - atomic_set(&fc->num_waiting, 0); - fc->max_background = FUSE_DEFAULT_MAX_BACKGROUND; fc->congestion_threshold = FUSE_DEFAULT_CONGESTION_THRESHOLD; atomic64_set(&fc->khctr, 0); fc->polled_files = RB_ROOT; - fc->blocked = 0; - fc->initialized = 0; - fc->connected = 1; atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); @@ -1028,7 +974,6 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ; fc->max_pages_limit = fuse_max_pages_limit; fc->name_max = FUSE_NAME_LOW_MAX; - fc->timeout.req_timeout = 0; if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_init(fc); @@ -1036,6 +981,8 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, INIT_LIST_HEAD(&fc->mounts); list_add(&fm->fc_entry, &fc->mounts); fm->fc = fc; + fuse_chan_set_fc(fch, fc); + fc->chan = fch; } EXPORT_SYMBOL_GPL(fuse_conn_init); @@ -1043,7 +990,8 @@ static void delayed_release(struct rcu_head *p) { struct fuse_conn *fc = container_of(p, struct fuse_conn, rcu); - fuse_uring_destruct(fc); + fuse_uring_destruct(fc->chan); + fuse_chan_free(fc->chan); put_user_ns(fc->user_ns); fc->release(fc); @@ -1051,7 +999,6 @@ static void delayed_release(struct rcu_head *p) void fuse_conn_put(struct fuse_conn *fc) { - struct fuse_iqueue *fiq = &fc->iq; struct fuse_sync_bucket *bucket; if (!refcount_dec_and_test(&fc->count)) @@ -1059,11 +1006,8 @@ void fuse_conn_put(struct fuse_conn *fc) if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_conn_free(fc); - if (fc->timeout.req_timeout) - cancel_delayed_work_sync(&fc->timeout.work); cancel_work_sync(&fc->epoch_work); - if (fiq->ops->release) - fiq->ops->release(fiq); + fuse_chan_release(fc->chan); put_pid_ns(fc->pid_ns); bucket = rcu_dereference_protected(fc->curr_bucket, 1); if (bucket) { @@ -1083,6 +1027,11 @@ struct fuse_conn *fuse_conn_get(struct fuse_conn *fc) } EXPORT_SYMBOL_GPL(fuse_conn_get); +dev_t fuse_conn_get_id(struct fuse_conn *fc) +{ + return fc->dev; +} + static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned int mode) { struct fuse_attr attr; @@ -1113,12 +1062,11 @@ static struct dentry *fuse_get_dentry(struct super_block *sb, inode = ilookup5(sb, handle->nodeid, fuse_inode_eq, &handle->nodeid); if (!inode) { struct fuse_entry_out outarg; - const struct qstr name = QSTR_INIT(".", 1); if (!fc->export_support) goto out_err; - err = fuse_lookup_name(sb, handle->nodeid, &name, &outarg, + err = fuse_lookup_name(sb, handle->nodeid, &QSTR("."), &outarg, &inode); if (err && err != -ENOENT) goto out_err; @@ -1294,12 +1242,13 @@ static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg) sanitize_global_limit(&max_user_bgreq); sanitize_global_limit(&max_user_congthresh); - spin_lock(&fc->bg_lock); if (arg->max_background) { - fc->max_background = arg->max_background; + unsigned int max_background = arg->max_background; + + if (!cap_sys_admin && max_background > max_user_bgreq) + max_background = max_user_bgreq; - if (!cap_sys_admin && fc->max_background > max_user_bgreq) - fc->max_background = max_user_bgreq; + fuse_chan_max_background_set(fc->chan, max_background); } if (arg->congestion_threshold) { fc->congestion_threshold = arg->congestion_threshold; @@ -1308,48 +1257,20 @@ static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg) fc->congestion_threshold > max_user_congthresh) fc->congestion_threshold = max_user_congthresh; } - spin_unlock(&fc->bg_lock); -} - -static void set_request_timeout(struct fuse_conn *fc, unsigned int timeout) -{ - fc->timeout.req_timeout = secs_to_jiffies(timeout); - INIT_DELAYED_WORK(&fc->timeout.work, fuse_check_timeout); - queue_delayed_work(system_percpu_wq, &fc->timeout.work, - fuse_timeout_timer_freq); -} - -static void init_server_timeout(struct fuse_conn *fc, unsigned int timeout) -{ - if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout) - return; - - if (!timeout) - timeout = fuse_default_req_timeout; - - if (fuse_max_req_timeout) { - if (timeout) - timeout = min(fuse_max_req_timeout, timeout); - else - timeout = fuse_max_req_timeout; - } - - timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout); - - set_request_timeout(fc, timeout); } struct fuse_init_args { struct fuse_args args; struct fuse_init_in in; struct fuse_init_out out; + struct fuse_mount *fm; }; -static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, - int error) +static void process_init_reply(struct fuse_args *args, int error) { - struct fuse_conn *fc = fm->fc; struct fuse_init_args *ia = container_of(args, typeof(*ia), args); + struct fuse_mount *fm = ia->fm; + struct fuse_conn *fc = fm->fc; struct fuse_init_out *arg = &ia->out; bool ok = true; @@ -1481,7 +1402,7 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, ok = false; } if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled()) - fc->io_uring = 1; + fuse_chan_io_uring_enable(fc->chan); if (flags & FUSE_REQUEST_TIMEOUT) timeout = arg->request_timeout; @@ -1491,7 +1412,7 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, fc->no_flock = 1; } - init_server_timeout(fc, timeout); + fuse_init_server_timeout(fc->chan, timeout); fm->sb->s_bdi->ra_pages = min(fm->sb->s_bdi->ra_pages, ra_pages); @@ -1505,10 +1426,15 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, if (!ok) { fc->conn_init = 0; fc->conn_error = 1; + fuse_chan_set_initialized(fc->chan, NULL); + } else { + struct fuse_chan_param cp = { + .minor = fc->minor, + .max_write = fc->max_write, + .max_pages = fc->max_pages, + }; + fuse_chan_set_initialized(fc->chan, &cp); } - - fuse_set_initialized(fc); - wake_up_all(&fc->blocked_waitq); } static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) @@ -1518,6 +1444,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) ia = kzalloc_obj(*ia, GFP_KERNEL | __GFP_NOFAIL); + ia->fm = fm; ia->in.major = FUSE_KERNEL_VERSION; ia->in.minor = FUSE_KERNEL_MINOR_VERSION; ia->in.max_readahead = fm->sb->s_bdi->ra_pages * PAGE_SIZE; @@ -1591,7 +1518,7 @@ int fuse_send_init(struct fuse_mount *fm) if (!err) return 0; } - process_init_reply(fm, &ia->args, err); + process_init_reply(&ia->args, err); if (fm->fc->conn_error) return -ENOTCONN; return 0; @@ -1600,7 +1527,6 @@ EXPORT_SYMBOL_GPL(fuse_send_init); void fuse_free_conn(struct fuse_conn *fc) { - WARN_ON(!list_empty(&fc->devices)); kfree(fc); } EXPORT_SYMBOL_GPL(fuse_free_conn); @@ -1643,89 +1569,6 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) return 0; } -struct fuse_dev *fuse_dev_alloc(void) -{ - struct fuse_dev *fud; - struct list_head *pq; - - fud = kzalloc_obj(struct fuse_dev); - if (!fud) - return NULL; - - refcount_set(&fud->ref, 1); - pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE); - if (!pq) { - kfree(fud); - return NULL; - } - - fud->pq.processing = pq; - fuse_pqueue_init(&fud->pq); - - return fud; -} -EXPORT_SYMBOL_GPL(fuse_dev_alloc); - -void fuse_dev_install(struct fuse_dev *fud, struct fuse_conn *fc) -{ - struct fuse_conn *old_fc; - - spin_lock(&fc->lock); - /* - * Pairs with: - * - xchg() in fuse_dev_release() - * - smp_load_acquire() in fuse_dev_fc_get() - */ - old_fc = cmpxchg(&fud->fc, NULL, fc); - if (old_fc) { - /* - * failed to set fud->fc because - * - it was already set to a different fc - * - it was set to disconneted - */ - fc->connected = 0; - } else { - list_add_tail(&fud->entry, &fc->devices); - fuse_conn_get(fc); - } - spin_unlock(&fc->lock); -} -EXPORT_SYMBOL_GPL(fuse_dev_install); - -struct fuse_dev *fuse_dev_alloc_install(struct fuse_conn *fc) -{ - struct fuse_dev *fud; - - fud = fuse_dev_alloc(); - if (!fud) - return NULL; - - fuse_dev_install(fud, fc); - return fud; -} -EXPORT_SYMBOL_GPL(fuse_dev_alloc_install); - -void fuse_dev_put(struct fuse_dev *fud) -{ - struct fuse_conn *fc; - - if (!refcount_dec_and_test(&fud->ref)) - return; - - fc = fuse_dev_fc_get(fud); - if (fc && fc != FUSE_DEV_FC_DISCONNECTED) { - /* This is the virtiofs case (fuse_dev_release() not called) */ - spin_lock(&fc->lock); - list_del(&fud->entry); - spin_unlock(&fc->lock); - - fuse_conn_put(fc); - } - kfree(fud->pq.processing); - kfree(fud); -} -EXPORT_SYMBOL_GPL(fuse_dev_put); - static void fuse_fill_attr_from_inode(struct fuse_attr *attr, const struct fuse_inode *fi) { @@ -1941,9 +1784,9 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) mutex_lock(&fuse_mutex); err = -EINVAL; if (fud) { - if (fuse_dev_fc_get(fud)) + if (fuse_dev_is_installed(fud)) goto err_unlock; - if (fud->sync_init) + if (fuse_dev_is_sync_init(fud)) fc->sync_init = 1; } @@ -1953,10 +1796,9 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) list_add_tail(&fc->entry, &fuse_conn_list); sb->s_root = root_dentry; - if (fud) { - fuse_dev_install(fud, fc); - wake_up_all(&fuse_dev_waitq); - } + if (fud) + fuse_dev_install(fud, fc->chan); + mutex_unlock(&fuse_mutex); return 0; @@ -2001,9 +1843,7 @@ static int fuse_set_no_super(struct super_block *sb, struct fs_context *fsc) static int fuse_test_super(struct super_block *sb, struct fs_context *fsc) { - struct fuse_dev *fud = fsc->sget_key; - - return fuse_dev_fc_get(fud) == get_fuse_conn_super(sb); + return fuse_dev_verify(fsc->sget_key, get_fuse_conn_super(sb)->chan); } static int fuse_get_tree(struct fs_context *fsc) @@ -2012,8 +1852,12 @@ static int fuse_get_tree(struct fs_context *fsc) struct fuse_conn *fc; struct fuse_mount *fm; struct super_block *sb; + struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new(); int err; + if (!fch) + return -ENOMEM; + fc = kmalloc_obj(*fc); if (!fc) return -ENOMEM; @@ -2024,7 +1868,7 @@ static int fuse_get_tree(struct fs_context *fsc) return -ENOMEM; } - fuse_conn_init(fc, fm, fsc->user_ns, &fuse_dev_fiq_ops, NULL); + fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch)); fc->release = fuse_free_conn; fsc->s_fs_info = fm; @@ -2045,7 +1889,7 @@ static int fuse_get_tree(struct fs_context *fsc) * Allow creating a fuse mount with an already initialized fuse * connection */ - if (fuse_dev_fc_get(ctx->fud)) { + if (fuse_dev_is_installed(ctx->fud)) { fsc->sget_key = ctx->fud; sb = sget_fc(fsc, fuse_test_super, fuse_set_no_super); err = PTR_ERR_OR_ZERO(sb); @@ -2116,8 +1960,8 @@ void fuse_conn_destroy(struct fuse_mount *fm) if (fc->destroy) fuse_send_destroy(fm); - fuse_abort_conn(fc); - fuse_wait_aborted(fc); + fuse_chan_abort(fc->chan, false); + fuse_chan_wait_aborted(fc->chan); if (!list_empty(&fc->entry)) { mutex_lock(&fuse_mutex); diff --git a/fs/fuse/ioctl.c b/fs/fuse/ioctl.c index fdc175e93f74..3614ea603913 100644 --- a/fs/fuse/ioctl.c +++ b/fs/fuse/ioctl.c @@ -10,6 +10,7 @@ #include <linux/fileattr.h> #include <linux/fsverity.h> +#include <linux/slab.h> #define FUSE_VERITY_ENABLE_ARG_MAX_PAGES 256 static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args, @@ -252,7 +253,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, err = -ENOMEM; ap.folios = fuse_folios_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); - iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); + iov_page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!ap.folios || !iov_page) goto out; @@ -400,7 +401,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, } err = 0; out: - free_page((unsigned long) iov_page); + kfree(iov_page); while (ap.num_folios) folio_put(ap.folios[--ap.num_folios]); kfree(ap.folios); diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c new file mode 100644 index 000000000000..29578104ae6c --- /dev/null +++ b/fs/fuse/notify.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "fuse_i.h" +#include <linux/pagemap.h> + +static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_poll_wakeup_out outarg; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + fuse_copy_finish(cs); + return fuse_notify_poll_wakeup(fc, &outarg); +} + +static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_inval_inode_out outarg; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + fuse_copy_finish(cs); + + down_read(&fc->killsb); + err = fuse_reverse_inval_inode(fc, outarg.ino, + outarg.off, outarg.len); + up_read(&fc->killsb); + return err; +} + +static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_inval_entry_out outarg; + int err; + char *buf; + struct qstr name; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (outarg.namelen > fc->name_max) + return -ENAMETOOLONG; + + err = -EINVAL; + if (size != sizeof(outarg) + outarg.namelen + 1) + return -EINVAL; + + buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + name.name = buf; + name.len = outarg.namelen; + err = fuse_copy_one(cs, buf, outarg.namelen + 1); + if (err) + goto err; + fuse_copy_finish(cs); + buf[outarg.namelen] = 0; + + down_read(&fc->killsb); + err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags); + up_read(&fc->killsb); +err: + kfree(buf); + return err; +} + +static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_delete_out outarg; + int err; + char *buf; + struct qstr name; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (outarg.namelen > fc->name_max) + return -ENAMETOOLONG; + + if (size != sizeof(outarg) + outarg.namelen + 1) + return -EINVAL; + + buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + name.name = buf; + name.len = outarg.namelen; + err = fuse_copy_one(cs, buf, outarg.namelen + 1); + if (err) + goto err; + fuse_copy_finish(cs); + buf[outarg.namelen] = 0; + + down_read(&fc->killsb); + err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0); + up_read(&fc->killsb); +err: + kfree(buf); + return err; +} + +static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_store_out outarg; + struct inode *inode; + struct address_space *mapping; + u64 nodeid; + int err; + unsigned int num; + loff_t file_size; + loff_t pos; + loff_t end; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (size - sizeof(outarg) != outarg.size) + return -EINVAL; + + if (outarg.offset >= MAX_LFS_FILESIZE) + return -EINVAL; + + nodeid = outarg.nodeid; + pos = outarg.offset; + num = min(outarg.size, MAX_LFS_FILESIZE - pos); + + down_read(&fc->killsb); + + err = -ENOENT; + inode = fuse_ilookup(fc, nodeid, NULL); + if (!inode) + goto out_up_killsb; + if (!S_ISREG(inode->i_mode)) { + err = -EINVAL; + goto out_iput; + } + + mapping = inode->i_mapping; + file_size = i_size_read(inode); + end = pos + num; + if (end > file_size) { + file_size = end; + fuse_write_update_attr(inode, file_size, num); + } + + while (num) { + struct folio *folio; + unsigned int folio_offset; + unsigned int nr_bytes; + pgoff_t index = pos >> PAGE_SHIFT; + + folio = filemap_grab_folio(mapping, index); + err = PTR_ERR(folio); + if (IS_ERR(folio)) + goto out_iput; + + folio_offset = offset_in_folio(folio, pos); + nr_bytes = min(num, folio_size(folio) - folio_offset); + + err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0); + if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && + (nr_bytes == folio_size(folio) || file_size == end)) { + folio_zero_segment(folio, nr_bytes, folio_size(folio)); + folio_mark_uptodate(folio); + } + folio_unlock(folio); + folio_put(folio); + + if (err) + goto out_iput; + + pos += nr_bytes; + num -= nr_bytes; + } + + err = 0; + +out_iput: + iput(inode); +out_up_killsb: + up_read(&fc->killsb); + return err; +} + +struct fuse_retrieve_args { + struct fuse_args_pages ap; + struct fuse_notify_retrieve_in inarg; +}; + +static void fuse_retrieve_end(struct fuse_args *args, int error) +{ + struct fuse_retrieve_args *ra = + container_of(args, typeof(*ra), ap.args); + + release_pages(ra->ap.folios, ra->ap.num_folios); + kfree(ra); +} + +static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, + struct fuse_notify_retrieve_out *outarg) +{ + int err; + struct address_space *mapping = inode->i_mapping; + loff_t file_size; + unsigned int num; + unsigned int offset; + size_t total_len = 0; + unsigned int num_pages; + struct fuse_conn *fc = fm->fc; + struct fuse_retrieve_args *ra; + size_t args_size = sizeof(*ra); + struct fuse_args_pages *ap; + struct fuse_args *args; + loff_t pos = outarg->offset; + + offset = offset_in_page(pos); + file_size = i_size_read(inode); + + num = min(outarg->size, fc->max_write); + if (pos > file_size) + num = 0; + else if (num > file_size - pos) + num = file_size - pos; + + num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE); + num_pages = min(num_pages, fc->max_pages); + num = min(num, num_pages << PAGE_SHIFT); + + args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); + + ra = kzalloc(args_size, GFP_KERNEL); + if (!ra) + return -ENOMEM; + + ap = &ra->ap; + ap->folios = (void *) (ra + 1); + ap->descs = (void *) (ap->folios + num_pages); + + args = &ap->args; + args->nodeid = outarg->nodeid; + args->opcode = FUSE_NOTIFY_REPLY; + args->in_numargs = 3; + args->in_pages = true; + args->end = fuse_retrieve_end; + + while (num && ap->num_folios < num_pages) { + struct folio *folio; + unsigned int folio_offset; + unsigned int nr_bytes; + pgoff_t index = pos >> PAGE_SHIFT; + + folio = filemap_get_folio(mapping, index); + if (IS_ERR(folio)) + break; + if (!folio_test_uptodate(folio)) { + folio_put(folio); + break; + } + + folio_offset = offset_in_folio(folio, pos); + nr_bytes = min(folio_size(folio) - folio_offset, num); + + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].offset = folio_offset; + ap->descs[ap->num_folios].length = nr_bytes; + ap->num_folios++; + + pos += nr_bytes; + num -= nr_bytes; + total_len += nr_bytes; + } + ra->inarg.offset = outarg->offset; + ra->inarg.size = total_len; + fuse_set_zero_arg0(args); + args->in_args[1].size = sizeof(ra->inarg); + args->in_args[1].value = &ra->inarg; + args->in_args[2].size = total_len; + + err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); + if (err) + fuse_retrieve_end(args, err); + + return err; +} + +static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_retrieve_out outarg; + struct fuse_mount *fm; + struct inode *inode; + u64 nodeid; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + fuse_copy_finish(cs); + + if (outarg.offset >= MAX_LFS_FILESIZE) + return -EINVAL; + + down_read(&fc->killsb); + err = -ENOENT; + nodeid = outarg.nodeid; + + inode = fuse_ilookup(fc, nodeid, &fm); + if (inode) { + err = -EINVAL; + if (S_ISREG(inode->i_mode)) + err = fuse_retrieve(fm, inode, &outarg); + iput(inode); + } + up_read(&fc->killsb); + + return err; +} + +static int fuse_notify_resend(struct fuse_conn *fc) +{ + fuse_chan_resend(fc->chan); + return 0; +} + +/* + * Increments the fuse connection epoch. This will cause dentries and + * readdir caches from previous epochs to be invalidated. Additionally, + * if inval_wq is set, a work queue is scheduled to trigger the invalidation. + */ +static int fuse_notify_inc_epoch(struct fuse_conn *fc) +{ + atomic_inc(&fc->epoch); + if (inval_wq) + schedule_work(&fc->epoch_work); + + return 0; +} + +static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_prune_out outarg; + const unsigned int batch = 512; + u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL); + unsigned int num, i; + int err; + + if (!nodeids) + return -ENOMEM; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (size - sizeof(outarg) != array_size(outarg.count, sizeof(u64))) + return -EINVAL; + + for (; outarg.count; outarg.count -= num) { + num = min(batch, outarg.count); + err = fuse_copy_one(cs, nodeids, num * sizeof(u64)); + if (err) + return err; + + scoped_guard(rwsem_read, &fc->killsb) { + for (i = 0; i < num; i++) + fuse_try_prune_one_inode(fc, nodeids[i]); + } + } + return 0; +} + +int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, + unsigned int size, struct fuse_copy_state *cs) +{ + switch (code) { + case FUSE_NOTIFY_POLL: + return fuse_notify_poll(fc, size, cs); + + case FUSE_NOTIFY_INVAL_INODE: + return fuse_notify_inval_inode(fc, size, cs); + + case FUSE_NOTIFY_INVAL_ENTRY: + return fuse_notify_inval_entry(fc, size, cs); + + case FUSE_NOTIFY_STORE: + return fuse_notify_store(fc, size, cs); + + case FUSE_NOTIFY_RETRIEVE: + return fuse_notify_retrieve(fc, size, cs); + + case FUSE_NOTIFY_DELETE: + return fuse_notify_delete(fc, size, cs); + + case FUSE_NOTIFY_RESEND: + return fuse_notify_resend(fc); + + case FUSE_NOTIFY_INC_EPOCH: + return fuse_notify_inc_epoch(fc); + + case FUSE_NOTIFY_PRUNE: + return fuse_notify_prune(fc, size, cs); + + default: + return -EINVAL; + } +} diff --git a/fs/fuse/poll.c b/fs/fuse/poll.c new file mode 100644 index 000000000000..bce3ee2e861e --- /dev/null +++ b/fs/fuse/poll.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "fuse_i.h" + +void fuse_end_polls(struct fuse_conn *fc) +{ + struct rb_node *p; + + spin_lock(&fc->lock); + p = rb_first(&fc->polled_files); + + while (p) { + struct fuse_file *ff; + ff = rb_entry(p, struct fuse_file, polled_node); + wake_up_interruptible_all(&ff->poll_wait); + + p = rb_next(p); + } + spin_unlock(&fc->lock); +} + +/* + * All files which have been polled are linked to RB tree + * fuse_conn->polled_files which is indexed by kh. Walk the tree and + * find the matching one. + */ +static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh, + struct rb_node **parent_out) +{ + struct rb_node **link = &fc->polled_files.rb_node; + struct rb_node *last = NULL; + + while (*link) { + struct fuse_file *ff; + + last = *link; + ff = rb_entry(last, struct fuse_file, polled_node); + + if (kh < ff->kh) + link = &last->rb_left; + else if (kh > ff->kh) + link = &last->rb_right; + else + return link; + } + + if (parent_out) + *parent_out = last; + return link; +} + +/* + * The file is about to be polled. Make sure it's on the polled_files + * RB tree. Note that files once added to the polled_files tree are + * not removed before the file is released. This is because a file + * polled once is likely to be polled again. + */ +static void fuse_register_polled_file(struct fuse_conn *fc, + struct fuse_file *ff) +{ + spin_lock(&fc->lock); + if (RB_EMPTY_NODE(&ff->polled_node)) { + struct rb_node **link, *parent; + + link = fuse_find_polled_node(fc, ff->kh, &parent); + BUG_ON(*link); + rb_link_node(&ff->polled_node, parent, link); + rb_insert_color(&ff->polled_node, &fc->polled_files); + } + spin_unlock(&fc->lock); +} + +__poll_t fuse_file_poll(struct file *file, poll_table *wait) +{ + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; + struct fuse_poll_out outarg; + FUSE_ARGS(args); + int err; + + if (fm->fc->no_poll) + return DEFAULT_POLLMASK; + + poll_wait(file, &ff->poll_wait, wait); + inarg.events = mangle_poll(poll_requested_events(wait)); + + /* + * Ask for notification iff there's someone waiting for it. + * The client may ignore the flag and always notify. + */ + if (waitqueue_active(&ff->poll_wait)) { + inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; + fuse_register_polled_file(fm->fc, ff); + } + + args.opcode = FUSE_POLL; + args.nodeid = ff->nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + + if (!err) + return demangle_poll(outarg.revents); + if (err == -ENOSYS) { + fm->fc->no_poll = 1; + return DEFAULT_POLLMASK; + } + return EPOLLERR; +} +EXPORT_SYMBOL_GPL(fuse_file_poll); + +/* + * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and + * wakes up the poll waiters. + */ +int fuse_notify_poll_wakeup(struct fuse_conn *fc, + struct fuse_notify_poll_wakeup_out *outarg) +{ + u64 kh = outarg->kh; + struct rb_node **link; + + spin_lock(&fc->lock); + + link = fuse_find_polled_node(fc, kh, NULL); + if (*link) { + struct fuse_file *ff; + + ff = rb_entry(*link, struct fuse_file, polled_node); + wake_up_interruptible_sync(&ff->poll_wait); + } + + spin_unlock(&fc->lock); + return 0; +} + diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index db5ae8ec1030..0e1321491747 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -1,9 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2018 Miklos Szeredi <miklos@szeredi.hu> - - This program can be distributed under the terms of the GNU GPL. - See the file COPYING. */ @@ -12,6 +10,7 @@ #include <linux/posix_acl.h> #include <linux/pagemap.h> #include <linux/highmem.h> +#include <linux/vmalloc.h> static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) { @@ -164,7 +163,6 @@ static int fuse_direntplus_link(struct file *file, struct inode *dir = d_inode(parent); struct fuse_conn *fc; struct inode *inode; - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); int epoch; if (!o->nodeid) { @@ -201,7 +199,7 @@ static int fuse_direntplus_link(struct file *file, dentry = d_lookup(parent, &name); if (!dentry) { retry: - dentry = d_alloc_parallel(parent, &name, &wq); + dentry = d_alloc_parallel(parent, &name); if (IS_ERR(dentry)) return PTR_ERR(dentry); } @@ -335,6 +333,43 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, return 0; } +static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize) +{ + unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE); + struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + + if (!pages) + return NULL; + + nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages); + if (!nr_alloc) + goto free_array; + + if (nr_alloc < nr_pages) { + nr_pages = nr_alloc; + *bufsize = (size_t) nr_pages << PAGE_SHIFT; + } + + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); + if (!ap->folios) + goto release_pages; + + for (i = 0; i < nr_pages; i++) { + ap->folios[i] = page_folio(pages[i]); + ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE); + } + ap->num_folios = nr_pages; + ap->args.out_pages = true; + + return pages; + +release_pages: + release_pages(pages, nr_pages); +free_array: + kfree(pages); + return NULL; +} + static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) { int plus; @@ -343,18 +378,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_conn *fc = fm->fc; struct fuse_io_args ia = {}; - struct fuse_args *args = &ia.ap.args; + struct fuse_args_pages *ap = &ia.ap; void *buf; size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT); u64 attr_version = 0, evict_ctr = 0; bool locked; + struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize); - buf = kvmalloc(bufsize, GFP_KERNEL); - if (!buf) + if (!pages) return -ENOMEM; - args->out_args[0].value = buf; - plus = fuse_use_readdirplus(inode, ctx); if (plus) { attr_version = fuse_get_attr_version(fm->fc); @@ -364,24 +397,37 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR); } locked = fuse_lock_inode(inode); - res = fuse_simple_request(fm, args); + res = fuse_simple_request(fm, &ap->args); fuse_unlock_inode(inode, locked); - if (res >= 0) { - if (!res) { - struct fuse_file *ff = file->private_data; - - if (ff->open_flags & FOPEN_CACHE_DIR) - fuse_readdir_cache_end(file, ctx->pos); - } else if (plus) { - res = parse_dirplusfile(buf, res, file, ctx, attr_version, - evict_ctr); - } else { + if (res < 0) + goto out; + + if (!res) { + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_readdir_cache_end(file, ctx->pos); + goto out; + } + + buf = vm_map_ram(pages, ap->num_folios, -1); + if (!buf) { + res = -ENOMEM; + } else { + if (plus) + res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr); + else res = parse_dirfile(buf, res, file, ctx); - } + + vm_unmap_ram(buf, ap->num_folios); } +out: + kfree(ap->folios); + release_pages(pages, ap->num_folios); + kfree(pages); - kvfree(buf); fuse_invalidate_atime(inode); + return res; } @@ -442,6 +488,7 @@ static void fuse_rdc_reset(struct inode *inode) fi->rdc.version++; fi->rdc.size = 0; fi->rdc.pos = 0; + fi->rdc.epoch = 0; } #define UNCACHED 1 @@ -483,6 +530,7 @@ retry_locked: if (!ctx->pos && !fi->rdc.size) { fi->rdc.mtime = inode_get_mtime(inode); fi->rdc.iversion = inode_query_iversion(inode); + fi->rdc.epoch = atomic_read(&fc->epoch); } spin_unlock(&fi->rdc.lock); return UNCACHED; @@ -496,7 +544,8 @@ retry_locked: struct timespec64 mtime = inode_get_mtime(inode); if (inode_peek_iversion(inode) != fi->rdc.iversion || - !timespec64_equal(&fi->rdc.mtime, &mtime)) { + !timespec64_equal(&fi->rdc.mtime, &mtime) || + fi->rdc.epoch != atomic_read(&fc->epoch)) { fuse_rdc_reset(inode); goto retry_locked; } diff --git a/fs/fuse/req.c b/fs/fuse/req.c new file mode 100644 index 000000000000..a01ee743d31e --- /dev/null +++ b/fs/fuse/req.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "fuse_i.h" + +static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap) +{ + struct fuse_conn *fc = fm->fc; + bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP); + kuid_t fsuid = mapped_fsuid(idmap, fc->user_ns); + kgid_t fsgid = mapped_fsgid(idmap, fc->user_ns); + + args->pid = pid_nr_ns(task_pid(current), fc->pid_ns); + + if (args->force) { + if (args->nocreds) + return 0; + + if (no_idmap) { + args->uid = from_kuid_munged(fc->user_ns, current_fsuid()); + args->gid = from_kgid_munged(fc->user_ns, current_fsgid()); + } else { + args->uid = FUSE_INVALID_UIDGID; + args->gid = FUSE_INVALID_UIDGID; + } + return 0; + } + + WARN_ON(args->nocreds); + /* + * Keep the old behavior when idmappings support was not + * declared by a FUSE server. + * + * For those FUSE servers who support idmapped mounts, we send UID/GID + * only along with "inode creation" fuse requests, otherwise idmap == + * &invalid_mnt_idmap and req->in.h.{u,g}id will be equal to + * FUSE_INVALID_UIDGID. + */ + if (no_idmap) { + fsuid = current_fsuid(); + fsgid = current_fsgid(); + } + args->uid = from_kuid(fc->user_ns, fsuid); + args->gid = from_kgid(fc->user_ns, fsgid); + + if (no_idmap && unlikely(args->uid == ((uid_t)-1) || args->gid == ((gid_t)-1))) + return -EOVERFLOW; + + return 0; +} + +static int fuse_req_prep(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap) +{ + if (!args->force && fm->fc->conn_error) + return -ECONNREFUSED; + + return fuse_fill_creds(fm, args, idmap); +} + +ssize_t __fuse_simple_request(struct mnt_idmap *idmap, struct fuse_mount *fm, + struct fuse_args *args) +{ + struct fuse_conn *fc = fm->fc; + int err = fuse_req_prep(fm, args, idmap); + + if (err) + return err; + + return fuse_chan_send(fc->chan, args); +} + +int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags) +{ + struct fuse_conn *fc = fm->fc; + int err; + + WARN_ON(args->force && !args->nocreds); + + err = fuse_req_prep(fm, args, &invalid_mnt_idmap); + if (err) + return err; + + return fuse_chan_send_bg(fc->chan, args, gfp_flags); +} +EXPORT_SYMBOL_GPL(fuse_simple_background); + +int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique) +{ + struct fuse_conn *fc = fm->fc; + int err; + + WARN_ON(args->force && !args->nocreds); + + err = fuse_req_prep(fm, args, &invalid_mnt_idmap); + if (err) + return err; + + return fuse_chan_send_notify_reply(fc->chan, args, unique); +} diff --git a/fs/fuse/req_timeout.c b/fs/fuse/req_timeout.c new file mode 100644 index 000000000000..6cc6fc491343 --- /dev/null +++ b/fs/fuse/req_timeout.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "sysctl.h" +#include "fuse_dev_i.h" +#include "dev_uring_i.h" + +/* Frequency (in seconds) of request timeout checks, if opted into */ +#define FUSE_TIMEOUT_TIMER_FREQ 15 + +/* Frequency (in jiffies) of request timeout checks, if opted into */ +static const unsigned long fuse_timeout_timer_freq = + secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ); + +/* + * Default timeout (in seconds) for the server to reply to a request + * before the connection is aborted, if no timeout was specified on mount. + * + * Exported via sysctl + */ +unsigned int fuse_default_req_timeout; + +/* + * Max timeout (in seconds) for the server to reply to a request before + * the connection is aborted. + * + * Exported via sysctl + */ +unsigned int fuse_max_req_timeout; + +bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list) +{ + struct fuse_req *req; + + req = list_first_entry_or_null(list, struct fuse_req, list); + if (!req) + return false; + return time_is_before_jiffies(req->create_time + fch->timeout.req_timeout); +} + +static bool fuse_fpq_processing_expired(struct fuse_chan *fch, struct list_head *processing) +{ + int i; + + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) + if (fuse_request_expired(fch, &processing[i])) + return true; + + return false; +} + +/* + * Check if any requests aren't being completed by the time the request timeout + * elapses. To do so, we: + * - check the fiq pending list + * - check the bg queue + * - check the fpq io and processing lists + * + * To make this fast, we only check against the head request on each list since + * these are generally queued in order of creation time (eg newer requests get + * queued to the tail). We might miss a few edge cases (eg requests transitioning + * between lists, re-sent requests at the head of the pending list having a + * later creation time than other requests on that list, etc.) but that is fine + * since if the request never gets fulfilled, it will eventually be caught. + */ +static void fuse_check_timeout(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct fuse_chan *fch = container_of(dwork, struct fuse_chan, timeout.work); + struct fuse_iqueue *fiq = &fch->iq; + struct fuse_dev *fud; + struct fuse_pqueue *fpq; + bool expired = false; + + if (!atomic_read(&fch->num_waiting)) + goto out; + + spin_lock(&fiq->lock); + expired = fuse_request_expired(fch, &fiq->pending); + spin_unlock(&fiq->lock); + if (expired) + goto chan_abort; + + spin_lock(&fch->bg_lock); + expired = fuse_request_expired(fch, &fch->bg_queue); + spin_unlock(&fch->bg_lock); + if (expired) + goto chan_abort; + + spin_lock(&fch->lock); + if (!fch->connected) { + spin_unlock(&fch->lock); + return; + } + list_for_each_entry(fud, &fch->devices, entry) { + fpq = &fud->pq; + spin_lock(&fpq->lock); + if (fuse_request_expired(fch, &fpq->io) || + fuse_fpq_processing_expired(fch, fpq->processing)) { + spin_unlock(&fpq->lock); + spin_unlock(&fch->lock); + goto chan_abort; + } + + spin_unlock(&fpq->lock); + } + spin_unlock(&fch->lock); + + if (fuse_uring_request_expired(fch)) + goto chan_abort; + +out: + queue_delayed_work(system_percpu_wq, &fch->timeout.work, + fuse_timeout_timer_freq); + return; + +chan_abort: + fuse_chan_abort(fch, false); +} + +static void set_request_timeout(struct fuse_chan *fch, unsigned int timeout) +{ + fch->timeout.req_timeout = secs_to_jiffies(timeout); + INIT_DELAYED_WORK(&fch->timeout.work, fuse_check_timeout); + queue_delayed_work(system_percpu_wq, &fch->timeout.work, + fuse_timeout_timer_freq); +} + +void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout) +{ + if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout) + return; + + if (!timeout) + timeout = fuse_default_req_timeout; + + if (fuse_max_req_timeout) { + if (timeout) + timeout = min(fuse_max_req_timeout, timeout); + else + timeout = fuse_max_req_timeout; + } + + timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout); + + set_request_timeout(fch, timeout); +} + diff --git a/fs/fuse/sysctl.c b/fs/fuse/sysctl.c index e2d921abcb88..74eca5ce9a2c 100644 --- a/fs/fuse/sysctl.c +++ b/fs/fuse/sysctl.c @@ -6,6 +6,7 @@ */ #include <linux/sysctl.h> +#include "sysctl.h" #include "fuse_i.h" static struct ctl_table_header *fuse_table_header; diff --git a/fs/fuse/sysctl.h b/fs/fuse/sysctl.h new file mode 100644 index 000000000000..948d88417133 --- /dev/null +++ b/fs/fuse/sysctl.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FS_FUSE_SYSCTL_H +#define _FS_FUSE_SYSCTL_H + +extern unsigned int fuse_default_req_timeout; +extern unsigned int fuse_max_req_timeout; + +#endif /* _FS_FUSE_SYSCTL_H */ diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 12300651a0f1..df25d4faca41 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -19,6 +19,7 @@ #include <linux/highmem.h> #include <linux/cleanup.h> #include <linux/uio.h> +#include "dev.h" #include "fuse_i.h" #include "fuse_dev_i.h" @@ -1009,7 +1010,9 @@ out: kfree(vqs); if (ret) { kfree(fs->vqs); + fs->vqs = NULL; kfree(fs->mq_map); + fs->mq_map = NULL; } return ret; } @@ -1519,7 +1522,7 @@ static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req) if (ret == -ENOSPC) { /* * Virtqueue full. Retry submission from worker - * context as we might be holding fc->bg_lock. + * context as we might be holding fc->chan->bg_lock. */ spin_lock(&fsvq->lock); list_add_tail(&req->list, &fsvq->queued_reqs); @@ -1562,7 +1565,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) { struct fuse_mount *fm = get_fuse_mount_super(sb); struct fuse_conn *fc = fm->fc; - struct virtio_fs *fs = fc->iq.priv; + struct virtio_fs *fs = fc->chan->iq.priv; struct fuse_fs_context *ctx = fsc->fs_private; unsigned int i; int err; @@ -1606,7 +1609,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) for (i = 0; i < fs->nvqs; i++) { struct virtio_fs_vq *fsvq = &fs->vqs[i]; - fuse_dev_install(fsvq->fud, fc); + fuse_dev_install(fsvq->fud, fc->chan); } /* Previous unmount will stop all queues. Start these again */ @@ -1625,7 +1628,7 @@ err: static void virtio_fs_conn_destroy(struct fuse_mount *fm) { struct fuse_conn *fc = fm->fc; - struct virtio_fs *vfs = fc->iq.priv; + struct virtio_fs *vfs = fc->chan->iq.priv; struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO]; /* Stop dax worker. Soon evict_inodes() will be called which @@ -1673,7 +1676,7 @@ static int virtio_fs_test_super(struct super_block *sb, struct fuse_mount *fsc_fm = fsc->s_fs_info; struct fuse_mount *sb_fm = get_fuse_mount_super(sb); - return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv; + return fsc_fm->fc->chan->iq.priv == sb_fm->fc->chan->iq.priv; } static int virtio_fs_get_tree(struct fs_context *fsc) @@ -1683,13 +1686,17 @@ static int virtio_fs_get_tree(struct fs_context *fsc) struct fuse_conn *fc = NULL; struct fuse_mount *fm; unsigned int virtqueue_size; + struct fuse_chan *fch __free(fuse_chan_free) = fuse_chan_new(); int err = -EIO; + if (!fch) + return -ENOMEM; + if (!fsc->source) return invalf(fsc, "No source specified"); /* This gets a reference on virtio_fs object. This ptr gets installed - * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() + * in chan->iq->priv. Once fuse_conn is going away, it calls ->put() * to drop the reference to this object. */ fs = virtio_fs_find_instance(fsc->source); @@ -1711,7 +1718,9 @@ static int virtio_fs_get_tree(struct fs_context *fsc) if (!fm) goto out_err; - fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs); + fuse_iqueue_init(&fch->iq, &virtio_fs_fiq_ops, fs); + fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch)); + fc->release = fuse_free_conn; fc->delete_stale = true; fc->auto_submounts = true; diff --git a/fs/fuse/xattr.c b/fs/fuse/xattr.c index 93dfb06b6cea..cab2685acc65 100644 --- a/fs/fuse/xattr.c +++ b/fs/fuse/xattr.c @@ -1,9 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * FUSE: Filesystem in Userspace * Copyright (C) 2001-2016 Miklos Szeredi <miklos@szeredi.hu> - * - * This program can be distributed under the terms of the GNU GPL. - * See the file COPYING. */ #include "fuse_i.h" diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index b3d7fcd95f03..51ac1fd44f78 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -304,14 +304,15 @@ static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end) rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE); if (trylock_buffer(rabh)) { if (!buffer_uptodate(rabh)) { - rabh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META | - REQ_PRIO, rabh); - continue; + bh_submit(rabh, + REQ_OP_READ | REQ_RAHEAD | REQ_META | + REQ_PRIO, + bh_end_read); + } else { + unlock_buffer(rabh); } - unlock_buffer(rabh); } - brelse(rabh); + put_bh(rabh); } } @@ -1321,6 +1322,19 @@ static int gfs2_block_zero_range(struct inode *inode, loff_t from, loff_t length &gfs2_iomap_write_ops, NULL); } +int gfs2_clear_beyond_eof(struct inode *inode, loff_t end) +{ + loff_t isize = i_size_read(inode); + unsigned int len = isize & ~PAGE_MASK; + + if (!len || isize >= end) + return 0; + len = PAGE_SIZE - len; + if (end - isize < len) + len = end - isize; + return gfs2_block_zero_range(inode, isize, len); +} + #define GFS2_JTRUNC_REVOKES 8192 /** @@ -2096,6 +2110,12 @@ static int do_grow(struct inode *inode, u64 size) unstuff = 1; } + if (!unstuff) { + error = gfs2_clear_beyond_eof(inode, size); + if (error) + goto do_grow_qunlock; + } + error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT + (unstuff && gfs2_is_jdata(ip) ? RES_JDATA : 0) + diff --git a/fs/gfs2/bmap.h b/fs/gfs2/bmap.h index 6cdc72dd55a3..e3d6efdfd890 100644 --- a/fs/gfs2/bmap.h +++ b/fs/gfs2/bmap.h @@ -58,6 +58,7 @@ int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock, unsigned int *extlen); int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock, unsigned *extlen, bool *new); +int gfs2_clear_beyond_eof(struct inode *inode, loff_t end); int gfs2_setattr_size(struct inode *inode, u64 size); int gfs2_truncatei_resume(struct gfs2_inode *ip); int gfs2_file_dealloc(struct gfs2_inode *ip); diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c index 022dbb31e0d9..0237b36b9eb1 100644 --- a/fs/gfs2/dir.c +++ b/fs/gfs2/dir.c @@ -1505,15 +1505,13 @@ static void gfs2_dir_readahead(struct inode *inode, unsigned hsize, u32 index, if (trylock_buffer(bh)) { if (buffer_uptodate(bh)) { unlock_buffer(bh); - brelse(bh); - continue; + } else { + bh_submit(bh, REQ_OP_READ | REQ_RAHEAD | + REQ_META | REQ_PRIO, + bh_end_read); } - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META | - REQ_PRIO, bh); - continue; } - brelse(bh); + put_bh(bh); } } diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index 9704f1ef6ad1..b8c10de113ba 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -1057,6 +1057,10 @@ retry: goto out_unlock; } + ret = gfs2_clear_beyond_eof(inode, iocb->ki_pos); + if (ret) + goto out_unlock; + pagefault_disable(); ret = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops, &gfs2_iomap_write_ops, NULL); @@ -1173,8 +1177,7 @@ out_unlock: return ret; } -static int fallocate_chunk(struct inode *inode, loff_t offset, loff_t len, - int mode) +static int fallocate_chunk(struct inode *inode, loff_t offset, loff_t len) { struct super_block *sb = inode->i_sb; struct gfs2_inode *ip = GFS2_I(inode); @@ -1266,6 +1269,12 @@ static long __gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t next = (next + 1) << sdp->sd_sb.sb_bsize_shift; + if (!(mode & FALLOC_FL_KEEP_SIZE)) { + error = gfs2_clear_beyond_eof(inode, offset + len); + if (error) + return error; + } + offset &= bsize_mask; len = next - offset; @@ -1336,7 +1345,7 @@ static long __gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t if (error) goto out_trans_fail; - error = fallocate_chunk(inode, offset, max_bytes, mode); + error = fallocate_chunk(inode, offset, max_bytes); gfs2_trans_end(sdp); if (error) diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index e9bf4879c07f..8a77794bbd4a 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -738,6 +738,13 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, inode = gfs2_dir_search(dir, &dentry->d_name, !S_ISREG(mode) || excl); error = PTR_ERR(inode); if (!IS_ERR(inode)) { + if (file && (file->f_flags & __O_REGULAR) && + !S_ISREG(inode->i_mode)) { + iput(inode); + inode = NULL; + error = -EFTYPE; + goto fail_gunlock; + } if (S_ISDIR(inode->i_mode)) { iput(inode); inode = NULL; @@ -987,12 +994,8 @@ static struct dentry *__gfs2_lookup(struct inode *dir, struct dentry *dentry, int error; inode = gfs2_lookupi(dir, &dentry->d_name, 0); - if (inode == NULL) { - d_add(dentry, NULL); - return NULL; - } - if (IS_ERR(inode)) - return ERR_CAST(inode); + if (inode == NULL || IS_ERR(inode)) + return d_splice_alias(inode, dentry); gl = GFS2_I(inode)->i_gl; error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &gh); diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index d407dd476e72..a87cfbf0df38 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -59,7 +59,7 @@ static void gfs2_aspace_write_folio(struct folio *folio, continue; } if (test_clear_buffer_dirty(bh)) { - mark_buffer_async_write(bh); + set_buffer_async_write(bh); } else { unlock_buffer(bh); } @@ -75,7 +75,8 @@ static void gfs2_aspace_write_folio(struct folio *folio, do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh(REQ_OP_WRITE | write_flags, bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, + bh_end_async_write); nr_underway++; } bh = next; @@ -212,7 +213,7 @@ static void gfs2_meta_read_endio(struct bio *bio) do { struct buffer_head *next = bh->b_this_page; len -= bh->b_size; - bh->b_end_io(bh, !bio->bi_status); + end_buffer_read_sync(bh, bio->bi_status == BLK_STS_OK); bh = next; } while (bh && len); } @@ -221,7 +222,7 @@ static void gfs2_meta_read_endio(struct bio *bio) /* * Submit several consecutive buffer head I/O requests as a single bio I/O - * request. (See submit_bh_wbc.) + * request. (See bh_submit.) */ static void gfs2_submit_bhs(blk_opf_t opf, struct buffer_head *bhs[], int num) { @@ -275,7 +276,6 @@ int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, unlock_buffer(bh); flags &= ~DIO_WAIT; } else { - bh->b_end_io = end_buffer_read_sync; get_bh(bh); bhs[num++] = bh; } @@ -286,11 +286,10 @@ int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, lock_buffer(bh); if (buffer_uptodate(bh)) { unlock_buffer(bh); - brelse(bh); } else { - bh->b_end_io = end_buffer_read_sync; bhs[num++] = bh; } + brelse(bh); } gfs2_submit_bhs(REQ_OP_READ | REQ_META | REQ_PRIO, bhs, num); diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index 5290865f27f1..91e9975d25e8 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -254,9 +254,13 @@ fail: return NULL; } -static struct gfs2_quota_data *gfs2_qd_search_bucket(unsigned int hash, - const struct gfs2_sbd *sdp, - struct kqid qid) +/* + * Lookup variant for callers which already hold qd_lock + bucket lock. + */ +static struct gfs2_quota_data * +gfs2_qd_search_bucket_noref(unsigned int hash, + const struct gfs2_sbd *sdp, + struct kqid qid) { struct gfs2_quota_data *qd; struct hlist_bl_node *h; @@ -264,12 +268,22 @@ static struct gfs2_quota_data *gfs2_qd_search_bucket(unsigned int hash, hlist_bl_for_each_entry_rcu(qd, h, &qd_hash_table[hash], qd_hlist) { if (!qid_eq(qd->qd_id, qid)) continue; - if (qd->qd_sbd != sdp) - continue; - if (lockref_get_not_dead(&qd->qd_lockref)) { - list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); + if (qd->qd_sbd == sdp) return qd; - } + } + + return NULL; +} + +static struct gfs2_quota_data * +gfs2_qd_search_bucket(unsigned int hash, const struct gfs2_sbd *sdp, struct kqid qid) +{ + struct gfs2_quota_data *qd; + + qd = gfs2_qd_search_bucket_noref(hash, sdp, qid); + if (qd && lockref_get_not_dead(&qd->qd_lockref)) { + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); + return qd; } return NULL; @@ -1433,7 +1447,7 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) qc = (struct gfs2_quota_change *)(bh->b_data + sizeof(struct gfs2_meta_header)); for (y = 0; y < sdp->sd_qc_per_block && slot < sdp->sd_quota_slots; - y++, slot++) { + y++, slot++, qc++) { struct gfs2_quota_data *old_qd, *qd; s64 qc_change = be64_to_cpu(qc->qc_change); u32 qc_flags = be32_to_cpu(qc->qc_flags); @@ -1441,7 +1455,6 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) USRQUOTA : GRPQUOTA; struct kqid qc_id = make_kqid(&init_user_ns, qtype, be32_to_cpu(qc->qc_id)); - qc++; if (!qc_change) continue; @@ -1458,7 +1471,7 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) spin_lock(&qd_lock); spin_lock_bucket(hash); - old_qd = gfs2_qd_search_bucket(hash, sdp, qc_id); + old_qd = gfs2_qd_search_bucket_noref(hash, sdp, qc_id); if (old_qd) { fs_err(sdp, "Corruption found in quota_change%u" "file: duplicate identifier in " @@ -1467,7 +1480,6 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) spin_unlock_bucket(hash); spin_unlock(&qd_lock); - qd_put(old_qd); gfs2_glock_put(qd->qd_gl); kmem_cache_free(gfs2_quotad_cachep, qd); diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index a2ea121331f1..4d854556b529 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -643,6 +643,7 @@ restart: gfs2_delete_debugfs_file(sdp); gfs2_sys_fs_del(sdp); + rcu_barrier(); free_sbd(sdp); } diff --git a/fs/hfs/bnode.c b/fs/hfs/bnode.c index 13d58c51fc46..da8e5342c91c 100644 --- a/fs/hfs/bnode.c +++ b/fs/hfs/bnode.c @@ -41,7 +41,7 @@ u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) node_size = node->tree->node_size; - if ((off + len) > node_size) { + if ((u64)off + len > node_size) { u32 new_len = node_size - off; pr_err("requested length has been corrected: " @@ -64,6 +64,8 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) u32 bytes_read; u32 bytes_to_read; + memset(buf, 0, len); + if (!is_bnode_offset_valid(node, off)) return; @@ -344,7 +346,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) struct hfs_bnode *node, *node2; struct address_space *mapping; struct page *page; - int size, block, i, hash; + int block, i, hash; loff_t off; if (cnid >= tree->node_count) { @@ -352,9 +354,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) return NULL; } - size = sizeof(struct hfs_bnode) + tree->pages_per_bnode * - sizeof(struct page *); - node = kzalloc(size, GFP_KERNEL); + node = kzalloc_flex(*node, page, tree->pages_per_bnode, GFP_KERNEL); if (!node) return NULL; node->tree = tree; diff --git a/fs/hfs/catalog.c b/fs/hfs/catalog.c index 7f5339ee57c1..1bfa36d71e24 100644 --- a/fs/hfs/catalog.c +++ b/fs/hfs/catalog.c @@ -340,7 +340,6 @@ int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str) { struct super_block *sb; struct hfs_find_data fd; - struct hfs_readdir_data *rd; int res, type; hfs_dbg("name %s, cnid %u\n", str ? str->name : NULL, cnid); @@ -366,14 +365,6 @@ int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str) } } - /* we only need to take spinlock for exclusion with ->release() */ - spin_lock(&HFS_I(dir)->open_dir_lock); - list_for_each_entry(rd, &HFS_I(dir)->open_dir_list, list) { - if (fd.tree->keycmp(fd.search_key, (void *)&rd->key) < 0) - rd->file->f_pos--; - } - spin_unlock(&HFS_I(dir)->open_dir_lock); - res = hfs_brec_remove(&fd); if (res) goto out; diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c index f5e7efe924e7..e13450bb933e 100644 --- a/fs/hfs/dir.c +++ b/fs/hfs/dir.c @@ -97,7 +97,15 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) } if (ctx->pos >= inode->i_size) goto out; - err = hfs_brec_goto(&fd, ctx->pos - 1); + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfs_cat_key)); + err = hfs_brec_find(&fd); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } if (err) goto out; @@ -146,7 +154,6 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) if (err) goto out; } - rd = file->private_data; if (!rd) { rd = kmalloc_obj(struct hfs_readdir_data); if (!rd) { @@ -154,15 +161,8 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) goto out; } file->private_data = rd; - rd->file = file; - spin_lock(&HFS_I(inode)->open_dir_lock); - list_add(&rd->list, &HFS_I(inode)->open_dir_list); - spin_unlock(&HFS_I(inode)->open_dir_lock); } - /* - * Can be done after the list insertion; exclusion with - * hfs_delete_cat() is provided by directory lock. - */ + rd->pos = ctx->pos; memcpy(&rd->key, &fd.key->cat, sizeof(struct hfs_cat_key)); out: hfs_find_exit(&fd); @@ -171,13 +171,7 @@ out: static int hfs_dir_release(struct inode *inode, struct file *file) { - struct hfs_readdir_data *rd = file->private_data; - if (rd) { - spin_lock(&HFS_I(inode)->open_dir_lock); - list_del(&rd->list); - spin_unlock(&HFS_I(inode)->open_dir_lock); - kfree(rd); - } + kfree(file->private_data); return 0; } @@ -306,10 +300,15 @@ static int hfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, res = hfs_cat_move(d_inode(old_dentry)->i_ino, old_dir, &old_dentry->d_name, new_dir, &new_dentry->d_name); - if (!res) + if (!res) { + struct inode *inode = d_inode(old_dentry); + hfs_cat_build_key(old_dir->i_sb, - (btree_key *)&HFS_I(d_inode(old_dentry))->cat_key, + (btree_key *)&HFS_I(inode)->cat_key, new_dir->i_ino, &new_dentry->d_name); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } return res; } @@ -328,4 +327,5 @@ const struct inode_operations hfs_dir_inode_operations = { .rmdir = hfs_remove, .rename = hfs_rename, .setattr = hfs_inode_setattr, + .fileattr_get = hfs_fileattr_get, }; diff --git a/fs/hfs/hfs.h b/fs/hfs/hfs.h index 3f2293ff6fdd..2e958c46bc0a 100644 --- a/fs/hfs/hfs.h +++ b/fs/hfs/hfs.h @@ -14,8 +14,7 @@ /*======== Data structures kept in memory ========*/ struct hfs_readdir_data { - struct list_head list; - struct file *file; + loff_t pos; struct hfs_cat_key key; }; diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h index ac0e83f77a0f..f3624514fcb0 100644 --- a/fs/hfs/hfs_fs.h +++ b/fs/hfs/hfs_fs.h @@ -36,8 +36,6 @@ struct hfs_inode_info { struct hfs_cat_key cat_key; - struct list_head open_dir_list; - spinlock_t open_dir_lock; struct inode *rsrc_inode; struct mutex extents_lock; @@ -177,6 +175,8 @@ extern int hfs_get_block(struct inode *inode, sector_t block, extern const struct address_space_operations hfs_aops; extern const struct address_space_operations hfs_btree_aops; +struct file_kattr; +int hfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); int hfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, loff_t pos, unsigned int len, struct folio **foliop, void **fsdata); diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index 89b33a9d46d5..ac4a9055c5c0 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -18,6 +18,7 @@ #include <linux/uio.h> #include <linux/xattr.h> #include <linux/blkdev.h> +#include <linux/fileattr.h> #include "hfs_fs.h" #include "btree.h" @@ -195,8 +196,6 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t err = -ERANGE; mutex_init(&HFS_I(inode)->extents_lock); - INIT_LIST_HEAD(&HFS_I(inode)->open_dir_list); - spin_lock_init(&HFS_I(inode)->open_dir_lock); hfs_cat_build_key(sb, (btree_key *)&HFS_I(inode)->cat_key, dir->i_ino, name); next_id = atomic64_inc_return(&HFS_SB(sb)->next_id); if (next_id > U32_MAX) { @@ -204,7 +203,7 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t pr_err("cannot create new inode: next CNID exceeds limit\n"); goto out_discard; } - inode->i_ino = (u32)next_id; + inode->i_ino = (u32)next_id - 1; inode->i_mode = mode; inode->i_uid = current_fsuid(); inode->i_gid = current_fsgid(); @@ -348,12 +347,11 @@ static int hfs_read_inode(struct inode *inode, void *data) struct hfs_iget_data *idata = data; struct hfs_sb_info *hsb = HFS_SB(inode->i_sb); hfs_cat_rec *rec; + struct timespec64 mtime; HFS_I(inode)->flags = 0; HFS_I(inode)->rsrc_inode = NULL; mutex_init(&HFS_I(inode)->extents_lock); - INIT_LIST_HEAD(&HFS_I(inode)->open_dir_list); - spin_lock_init(&HFS_I(inode)->open_dir_lock); /* Initialize the inode */ inode->i_uid = hsb->s_uid; @@ -383,8 +381,10 @@ static int hfs_read_inode(struct inode *inode, void *data) inode->i_mode |= S_IWUGO; inode->i_mode &= ~hsb->s_file_umask; inode->i_mode |= S_IFREG; - inode_set_mtime_to_ts(inode, - inode_set_atime_to_ts(inode, inode_set_ctime_to_ts(inode, hfs_m_to_utime(rec->file.MdDat)))); + mtime = hfs_m_to_utime(rec->file.MdDat); + inode_set_ctime_to_ts(inode, mtime); + inode_set_atime_to_ts(inode, mtime); + inode_set_mtime_to_ts(inode, mtime); inode->i_op = &hfs_file_inode_operations; inode->i_fop = &hfs_file_operations; inode->i_mapping->a_ops = &hfs_aops; @@ -394,8 +394,10 @@ static int hfs_read_inode(struct inode *inode, void *data) inode->i_size = be16_to_cpu(rec->dir.Val) + 2; HFS_I(inode)->fs_blocks = 0; inode->i_mode = S_IFDIR | (S_IRWXUGO & ~hsb->s_dir_umask); - inode_set_mtime_to_ts(inode, - inode_set_atime_to_ts(inode, inode_set_ctime_to_ts(inode, hfs_m_to_utime(rec->dir.MdDat)))); + mtime = hfs_m_to_utime(rec->dir.MdDat); + inode_set_ctime_to_ts(inode, mtime); + inode_set_atime_to_ts(inode, mtime); + inode_set_mtime_to_ts(inode, mtime); inode->i_op = &hfs_dir_inode_operations; inode->i_fop = &hfs_dir_operations; break; @@ -665,7 +667,7 @@ int hfs_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, truncate_setsize(inode, attr->ia_size); hfs_file_truncate(inode); - simple_inode_init_ts(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); } setattr_copy(&nop_mnt_idmap, inode, attr); @@ -699,6 +701,18 @@ static int hfs_file_fsync(struct file *filp, loff_t start, loff_t end, return ret; } +int hfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + /* + * HFS compares filenames using Mac OS Roman case folding, so + * lookup is always case-insensitive. Names are stored on disk + * with case intact; CASENONPRESERVING stays clear. + */ + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + return 0; +} + static const struct file_operations hfs_file_operations = { .llseek = generic_file_llseek, .read_iter = generic_file_read_iter, @@ -715,4 +729,5 @@ static const struct inode_operations hfs_file_inode_operations = { .lookup = hfs_file_lookup, .setattr = hfs_inode_setattr, .listxattr = generic_listxattr, + .fileattr_get = hfs_fileattr_get, }; diff --git a/fs/hfs/super.c b/fs/hfs/super.c index a4f2a2bfa6d3..a466c401f6bb 100644 --- a/fs/hfs/super.c +++ b/fs/hfs/super.c @@ -338,7 +338,7 @@ static int hfs_fill_super(struct super_block *sb, struct fs_context *fc) sbi->sb = sb; sb->s_op = &hfs_super_operations; sb->s_xattr = hfs_xattr_handlers; - sb->s_flags |= SB_NODIRATIME; + sb->s_flags |= SB_NOATIME | SB_NODIRATIME; mutex_init(&sbi->bitmap_lock); res = hfs_mdb_get(sb); diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index f8b5a8ae58ff..d088fb7eb0df 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -25,6 +25,8 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) struct page **pagep; u32 l; + memset(buf, 0, len); + if (!is_bnode_offset_valid(node, off)) return; @@ -455,7 +457,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) struct hfs_bnode *node, *node2; struct address_space *mapping; struct page *page; - int size, block, i, hash; + int block, i, hash; loff_t off; if (cnid >= tree->node_count) { @@ -464,9 +466,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) return NULL; } - size = sizeof(struct hfs_bnode) + tree->pages_per_bnode * - sizeof(struct page *); - node = kzalloc(size, GFP_KERNEL); + node = kzalloc_flex(*node, page, tree->pages_per_bnode); if (!node) return NULL; node->tree = tree; diff --git a/fs/hfsplus/btree.c b/fs/hfsplus/btree.c index 761c74ccd653..394542a47e60 100644 --- a/fs/hfsplus/btree.c +++ b/fs/hfsplus/btree.c @@ -365,6 +365,8 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id) } size = tree->node_size; + if (size < HFSPLUS_NODE_MINSZ || size > HFSPLUS_NODE_MXSZ) + goto fail_page; if (!is_power_of_2(size)) goto fail_page; if (!tree->node_count) diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c index 9bc8a6c48fd3..776ce36cf076 100644 --- a/fs/hfsplus/catalog.c +++ b/fs/hfsplus/catalog.c @@ -333,7 +333,6 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) struct super_block *sb = dir->i_sb; struct hfs_find_data fd; struct hfsplus_fork_raw fork; - struct list_head *pos; int err, off; u16 type; @@ -392,16 +391,6 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) hfsplus_free_fork(sb, cnid, &fork, HFSPLUS_TYPE_RSRC); } - /* we only need to take spinlock for exclusion with ->release() */ - spin_lock(&HFSPLUS_I(dir)->open_dir_lock); - list_for_each(pos, &HFSPLUS_I(dir)->open_dir_list) { - struct hfsplus_readdir_data *rd = - list_entry(pos, struct hfsplus_readdir_data, list); - if (fd.tree->keycmp(fd.search_key, (void *)&rd->key) < 0) - rd->file->f_pos--; - } - spin_unlock(&HFSPLUS_I(dir)->open_dir_lock); - err = hfs_brec_remove(&fd); if (err) goto out; diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index 47194370c2c5..8bf6c7cdd9a8 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -185,7 +185,15 @@ static int hfsplus_readdir(struct file *file, struct dir_context *ctx) } if (ctx->pos >= inode->i_size) goto out; - err = hfs_brec_goto(&fd, ctx->pos - 1); + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfsplus_cat_key)); + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } if (err) goto out; for (;;) { @@ -261,7 +269,6 @@ next: if (err) goto out; } - rd = file->private_data; if (!rd) { rd = kmalloc_obj(struct hfsplus_readdir_data); if (!rd) { @@ -269,15 +276,8 @@ next: goto out; } file->private_data = rd; - rd->file = file; - spin_lock(&HFSPLUS_I(inode)->open_dir_lock); - list_add(&rd->list, &HFSPLUS_I(inode)->open_dir_list); - spin_unlock(&HFSPLUS_I(inode)->open_dir_lock); } - /* - * Can be done after the list insertion; exclusion with - * hfsplus_delete_cat() is provided by directory lock. - */ + rd->pos = ctx->pos; memcpy(&rd->key, fd.key, sizeof(struct hfsplus_cat_key)); out: kfree(strbuf); @@ -287,13 +287,7 @@ out: static int hfsplus_dir_release(struct inode *inode, struct file *file) { - struct hfsplus_readdir_data *rd = file->private_data; - if (rd) { - spin_lock(&HFSPLUS_I(inode)->open_dir_lock); - list_del(&rd->list); - spin_unlock(&HFSPLUS_I(inode)->open_dir_lock); - kfree(rd); - } + kfree(file->private_data); return 0; } diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index 3545b8dbf11c..ec04b82ad927 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -214,8 +214,6 @@ struct hfsplus_inode_info { sector_t fs_blocks; u8 userflags; /* BSD user file flags */ u32 subfolders; /* Subfolder count (HFSX only) */ - struct list_head open_dir_list; - spinlock_t open_dir_lock; loff_t phys_size; struct inode vfs_inode; @@ -272,8 +270,7 @@ struct hfs_find_data { }; struct hfsplus_readdir_data { - struct list_head list; - struct file *file; + loff_t pos; struct hfsplus_cat_key key; }; @@ -600,7 +597,7 @@ u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) node_size = node->tree->node_size; - if ((off + len) > node_size) { + if ((u64)off + len > node_size) { u32 new_len = node_size - off; pr_err("requested length has been corrected: " diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index d05891ec492e..1c57c873f05d 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -125,9 +125,44 @@ static ssize_t hfsplus_direct_IO(struct kiocb *iocb, struct iov_iter *iter) struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; + loff_t isize; size_t count = iov_iter_count(iter); + loff_t end = iocb->ki_pos + count; ssize_t ret; + /* + * The hfsplus_get_block() only allows creating the next sequential block. + * For direct writes beyond EOF, expand the file first. + */ + if (iov_iter_rw(iter) == WRITE && iocb->ki_pos > i_size_read(inode)) { + loff_t start_off, end_off; + loff_t start_page, end_page; + + isize = i_size_read(inode); + + /* + * Wait for any in-flight DIO on this inode to finish before + * calling generic_cont_expand_simple(). + */ + inode_dio_wait(inode); + + ret = generic_cont_expand_simple(inode, iocb->ki_pos); + if (ret) + return ret; + + start_off = isize; + end_off = (end > 0) ? end - 1 : end; + + ret = filemap_write_and_wait_range(mapping, start_off, end_off); + if (ret) + return ret; + + start_page = start_off >> PAGE_SHIFT; + end_page = end_off >> PAGE_SHIFT; + + invalidate_inode_pages2_range(mapping, start_page, end_page); + } + ret = blockdev_direct_IO(iocb, inode, iter, hfsplus_get_block); /* @@ -135,8 +170,7 @@ static ssize_t hfsplus_direct_IO(struct kiocb *iocb, struct iov_iter *iter) * blocks outside i_size. Trim these off again. */ if (unlikely(iov_iter_rw(iter) == WRITE && ret < 0)) { - loff_t isize = i_size_read(inode); - loff_t end = iocb->ki_pos + count; + isize = i_size_read(inode); if (end > isize) hfsplus_write_failed(mapping, end); @@ -449,8 +483,6 @@ struct inode *hfsplus_new_inode(struct super_block *sb, struct inode *dir, simple_inode_init_ts(inode); hip = HFSPLUS_I(inode); - INIT_LIST_HEAD(&hip->open_dir_list); - spin_lock_init(&hip->open_dir_lock); mutex_init(&hip->extents_lock); atomic_set(&hip->opencnt, 0); hip->extent_state = 0; @@ -740,6 +772,7 @@ int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa) { struct inode *inode = d_inode(dentry); struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); unsigned int flags = 0; if (inode->i_flags & S_IMMUTABLE) @@ -748,6 +781,8 @@ int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa) flags |= FS_APPEND_FL; if (hip->userflags & HFSPLUS_FLG_NODUMP) flags |= FS_NODUMP_FL; + if (test_bit(HFSPLUS_SB_CASEFOLD, &sbi->flags)) + flags |= FS_CASEFOLD_FL; fileattr_fill_flags(fa, flags); @@ -759,13 +794,24 @@ int hfsplus_fileattr_set(struct mnt_idmap *idmap, { struct inode *inode = d_inode(dentry); struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + unsigned int allowed = FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL; unsigned int new_fl = 0; if (fileattr_has_fsx(fa)) return -EOPNOTSUPP; + /* + * FS_CASEFOLD_FL reflects HFSPLUS_SB_CASEFOLD, a mount-time + * property. Accept it as a no-op so chattr's RMW round-trip + * succeeds; reject any attempt to enable it on a volume that + * was not formatted case-insensitive. + */ + if (test_bit(HFSPLUS_SB_CASEFOLD, &sbi->flags)) + allowed |= FS_CASEFOLD_FL; + /* don't silently ignore unsupported ext2 flags */ - if (fa->flags & ~(FS_IMMUTABLE_FL|FS_APPEND_FL|FS_NODUMP_FL)) + if (fa->flags & ~allowed) return -EOPNOTSUPP; if (fa->flags & FS_IMMUTABLE_FL) diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 40a0feda716b..5777e31de45a 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -91,8 +91,6 @@ struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino) HFSPLUS_I(inode)->fs_blocks = 0; HFSPLUS_I(inode)->userflags = 0; HFSPLUS_I(inode)->subfolders = 0; - INIT_LIST_HEAD(&HFSPLUS_I(inode)->open_dir_list); - spin_lock_init(&HFSPLUS_I(inode)->open_dir_lock); HFSPLUS_I(inode)->phys_size = 0; if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID || diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c index 452a1f9becb2..21a1c196c71f 100644 --- a/fs/hfsplus/xattr.c +++ b/fs/hfsplus/xattr.c @@ -317,7 +317,6 @@ check_attr_tree_state_again: next_node++; } - hfsplus_mark_inode_dirty(HFSPLUS_ATTR_TREE_I(sb), HFSPLUS_I_ATTR_DIRTY); hfsplus_mark_inode_dirty(attr_file, HFSPLUS_I_ATTR_DIRTY); sbi->attr_tree = hfs_btree_open(sb, HFSPLUS_ATTR_CNID); diff --git a/fs/hpfs/alloc.c b/fs/hpfs/alloc.c index 66617b1557c6..f5150372618e 100644 --- a/fs/hpfs/alloc.c +++ b/fs/hpfs/alloc.c @@ -372,8 +372,8 @@ int hpfs_check_free_dnodes(struct super_block *s, int n) return 0; } } + hpfs_brelse4(&qbh); } - hpfs_brelse4(&qbh); i = 0; if (hpfs_sb(s)->sb_c_bitmap != -1) { bmp = hpfs_map_bitmap(s, b, &qbh, "chkdn1"); diff --git a/fs/hpfs/inode.c b/fs/hpfs/inode.c index 0e932cc8be1b..1b4fcf760aad 100644 --- a/fs/hpfs/inode.c +++ b/fs/hpfs/inode.c @@ -184,7 +184,7 @@ void hpfs_write_inode(struct inode *i) struct hpfs_inode_info *hpfs_inode = hpfs_i(i); struct inode *parent; if (i->i_ino == hpfs_sb(i->i_sb)->sb_root) return; - if (hpfs_inode->i_rddir_off && !icount_read(i)) { + if (hpfs_inode->i_rddir_off && !icount_read_once(i)) { if (*hpfs_inode->i_rddir_off) pr_err("write_inode: some position still there\n"); kfree(hpfs_inode->i_rddir_off); diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c index c16d5d4caead..8fbdbf080627 100644 --- a/fs/hpfs/super.c +++ b/fs/hpfs/super.c @@ -523,7 +523,8 @@ static int hpfs_fill_super(struct super_block *s, struct fs_context *fc) hpfs_lock(s); /*sbi->sb_mounting = 1;*/ - sb_set_blocksize(s, 512); + if (!sb_set_blocksize(s, 512)) + goto bail0; sbi->sb_fs_size = -1; if (!(bootblock = hpfs_map_sector(s, 0, &bh0, 0))) goto bail1; if (!(superblock = hpfs_map_sector(s, 16, &bh1, 1))) goto bail2; diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 8b05bec08e04..216e1a0dd0b2 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -96,15 +96,8 @@ static const struct fs_parameter_spec hugetlb_fs_parameters[] = { #define PGOFF_LOFFT_MAX \ (((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1))) -static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma) +static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) { - /* Unfortunate we have to reassign vma->vm_private_data. */ - return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma); -} - -static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc) -{ - struct file *file = desc->file; struct inode *inode = file_inode(file); loff_t len, vma_len; int ret; @@ -119,8 +112,8 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc) * way when do_mmap unwinds (may be important on powerpc * and ia64). */ - vma_desc_set_flags(desc, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT); - desc->vm_ops = &hugetlb_vm_ops; + vma_set_flags(vma, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT); + vma->vm_ops = &hugetlb_vm_ops; /* * page based offset in vm_pgoff could be sufficiently large to @@ -129,16 +122,16 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc) * sizeof(unsigned long). So, only check in those instances. */ if (sizeof(unsigned long) == sizeof(loff_t)) { - if (desc->pgoff & PGOFF_LOFFT_MAX) + if (vma->vm_pgoff & PGOFF_LOFFT_MAX) return -EINVAL; } /* must be huge page aligned */ - if (desc->pgoff & (~huge_page_mask(h) >> PAGE_SHIFT)) + if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT)) return -EINVAL; - vma_len = (loff_t)vma_desc_size(desc); - len = vma_len + ((loff_t)desc->pgoff << PAGE_SHIFT); + vma_len = (loff_t)(vma->vm_end - vma->vm_start); + len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); /* check for overflow */ if (len < vma_len) return -EINVAL; @@ -148,7 +141,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc) ret = -ENOMEM; - vma_flags = desc->vma_flags; + vma_flags = vma->flags; /* * for SHM_HUGETLB, the pages are reserved in the shmget() call so skip * reserving here. Note: only for SHM hugetlbfs file, the inode @@ -158,30 +151,17 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc) vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); if (hugetlb_reserve_pages(inode, - desc->pgoff >> huge_page_order(h), - len >> huge_page_shift(h), desc, - vma_flags) < 0) + vma->vm_pgoff >> huge_page_order(h), + len >> huge_page_shift(h), vma, + vma_flags) < 0) goto out; ret = 0; - if (vma_desc_test(desc, VMA_WRITE_BIT) && inode->i_size < len) + if (vma_test(vma, VMA_WRITE_BIT) && inode->i_size < len) i_size_write(inode, len); out: inode_unlock(inode); - if (!ret) { - /* Allocate the VMA lock after we set it up. */ - desc->action.success_hook = hugetlb_file_mmap_prepare_success; - /* - * We cannot permit the rmap finding this VMA in the time - * between the VMA being inserted into the VMA tree and the - * completion/success hook being invoked. - * - * This is because we establish a per-VMA hugetlb lock which can - * be raced by rmap. - */ - desc->action.hide_from_rmap_until_complete = true; - } return ret; } @@ -634,7 +614,7 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) i_size_write(inode, offset); i_mmap_lock_write(mapping); - if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)) + if (mapping_mapped(mapping)) hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0, ZAP_FLAG_DROP_MARKER); i_mmap_unlock_write(mapping); @@ -695,7 +675,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) /* Unmap users of full pages in the hole. */ if (hole_end > hole_start) { - if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)) + if (mapping_mapped(mapping)) hugetlb_vmdelete_list(&mapping->i_mmap, hole_start >> PAGE_SHIFT, hole_end >> PAGE_SHIFT, 0); @@ -1227,7 +1207,7 @@ static void init_once(void *foo) static const struct file_operations hugetlbfs_file_operations = { .read_iter = hugetlbfs_read_iter, - .mmap_prepare = hugetlbfs_file_mmap_prepare, + .mmap = hugetlbfs_file_mmap, .fsync = noop_fsync, .get_unmapped_area = hugetlb_get_unmapped_area, .llseek = default_llseek, diff --git a/fs/inode.c b/fs/inode.c index 6a3cbc7dcd28..31c5b9ee3a81 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -53,11 +53,7 @@ * inode->i_lock * * inode_hash_lock - * inode->i_sb->s_inode_list_lock * inode->i_lock - * - * iunique_lock - * inode_hash_lock */ static unsigned int i_hash_mask __ro_after_init; @@ -279,9 +275,6 @@ int inode_init_always_gfp(struct super_block *sb, struct inode *inode, gfp_t gfp mapping->flags = 0; mapping->wb_err = 0; atomic_set(&mapping->i_mmap_writable, 0); -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - atomic_set(&mapping->nr_thps, 0); -#endif mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE); mapping->writeback_index = 0; init_rwsem(&mapping->invalidate_lock); @@ -518,15 +511,6 @@ static void init_once(void *foo) inode_init_once(inode); } -/* - * get additional reference to inode; caller must already hold one. - */ -void ihold(struct inode *inode) -{ - WARN_ON(atomic_inc_return(&inode->i_count) < 2); -} -EXPORT_SYMBOL(ihold); - struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe, struct inode *inode, u32 bit) { @@ -902,7 +886,7 @@ void evict_inodes(struct super_block *sb) again: spin_lock(&sb->s_inode_list_lock); list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { - if (icount_read(inode)) + if (icount_read_once(inode)) continue; spin_lock(&inode->i_lock); @@ -1032,6 +1016,7 @@ long prune_icache_sb(struct super_block *sb, struct shrink_control *sc) } static void __wait_on_freeing_inode(struct inode *inode, bool hash_locked, bool rcu_locked); +static bool igrab_from_hash(struct inode *inode); /* * Called with the inode lock held. @@ -1056,6 +1041,11 @@ repeat: continue; if (!test(inode, data)) continue; + if (igrab_from_hash(inode)) { + rcu_read_unlock(); + *isnew = false; + return inode; + } spin_lock(&inode->i_lock); if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) { __wait_on_freeing_inode(inode, hash_locked, true); @@ -1098,6 +1088,11 @@ repeat: continue; if (inode->i_sb != sb) continue; + if (igrab_from_hash(inode)) { + rcu_read_unlock(); + *isnew = false; + return inode; + } spin_lock(&inode->i_lock); if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) { __wait_on_freeing_inode(inode, hash_locked, true); @@ -1215,6 +1210,10 @@ void unlock_new_inode(struct inode *inode) lockdep_annotate_inode_mutex_key(inode); spin_lock(&inode->i_lock); WARN_ON(!(inode_state_read(inode) & I_NEW)); + /* + * Paired with igrab_from_hash() + */ + smp_wmb(); inode_state_clear(inode, I_NEW | I_CREATING); inode_wake_up_bit(inode, __I_NEW); spin_unlock(&inode->i_lock); @@ -1226,6 +1225,10 @@ void discard_new_inode(struct inode *inode) lockdep_annotate_inode_mutex_key(inode); spin_lock(&inode->i_lock); WARN_ON(!(inode_state_read(inode) & I_NEW)); + /* + * Paired with igrab_from_hash() + */ + smp_wmb(); inode_state_clear(inode, I_NEW); inode_wake_up_bit(inode, __I_NEW); spin_unlock(&inode->i_lock); @@ -1572,8 +1575,27 @@ ino_t iunique(struct super_block *sb, ino_t max_reserved) } EXPORT_SYMBOL(iunique); +/** + * ihold - get a reference on the inode, provided you already have one + * @inode: inode to operate on + */ +void ihold(struct inode *inode) +{ + VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); + WARN_ON(atomic_inc_return(&inode->i_count) < 2); +} +EXPORT_SYMBOL(ihold); + struct inode *igrab(struct inode *inode) { + /* + * Read commentary above igrab_from_hash() for an explanation why this works. + */ + if (atomic_add_unless(&inode->i_count, 1, 0)) { + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode); + return inode; + } + spin_lock(&inode->i_lock); if (!(inode_state_read(inode) & (I_FREEING | I_WILL_FREE))) { __iget(inode); @@ -1591,6 +1613,43 @@ struct inode *igrab(struct inode *inode) } EXPORT_SYMBOL(igrab); +/* + * igrab_from_hash - special inode refcount acquire primitive for the inode hash + * + * It provides lockless refcount acquire in the common case of no problematic + * flags being set and the count being > 0. + * + * There are 4 state flags to worry about and the routine makes sure to not bump the + * ref if any of them is present. + * + * I_NEW and I_CREATING can only legally get set *before* the inode becomes visible + * during lookup. Thus if the flags are not spotted, they are guaranteed to not be + * a factor. However, we need an acquire fence before returning the inode just + * in case we raced against clearing the state to make sure our consumer picks up + * any other changes made prior. atomic_add_unless provides a full fence, which + * takes care of it. + * + * I_FREEING and I_WILL_FREE can only legally get set if ->i_count == 0 and it is + * illegal to bump the ref if either is present. Consequently if atomic_add_unless + * managed to replace a non-0 value with a bigger one, we have a guarantee neither + * of these flags is set. Note this means explicitly checking of these flags below + * is not necessary, it is only done because it does not cost anything on top of the + * load which already needs to be done to handle the other flags. + */ +static bool igrab_from_hash(struct inode *inode) +{ + if (inode_state_read_once(inode) & (I_NEW | I_CREATING | I_FREEING | I_WILL_FREE)) + return false; + /* + * Paired with routines clearing I_NEW + */ + if (atomic_add_unless(&inode->i_count, 1, 0)) { + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode); + return true; + } + return false; +} + /** * ilookup5_nowait - search for an inode in the inode cache * @sb: super block of file system to search @@ -1920,7 +1979,7 @@ static void iput_final(struct inode *inode) int drop; WARN_ON(inode_state_read(inode) & I_NEW); - VFS_BUG_ON_INODE(atomic_read(&inode->i_count) != 0, inode); + VFS_BUG_ON_INODE(icount_read(inode) != 0, inode); if (op->drop_inode) drop = op->drop_inode(inode); @@ -1939,7 +1998,7 @@ static void iput_final(struct inode *inode) * Re-check ->i_count in case the ->drop_inode() hooks played games. * Note we only execute this if the verdict was to drop the inode. */ - VFS_BUG_ON_INODE(atomic_read(&inode->i_count) != 0, inode); + VFS_BUG_ON_INODE(icount_read(inode) != 0, inode); if (drop) { inode_state_set(inode, I_FREEING); @@ -1983,7 +2042,7 @@ retry: * equal to one, then two CPUs racing to further drop it can both * conclude it's fine. */ - VFS_BUG_ON_INODE(atomic_read(&inode->i_count) < 1, inode); + VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); if (atomic_add_unless(&inode->i_count, -1, 1)) return; @@ -2017,7 +2076,7 @@ EXPORT_SYMBOL(iput); void iput_not_last(struct inode *inode) { VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); - VFS_BUG_ON_INODE(atomic_read(&inode->i_count) < 2, inode); + VFS_BUG_ON_INODE(icount_read_once(inode) < 2, inode); WARN_ON(atomic_sub_return(1, &inode->i_count) == 0); } @@ -2124,7 +2183,13 @@ static int inode_update_cmtime(struct inode *inode, unsigned int flags) inode_iversion_need_inc(inode)) return -EAGAIN; } else { - if (inode_maybe_inc_iversion(inode, !!dirty)) + /* + * Don't force iversion increment for pure lazytime + * updates (I_DIRTY_TIME only), let I_VERSION_QUERIED + * dictate whether the increment is needed. + */ + if (inode_maybe_inc_iversion(inode, + dirty != I_DIRTY_TIME)) dirty |= I_DIRTY_SYNC; } } @@ -3040,7 +3105,7 @@ void dump_inode(struct inode *inode, const char *reason) } state = inode_state_read_once(inode); - count = atomic_read(&inode->i_count); + count = icount_read_once(inode); if (!sb || get_kernel_nofault(s_type, &sb->s_type) || !s_type || diff --git a/fs/internal.h b/fs/internal.h index d77578d66d42..355d93f92208 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -138,7 +138,6 @@ extern bool super_trylock_shared(struct super_block *sb); struct super_block *user_get_super(dev_t, bool excl); void put_super(struct super_block *sb); extern bool mount_capable(struct fs_context *); -int sb_init_dio_done_wq(struct super_block *sb); /* * Prepare superblock for changing its read-only state (i.e., either remount diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c index 4504f4633f17..dc8ac7e370a5 100644 --- a/fs/iomap/bio.c +++ b/fs/iomap/bio.c @@ -78,14 +78,24 @@ u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend) return __iomap_read_end_io(&ioend->io_bio, ioend->io_error); } -static void iomap_bio_submit_read(const struct iomap_iter *iter, - struct iomap_read_folio_ctx *ctx) +void iomap_bio_submit_read_endio(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io) { struct bio *bio = ctx->read_ctx; + bio->bi_end_io = end_io; if (iter->iomap.flags & IOMAP_F_INTEGRITY) fs_bio_integrity_alloc(bio); submit_bio(bio); + + ctx->read_ctx = NULL; +} +EXPORT_SYMBOL_GPL(iomap_bio_submit_read_endio); + +static void iomap_bio_submit_read(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + return iomap_bio_submit_read_endio(iter, ctx, iomap_read_end_io); } static struct bio_set *iomap_read_bio_set(struct iomap_read_folio_ctx *ctx) @@ -127,7 +137,6 @@ static void iomap_read_alloc_bio(const struct iomap_iter *iter, if (ctx->rac) bio->bi_opf |= REQ_RAHEAD; bio->bi_iter.bi_sector = iomap_sector(iomap, iter->pos); - bio->bi_end_io = iomap_read_end_io; bio_add_folio_nofail(bio, folio, plen, offset_in_folio(folio, iter->pos)); ctx->read_ctx = bio; diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index d7b648421a70..276720bc18dc 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -9,6 +9,7 @@ #include <linux/swap.h> #include <linux/migrate.h> #include <linux/fserror.h> +#include <linux/fsverity.h> #include "internal.h" #include "trace.h" @@ -353,9 +354,26 @@ static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter, { const struct iomap *srcmap = iomap_iter_srcmap(iter); - return srcmap->type != IOMAP_MAPPED || - (srcmap->flags & IOMAP_F_NEW) || - pos >= i_size_read(iter->inode); + /* + * If this block has not been written, there's nothing to read + */ + if (srcmap->type != IOMAP_MAPPED) + return true; + + /* + * Newly allocated blocks have not been written + */ + if (srcmap->flags & IOMAP_F_NEW) + return true; + + /* + * fsverity metadata is stored past i_size, we need to read it instead + * of zeroing + */ + if (srcmap->flags & IOMAP_F_FSVERITY) + return false; + + return pos >= i_size_read(iter->inode); } /** @@ -400,6 +418,11 @@ void iomap_finish_folio_read(struct folio *folio, size_t off, size_t len, bool uptodate = !error; bool finished = true; + if (error) + fserror_report_io(folio->mapping->host, FSERR_BUFFERED_READ, + folio_pos(folio) + off, len, error, + GFP_ATOMIC); + if (ifs) { unsigned long flags; @@ -411,11 +434,6 @@ void iomap_finish_folio_read(struct folio *folio, size_t off, size_t len, spin_unlock_irqrestore(&ifs->state_lock, flags); } - if (error) - fserror_report_io(folio->mapping->host, FSERR_BUFFERED_READ, - folio_pos(folio) + off, len, error, - GFP_ATOMIC); - if (finished) folio_end_read(folio, uptodate); } @@ -544,9 +562,27 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, if (plen == 0) return 0; - /* zero post-eof blocks as the page may be mapped */ - if (iomap_block_needs_zeroing(iter, pos)) { + /* + * Handling of fsverity "holes". We hit this for two case: + * 1. No need to go further, the hole after fsverity + * descriptor is the end of the fsverity metadata. + * + * 2. This folio contains merkle tree blocks which need to be + * synthesized. If we already have fsverity info (ctx->vi) + * synthesize these blocks. + */ + if ((iomap->flags & IOMAP_F_FSVERITY) && + iomap->type == IOMAP_HOLE) { + if (ctx->vi) + fsverity_fill_zerohash(folio, poff, plen, + ctx->vi); + iomap_set_range_uptodate(folio, poff, plen); + } else if (iomap_block_needs_zeroing(iter, pos)) { + /* zero post-eof blocks as the page may be mapped */ folio_zero_range(folio, poff, plen); + if (ctx->vi && + !fsverity_verify_blocks(ctx->vi, folio, plen, poff)) + return -EIO; iomap_set_range_uptodate(folio, poff, plen); } else { if (!*bytes_submitted) @@ -597,12 +633,21 @@ void iomap_read_folio(const struct iomap_ops *ops, trace_iomap_readpage(iter.inode, 1); - while ((ret = iomap_iter(&iter, ops)) > 0) + /* + * Fetch fsverity_info for both data and fsverity metadata, as iomap + * needs zeroed hash for merkle tree block synthesis + */ + ctx->vi = fsverity_get_info(iter.inode); + if (ctx->vi && iter.pos < i_size_read(iter.inode)) + fsverity_readahead(ctx->vi, folio->index, + folio_nr_pages(folio)); + + while ((ret = iomap_iter(&iter, ops)) > 0) { iter.status = iomap_read_folio_iter(&iter, ctx, &bytes_submitted); - - if (ctx->read_ctx && ctx->ops->submit_read) - ctx->ops->submit_read(&iter, ctx); + if (ctx->read_ctx && ctx->ops->submit_read) + ctx->ops->submit_read(&iter, ctx); + } if (ctx->cur_folio) iomap_read_end(ctx->cur_folio, bytes_submitted); @@ -664,12 +709,21 @@ void iomap_readahead(const struct iomap_ops *ops, trace_iomap_readahead(rac->mapping->host, readahead_count(rac)); - while (iomap_iter(&iter, ops) > 0) + /* + * Fetch fsverity_info for both data and fsverity metadata, as iomap + * needs zeroed hash for merkle tree block synthesis + */ + ctx->vi = fsverity_get_info(iter.inode); + if (ctx->vi && iter.pos < i_size_read(iter.inode)) + fsverity_readahead(ctx->vi, readahead_index(rac), + readahead_count(rac)); + + while (iomap_iter(&iter, ops) > 0) { iter.status = iomap_readahead_iter(&iter, ctx, &cur_bytes_submitted); - - if (ctx->read_ctx && ctx->ops->submit_read) - ctx->ops->submit_read(&iter, ctx); + if (ctx->read_ctx && ctx->ops->submit_read) + ctx->ops->submit_read(&iter, ctx); + } if (ctx->cur_folio) iomap_read_end(ctx->cur_folio, cur_bytes_submitted); @@ -836,6 +890,7 @@ static int __iomap_write_begin(const struct iomap_iter *iter, return -EIO; folio_zero_segments(folio, poff, from, to, poff + plen); } else { + const struct iomap *iomap = iomap_iter_srcmap(iter); int status; if (iter->flags & IOMAP_NOWAIT) @@ -850,9 +905,12 @@ static int __iomap_write_begin(const struct iomap_iter *iter, if (status < 0) fserror_report_io(iter->inode, FSERR_BUFFERED_READ, pos, - len, status, GFP_NOFS); + plen, status, GFP_NOFS); if (status) return status; + + if (iomap->flags & IOMAP_F_ZERO_TAIL) + folio_zero_segment(folio, to, poff + plen); } iomap_set_range_uptodate(folio, poff, plen); } while ((block_start += plen) < block_end); @@ -1058,7 +1116,6 @@ static bool iomap_write_end_inline(const struct iomap_iter *iter, void *addr; WARN_ON_ONCE(!folio_test_uptodate(folio)); - BUG_ON(!iomap_inline_data_valid(iomap)); if (WARN_ON_ONCE(!iomap->inline_data)) return false; @@ -1167,13 +1224,14 @@ retry: * unlock and release the folio. */ old_size = iter->inode->i_size; - if (pos + written > old_size) { + if (pos + written > old_size && + !(iter->iomap.flags & IOMAP_F_FSVERITY)) { i_size_write(iter->inode, pos + written); iter->iomap.flags |= IOMAP_F_SIZE_CHANGED; } __iomap_put_folio(iter, write_ops, written, folio); - if (old_size < pos) + if (old_size < pos && !(iter->iomap.flags & IOMAP_F_FSVERITY)) pagecache_isize_extended(iter->inode, old_size, pos); cond_resched(); @@ -1232,6 +1290,31 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, } EXPORT_SYMBOL_GPL(iomap_file_buffered_write); +int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, + const void *buf, const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops) +{ + int ret; + struct iov_iter iiter; + struct kvec kvec = { + .iov_base = (void *)buf, + .iov_len = length, + }; + struct kiocb iocb = { + .ki_filp = file, + .ki_ioprio = get_current_ioprio(), + .ki_pos = pos, + }; + + iov_iter_kvec(&iiter, WRITE, &kvec, 1, length); + + ret = iomap_file_buffered_write(&iocb, &iiter, ops, write_ops, NULL); + if (ret < 0) + return ret; + return ret == length ? 0 : -EIO; +} +EXPORT_SYMBOL_GPL(iomap_fsverity_write); + static void iomap_write_delalloc_ifs_punch(struct inode *inode, struct folio *folio, loff_t start_byte, loff_t end_byte, struct iomap *iomap, iomap_punch_t punch) @@ -1543,6 +1626,8 @@ static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero, size_t offset; bool ret; + balance_dirty_pages_ratelimited(iter->inode->i_mapping); + bytes = min_t(u64, SIZE_MAX, bytes); status = iomap_write_begin(iter, write_ops, &folio, &offset, &bytes); @@ -1797,13 +1882,20 @@ static int iomap_writeback_range(struct iomap_writepage_ctx *wpc, * Check interaction of the folio with the file end. * * If the folio is entirely beyond i_size, return false. If it straddles - * i_size, adjust end_pos and zero all data beyond i_size. + * i_size, adjust end_pos and zero all data beyond i_size. Don't skip fsverity + * folios as those are beyond i_size. */ -static bool iomap_writeback_handle_eof(struct folio *folio, struct inode *inode, - u64 *end_pos) +static bool iomap_writeback_handle_eof(struct folio *folio, + struct iomap_writepage_ctx *wpc, u64 *end_pos) { + struct inode *inode = wpc->inode; u64 isize = i_size_read(inode); + if (wpc->iomap.flags & IOMAP_F_FSVERITY) { + WARN_ON_ONCE(folio_pos(folio) < isize); + return true; + } + if (*end_pos > isize) { size_t poff = offset_in_folio(folio, isize); pgoff_t end_index = isize >> PAGE_SHIFT; @@ -1869,7 +1961,7 @@ int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio) trace_iomap_writeback_folio(inode, pos, folio_size(folio)); - if (!iomap_writeback_handle_eof(folio, inode, &end_pos)) + if (!iomap_writeback_handle_eof(folio, wpc, &end_pos)) return 0; WARN_ON_ONCE(end_pos <= pos); diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index b0a6549b3848..e2cd5f92babe 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -69,7 +69,7 @@ static void iomap_dio_submit_bio(const struct iomap_iter *iter, /* Sync dio can't be polled reliably */ if ((iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(iocb)) { - bio_set_polled(bio, iocb); + bio->bi_opf |= REQ_POLLED; WRITE_ONCE(iocb->private, bio); } @@ -355,7 +355,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, if (dio->flags & IOMAP_DIO_BOUNCE) ret = bio_iov_iter_bounce(bio, dio->submit.iter, - iomap_max_bio_size(&iter->iomap)); + iomap_max_bio_size(&iter->iomap), alignment); else ret = bio_iov_iter_get_pages(bio, dio->submit.iter, alignment - 1); @@ -369,7 +369,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, */ if ((op & REQ_ATOMIC) && WARN_ON_ONCE(ret != iomap_length(iter))) { ret = -EINVAL; - goto out_put_bio; + goto out_bio_release_pages; } if (iter->iomap.flags & IOMAP_F_INTEGRITY) { @@ -393,6 +393,11 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, iomap_dio_submit_bio(iter, dio, bio, pos); return ret; +out_bio_release_pages: + if (dio->flags & IOMAP_DIO_BOUNCE) + bio_iov_iter_unbounce(bio, true, false); + else + bio_release_pages(bio, false); out_put_bio: bio_put(bio); return ret; @@ -603,9 +608,6 @@ static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio) if (WARN_ON_ONCE(!inline_data)) return -EIO; - if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap))) - return -EIO; - if (dio->flags & IOMAP_DIO_WRITE) { loff_t size = iomi->inode->i_size; diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index acf3cf98b23a..0565328764c1 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -28,6 +28,7 @@ struct iomap_ioend *iomap_init_ioend(struct inode *inode, ioend->io_offset = file_offset; ioend->io_size = bio->bi_iter.bi_size; ioend->io_sector = bio->bi_iter.bi_sector; + ioend->io_vi = NULL; ioend->io_private = NULL; return ioend; } @@ -297,8 +298,12 @@ new_ioend: * appending writes. */ ioend->io_size += map_len; - if (ioend->io_offset + ioend->io_size > end_pos) - ioend->io_size = end_pos - ioend->io_offset; + if (ioend->io_offset + ioend->io_size > end_pos) { + if (ioend->io_offset >= end_pos) + ioend->io_size = 0; + else + ioend->io_size = end_pos - ioend->io_offset; + } wbc_account_cgroup_owner(wpc->wbc, folio, map_len); return map_len; diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index c04796f6e57f..e4a29829591a 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -6,17 +6,13 @@ #include <linux/iomap.h> #include "trace.h" -static inline void iomap_iter_reset_iomap(struct iomap_iter *iter) +static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter) { if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) { folio_batch_release(iter->fbatch); folio_batch_reinit(iter->fbatch); iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; } - - iter->status = 0; - memset(&iter->iomap, 0, sizeof(iter->iomap)); - memset(&iter->srcmap, 0, sizeof(iter->srcmap)); } /* Advance the current iterator position and decrement the remaining length */ @@ -102,10 +98,14 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) ret = 0; else ret = 1; - iomap_iter_reset_iomap(iter); + iomap_iter_clean_fbatch(iter); + iter->status = 0; if (ret <= 0) return ret; + memset(&iter->iomap, 0, sizeof(iter->iomap)); + memset(&iter->srcmap, 0, sizeof(iter->srcmap)); + begin: ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags, &iter->iomap, &iter->srcmap); diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h index 097773c6db80..e1ea8392cf47 100644 --- a/fs/iomap/trace.h +++ b/fs/iomap/trace.h @@ -118,7 +118,9 @@ DEFINE_RANGE_EVENT(iomap_zero_iter); { IOMAP_F_ATOMIC_BIO, "ATOMIC_BIO" }, \ { IOMAP_F_PRIVATE, "PRIVATE" }, \ { IOMAP_F_SIZE_CHANGED, "SIZE_CHANGED" }, \ - { IOMAP_F_STALE, "STALE" } + { IOMAP_F_STALE, "STALE" }, \ + { IOMAP_F_FSVERITY, "FSVERITY" }, \ + { IOMAP_F_ZERO_TAIL, "ZERO TAIL" } #define IOMAP_DIO_STRINGS \ diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c index 2fd9948d606e..cc587cd25162 100644 --- a/fs/isofs/dir.c +++ b/fs/isofs/dir.c @@ -13,7 +13,9 @@ */ #include <linux/gfp.h> #include <linux/filelock.h> +#include <linux/slab.h> #include "isofs.h" +#include <linux/fileattr.h> int isofs_name_translate(struct iso_directory_record *de, char *new, struct inode *inode) { @@ -255,7 +257,7 @@ static int isofs_readdir(struct file *file, struct dir_context *ctx) struct iso_directory_record *tmpde; struct inode *inode = file_inode(file); - tmpname = (char *)__get_free_page(GFP_KERNEL); + tmpname = kmalloc(PAGE_SIZE, GFP_KERNEL); if (tmpname == NULL) return -ENOMEM; @@ -263,10 +265,24 @@ static int isofs_readdir(struct file *file, struct dir_context *ctx) result = do_isofs_readdir(inode, file, ctx, tmpname, tmpde); - free_page((unsigned long) tmpname); + kfree(tmpname); return result; } +int isofs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct isofs_sb_info *sbi = ISOFS_SB(dentry->d_sb); + + if (sbi->s_check == 'r') { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + if (!sbi->s_joliet_level && !sbi->s_rock && + (sbi->s_mapping == 'n' || sbi->s_mapping == 'a')) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + return 0; +} + const struct file_operations isofs_dir_operations = { .llseek = generic_file_llseek, @@ -281,6 +297,7 @@ const struct file_operations isofs_dir_operations = const struct inode_operations isofs_dir_inode_operations = { .lookup = isofs_lookup, + .fileattr_get = isofs_fileattr_get, }; diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index efee53717f1c..337836a0a170 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -818,7 +818,8 @@ root_found: * entries. By forcing the blocksize in this way, we ensure * that we will never be required to do this. */ - sb_set_blocksize(s, orig_zonesize); + if (!sb_set_blocksize(s, orig_zonesize)) + goto out_freesbi; sbi->s_nls_iocharset = NULL; diff --git a/fs/isofs/isofs.h b/fs/isofs/isofs.h index 506555837533..0ec8b24a42ed 100644 --- a/fs/isofs/isofs.h +++ b/fs/isofs/isofs.h @@ -197,6 +197,9 @@ isofs_normalize_block_and_offset(struct iso_directory_record* de, } } +struct file_kattr; +int isofs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + extern const struct inode_operations isofs_dir_inode_operations; extern const struct file_operations isofs_dir_operations; extern const struct address_space_operations isofs_symlink_aops; diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c index 1232fab59a4e..2628f31bd3a5 100644 --- a/fs/isofs/rock.c +++ b/fs/isofs/rock.c @@ -466,6 +466,9 @@ repeat: inode->i_size = symlink_len; while (slen > 1) { rootflag = 0; + /* keep the component within the SL record */ + if (slp->len + 2 > slen) + goto eio; switch (slp->flags & ~1) { case 0: inode->i_size += @@ -621,6 +624,14 @@ static char *get_symlink_chunk(char *rpnt, struct rock_ridge *rr, char *plimit) slp = &rr->u.SL.link; while (slen > 1) { rootflag = 0; + /* + * A component is slp->len + 2 bytes (a two-byte header plus + * len bytes of text). If it does not fit in the bytes left in + * the SL record the record is malformed: fail like the plimit + * checks below so readlink() returns -EIO, not a truncated path. + */ + if (slp->len + 2 > slen) + return NULL; switch (slp->flags & ~1) { case 0: if (slp->len > plimit - rpnt) diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index 8cf61e7185c4..3029cb6f6d64 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -29,8 +29,10 @@ /* * IO end handler for temporary buffer_heads handling writes to the journal. */ -static void journal_end_buffer_io_sync(struct buffer_head *bh, int uptodate) +static void journal_end_buffer_io_sync(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); struct buffer_head *orig_bh = bh->b_private; BUFFER_TRACE(bh, ""); @@ -39,9 +41,7 @@ static void journal_end_buffer_io_sync(struct buffer_head *bh, int uptodate) else clear_buffer_uptodate(bh); if (orig_bh) { - clear_bit_unlock(BH_Shadow, &orig_bh->b_state); - smp_mb__after_atomic(); - wake_up_bit(&orig_bh->b_state, BH_Shadow); + clear_and_wake_up_bit(BH_Shadow, &orig_bh->b_state); } unlock_buffer(bh); } @@ -147,13 +147,12 @@ static int journal_submit_commit_record(journal_t *journal, lock_buffer(bh); clear_buffer_dirty(bh); set_buffer_uptodate(bh); - bh->b_end_io = journal_end_buffer_io_sync; if (journal->j_flags & JBD2_BARRIER && !jbd2_has_feature_async_commit(journal)) write_flags |= REQ_PREFLUSH | REQ_FUA; - submit_bh(write_flags, bh); + bh_submit(bh, write_flags, journal_end_buffer_io_sync); *cbh = bh; return 0; } @@ -513,10 +512,8 @@ void jbd2_journal_commit_transaction(journal_t *journal) * leave undo-committed data. */ if (jh->b_committed_data) { - struct buffer_head *bh = jh2bh(jh); - spin_lock(&jh->b_state_lock); - jbd2_free(jh->b_committed_data, bh->b_size); + kfree(jh->b_committed_data); jh->b_committed_data = NULL; spin_unlock(&jh->b_state_lock); } @@ -751,9 +748,9 @@ start_journal_io: lock_buffer(bh); clear_buffer_dirty(bh); set_buffer_uptodate(bh); - bh->b_end_io = journal_end_buffer_io_sync; - submit_bh(REQ_OP_WRITE | JBD2_JOURNAL_REQ_FLAGS, - bh); + bh_submit(bh, + REQ_OP_WRITE | JBD2_JOURNAL_REQ_FLAGS, + journal_end_buffer_io_sync); } cond_resched(); @@ -977,7 +974,7 @@ restart_loop: * its triggers if they exist, so we can clear that too. */ if (jh->b_committed_data) { - jbd2_free(jh->b_committed_data, bh->b_size); + kfree(jh->b_committed_data); jh->b_committed_data = NULL; if (jh->b_frozen_data) { jh->b_committed_data = jh->b_frozen_data; @@ -985,7 +982,7 @@ restart_loop: jh->b_frozen_triggers = NULL; } } else if (jh->b_frozen_data) { - jbd2_free(jh->b_frozen_data, bh->b_size); + kfree(jh->b_frozen_data); jh->b_frozen_data = NULL; jh->b_frozen_triggers = NULL; } diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 4f397fcdb13c..09efa337649e 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -95,8 +95,6 @@ EXPORT_SYMBOL(jbd2_journal_release_jbd_inode); EXPORT_SYMBOL(jbd2_journal_begin_ordered_truncate); EXPORT_SYMBOL(jbd2_inode_cache); -static int jbd2_journal_create_slab(size_t slab_size); - #ifdef CONFIG_JBD2_DEBUG void __jbd2_debug(int level, const char *file, const char *func, unsigned int line, const char *fmt, ...) @@ -385,10 +383,10 @@ int jbd2_journal_write_metadata_buffer(transaction_t *transaction, goto escape_done; spin_unlock(&jh_in->b_state_lock); - tmp = jbd2_alloc(bh_in->b_size, GFP_NOFS | __GFP_NOFAIL); + tmp = kmalloc(bh_in->b_size, GFP_NOFS | __GFP_NOFAIL); spin_lock(&jh_in->b_state_lock); if (jh_in->b_frozen_data) { - jbd2_free(tmp, bh_in->b_size); + kfree(tmp); goto copy_done; } @@ -1820,9 +1818,7 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) } if (jbd2_journal_has_csum_v2or3(journal)) sb->s_checksum = jbd2_superblock_csum(sb); - get_bh(bh); - bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | write_flags, bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); wait_on_buffer(bh); if (buffer_write_io_error(bh)) { clear_buffer_write_io_error(bh); @@ -2064,14 +2060,6 @@ EXPORT_SYMBOL(jbd2_journal_update_sb_errno); int jbd2_journal_load(journal_t *journal) { int err; - journal_superblock_t *sb = journal->j_superblock; - - /* - * Create a slab for this blocksize - */ - err = jbd2_journal_create_slab(be32_to_cpu(sb->s_blocksize)); - if (err) - return err; /* Let the recovery code check whether it needs to recover any * data from the journal. */ @@ -2263,6 +2251,8 @@ jbd2_journal_initialize_fast_commit(journal_t *journal) unsigned long long num_fc_blks; num_fc_blks = jbd2_journal_get_num_fc_blks(sb); + if (num_fc_blks > journal->j_last) + return -EFSCORRUPTED; if (journal->j_last - num_fc_blks < JBD2_MIN_JOURNAL_BLOCKS) return -ENOSPC; @@ -2700,108 +2690,6 @@ size_t journal_tag_bytes(journal_t *journal) } /* - * JBD memory management - * - * These functions are used to allocate block-sized chunks of memory - * used for making copies of buffer_head data. Very often it will be - * page-sized chunks of data, but sometimes it will be in - * sub-page-size chunks. (For example, 16k pages on Power systems - * with a 4k block file system.) For blocks smaller than a page, we - * use a SLAB allocator. There are slab caches for each block size, - * which are allocated at mount time, if necessary, and we only free - * (all of) the slab caches when/if the jbd2 module is unloaded. For - * this reason we don't need to a mutex to protect access to - * jbd2_slab[] allocating or releasing memory; only in - * jbd2_journal_create_slab(). - */ -#define JBD2_MAX_SLABS 8 -static struct kmem_cache *jbd2_slab[JBD2_MAX_SLABS]; - -static const char *jbd2_slab_names[JBD2_MAX_SLABS] = { - "jbd2_1k", "jbd2_2k", "jbd2_4k", "jbd2_8k", - "jbd2_16k", "jbd2_32k", "jbd2_64k", "jbd2_128k" -}; - - -static void jbd2_journal_destroy_slabs(void) -{ - int i; - - for (i = 0; i < JBD2_MAX_SLABS; i++) { - kmem_cache_destroy(jbd2_slab[i]); - jbd2_slab[i] = NULL; - } -} - -static int jbd2_journal_create_slab(size_t size) -{ - static DEFINE_MUTEX(jbd2_slab_create_mutex); - int i = order_base_2(size) - 10; - size_t slab_size; - - if (size == PAGE_SIZE) - return 0; - - if (i >= JBD2_MAX_SLABS) - return -EINVAL; - - if (unlikely(i < 0)) - i = 0; - mutex_lock(&jbd2_slab_create_mutex); - if (jbd2_slab[i]) { - mutex_unlock(&jbd2_slab_create_mutex); - return 0; /* Already created */ - } - - slab_size = 1 << (i+10); - jbd2_slab[i] = kmem_cache_create(jbd2_slab_names[i], slab_size, - slab_size, 0, NULL); - mutex_unlock(&jbd2_slab_create_mutex); - if (!jbd2_slab[i]) { - printk(KERN_EMERG "JBD2: no memory for jbd2_slab cache\n"); - return -ENOMEM; - } - return 0; -} - -static struct kmem_cache *get_slab(size_t size) -{ - int i = order_base_2(size) - 10; - - BUG_ON(i >= JBD2_MAX_SLABS); - if (unlikely(i < 0)) - i = 0; - BUG_ON(jbd2_slab[i] == NULL); - return jbd2_slab[i]; -} - -void *jbd2_alloc(size_t size, gfp_t flags) -{ - void *ptr; - - BUG_ON(size & (size-1)); /* Must be a power of 2 */ - - if (size < PAGE_SIZE) - ptr = kmem_cache_alloc(get_slab(size), flags); - else - ptr = (void *)__get_free_pages(flags, get_order(size)); - - /* Check alignment; SLUB has gotten this wrong in the past, - * and this can lead to user data corruption! */ - BUG_ON(((unsigned long) ptr) & (size-1)); - - return ptr; -} - -void jbd2_free(void *ptr, size_t size) -{ - if (size < PAGE_SIZE) - kmem_cache_free(get_slab(size), ptr); - else - free_pages((unsigned long)ptr, get_order(size)); -}; - -/* * Journal_head storage management */ static struct kmem_cache *jbd2_journal_head_cache; @@ -2974,15 +2862,15 @@ static void __journal_remove_journal_head(struct buffer_head *bh) clear_buffer_jbd(bh); } -static void journal_release_journal_head(struct journal_head *jh, size_t b_size) +static void journal_release_journal_head(struct journal_head *jh) { if (jh->b_frozen_data) { printk(KERN_WARNING "%s: freeing b_frozen_data\n", __func__); - jbd2_free(jh->b_frozen_data, b_size); + kfree(jh->b_frozen_data); } if (jh->b_committed_data) { printk(KERN_WARNING "%s: freeing b_committed_data\n", __func__); - jbd2_free(jh->b_committed_data, b_size); + kfree(jh->b_committed_data); } journal_free_journal_head(jh); } @@ -3001,7 +2889,7 @@ void jbd2_journal_put_journal_head(struct journal_head *jh) if (!jh->b_jcount) { __journal_remove_journal_head(bh); jbd_unlock_bh_journal_head(bh); - journal_release_journal_head(jh, bh->b_size); + journal_release_journal_head(jh); __brelse(bh); } else { jbd_unlock_bh_journal_head(bh); @@ -3143,7 +3031,6 @@ static void jbd2_journal_destroy_caches(void) jbd2_journal_destroy_handle_cache(); jbd2_journal_destroy_inode_cache(); jbd2_journal_destroy_transaction_cache(); - jbd2_journal_destroy_slabs(); } static int __init journal_init(void) diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c index 4885903bbd10..5cc7d097b2ac 100644 --- a/fs/jbd2/transaction.c +++ b/fs/jbd2/transaction.c @@ -1131,7 +1131,7 @@ repeat: if (!frozen_buffer) { JBUFFER_TRACE(jh, "allocate memory for buffer"); spin_unlock(&jh->b_state_lock); - frozen_buffer = jbd2_alloc(jh2bh(jh)->b_size, + frozen_buffer = kmalloc(jh2bh(jh)->b_size, GFP_NOFS | __GFP_NOFAIL); goto repeat; } @@ -1159,7 +1159,7 @@ done: out: if (unlikely(frozen_buffer)) /* It's usually NULL */ - jbd2_free(frozen_buffer, bh->b_size); + kfree(frozen_buffer); JBUFFER_TRACE(jh, "exit"); return error; @@ -1424,7 +1424,7 @@ int jbd2_journal_get_undo_access(handle_t *handle, struct buffer_head *bh) repeat: if (!jh->b_committed_data) - committed_data = jbd2_alloc(jh2bh(jh)->b_size, + committed_data = kmalloc(jh2bh(jh)->b_size, GFP_NOFS|__GFP_NOFAIL); spin_lock(&jh->b_state_lock); @@ -1445,7 +1445,7 @@ repeat: out: jbd2_journal_put_journal_head(jh); if (unlikely(committed_data)) - jbd2_free(committed_data, bh->b_size); + kfree(committed_data); return err; } @@ -1516,14 +1516,19 @@ void jbd2_buffer_abort_trigger(struct journal_head *jh, */ int jbd2_journal_dirty_metadata(handle_t *handle, struct buffer_head *bh) { - transaction_t *transaction = handle->h_transaction; - journal_t *journal = transaction->t_journal; + transaction_t *transaction; + journal_t *journal; struct journal_head *jh; int ret = 0; + if (is_handle_aborted(handle)) + return -EROFS; if (!buffer_jbd(bh)) return -EUCLEAN; + transaction = handle->h_transaction; + journal = transaction->t_journal; + /* * We don't grab jh reference here since the buffer must be part * of the running transaction. diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c index ac0f79fafaca..8ce6e4458cc2 100644 --- a/fs/jfs/jfs_dtree.c +++ b/fs/jfs/jfs_dtree.c @@ -2729,7 +2729,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) struct ldtentry *d; struct dtslot *t; int d_namleft, len, outlen; - unsigned long dirent_buf; + void *dirent_buf; char *name_ptr; u32 dir_index; int do_index = 0; @@ -2884,7 +2884,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) } } - dirent_buf = __get_free_page(GFP_KERNEL); + dirent_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); if (dirent_buf == 0) { DT_PUTPAGE(mp); jfs_warn("jfs_readdir: __get_free_page failed!"); @@ -2893,7 +2893,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) } while (1) { - jfs_dirent = (struct jfs_dirent *) dirent_buf; + jfs_dirent = dirent_buf; jfs_dirents = 0; overflow = fix_page = 0; @@ -2903,7 +2903,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) if (stbl[i] < 0) { jfs_err("JFS: Invalid stbl[%d] = %d for inode %ld, block = %lld", i, stbl[i], (long)ip->i_ino, (long long)bn); - free_page(dirent_buf); + kfree(dirent_buf); DT_PUTPAGE(mp); return -EIO; } @@ -2911,7 +2911,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) d = (struct ldtentry *) & p->slot[stbl[i]]; if (((long) jfs_dirent + d->namlen + 1) > - (dirent_buf + PAGE_SIZE)) { + ((long)dirent_buf + PAGE_SIZE)) { /* DBCS codepages could overrun dirent_buf */ index = i; overflow = 1; @@ -3014,7 +3014,7 @@ skip_one: /* unpin previous leaf page */ DT_PUTPAGE(mp); - jfs_dirent = (struct jfs_dirent *) dirent_buf; + jfs_dirent = dirent_buf; while (jfs_dirents--) { ctx->pos = jfs_dirent->position; if (!dir_emit(ctx, jfs_dirent->name, @@ -3037,13 +3037,13 @@ skip_one: DT_GETPAGE(ip, bn, mp, PSIZE, p, rc); if (rc) { - free_page(dirent_buf); + kfree(dirent_buf); return rc; } } out: - free_page(dirent_buf); + kfree(dirent_buf); return rc; } diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c index 60c4a0e0fca5..442d62679262 100644 --- a/fs/jfs/namei.c +++ b/fs/jfs/namei.c @@ -309,7 +309,7 @@ static struct dentry *jfs_mkdir(struct mnt_idmap *idmap, struct inode *dip, out1: jfs_info("jfs_mkdir: rc:%d", rc); - return ERR_PTR(rc); + return rc ? ERR_PTR(rc) : NULL; } /* diff --git a/fs/jfs/super.c b/fs/jfs/super.c index 61575f7397ae..8180d83d33fe 100644 --- a/fs/jfs/super.c +++ b/fs/jfs/super.c @@ -491,7 +491,8 @@ static int jfs_fill_super(struct super_block *sb, struct fs_context *fc) /* * Initialize blocksize to 4K. */ - sb_set_blocksize(sb, PSIZE); + if (!sb_set_blocksize(sb, PSIZE)) + goto out_unload; /* * Set method vectors. diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index 4f9ade82b08a..97d9d227b66d 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -597,19 +597,19 @@ void kernfs_put(struct kernfs_node *kn) */ parent = kernfs_parent(kn); - WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS, - "kernfs_put: %s/%s: released with incorrect active_ref %d\n", - parent ? rcu_dereference(parent->name) : "", - rcu_dereference(kn->name), atomic_read(&kn->active)); + if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS) { + guard(rcu)(); + WARN_ONCE(1, + "kernfs_put: %s/%s: released with incorrect active_ref %d\n", + parent ? rcu_dereference(parent->name) : "", + rcu_dereference(kn->name), atomic_read(&kn->active)); + } if (kernfs_type(kn) == KERNFS_LINK) kernfs_put(kn->symlink.target_kn); - if (kn->iattr && kn->iattr->xattrs) { - simple_xattrs_free(kn->iattr->xattrs, NULL); - kfree(kn->iattr->xattrs); - kn->iattr->xattrs = NULL; - } + if (kn->iattr) + simple_xattrs_free(&root->xa_cache, &kn->iattr->xattrs, NULL); spin_lock(&root->kernfs_idr_lock); idr_remove(&root->ino_idr, (u32)kernfs_ino(kn)); @@ -624,6 +624,7 @@ void kernfs_put(struct kernfs_node *kn) } else { /* just released the root kn, free @root too */ idr_destroy(&root->ino_idr); + simple_xattr_cache_cleanup(&root->xa_cache); kfree_rcu(root, rcu); } } @@ -700,6 +701,9 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, } if (parent) { + kernfs_get(parent); + rcu_assign_pointer(kn->__parent, parent); + ret = security_kernfs_init_security(parent, kn); if (ret) goto err_out4; @@ -708,11 +712,10 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, return kn; err_out4: + RCU_INIT_POINTER(kn->__parent, NULL); + kernfs_put(parent); if (kn->iattr) { - if (kn->iattr->xattrs) { - simple_xattrs_free(kn->iattr->xattrs, NULL); - kfree(kn->iattr->xattrs); - } + simple_xattrs_free(&root->xa_cache, &kn->iattr->xattrs, NULL); kmem_cache_free(kernfs_iattrs_cache, kn->iattr); } err_out3: @@ -747,10 +750,6 @@ struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, kn = __kernfs_new_node(kernfs_root(parent), parent, name, mode, uid, gid, flags); - if (kn) { - kernfs_get(parent); - rcu_assign_pointer(kn->__parent, parent); - } return kn; } diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 1163aa769738..8e0e90c93372 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -40,22 +40,15 @@ struct kernfs_open_node { static DEFINE_SPINLOCK(kernfs_notify_lock); static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL; +/* Compatibility wrappers - use the common hashed node lock */ static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn) { - int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS); - - return &kernfs_locks->open_file_mutex[idx]; + return kernfs_node_lock_ptr(kn); } static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn) { - struct mutex *lock; - - lock = kernfs_open_file_mutex_ptr(kn); - - mutex_lock(lock); - - return lock; + return kernfs_node_lock(kn); } /** diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index 38b28aa7cd02..2cb20294aaf5 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -37,6 +37,7 @@ static struct kernfs_iattrs *__kernfs_iattrs(struct kernfs_node *kn, bool alloc) if (!ret) return NULL; + INIT_LIST_HEAD_RCU(&ret->xattrs); /* assign default attributes */ ret->ia_uid = GLOBAL_ROOT_UID; ret->ia_gid = GLOBAL_ROOT_GID; @@ -144,8 +145,7 @@ ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size) if (!attrs) return -ENOMEM; - return simple_xattr_list(d_inode(dentry), READ_ONCE(attrs->xattrs), - buf, size); + return simple_xattr_list(d_inode(dentry), &attrs->xattrs, buf, size); } static inline void set_default_inode_attr(struct inode *inode, umode_t mode) @@ -297,34 +297,35 @@ int kernfs_xattr_get(struct kernfs_node *kn, const char *name, void *value, size_t size) { struct kernfs_iattrs *attrs = kernfs_iattrs_noalloc(kn); - struct simple_xattrs *xattrs; + struct simple_xattr_cache *cache = &kernfs_root(kn)->xa_cache; if (!attrs) return -ENODATA; - xattrs = READ_ONCE(attrs->xattrs); - if (!xattrs) - return -ENODATA; - - return simple_xattr_get(xattrs, name, value, size); + return simple_xattr_get(cache, &attrs->xattrs, name, value, size); } int kernfs_xattr_set(struct kernfs_node *kn, const char *name, const void *value, size_t size, int flags) { struct simple_xattr *old_xattr; - struct simple_xattrs *xattrs; struct kernfs_iattrs *attrs; + struct simple_xattr_cache *cache = &kernfs_root(kn)->xa_cache; attrs = kernfs_iattrs(kn); if (!attrs) return -ENOMEM; - xattrs = simple_xattrs_lazy_alloc(&attrs->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); + /* + * Protect xattr modifications with the hashed per-node mutex. + * Multiple superblocks (with different namespaces) can share the same + * kernfs_node, so inode locking alone is insufficient. The hashed mutex + * ensures serialization of concurrent xattr operations on the same node, + * including the lazy allocation of the xattrs structure itself. + */ + CLASS(kernfs_node_lock, lock)(kn); - old_xattr = simple_xattr_set(xattrs, name, value, size, flags); + old_xattr = simple_xattr_set(cache, &attrs->xattrs, name, value, size, flags); if (IS_ERR(old_xattr)) return PTR_ERR(old_xattr); @@ -362,7 +363,6 @@ static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler, { const char *full_name = xattr_full_name(handler, suffix); struct kernfs_node *kn = inode->i_private; - struct simple_xattrs *xattrs; struct kernfs_iattrs *attrs; if (!(kernfs_root(kn)->flags & KERNFS_ROOT_SUPPORT_USER_XATTR)) @@ -372,11 +372,11 @@ static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler, if (!attrs) return -ENOMEM; - xattrs = simple_xattrs_lazy_alloc(&attrs->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); + /* See comment in kernfs_xattr_set() about locking. */ + CLASS(kernfs_node_lock, lock)(kn); - return simple_xattr_set_limited(xattrs, &attrs->xattr_limits, + return simple_xattr_set_limited(&kernfs_root(kn)->xa_cache, + &attrs->xattrs, &attrs->xattr_limits, full_name, value, size, flags); } diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index 8d8912f50b05..aa784b540b36 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -26,7 +26,7 @@ struct kernfs_iattrs { struct timespec64 ia_mtime; struct timespec64 ia_ctime; - struct simple_xattrs *xattrs; + struct list_head xattrs; struct simple_xattr_limits xattr_limits; }; @@ -54,6 +54,8 @@ struct kernfs_root { rwlock_t kernfs_rename_lock; struct rcu_head rcu; + + struct simple_xattr_cache xa_cache; }; /* +1 to avoid triggering overflow warning when negating it */ @@ -211,4 +213,24 @@ extern const struct inode_operations kernfs_symlink_iops; * kernfs locks */ extern struct kernfs_global_locks *kernfs_locks; + +/* Hashed mutex helpers - protect per-node data structures */ +static inline struct mutex *kernfs_node_lock_ptr(struct kernfs_node *kn) +{ + int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS); + + return &kernfs_locks->node_mutex[idx]; +} + +static inline struct mutex *kernfs_node_lock(struct kernfs_node *kn) +{ + struct mutex *lock = kernfs_node_lock_ptr(kn); + + mutex_lock(lock); + return lock; +} + +DEFINE_CLASS(kernfs_node_lock, struct mutex *, + mutex_unlock(_T), kernfs_node_lock(kn), struct kernfs_node *kn) + #endif /* __KERNFS_INTERNAL_H */ diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index 6e3217b6e481..f183a96778b9 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -446,7 +446,7 @@ static void __init kernfs_mutex_init(void) int count; for (count = 0; count < NR_KERNFS_LOCKS; count++) - mutex_init(&kernfs_locks->open_file_mutex[count]); + mutex_init(&kernfs_locks->node_mutex[count]); } static void __init kernfs_lock_init(void) diff --git a/fs/libfs.c b/fs/libfs.c index 1bbea5e7bae3..5a0d276379d1 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -78,8 +78,7 @@ struct dentry *simple_lookup(struct inode *dir, struct dentry *dentry, unsigned if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir)) return NULL; - d_add(dentry, NULL); - return NULL; + return d_splice_alias(NULL, dentry); } EXPORT_SYMBOL(simple_lookup); @@ -736,6 +735,7 @@ struct pseudo_fs_context *init_pseudo(struct fs_context *fc, fc->fs_private = ctx; fc->ops = &pseudo_fs_context_ops; fc->sb_flags |= SB_NOUSER; + fc->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; fc->global = true; } return ctx; @@ -1258,7 +1258,7 @@ char *simple_transaction_get(struct file *file, const char __user *buf, size_t s if (size > SIMPLE_TRANSACTION_LIMIT - 1) return ERR_PTR(-EFBIG); - ar = (struct simple_transaction_argresp *)get_zeroed_page(GFP_KERNEL); + ar = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!ar) return ERR_PTR(-ENOMEM); @@ -1267,7 +1267,7 @@ char *simple_transaction_get(struct file *file, const char __user *buf, size_t s /* only one write allowed per open */ if (file->private_data) { spin_unlock(&simple_transaction_lock); - free_page((unsigned long)ar); + kfree(ar); return ERR_PTR(-EBUSY); } @@ -1294,7 +1294,7 @@ EXPORT_SYMBOL(simple_transaction_read); int simple_transaction_release(struct inode *inode, struct file *file) { - free_page((unsigned long)file->private_data); + kfree(file->private_data); return 0; } EXPORT_SYMBOL(simple_transaction_release); diff --git a/fs/lockd/Makefile b/fs/lockd/Makefile index 808f0f2a7be1..3f9569d3ba0e 100644 --- a/fs/lockd/Makefile +++ b/fs/lockd/Makefile @@ -8,7 +8,8 @@ ccflags-y += -I$(src) # needed for trace events obj-$(CONFIG_LOCKD) += lockd.o lockd-y := clntlock.o clntproc.o clntxdr.o host.o svc.o svclock.o \ - svcshare.o svcproc.o svcsubs.o mon.o trace.o xdr.o netlink.o + svcshare.o svcproc.o svcsubs.o mon.o trace.o netlink.o \ + nlm3xdr_gen.o lockd-$(CONFIG_LOCKD_V4) += clnt4xdr.o svc4proc.o nlm4xdr_gen.o lockd-$(CONFIG_PROC_FS) += procfs.o @@ -25,17 +26,27 @@ lockd-$(CONFIG_PROC_FS) += procfs.o XDRGEN = ../../tools/net/sunrpc/xdrgen/xdrgen -XDRGEN_DEFINITIONS = ../../include/linux/sunrpc/xdrgen/nlm4.h -XDRGEN_DECLARATIONS = nlm4xdr_gen.h -XDRGEN_SOURCE = nlm4xdr_gen.c +XDRGEN_DEFINITIONS = ../../include/linux/sunrpc/xdrgen/nlm4.h \ + ../../include/linux/sunrpc/xdrgen/nlm3.h +XDRGEN_DECLARATIONS = nlm4xdr_gen.h nlm3xdr_gen.h +XDRGEN_SOURCE = nlm4xdr_gen.c nlm3xdr_gen.c xdrgen: $(XDRGEN_DEFINITIONS) $(XDRGEN_DECLARATIONS) $(XDRGEN_SOURCE) ../../include/linux/sunrpc/xdrgen/nlm4.h: ../../Documentation/sunrpc/xdr/nlm4.x $(XDRGEN) definitions $< > $@ +../../include/linux/sunrpc/xdrgen/nlm3.h: ../../Documentation/sunrpc/xdr/nlm3.x + $(XDRGEN) definitions $< > $@ + nlm4xdr_gen.h: ../../Documentation/sunrpc/xdr/nlm4.x $(XDRGEN) declarations $< > $@ +nlm3xdr_gen.h: ../../Documentation/sunrpc/xdr/nlm3.x + $(XDRGEN) declarations $< > $@ + nlm4xdr_gen.c: ../../Documentation/sunrpc/xdr/nlm4.x $(XDRGEN) source --peer server $< > $@ + +nlm3xdr_gen.c: ../../Documentation/sunrpc/xdr/nlm3.x + $(XDRGEN) source --peer server $< > $@ diff --git a/fs/lockd/clnt4xdr.c b/fs/lockd/clnt4xdr.c index 2058733eacf8..96a4a1e6a6b6 100644 --- a/fs/lockd/clnt4xdr.c +++ b/fs/lockd/clnt4xdr.c @@ -63,7 +63,7 @@ static s64 loff_t_to_s64(loff_t offset) return res; } -static void nlm4_compute_offsets(const struct nlm_lock *lock, +static void nlm4_compute_offsets(const struct lockd_lock *lock, u64 *l_offset, u64 *l_len) { const struct file_lock *fl = &lock->fl; @@ -132,13 +132,13 @@ static int decode_netobj(struct xdr_stream *xdr, * netobj cookie; */ static void encode_cookie(struct xdr_stream *xdr, - const struct nlm_cookie *cookie) + const struct lockd_cookie *cookie) { encode_netobj(xdr, (u8 *)&cookie->data, cookie->len); } static int decode_cookie(struct xdr_stream *xdr, - struct nlm_cookie *cookie) + struct lockd_cookie *cookie) { u32 length; __be32 *p; @@ -238,9 +238,9 @@ out_overflow: * }; */ static void encode_nlm4_holder(struct xdr_stream *xdr, - const struct nlm_res *result) + const struct lockd_res *result) { - const struct nlm_lock *lock = &result->lock; + const struct lockd_lock *lock = &result->lock; u64 l_offset, l_len; __be32 *p; @@ -254,9 +254,9 @@ static void encode_nlm4_holder(struct xdr_stream *xdr, xdr_encode_hyper(p, l_len); } -static int decode_nlm4_holder(struct xdr_stream *xdr, struct nlm_res *result) +static int decode_nlm4_holder(struct xdr_stream *xdr, struct lockd_res *result) { - struct nlm_lock *lock = &result->lock; + struct lockd_lock *lock = &result->lock; struct file_lock *fl = &lock->fl; u64 l_offset, l_len; u32 exclusive; @@ -317,7 +317,7 @@ static void encode_caller_name(struct xdr_stream *xdr, const char *name) * }; */ static void encode_nlm4_lock(struct xdr_stream *xdr, - const struct nlm_lock *lock) + const struct lockd_lock *lock) { u64 l_offset, l_len; __be32 *p; @@ -354,8 +354,8 @@ static void nlm4_xdr_enc_testargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); @@ -376,8 +376,8 @@ static void nlm4_xdr_enc_lockargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_bool(xdr, args->block); @@ -399,8 +399,8 @@ static void nlm4_xdr_enc_cancargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_bool(xdr, args->block); @@ -418,8 +418,8 @@ static void nlm4_xdr_enc_unlockargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_nlm4_lock(xdr, lock); @@ -435,7 +435,7 @@ static void nlm4_xdr_enc_res(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_res *result = data; + const struct lockd_res *result = data; encode_cookie(xdr, &result->cookie); encode_nlm4_stat(xdr, result->status); @@ -458,7 +458,7 @@ static void nlm4_xdr_enc_testres(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_res *result = data; + const struct lockd_res *result = data; encode_cookie(xdr, &result->cookie); encode_nlm4_stat(xdr, result->status); @@ -489,7 +489,7 @@ static void nlm4_xdr_enc_testres(struct rpc_rqst *req, * }; */ static int decode_nlm4_testrply(struct xdr_stream *xdr, - struct nlm_res *result) + struct lockd_res *result) { int error; @@ -506,7 +506,7 @@ static int nlm4_xdr_dec_testres(struct rpc_rqst *req, struct xdr_stream *xdr, void *data) { - struct nlm_res *result = data; + struct lockd_res *result = data; int error; error = decode_cookie(xdr, &result->cookie); @@ -527,7 +527,7 @@ static int nlm4_xdr_dec_res(struct rpc_rqst *req, struct xdr_stream *xdr, void *data) { - struct nlm_res *result = data; + struct lockd_res *result = data; int error; error = decode_cookie(xdr, &result->cookie); diff --git a/fs/lockd/clntlock.c b/fs/lockd/clntlock.c index 8fa30c42c92a..f797cc99f94d 100644 --- a/fs/lockd/clntlock.c +++ b/fs/lockd/clntlock.c @@ -158,7 +158,7 @@ int nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout) /* * The server lockd has called us back to tell us the lock was granted */ -__be32 nlmclnt_grant(const struct sockaddr *addr, const struct nlm_lock *lock) +__be32 nlmclnt_grant(const struct sockaddr *addr, const struct lockd_lock *lock) { const struct file_lock *fl = &lock->fl; const struct nfs_fh *fh = &lock->fh; diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c index 7f211008a5d2..f06faf577cea 100644 --- a/fs/lockd/clntproc.c +++ b/fs/lockd/clntproc.c @@ -42,7 +42,7 @@ static const struct rpc_call_ops nlmclnt_cancel_ops; */ static atomic_t nlm_cookie = ATOMIC_INIT(0x1234); -void nlmclnt_next_cookie(struct nlm_cookie *c) +void nlmclnt_next_cookie(struct lockd_cookie *c) { u32 cookie = atomic_inc_return(&nlm_cookie); @@ -128,8 +128,8 @@ static struct nlm_lockowner *nlmclnt_find_lockowner(struct nlm_host *host, fl_ow */ static void nlmclnt_setlockargs(struct nlm_rqst *req, struct file_lock *fl) { - struct nlm_args *argp = &req->a_args; - struct nlm_lock *lock = &argp->lock; + struct lockd_args *argp = &req->a_args; + struct lockd_lock *lock = &argp->lock; char *nodename = req->a_host->h_rpcclnt->cl_nodename; nlmclnt_next_cookie(&argp->cookie); @@ -266,8 +266,8 @@ nlmclnt_call(const struct cred *cred, struct nlm_rqst *req, u32 proc) { struct nlm_host *host = req->a_host; struct rpc_clnt *clnt; - struct nlm_args *argp = &req->a_args; - struct nlm_res *resp = &req->a_res; + struct lockd_args *argp = &req->a_args; + struct lockd_res *resp = &req->a_res; struct rpc_message msg = { .rpc_argp = argp, .rpc_resp = resp, @@ -523,7 +523,7 @@ nlmclnt_lock(struct nlm_rqst *req, struct file_lock *fl) { const struct cred *cred = nfs_file_cred(fl->c.flc_file); struct nlm_host *host = req->a_host; - struct nlm_res *resp = &req->a_res; + struct lockd_res *resp = &req->a_res; struct nlm_wait block; unsigned char flags = fl->c.flc_flags; unsigned char type; @@ -686,7 +686,7 @@ static int nlmclnt_unlock(struct nlm_rqst *req, struct file_lock *fl) { struct nlm_host *host = req->a_host; - struct nlm_res *resp = &req->a_res; + struct lockd_res *resp = &req->a_res; int status; unsigned char flags = fl->c.flc_flags; diff --git a/fs/lockd/clntxdr.c b/fs/lockd/clntxdr.c index 65555f5224b1..3789ecb0b984 100644 --- a/fs/lockd/clntxdr.c +++ b/fs/lockd/clntxdr.c @@ -60,7 +60,7 @@ static s32 loff_t_to_s32(loff_t offset) return res; } -static void nlm_compute_offsets(const struct nlm_lock *lock, +static void nlm_compute_offsets(const struct lockd_lock *lock, u32 *l_offset, u32 *l_len) { const struct file_lock *fl = &lock->fl; @@ -130,13 +130,13 @@ static int decode_netobj(struct xdr_stream *xdr, * netobj cookie; */ static void encode_cookie(struct xdr_stream *xdr, - const struct nlm_cookie *cookie) + const struct lockd_cookie *cookie) { encode_netobj(xdr, (u8 *)&cookie->data, cookie->len); } static int decode_cookie(struct xdr_stream *xdr, - struct nlm_cookie *cookie) + struct lockd_cookie *cookie) { u32 length; __be32 *p; @@ -234,9 +234,9 @@ out_overflow: * }; */ static void encode_nlm_holder(struct xdr_stream *xdr, - const struct nlm_res *result) + const struct lockd_res *result) { - const struct nlm_lock *lock = &result->lock; + const struct lockd_lock *lock = &result->lock; u32 l_offset, l_len; __be32 *p; @@ -250,9 +250,9 @@ static void encode_nlm_holder(struct xdr_stream *xdr, *p = cpu_to_be32(l_len); } -static int decode_nlm_holder(struct xdr_stream *xdr, struct nlm_res *result) +static int decode_nlm_holder(struct xdr_stream *xdr, struct lockd_res *result) { - struct nlm_lock *lock = &result->lock; + struct lockd_lock *lock = &result->lock; struct file_lock *fl = &lock->fl; u32 exclusive, l_offset, l_len; int error; @@ -319,7 +319,7 @@ static void encode_caller_name(struct xdr_stream *xdr, const char *name) * }; */ static void encode_nlm_lock(struct xdr_stream *xdr, - const struct nlm_lock *lock) + const struct lockd_lock *lock) { u32 l_offset, l_len; __be32 *p; @@ -355,8 +355,8 @@ static void nlm_xdr_enc_testargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); @@ -377,8 +377,8 @@ static void nlm_xdr_enc_lockargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_bool(xdr, args->block); @@ -400,8 +400,8 @@ static void nlm_xdr_enc_cancargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_bool(xdr, args->block); @@ -419,8 +419,8 @@ static void nlm_xdr_enc_unlockargs(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_args *args = data; - const struct nlm_lock *lock = &args->lock; + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; encode_cookie(xdr, &args->cookie); encode_nlm_lock(xdr, lock); @@ -436,7 +436,7 @@ static void nlm_xdr_enc_res(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_res *result = data; + const struct lockd_res *result = data; encode_cookie(xdr, &result->cookie); encode_nlm_stat(xdr, result->status); @@ -456,7 +456,7 @@ static void nlm_xdr_enc_res(struct rpc_rqst *req, * }; */ static void encode_nlm_testrply(struct xdr_stream *xdr, - const struct nlm_res *result) + const struct lockd_res *result) { if (result->status == nlm_lck_denied) encode_nlm_holder(xdr, result); @@ -466,7 +466,7 @@ static void nlm_xdr_enc_testres(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) { - const struct nlm_res *result = data; + const struct lockd_res *result = data; encode_cookie(xdr, &result->cookie); encode_nlm_stat(xdr, result->status); @@ -495,7 +495,7 @@ static void nlm_xdr_enc_testres(struct rpc_rqst *req, * }; */ static int decode_nlm_testrply(struct xdr_stream *xdr, - struct nlm_res *result) + struct lockd_res *result) { int error; @@ -512,7 +512,7 @@ static int nlm_xdr_dec_testres(struct rpc_rqst *req, struct xdr_stream *xdr, void *data) { - struct nlm_res *result = data; + struct lockd_res *result = data; int error; error = decode_cookie(xdr, &result->cookie); @@ -533,7 +533,7 @@ static int nlm_xdr_dec_res(struct rpc_rqst *req, struct xdr_stream *xdr, void *data) { - struct nlm_res *result = data; + struct lockd_res *result = data; int error; error = decode_cookie(xdr, &result->cookie); diff --git a/fs/lockd/host.c b/fs/lockd/host.c index ea8a8e166f7e..d572cb27533f 100644 --- a/fs/lockd/host.c +++ b/fs/lockd/host.c @@ -552,7 +552,7 @@ struct nlm_host * nlm_get_host(struct nlm_host *host) static struct nlm_host *next_host_state(struct hlist_head *cache, struct nsm_handle *nsm, - const struct nlm_reboot *info) + const struct lockd_reboot *info) { struct nlm_host *host; struct hlist_head *chain; @@ -582,7 +582,7 @@ static struct nlm_host *next_host_state(struct hlist_head *cache, * We were notified that the specified host has rebooted. Release * all resources held by that peer. */ -void nlm_host_rebooted(const struct net *net, const struct nlm_reboot *info) +void nlm_host_rebooted(const struct net *net, const struct lockd_reboot *info) { struct nsm_handle *nsm; struct nlm_host *host; diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h index a7c85ab6d4b5..e418a50c4180 100644 --- a/fs/lockd/lockd.h +++ b/fs/lockd/lockd.h @@ -52,6 +52,14 @@ */ #define LOCKD_DFLT_TIMEO 10 +/* + * Number of leading bytes of nfs_fh.data that file_hash() + * digests when bucketing nlm_files[]. Sized for historical + * NFSv2 handles; nfs_fh.data must be initialized at least + * this far before lookup, regardless of fh.size. + */ +#define LOCKD_FH_HASH_SIZE 32 + /* error codes new to NLMv4 */ #define nlm4_deadlock cpu_to_be32(NLM_DEADLCK) #define nlm4_rofs cpu_to_be32(NLM_ROFS) @@ -171,15 +179,15 @@ struct nlm_rqst { refcount_t a_count; unsigned int a_flags; /* initial RPC task flags */ struct nlm_host * a_host; /* host handle */ - struct nlm_args a_args; /* arguments */ - struct nlm_res a_res; /* result */ + struct lockd_args a_args; /* arguments */ + struct lockd_res a_res; /* result */ struct nlm_block * a_block; unsigned int a_retries; /* Retry count */ u8 a_owner[NLMCLNT_OHSIZE]; void * a_callback_data; /* sent to nlmclnt_operations callbacks */ }; -struct nlm_share; +struct lockd_share; /* * This struct describes a file held open by lockd on behalf of @@ -190,7 +198,7 @@ struct nlm_file { struct nfs_fh f_handle; /* NFS file handle */ struct file * f_file[2]; /* VFS file pointers, indexed by O_ flags */ - struct nlm_share * f_shares; /* DOS shares */ + struct lockd_share * f_shares; /* DOS shares */ struct list_head f_blocks; /* blocked locks */ unsigned int f_locks; /* guesstimate # of locks */ unsigned int f_count; /* reference count */ @@ -253,11 +261,11 @@ void nlmclnt_queue_block(struct nlm_wait *block); __be32 nlmclnt_dequeue_block(struct nlm_wait *block); int nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout); __be32 nlmclnt_grant(const struct sockaddr *addr, - const struct nlm_lock *lock); + const struct lockd_lock *lock); void nlmclnt_recovery(struct nlm_host *); int nlmclnt_reclaim(struct nlm_host *, struct file_lock *, struct nlm_rqst *); -void nlmclnt_next_cookie(struct nlm_cookie *); +void nlmclnt_next_cookie(struct lockd_cookie *); #ifdef CONFIG_LOCKD_V4 extern const struct rpc_version nlm_version4; @@ -285,7 +293,7 @@ struct nlm_host * nlm_get_host(struct nlm_host *); void nlm_shutdown_hosts(void); void nlm_shutdown_hosts_net(struct net *net); void nlm_host_rebooted(const struct net *net, - const struct nlm_reboot *); + const struct lockd_reboot *); /* * Host monitoring @@ -299,7 +307,7 @@ struct nsm_handle *nsm_get_handle(const struct net *net, const char *hostname, const size_t hostname_len); struct nsm_handle *nsm_reboot_lookup(const struct net *net, - const struct nlm_reboot *info); + const struct lockd_reboot *info); void nsm_release(struct nsm_handle *nsm); /* @@ -313,17 +321,17 @@ typedef int (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref); */ int lock_to_openmode(struct file_lock *); __be32 nlmsvc_lock(struct svc_rqst *, struct nlm_file *, - struct nlm_host *, struct nlm_lock *, int, - struct nlm_cookie *, int); -__be32 nlmsvc_unlock(struct net *net, struct nlm_file *, struct nlm_lock *); + struct nlm_host *, struct lockd_lock *, int, + struct lockd_cookie *, int); +__be32 nlmsvc_unlock(struct net *net, struct nlm_file *, struct lockd_lock *); __be32 nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, - struct nlm_host *host, struct nlm_lock *lock, - struct nlm_lock *conflock); -__be32 nlmsvc_cancel_blocked(struct net *net, struct nlm_file *, struct nlm_lock *); + struct nlm_host *host, struct lockd_lock *lock, + struct lockd_lock *conflock); +__be32 nlmsvc_cancel_blocked(struct net *net, struct nlm_file *, struct lockd_lock *); void nlmsvc_retry_blocked(struct svc_rqst *rqstp); void nlmsvc_traverse_blocks(struct nlm_host *, struct nlm_file *, nlm_host_match_fn_t match); -void nlmsvc_grant_reply(struct nlm_cookie *, __be32); +void nlmsvc_grant_reply(struct lockd_cookie *, __be32); void nlmsvc_release_call(struct nlm_rqst *); void nlmsvc_locks_init_private(struct file_lock *, struct nlm_host *, pid_t); int nlmsvc_dispatch(struct svc_rqst *rqstp); @@ -332,10 +340,10 @@ int nlmsvc_dispatch(struct svc_rqst *rqstp); * File handling for the server personality */ __be32 nlm_lookup_file(struct svc_rqst *, struct nlm_file **, - struct nlm_lock *); + struct lockd_lock *, int); void nlm_release_file(struct nlm_file *); void nlmsvc_put_lockowner(struct nlm_lockowner *); -void nlmsvc_release_lockowner(struct nlm_lock *); +void nlmsvc_release_lockowner(struct lockd_lock *); void nlmsvc_mark_resources(struct net *); void nlmsvc_free_host_resources(struct nlm_host *); void nlmsvc_invalidate_all(void); @@ -423,6 +431,29 @@ static inline int nlm_compare_locks(const struct file_lock *fl1, } /** + * lockd_set_file_lock_range3 - set the byte range of a file_lock + * @fl: file_lock whose length fields are to be initialized + * @off: starting offset of the lock, in bytes + * @len: length of the byte range, in bytes, or zero + * + * NLMv3 uses a (start, length) representation for lock byte ranges, + * while the kernel's file_lock uses (start, end). Treat a length of + * zero or arithmetic overflow (end wrapping negative when the sum + * exceeds S32_MAX) as "lock to end of file." + */ +static inline void +lockd_set_file_lock_range3(struct file_lock *fl, u32 off, u32 len) +{ + s32 end = off + len - 1; + + fl->fl_start = off; + if (len == 0 || end < 0) + fl->fl_end = OFFSET_MAX; + else + fl->fl_end = end; +} + +/** * lockd_set_file_lock_range4 - set the byte range of a file_lock * @fl: file_lock whose length fields are to be initialized * @off: starting offset of the lock, in bytes diff --git a/fs/lockd/mon.c b/fs/lockd/mon.c index 3d3ee88ca4dc..a8f5ac6f0577 100644 --- a/fs/lockd/mon.c +++ b/fs/lockd/mon.c @@ -377,7 +377,7 @@ retry: * error occurred. */ struct nsm_handle *nsm_reboot_lookup(const struct net *net, - const struct nlm_reboot *info) + const struct lockd_reboot *info) { struct nsm_handle *cached; struct lockd_net *ln = net_generic(net, lockd_net_id); diff --git a/fs/lockd/nlm3xdr_gen.c b/fs/lockd/nlm3xdr_gen.c new file mode 100644 index 000000000000..9ed5a41b5daf --- /dev/null +++ b/fs/lockd/nlm3xdr_gen.c @@ -0,0 +1,714 @@ +// SPDX-License-Identifier: GPL-2.0 +// Generated by xdrgen. Manual edits will be lost. +// XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x +// XDR specification modification time: Thu Apr 23 10:56:34 2026 + +#include <linux/sunrpc/svc.h> + +#include "nlm3xdr_gen.h" + +static bool __maybe_unused +xdrgen_decode_netobj(struct xdr_stream *xdr, netobj *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, MAXNETOBJ_SZ); +} + +static bool __maybe_unused +xdrgen_decode_nlm_stats(struct xdr_stream *xdr, nlm_stats *ptr) +{ + __be32 raw; + u32 val; + + if (xdr_stream_decode_be32(xdr, &raw) < 0) + return false; + val = be32_to_cpu(raw); + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case LCK_GRANTED: + case LCK_DENIED: + case LCK_DENIED_NOLOCKS: + case LCK_BLOCKED: + case LCK_DENIED_GRACE_PERIOD: + break; + default: + return false; + } + *ptr = raw; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_stat(struct xdr_stream *xdr, struct nlm_stat *ptr) +{ + if (!xdrgen_decode_nlm_stats(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_res(struct xdr_stream *xdr, struct nlm_res *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_stat(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_holder(struct xdr_stream *xdr, struct nlm_holder *ptr) +{ + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->uppid)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_testrply(struct xdr_stream *xdr, struct nlm_testrply *ptr) +{ + if (!xdrgen_decode_nlm_stats(xdr, &ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(LCK_DENIED): + if (!xdrgen_decode_nlm_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_testres(struct xdr_stream *xdr, struct nlm_testres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_testrply(xdr, &ptr->test_stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_lock(struct xdr_stream *xdr, struct nlm_lock *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->uppid)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_lockargs(struct xdr_stream *xdr, struct nlm_lockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_cancargs(struct xdr_stream *xdr, struct nlm_cancargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_testargs(struct xdr_stream *xdr, struct nlm_testargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_unlockargs(struct xdr_stream *xdr, struct nlm_unlockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fsh_mode(struct xdr_stream *xdr, fsh_mode *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsm_DN: + case fsm_DR: + case fsm_DW: + case fsm_DRW: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fsh_access(struct xdr_stream *xdr, fsh_access *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsa_NONE: + case fsa_R: + case fsa_W: + case fsa_RW: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_share(struct xdr_stream *xdr, struct nlm_share *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_fsh_mode(xdr, &ptr->mode)) + return false; + if (!xdrgen_decode_fsh_access(xdr, &ptr->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_shareargs(struct xdr_stream *xdr, struct nlm_shareargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_share(xdr, &ptr->share)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_shareres(struct xdr_stream *xdr, struct nlm_shareres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_stats(xdr, &ptr->stat)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_notify(struct xdr_stream *xdr, struct nlm_notify *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXNAMELEN)) + return false; + if (!xdrgen_decode_long(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_notifyargs(struct xdr_stream *xdr, struct nlm_notifyargs *ptr) +{ + if (!xdrgen_decode_nlm_notify(xdr, &ptr->notify)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm_svc_decode_void - Decode a void argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_decode_void(xdr); +} + +/** + * nlm_svc_decode_nlm_testargs - Decode a nlm_testargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_testargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_testargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_lockargs - Decode a nlm_lockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_lockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_lockargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_cancargs - Decode a nlm_cancargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_cancargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_cancargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_unlockargs - Decode a nlm_unlockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_unlockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_unlockargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_testres - Decode a nlm_testres argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_testres *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_testres(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_res - Decode a nlm_res argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_res *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_res(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_notifyargs - Decode a nlm_notifyargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_notifyargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_notifyargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_shareargs - Decode a nlm_shareargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_shareargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_shareargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_notify - Decode a nlm_notify argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_notify *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_notify(xdr, argp); +} + +static bool __maybe_unused +xdrgen_encode_netobj(struct xdr_stream *xdr, const netobj value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_nlm_stats(struct xdr_stream *xdr, nlm_stats value) +{ + return xdr_stream_encode_be32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_nlm_stat(struct xdr_stream *xdr, const struct nlm_stat *value) +{ + if (!xdrgen_encode_nlm_stats(xdr, value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_res(struct xdr_stream *xdr, const struct nlm_res *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_stat(xdr, &value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_holder(struct xdr_stream *xdr, const struct nlm_holder *value) +{ + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_int(xdr, value->uppid)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_testrply(struct xdr_stream *xdr, const struct nlm_testrply *ptr) +{ + if (!xdrgen_encode_nlm_stats(xdr, ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(LCK_DENIED): + if (!xdrgen_encode_nlm_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_testres(struct xdr_stream *xdr, const struct nlm_testres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_testrply(xdr, &value->test_stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_lock(struct xdr_stream *xdr, const struct nlm_lock *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_int(xdr, value->uppid)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_lockargs(struct xdr_stream *xdr, const struct nlm_lockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + if (!xdrgen_encode_int(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_cancargs(struct xdr_stream *xdr, const struct nlm_cancargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_testargs(struct xdr_stream *xdr, const struct nlm_testargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_unlockargs(struct xdr_stream *xdr, const struct nlm_unlockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_fsh_mode(struct xdr_stream *xdr, fsh_mode value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_fsh_access(struct xdr_stream *xdr, fsh_access value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_nlm_share(struct xdr_stream *xdr, const struct nlm_share *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_fsh_mode(xdr, value->mode)) + return false; + if (!xdrgen_encode_fsh_access(xdr, value->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_shareargs(struct xdr_stream *xdr, const struct nlm_shareargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_share(xdr, &value->share)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_shareres(struct xdr_stream *xdr, const struct nlm_shareres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_stats(xdr, value->stat)) + return false; + if (!xdrgen_encode_int(xdr, value->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_notify(struct xdr_stream *xdr, const struct nlm_notify *value) +{ + if (value->name.len > LM_MAXNAMELEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->name.data, value->name.len) < 0) + return false; + if (!xdrgen_encode_long(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_notifyargs(struct xdr_stream *xdr, const struct nlm_notifyargs *value) +{ + if (!xdrgen_encode_nlm_notify(xdr, &value->notify)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm_svc_encode_void - Encode a void result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_encode_void(xdr); +} + +/** + * nlm_svc_encode_nlm_testres - Encode a nlm_testres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm_svc_encode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_testres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm_testres(xdr, resp); +} + +/** + * nlm_svc_encode_nlm_res - Encode a nlm_res result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm_svc_encode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_res *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm_res(xdr, resp); +} + +/** + * nlm_svc_encode_nlm_shareres - Encode a nlm_shareres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm_svc_encode_nlm_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_shareres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm_shareres(xdr, resp); +} diff --git a/fs/lockd/nlm3xdr_gen.h b/fs/lockd/nlm3xdr_gen.h new file mode 100644 index 000000000000..c99038e99805 --- /dev/null +++ b/fs/lockd/nlm3xdr_gen.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ +/* XDR specification modification time: Thu Apr 23 10:56:34 2026 */ + +#ifndef _LINUX_XDRGEN_NLM3_DECL_H +#define _LINUX_XDRGEN_NLM3_DECL_H + +#include <linux/types.h> + +#include <linux/sunrpc/xdr.h> +#include <linux/sunrpc/xdrgen/_defs.h> +#include <linux/sunrpc/xdrgen/_builtins.h> +#include <linux/sunrpc/xdrgen/nlm3.h> + +bool nlm_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +bool nlm_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_encode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_encode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_encode_nlm_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +#endif /* _LINUX_XDRGEN_NLM3_DECL_H */ diff --git a/fs/lockd/share.h b/fs/lockd/share.h index 20ea8ee49168..1ec3ccdb2aef 100644 --- a/fs/lockd/share.h +++ b/fs/lockd/share.h @@ -14,8 +14,8 @@ /* * DOS share for a specific file */ -struct nlm_share { - struct nlm_share * s_next; /* linked list */ +struct lockd_share { + struct lockd_share * s_next; /* linked list */ struct nlm_host * s_host; /* client host */ struct nlm_file * s_file; /* shared file */ struct xdr_netobj s_owner; /* owner handle */ diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index 5de41e249534..78e675470c4b 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -26,42 +26,42 @@ #include "nlm4xdr_gen.h" /* - * Wrapper structures combine xdrgen types with legacy nlm_lock. + * Wrapper structures combine xdrgen types with legacy lockd_lock. * The xdrgen field must be first so the structure can be cast * to its XDR type for the RPC dispatch layer. */ struct nlm4_testargs_wrapper { struct nlm4_testargs xdrgen; - struct nlm_lock lock; + struct lockd_lock lock; }; static_assert(offsetof(struct nlm4_testargs_wrapper, xdrgen) == 0); struct nlm4_lockargs_wrapper { struct nlm4_lockargs xdrgen; - struct nlm_cookie cookie; - struct nlm_lock lock; + struct lockd_cookie cookie; + struct lockd_lock lock; }; static_assert(offsetof(struct nlm4_lockargs_wrapper, xdrgen) == 0); struct nlm4_cancargs_wrapper { struct nlm4_cancargs xdrgen; - struct nlm_lock lock; + struct lockd_lock lock; }; static_assert(offsetof(struct nlm4_cancargs_wrapper, xdrgen) == 0); struct nlm4_unlockargs_wrapper { struct nlm4_unlockargs xdrgen; - struct nlm_lock lock; + struct lockd_lock lock; }; static_assert(offsetof(struct nlm4_unlockargs_wrapper, xdrgen) == 0); struct nlm4_notifyargs_wrapper { struct nlm4_notifyargs xdrgen; - struct nlm_reboot reboot; + struct lockd_reboot reboot; }; static_assert(offsetof(struct nlm4_notifyargs_wrapper, xdrgen) == 0); @@ -74,12 +74,12 @@ static_assert(offsetof(struct nlm4_notify_wrapper, xdrgen) == 0); struct nlm4_testres_wrapper { struct nlm4_testres xdrgen; - struct nlm_lock lock; + struct lockd_lock lock; }; struct nlm4_shareargs_wrapper { struct nlm4_shareargs xdrgen; - struct nlm_lock lock; + struct lockd_lock lock; }; static_assert(offsetof(struct nlm4_shareargs_wrapper, xdrgen) == 0); @@ -88,7 +88,7 @@ static_assert(offsetof(struct nlm4_testres_wrapper, xdrgen) == 0); struct nlm4_res_wrapper { struct nlm4_res xdrgen; - struct nlm_cookie cookie; + struct lockd_cookie cookie; }; static_assert(offsetof(struct nlm4_res_wrapper, xdrgen) == 0); @@ -100,7 +100,7 @@ struct nlm4_shareres_wrapper { static_assert(offsetof(struct nlm4_shareres_wrapper, xdrgen) == 0); static __be32 -nlm4_netobj_to_cookie(struct nlm_cookie *cookie, netobj *object) +nlm4_netobj_to_cookie(struct lockd_cookie *cookie, netobj *object) { if (object->len > NLM_MAXCOOKIELEN) return nlm_lck_denied_nolocks; @@ -110,7 +110,7 @@ nlm4_netobj_to_cookie(struct nlm_cookie *cookie, netobj *object) } static __be32 -nlm4_lock_to_nlm_lock(struct nlm_lock *lock, struct nlm4_lock *alock) +nlm4_lock_to_lockd_lock(struct lockd_lock *lock, struct nlm4_lock *alock) { if (alock->fh.len > NFS_MAXFHSIZE) return nlm_lck_denied; @@ -119,7 +119,6 @@ nlm4_lock_to_nlm_lock(struct nlm_lock *lock, struct nlm4_lock *alock) lock->oh.len = alock->oh.len; lock->oh.data = alock->oh.data; lock->svid = alock->svid; - locks_init_lock(&lock->fl); lockd_set_file_lock_range4(&lock->fl, alock->l_offset, alock->l_len); return nlm_granted; } @@ -143,17 +142,23 @@ nlm4svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) static __be32 nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host, - struct nlm_lock *lock, struct nlm_file **filp, + struct lockd_lock *lock, struct nlm_file **filp, struct nlm4_lock *xdr_lock, unsigned char type) { + bool is_test = (rqstp->rq_proc == NLMPROC4_TEST || + rqstp->rq_proc == NLMPROC4_TEST_MSG); struct file_lock *fl = &lock->fl; struct nlm_file *file = NULL; + int mode; __be32 error; if (xdr_lock->fh.len > NFS_MAXFHSIZE) return nlm_lck_denied_nolocks; lock->fh.size = xdr_lock->fh.len; memcpy(lock->fh.data, xdr_lock->fh.data, xdr_lock->fh.len); + if (xdr_lock->fh.len < LOCKD_FH_HASH_SIZE) + memset(lock->fh.data + xdr_lock->fh.len, 0, + LOCKD_FH_HASH_SIZE - xdr_lock->fh.len); lock->oh.len = xdr_lock->oh.len; lock->oh.data = xdr_lock->oh.data; @@ -170,7 +175,8 @@ nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host, fl->c.flc_type = type; lockd_set_file_lock_range4(fl, lock->lock_start, lock->lock_len); - error = nlm_lookup_file(rqstp, &file, lock); + mode = is_test ? O_RDWR : lock_to_openmode(fl); + error = nlm_lookup_file(rqstp, &file, lock, mode); switch (error) { case nlm_granted: break; @@ -184,7 +190,8 @@ nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host, *filp = file; fl->c.flc_flags = FL_POSIX; - fl->c.flc_file = file->f_file[lock_to_openmode(fl)]; + fl->c.flc_file = is_test ? nlmsvc_file_file(file) + : file->f_file[mode]; fl->c.flc_pid = current->tgid; fl->fl_lmops = &nlmsvc_lock_operations; nlmsvc_locks_init_private(fl, host, (pid_t)lock->svid); @@ -265,7 +272,7 @@ static __be32 nlm4svc_proc_test(struct svc_rqst *rqstp) nlmsvc_release_lockowner(&argp->lock); if (resp->xdrgen.stat.stat == nlm_lck_denied) { - struct nlm_lock *conf = &resp->lock; + struct lockd_lock *conf = &resp->lock; struct nlm4_holder *holder = &resp->xdrgen.stat.u.holder; holder->exclusive = (conf->fl.c.flc_type != F_RDLCK); @@ -508,12 +515,12 @@ out: * nlm4_res NLMPROC4_GRANTED(nlm4_testargs) = 5; * * Permissible procedure status codes: - * %NLM4_GRANTED: The requested lock was granted. - * %NLM4_DENIED: The server could not allocate the resources - * needed to process the request. - * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is - * re-establishing existing locks, and is not - * yet ready to accept normal service requests. + * %NLM4_GRANTED: The granted lock was accepted. + * %NLM4_DENIED: The procedure failed, possibly due to + * internal resource constraints. + * %NLM4_DENIED_GRACE_PERIOD: The client host recently restarted and + * its NLM is re-establishing existing locks, + * so it is not yet ready to accept callbacks. */ static __be32 nlm4svc_proc_granted(struct svc_rqst *rqstp) @@ -523,8 +530,8 @@ nlm4svc_proc_granted(struct svc_rqst *rqstp) resp->xdrgen.cookie = argp->xdrgen.cookie; - resp->xdrgen.stat.stat = nlm4_lock_to_nlm_lock(&argp->lock, - &argp->xdrgen.alock); + resp->xdrgen.stat.stat = nlm4_lock_to_lockd_lock(&argp->lock, + &argp->xdrgen.alock); if (resp->xdrgen.stat.stat) goto out; @@ -558,7 +565,7 @@ static const struct rpc_call_ops nlm4svc_callback_ops = { */ static __be32 nlm4svc_callback(struct svc_rqst *rqstp, struct nlm_host *host, u32 proc, - __be32 (*func)(struct svc_rqst *, struct nlm_res *)) + __be32 (*func)(struct svc_rqst *, struct lockd_res *)) { struct nlm_rqst *call; __be32 stat; @@ -581,7 +588,7 @@ nlm4svc_callback(struct svc_rqst *rqstp, struct nlm_host *host, u32 proc, } static __be32 -__nlm4svc_proc_test_msg(struct svc_rqst *rqstp, struct nlm_res *resp) +__nlm4svc_proc_test_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; @@ -641,7 +648,7 @@ static __be32 nlm4svc_proc_test_msg(struct svc_rqst *rqstp) } static __be32 -__nlm4svc_proc_lock_msg(struct svc_rqst *rqstp, struct nlm_res *resp) +__nlm4svc_proc_lock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp; unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; @@ -664,6 +671,8 @@ __nlm4svc_proc_lock_msg(struct svc_rqst *rqstp, struct nlm_res *resp) resp->status = nlmsvc_lock(rqstp, file, host, &argp->lock, argp->xdrgen.block, &resp->cookie, argp->xdrgen.reclaim); + if (resp->status == nlm__int__deadlock) + resp->status = nlm4_deadlock; nlmsvc_release_lockowner(&argp->lock); out: @@ -692,7 +701,7 @@ static __be32 nlm4svc_proc_lock_msg(struct svc_rqst *rqstp) struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp; struct nlm_host *host; - host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, true); + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); if (!host) return rpc_system_err; @@ -701,7 +710,7 @@ static __be32 nlm4svc_proc_lock_msg(struct svc_rqst *rqstp) } static __be32 -__nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp, struct nlm_res *resp) +__nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { struct nlm4_cancargs_wrapper *argp = rqstp->rq_argp; unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; @@ -765,7 +774,7 @@ static __be32 nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp) } static __be32 -__nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp, struct nlm_res *resp) +__nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { struct nlm4_unlockargs_wrapper *argp = rqstp->rq_argp; struct net *net = SVC_NET(rqstp); @@ -828,7 +837,7 @@ static __be32 nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp) } static __be32 -__nlm4svc_proc_granted_msg(struct svc_rqst *rqstp, struct nlm_res *resp) +__nlm4svc_proc_granted_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; @@ -836,7 +845,7 @@ __nlm4svc_proc_granted_msg(struct svc_rqst *rqstp, struct nlm_res *resp) if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) goto out; - if (nlm4_lock_to_nlm_lock(&argp->lock, &argp->xdrgen.alock)) + if (nlm4_lock_to_lockd_lock(&argp->lock, &argp->xdrgen.alock)) goto out; resp->status = nlmclnt_grant(svc_addr(rqstp), &argp->lock); @@ -912,7 +921,7 @@ static __be32 nlm4svc_proc_granted_res(struct svc_rqst *rqstp) static __be32 nlm4svc_proc_sm_notify(struct svc_rqst *rqstp) { struct nlm4_notifyargs_wrapper *argp = rqstp->rq_argp; - struct nlm_reboot *reboot = &argp->reboot; + struct lockd_reboot *reboot = &argp->reboot; if (!nlm_privileged_requester(rqstp)) { char buf[RPC_MAX_ADDRBUFLEN]; @@ -976,7 +985,7 @@ static __be32 nlm4svc_proc_share(struct svc_rqst *rqstp) { struct nlm4_shareargs_wrapper *argp = rqstp->rq_argp; struct nlm4_shareres_wrapper *resp = rqstp->rq_resp; - struct nlm_lock *lock = &argp->lock; + struct lockd_lock *lock = &argp->lock; struct nlm_host *host = NULL; struct nlm_file *file = NULL; struct nlm4_lock xdr_lock = { @@ -1044,7 +1053,7 @@ static __be32 nlm4svc_proc_unshare(struct svc_rqst *rqstp) { struct nlm4_shareargs_wrapper *argp = rqstp->rq_argp; struct nlm4_shareres_wrapper *resp = rqstp->rq_resp; - struct nlm_lock *lock = &argp->lock; + struct lockd_lock *lock = &argp->lock; struct nlm4_lock xdr_lock = { .fh = argp->xdrgen.share.fh, .oh = argp->xdrgen.share.oh, diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index b98b1d0ada35..e48d31f14a65 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -37,7 +37,7 @@ static void nlmsvc_release_block(struct nlm_block *block); static void nlmsvc_insert_block(struct nlm_block *block, unsigned long); static void nlmsvc_remove_block(struct nlm_block *block); -static int nlmsvc_setgrantargs(struct nlm_rqst *call, struct nlm_lock *lock); +static int nlmsvc_setgrantargs(struct nlm_rqst *call, struct lockd_lock *lock); static void nlmsvc_freegrantargs(struct nlm_rqst *call); static const struct rpc_call_ops nlmsvc_grant_ops; @@ -48,7 +48,7 @@ static LIST_HEAD(nlm_blocked); static DEFINE_SPINLOCK(nlm_blocked_lock); #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) -static const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie) +static const char *nlmdbg_cookie2a(const struct lockd_cookie *cookie) { /* * We can get away with a static buffer because this is only called @@ -75,7 +75,7 @@ static const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie) return buf; } #else -static inline const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie) +static inline const char *nlmdbg_cookie2a(const struct lockd_cookie *cookie) { return "???"; } @@ -142,7 +142,7 @@ nlmsvc_remove_block(struct nlm_block *block) * Find a block for a given lock */ static struct nlm_block * -nlmsvc_lookup_block(struct nlm_file *file, struct nlm_lock *lock) +nlmsvc_lookup_block(struct nlm_file *file, struct lockd_lock *lock) { struct nlm_block *block; struct file_lock *fl; @@ -171,7 +171,7 @@ nlmsvc_lookup_block(struct nlm_file *file, struct nlm_lock *lock) return NULL; } -static inline int nlm_cookie_match(struct nlm_cookie *a, struct nlm_cookie *b) +static int lockd_cookie_match(struct lockd_cookie *a, struct lockd_cookie *b) { if (a->len != b->len) return 0; @@ -184,13 +184,13 @@ static inline int nlm_cookie_match(struct nlm_cookie *a, struct nlm_cookie *b) * Find a block with a given NLM cookie. */ static inline struct nlm_block * -nlmsvc_find_block(struct nlm_cookie *cookie) +nlmsvc_find_block(struct lockd_cookie *cookie) { struct nlm_block *block; spin_lock(&nlm_blocked_lock); list_for_each_entry(block, &nlm_blocked, b_list) { - if (nlm_cookie_match(&block->b_call->a_args.cookie,cookie)) + if (lockd_cookie_match(&block->b_call->a_args.cookie, cookie)) goto found; } spin_unlock(&nlm_blocked_lock); @@ -221,8 +221,8 @@ found: */ static struct nlm_block * nlmsvc_create_block(struct svc_rqst *rqstp, struct nlm_host *host, - struct nlm_file *file, struct nlm_lock *lock, - struct nlm_cookie *cookie) + struct nlm_file *file, struct lockd_lock *lock, + struct lockd_cookie *cookie) { struct nlm_block *block; struct nlm_rqst *call = NULL; @@ -399,7 +399,7 @@ static struct nlm_lockowner *nlmsvc_find_lockowner(struct nlm_host *host, pid_t } void -nlmsvc_release_lockowner(struct nlm_lock *lock) +nlmsvc_release_lockowner(struct lockd_lock *lock) { if (lock->fl.c.flc_owner) nlmsvc_put_lockowner(lock->fl.c.flc_owner); @@ -415,7 +415,7 @@ void nlmsvc_locks_init_private(struct file_lock *fl, struct nlm_host *host, * Initialize arguments for GRANTED call. The nlm_rqst structure * has been cleared already. */ -static int nlmsvc_setgrantargs(struct nlm_rqst *call, struct nlm_lock *lock) +static int nlmsvc_setgrantargs(struct nlm_rqst *call, struct lockd_lock *lock) { locks_copy_lock(&call->a_args.lock.fl, &lock->fl); memcpy(&call->a_args.lock.fh, &lock->fh, sizeof(call->a_args.lock.fh)); @@ -476,8 +476,8 @@ nlmsvc_defer_lock_rqst(struct svc_rqst *rqstp, struct nlm_block *block) */ __be32 nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file, - struct nlm_host *host, struct nlm_lock *lock, int wait, - struct nlm_cookie *cookie, int reclaim) + struct nlm_host *host, struct lockd_lock *lock, int wait, + struct lockd_cookie *cookie, int reclaim) { struct inode *inode __maybe_unused = nlmsvc_file_inode(file); struct nlm_block *block = NULL; @@ -609,11 +609,10 @@ out: */ __be32 nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, - struct nlm_host *host, struct nlm_lock *lock, - struct nlm_lock *conflock) + struct nlm_host *host, struct lockd_lock *lock, + struct lockd_lock *conflock) { int error; - int mode; __be32 ret; dprintk("lockd: nlmsvc_testlock(%s/%llu, ty=%d, %Ld-%Ld)\n", @@ -631,14 +630,13 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, goto out; } - mode = lock_to_openmode(&lock->fl); locks_init_lock(&conflock->fl); /* vfs_test_lock only uses start, end, and owner, but tests flc_file */ conflock->fl.c.flc_file = lock->fl.c.flc_file; conflock->fl.fl_start = lock->fl.fl_start; conflock->fl.fl_end = lock->fl.fl_end; conflock->fl.c.flc_owner = lock->fl.c.flc_owner; - error = vfs_test_lock(file->f_file[mode], &conflock->fl); + error = vfs_test_lock(lock->fl.c.flc_file, &conflock->fl); if (error) { ret = nlm_lck_denied_nolocks; goto out; @@ -671,7 +669,7 @@ out: * must be removed. */ __be32 -nlmsvc_unlock(struct net *net, struct nlm_file *file, struct nlm_lock *lock) +nlmsvc_unlock(struct net *net, struct nlm_file *file, struct lockd_lock *lock) { int error = 0; @@ -709,7 +707,7 @@ nlmsvc_unlock(struct net *net, struct nlm_file *file, struct nlm_lock *lock) * The calling procedure must check whether the file can be closed. */ __be32 -nlmsvc_cancel_blocked(struct net *net, struct nlm_file *file, struct nlm_lock *lock) +nlmsvc_cancel_blocked(struct net *net, struct nlm_file *file, struct lockd_lock *lock) { struct nlm_block *block; int status = 0; @@ -850,7 +848,7 @@ static void nlmsvc_grant_blocked(struct nlm_block *block) { struct nlm_file *file = block->b_file; - struct nlm_lock *lock = &block->b_call->a_args.lock; + struct lockd_lock *lock = &block->b_call->a_args.lock; int mode; int error; loff_t fl_start, fl_end; @@ -984,7 +982,7 @@ static const struct rpc_call_ops nlmsvc_grant_ops = { * block. */ void -nlmsvc_grant_reply(struct nlm_cookie *cookie, __be32 status) +nlmsvc_grant_reply(struct lockd_cookie *cookie, __be32 status) { struct nlm_block *block; struct file_lock *fl; diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index 749abf8886ba..4836887f11ef 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -13,11 +13,196 @@ #include <linux/sunrpc/svc_xprt.h> #include "lockd.h" + +/* + * xdr.h defines SM_PRIV_SIZE as a macro. nlm3xdr_gen.h defines + * it as an enum constant. Undefine the macro before including + * the generated header. + */ +#undef SM_PRIV_SIZE + #include "share.h" +#include "nlm3xdr_gen.h" + +/* + * Size of an NFSv2 file handle, in bytes, which is 32. + * Defined locally to avoid including uapi/linux/nfs2.h. + */ +#define NLM3_FHSIZE 32 + +/* + * Wrapper structures combine xdrgen types with legacy lockd_lock. + * The xdrgen field must be first so the structure can be cast + * to its XDR type for the RPC dispatch layer. + */ +struct nlm_testargs_wrapper { + struct nlm_testargs xdrgen; + struct lockd_lock lock; +}; -#define NLMDBG_FACILITY NLMDBG_CLIENT +static_assert(offsetof(struct nlm_testargs_wrapper, xdrgen) == 0); + +struct nlm_testres_wrapper { + struct nlm_testres xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_testres_wrapper, xdrgen) == 0); + +struct nlm_lockargs_wrapper { + struct nlm_lockargs xdrgen; + struct lockd_cookie cookie; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_lockargs_wrapper, xdrgen) == 0); + +struct nlm_res_wrapper { + struct nlm_res xdrgen; + struct lockd_cookie cookie; +}; + +static_assert(offsetof(struct nlm_res_wrapper, xdrgen) == 0); + +struct nlm_cancargs_wrapper { + struct nlm_cancargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_cancargs_wrapper, xdrgen) == 0); + +struct nlm_unlockargs_wrapper { + struct nlm_unlockargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_unlockargs_wrapper, xdrgen) == 0); + +struct nlm_notifyargs_wrapper { + struct nlm_notifyargs xdrgen; + struct lockd_reboot reboot; +}; + +static_assert(offsetof(struct nlm_notifyargs_wrapper, xdrgen) == 0); + +struct nlm_shareargs_wrapper { + struct nlm_shareargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_shareargs_wrapper, xdrgen) == 0); + +struct nlm_shareres_wrapper { + struct nlm_shareres xdrgen; +}; + +static_assert(offsetof(struct nlm_shareres_wrapper, xdrgen) == 0); + +struct nlm_notify_wrapper { + struct nlm_notify xdrgen; +}; + +static_assert(offsetof(struct nlm_notify_wrapper, xdrgen) == 0); + +static __be32 +nlm_netobj_to_cookie(struct lockd_cookie *cookie, netobj *object) +{ + if (object->len > NLM_MAXCOOKIELEN) + return nlm_lck_denied_nolocks; + cookie->len = object->len; + memcpy(cookie->data, object->data, object->len); + return nlm_granted; +} + +static __be32 +nlm_lock_to_lockd_lock(struct lockd_lock *lock, struct nlm_lock *alock) +{ + if (alock->fh.len != NLM3_FHSIZE) + return nlm_lck_denied; + lock->fh.size = alock->fh.len; + memcpy(lock->fh.data, alock->fh.data, alock->fh.len); + lock->oh.len = alock->oh.len; + lock->oh.data = alock->oh.data; + lock->svid = alock->uppid; + lockd_set_file_lock_range3(&lock->fl, alock->l_offset, alock->l_len); + return nlm_granted; +} + +static struct nlm_host * +nlm3svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) +{ + struct nlm_host *host; + + if (!nlmsvc_ops) + return NULL; + host = nlmsvc_lookup_host(rqstp, caller.data, caller.len); + if (!host) + return NULL; + if (monitored && nsm_monitor(host) < 0) { + nlmsvc_release_host(host); + return NULL; + } + return host; +} + +static __be32 +nlm3svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host, + struct lockd_lock *lock, struct nlm_file **filp, + struct nlm_lock *xdr_lock, unsigned char type) +{ + bool is_test = (rqstp->rq_proc == NLMPROC_TEST || + rqstp->rq_proc == NLMPROC_TEST_MSG); + struct file_lock *fl = &lock->fl; + struct nlm_file *file = NULL; + __be32 error; + int mode; + + if (xdr_lock->fh.len != NLM3_FHSIZE) + return nlm_lck_denied_nolocks; + lock->fh.size = xdr_lock->fh.len; + memcpy(lock->fh.data, xdr_lock->fh.data, xdr_lock->fh.len); + + lock->oh.len = xdr_lock->oh.len; + lock->oh.data = xdr_lock->oh.data; + + lock->svid = xdr_lock->uppid; + lock->lock_start = xdr_lock->l_offset; + lock->lock_len = xdr_lock->l_len; + + locks_init_lock(fl); + fl->c.flc_type = type; + lockd_set_file_lock_range3(fl, lock->lock_start, lock->lock_len); + + mode = lock_to_openmode(fl); + if (is_test) + mode = O_RDWR; + + error = nlm_lookup_file(rqstp, &file, lock, mode); + switch (error) { + case nlm_granted: + break; + case nlm__int__stale_fh: + case nlm__int__failed: + return nlm_lck_denied_nolocks; + default: + return error; + } + *filp = file; + + fl->c.flc_flags = FL_POSIX; + if (is_test) + fl->c.flc_file = nlmsvc_file_file(file); + else + fl->c.flc_file = file->f_file[mode]; + fl->c.flc_pid = current->tgid; + fl->fl_lmops = &nlmsvc_lock_operations; + nlmsvc_locks_init_private(fl, host, (pid_t)lock->svid); + if (!fl->c.flc_owner) + return nlm_lck_denied_nolocks; + + return nlm_granted; +} -#ifdef CONFIG_LOCKD_V4 static inline __be32 cast_status(__be32 status) { switch (status) { @@ -32,275 +217,330 @@ static inline __be32 cast_status(__be32 status) status = nlm_lck_denied; break; default: + pr_warn_once("lockd: unhandled internal status %u\n", + be32_to_cpu(status)); status = nlm_lck_denied_nolocks; - } - - return status; -} -#else -static inline __be32 cast_status(__be32 status) -{ - switch (status) { - case nlm__int__deadlock: - status = nlm_lck_denied; - break; - case nlm__int__stale_fh: - case nlm__int__failed: - status = nlm_lck_denied_nolocks; - break; - default: - if (be32_to_cpu(status) >= 30000) - pr_warn_once("lockd: unhandled internal status %u\n", - be32_to_cpu(status)); break; } return status; } -#endif -/* - * Obtain client and file from arguments - */ -static __be32 -nlmsvc_retrieve_args(struct svc_rqst *rqstp, struct nlm_args *argp, - struct nlm_host **hostp, struct nlm_file **filp) -{ - struct nlm_host *host = NULL; - struct nlm_file *file = NULL; - struct nlm_lock *lock = &argp->lock; - int mode; - __be32 error = 0; - - /* nfsd callbacks must have been installed for this procedure */ - if (!nlmsvc_ops) - return nlm_lck_denied_nolocks; - - /* Obtain host handle */ - if (!(host = nlmsvc_lookup_host(rqstp, lock->caller, lock->len)) - || (argp->monitor && nsm_monitor(host) < 0)) - goto no_locks; - *hostp = host; - - /* Obtain file pointer. Not used by FREE_ALL call. */ - if (filp != NULL) { - error = cast_status(nlm_lookup_file(rqstp, &file, lock)); - if (error != 0) - goto no_locks; - *filp = file; - - /* Set up the missing parts of the file_lock structure */ - mode = lock_to_openmode(&lock->fl); - lock->fl.c.flc_flags = FL_POSIX; - lock->fl.c.flc_file = file->f_file[mode]; - lock->fl.c.flc_pid = current->tgid; - lock->fl.fl_lmops = &nlmsvc_lock_operations; - nlmsvc_locks_init_private(&lock->fl, host, (pid_t)lock->svid); - if (!lock->fl.c.flc_owner) { - /* lockowner allocation has failed */ - nlmsvc_release_host(host); - return nlm_lck_denied_nolocks; - } - } - - return 0; - -no_locks: - nlmsvc_release_host(host); - if (error) - return error; - return nlm_lck_denied_nolocks; -} - -/* - * NULL: Test for presence of service +/** + * nlmsvc_proc_null - NULL: Test for presence of service + * @rqstp: RPC transaction context + * + * Return: + * %rpc_success: RPC executed successfully + * + * RPC synopsis: + * void NLM_NULL(void) = 0; */ -static __be32 -nlmsvc_proc_null(struct svc_rqst *rqstp) +static __be32 nlmsvc_proc_null(struct svc_rqst *rqstp) { - dprintk("lockd: NULL called\n"); return rpc_success; } -/* - * TEST: Check for conflicting lock +/** + * nlmsvc_proc_test - TEST: Check for conflicting lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_testres NLM_TEST(nlm_testargs) = 1; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The server would be able to grant the + * requested lock. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. */ -static __be32 -__nlmsvc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) +static __be32 nlmsvc_proc_test(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_testres_wrapper *resp = rqstp->rq_resp; + struct nlm_file *file = NULL; struct nlm_host *host; - struct nlm_file *file; - __be32 rc = rpc_success; - dprintk("lockd: TEST called\n"); - resp->cookie = argp->cookie; + resp->xdrgen.cookie = argp->xdrgen.cookie; - /* Obtain client and file */ - if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) - return resp->status == nlm__int__drop_reply ? - rpc_drop_reply : rpc_success; + resp->xdrgen.test_stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; - /* Now check for conflicting locks */ - resp->status = cast_status(nlmsvc_testlock(rqstp, file, host, - &argp->lock, &resp->lock)); - if (resp->status == nlm__int__drop_reply) - rc = rpc_drop_reply; - else - dprintk("lockd: TEST status %d vers %d\n", - ntohl(resp->status), rqstp->rq_vers); + resp->xdrgen.test_stat.stat = + nlm3svc_lookup_file(rqstp, host, &argp->lock, &file, + &argp->xdrgen.alock, type); + if (resp->xdrgen.test_stat.stat) + goto out; + resp->xdrgen.test_stat.stat = + cast_status(nlmsvc_testlock(rqstp, file, host, &argp->lock, + &resp->lock)); nlmsvc_release_lockowner(&argp->lock); - nlmsvc_release_host(host); - nlm_release_file(file); - return rc; -} -static __be32 -nlmsvc_proc_test(struct svc_rqst *rqstp) -{ - return __nlmsvc_proc_test(rqstp, rqstp->rq_resp); + if (resp->xdrgen.test_stat.stat == nlm_lck_denied) { + struct lockd_lock *conf = &resp->lock; + struct nlm_holder *holder = &resp->xdrgen.test_stat.u.holder; + + holder->exclusive = (conf->fl.c.flc_type != F_RDLCK); + holder->uppid = conf->svid; + holder->oh.len = conf->oh.len; + holder->oh.data = conf->oh.data; + holder->l_offset = min_t(loff_t, conf->fl.fl_start, + NLM_OFFSET_MAX); + if (conf->fl.fl_end == OFFSET_MAX) + holder->l_len = 0; + else + holder->l_len = min_t(loff_t, + conf->fl.fl_end - + conf->fl.fl_start + 1, + NLM_OFFSET_MAX); + } + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.test_stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; } static __be32 -__nlmsvc_proc_lock(struct svc_rqst *rqstp, struct nlm_res *resp) +nlmsvc_do_lock(struct svc_rqst *rqstp, bool monitored) { - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_host *host; - struct nlm_file *file; - __be32 rc = rpc_success; - - dprintk("lockd: LOCK called\n"); - - resp->cookie = argp->cookie; - - /* Obtain client and file */ - if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) - return resp->status == nlm__int__drop_reply ? - rpc_drop_reply : rpc_success; - - /* Now try to lock the file */ - resp->status = cast_status(nlmsvc_lock(rqstp, file, host, &argp->lock, - argp->block, &argp->cookie, - argp->reclaim)); - if (resp->status == nlm__int__drop_reply) - rc = rpc_drop_reply; - else - dprintk("lockd: LOCK status %d\n", ntohl(resp->status)); + struct nlm_lockargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_res_wrapper *resp = rqstp->rq_resp; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_netobj_to_cookie(&argp->cookie, + &argp->xdrgen.cookie); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, + monitored); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + type); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = cast_status(nlmsvc_lock(rqstp, file, host, + &argp->lock, + argp->xdrgen.block, + &argp->cookie, + argp->xdrgen.reclaim)); nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); nlmsvc_release_host(host); - nlm_release_file(file); - return rc; + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; } +/** + * nlmsvc_proc_lock - LOCK: Establish a monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_LOCK(nlm_lockargs) = 2; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was granted. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %LCK_BLOCKED: The blocking request cannot be granted + * immediately. The server will send an + * NLM_GRANTED callback to the client when + * the lock can be granted. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + */ static __be32 nlmsvc_proc_lock(struct svc_rqst *rqstp) { - return __nlmsvc_proc_lock(rqstp, rqstp->rq_resp); + return nlmsvc_do_lock(rqstp, true); } +/** + * nlmsvc_proc_cancel - CANCEL: Cancel an outstanding blocked lock request + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_CANCEL(nlm_cancargs) = 3; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was canceled. + * %LCK_DENIED: There was no lock to cancel. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. + */ static __be32 -__nlmsvc_proc_cancel(struct svc_rqst *rqstp, struct nlm_res *resp) +nlmsvc_proc_cancel(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_host *host; - struct nlm_file *file; + struct nlm_cancargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_res_wrapper *resp = rqstp->rq_resp; struct net *net = SVC_NET(rqstp); + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; - dprintk("lockd: CANCEL called\n"); + resp->xdrgen.cookie = argp->xdrgen.cookie; - resp->cookie = argp->cookie; + resp->xdrgen.stat.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; - /* Don't accept requests during grace period */ - if (locks_in_grace(net)) { - resp->status = nlm_lck_denied_grace_period; - return rpc_success; - } - - /* Obtain client and file */ - if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) - return resp->status == nlm__int__drop_reply ? - rpc_drop_reply : rpc_success; + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; - /* Try to cancel request. */ - resp->status = cast_status(nlmsvc_cancel_blocked(net, file, &argp->lock)); + resp->xdrgen.stat.stat = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + type); + if (resp->xdrgen.stat.stat) + goto out; - dprintk("lockd: CANCEL status %d\n", ntohl(resp->status)); + resp->xdrgen.stat.stat = nlmsvc_cancel_blocked(net, file, &argp->lock); nlmsvc_release_lockowner(&argp->lock); - nlmsvc_release_host(host); - nlm_release_file(file); - return rpc_success; -} -static __be32 -nlmsvc_proc_cancel(struct svc_rqst *rqstp) -{ - return __nlmsvc_proc_cancel(rqstp, rqstp->rq_resp); +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; } -/* - * UNLOCK: release a lock +/** + * nlmsvc_proc_unlock - UNLOCK: Remove a lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_UNLOCK(nlm_unlockargs) = 4; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was released. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. */ static __be32 -__nlmsvc_proc_unlock(struct svc_rqst *rqstp, struct nlm_res *resp) +nlmsvc_proc_unlock(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_host *host; - struct nlm_file *file; + struct nlm_unlockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_res_wrapper *resp = rqstp->rq_resp; struct net *net = SVC_NET(rqstp); + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; - dprintk("lockd: UNLOCK called\n"); + resp->xdrgen.cookie = argp->xdrgen.cookie; - resp->cookie = argp->cookie; + resp->xdrgen.stat.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; - /* Don't accept new lock requests during grace period */ - if (locks_in_grace(net)) { - resp->status = nlm_lck_denied_grace_period; - return rpc_success; - } + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; - /* Obtain client and file */ - if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) - return resp->status == nlm__int__drop_reply ? - rpc_drop_reply : rpc_success; + resp->xdrgen.stat.stat = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + F_UNLCK); + if (resp->xdrgen.stat.stat) + goto out; - /* Now try to remove the lock */ - resp->status = cast_status(nlmsvc_unlock(net, file, &argp->lock)); - - dprintk("lockd: UNLOCK status %d\n", ntohl(resp->status)); + resp->xdrgen.stat.stat = nlmsvc_unlock(net, file, &argp->lock); nlmsvc_release_lockowner(&argp->lock); - nlmsvc_release_host(host); - nlm_release_file(file); - return rpc_success; -} -static __be32 -nlmsvc_proc_unlock(struct svc_rqst *rqstp) -{ - return __nlmsvc_proc_unlock(rqstp, rqstp->rq_resp); +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; } -/* - * GRANTED: A server calls us to tell that a process' lock request - * was granted +/** + * nlmsvc_proc_granted - GRANTED: Blocked lock has been granted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * nlm_res NLM_GRANTED(nlm_testargs) = 5; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The granted lock was accepted. + * %LCK_DENIED: The procedure failed, possibly due to + * internal resource constraints. + * %LCK_DENIED_GRACE_PERIOD: The client host recently restarted and + * its NLM is re-establishing existing locks, + * so it is not yet ready to accept callbacks. */ static __be32 -__nlmsvc_proc_granted(struct svc_rqst *rqstp, struct nlm_res *resp) +nlmsvc_proc_granted(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_res_wrapper *resp = rqstp->rq_resp; - resp->cookie = argp->cookie; + resp->xdrgen.cookie = argp->xdrgen.cookie; - dprintk("lockd: GRANTED called\n"); - resp->status = nlmclnt_grant(svc_addr(rqstp), &argp->lock); - dprintk("lockd: GRANTED status %d\n", ntohl(resp->status)); - return rpc_success; -} + resp->xdrgen.stat.stat = nlm_lock_to_lockd_lock(&argp->lock, + &argp->xdrgen.alock); + if (resp->xdrgen.stat.stat) + goto out; -static __be32 -nlmsvc_proc_granted(struct svc_rqst *rqstp) -{ - return __nlmsvc_proc_granted(rqstp, rqstp->rq_resp); + resp->xdrgen.stat.stat = nlmclnt_grant(svc_addr(rqstp), &argp->lock); + +out: + return rpc_success; } /* @@ -333,20 +573,13 @@ static const struct rpc_call_ops nlmsvc_callback_ops = { * because we send the callback before the reply proper. I hope this * doesn't break any clients. */ -static __be32 nlmsvc_callback(struct svc_rqst *rqstp, u32 proc, - __be32 (*func)(struct svc_rqst *, struct nlm_res *)) +static __be32 +nlmsvc_callback(struct svc_rqst *rqstp, struct nlm_host *host, u32 proc, + __be32 (*func)(struct svc_rqst *, struct lockd_res *)) { - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_host *host; struct nlm_rqst *call; __be32 stat; - host = nlmsvc_lookup_host(rqstp, - argp->lock.caller, - argp->lock.len); - if (host == NULL) - return rpc_system_err; - call = nlm_alloc_call(host); nlmsvc_release_host(host); if (call == NULL) @@ -364,440 +597,819 @@ static __be32 nlmsvc_callback(struct svc_rqst *rqstp, u32 proc, return rpc_success; } -static __be32 nlmsvc_proc_test_msg(struct svc_rqst *rqstp) +static __be32 +__nlmsvc_proc_test_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { - dprintk("lockd: TEST_MSG called\n"); - return nlmsvc_callback(rqstp, NLMPROC_TEST_RES, __nlmsvc_proc_test); + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_lockowner *owner; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + owner = argp->lock.fl.c.flc_owner; + resp->status = cast_status(nlmsvc_testlock(rqstp, file, host, + &argp->lock, &resp->lock)); + nlmsvc_put_lockowner(owner); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; } -static __be32 nlmsvc_proc_lock_msg(struct svc_rqst *rqstp) +/** + * nlmsvc_proc_test_msg - TEST_MSG: Check for conflicting lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_TEST_MSG(nlm_testargs) = 6; + * + * The response to this request is delivered via the TEST_RES procedure. + */ +static __be32 nlmsvc_proc_test_msg(struct svc_rqst *rqstp) { - dprintk("lockd: LOCK_MSG called\n"); - return nlmsvc_callback(rqstp, NLMPROC_LOCK_RES, __nlmsvc_proc_lock); + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlmsvc_callback(rqstp, host, NLMPROC_TEST_RES, + __nlmsvc_proc_test_msg); } -static __be32 nlmsvc_proc_cancel_msg(struct svc_rqst *rqstp) +static __be32 +__nlmsvc_proc_lock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { - dprintk("lockd: CANCEL_MSG called\n"); - return nlmsvc_callback(rqstp, NLMPROC_CANCEL_RES, __nlmsvc_proc_cancel); + struct nlm_lockargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, true); + if (!host) + goto out; + + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + resp->status = cast_status(nlmsvc_lock(rqstp, file, host, &argp->lock, + argp->xdrgen.block, &resp->cookie, + argp->xdrgen.reclaim)); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; } -static __be32 -nlmsvc_proc_unlock_msg(struct svc_rqst *rqstp) +/** + * nlmsvc_proc_lock_msg - LOCK_MSG: Establish a monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_LOCK_MSG(nlm_lockargs) = 7; + * + * The response to this request is delivered via the LOCK_RES procedure. + */ +static __be32 nlmsvc_proc_lock_msg(struct svc_rqst *rqstp) { - dprintk("lockd: UNLOCK_MSG called\n"); - return nlmsvc_callback(rqstp, NLMPROC_UNLOCK_RES, __nlmsvc_proc_unlock); + struct nlm_lockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlmsvc_callback(rqstp, host, NLMPROC_LOCK_RES, + __nlmsvc_proc_lock_msg); } static __be32 -nlmsvc_proc_granted_msg(struct svc_rqst *rqstp) +__nlmsvc_proc_cancel_msg(struct svc_rqst *rqstp, struct lockd_res *resp) { - dprintk("lockd: GRANTED_MSG called\n"); - return nlmsvc_callback(rqstp, NLMPROC_GRANTED_RES, __nlmsvc_proc_granted); + struct nlm_cancargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct net *net = SVC_NET(rqstp); + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + resp->status = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->status = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + resp->status = nlmsvc_cancel_blocked(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; } -/* - * SHARE: create a DOS share or alter existing share. +/** + * nlmsvc_proc_cancel_msg - CANCEL_MSG: Cancel an outstanding lock request + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_CANCEL_MSG(nlm_cancargs) = 8; + * + * The response to this request is delivered via the CANCEL_RES procedure. */ -static __be32 -nlmsvc_proc_share(struct svc_rqst *rqstp) +static __be32 nlmsvc_proc_cancel_msg(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_res *resp = rqstp->rq_resp; - struct nlm_host *host; - struct nlm_file *file; + struct nlm_cancargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; - dprintk("lockd: SHARE called\n"); + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; - resp->cookie = argp->cookie; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; - /* Don't accept new lock requests during grace period */ - if (locks_in_grace(SVC_NET(rqstp)) && !argp->reclaim) { - resp->status = nlm_lck_denied_grace_period; - return rpc_success; - } + return nlmsvc_callback(rqstp, host, NLMPROC_CANCEL_RES, + __nlmsvc_proc_cancel_msg); +} - /* Obtain client and file */ - if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) - return resp->status == nlm__int__drop_reply ? - rpc_drop_reply : rpc_success; +static __be32 +__nlmsvc_proc_unlock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm_unlockargs_wrapper *argp = rqstp->rq_argp; + struct net *net = SVC_NET(rqstp); + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + resp->status = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->status = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; - /* Now try to create the share */ - resp->status = cast_status(nlmsvc_share_file(host, file, &argp->lock.oh, - argp->fsm_access, - argp->fsm_mode)); + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, F_UNLCK); + if (resp->status) + goto out; - dprintk("lockd: SHARE status %d\n", ntohl(resp->status)); + resp->status = nlmsvc_unlock(net, file, &argp->lock); nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); nlmsvc_release_host(host); - nlm_release_file(file); - return rpc_success; + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; } -/* - * UNSHARE: Release a DOS share. +/** + * nlmsvc_proc_unlock_msg - UNLOCK_MSG: Remove an existing lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_UNLOCK_MSG(nlm_unlockargs) = 9; + * + * The response to this request is delivered via the UNLOCK_RES procedure. */ -static __be32 -nlmsvc_proc_unshare(struct svc_rqst *rqstp) +static __be32 nlmsvc_proc_unlock_msg(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_res *resp = rqstp->rq_resp; - struct nlm_host *host; - struct nlm_file *file; + struct nlm_unlockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; - dprintk("lockd: UNSHARE called\n"); + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; - resp->cookie = argp->cookie; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; - /* Don't accept requests during grace period */ - if (locks_in_grace(SVC_NET(rqstp))) { - resp->status = nlm_lck_denied_grace_period; - return rpc_success; - } + return nlmsvc_callback(rqstp, host, NLMPROC_UNLOCK_RES, + __nlmsvc_proc_unlock_msg); +} - /* Obtain client and file */ - if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) - return resp->status == nlm__int__drop_reply ? - rpc_drop_reply : rpc_success; +static __be32 +__nlmsvc_proc_granted_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; - /* Now try to unshare the file */ - resp->status = cast_status(nlmsvc_unshare_file(host, file, - &argp->lock.oh)); + resp->status = nlm_lck_denied; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; - dprintk("lockd: UNSHARE status %d\n", ntohl(resp->status)); - nlmsvc_release_lockowner(&argp->lock); - nlmsvc_release_host(host); - nlm_release_file(file); + if (nlm_lock_to_lockd_lock(&argp->lock, &argp->xdrgen.alock)) + goto out; + + resp->status = nlmclnt_grant(svc_addr(rqstp), &argp->lock); + +out: return rpc_success; } -/* - * NM_LOCK: Create an unmonitored lock +/** + * nlmsvc_proc_granted_msg - GRANTED_MSG: Blocked lock has been granted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_GRANTED_MSG(nlm_testargs) = 10; + * + * The response to this request is delivered via the GRANTED_RES procedure. */ -static __be32 -nlmsvc_proc_nm_lock(struct svc_rqst *rqstp) +static __be32 nlmsvc_proc_granted_msg(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; - dprintk("lockd: NM_LOCK called\n"); + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; - argp->monitor = 0; /* just clean the monitor flag */ - return nlmsvc_proc_lock(rqstp); + return nlmsvc_callback(rqstp, host, NLMPROC_GRANTED_RES, + __nlmsvc_proc_granted_msg); } -/* - * FREE_ALL: Release all locks and shares held by client +/** + * nlmsvc_proc_granted_res - GRANTED_RES: Lock Granted result + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * void NLMPROC_GRANTED_RES(nlm_res) = 15; */ -static __be32 -nlmsvc_proc_free_all(struct svc_rqst *rqstp) +static __be32 nlmsvc_proc_granted_res(struct svc_rqst *rqstp) { - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_host *host; + struct nlm_res_wrapper *argp = rqstp->rq_argp; - /* Obtain client */ - if (nlmsvc_retrieve_args(rqstp, argp, &host, NULL)) + if (!nlmsvc_ops) return rpc_success; - nlmsvc_free_host_resources(host); - nlmsvc_release_host(host); + if (nlm_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie)) + return rpc_success; + nlmsvc_grant_reply(&argp->cookie, argp->xdrgen.stat.stat); + return rpc_success; } -/* - * SM_NOTIFY: private callback from statd (not part of official NLM proto) +/** + * nlmsvc_proc_sm_notify - SM_NOTIFY: Peer has rebooted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * The SM_NOTIFY procedure is a private callback from Linux statd and is + * not part of the official NLM protocol. + * + * RPC synopsis: + * void NLM_SM_NOTIFY(nlm_notifyargs) = 16; */ -static __be32 -nlmsvc_proc_sm_notify(struct svc_rqst *rqstp) +static __be32 nlmsvc_proc_sm_notify(struct svc_rqst *rqstp) { - struct nlm_reboot *argp = rqstp->rq_argp; - - dprintk("lockd: SM_NOTIFY called\n"); + struct nlm_notifyargs_wrapper *argp = rqstp->rq_argp; + struct lockd_reboot *reboot = &argp->reboot; if (!nlm_privileged_requester(rqstp)) { char buf[RPC_MAX_ADDRBUFLEN]; - printk(KERN_WARNING "lockd: rejected NSM callback from %s\n", - svc_print_addr(rqstp, buf, sizeof(buf))); + + pr_warn("lockd: rejected NSM callback from %s\n", + svc_print_addr(rqstp, buf, sizeof(buf))); return rpc_system_err; } - nlm_host_rebooted(SVC_NET(rqstp), argp); + reboot->len = argp->xdrgen.notify.name.len; + reboot->mon = (char *)argp->xdrgen.notify.name.data; + reboot->state = argp->xdrgen.notify.state; + memcpy(&reboot->priv.data, argp->xdrgen.private, + sizeof(reboot->priv.data)); + + nlm_host_rebooted(SVC_NET(rqstp), reboot); + return rpc_success; } -/* - * client sent a GRANTED_RES, let's remove the associated block +/** + * nlmsvc_proc_unused - stub for unused procedures + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_proc_unavail: Program can't support procedure. */ -static __be32 -nlmsvc_proc_granted_res(struct svc_rqst *rqstp) +static __be32 nlmsvc_proc_unused(struct svc_rqst *rqstp) { - struct nlm_res *argp = rqstp->rq_argp; - - if (!nlmsvc_ops) - return rpc_success; + return rpc_proc_unavail; +} - dprintk("lockd: GRANTED_RES called\n"); +/** + * nlmsvc_proc_share - SHARE: Open a file using DOS file-sharing modes + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_shareres NLM_SHARE(nlm_shareargs) = 20; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested share lock was granted. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. + */ +static __be32 nlmsvc_proc_share(struct svc_rqst *rqstp) +{ + struct nlm_shareargs_wrapper *argp = rqstp->rq_argp; + struct nlm_shareres_wrapper *resp = rqstp->rq_resp; + struct lockd_lock *lock = &argp->lock; + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + struct nlm_lock xdr_lock = { + .fh = argp->xdrgen.share.fh, + .oh = argp->xdrgen.share.oh, + .uppid = LOCKD_SHARE_SVID, + }; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(SVC_NET(rqstp)) && !argp->xdrgen.reclaim) + goto out; + + resp->xdrgen.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat = nlm3svc_lookup_file(rqstp, host, lock, &file, + &xdr_lock, F_RDLCK); + if (resp->xdrgen.stat) + goto out; + + resp->xdrgen.stat = nlmsvc_share_file(host, file, &lock->oh, + argp->xdrgen.share.access, + argp->xdrgen.share.mode); + + nlmsvc_release_lockowner(lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} - nlmsvc_grant_reply(&argp->cookie, argp->status); - return rpc_success; +/** + * nlmsvc_proc_unshare - UNSHARE: Release a share reservation + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_shareres NLM_UNSHARE(nlm_shareargs) = 21; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The share reservation was released. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. + */ +static __be32 nlmsvc_proc_unshare(struct svc_rqst *rqstp) +{ + struct nlm_shareargs_wrapper *argp = rqstp->rq_argp; + struct nlm_shareres_wrapper *resp = rqstp->rq_resp; + struct lockd_lock *lock = &argp->lock; + struct nlm_lock xdr_lock = { + .fh = argp->xdrgen.share.fh, + .oh = argp->xdrgen.share.oh, + .uppid = LOCKD_SHARE_SVID, + }; + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(SVC_NET(rqstp))) + goto out; + + resp->xdrgen.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat = nlm3svc_lookup_file(rqstp, host, lock, &file, + &xdr_lock, F_RDLCK); + if (resp->xdrgen.stat) + goto out; + + resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh); + + nlmsvc_release_lockowner(lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; } -static __be32 -nlmsvc_proc_unused(struct svc_rqst *rqstp) +/** + * nlmsvc_proc_nm_lock - NM_LOCK: Establish a non-monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_NM_LOCK(nlm_lockargs) = 22; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was granted. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %LCK_BLOCKED: The blocking request cannot be granted + * immediately. The server will send an + * NLM_GRANTED callback to the client when + * the lock can be granted. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + */ +static __be32 nlmsvc_proc_nm_lock(struct svc_rqst *rqstp) { - return rpc_proc_unavail; + return nlmsvc_do_lock(rqstp, false); } -/* - * NLM Server procedures. +/** + * nlmsvc_proc_free_all - FREE_ALL: Discard client's lock and share state + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * void NLMPROC_FREE_ALL(nlm_notify) = 23; */ +static __be32 nlmsvc_proc_free_all(struct svc_rqst *rqstp) +{ + struct nlm_notify_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.name, false); + if (!host) + goto out; + + nlmsvc_free_host_resources(host); + + nlmsvc_release_host(host); -struct nlm_void { int dummy; }; +out: + return rpc_success; +} -#define Ck (1+XDR_QUADLEN(NLM_MAXCOOKIELEN)) /* cookie */ -#define St 1 /* status */ -#define No (1+1024/4) /* Net Obj */ -#define Rg 2 /* range - offset + size */ +/* + * NLMv3 Server procedures. + */ static const struct svc_procedure nlmsvc_procedures[24] = { - [NLMPROC_NULL] = { - .pc_func = nlmsvc_proc_null, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_void), - .pc_argzero = sizeof(struct nlm_void), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "NULL", + [NLM_NULL] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_void, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = XDR_void, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "NULL", }, - [NLMPROC_TEST] = { - .pc_func = nlmsvc_proc_test, - .pc_decode = nlmsvc_decode_testargs, - .pc_encode = nlmsvc_encode_testres, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St+2+No+Rg, - .pc_name = "TEST", + [NLM_TEST] = { + .pc_func = nlmsvc_proc_test, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = nlm_svc_encode_nlm_testres, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_testres_wrapper), + .pc_xdrressize = NLM3_nlm_testres_sz, + .pc_name = "TEST", }, - [NLMPROC_LOCK] = { - .pc_func = nlmsvc_proc_lock, - .pc_decode = nlmsvc_decode_lockargs, - .pc_encode = nlmsvc_encode_res, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St, - .pc_name = "LOCK", + [NLM_LOCK] = { + .pc_func = nlmsvc_proc_lock, + .pc_decode = nlm_svc_decode_nlm_lockargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "LOCK", }, - [NLMPROC_CANCEL] = { - .pc_func = nlmsvc_proc_cancel, - .pc_decode = nlmsvc_decode_cancargs, - .pc_encode = nlmsvc_encode_res, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St, - .pc_name = "CANCEL", + [NLM_CANCEL] = { + .pc_func = nlmsvc_proc_cancel, + .pc_decode = nlm_svc_decode_nlm_cancargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_cancargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "CANCEL", }, - [NLMPROC_UNLOCK] = { - .pc_func = nlmsvc_proc_unlock, - .pc_decode = nlmsvc_decode_unlockargs, - .pc_encode = nlmsvc_encode_res, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St, - .pc_name = "UNLOCK", + [NLM_UNLOCK] = { + .pc_func = nlmsvc_proc_unlock, + .pc_decode = nlm_svc_decode_nlm_unlockargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_unlockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "UNLOCK", }, - [NLMPROC_GRANTED] = { - .pc_func = nlmsvc_proc_granted, - .pc_decode = nlmsvc_decode_testargs, - .pc_encode = nlmsvc_encode_res, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St, - .pc_name = "GRANTED", + [NLM_GRANTED] = { + .pc_func = nlmsvc_proc_granted, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "GRANTED", }, - [NLMPROC_TEST_MSG] = { - .pc_func = nlmsvc_proc_test_msg, - .pc_decode = nlmsvc_decode_testargs, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "TEST_MSG", + [NLM_TEST_MSG] = { + .pc_func = nlmsvc_proc_test_msg, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "TEST_MSG", }, - [NLMPROC_LOCK_MSG] = { - .pc_func = nlmsvc_proc_lock_msg, - .pc_decode = nlmsvc_decode_lockargs, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "LOCK_MSG", + [NLM_LOCK_MSG] = { + .pc_func = nlmsvc_proc_lock_msg, + .pc_decode = nlm_svc_decode_nlm_lockargs, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "LOCK_MSG", }, - [NLMPROC_CANCEL_MSG] = { - .pc_func = nlmsvc_proc_cancel_msg, - .pc_decode = nlmsvc_decode_cancargs, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "CANCEL_MSG", + [NLM_CANCEL_MSG] = { + .pc_func = nlmsvc_proc_cancel_msg, + .pc_decode = nlm_svc_decode_nlm_cancargs, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_cancargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "CANCEL_MSG", }, - [NLMPROC_UNLOCK_MSG] = { - .pc_func = nlmsvc_proc_unlock_msg, - .pc_decode = nlmsvc_decode_unlockargs, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "UNLOCK_MSG", + [NLM_UNLOCK_MSG] = { + .pc_func = nlmsvc_proc_unlock_msg, + .pc_decode = nlm_svc_decode_nlm_unlockargs, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_unlockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNLOCK_MSG", }, - [NLMPROC_GRANTED_MSG] = { - .pc_func = nlmsvc_proc_granted_msg, - .pc_decode = nlmsvc_decode_testargs, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "GRANTED_MSG", + [NLM_GRANTED_MSG] = { + .pc_func = nlmsvc_proc_granted_msg, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "GRANTED_MSG", }, - [NLMPROC_TEST_RES] = { - .pc_func = nlmsvc_proc_null, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_res), - .pc_argzero = sizeof(struct nlm_res), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "TEST_RES", + [NLM_TEST_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_testres, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_testres), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "TEST_RES", }, - [NLMPROC_LOCK_RES] = { - .pc_func = nlmsvc_proc_null, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_res), - .pc_argzero = sizeof(struct nlm_res), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "LOCK_RES", + [NLM_LOCK_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "LOCK_RES", }, - [NLMPROC_CANCEL_RES] = { - .pc_func = nlmsvc_proc_null, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_res), - .pc_argzero = sizeof(struct nlm_res), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "CANCEL_RES", + [NLM_CANCEL_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "CANCEL_RES", }, - [NLMPROC_UNLOCK_RES] = { - .pc_func = nlmsvc_proc_null, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_res), - .pc_argzero = sizeof(struct nlm_res), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "UNLOCK_RES", + [NLM_UNLOCK_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNLOCK_RES", }, - [NLMPROC_GRANTED_RES] = { - .pc_func = nlmsvc_proc_granted_res, - .pc_decode = nlmsvc_decode_res, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_res), - .pc_argzero = sizeof(struct nlm_res), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "GRANTED_RES", + [NLM_GRANTED_RES] = { + .pc_func = nlmsvc_proc_granted_res, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "GRANTED_RES", }, - [NLMPROC_NSM_NOTIFY] = { - .pc_func = nlmsvc_proc_sm_notify, - .pc_decode = nlmsvc_decode_reboot, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_reboot), - .pc_argzero = sizeof(struct nlm_reboot), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "SM_NOTIFY", + [NLM_SM_NOTIFY] = { + .pc_func = nlmsvc_proc_sm_notify, + .pc_decode = nlm_svc_decode_nlm_notifyargs, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_notifyargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "SM_NOTIFY", }, [17] = { - .pc_func = nlmsvc_proc_unused, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_void), - .pc_argzero = sizeof(struct nlm_void), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "UNUSED", + .pc_func = nlmsvc_proc_unused, + .pc_decode = nlm_svc_decode_void, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", }, [18] = { - .pc_func = nlmsvc_proc_unused, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_void), - .pc_argzero = sizeof(struct nlm_void), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "UNUSED", + .pc_func = nlmsvc_proc_unused, + .pc_decode = nlm_svc_decode_void, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", }, [19] = { - .pc_func = nlmsvc_proc_unused, - .pc_decode = nlmsvc_decode_void, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_void), - .pc_argzero = sizeof(struct nlm_void), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = St, - .pc_name = "UNUSED", + .pc_func = nlmsvc_proc_unused, + .pc_decode = nlm_svc_decode_void, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", }, - [NLMPROC_SHARE] = { - .pc_func = nlmsvc_proc_share, - .pc_decode = nlmsvc_decode_shareargs, - .pc_encode = nlmsvc_encode_shareres, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St+1, - .pc_name = "SHARE", + [NLM_SHARE] = { + .pc_func = nlmsvc_proc_share, + .pc_decode = nlm_svc_decode_nlm_shareargs, + .pc_encode = nlm_svc_encode_nlm_shareres, + .pc_argsize = sizeof(struct nlm_shareargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_shareres_wrapper), + .pc_xdrressize = NLM3_nlm_shareres_sz, + .pc_name = "SHARE", }, - [NLMPROC_UNSHARE] = { - .pc_func = nlmsvc_proc_unshare, - .pc_decode = nlmsvc_decode_shareargs, - .pc_encode = nlmsvc_encode_shareres, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St+1, - .pc_name = "UNSHARE", + [NLM_UNSHARE] = { + .pc_func = nlmsvc_proc_unshare, + .pc_decode = nlm_svc_decode_nlm_shareargs, + .pc_encode = nlm_svc_encode_nlm_shareres, + .pc_argsize = sizeof(struct nlm_shareargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_shareres_wrapper), + .pc_xdrressize = NLM3_nlm_shareres_sz, + .pc_name = "UNSHARE", }, - [NLMPROC_NM_LOCK] = { - .pc_func = nlmsvc_proc_nm_lock, - .pc_decode = nlmsvc_decode_lockargs, - .pc_encode = nlmsvc_encode_res, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_res), - .pc_xdrressize = Ck+St, - .pc_name = "NM_LOCK", + [NLM_NM_LOCK] = { + .pc_func = nlmsvc_proc_nm_lock, + .pc_decode = nlm_svc_decode_nlm_lockargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "NM_LOCK", }, - [NLMPROC_FREE_ALL] = { - .pc_func = nlmsvc_proc_free_all, - .pc_decode = nlmsvc_decode_notify, - .pc_encode = nlmsvc_encode_void, - .pc_argsize = sizeof(struct nlm_args), - .pc_argzero = sizeof(struct nlm_args), - .pc_ressize = sizeof(struct nlm_void), - .pc_xdrressize = 0, - .pc_name = "FREE_ALL", + [NLM_FREE_ALL] = { + .pc_func = nlmsvc_proc_free_all, + .pc_decode = nlm_svc_decode_nlm_notify, + .pc_encode = nlm_svc_encode_void, + .pc_argsize = sizeof(struct nlm_notify_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "FREE_ALL", }, }; @@ -805,9 +1417,16 @@ static const struct svc_procedure nlmsvc_procedures[24] = { * Storage requirements for XDR arguments and results */ union nlmsvc_xdrstore { - struct nlm_args args; - struct nlm_res res; - struct nlm_reboot reboot; + struct nlm_testargs_wrapper testargs; + struct nlm_lockargs_wrapper lockargs; + struct nlm_cancargs_wrapper cancargs; + struct nlm_unlockargs_wrapper unlockargs; + struct nlm_notifyargs_wrapper notifyargs; + struct nlm_shareargs_wrapper shareargs; + struct nlm_notify_wrapper notify; + struct nlm_testres_wrapper testres; + struct nlm_res_wrapper res; + struct nlm_shareres_wrapper shareres; }; /* diff --git a/fs/lockd/svcshare.c b/fs/lockd/svcshare.c index 53f5655c128c..5ac0ec25d62d 100644 --- a/fs/lockd/svcshare.c +++ b/fs/lockd/svcshare.c @@ -19,7 +19,7 @@ #include "share.h" static inline int -nlm_cmp_owner(struct nlm_share *share, struct xdr_netobj *oh) +nlm_cmp_owner(struct lockd_share *share, struct xdr_netobj *oh) { return share->s_owner.len == oh->len && !memcmp(share->s_owner.data, oh->data, oh->len); @@ -39,7 +39,7 @@ __be32 nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file, struct xdr_netobj *oh, u32 access, u32 mode) { - struct nlm_share *share; + struct lockd_share *share; u8 *ohdata; if (nlmsvc_file_cannot_lock(file)) @@ -85,7 +85,7 @@ __be32 nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, struct xdr_netobj *oh) { - struct nlm_share *share, **shpp; + struct lockd_share *share, **shpp; if (nlmsvc_file_cannot_lock(file)) return nlm_lck_denied_nolocks; @@ -111,7 +111,7 @@ nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, void nlmsvc_traverse_shares(struct nlm_host *host, struct nlm_file *file, nlm_host_match_fn_t match) { - struct nlm_share *share, **shpp; + struct lockd_share *share, **shpp; shpp = &file->f_shares; while ((share = *shpp) != NULL) { diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c index 344e6c187cde..a0d1a6fbf61e 100644 --- a/fs/lockd/svcsubs.c +++ b/fs/lockd/svcsubs.c @@ -17,7 +17,6 @@ #include <linux/sunrpc/addr.h> #include <linux/module.h> #include <linux/mount.h> -#include <uapi/linux/nfs2.h> #include "lockd.h" #include "share.h" @@ -67,7 +66,7 @@ static inline unsigned int file_hash(struct nfs_fh *f) { unsigned int tmp=0; int i; - for (i=0; i<NFS2_FHSIZE;i++) + for (i = 0; i < LOCKD_FH_HASH_SIZE; i++) tmp += f->data[i]; return tmp & (FILE_NRHASH - 1); } @@ -83,23 +82,36 @@ int lock_to_openmode(struct file_lock *lock) * * We have to make sure we have the right credential to open * the file. + * + * @mode is O_RDONLY, O_WRONLY, or O_RDWR. O_RDWR means success + * is achieved with EITHER O_RDONLY or O_WRONLY; it does not + * require both. */ static __be32 nlm_do_fopen(struct svc_rqst *rqstp, struct nlm_file *file, int mode) { - struct file **fp = &file->f_file[mode]; - __be32 nlmerr = nlm_granted; + __be32 nlmerr = nlm__int__failed; + __be32 deferred = 0; int error; + int m; + + for (m = O_RDONLY; m <= O_WRONLY; m++) { + struct file **fp = &file->f_file[m]; - if (*fp) - return nlmerr; + if (mode != O_RDWR && mode != m) + continue; + if (*fp) + return nlm_granted; + + error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, m); + if (!error) + return nlm_granted; - error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, mode); - if (error) { dprintk("lockd: open failed (errno %d)\n", error); switch (error) { case -EWOULDBLOCK: nlmerr = nlm__int__drop_reply; + deferred = nlmerr; break; case -ESTALE: nlmerr = nlm__int__stale_fh; @@ -110,7 +122,7 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp, } } - return nlmerr; + return deferred ? deferred : nlmerr; } /* @@ -119,17 +131,15 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp, */ __be32 nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result, - struct nlm_lock *lock) + struct lockd_lock *lock, int mode) { struct nlm_file *file; unsigned int hash; __be32 nfserr; - int mode; nlm_debug_print_fh("nlm_lookup_file", &lock->fh); hash = file_hash(&lock->fh); - mode = lock_to_openmode(&lock->fl); /* Lock file table */ mutex_lock(&nlm_file_mutex); @@ -139,6 +149,8 @@ nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result, mutex_lock(&file->f_mutex); nfserr = nlm_do_fopen(rqstp, file, mode); mutex_unlock(&file->f_mutex); + if (nfserr) + goto out_unlock; goto found; } nlm_debug_print_fh("creating file for", &lock->fh); @@ -155,7 +167,7 @@ nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result, nfserr = nlm_do_fopen(rqstp, file, mode); if (nfserr) - goto out_unlock; + goto out_free; hlist_add_head(&file->f_list, &nlm_files[hash]); diff --git a/fs/lockd/svcxdr.h b/fs/lockd/svcxdr.h deleted file mode 100644 index 4f1a451da5ba..000000000000 --- a/fs/lockd/svcxdr.h +++ /dev/null @@ -1,142 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Encode/decode NLM basic data types - * - * Basic NLMv3 XDR data types are not defined in an IETF standards - * document. X/Open has a description of these data types that - * is useful. See Chapter 10 of "Protocols for Interworking: - * XNFS, Version 3W". - * - * Basic NLMv4 XDR data types are defined in Appendix II.1.4 of - * RFC 1813: "NFS Version 3 Protocol Specification". - * - * Author: Chuck Lever <chuck.lever@oracle.com> - * - * Copyright (c) 2020, Oracle and/or its affiliates. - */ - -#ifndef _LOCKD_SVCXDR_H_ -#define _LOCKD_SVCXDR_H_ - -static inline bool -svcxdr_decode_stats(struct xdr_stream *xdr, __be32 *status) -{ - __be32 *p; - - p = xdr_inline_decode(xdr, XDR_UNIT); - if (!p) - return false; - *status = *p; - - return true; -} - -static inline bool -svcxdr_encode_stats(struct xdr_stream *xdr, __be32 status) -{ - __be32 *p; - - p = xdr_reserve_space(xdr, XDR_UNIT); - if (!p) - return false; - *p = status; - - return true; -} - -static inline bool -svcxdr_decode_string(struct xdr_stream *xdr, char **data, unsigned int *data_len) -{ - __be32 *p; - u32 len; - - if (xdr_stream_decode_u32(xdr, &len) < 0) - return false; - if (len > NLM_MAXSTRLEN) - return false; - p = xdr_inline_decode(xdr, len); - if (!p) - return false; - *data_len = len; - *data = (char *)p; - - return true; -} - -/* - * NLM cookies are defined by specification to be a variable-length - * XDR opaque no longer than 1024 bytes. However, this implementation - * limits their length to 32 bytes, and treats zero-length cookies - * specially. - */ -static inline bool -svcxdr_decode_cookie(struct xdr_stream *xdr, struct nlm_cookie *cookie) -{ - __be32 *p; - u32 len; - - if (xdr_stream_decode_u32(xdr, &len) < 0) - return false; - if (len > NLM_MAXCOOKIELEN) - return false; - if (!len) - goto out_hpux; - - p = xdr_inline_decode(xdr, len); - if (!p) - return false; - cookie->len = len; - memcpy(cookie->data, p, len); - - return true; - - /* apparently HPUX can return empty cookies */ -out_hpux: - cookie->len = 4; - memset(cookie->data, 0, 4); - return true; -} - -static inline bool -svcxdr_encode_cookie(struct xdr_stream *xdr, const struct nlm_cookie *cookie) -{ - __be32 *p; - - if (xdr_stream_encode_u32(xdr, cookie->len) < 0) - return false; - p = xdr_reserve_space(xdr, cookie->len); - if (!p) - return false; - memcpy(p, cookie->data, cookie->len); - - return true; -} - -static inline bool -svcxdr_decode_owner(struct xdr_stream *xdr, struct xdr_netobj *obj) -{ - __be32 *p; - u32 len; - - if (xdr_stream_decode_u32(xdr, &len) < 0) - return false; - if (len > XDR_MAX_NETOBJ) - return false; - p = xdr_inline_decode(xdr, len); - if (!p) - return false; - obj->len = len; - obj->data = (u8 *)p; - - return true; -} - -static inline bool -svcxdr_encode_owner(struct xdr_stream *xdr, const struct xdr_netobj *obj) -{ - if (obj->len > XDR_MAX_NETOBJ) - return false; - return xdr_stream_encode_opaque(xdr, obj->data, obj->len) > 0; -} - -#endif /* _LOCKD_SVCXDR_H_ */ diff --git a/fs/lockd/trace.h b/fs/lockd/trace.h index 7214d7e96a42..a11d04e8c835 100644 --- a/fs/lockd/trace.h +++ b/fs/lockd/trace.h @@ -48,7 +48,7 @@ NLM_STATUS_LIST DECLARE_EVENT_CLASS(nlmclnt_lock_event, TP_PROTO( - const struct nlm_lock *lock, + const struct lockd_lock *lock, const struct sockaddr *addr, unsigned int addrlen, __be32 status @@ -61,8 +61,8 @@ DECLARE_EVENT_CLASS(nlmclnt_lock_event, __field(u32, svid) __field(u32, fh) __field(unsigned long, status) - __field(u64, start) - __field(u64, len) + __field(loff_t, start) + __field(loff_t, end) __sockaddr(addr, addrlen) ), @@ -70,16 +70,16 @@ DECLARE_EVENT_CLASS(nlmclnt_lock_event, __entry->oh = ~crc32_le(0xffffffff, lock->oh.data, lock->oh.len); __entry->svid = lock->svid; __entry->fh = nfs_fhandle_hash(&lock->fh); - __entry->start = lock->lock_start; - __entry->len = lock->lock_len; + __entry->start = lock->fl.fl_start; + __entry->end = lock->fl.fl_end; __entry->status = be32_to_cpu(status); __assign_sockaddr(addr, addr, addrlen); ), TP_printk( - "addr=%pISpc oh=0x%08x svid=0x%08x fh=0x%08x start=%llu len=%llu status=%s", + "addr=%pISpc oh=0x%08x svid=0x%08x fh=0x%08x start=%lld end=%lld status=%s", __get_sockaddr(addr), __entry->oh, __entry->svid, - __entry->fh, __entry->start, __entry->len, + __entry->fh, __entry->start, __entry->end, show_nlm_status(__entry->status) ) ); @@ -87,7 +87,7 @@ DECLARE_EVENT_CLASS(nlmclnt_lock_event, #define DEFINE_NLMCLNT_EVENT(name) \ DEFINE_EVENT(nlmclnt_lock_event, name, \ TP_PROTO( \ - const struct nlm_lock *lock, \ + const struct lockd_lock *lock, \ const struct sockaddr *addr, \ unsigned int addrlen, \ __be32 status \ diff --git a/fs/lockd/xdr.c b/fs/lockd/xdr.c deleted file mode 100644 index dfca8b8dab73..000000000000 --- a/fs/lockd/xdr.c +++ /dev/null @@ -1,354 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/fs/lockd/xdr.c - * - * XDR support for lockd and the lock client. - * - * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> - */ - -#include <linux/types.h> -#include <linux/sched.h> -#include <linux/nfs.h> - -#include <linux/sunrpc/xdr.h> -#include <linux/sunrpc/clnt.h> -#include <linux/sunrpc/svc.h> -#include <linux/sunrpc/stats.h> - -#include <uapi/linux/nfs2.h> - -#include "lockd.h" -#include "share.h" -#include "svcxdr.h" - -static inline loff_t -s32_to_loff_t(__s32 offset) -{ - return (loff_t)offset; -} - -static inline __s32 -loff_t_to_s32(loff_t offset) -{ - __s32 res; - if (offset >= NLM_OFFSET_MAX) - res = NLM_OFFSET_MAX; - else if (offset <= -NLM_OFFSET_MAX) - res = -NLM_OFFSET_MAX; - else - res = offset; - return res; -} - -/* - * NLM file handles are defined by specification to be a variable-length - * XDR opaque no longer than 1024 bytes. However, this implementation - * constrains their length to exactly the length of an NFSv2 file - * handle. - */ -static bool -svcxdr_decode_fhandle(struct xdr_stream *xdr, struct nfs_fh *fh) -{ - __be32 *p; - u32 len; - - if (xdr_stream_decode_u32(xdr, &len) < 0) - return false; - if (len != NFS2_FHSIZE) - return false; - - p = xdr_inline_decode(xdr, len); - if (!p) - return false; - fh->size = NFS2_FHSIZE; - memcpy(fh->data, p, len); - memset(fh->data + NFS2_FHSIZE, 0, sizeof(fh->data) - NFS2_FHSIZE); - - return true; -} - -static bool -svcxdr_decode_lock(struct xdr_stream *xdr, struct nlm_lock *lock) -{ - struct file_lock *fl = &lock->fl; - s32 start, len, end; - - if (!svcxdr_decode_string(xdr, &lock->caller, &lock->len)) - return false; - if (!svcxdr_decode_fhandle(xdr, &lock->fh)) - return false; - if (!svcxdr_decode_owner(xdr, &lock->oh)) - return false; - if (xdr_stream_decode_u32(xdr, &lock->svid) < 0) - return false; - if (xdr_stream_decode_u32(xdr, &start) < 0) - return false; - if (xdr_stream_decode_u32(xdr, &len) < 0) - return false; - - locks_init_lock(fl); - fl->c.flc_flags = FL_POSIX; - fl->c.flc_type = F_RDLCK; - end = start + len - 1; - fl->fl_start = s32_to_loff_t(start); - if (len == 0 || end < 0) - fl->fl_end = OFFSET_MAX; - else - fl->fl_end = s32_to_loff_t(end); - - return true; -} - -static bool -svcxdr_encode_holder(struct xdr_stream *xdr, const struct nlm_lock *lock) -{ - const struct file_lock *fl = &lock->fl; - s32 start, len; - - /* exclusive */ - if (xdr_stream_encode_bool(xdr, fl->c.flc_type != F_RDLCK) < 0) - return false; - if (xdr_stream_encode_u32(xdr, lock->svid) < 0) - return false; - if (!svcxdr_encode_owner(xdr, &lock->oh)) - return false; - start = loff_t_to_s32(fl->fl_start); - if (fl->fl_end == OFFSET_MAX) - len = 0; - else - len = loff_t_to_s32(fl->fl_end - fl->fl_start + 1); - if (xdr_stream_encode_u32(xdr, start) < 0) - return false; - if (xdr_stream_encode_u32(xdr, len) < 0) - return false; - - return true; -} - -static bool -svcxdr_encode_testrply(struct xdr_stream *xdr, const struct nlm_res *resp) -{ - if (!svcxdr_encode_stats(xdr, resp->status)) - return false; - switch (resp->status) { - case nlm_lck_denied: - if (!svcxdr_encode_holder(xdr, &resp->lock)) - return false; - } - - return true; -} - - -/* - * Decode Call arguments - */ - -bool -nlmsvc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - return true; -} - -bool -nlmsvc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_args *argp = rqstp->rq_argp; - u32 exclusive; - - if (!svcxdr_decode_cookie(xdr, &argp->cookie)) - return false; - if (xdr_stream_decode_bool(xdr, &exclusive) < 0) - return false; - if (!svcxdr_decode_lock(xdr, &argp->lock)) - return false; - if (exclusive) - argp->lock.fl.c.flc_type = F_WRLCK; - - return true; -} - -bool -nlmsvc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_args *argp = rqstp->rq_argp; - u32 exclusive; - - if (!svcxdr_decode_cookie(xdr, &argp->cookie)) - return false; - if (xdr_stream_decode_bool(xdr, &argp->block) < 0) - return false; - if (xdr_stream_decode_bool(xdr, &exclusive) < 0) - return false; - if (!svcxdr_decode_lock(xdr, &argp->lock)) - return false; - if (exclusive) - argp->lock.fl.c.flc_type = F_WRLCK; - if (xdr_stream_decode_bool(xdr, &argp->reclaim) < 0) - return false; - if (xdr_stream_decode_u32(xdr, &argp->state) < 0) - return false; - argp->monitor = 1; /* monitor client by default */ - - return true; -} - -bool -nlmsvc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_args *argp = rqstp->rq_argp; - u32 exclusive; - - if (!svcxdr_decode_cookie(xdr, &argp->cookie)) - return false; - if (xdr_stream_decode_bool(xdr, &argp->block) < 0) - return false; - if (xdr_stream_decode_bool(xdr, &exclusive) < 0) - return false; - if (!svcxdr_decode_lock(xdr, &argp->lock)) - return false; - if (exclusive) - argp->lock.fl.c.flc_type = F_WRLCK; - - return true; -} - -bool -nlmsvc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_args *argp = rqstp->rq_argp; - - if (!svcxdr_decode_cookie(xdr, &argp->cookie)) - return false; - if (!svcxdr_decode_lock(xdr, &argp->lock)) - return false; - argp->lock.fl.c.flc_type = F_UNLCK; - - return true; -} - -bool -nlmsvc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_res *resp = rqstp->rq_argp; - - if (!svcxdr_decode_cookie(xdr, &resp->cookie)) - return false; - if (!svcxdr_decode_stats(xdr, &resp->status)) - return false; - - return true; -} - -bool -nlmsvc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_reboot *argp = rqstp->rq_argp; - __be32 *p; - u32 len; - - if (xdr_stream_decode_u32(xdr, &len) < 0) - return false; - if (len > SM_MAXSTRLEN) - return false; - p = xdr_inline_decode(xdr, len); - if (!p) - return false; - argp->len = len; - argp->mon = (char *)p; - if (xdr_stream_decode_u32(xdr, &argp->state) < 0) - return false; - p = xdr_inline_decode(xdr, SM_PRIV_SIZE); - if (!p) - return false; - memcpy(&argp->priv.data, p, sizeof(argp->priv.data)); - - return true; -} - -bool -nlmsvc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_lock *lock = &argp->lock; - - memset(lock, 0, sizeof(*lock)); - locks_init_lock(&lock->fl); - lock->svid = LOCKD_SHARE_SVID; - - if (!svcxdr_decode_cookie(xdr, &argp->cookie)) - return false; - if (!svcxdr_decode_string(xdr, &lock->caller, &lock->len)) - return false; - if (!svcxdr_decode_fhandle(xdr, &lock->fh)) - return false; - if (!svcxdr_decode_owner(xdr, &lock->oh)) - return false; - /* XXX: Range checks are missing in the original code */ - if (xdr_stream_decode_u32(xdr, &argp->fsm_mode) < 0) - return false; - if (xdr_stream_decode_u32(xdr, &argp->fsm_access) < 0) - return false; - - return true; -} - -bool -nlmsvc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_args *argp = rqstp->rq_argp; - struct nlm_lock *lock = &argp->lock; - - if (!svcxdr_decode_string(xdr, &lock->caller, &lock->len)) - return false; - if (xdr_stream_decode_u32(xdr, &argp->state) < 0) - return false; - - return true; -} - - -/* - * Encode Reply results - */ - -bool -nlmsvc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - return true; -} - -bool -nlmsvc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_res *resp = rqstp->rq_resp; - - return svcxdr_encode_cookie(xdr, &resp->cookie) && - svcxdr_encode_testrply(xdr, resp); -} - -bool -nlmsvc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_res *resp = rqstp->rq_resp; - - return svcxdr_encode_cookie(xdr, &resp->cookie) && - svcxdr_encode_stats(xdr, resp->status); -} - -bool -nlmsvc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - struct nlm_res *resp = rqstp->rq_resp; - - if (!svcxdr_encode_cookie(xdr, &resp->cookie)) - return false; - if (!svcxdr_encode_stats(xdr, resp->status)) - return false; - /* sequence */ - if (xdr_stream_encode_u32(xdr, 0) < 0) - return false; - - return true; -} diff --git a/fs/lockd/xdr.h b/fs/lockd/xdr.h index 3c60817c4349..a1126cca98c6 100644 --- a/fs/lockd/xdr.h +++ b/fs/lockd/xdr.h @@ -20,8 +20,6 @@ struct nsm_private { unsigned char data[SM_PRIV_SIZE]; }; -struct svc_rqst; - #define NLM_MAXCOOKIELEN 32 #define NLM_MAXSTRLEN 1024 @@ -32,7 +30,7 @@ struct svc_rqst; #define nlm_lck_denied_grace_period cpu_to_be32(NLM_LCK_DENIED_GRACE_PERIOD) /* Lock info passed via NLM */ -struct nlm_lock { +struct lockd_lock { char * caller; unsigned int len; /* length of "caller" */ struct nfs_fh fh; @@ -49,8 +47,7 @@ struct nlm_lock { * 32 bytes. */ -struct nlm_cookie -{ +struct lockd_cookie { unsigned char data[NLM_MAXCOOKIELEN]; unsigned int len; }; @@ -58,49 +55,31 @@ struct nlm_cookie /* * Generic lockd arguments for all but sm_notify */ -struct nlm_args { - struct nlm_cookie cookie; - struct nlm_lock lock; +struct lockd_args { + struct lockd_cookie cookie; + struct lockd_lock lock; u32 block; u32 reclaim; u32 state; - u32 monitor; - u32 fsm_access; - u32 fsm_mode; }; /* * Generic lockd result */ -struct nlm_res { - struct nlm_cookie cookie; +struct lockd_res { + struct lockd_cookie cookie; __be32 status; - struct nlm_lock lock; + struct lockd_lock lock; }; /* * statd callback when client has rebooted */ -struct nlm_reboot { +struct lockd_reboot { char *mon; unsigned int len; u32 state; struct nsm_private priv; }; -bool nlmsvc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); - -bool nlmsvc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); - #endif /* _LOCKD_XDR_H */ diff --git a/fs/locks.c b/fs/locks.c index fead53474c30..6e4ff7fcec05 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -1582,30 +1582,96 @@ trace: return rc; } -static bool -any_leases_conflict(struct inode *inode, struct file_lease *breaker) +#define IGNORE_MASK (FL_IGN_DIR_CREATE | FL_IGN_DIR_DELETE | FL_IGN_DIR_RENAME) + +/** + * inode_lease_ignore_mask - return union of all ignored inode events for this inode + * @inode: inode of which to get ignore mask + * + * Walk the list of leases, and return the result of all of + * their FL_IGN_DIR_* bits or'ed together. + */ +u32 +inode_lease_ignore_mask(struct inode *inode) { - struct file_lock_context *ctx = inode->i_flctx; + struct file_lock_context *ctx; struct file_lock_core *flc; + u32 mask = 0; - lockdep_assert_held(&ctx->flc_lock); + ctx = locks_inode_context(inode); + if (!ctx) + return 0; + spin_lock(&ctx->flc_lock); list_for_each_entry(flc, &ctx->flc_lease, flc_list) { - if (leases_conflict(flc, &breaker->c)) - return true; + mask |= flc->flc_flags & IGNORE_MASK; + /* If we already have everything, we can stop */ + if (mask == IGNORE_MASK) + break; } + spin_unlock(&ctx->flc_lock); + return mask; +} +EXPORT_SYMBOL_GPL(inode_lease_ignore_mask); + +static bool +ignore_dir_deleg_break(struct file_lease *fl, unsigned int flags) +{ + if ((flags & LEASE_BREAK_DIR_CREATE) && (fl->c.flc_flags & FL_IGN_DIR_CREATE)) + return true; + if ((flags & LEASE_BREAK_DIR_DELETE) && (fl->c.flc_flags & FL_IGN_DIR_DELETE)) + return true; + if ((flags & LEASE_BREAK_DIR_RENAME) && (fl->c.flc_flags & FL_IGN_DIR_RENAME)) + return true; + return false; } +static unsigned int +break_lease_flags_to_type(unsigned int flags) +{ + if (flags & LEASE_BREAK_LEASE) + return FL_LEASE; + else if (flags & LEASE_BREAK_DELEG) + return FL_DELEG; + else if (flags & LEASE_BREAK_LAYOUT) + return FL_LAYOUT; + else + return 0; + +} + +static struct file_lease * +first_visible_lease(struct inode *inode, struct file_lease *new_fl, unsigned int flags) +{ + struct file_lock_context *ctx = locks_inode_context(inode); + struct file_lease *fl; + + lockdep_assert_held(&ctx->flc_lock); + + list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) { + if (!leases_conflict(&fl->c, &new_fl->c)) + continue; + if (S_ISDIR(inode->i_mode) && ignore_dir_deleg_break(fl, flags)) + continue; + return fl; + } + return NULL; +} + + /** - * __break_lease - revoke all outstanding leases on file - * @inode: the inode of the file to return - * @flags: LEASE_BREAK_* flags + * __break_lease - revoke all outstanding leases on file + * @inode: the inode of the file to return + * @flags: LEASE_BREAK_* flags * - * break_lease (inlined for speed) has checked there already is at least - * some kind of lock (maybe a lease) on this file. Leases are broken on - * a call to open() or truncate(). This function can block waiting for the - * lease break unless you specify LEASE_BREAK_NONBLOCK. + * break_lease (inlined for speed) has checked there already is at least + * some kind of lock (maybe a lease) on this file. Leases and Delegations + * are broken on a call to open() or truncate(). Delegations are also + * broken on any event that would change the ctime. Directory delegations + * are broken whenever the directory changes (unless the delegation is set + * up to ignore the event). This function can block waiting for the lease + * break unless you specify LEASE_BREAK_NONBLOCK. */ int __break_lease(struct inode *inode, unsigned int flags) { @@ -1617,13 +1683,10 @@ int __break_lease(struct inode *inode, unsigned int flags) bool want_write = !(flags & LEASE_BREAK_OPEN_RDONLY); int error = 0; - if (flags & LEASE_BREAK_LEASE) - type = FL_LEASE; - else if (flags & LEASE_BREAK_DELEG) - type = FL_DELEG; - else if (flags & LEASE_BREAK_LAYOUT) - type = FL_LAYOUT; - else + trace_break_lease(inode, flags); + + type = break_lease_flags_to_type(flags); + if (!type) return -EINVAL; new_fl = lease_alloc(NULL, type, want_write ? F_WRLCK : F_RDLCK); @@ -1642,7 +1705,7 @@ int __break_lease(struct inode *inode, unsigned int flags) time_out_leases(inode, &dispose); - if (!any_leases_conflict(inode, new_fl)) + if (!first_visible_lease(inode, new_fl, flags)) goto out; break_time = 0; @@ -1655,6 +1718,8 @@ int __break_lease(struct inode *inode, unsigned int flags) list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) { if (!leases_conflict(&fl->c, &new_fl->c)) continue; + if (S_ISDIR(inode->i_mode) && ignore_dir_deleg_break(fl, flags)) + continue; if (want_write) { if (fl->c.flc_flags & FL_UNLOCK_PENDING) continue; @@ -1670,7 +1735,8 @@ int __break_lease(struct inode *inode, unsigned int flags) locks_delete_lock_ctx(&fl->c, &dispose); } - if (list_empty(&ctx->flc_lease)) + fl = first_visible_lease(inode, new_fl, flags); + if (!fl) goto out; if (flags & LEASE_BREAK_NONBLOCK) { @@ -1680,7 +1746,6 @@ int __break_lease(struct inode *inode, unsigned int flags) } restart: - fl = list_first_entry(&ctx->flc_lease, struct file_lease, c.flc_list); break_time = fl->fl_break_time; if (break_time != 0) { if (time_after(jiffies, break_time)) { @@ -1691,7 +1756,7 @@ restart: } else break_time++; locks_insert_block(&fl->c, &new_fl->c, leases_conflict); - trace_break_lease_block(inode, new_fl); + trace_break_lease_block(inode, fl); spin_unlock(&ctx->flc_lock); percpu_up_read(&file_rwsem); @@ -1702,7 +1767,7 @@ restart: percpu_down_read(&file_rwsem); spin_lock(&ctx->flc_lock); - trace_break_lease_unblock(inode, new_fl); + trace_break_lease_unblock(inode, NULL); __locks_delete_block(&new_fl->c); if (error >= 0) { /* @@ -1711,7 +1776,8 @@ restart: */ if (error == 0) time_out_leases(inode, &dispose); - if (any_leases_conflict(inode, new_fl)) + fl = first_visible_lease(inode, new_fl, flags); + if (fl) goto restart; error = 0; } diff --git a/fs/minix/inode.c b/fs/minix/inode.c index 9c6bac248907..c30cc590698d 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -292,7 +292,8 @@ static int minix_fill_super(struct super_block *s, struct fs_context *fc) sbi->s_namelen = 60; sbi->s_version = MINIX_V3; sbi->s_mount_state = MINIX_VALID_FS; - sb_set_blocksize(s, m3s->s_blocksize); + if (!sb_set_blocksize(s, m3s->s_blocksize)) + goto out_release; s->s_max_links = MINIX2_LINK_MAX; } else goto out_no_fs; diff --git a/fs/minix/minix.h b/fs/minix/minix.h index f2025c9b5825..9e52d4302f0d 100644 --- a/fs/minix/minix.h +++ b/fs/minix/minix.h @@ -97,7 +97,7 @@ static inline struct minix_inode_info *minix_i(struct inode *inode) static inline unsigned minix_blocks_needed(unsigned bits, unsigned blocksize) { - return DIV_ROUND_UP(bits, blocksize * 8); + return DIV_ROUND_UP_POW2(bits, blocksize * 8); } #if defined(CONFIG_MINIX_FS_NATIVE_ENDIAN) && \ diff --git a/fs/mnt_idmapping.c b/fs/mnt_idmapping.c index 6472c4ea3d1e..cb61fbdb52e9 100644 --- a/fs/mnt_idmapping.c +++ b/fs/mnt_idmapping.c @@ -375,6 +375,8 @@ int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_ continue; seq_printf(seq, "%u %u %u", extent->first, lower, extent->count); + if (seq_has_overflowed(seq)) + return -EAGAIN; seq->count++; /* mappings are separated by \0 */ if (seq_has_overflowed(seq)) diff --git a/fs/mount.h b/fs/mount.h index e0816c11a198..94fcc306d21e 100644 --- a/fs/mount.h +++ b/fs/mount.h @@ -25,6 +25,7 @@ struct mnt_namespace { __u32 n_fsnotify_mask; struct fsnotify_mark_connector __rcu *n_fsnotify_marks; #endif + struct hlist_head mnt_visible_mounts; /* SB_I_USERNS_VISIBLE mounts */ unsigned int nr_mounts; /* # of mounts in the namespace */ unsigned int pending_mounts; refcount_t passive; /* number references not pinning @mounts */ @@ -71,7 +72,15 @@ struct mount { struct hlist_head mnt_slave_list;/* list of slave mounts */ struct hlist_node mnt_slave; /* slave list entry */ struct mount *mnt_master; /* slave is on master->mnt_slave_list */ - struct mnt_namespace *mnt_ns; /* containing namespace */ + /* + * Containing namespace (active or deactivating, non-refcounted). + * Normally protected by namespace_sem. + * Can also be accessed locklessly under RCU. RCU readers can't rely on + * the namespace still being active, but implicitly hold a passive + * reference (because an RCU delay happens between a namespace being + * deactivated and the corresponding passive refcount drop). + */ + struct mnt_namespace *mnt_ns; struct mountpoint *mnt_mp; /* where is it mounted */ union { struct hlist_node mnt_mp_list; /* list mounts with the same mountpoint */ @@ -90,6 +99,7 @@ struct mount { int mnt_expiry_mark; /* true if marked for expiry */ struct hlist_head mnt_pins; struct hlist_head mnt_stuck_children; + struct hlist_node mnt_ns_visible; /* link in ns->mnt_visible_mounts */ struct mount *overmount; /* mounted on ->mnt_root */ } __randomize_layout; @@ -207,6 +217,8 @@ static inline void move_from_ns(struct mount *mnt) ns->mnt_first_node = rb_next(&mnt->mnt_node); rb_erase(&mnt->mnt_node, &ns->mounts); RB_CLEAR_NODE(&mnt->mnt_node); + if (!hlist_unhashed(&mnt->mnt_ns_visible)) + hlist_del_init(&mnt->mnt_ns_visible); } bool has_locked_children(struct mount *mnt, struct dentry *dentry); diff --git a/fs/namei.c b/fs/namei.c index c7fac83c9a85..19ce43c9a6e6 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -129,6 +129,11 @@ static struct kmem_cache *__names_cache __ro_after_init; #define names_cache runtime_const_ptr(__names_cache) +/* + * Type of the last component on LOOKUP_PARENT + */ +enum last_type {LAST_NORM, LAST_ROOT, LAST_DOT, LAST_DOTDOT}; + void __init filename_init(void) { __names_cache = kmem_cache_create_usercopy("names_cache", sizeof(struct filename), 0, @@ -727,7 +732,7 @@ struct nameidata { struct inode *inode; /* path.dentry.d_inode */ unsigned int flags, state; unsigned seq, next_seq, m_seq, r_seq; - int last_type; + enum last_type last_type; unsigned depth; int total_link_count; struct saved { @@ -1891,13 +1896,12 @@ static struct dentry *__lookup_slow(const struct qstr *name, { struct dentry *dentry, *old; struct inode *inode = dir->d_inode; - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); /* Don't go there if it's already dead */ if (unlikely(IS_DEADDIR(inode))) return ERR_PTR(-ENOENT); again: - dentry = d_alloc_parallel(dir, name, &wq); + dentry = d_alloc_parallel(dir, name); if (IS_ERR(dentry)) return dentry; if (unlikely(!d_in_lookup(dentry))) { @@ -2220,7 +2224,7 @@ in_root: return dget(nd->path.dentry); } -static const char *handle_dots(struct nameidata *nd, int type) +static const char *handle_dots(struct nameidata *nd, enum last_type type) { if (type == LAST_DOTDOT) { const char *error = NULL; @@ -2868,7 +2872,7 @@ static int path_parentat(struct nameidata *nd, unsigned flags, /* Note: this does not consume "name" */ static int __filename_parentat(int dfd, struct filename *name, unsigned int flags, struct path *parent, - struct qstr *last, int *type, + struct qstr *last, enum last_type *type, const struct path *root) { int retval; @@ -2893,7 +2897,7 @@ static int __filename_parentat(int dfd, struct filename *name, static int filename_parentat(int dfd, struct filename *name, unsigned int flags, struct path *parent, - struct qstr *last, int *type) + struct qstr *last, enum last_type *type) { return __filename_parentat(dfd, name, flags, parent, last, type, NULL); } @@ -2955,15 +2959,17 @@ void end_dirop(struct dentry *de) EXPORT_SYMBOL(end_dirop); /* does lookup, returns the object with parent locked */ -static struct dentry *__start_removing_path(int dfd, struct filename *name, - struct path *path) +struct dentry *start_removing_path(const char *name, struct path *path) { + CLASS(filename_kernel, filename)(name); struct path parent_path __free(path_put) = {}; struct dentry *d; struct qstr last; - int type, error; + enum last_type type; + int error; - error = filename_parentat(dfd, name, 0, &parent_path, &last, &type); + error = filename_parentat(AT_FDCWD, filename, 0, &parent_path, &last, + &type); if (error) return ERR_PTR(error); if (unlikely(type != LAST_NORM)) @@ -3007,7 +3013,8 @@ struct dentry *kern_path_parent(const char *name, struct path *path) CLASS(filename_kernel, filename)(name); struct dentry *d; struct qstr last; - int type, error; + enum last_type type; + int error; error = filename_parentat(AT_FDCWD, filename, 0, &parent_path, &last, &type); if (error) @@ -3023,21 +3030,6 @@ struct dentry *kern_path_parent(const char *name, struct path *path) return d; } -struct dentry *start_removing_path(const char *name, struct path *path) -{ - CLASS(filename_kernel, filename)(name); - return __start_removing_path(AT_FDCWD, filename, path); -} - -struct dentry *start_removing_user_path_at(int dfd, - const char __user *name, - struct path *path) -{ - CLASS(filename, filename)(name); - return __start_removing_path(dfd, filename, path); -} -EXPORT_SYMBOL(start_removing_user_path_at); - int kern_path(const char *name, unsigned int flags, struct path *path) { CLASS(filename_kernel, filename)(name); @@ -3051,15 +3043,22 @@ EXPORT_SYMBOL(kern_path); * @flags: lookup flags * @parent: pointer to struct path to fill * @last: last component - * @type: type of the last component * @root: pointer to struct path of the base directory */ int vfs_path_parent_lookup(struct filename *filename, unsigned int flags, - struct path *parent, struct qstr *last, int *type, + struct path *parent, struct qstr *last, const struct path *root) { - return __filename_parentat(AT_FDCWD, filename, flags, parent, last, - type, root); + enum last_type type; + int err = __filename_parentat(AT_FDCWD, filename, flags, parent, last, + &type, root); + if (err) + return err; + if (unlikely(type != LAST_NORM)) { + path_put(parent); + return -EINVAL; + } + return 0; } EXPORT_SYMBOL(vfs_path_parent_lookup); @@ -3617,7 +3616,6 @@ int path_pts(struct path *path) */ struct dentry *parent = dget_parent(path->dentry); struct dentry *child; - struct qstr this = QSTR_INIT("pts", 3); if (unlikely(!path_connected(path->mnt, parent))) { dput(parent); @@ -3625,7 +3623,7 @@ int path_pts(struct path *path) } dput(path->dentry); path->dentry = parent; - child = d_hash_and_lookup(parent, &this); + child = d_hash_and_lookup(parent, &QSTR("pts")); if (IS_ERR_OR_NULL(child)) return -ENOENT; @@ -4198,7 +4196,7 @@ int vfs_create(struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode, error = security_inode_create(dir, dentry, mode); if (error) return error; - error = try_break_deleg(dir, di); + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, di); if (error) return error; error = dir->i_op->create(idmap, dir, dentry, mode, true); @@ -4414,7 +4412,6 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file, struct dentry *dentry; int error, create_error = 0; umode_t mode = op->mode; - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); if (unlikely(IS_DEADDIR(dir_inode))) return ERR_PTR(-ENOENT); @@ -4423,7 +4420,7 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file, dentry = d_lookup(dir, &nd->last); for (;;) { if (!dentry) { - dentry = d_alloc_parallel(dir, &nd->last, &wq); + dentry = d_alloc_parallel(dir, &nd->last); if (IS_ERR(dentry)) return dentry; } @@ -4497,7 +4494,7 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file, /* Negative dentry, just create the file */ if (!dentry->d_inode && (open_flag & O_CREAT)) { /* but break the directory lease first! */ - error = try_break_deleg(dir_inode, delegated_inode); + error = try_break_deleg(dir_inode, LEASE_BREAK_DIR_CREATE, delegated_inode); if (error) goto out_dput; @@ -4679,6 +4676,10 @@ static int do_open(struct nameidata *nd, if (unlikely(error)) return error; } + + if ((open_flag & __O_REGULAR) && !d_is_reg(nd->path.dentry)) + return -EFTYPE; + if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry)) return -ENOTDIR; @@ -4735,6 +4736,10 @@ int vfs_tmpfile(struct mnt_idmap *idmap, int error; int open_flag = file->f_flags; + /* A tmpfile is I_LINKABLE, so guard its owner like may_o_create(). */ + if (!fsuidgid_has_mapping(dir->i_sb, idmap)) + return -EOVERFLOW; + /* we want directory to be writable */ error = inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC); if (error) @@ -4925,7 +4930,7 @@ static struct dentry *filename_create(int dfd, struct filename *name, bool want_dir = lookup_flags & LOOKUP_DIRECTORY; unsigned int reval_flag = lookup_flags & LOOKUP_REVAL; unsigned int create_flags = LOOKUP_CREATE | LOOKUP_EXCL; - int type; + enum last_type type; int error; error = filename_parentat(dfd, name, reval_flag, path, &last, &type); @@ -5024,6 +5029,7 @@ struct file *dentry_create(struct path *path, int flags, umode_t mode, { struct file *file __free(fput) = NULL; struct dentry *dentry = path->dentry; + struct dentry *orig_dentry = dentry; struct dentry *dir = dentry->d_parent; struct inode *dir_inode = d_inode(dir); struct mnt_idmap *idmap; @@ -5043,9 +5049,18 @@ struct file *dentry_create(struct path *path, int flags, umode_t mode, if (create_error) flags &= ~O_CREAT; + /* atomic_open will dput(dentry) on error */ + dget(orig_dentry); dentry = atomic_open(path, dentry, file, flags, mode); error = PTR_ERR_OR_ZERO(dentry); + if (IS_ERR(dentry)) + /* keep the original */ + dentry = orig_dentry; + else + /* Drop the extra reference */ + dput(orig_dentry); + if (unlikely(create_error) && error == -ENOENT) error = create_error; @@ -5113,7 +5128,7 @@ int vfs_mknod(struct mnt_idmap *idmap, struct inode *dir, if (error) return error; - error = try_break_deleg(dir, delegated_inode); + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); if (error) return error; @@ -5254,7 +5269,7 @@ struct dentry *vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, if (max_links && dir->i_nlink >= max_links) goto err; - error = try_break_deleg(dir, delegated_inode); + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); if (error) goto err; @@ -5359,7 +5374,7 @@ int vfs_rmdir(struct mnt_idmap *idmap, struct inode *dir, if (error) goto out; - error = try_break_deleg(dir, delegated_inode); + error = try_break_deleg(dir, LEASE_BREAK_DIR_DELETE, delegated_inode); if (error) goto out; @@ -5387,7 +5402,7 @@ int filename_rmdir(int dfd, struct filename *name) struct dentry *dentry; struct path path; struct qstr last; - int type; + enum last_type type; unsigned int lookup_flags = 0; struct delegated_inode delegated_inode = { }; retry: @@ -5396,6 +5411,8 @@ retry: return error; switch (type) { + case LAST_NORM: + break; case LAST_DOTDOT: error = -ENOTEMPTY; goto exit2; @@ -5489,10 +5506,10 @@ int vfs_unlink(struct mnt_idmap *idmap, struct inode *dir, else { error = security_inode_unlink(dir, dentry); if (!error) { - error = try_break_deleg(dir, delegated_inode); + error = try_break_deleg(dir, LEASE_BREAK_DIR_DELETE, delegated_inode); if (error) goto out; - error = try_break_deleg(target, delegated_inode); + error = try_break_deleg(target, 0, delegated_inode); if (error) goto out; error = dir->i_op->unlink(dir, dentry); @@ -5529,7 +5546,7 @@ int filename_unlinkat(int dfd, struct filename *name) struct dentry *dentry; struct path path; struct qstr last; - int type; + enum last_type type; struct inode *inode; struct delegated_inode delegated_inode = { }; unsigned int lookup_flags = 0; @@ -5636,7 +5653,7 @@ int vfs_symlink(struct mnt_idmap *idmap, struct inode *dir, if (error) return error; - error = try_break_deleg(dir, delegated_inode); + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); if (error) return error; @@ -5767,9 +5784,9 @@ int vfs_link(struct dentry *old_dentry, struct mnt_idmap *idmap, else if (max_links && inode->i_nlink >= max_links) error = -EMLINK; else { - error = try_break_deleg(dir, delegated_inode); + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); if (!error) - error = try_break_deleg(inode, delegated_inode); + error = try_break_deleg(inode, 0, delegated_inode); if (!error) error = dir->i_op->link(old_dentry, dir, new_dentry); } @@ -6033,21 +6050,24 @@ int vfs_rename(struct renamedata *rd) old_dir->i_nlink >= max_links) goto out; } - error = try_break_deleg(old_dir, delegated_inode); + error = try_break_deleg(old_dir, + old_dir == new_dir ? LEASE_BREAK_DIR_RENAME : + LEASE_BREAK_DIR_DELETE, + delegated_inode); if (error) goto out; if (new_dir != old_dir) { - error = try_break_deleg(new_dir, delegated_inode); + error = try_break_deleg(new_dir, LEASE_BREAK_DIR_CREATE, delegated_inode); if (error) goto out; } if (!is_dir) { - error = try_break_deleg(source, delegated_inode); + error = try_break_deleg(source, 0, delegated_inode); if (error) goto out; } if (target && !new_is_dir) { - error = try_break_deleg(target, delegated_inode); + error = try_break_deleg(target, 0, delegated_inode); if (error) goto out; } @@ -6096,7 +6116,7 @@ int filename_renameat2(int olddfd, struct filename *from, struct renamedata rd; struct path old_path, new_path; struct qstr old_last, new_last; - int old_type, new_type; + enum last_type old_type, new_type; struct delegated_inode delegated_inode = { }; unsigned int lookup_flags = 0; bool should_retry = false; diff --git a/fs/namespace.c b/fs/namespace.c index fe919abd2f01..3d5cd5bf3b05 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -321,6 +321,7 @@ static struct mount *alloc_vfsmnt(const char *name) INIT_HLIST_NODE(&mnt->mnt_slave); INIT_HLIST_NODE(&mnt->mnt_mp_list); INIT_HLIST_HEAD(&mnt->mnt_stuck_children); + INIT_HLIST_NODE(&mnt->mnt_ns_visible); RB_CLEAR_NODE(&mnt->mnt_node); mnt->mnt.mnt_idmap = &nop_mnt_idmap; } @@ -1079,7 +1080,7 @@ static void mnt_add_to_ns(struct mnt_namespace *ns, struct mount *mnt) bool mnt_first_node = true, mnt_last_node = true; WARN_ON(mnt_ns_attached(mnt)); - mnt->mnt_ns = ns; + WRITE_ONCE(mnt->mnt_ns, ns); while (*link) { parent = *link; if (mnt->mnt_id_unique < node_to_mount(parent)->mnt_id_unique) { @@ -1098,6 +1099,10 @@ static void mnt_add_to_ns(struct mnt_namespace *ns, struct mount *mnt) rb_link_node(&mnt->mnt_node, parent, link); rb_insert_color(&mnt->mnt_node, &ns->mounts); + if ((mnt->mnt.mnt_sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED) && + mnt->mnt.mnt_root == mnt->mnt.mnt_sb->s_root) + hlist_add_head(&mnt->mnt_ns_visible, &ns->mnt_visible_mounts); + mnt_notify_add(mnt); } @@ -1434,7 +1439,7 @@ EXPORT_SYMBOL(mntget); void mnt_make_shortterm(struct vfsmount *mnt) { if (mnt) - real_mount(mnt)->mnt_ns = NULL; + WRITE_ONCE(real_mount(mnt)->mnt_ns, NULL); } /** @@ -1806,7 +1811,7 @@ static void umount_tree(struct mount *mnt, enum umount_tree_flags how) ns->nr_mounts--; __touch_mnt_namespace(ns); } - p->mnt_ns = NULL; + WRITE_ONCE(p->mnt_ns, NULL); if (how & UMOUNT_SYNC) p->mnt.mnt_flags |= MNT_SYNC_UMOUNT; @@ -3103,6 +3108,9 @@ static struct mnt_namespace *create_new_namespace(struct path *path, unsigned int copy_flags = 0; bool locked = false, recurse = flags & MOUNT_COPY_RECURSIVE; + if (unlikely(!d_can_lookup(path->dentry))) + return ERR_PTR(-ENOTDIR); + if (user_ns != ns->user_ns) copy_flags |= CL_SLAVE; @@ -3303,9 +3311,9 @@ static void mnt_warn_timestamp_expiry(const struct path *mountpoint, (ktime_get_real_seconds() + TIME_UPTIME_SEC_MAX > sb->s_time_max)) { char *buf, *mntpath; - buf = (char *)__get_free_page(GFP_KERNEL); + buf = __getname(); if (buf) - mntpath = d_path(mountpoint, buf, PAGE_SIZE); + mntpath = d_path(mountpoint, buf, PATH_MAX); else mntpath = ERR_PTR(-ENOMEM); if (IS_ERR(mntpath)) @@ -3318,8 +3326,7 @@ static void mnt_warn_timestamp_expiry(const struct path *mountpoint, (unsigned long long)sb->s_time_max); sb->s_iflags |= SB_I_TS_EXPIRY_WARNED; - if (buf) - free_page((unsigned long)buf); + __putname(buf); } } @@ -4499,6 +4506,10 @@ SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags, new_mnt = vfs_create_mount(fc); if (IS_ERR(new_mnt)) return PTR_ERR(new_mnt); + if (new_mnt->mnt_sb->s_flags & SB_NOUSER) { + mntput(new_mnt); + return -EINVAL; + } new_mnt->mnt_flags = mnt_flags; new_path.dentry = dget(fc->root); @@ -6340,20 +6351,26 @@ static bool mnt_already_visible(struct mnt_namespace *ns, int *new_mnt_flags) { int new_flags = *new_mnt_flags; - struct mount *mnt, *n; + struct mount *mnt; + + /* Don't acquire namespace semaphore without a good reason. */ + if (hlist_empty(&ns->mnt_visible_mounts)) + return false; guard(namespace_shared)(); - rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) { + hlist_for_each_entry(mnt, &ns->mnt_visible_mounts, mnt_ns_visible) { + const struct super_block *sb_visible = mnt->mnt.mnt_sb; struct mount *child; int mnt_flags; - if (mnt->mnt.mnt_sb->s_type != sb->s_type) + if (sb_visible->s_type != sb->s_type) continue; - /* This mount is not fully visible if it's root directory - * is not the root directory of the filesystem. + /* + * Restricted variants are not compatible with anything, even + * other restricted variants. */ - if (mnt->mnt.mnt_root != mnt->mnt.mnt_sb->s_root) + if (sb_visible->s_iflags & SB_I_RESTRICTED_VARIANT) continue; /* A local view of the mount flags */ @@ -6405,16 +6422,23 @@ static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags return false; /* Can this filesystem be too revealing? */ - s_iflags = sb->s_iflags; - if (!(s_iflags & SB_I_USERNS_VISIBLE)) + if (!(sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED)) return false; + s_iflags = sb->s_iflags; if ((s_iflags & required_iflags) != required_iflags) { WARN_ONCE(1, "Expected s_iflags to contain 0x%lx\n", required_iflags); return true; } + /* + * Restricted variants don't need an already visible mount because they + * don't expose the full filesystem view. + */ + if (s_iflags & SB_I_RESTRICTED_VARIANT) + return false; + return !mnt_already_visible(ns, sb, new_mnt_flags); } diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index a8c0d86118c5..24a8a5418e31 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -156,9 +156,8 @@ static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq, netfs_cache_read_terminated, subreq); } -static void netfs_queue_read(struct netfs_io_request *rreq, - struct netfs_io_subrequest *subreq, - bool last_subreq) +void netfs_queue_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) { struct netfs_io_stream *stream = &rreq->io_streams[0]; @@ -169,7 +168,8 @@ static void netfs_queue_read(struct netfs_io_request *rreq, * remove entries off of the front. */ spin_lock(&rreq->lock); - list_add_tail(&subreq->rreq_link, &stream->subrequests); + /* Write IN_PROGRESS before pointer to new subreq */ + list_add_tail_release(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { if (!stream->active) { stream->collected_to = subreq->start; @@ -178,11 +178,6 @@ static void netfs_queue_read(struct netfs_io_request *rreq, } } - if (last_subreq) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); - } - spin_unlock(&rreq->lock); } @@ -214,7 +209,6 @@ static void netfs_issue_read(struct netfs_io_request *rreq, static void netfs_read_to_pagecache(struct netfs_io_request *rreq, struct readahead_control *ractl) { - struct netfs_inode *ictx = netfs_inode(rreq->inode); unsigned long long start = rreq->start; ssize_t size = rreq->len; int ret = 0; @@ -233,10 +227,13 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq, subreq->start = start; subreq->len = size; + netfs_queue_read(rreq, subreq); + source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size); subreq->source = source; if (source == NETFS_DOWNLOAD_FROM_SERVER) { - unsigned long long zp = umin(ictx->zero_point, rreq->i_size); + unsigned long long zero_point = netfs_read_zero_point(rreq->inode); + unsigned long long zp = umin(zero_point, rreq->i_size); size_t len = subreq->len; if (unlikely(rreq->origin == NETFS_READ_SINGLE)) @@ -252,7 +249,8 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq, pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx", rreq->debug_id, subreq->debug_index, subreq->len, size, - subreq->start, ictx->zero_point, rreq->i_size); + subreq->start, zero_point, rreq->i_size); + netfs_cancel_read(subreq, ret); break; } subreq->len = len; @@ -261,12 +259,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq, if (rreq->netfs_ops->prepare_read) { ret = rreq->netfs_ops->prepare_read(subreq); if (ret < 0) { - subreq->error = ret; - /* Not queued - release both refs. */ - netfs_put_subrequest(subreq, - netfs_sreq_trace_put_cancel); - netfs_put_subrequest(subreq, - netfs_sreq_trace_put_cancel); + netfs_cancel_read(subreq, ret); break; } trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); @@ -289,24 +282,29 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq, pr_err("Unexpected read source %u\n", source); WARN_ON_ONCE(1); + netfs_cancel_read(subreq, ret); break; issue: slice = netfs_prepare_read_iterator(subreq, ractl); if (slice < 0) { ret = slice; - subreq->error = ret; - trace_netfs_sreq(subreq, netfs_sreq_trace_cancel); - /* Not queued - release both refs. */ - netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel); - netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel); + netfs_cancel_read(subreq, ret); break; } - size -= slice; start += slice; + size -= slice; + if (size <= 0) { + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + } - netfs_queue_read(rreq, subreq, size <= 0); netfs_issue_read(rreq, subreq); + + if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags)) + netfs_wait_for_paused_read(rreq); + if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) + break; cond_resched(); } while (size > 0); @@ -397,6 +395,7 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) { struct netfs_io_request *rreq; struct address_space *mapping = folio->mapping; + struct netfs_group *group = netfs_folio_group(folio); struct netfs_folio *finfo = netfs_folio_info(folio); struct netfs_inode *ctx = netfs_inode(mapping->host); struct folio *sink = NULL; @@ -458,14 +457,20 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) netfs_read_to_pagecache(rreq, NULL); - if (sink) - folio_put(sink); - ret = netfs_wait_for_read(rreq); if (ret >= 0) { + if (group) + folio_change_private(folio, group); + else + folio_detach_private(folio); + kfree(finfo); + trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); flush_dcache_folio(folio); folio_mark_uptodate(folio); } + + if (sink) + folio_put(sink); folio_unlock(folio); netfs_put_request(rreq, netfs_rreq_trace_put_return); return ret < 0 ? ret : 0; @@ -498,10 +503,10 @@ int netfs_read_folio(struct file *file, struct folio *folio) struct netfs_inode *ctx = netfs_inode(mapping->host); int ret; - if (folio_test_dirty(folio)) { - trace_netfs_folio(folio, netfs_folio_trace_read_gaps); + folio_wait_writeback(folio); + + if (folio_test_dirty(folio)) return netfs_read_gaps(file, folio); - } _enter("%lx", folio->index); @@ -654,7 +659,7 @@ retry: * within the cache granule containing the EOF, in which case we need * to preload the granule. */ - if (!netfs_is_cache_enabled(ctx) && + if (!netfs_is_cache_maybe_enabled(ctx) && netfs_skip_folio_read(folio, pos, len, false)) { netfs_stat(&netfs_n_rh_write_zskip); goto have_folio_no_wait; @@ -667,7 +672,7 @@ retry: ret = PTR_ERR(rreq); goto error; } - rreq->no_unlock_folio = folio->index; + rreq->no_unlock_folio = folio; __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags); ret = netfs_begin_cache_read(rreq, ctx); @@ -684,9 +689,9 @@ retry: netfs_read_to_pagecache(rreq, NULL); ret = netfs_wait_for_read(rreq); + netfs_put_request(rreq, netfs_rreq_trace_put_return); if (ret < 0) goto error; - netfs_put_request(rreq, netfs_rreq_trace_put_return); have_folio: ret = folio_wait_private_2_killable(folio); @@ -733,7 +738,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, goto error; } - rreq->no_unlock_folio = folio->index; + rreq->no_unlock_folio = folio; __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags); ret = netfs_begin_cache_read(rreq, ctx); if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c index 05ea5b0cc0e8..2cdb68e6b16f 100644 --- a/fs/netfs/buffered_write.c +++ b/fs/netfs/buffered_write.c @@ -12,24 +12,6 @@ #include <linux/slab.h> #include "internal.h" -static void __netfs_set_group(struct folio *folio, struct netfs_group *netfs_group) -{ - if (netfs_group) - folio_attach_private(folio, netfs_get_group(netfs_group)); -} - -static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group) -{ - void *priv = folio_get_private(folio); - - if (unlikely(priv != netfs_group)) { - if (netfs_group && (!priv || priv == NETFS_FOLIO_COPY_TO_CACHE)) - folio_attach_private(folio, netfs_get_group(netfs_group)); - else if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE) - folio_detach_private(folio); - } -} - /* * Grab a folio for writing and lock it. Attempt to allocate as large a folio * as possible to hold as much of the remaining length as possible in one go. @@ -149,6 +131,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, } do { + enum netfs_folio_trace trace; struct netfs_folio *finfo; struct netfs_group *group; unsigned long long fpos; @@ -156,6 +139,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, size_t offset; /* Offset into pagecache folio */ size_t part; /* Bytes to write to folio */ size_t copied; /* Bytes copied from user */ + void *priv; offset = pos & (max_chunk - 1); part = min(max_chunk - offset, iov_iter_count(iter)); @@ -201,73 +185,99 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, goto error_folio_unlock; } - /* Decide how we should modify a folio. We might be attempting - * to do write-streaming, in which case we don't want to a - * local RMW cycle if we can avoid it. If we're doing local - * caching or content crypto, we award that priority over - * avoiding RMW. If the file is open readably, then we also - * assume that we may want to read what we wrote. - */ finfo = netfs_folio_info(folio); group = netfs_folio_group(folio); + /* If the requested group differs from the group set on the + * page, then we need to flush out the folio if it has a group + * set (ie. is non-NULL). Note that COPY_TO_CACHE is a special + * case, being a netfs annotation rather than an actual group. + * + * The filesystem isn't permitted to mix writes with groups and + * writes without groups as the NULL group is used to indicate + * that no group is set. + */ if (unlikely(group != netfs_group) && - group != NETFS_FOLIO_COPY_TO_CACHE) + group != NETFS_FOLIO_COPY_TO_CACHE && + group) { + WARN_ON_ONCE(!netfs_group); goto flush_content; + } + /* Decide how we should modify a folio. We might be attempting + * to do write-streaming, as we don't want to a local RMW cycle + * if we can avoid it. If we're doing local caching or content + * crypto, we award that priority over avoiding RMW. If the + * file is open readably, then we let ->read_folio() fill in + * the gaps. + */ if (folio_test_uptodate(folio)) { if (mapping_writably_mapped(mapping)) flush_dcache_folio(folio); copied = copy_folio_from_iter_atomic(folio, offset, part, iter); if (unlikely(copied == 0)) goto copy_failed; - netfs_set_group(folio, netfs_group); - trace_netfs_folio(folio, netfs_folio_is_uptodate); - goto copied; + trace = netfs_folio_is_uptodate; + goto copied_uptodate; } /* If the page is above the zero-point then we assume that the * server would just return a block of zeros or a short read if * we try to read it. */ - if (fpos >= ctx->zero_point) { + if (fpos >= netfs_read_zero_point(inode)) { folio_zero_segment(folio, 0, offset); copied = copy_folio_from_iter_atomic(folio, offset, part, iter); if (unlikely(copied == 0)) goto copy_failed; folio_zero_segment(folio, offset + copied, flen); - __netfs_set_group(folio, netfs_group); - folio_mark_uptodate(folio); - trace_netfs_folio(folio, netfs_modify_and_clear); - goto copied; + if (finfo) + trace = netfs_modify_and_clear_rm_finfo; + else + trace = netfs_modify_and_clear; + goto mark_uptodate; } /* See if we can write a whole folio in one go. */ if (!maybe_trouble && offset == 0 && part >= flen) { copied = copy_folio_from_iter_atomic(folio, offset, part, iter); - if (unlikely(copied == 0)) + if (likely(copied == part)) { + if (finfo) + trace = netfs_whole_folio_modify_filled; + else + trace = netfs_whole_folio_modify; + goto mark_uptodate; + } + if (copied == 0) goto copy_failed; - if (unlikely(copied < part)) { + if (!finfo || copied <= finfo->dirty_offset) { maybe_trouble = true; iov_iter_revert(iter, copied); copied = 0; folio_unlock(folio); goto retry; } - __netfs_set_group(folio, netfs_group); - folio_mark_uptodate(folio); - trace_netfs_folio(folio, netfs_whole_folio_modify); + + /* We overwrote some existing dirty data, so we have to + * accept the partial write. + */ + finfo->dirty_len += finfo->dirty_offset; + if (finfo->dirty_len == flen) { + trace = netfs_whole_folio_modify_filled_efault; + goto mark_uptodate; + } + if (copied > finfo->dirty_len) + finfo->dirty_len = copied; + finfo->dirty_offset = 0; + trace = netfs_whole_folio_modify_efault; goto copied; } /* We don't want to do a streaming write on a file that loses * caching service temporarily because the backing store got - * culled and we don't really want to get a streaming write on - * a file that's open for reading as ->read_folio() then has to - * be able to flush it. + * culled. */ - if ((file->f_mode & FMODE_READ) || - netfs_is_cache_enabled(ctx)) { + if (netfs_is_cache_maybe_enabled(ctx)) { if (finfo) { netfs_stat(&netfs_n_wh_wstream_conflict); goto flush_content; @@ -282,11 +292,11 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, copied = copy_folio_from_iter_atomic(folio, offset, part, iter); if (unlikely(copied == 0)) goto copy_failed; - netfs_set_group(folio, netfs_group); - trace_netfs_folio(folio, netfs_just_prefetch); - goto copied; + trace = netfs_just_prefetch; + goto copied_uptodate; } + /* Do a streaming write on a folio that has nothing in it yet. */ if (!finfo) { ret = -EIO; if (WARN_ON(folio_get_private(folio))) @@ -295,10 +305,8 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, if (unlikely(copied == 0)) goto copy_failed; if (offset == 0 && copied == flen) { - __netfs_set_group(folio, netfs_group); - folio_mark_uptodate(folio); - trace_netfs_folio(folio, netfs_streaming_filled_page); - goto copied; + trace = netfs_streaming_filled_page; + goto mark_uptodate; } finfo = kzalloc_obj(*finfo); @@ -312,7 +320,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, finfo->dirty_len = copied; folio_attach_private(folio, (void *)((unsigned long)finfo | NETFS_FOLIO_INFO)); - trace_netfs_folio(folio, netfs_streaming_write); + trace = netfs_streaming_write; goto copied; } @@ -326,16 +334,10 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, goto copy_failed; finfo->dirty_len += copied; if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) { - if (finfo->netfs_group) - folio_change_private(folio, finfo->netfs_group); - else - folio_detach_private(folio); - folio_mark_uptodate(folio); - kfree(finfo); - trace_netfs_folio(folio, netfs_streaming_cont_filled_page); - } else { - trace_netfs_folio(folio, netfs_streaming_write_cont); + trace = netfs_streaming_cont_filled_page; + goto mark_uptodate; } + trace = netfs_streaming_write_cont; goto copied; } @@ -349,7 +351,38 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, goto out; continue; + /* Mark a folio as being up to data when we've filled it + * completely. If the folio has a group attached, then it must + * be the same group, otherwise we should have flushed it out + * above. We have to get rid of the netfs_folio struct if + * there was one. + */ + mark_uptodate: + folio_mark_uptodate(folio); + + copied_uptodate: + priv = folio_get_private(folio); + if (likely(priv == netfs_group)) { + /* Already set correctly; no change required. */ + } else if (priv == NETFS_FOLIO_COPY_TO_CACHE) { + if (!netfs_group) + folio_detach_private(folio); + else + folio_change_private(folio, netfs_get_group(netfs_group)); + } else if (!priv) { + folio_attach_private(folio, netfs_get_group(netfs_group)); + } else { + WARN_ON_ONCE(!finfo); + if (netfs_group) + /* finfo->netfs_group has a ref */ + folio_change_private(folio, netfs_group); + else + folio_detach_private(folio); + kfree(finfo); + } + copied: + trace_netfs_folio(folio, trace); flush_dcache_folio(folio); /* Update the inode size if we moved the EOF marker */ @@ -510,6 +543,7 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr struct inode *inode = file_inode(file); struct netfs_inode *ictx = netfs_inode(inode); vm_fault_t ret = VM_FAULT_NOPAGE; + void *priv; int err; _enter("%lx", folio->index); @@ -530,7 +564,9 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr } group = netfs_folio_group(folio); - if (group != netfs_group && group != NETFS_FOLIO_COPY_TO_CACHE) { + if (group && + group != netfs_group && + group != NETFS_FOLIO_COPY_TO_CACHE) { folio_unlock(folio); err = filemap_fdatawrite_range(mapping, folio_pos(folio), @@ -552,7 +588,19 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus); else trace_netfs_folio(folio, netfs_folio_trace_mkwrite); - netfs_set_group(folio, netfs_group); + + priv = folio_get_private(folio); + if (priv != netfs_group) { + if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE) + folio_detach_private(folio); + else if (netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE) + folio_change_private(folio, netfs_get_group(netfs_group)); + else if (netfs_group && !priv) + folio_attach_private(folio, netfs_get_group(netfs_group)); + else + WARN_ON_ONCE(1); + } + file_update_time(file); set_bit(NETFS_ICTX_MODIFIED_ATTR, &ictx->flags); if (ictx->ops->post_modify) diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c index f72e6da88cca..6a8fb0d55e04 100644 --- a/fs/netfs/direct_read.c +++ b/fs/netfs/direct_read.c @@ -45,12 +45,11 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) * Perform a read to a buffer from the server, slicing up the region to be read * according to the network rsize. */ -static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) +static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) { - struct netfs_io_stream *stream = &rreq->io_streams[0]; unsigned long long start = rreq->start; ssize_t size = rreq->len; - int ret = 0; + int ret; do { struct netfs_io_subrequest *subreq; @@ -58,7 +57,10 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) subreq = netfs_alloc_subrequest(rreq); if (!subreq) { - ret = -ENOMEM; + /* Stash the error in the request if there's not + * already an error set. + */ + cmpxchg(&rreq->error, 0, -ENOMEM); break; } @@ -66,25 +68,13 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) subreq->start = start; subreq->len = size; - __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); - - spin_lock(&rreq->lock); - list_add_tail(&subreq->rreq_link, &stream->subrequests); - if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { - if (!stream->active) { - stream->collected_to = subreq->start; - /* Store list pointers before active flag */ - smp_store_release(&stream->active, true); - } - } - trace_netfs_sreq(subreq, netfs_sreq_trace_added); - spin_unlock(&rreq->lock); + netfs_queue_read(rreq, subreq); netfs_stat(&netfs_n_rh_download); if (rreq->netfs_ops->prepare_read) { ret = rreq->netfs_ops->prepare_read(subreq); if (ret < 0) { - netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel); + netfs_cancel_read(subreq, ret); break; } } @@ -113,8 +103,6 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); netfs_wake_collector(rreq); } - - return ret; } /* @@ -137,21 +125,17 @@ static ssize_t netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync) // TODO: Use bounce buffer if requested inode_dio_begin(rreq->inode); + netfs_dispatch_unbuffered_reads(rreq); - ret = netfs_dispatch_unbuffered_reads(rreq); - - if (!rreq->submitted) { - netfs_put_request(rreq, netfs_rreq_trace_put_no_submit); - inode_dio_end(rreq->inode); - ret = 0; - goto out; - } + /* The collector will get run, even if we don't manage to submit any + * subreqs, so we shouldn't call inode_dio_end() here. + */ if (sync) ret = netfs_wait_for_read(rreq); else ret = -EIOCBQUEUED; -out: + _leave(" = %zd", ret); return ret; } diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index f9ab69de3e29..c16fbad286a1 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -166,13 +166,16 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) */ subreq->error = -EAGAIN; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); - if (subreq->transferred > 0) + if (subreq->transferred > 0) { iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + wreq->transferred += subreq->transferred; + } if (stream->source == NETFS_UPLOAD_TO_SERVER && wreq->netfs_ops->retry_request) wreq->netfs_ops->retry_request(wreq, stream); + __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); @@ -186,17 +189,10 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); - if (stream->prepare_write) { + if (stream->prepare_write) stream->prepare_write(subreq); - __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); - netfs_stat(&netfs_n_wh_retry_write_subreq); - } else { - struct iov_iter source; - - netfs_reset_iter(subreq); - source = subreq->io_iter; - netfs_reissue_write(stream, subreq, &source); - } + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + netfs_stat(&netfs_n_wh_retry_write_subreq); } netfs_unbuffered_write_done(wreq); @@ -376,8 +372,10 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from) if (ret < 0) goto out; end = iocb->ki_pos + iov_iter_count(from); - if (end > ictx->zero_point) - ictx->zero_point = end; + spin_lock(&inode->i_lock); + if (end > ictx->_zero_point) + netfs_write_zero_point(inode, end); + spin_unlock(&inode->i_lock); fscache_invalidate(netfs_i_cookie(ictx), NULL, i_size_read(inode), FSCACHE_INVAL_DIO_WRITE); diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index d436e20d3418..d889caa401dc 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -23,6 +23,8 @@ /* * buffered_read.c */ +void netfs_queue_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq); void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error); int netfs_prefetch_for_write(struct file *file, struct folio *folio, size_t offset, size_t len); @@ -108,6 +110,7 @@ static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq, */ bool netfs_read_collection(struct netfs_io_request *rreq); void netfs_read_collection_worker(struct work_struct *work); +void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error); void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error); /* @@ -236,6 +239,18 @@ static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx) #endif } +static inline bool netfs_is_cache_maybe_enabled(struct netfs_inode *ctx) +{ +#if IS_ENABLED(CONFIG_FSCACHE) + struct fscache_cookie *cookie = ctx->cache; + + return fscache_cookie_valid(cookie) && + test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); +#else + return false; +#endif +} + /* * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap). */ diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c index 154a14bb2d7f..b375567e0520 100644 --- a/fs/netfs/iterator.c +++ b/fs/netfs/iterator.c @@ -22,7 +22,7 @@ * * Extract the page fragments from the given amount of the source iterator and * build up a second iterator that refers to all of those bits. This allows - * the original iterator to disposed of. + * the original iterator to be disposed of. * * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be * allowed on the pages extracted. @@ -43,7 +43,7 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, unsigned int max_pages; unsigned int npages = 0; unsigned int i; - ssize_t ret; + ssize_t ret = 0; size_t count = orig_len, offset, len; size_t bv_size, pg_size; @@ -67,26 +67,29 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, ret = iov_iter_extract_pages(orig, &pages, count, max_pages - npages, extraction_flags, &offset); - if (ret < 0) { - pr_err("Couldn't get user pages (rc=%zd)\n", ret); + if (unlikely(ret <= 0)) { + ret = ret ?: -EIO; break; } - if (ret > count) { - pr_err("get_pages rc=%zd more than %zu\n", ret, count); + if (WARN(ret > count, + "%s: extract_pages overrun %zd > %zu bytes\n", + __func__, ret, count)) { + ret = -EIO; break; } - count -= ret; - ret += offset; - cur_npages = DIV_ROUND_UP(ret, PAGE_SIZE); - - if (npages + cur_npages > max_pages) { - pr_err("Out of bvec array capacity (%u vs %u)\n", - npages + cur_npages, max_pages); + cur_npages = DIV_ROUND_UP(offset + ret, PAGE_SIZE); + if (WARN(cur_npages > max_pages - npages, + "%s: extract_pages overrun %u > %u pages\n", + __func__, npages + cur_npages, max_pages)) { + ret = -EIO; break; } + count -= ret; + ret += offset; + for (i = 0; i < cur_npages; i++) { len = ret > PAGE_SIZE ? PAGE_SIZE : ret; bvec_set_page(bv + npages + i, *pages++, len - offset, offset); @@ -97,6 +100,18 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, npages += cur_npages; } + /* Note: Don't try to clean up after EIO. Either we got no pages, so + * nothing to clean up, or we got a buffer overrun, memory corruption + * and can't trust the stuff in the buffer (a WARN was emitted). + */ + + if (ret < 0 && (ret == -ENOMEM || npages == 0)) { + for (i = 0; i < npages; i++) + unpin_user_page(bv[i].bv_page); + kvfree(bv); + return ret; + } + iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count); return npages; } diff --git a/fs/netfs/locking.c b/fs/netfs/locking.c index 2249ecd09d0a..4e3be2b81504 100644 --- a/fs/netfs/locking.c +++ b/fs/netfs/locking.c @@ -9,6 +9,11 @@ #include <linux/netfs.h> #include "internal.h" +struct netfs_wb_waiter { + struct list_head link; /* Link in ictx->wb_queue */ + struct task_struct *waiter; /* Waiter task; cleared when lock granted */ +}; + /* * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish * @inode: inode to wait for @@ -203,3 +208,93 @@ void netfs_end_io_direct(struct inode *inode) up_read(&inode->i_rwsem); } EXPORT_SYMBOL(netfs_end_io_direct); + +/* + * Wait to have exclusive access to writeback. + */ +static bool netfs_wb_begin_wait(struct netfs_inode *ictx) +{ + struct netfs_wb_waiter waiter = {}; + struct task_struct *tsk = current; + bool got = false; + + spin_lock(&ictx->lock); + + if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) { + get_task_struct(tsk); + waiter.waiter = tsk; + list_add_tail(&waiter.link, &ictx->wb_queue); + } else { + got = true; + } + spin_unlock(&ictx->lock); + + if (!got) { + for (;;) { + set_current_state(TASK_UNINTERRUPTIBLE); + /* Read waiter before accessing inode state. */ + if (smp_load_acquire(&waiter.waiter) == NULL) + break; + schedule(); + } + } + __set_current_state(TASK_RUNNING); + return true; +} + +/** + * netfs_wb_begin - Begin writeback, waiting if need be + * @ictx: The inode to get writeback access on + * @nowait: Return failure immediately rather than waiting if true + * + * Begin writeback to an inode, waiting for exclusive access if @nowait is + * false. This prevents collection from being done out of order with respect + * to the issuance of write subrequests. + * + * Note that writeback may be ended in a different process (e.g. the collection + * function on a workqueue) than started it. + * + * Return: True if can proceed, false if denied. + */ +bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait) +{ + if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) + return true; + if (nowait) { + netfs_stat(&netfs_n_wb_lock_skip); + return false; + } + netfs_stat(&netfs_n_wb_lock_wait); + return netfs_wb_begin_wait(ictx); +} +EXPORT_SYMBOL(netfs_wb_begin); + +/* netfs_wb_end - End writeback + * @ictx: The inode we have writeback access to + * + * End writeback access on an inode, waking up the next writeback request. + */ +void netfs_wb_end(struct netfs_inode *ictx) +{ + struct netfs_wb_waiter *waiter; + struct task_struct *tsk; + + WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags)); + + spin_lock(&ictx->lock); + + waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link); + if (waiter) { + list_del(&waiter->link); + tsk = waiter->waiter; + /* Write inode state before clearing waiter. */ + smp_store_release(&waiter->waiter, NULL); + wake_up_process(tsk); + put_task_struct(tsk); + } else { + clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags); + } + + spin_unlock(&ictx->lock); +} +EXPORT_SYMBOL(netfs_wb_end); diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index 6df89c92b10b..5d554512ed23 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -211,18 +211,25 @@ EXPORT_SYMBOL(netfs_clear_inode_writeback); void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length) { struct netfs_folio *finfo; - struct netfs_inode *ctx = netfs_inode(folio_inode(folio)); + struct inode *inode = folio_inode(folio); + struct netfs_inode *ctx = netfs_inode(inode); size_t flen = folio_size(folio); _enter("{%lx},%zx,%zx", folio->index, offset, length); if (offset == 0 && length == flen) { - unsigned long long i_size = i_size_read(&ctx->inode); + unsigned long long i_size, remote_i_size, zero_point; unsigned long long fpos = folio_pos(folio), end; + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); end = umin(fpos + flen, i_size); - if (fpos < i_size && end > ctx->zero_point) - ctx->zero_point = end; + if (fpos < i_size && end > zero_point) { + spin_lock(&inode->i_lock); + end = umin(fpos + flen, inode->i_size); + if (fpos < i_size && end > ctx->_zero_point) + netfs_write_zero_point(inode, end); + spin_unlock(&inode->i_lock); + } } folio_wait_private_2(folio); /* [DEPRECATED] */ @@ -255,7 +262,8 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length) goto erase_completely; /* Move the start of the data. */ finfo->dirty_len = fend - iend; - finfo->dirty_offset = offset; + finfo->dirty_offset = iend; + trace_netfs_folio(folio, netfs_folio_trace_invalidate_front); return; } @@ -264,12 +272,14 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length) */ if (iend >= fend) { finfo->dirty_len = offset - fstart; + trace_netfs_folio(folio, netfs_folio_trace_invalidate_tail); return; } /* A partial write was split. The caller has already zeroed * it, so just absorb the hole. */ + trace_netfs_folio(folio, netfs_folio_trace_invalidate_middle); } return; @@ -277,8 +287,9 @@ erase_completely: netfs_put_group(netfs_folio_group(folio)); folio_detach_private(folio); folio_clear_uptodate(folio); + folio_cancel_dirty(folio); kfree(finfo); - return; + trace_netfs_folio(folio, netfs_folio_trace_invalidate_all); } EXPORT_SYMBOL(netfs_invalidate_folio); @@ -292,15 +303,22 @@ EXPORT_SYMBOL(netfs_invalidate_folio); */ bool netfs_release_folio(struct folio *folio, gfp_t gfp) { - struct netfs_inode *ctx = netfs_inode(folio_inode(folio)); - unsigned long long end; + struct inode *inode = folio_inode(folio); + struct netfs_inode *ctx = netfs_inode(inode); + unsigned long long i_size, remote_i_size, zero_point, end; if (folio_test_dirty(folio)) return false; - end = umin(folio_next_pos(folio), i_size_read(&ctx->inode)); - if (end > ctx->zero_point) - ctx->zero_point = end; + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); + end = folio_next_pos(folio); + if (end > zero_point) { + spin_lock(&inode->i_lock); + end = umin(end, ctx->_remote_i_size); + if (end > ctx->_zero_point) + netfs_write_zero_point(inode, end); + spin_unlock(&inode->i_lock); + } if (folio_test_private(folio)) return false; @@ -356,6 +374,7 @@ void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq, DEFINE_WAIT(myself); list_for_each_entry(subreq, &stream->subrequests, rreq_link) { + smp_rmb(); /* Read ->next before IN_PROGRESS. */ if (!netfs_check_subreq_in_progress(subreq)) continue; diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index e5f6665b3341..23660a590124 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -83,7 +83,7 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq, } just_unlock: - if (folio->index == rreq->no_unlock_folio && + if (folio == rreq->no_unlock_folio && test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) { _debug("no unlock"); } else { @@ -205,8 +205,10 @@ reassess: * in progress. The issuer thread may be adding stuff to the tail * whilst we're doing this. */ - front = list_first_entry_or_null(&stream->subrequests, - struct netfs_io_subrequest, rreq_link); + front = list_first_entry_or_null_acquire(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + /* Read first subreq pointer before IN_PROGRESS flag. */ + while (front) { size_t transferred; @@ -576,6 +578,17 @@ skip_error_checks: EXPORT_SYMBOL(netfs_read_subreq_terminated); /* + * Cancel a read subrequest due to preparation failure. + */ +void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error) +{ + trace_netfs_sreq(subreq, netfs_sreq_trace_cancel); + subreq->error = error; + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + netfs_read_subreq_terminated(subreq); +} + +/* * Handle termination of a read from the cache. */ void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error) diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index cca9ac43c077..2b42758e01ec 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -98,7 +98,12 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) goto abandon; } - list_for_each_continue(next, &stream->subrequests) { + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); if (subreq->start + subreq->transferred != start + len || test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || @@ -175,7 +180,9 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) list_for_each_entry_safe_from(subreq, tmp, &stream->subrequests, rreq_link) { trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); + spin_lock(&rreq->lock); list_del(&subreq->rreq_link); + spin_unlock(&rreq->lock); netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); if (subreq == to) break; @@ -203,8 +210,10 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) refcount_read(&subreq->ref), netfs_sreq_trace_new); + spin_lock(&rreq->lock); list_add(&subreq->rreq_link, &to->rreq_link); - to = list_next_entry(to, rreq_link); + spin_unlock(&rreq->lock); + to = subreq; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); stream->sreq_max_len = umin(len, rreq->rsize); @@ -288,8 +297,15 @@ void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) struct folio *folio = folioq_folio(p, slot); if (folio && !folioq_is_marked2(p, slot)) { - trace_netfs_folio(folio, netfs_folio_trace_abandon); - folio_unlock(folio); + if (folio == rreq->no_unlock_folio && + test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, + &rreq->flags)) { + _debug("no unlock"); + } else { + trace_netfs_folio(folio, + netfs_folio_trace_abandon); + folio_unlock(folio); + } } } } diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c index d0e23bc42445..8833550d2eb6 100644 --- a/fs/netfs/read_single.c +++ b/fs/netfs/read_single.c @@ -89,7 +89,6 @@ static void netfs_single_read_cache(struct netfs_io_request *rreq, */ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) { - struct netfs_io_stream *stream = &rreq->io_streams[0]; struct netfs_io_subrequest *subreq; int ret = 0; @@ -102,14 +101,7 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) subreq->len = rreq->len; subreq->io_iter = rreq->buffer.iter; - __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); - - spin_lock(&rreq->lock); - list_add_tail(&subreq->rreq_link, &stream->subrequests); - trace_netfs_sreq(subreq, netfs_sreq_trace_added); - /* Store list pointers before active flag */ - smp_store_release(&stream->active, true); - spin_unlock(&rreq->lock); + netfs_queue_read(rreq, subreq); netfs_single_cache_prepare_read(rreq, subreq); switch (subreq->source) { @@ -121,10 +113,14 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) goto cancel; } + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); rreq->netfs_ops->issue_read(subreq); rreq->submitted += subreq->len; break; case NETFS_READ_FROM_CACHE: + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); trace_netfs_sreq(subreq, netfs_sreq_trace_submit); netfs_single_read_cache(rreq, subreq); rreq->submitted += subreq->len; @@ -134,14 +130,15 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) pr_warn("Unexpected single-read source %u\n", subreq->source); WARN_ON_ONCE(true); ret = -EIO; - break; + goto cancel; } - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); return ret; cancel: - netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel); + netfs_cancel_read(subreq, ret); + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_wake_collector(rreq); return ret; } diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c index b194447f4b11..210eb8f3958d 100644 --- a/fs/netfs/write_collect.c +++ b/fs/netfs/write_collect.c @@ -57,7 +57,8 @@ static void netfs_dump_request(const struct netfs_io_request *rreq) int netfs_folio_written_back(struct folio *folio) { enum netfs_folio_trace why = netfs_folio_trace_clear; - struct netfs_inode *ictx = netfs_inode(folio->mapping->host); + struct inode *inode = folio_inode(folio); + struct netfs_inode *ictx = netfs_inode(inode); struct netfs_folio *finfo; struct netfs_group *group = NULL; int gcount = 0; @@ -69,8 +70,10 @@ int netfs_folio_written_back(struct folio *folio) unsigned long long fend; fend = folio_pos(folio) + finfo->dirty_offset + finfo->dirty_len; - if (fend > ictx->zero_point) - ictx->zero_point = fend; + spin_lock(&ictx->inode.i_lock); + if (fend > ictx->_zero_point) + netfs_write_zero_point(inode, fend); + spin_unlock(&ictx->inode.i_lock); folio_detach_private(folio); group = finfo->netfs_group; @@ -228,8 +231,10 @@ reassess_streams: if (!smp_load_acquire(&stream->active)) continue; - front = list_first_entry_or_null(&stream->subrequests, - struct netfs_io_subrequest, rreq_link); + front = list_first_entry_or_null_acquire(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + /* Read first subreq pointer before IN_PROGRESS flag. */ + while (front) { trace_netfs_collect_sreq(wreq, front); //_debug("sreq [%x] %llx %zx/%zx", @@ -403,6 +408,16 @@ bool netfs_write_collection(struct netfs_io_request *wreq) netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ + switch (wreq->origin) { + case NETFS_WRITEBACK: + case NETFS_WRITEBACK_SINGLE: + case NETFS_WRITETHROUGH: + netfs_wb_end(ictx); + break; + default: + break; + } + if (wreq->iocb) { size_t written = min(wreq->transferred, wreq->len); wreq->iocb->ki_pos += written; diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index 2db688f94125..f2761c99795a 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -106,7 +106,7 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, _enter("R=%x", wreq->debug_id); ictx = netfs_inode(wreq->inode); - if (is_cacheable && netfs_is_cache_enabled(ictx)) + if (is_cacheable) fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx)); if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE) < 0) goto nomem; @@ -204,7 +204,8 @@ void netfs_prepare_write(struct netfs_io_request *wreq, * remove entries off of the front. */ spin_lock(&wreq->lock); - list_add_tail(&subreq->rreq_link, &stream->subrequests); + /* Write IN_PROGRESS before pointer to new subreq */ + list_add_tail_release(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { if (!stream->active) { stream->collected_to = subreq->start; @@ -413,12 +414,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq, if (streamw) netfs_issue_write(wreq, cache); - /* Flip the page to the writeback state and unlock. If we're called - * from write-through, then the page has already been put into the wb - * state. - */ - if (wreq->origin == NETFS_WRITEBACK) - folio_start_writeback(folio); + folio_start_writeback(folio); folio_unlock(folio); if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) { @@ -555,14 +551,8 @@ int netfs_writepages(struct address_space *mapping, struct folio *folio; int error = 0; - if (!mutex_trylock(&ictx->wb_lock)) { - if (wbc->sync_mode == WB_SYNC_NONE) { - netfs_stat(&netfs_n_wb_lock_skip); - return 0; - } - netfs_stat(&netfs_n_wb_lock_wait); - mutex_lock(&ictx->wb_lock); - } + if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) + return 0; /* Need the first folio to be able to set up the op. */ folio = writeback_iter(mapping, wbc, NULL, &error); @@ -592,13 +582,13 @@ int netfs_writepages(struct address_space *mapping, } error = netfs_write_folio(wreq, wbc, folio); - if (error < 0) - break; + if (error == -ENOMEM) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + } } while ((folio = writeback_iter(mapping, wbc, folio, &error))); netfs_end_issue_write(wreq); - - mutex_unlock(&ictx->wb_lock); netfs_wake_collector(wreq); netfs_put_request(wreq, netfs_rreq_trace_put_return); @@ -606,9 +596,16 @@ int netfs_writepages(struct address_space *mapping, return error; couldnt_start: - netfs_kill_dirty_pages(mapping, wbc, folio); + if (error == -ENOMEM) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + folio = writeback_iter(mapping, wbc, folio, &error); + WARN_ON_ONCE(folio != NULL); + } else { + netfs_kill_dirty_pages(mapping, wbc, folio); + } out: - mutex_unlock(&ictx->wb_lock); + netfs_wb_end(ictx); _leave(" = %d", error); return error; } @@ -622,16 +619,17 @@ struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len struct netfs_io_request *wreq = NULL; struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp)); - mutex_lock(&ictx->wb_lock); + netfs_wb_begin(ictx, false); wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp, iocb->ki_pos, NETFS_WRITETHROUGH); if (IS_ERR(wreq)) { - mutex_unlock(&ictx->wb_lock); + netfs_wb_end(ictx); return wreq; } wreq->io_streams[0].avail = true; + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); trace_netfs_write(wreq, netfs_write_trace_writethrough); return wreq; } @@ -646,29 +644,41 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c struct folio *folio, size_t copied, bool to_page_end, struct folio **writethrough_cache) { + int ret; + _enter("R=%x ic=%zu ws=%u cp=%zu tp=%u", wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end); - if (!*writethrough_cache) { - if (folio_test_dirty(folio)) - /* Sigh. mmap. */ - folio_clear_dirty_for_io(folio); + /* The folio is locked. */ + if (*writethrough_cache != folio) { + if (*writethrough_cache) { + /* Did the folio get moved? */ + folio_put(*writethrough_cache); + *writethrough_cache = NULL; + } /* We can make multiple writes to the folio... */ - folio_start_writeback(folio); if (wreq->len == 0) trace_netfs_folio(folio, netfs_folio_trace_wthru); else trace_netfs_folio(folio, netfs_folio_trace_wthru_plus); *writethrough_cache = folio; + folio_get(folio); } wreq->len += copied; - if (!to_page_end) + + if (!to_page_end) { + folio_mark_dirty(folio); + folio_unlock(folio); return 0; + } + ret = netfs_write_folio(wreq, wbc, folio); + folio_put(*writethrough_cache); *writethrough_cache = NULL; - return netfs_write_folio(wreq, wbc, folio); + wreq->submitted = wreq->len; + return ret; } /* @@ -677,18 +687,19 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, struct folio *writethrough_cache) { - struct netfs_inode *ictx = netfs_inode(wreq->inode); ssize_t ret; _enter("R=%x", wreq->debug_id); - if (writethrough_cache) + if (writethrough_cache) { + folio_lock(writethrough_cache); netfs_write_folio(wreq, wbc, writethrough_cache); + folio_put(writethrough_cache); + wreq->submitted = wreq->len; + } netfs_end_issue_write(wreq); - mutex_unlock(&ictx->wb_lock); - if (wreq->iocb) ret = -EIOCBQUEUED; else @@ -818,6 +829,9 @@ static int netfs_write_folio_single(struct netfs_io_request *wreq, * * Write a monolithic, non-pagecache object back to the server and/or * the cache. + * + * Return: 0 if successful; 1 if skipped due to lock conflict and WB_SYNC_NONE; + * or a negative error code. */ int netfs_writeback_single(struct address_space *mapping, struct writeback_control *wbc, @@ -832,13 +846,10 @@ int netfs_writeback_single(struct address_space *mapping, if (WARN_ON_ONCE(!iov_iter_is_folioq(iter))) return -EIO; - if (!mutex_trylock(&ictx->wb_lock)) { - if (wbc->sync_mode == WB_SYNC_NONE) { - netfs_stat(&netfs_n_wb_lock_skip); - return 0; - } - netfs_stat(&netfs_n_wb_lock_wait); - mutex_lock(&ictx->wb_lock); + if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) { + /* The VFS will have undirtied the inode. */ + netfs_single_mark_inode_dirty(&ictx->inode); + return 1; } wreq = netfs_create_write_req(mapping, NULL, 0, NETFS_WRITEBACK_SINGLE); @@ -876,7 +887,6 @@ stop: smp_wmb(); /* Write lists before ALL_QUEUED. */ set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); - mutex_unlock(&ictx->wb_lock); netfs_wake_collector(wreq); netfs_put_request(wreq, netfs_rreq_trace_put_return); @@ -884,7 +894,7 @@ stop: return ret; couldnt_start: - mutex_unlock(&ictx->wb_lock); + netfs_wb_end(ictx); _leave(" = %d", ret); return ret; } diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index 29489a23a220..058bc7a166a5 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -72,7 +72,12 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) return; - list_for_each_continue(next, &stream->subrequests) { + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); if (subreq->start + subreq->transferred != start + len || test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || @@ -130,7 +135,9 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, list_for_each_entry_safe_from(subreq, tmp, &stream->subrequests, rreq_link) { trace_netfs_sreq(subreq, netfs_sreq_trace_discard); + spin_lock(&wreq->lock); list_del(&subreq->rreq_link); + spin_unlock(&wreq->lock); netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); if (subreq == to) break; @@ -153,8 +160,10 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, netfs_sreq_trace_new); trace_netfs_sreq(subreq, netfs_sreq_trace_split); + spin_lock(&wreq->lock); list_add(&subreq->rreq_link, &to->rreq_link); - to = list_next_entry(to, rreq_link); + spin_unlock(&wreq->lock); + to = subreq; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); stream->sreq_max_len = len; diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index 4ea9221ded42..10f2354ba304 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -257,6 +257,7 @@ static u32 initiate_file_draining(struct nfs_client *clp, struct pnfs_layout_hdr *lo; u32 rv = NFS4ERR_NOMATCHING_LAYOUT; LIST_HEAD(free_me_list); + bool return_range = false; ino = nfs_layout_find_inode(clp, &args->cbl_fh, &args->cbl_stateid); if (IS_ERR(ino)) { @@ -301,13 +302,13 @@ static u32 initiate_file_draining(struct nfs_client *clp, /* Embrace your forgetfulness! */ rv = NFS4ERR_NOMATCHING_LAYOUT; - if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) { - NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, - &args->cbl_range); - } + return_range = true; } unlock: spin_unlock(&ino->i_lock); + if (return_range && NFS_SERVER(ino)->pnfs_curr_ld->return_range) + NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, + &args->cbl_range); pnfs_free_lseg_list(&free_me_list); /* Free all lsegs that are attached to commit buckets */ nfs_commit_inode(ino, 0); diff --git a/fs/nfs/client.c b/fs/nfs/client.c index be02bb227741..4dcb91ab3039 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -914,6 +914,7 @@ static void nfs_server_set_fsinfo(struct nfs_server *server, */ static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, struct nfs_fattr *fattr) { + struct nfs_pathconf pathinfo = { }; struct nfs_fsinfo fsinfo; struct nfs_client *clp = server->nfs_client; int error; @@ -933,15 +934,28 @@ static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, str nfs_server_set_fsinfo(server, &fsinfo); - /* Get some general file system info */ - if (server->namelen == 0) { - struct nfs_pathconf pathinfo; - - pathinfo.fattr = fattr; - nfs_fattr_init(fattr); - - if (clp->rpc_ops->pathconf(server, mntfh, &pathinfo) >= 0) - server->namelen = pathinfo.max_namelen; + pathinfo.fattr = fattr; + nfs_fattr_init(fattr); + + if (clp->rpc_ops->version < 4 || server->namelen == 0) { + if (clp->rpc_ops->pathconf(server, mntfh, &pathinfo) >= 0) { + if (server->namelen == 0) + server->namelen = pathinfo.max_namelen; + if (clp->rpc_ops->version < 4) { + unsigned int caps = server->caps; + + caps &= ~(NFS_CAP_CASE_INSENSITIVE | + NFS_CAP_CASE_NONPRESERVING); + if (pathinfo.case_insensitive) + caps |= NFS_CAP_CASE_INSENSITIVE; + if (!pathinfo.case_preserving) + caps |= NFS_CAP_CASE_NONPRESERVING; + server->caps = caps; + } + } else if (clp->rpc_ops->version < 4) { + server->caps &= ~(NFS_CAP_CASE_INSENSITIVE | + NFS_CAP_CASE_NONPRESERVING); + } } if (clp->rpc_ops->discover_trunking != NULL && @@ -1049,10 +1063,8 @@ struct nfs_server *nfs_alloc_server(void) return NULL; server->s_sysfs_id = ida_alloc(&s_sysfs_ids, GFP_KERNEL); - if (server->s_sysfs_id < 0) { - kfree(server); - return NULL; - } + if (server->s_sysfs_id < 0) + goto free_server; server->client = server->client_acl = ERR_PTR(-EINVAL); @@ -1074,8 +1086,8 @@ struct nfs_server *nfs_alloc_server(void) server->io_stats = nfs_alloc_iostats(); if (!server->io_stats) { - kfree(server); - return NULL; + ida_free(&s_sysfs_ids, server->s_sysfs_id); + goto free_server; } server->change_attr_type = NFS4_CHANGE_TYPE_IS_UNDEFINED; @@ -1089,6 +1101,10 @@ struct nfs_server *nfs_alloc_server(void) rpc_init_wait_queue(&server->uoc_rpcwaitq, "NFS UOC"); return server; + +free_server: + kfree(server); + return NULL; } EXPORT_SYMBOL_GPL(nfs_alloc_server); diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 122fb3f14ffb..9546d2195c25 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -173,6 +173,7 @@ int nfs4_check_delegation(struct inode *inode, fmode_t type) static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_stateid *stateid) { struct inode *inode = state->inode; + struct nfs_inode *nfsi = NFS_I(inode); struct file_lock *fl; struct file_lock_context *flctx = locks_inode_context(inode); struct list_head *list; @@ -182,6 +183,9 @@ static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_state goto out; list = &flctx->flc_posix; + + /* Guard against reclaim and new lock/unlock calls */ + down_write(&nfsi->rwsem); spin_lock(&flctx->flc_lock); restart: for_each_file_lock(fl, list) { @@ -189,8 +193,10 @@ restart: continue; spin_unlock(&flctx->flc_lock); status = nfs4_lock_delegation_recall(fl, state, stateid); - if (status < 0) + if (status < 0) { + up_write(&nfsi->rwsem); goto out; + } spin_lock(&flctx->flc_lock); } if (list == &flctx->flc_posix) { @@ -198,6 +204,7 @@ restart: goto restart; } spin_unlock(&flctx->flc_lock); + up_write(&nfsi->rwsem); out: return status; } diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index e9ce1883288c..c7caffb31935 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -650,7 +650,7 @@ int nfs_same_file(struct dentry *dentry, struct nfs_entry *entry) return 0; nfsi = NFS_I(inode); - if (entry->fattr->fileid != nfsi->fileid) + if (entry->fattr->fileid != inode->i_ino) return 0; if (entry->fh->size && nfs_compare_fh(entry->fh, &nfsi->fh) != 0) return 0; @@ -726,7 +726,6 @@ void nfs_prime_dcache(struct dentry *parent, struct nfs_entry *entry, unsigned long dir_verifier) { struct qstr filename = QSTR_INIT(entry->name, entry->len); - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); struct dentry *dentry; struct dentry *alias; struct inode *inode; @@ -755,7 +754,7 @@ void nfs_prime_dcache(struct dentry *parent, struct nfs_entry *entry, dentry = d_lookup(parent, &filename); again: if (!dentry) { - dentry = d_alloc_parallel(parent, &filename, &wq); + dentry = d_alloc_parallel(parent, &filename); if (IS_ERR(dentry)) return; } @@ -1106,7 +1105,7 @@ static void nfs_do_filldir(struct nfs_readdir_descriptor *desc, ent = &array->array[i]; if (!dir_emit(desc->ctx, ent->name, ent->name_len, - nfs_compat_user_ino64(ent->ino), ent->d_type)) { + ent->ino, ent->d_type)) { desc->eob = true; break; } @@ -2106,7 +2105,6 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry, struct file *file, unsigned open_flags, umode_t mode) { - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); struct nfs_open_context *ctx; struct dentry *res; struct iattr attr = { .ia_valid = ATTR_OPEN }; @@ -2162,7 +2160,7 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry, d_drop(dentry); switched = true; dentry = d_alloc_parallel(dentry->d_parent, - &dentry->d_name, &wq); + &dentry->d_name); if (IS_ERR(dentry)) return PTR_ERR(dentry); if (unlikely(!d_in_lookup(dentry))) @@ -2194,6 +2192,10 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry, break; case -EISDIR: case -ENOTDIR: + if (open_flags & __O_REGULAR) { + err = -EFTYPE; + break; + } goto no_open; case -ELOOP: if (!(open_flags & O_NOFOLLOW)) @@ -2299,7 +2301,7 @@ full_reval: return nfs_do_lookup_revalidate(dir, name, dentry, flags); } -#endif /* CONFIG_NFSV4 */ +#endif /* CONFIG_NFS_V4 */ int nfs_atomic_open_v23(struct inode *dir, struct dentry *dentry, struct file *file, unsigned int open_flags, @@ -3344,6 +3346,8 @@ static int nfs_open_permission_mask(int openflags) mask |= MAY_READ; if ((openflags & O_ACCMODE) != O_RDONLY) mask |= MAY_WRITE; + if (openflags & O_TRUNC) + mask |= MAY_WRITE; } return mask; diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 48d89716193a..e626c72495e6 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -466,14 +466,22 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, goto out_release; } dreq->l_ctx = l_ctx; - if (!is_sync_kiocb(iocb)) + if (!is_sync_kiocb(iocb)) { dreq->iocb = iocb; + } else if (iocb->ki_flags & IOCB_NOWAIT) { + result = -EAGAIN; + nfs_direct_req_release(dreq); + goto out_release; + } if (user_backed_iter(iter)) dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; if (!swap) { - result = nfs_start_io_direct(inode); + if (iocb->ki_flags & IOCB_NOWAIT) + result = nfs_start_io_direct_nowait(inode); + else + result = nfs_start_io_direct(inode); if (result) { /* release the reference that would usually be * consumed by nfs_direct_read_schedule_iovec() diff --git a/fs/nfs/dns_resolve.c b/fs/nfs/dns_resolve.c index 2ed2126201f4..acd3511c04a6 100644 --- a/fs/nfs/dns_resolve.c +++ b/fs/nfs/dns_resolve.c @@ -156,7 +156,7 @@ static int nfs_dns_upcall(struct cache_detail *cd, if (!nfs_cache_upcall(cd, key->hostname)) return 0; clear_bit(CACHE_PENDING, &ch->flags); - return sunrpc_cache_pipe_upcall_timeout(cd, ch); + return sunrpc_cache_upcall_warn(cd, ch); } static int nfs_dns_match(struct cache_head *ca, diff --git a/fs/nfs/export.c b/fs/nfs/export.c index a10dd5f9d078..8fb08bce0623 100644 --- a/fs/nfs/export.c +++ b/fs/nfs/export.c @@ -49,14 +49,14 @@ nfs_encode_fh(struct inode *inode, __u32 *p, int *max_len, struct inode *parent) return FILEID_INVALID; } - p[FILEID_HIGH_OFF] = NFS_FILEID(inode) >> 32; - p[FILEID_LOW_OFF] = NFS_FILEID(inode); + p[FILEID_HIGH_OFF] = inode->i_ino >> 32; + p[FILEID_LOW_OFF] = inode->i_ino; p[FILE_I_TYPE_OFF] = inode->i_mode & S_IFMT; p[len - 1] = 0; /* Padding */ nfs_copy_fh(clnt_fh, server_fh); *max_len = len; dprintk("%s: result fh fileid %llu mode %u size %d\n", - __func__, NFS_FILEID(inode), inode->i_mode, *max_len); + __func__, inode->i_ino, inode->i_mode, *max_len); return *max_len; } diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 25048a3c2364..a0d8f1c1cf10 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -72,8 +72,12 @@ nfs_file_open(struct inode *inode, struct file *filp) return res; res = nfs_open(inode, filp); - if (res == 0) + if (res == 0) { filp->f_mode |= FMODE_CAN_ODIRECT; + /* flag NOWAIT on read-only files only */ + if (!(filp->f_mode & FMODE_WRITE)) + filp->f_mode |= FMODE_NOWAIT; + } return res; } @@ -166,6 +170,10 @@ nfs_file_read(struct kiocb *iocb, struct iov_iter *to) if (iocb->ki_flags & IOCB_DIRECT) return nfs_file_direct_read(iocb, to, false); + /* NOWAIT only supported on direct reads */ + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + dprintk("NFS: read(%pD2, %zu@%lu)\n", iocb->ki_filp, iov_iter_count(to), (unsigned long) iocb->ki_pos); @@ -705,6 +713,12 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) trace_nfs_file_write(iocb, from); + /* + * FMODE_NOWAIT is not set for writable files + */ + if (WARN_ON_ONCE(iocb->ki_flags & IOCB_NOWAIT)) + return -EAGAIN; + result = nfs_key_timeout_notify(file, inode); if (result) return result; diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index e85380e3b11d..72e20b56fbc7 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -95,7 +95,7 @@ static void filelayout_reset_write(struct nfs_pgio_header *hdr) "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, hdr->task.tk_pid, hdr->inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(hdr->inode), + (unsigned long long)hdr->inode->i_ino, hdr->args.count, (unsigned long long)hdr->args.offset); @@ -112,7 +112,7 @@ static void filelayout_reset_read(struct nfs_pgio_header *hdr) "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, hdr->task.tk_pid, hdr->inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(hdr->inode), + (unsigned long long)hdr->inode->i_ino, hdr->args.count, (unsigned long long)hdr->args.offset); @@ -778,6 +778,8 @@ filelayout_alloc_lseg(struct pnfs_layout_hdr *layoutid, static bool filelayout_lseg_is_striped(const struct nfs4_filelayout_segment *flseg) { + if (flseg->dsaddr) + return flseg->dsaddr->stripe_count > 1; return flseg->num_fh > 1; } diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 8b1559171fe3..c4aa995026f6 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -551,6 +551,10 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, if (!p) goto out_err_free; fh_count = be32_to_cpup(p); + if (fh_count == 0) { + rc = -EINVAL; + goto out_err_free; + } dss_info->fh_versions = kzalloc_objs(struct nfs_fh, fh_count, gfp_flags); @@ -632,6 +636,9 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, if (!p) goto out_sort_mirrors; fls->flags = be32_to_cpup(p); + if (fls->flags & FF_FLAGS_NO_IO_THRU_MDS) + set_bit(NFS4_FF_HDR_NO_IO_THRU_MDS, + &FF_LAYOUT_FROM_HDR(lh)->flags); p = xdr_inline_decode(&stream, 4); if (!p) @@ -1181,6 +1188,16 @@ ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio, 0, NFS4_MAX_UINT64, IOMODE_RW, NFS_I(pgio->pg_inode)->layout, pgio->pg_lseg); + if (NFS_I(pgio->pg_inode)->layout && + ff_layout_hdr_no_fallback_to_mds(NFS_I(pgio->pg_inode)->layout)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: no current lseg but the server's + * policy forbids MDS fallback. Surface -EAGAIN so writeback + * retries rather than silently issuing the WRITE via MDS. + */ + pgio->pg_error = -EAGAIN; + goto out; + } /* no lseg means that pnfs is not in use, so no mirroring here */ nfs_pageio_reset_write_mds(pgio); out: @@ -1230,7 +1247,7 @@ static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs) "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, hdr->task.tk_pid, hdr->inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(hdr->inode), + (unsigned long long)hdr->inode->i_ino, hdr->args.count, (unsigned long long)hdr->args.offset); @@ -1243,7 +1260,7 @@ static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs) "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, hdr->task.tk_pid, hdr->inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(hdr->inode), + (unsigned long long)hdr->inode->i_ino, hdr->args.count, (unsigned long long)hdr->args.offset); @@ -1283,7 +1300,7 @@ static void ff_layout_reset_read(struct nfs_pgio_header *hdr) "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, hdr->task.tk_pid, hdr->inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(hdr->inode), + (unsigned long long)hdr->inode->i_ino, hdr->args.count, (unsigned long long)hdr->args.offset); @@ -2200,6 +2217,14 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) out_failed: if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; + if (ff_layout_no_fallback_to_mds(lseg)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, + * never fall through to MDS I/O. + */ + pnfs_error_mark_layout_for_return(hdr->inode, lseg); + return PNFS_TRY_AGAIN; + } trace_pnfs_mds_fallback_read_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, IOMODE_READ, NFS_I(hdr->inode)->layout, lseg); @@ -2285,6 +2310,14 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) out_failed: if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; + if (ff_layout_no_fallback_to_mds(lseg)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, + * never fall through to MDS I/O. + */ + pnfs_error_mark_layout_for_return(hdr->inode, lseg); + return PNFS_TRY_AGAIN; + } trace_pnfs_mds_fallback_write_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, IOMODE_RW, NFS_I(hdr->inode)->layout, lseg); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h index 17a008c8e97c..a5bd00f69e82 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.h +++ b/fs/nfs/flexfilelayout/flexfilelayout.h @@ -112,12 +112,16 @@ struct nfs4_ff_layout_segment { struct nfs4_ff_layout_mirror *mirror_array[] __counted_by(mirror_array_cnt); }; +/* nfs4_flexfile_layout::flags bit indices */ +#define NFS4_FF_HDR_NO_IO_THRU_MDS 0 /* any lseg has had FF_FLAGS_NO_IO_THRU_MDS */ + struct nfs4_flexfile_layout { struct pnfs_layout_hdr generic_hdr; struct pnfs_ds_commit_info commit_info; struct list_head mirrors; struct list_head error_list; /* nfs4_ff_layout_ds_err */ ktime_t last_report_time; /* Layoutstat report times */ + unsigned long flags; }; struct nfs4_flexfile_layoutreturn_args { @@ -184,6 +188,18 @@ ff_layout_no_fallback_to_mds(struct pnfs_layout_segment *lseg) return FF_LAYOUT_LSEG(lseg)->flags & FF_FLAGS_NO_IO_THRU_MDS; } +/* + * Sticky hdr-level mirror of FF_FLAGS_NO_IO_THRU_MDS so callers that have + * no current lseg (e.g. between LAYOUTRETURN and the next LAYOUTGET) can + * still honor the no-MDS-fallback policy. + */ +static inline bool +ff_layout_hdr_no_fallback_to_mds(struct pnfs_layout_hdr *lo) +{ + return test_bit(NFS4_FF_HDR_NO_IO_THRU_MDS, + &FF_LAYOUT_FROM_HDR(lo)->flags); +} + static inline bool ff_layout_no_read_on_rw(struct pnfs_layout_segment *lseg) { diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index c105882edd16..1967de7d1dff 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -1769,7 +1769,9 @@ struct file_system_type nfs_fs_type = { .init_fs_context = nfs_init_fs_context, .parameters = nfs_fs_parameters, .kill_sb = nfs_kill_super, - .fs_flags = FS_RENAME_DOES_D_MOVE|FS_BINARY_MOUNTDATA, + .fs_flags = FS_RENAME_DOES_D_MOVE | + FS_BINARY_MOUNTDATA | + FS_USERNS_DELEGATABLE, }; MODULE_ALIAS_FS("nfs"); EXPORT_SYMBOL_GPL(nfs_fs_type); @@ -1781,7 +1783,9 @@ struct file_system_type nfs4_fs_type = { .init_fs_context = nfs_init_fs_context, .parameters = nfs_fs_parameters, .kill_sb = nfs_kill_super, - .fs_flags = FS_RENAME_DOES_D_MOVE|FS_BINARY_MOUNTDATA, + .fs_flags = FS_RENAME_DOES_D_MOVE | + FS_BINARY_MOUNTDATA | + FS_USERNS_DELEGATABLE, }; MODULE_ALIAS_FS("nfs4"); MODULE_ALIAS("nfs4"); diff --git a/fs/nfs/getroot.c b/fs/nfs/getroot.c index eef0736beb67..ff7424bc4bec 100644 --- a/fs/nfs/getroot.c +++ b/fs/nfs/getroot.c @@ -33,35 +33,6 @@ #define NFSDBG_FACILITY NFSDBG_CLIENT /* - * Set the superblock root dentry. - * Note that this function frees the inode in case of error. - */ -static int nfs_superblock_set_dummy_root(struct super_block *sb, struct inode *inode) -{ - /* The mntroot acts as the dummy root dentry for this superblock */ - if (sb->s_root == NULL) { - sb->s_root = d_make_root(inode); - if (sb->s_root == NULL) - return -ENOMEM; - ihold(inode); - /* - * Ensure that this dentry is invisible to d_find_alias(). - * Otherwise, it may be spliced into the tree by - * d_splice_alias if a parent directory from the same - * filesystem gets mounted at a later time. - * This again causes shrink_dcache_for_umount_subtree() to - * Oops, since the test for IS_ROOT() will fail. - */ - spin_lock(&d_inode(sb->s_root)->i_lock); - spin_lock(&sb->s_root->d_lock); - hlist_del_init(&sb->s_root->d_alias); - spin_unlock(&sb->s_root->d_lock); - spin_unlock(&d_inode(sb->s_root)->i_lock); - } - return 0; -} - -/* * get a root dentry from the root filehandle */ int nfs_get_root(struct super_block *s, struct fs_context *fc) @@ -99,10 +70,6 @@ int nfs_get_root(struct super_block *s, struct fs_context *fc) goto out_fattr; } - error = nfs_superblock_set_dummy_root(s, inode); - if (error != 0) - goto out_fattr; - /* root dentries normally start off anonymous and get spliced in later * if the dentry tree reaches them; however if the dentry already * exists, we'll pick it up at this point and use it as the root @@ -123,6 +90,8 @@ int nfs_get_root(struct super_block *s, struct fs_context *fc) name = NULL; } spin_unlock(&root->d_lock); + if (!s->s_root) + s->s_root = dget(root); fc->root = root; if (server->caps & NFS_CAP_SECURITY_LABEL) kflags |= SECURITY_LSM_NATIVE_LABELS; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index e26030e73696..5bcd4027d203 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -41,6 +41,7 @@ #include <linux/freezer.h> #include <linux/uaccess.h> #include <linux/iversion.h> +#include <linux/fileattr.h> #include "nfs4_fs.h" #include "callback.h" @@ -57,21 +58,23 @@ #define NFSDBG_FACILITY NFSDBG_VFS -#define NFS_64_BIT_INODE_NUMBERS_ENABLED 1 +static bool enable_ino64; -/* Default is to see 64-bit inode numbers */ -static bool enable_ino64 = NFS_64_BIT_INODE_NUMBERS_ENABLED; +static int param_set_enable_ino64(const char *val, const struct kernel_param *kp) +{ + pr_notice("enable_ino64 is deprecated and has no effect\n"); + return 0; +} + +static const struct kernel_param_ops param_ops_enable_ino64 = { + .set = param_set_enable_ino64, + .get = param_get_bool, +}; static int nfs_update_inode(struct inode *, struct nfs_fattr *); static struct kmem_cache * nfs_inode_cachep; -static inline unsigned long -nfs_fattr_to_ino_t(struct nfs_fattr *fattr) -{ - return nfs_fileid_to_ino_t(fattr->fileid); -} - int nfs_wait_bit_killable(struct wait_bit_key *key, int mode) { if (unlikely(nfs_current_task_exiting())) @@ -83,29 +86,6 @@ int nfs_wait_bit_killable(struct wait_bit_key *key, int mode) } EXPORT_SYMBOL_GPL(nfs_wait_bit_killable); -/** - * nfs_compat_user_ino64 - returns the user-visible inode number - * @fileid: 64-bit fileid - * - * This function returns a 32-bit inode number if the boot parameter - * nfs.enable_ino64 is zero. - */ -u64 nfs_compat_user_ino64(u64 fileid) -{ -#ifdef CONFIG_COMPAT - compat_ulong_t ino; -#else - unsigned long ino; -#endif - - if (enable_ino64) - return fileid; - ino = fileid; - if (sizeof(ino) < sizeof(fileid)) - ino ^= fileid >> (sizeof(fileid)-sizeof(ino)) * 8; - return ino; -} - int nfs_drop_inode(struct inode *inode) { return NFS_STALE(inode) || inode_generic_drop(inode); @@ -313,8 +293,7 @@ struct nfs_find_desc { }; /* - * In NFSv3 we can have 64bit inode numbers. In order to support - * this, and re-exported directories (also seen in NFSv2) + * For re-exported directories (also seen in NFSv2) * we are forced to allow 2 different inodes to have the same * i_ino. */ @@ -325,7 +304,7 @@ nfs_find_actor(struct inode *inode, void *opaque) struct nfs_fh *fh = desc->fh; struct nfs_fattr *fattr = desc->fattr; - if (NFS_FILEID(inode) != fattr->fileid) + if (inode->i_ino != fattr->fileid) return 0; if (inode_wrong_type(inode, fattr->mode)) return 0; @@ -342,7 +321,7 @@ nfs_init_locked(struct inode *inode, void *opaque) struct nfs_find_desc *desc = opaque; struct nfs_fattr *fattr = desc->fattr; - set_nfs_fileid(inode, fattr->fileid); + inode->i_ino = fattr->fileid; inode->i_mode = fattr->mode; nfs_copy_fh(NFS_FH(inode), desc->fh); return 0; @@ -413,13 +392,13 @@ nfs_ilookup(struct super_block *sb, struct nfs_fattr *fattr, struct nfs_fh *fh) .fattr = fattr, }; struct inode *inode; - unsigned long hash; + u64 hash; if (!(fattr->valid & NFS_ATTR_FATTR_FILEID) || !(fattr->valid & NFS_ATTR_FATTR_TYPE)) return NULL; - hash = nfs_fattr_to_ino_t(fattr); + hash = fattr->fileid; inode = ilookup5(sb, hash, nfs_find_actor, &desc); dprintk("%s: returning %p\n", __func__, inode); @@ -456,7 +435,7 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) }; struct inode *inode = ERR_PTR(-ENOENT); u64 fattr_supported = NFS_SB(sb)->fattr_valid; - unsigned long hash; + u64 hash; nfs_attr_check_mountpoint(sb, fattr); @@ -467,7 +446,7 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) if ((fattr->valid & NFS_ATTR_FATTR_TYPE) == 0) goto out_no_inode; - hash = nfs_fattr_to_ino_t(fattr); + hash = fattr->fileid; inode = iget5_locked(sb, hash, nfs_find_actor, nfs_init_locked, &desc); if (inode == NULL) { @@ -479,10 +458,6 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) struct nfs_inode *nfsi = NFS_I(inode); unsigned long now = jiffies; - /* We set i_ino for the few things that still rely on it, - * such as stat(2) */ - inode->i_ino = hash; - /* We can't support update_atime(), since the server will reset it */ inode->i_flags |= S_NOATIME|S_NOCMTIME; inode->i_mode = fattr->mode; @@ -606,9 +581,9 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) } dprintk("NFS: nfs_fhget(%s/%Lu fh_crc=0x%08x ct=%d)\n", inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(inode), + (unsigned long long)inode->i_ino, nfs_display_fhandle_hash(fh), - icount_read(inode)); + icount_read_once(inode)); out: return inode; @@ -1066,7 +1041,6 @@ out_no_revalidate: stat->result_mask = nfs_get_valid_attrmask(inode) | request_mask; generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); - stat->ino = nfs_compat_user_ino64(NFS_FILEID(inode)); stat->change_cookie = inode_peek_iversion_raw(inode); stat->attributes_mask |= STATX_ATTR_CHANGE_MONOTONIC; if (server->change_attr_type != NFS4_CHANGE_TYPE_IS_UNDEFINED) @@ -1095,6 +1069,20 @@ out: } EXPORT_SYMBOL_GPL(nfs_getattr); +int nfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + + if (nfs_server_capable(inode, NFS_CAP_CASE_INSENSITIVE)) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + if (nfs_server_capable(inode, NFS_CAP_CASE_NONPRESERVING)) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + return 0; +} +EXPORT_SYMBOL_GPL(nfs_fileattr_get); + static void nfs_init_lock_context(struct nfs_lock_context *l_ctx) { refcount_set(&l_ctx->count, 1); @@ -1370,7 +1358,7 @@ __nfs_revalidate_inode(struct nfs_server *server, struct inode *inode) struct nfs_inode *nfsi = NFS_I(inode); dfprintk(PAGECACHE, "NFS: revalidating (%s/%Lu)\n", - inode->i_sb->s_id, (unsigned long long)NFS_FILEID(inode)); + inode->i_sb->s_id, (unsigned long long)inode->i_ino); trace_nfs_revalidate_inode_enter(inode); @@ -1384,7 +1372,8 @@ __nfs_revalidate_inode(struct nfs_server *server, struct inode *inode) status = pnfs_sync_inode(inode, false); if (status) goto out; - } else if (nfs_have_directory_delegation(inode)) { + } else if (nfs_have_directory_delegation(inode) && + !(NFS_I(inode)->cache_validity & NFS_INO_INVALID_ATTR)) { status = 0; goto out; } @@ -1400,7 +1389,7 @@ __nfs_revalidate_inode(struct nfs_server *server, struct inode *inode) if (status != 0) { dfprintk(PAGECACHE, "nfs_revalidate_inode: (%s/%Lu) getattr failed, error=%d\n", inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(inode), status); + (unsigned long long)inode->i_ino, status); switch (status) { case -ETIMEDOUT: /* A soft timeout occurred. Use cached information? */ @@ -1420,7 +1409,7 @@ __nfs_revalidate_inode(struct nfs_server *server, struct inode *inode) if (status) { dfprintk(PAGECACHE, "nfs_revalidate_inode: (%s/%Lu) refresh failed, error=%d\n", inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(inode), status); + (unsigned long long)inode->i_ino, status); goto out; } @@ -1431,7 +1420,7 @@ __nfs_revalidate_inode(struct nfs_server *server, struct inode *inode) dfprintk(PAGECACHE, "NFS: (%s/%Lu) revalidation complete\n", inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(inode)); + (unsigned long long)inode->i_ino); out: nfs_free_fattr(fattr); @@ -1480,7 +1469,7 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map dfprintk(PAGECACHE, "NFS: (%s/%Lu) data cache invalidated\n", inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(inode)); + (unsigned long long)inode->i_ino); return 0; } @@ -1672,10 +1661,10 @@ static int nfs_check_inode_attributes(struct inode *inode, struct nfs_fattr *fat if (fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) return 0; /* Has the inode gone and changed behind our back? */ - } else if (nfsi->fileid != fattr->fileid) { + } else if (inode->i_ino != fattr->fileid) { /* Is this perhaps the mounted-on fileid? */ if ((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) && - nfsi->fileid == fattr->mounted_on_fileid) + inode->i_ino == fattr->mounted_on_fileid) return 0; return -ESTALE; } @@ -2255,22 +2244,22 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) dfprintk(VFS, "NFS: %s(%s/%llu fh_crc=0x%08x ct=%d info=0x%llx)\n", __func__, inode->i_sb->s_id, inode->i_ino, nfs_display_fhandle_hash(NFS_FH(inode)), - icount_read(inode), fattr->valid); + icount_read_once(inode), fattr->valid); if (!(fattr->valid & NFS_ATTR_FATTR_FILEID)) { /* Only a mounted-on-fileid? Just exit */ if (fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) return 0; /* Has the inode gone and changed behind our back? */ - } else if (nfsi->fileid != fattr->fileid) { + } else if (inode->i_ino != fattr->fileid) { /* Is this perhaps the mounted-on fileid? */ if ((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) && - nfsi->fileid == fattr->mounted_on_fileid) + inode->i_ino == fattr->mounted_on_fileid) return 0; printk(KERN_ERR "NFS: server %s error: fileid changed\n" "fsid %s: expected fileid 0x%Lx, got 0x%Lx\n", NFS_SERVER(inode)->nfs_client->cl_hostname, - inode->i_sb->s_id, (long long)nfsi->fileid, + inode->i_sb->s_id, (long long)inode->i_ino, (long long)fattr->fileid); goto out_err; } @@ -2798,7 +2787,7 @@ static void __exit exit_nfs_fs(void) MODULE_AUTHOR("Olaf Kirch <okir@monad.swb.de>"); MODULE_DESCRIPTION("NFS client support"); MODULE_LICENSE("GPL"); -module_param(enable_ino64, bool, 0644); +module_param_cb(enable_ino64, ¶m_ops_enable_ino64, &enable_ino64, 0644); module_init(init_nfs_fs) module_exit(exit_nfs_fs) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 18d46b0e71dd..e4533f583632 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -451,6 +451,9 @@ extern void nfs_set_cache_invalid(struct inode *inode, unsigned long flags); extern bool nfs_check_cache_invalid(struct inode *, unsigned long); extern int nfs_wait_bit_killable(struct wait_bit_key *key, int mode); +struct file_kattr; +int nfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + #if IS_ENABLED(CONFIG_NFS_LOCALIO) /* localio.c */ struct nfs_local_dio { @@ -532,6 +535,7 @@ extern void nfs_end_io_read(struct inode *inode); extern __must_check int nfs_start_io_write(struct inode *inode); extern void nfs_end_io_write(struct inode *inode); extern __must_check int nfs_start_io_direct(struct inode *inode); +extern __must_check int nfs_start_io_direct_nowait(struct inode *inode); extern void nfs_end_io_direct(struct inode *inode); static inline bool nfs_file_io_is_buffered(struct nfs_inode *nfsi) @@ -847,17 +851,19 @@ void nfs_super_set_maxbytes(struct super_block *sb, __u64 maxfilesize) } /* - * Record the page as unstable (an extra writeback period) and mark its - * inode as dirty. + * Record the request's range as unstable (an extra writeback period) and + * mark its inode as dirty. */ -static inline void nfs_folio_mark_unstable(struct folio *folio, +static inline void nfs_folio_mark_unstable(struct nfs_page *req, struct nfs_commit_info *cinfo) { + struct folio *folio = nfs_page_to_folio(req); + if (folio && !cinfo->dreq) { struct inode *inode = folio->mapping->host; - long nr = folio_nr_pages(folio); + long nr = DIV_ROUND_UP(req->wb_bytes, PAGE_SIZE); - /* This page is really still in write-back - just that the + /* This range is really still in write-back - just that the * writeback is happening on the server now. */ node_stat_mod_folio(folio, NR_WRITEBACK, nr); diff --git a/fs/nfs/io.c b/fs/nfs/io.c index 8337f0ae852d..2faf2003faf6 100644 --- a/fs/nfs/io.c +++ b/fs/nfs/io.c @@ -109,6 +109,16 @@ static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode) } } +static int nfs_block_buffered_nowait(struct nfs_inode *nfsi, struct inode *inode) +{ + if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { + if (inode->i_mapping->nrpages != 0) + return 1; + set_bit(NFS_INO_ODIRECT, &nfsi->flags); + } + return 0; +} + /** * nfs_start_io_direct - declare the file is being used for direct i/o * @inode: file inode @@ -150,6 +160,37 @@ nfs_start_io_direct(struct inode *inode) } /** + * nfs_start_io_direct_nowait - non-blocking variant of nfs_start_io_direct() + * @inode: file inode + * + * Try to declare that a direct I/O operation is about to start without + * blocking. + * Ensure all buffered I/O is blocked. + * If this could not be done without blocking then returns -EAGAIN. + */ +int +nfs_start_io_direct_nowait(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + if (!down_read_trylock(&inode->i_rwsem)) + return -EAGAIN; + if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path: try to flip NFS_INO_ODIRECT without blocking. */ + if (!down_write_trylock(&inode->i_rwsem)) + return -EAGAIN; + if (nfs_block_buffered_nowait(nfsi, inode)) { + up_write(&inode->i_rwsem); + return -EAGAIN; + } + downgrade_write(&inode->i_rwsem); + return 0; +} + +/** * nfs_end_io_direct - declare that the direct i/o operation is done * @inode: file inode * diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index af9be0c5f516..6d0073c24771 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -246,11 +246,13 @@ nfs_namespace_setattr(struct mnt_idmap *idmap, struct dentry *dentry, const struct inode_operations nfs_mountpoint_inode_operations = { .getattr = nfs_getattr, .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, }; const struct inode_operations nfs_referral_inode_operations = { .getattr = nfs_namespace_getattr, .setattr = nfs_namespace_setattr, + .fileattr_get = nfs_fileattr_get, }; static void nfs_expire_automounts(struct work_struct *work) diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index 95d7cd564b74..b80d0c5efc27 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -1053,6 +1053,7 @@ static const struct inode_operations nfs3_dir_inode_operations = { .permission = nfs_permission, .getattr = nfs_getattr, .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, #ifdef CONFIG_NFS_V3_ACL .listxattr = nfs3_listxattr, .get_inode_acl = nfs3_get_acl, @@ -1064,6 +1065,7 @@ static const struct inode_operations nfs3_file_inode_operations = { .permission = nfs_permission, .getattr = nfs_getattr, .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, #ifdef CONFIG_NFS_V3_ACL .listxattr = nfs3_listxattr, .get_inode_acl = nfs3_get_acl, diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c index e17d72908412..e745e78faab0 100644 --- a/fs/nfs/nfs3xdr.c +++ b/fs/nfs/nfs3xdr.c @@ -2276,8 +2276,11 @@ static int decode_pathconf3resok(struct xdr_stream *xdr, if (unlikely(!p)) return -EIO; result->max_link = be32_to_cpup(p++); - result->max_namelen = be32_to_cpup(p); - /* ignore remaining fields */ + result->max_namelen = be32_to_cpup(p++); + p++; /* ignore no_trunc */ + p++; /* ignore chown_restricted */ + result->case_insensitive = be32_to_cpup(p++) != 0; + result->case_preserving = be32_to_cpup(p) != 0; return 0; } diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index 7602ede6f75f..ab86246fc364 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -81,12 +81,17 @@ static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); if (status == 0) { - if (nfs_should_remove_suid(inode)) { - spin_lock(&inode->i_lock); + loff_t newsize = offset + len; + + spin_lock(&inode->i_lock); + if (newsize > i_size_read(inode)) + i_size_write(inode, newsize); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); + if (nfs_should_remove_suid(inode)) nfs_set_cache_invalid(inode, - NFS_INO_REVAL_FORCED | NFS_INO_INVALID_MODE); - spin_unlock(&inode->i_lock); - } + NFS_INO_REVAL_FORCED | + NFS_INO_INVALID_MODE); + spin_unlock(&inode->i_lock); status = nfs_post_op_update_inode_force_wcc(inode, res.falloc_fattr); } diff --git a/fs/nfs/nfs4namespace.c b/fs/nfs/nfs4namespace.c index 14f72baf3b30..2a03f02bba7c 100644 --- a/fs/nfs/nfs4namespace.c +++ b/fs/nfs/nfs4namespace.c @@ -481,7 +481,6 @@ int nfs4_submount(struct fs_context *fc, struct nfs_server *server) * Returns zero on success, or a negative errno value. */ static int nfs4_try_replacing_one_location(struct nfs_server *server, - char *page, char *page2, const struct nfs4_fs_location *location) { struct net *net = rpc_net_ns(server->client); @@ -541,21 +540,12 @@ static int nfs4_try_replacing_one_location(struct nfs_server *server, int nfs4_replace_transport(struct nfs_server *server, const struct nfs4_fs_locations *locations) { - char *page = NULL, *page2 = NULL; int loc, error; error = -ENOENT; if (locations == NULL || locations->nlocations <= 0) goto out; - error = -ENOMEM; - page = (char *) __get_free_page(GFP_USER); - if (!page) - goto out; - page2 = (char *) __get_free_page(GFP_USER); - if (!page2) - goto out; - for (loc = 0; loc < locations->nlocations; loc++) { const struct nfs4_fs_location *location = &locations->locations[loc]; @@ -564,14 +554,11 @@ int nfs4_replace_transport(struct nfs_server *server, location->rootpath.ncomponents == 0) continue; - error = nfs4_try_replacing_one_location(server, page, - page2, location); + error = nfs4_try_replacing_one_location(server, location); if (error == 0) break; } out: - free_page((unsigned long)page); - free_page((unsigned long)page2); return error; } diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index a9b8d482d289..1360409d8de9 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -377,7 +377,7 @@ static void nfs4_setup_readdir(u64 cookie, __be32 *verifier, struct dentry *dent *p++ = htonl(attrs); /* bitmap */ *p++ = htonl(12); /* attribute buffer length */ *p++ = htonl(NF4DIR); - p = xdr_encode_hyper(p, NFS_FILEID(d_inode(dentry))); + p = xdr_encode_hyper(p, d_inode(dentry)->i_ino); } *p++ = xdr_one; /* next */ @@ -391,7 +391,7 @@ static void nfs4_setup_readdir(u64 cookie, __be32 *verifier, struct dentry *dent *p++ = htonl(12); /* attribute buffer length */ *p++ = htonl(NF4DIR); spin_lock(&dentry->d_lock); - p = xdr_encode_hyper(p, NFS_FILEID(d_inode(dentry->d_parent))); + p = xdr_encode_hyper(p, d_inode(dentry->d_parent)->i_ino); spin_unlock(&dentry->d_lock); readdir->pgbase = (char *)p - (char *)start; @@ -3933,7 +3933,8 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f server->caps &= ~(NFS_CAP_ACLS | NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS | NFS_CAP_SECURITY_LABEL | NFS_CAP_FS_LOCATIONS | - NFS_CAP_OPEN_XOR | NFS_CAP_DELEGTIME); + NFS_CAP_OPEN_XOR | NFS_CAP_DELEGTIME | + NFS_CAP_CASE_INSENSITIVE | NFS_CAP_CASE_NONPRESERVING); server->fattr_valid = NFS_ATTR_FATTR_V4; if (res.attr_bitmask[0] & FATTR4_WORD0_ACL && res.acl_bitmask & ACL4_SUPPORT_ALLOW_ACL) @@ -3944,8 +3945,9 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f server->caps |= NFS_CAP_SYMLINKS; if (res.case_insensitive) server->caps |= NFS_CAP_CASE_INSENSITIVE; - if (res.case_preserving) - server->caps |= NFS_CAP_CASE_PRESERVING; + if ((res.attr_bitmask[0] & FATTR4_WORD0_CASE_PRESERVING) && + !res.case_preserving) + server->caps |= NFS_CAP_CASE_NONPRESERVING; #ifdef CONFIG_NFS_V4_SECURITY_LABEL if (res.attr_bitmask[2] & FATTR4_WORD2_SECURITY_LABEL) server->caps |= NFS_CAP_SECURITY_LABEL; @@ -5302,10 +5304,9 @@ static struct dentry *nfs4_proc_mkdir(struct inode *dir, struct dentry *dentry, do { alias = _nfs4_proc_mkdir(dir, dentry, sattr, label, &err); trace_nfs4_mkdir(dir, &dentry->d_name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, &exception); if (err) - alias = ERR_PTR(nfs4_handle_exception(NFS_SERVER(dir), - err, - &exception)); + alias = ERR_PTR(err); } while (exception.retry); nfs4_label_release_security(label); @@ -7085,7 +7086,6 @@ static void nfs4_locku_done(struct rpc_task *task, void *data) switch (task->tk_status) { case 0: renew_lease(calldata->server, calldata->timestamp); - locks_lock_inode_wait(calldata->lsp->ls_state->inode, &calldata->fl); if (nfs4_update_lock_stateid(calldata->lsp, &calldata->res.stateid)) break; @@ -7353,11 +7353,6 @@ static void nfs4_lock_done(struct rpc_task *task, void *calldata) case 0: renew_lease(NFS_SERVER(d_inode(data->ctx->dentry)), data->timestamp); - if (data->arg.new_lock && !data->cancelled) { - data->fl.c.flc_flags &= ~(FL_SLEEP | FL_ACCESS); - if (locks_lock_inode_wait(lsp->ls_state->inode, &data->fl) < 0) - goto out_restart; - } if (data->arg.new_lock_owner != 0) { nfs_confirm_seqid(&lsp->ls_seqid, 0); nfs4_stateid_copy(&lsp->ls_stateid, &data->res.stateid); @@ -7468,11 +7463,10 @@ static int _nfs4_do_setlk(struct nfs4_state *state, int cmd, struct file_lock *f msg.rpc_argp = &data->arg; msg.rpc_resp = &data->res; task_setup_data.callback_data = data; - if (recovery_type > NFS_LOCK_NEW) { - if (recovery_type == NFS_LOCK_RECLAIM) - data->arg.reclaim = NFS_LOCK_RECLAIM; - } else - data->arg.new_lock = 1; + + if (recovery_type == NFS_LOCK_RECLAIM) + data->arg.reclaim = NFS_LOCK_RECLAIM; + task = rpc_run_task(&task_setup_data); if (IS_ERR(task)) return PTR_ERR(task); @@ -7582,6 +7576,13 @@ static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock up_read(&nfsi->rwsem); mutex_unlock(&sp->so_delegreturn_mutex); status = _nfs4_do_setlk(state, cmd, request, NFS_LOCK_NEW); + if (status) + goto out; + + down_read(&nfsi->rwsem); + request->c.flc_flags &= ~(FL_SLEEP | FL_ACCESS); + status = locks_lock_inode_wait(state->inode, request); + up_read(&nfsi->rwsem); out: request->c.flc_flags = flags; return status; @@ -9989,6 +9990,38 @@ nfs4_layoutcommit_done(struct rpc_task *task, void *calldata) case -NFS4ERR_GRACE: /* loca_recalim always false */ task->tk_status = 0; break; + case -NFS4ERR_OLD_STATEID: { + u32 old_seqid = be32_to_cpu(data->args.stateid.seqid); + struct pnfs_layout_range range = { + .iomode = IOMODE_ANY, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + + if (nfs4_layout_refresh_old_stateid(&data->args.stateid, + &range, + data->args.inode)) { + struct pnfs_layout_hdr *lo; + + spin_lock(&data->args.inode->i_lock); + lo = NFS_I(data->args.inode)->layout; + if (lo && pnfs_layout_is_valid(lo) && + nfs4_stateid_match_other(&data->args.stateid, + &lo->plh_stateid)) + pnfs_set_layout_stateid(lo, &data->args.stateid, + NULL, false); + spin_unlock(&data->args.inode->i_lock); + + dprintk("%s: refreshed OLD_STATEID inode %llu seq %u->%u\n", + __func__, data->args.inode->i_ino, + old_seqid, + be32_to_cpu(data->args.stateid.seqid)); + + rpc_restart_call_prepare(task); + return; + } + fallthrough; + } case 0: break; default: @@ -10562,13 +10595,10 @@ static ssize_t nfs4_listxattr(struct dentry *dentry, char *list, size_t size) left -= error; } - error2 = security_inode_listsecurity(d_inode(dentry), list, left); + error2 = security_inode_listsecurity(d_inode(dentry), &list, &left); if (error2 < 0) return error2; - if (list) { - list += error2; - left -= error2; - } + error2 = size - error - left; error3 = nfs4_listxattr_nfs4_user(d_inode(dentry), list, left); if (error3 < 0) @@ -10617,6 +10647,7 @@ static const struct inode_operations nfs4_dir_inode_operations = { .getattr = nfs_getattr, .setattr = nfs_setattr, .listxattr = nfs4_listxattr, + .fileattr_get = nfs_fileattr_get, }; static const struct inode_operations nfs4_file_inode_operations = { @@ -10624,6 +10655,7 @@ static const struct inode_operations nfs4_file_inode_operations = { .getattr = nfs_getattr, .setattr = nfs_setattr, .listxattr = nfs4_listxattr, + .fileattr_get = nfs_fileattr_get, }; static struct nfs_server *nfs4_clone_server(struct nfs_server *source, diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h index c939533b9881..1ed677810d9d 100644 --- a/fs/nfs/nfs4trace.h +++ b/fs/nfs/nfs4trace.h @@ -597,13 +597,13 @@ DECLARE_EVENT_CLASS(nfs4_open_event, __entry->openstateid_hash = 0; } if (inode != NULL) { - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); } else { __entry->fileid = 0; __entry->fhandle = 0; } - __entry->dir = NFS_FILEID(d_inode(ctx->dentry->d_parent)); + __entry->dir = d_inode(ctx->dentry->d_parent)->i_ino; __assign_str(name); ), @@ -658,7 +658,7 @@ TRACE_EVENT(nfs4_cached_open, const struct inode *inode = state->inode; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->fmode = (__force unsigned int)state->state; __entry->stateid_seq = @@ -703,7 +703,7 @@ TRACE_EVENT(nfs4_close, const struct inode *inode = state->inode; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->fmode = (__force unsigned int)state->state; __entry->error = error < 0 ? -error : 0; @@ -759,7 +759,7 @@ DECLARE_EVENT_CLASS(nfs4_lock_event, __entry->start = request->fl_start; __entry->end = request->fl_end; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->stateid_seq = be32_to_cpu(state->stateid.seqid); @@ -831,7 +831,7 @@ TRACE_EVENT(nfs4_set_lock, __entry->start = request->fl_start; __entry->end = request->fl_end; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->stateid_seq = be32_to_cpu(state->stateid.seqid); @@ -922,7 +922,7 @@ TRACE_EVENT(nfs4_state_lock_reclaim, const struct inode *inode = state->inode; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->state_flags = state->flags; __entry->lock_flags = lock->ls_flags; @@ -960,7 +960,7 @@ DECLARE_EVENT_CLASS(nfs4_set_delegation_event, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->fmode = (__force unsigned int)fmode; ), @@ -1087,7 +1087,7 @@ DECLARE_EVENT_CLASS(nfs4_test_stateid_event, __entry->error = error < 0 ? -error : 0; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->stateid_seq = be32_to_cpu(state->stateid.seqid); @@ -1137,7 +1137,7 @@ DECLARE_EVENT_CLASS(nfs4_lookup_event, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->error = -error; __assign_str(name); ), @@ -1185,7 +1185,7 @@ TRACE_EVENT(nfs4_lookupp, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->ino = NFS_FILEID(inode); + __entry->ino = inode->i_ino; __entry->error = error < 0 ? -error : 0; ), @@ -1220,8 +1220,8 @@ TRACE_EVENT(nfs4_rename, TP_fast_assign( __entry->dev = olddir->i_sb->s_dev; - __entry->olddir = NFS_FILEID(olddir); - __entry->newdir = NFS_FILEID(newdir); + __entry->olddir = olddir->i_ino; + __entry->newdir = newdir->i_ino; __entry->error = error < 0 ? -error : 0; __assign_str(oldname); __assign_str(newname); @@ -1258,7 +1258,7 @@ DECLARE_EVENT_CLASS(nfs4_inode_event, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->error = error < 0 ? -error : 0; ), @@ -1311,7 +1311,7 @@ DECLARE_EVENT_CLASS(nfs4_inode_stateid_event, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->error = error < 0 ? -error : 0; __entry->stateid_seq = @@ -1421,7 +1421,7 @@ DECLARE_EVENT_CLASS(nfs4_inode_callback_event, __entry->error = error < 0 ? -error : 0; __entry->fhandle = nfs_fhandle_hash(fhandle); if (!IS_ERR_OR_NULL(inode)) { - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->dev = inode->i_sb->s_dev; } else { __entry->fileid = 0; @@ -1478,7 +1478,7 @@ DECLARE_EVENT_CLASS(nfs4_inode_stateid_callback_event, __entry->error = error < 0 ? -error : 0; __entry->fhandle = nfs_fhandle_hash(fhandle); if (!IS_ERR_OR_NULL(inode)) { - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->dev = inode->i_sb->s_dev; } else { __entry->fileid = 0; @@ -1655,7 +1655,7 @@ DECLARE_EVENT_CLASS(nfs4_read_event, const struct pnfs_layout_segment *lseg = hdr->lseg; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); __entry->offset = hdr->args.offset; __entry->arg_count = hdr->args.count; @@ -1727,7 +1727,7 @@ DECLARE_EVENT_CLASS(nfs4_write_event, const struct pnfs_layout_segment *lseg = hdr->lseg; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); __entry->offset = hdr->args.offset; __entry->arg_count = hdr->args.count; @@ -1795,7 +1795,7 @@ DECLARE_EVENT_CLASS(nfs4_commit_event, const struct pnfs_layout_segment *lseg = data->lseg; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); __entry->offset = data->args.offset; __entry->count = data->args.count; @@ -1857,7 +1857,7 @@ TRACE_EVENT(nfs4_layoutget, const struct inode *inode = d_inode(ctx->dentry); const struct nfs4_state *state = ctx->state; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->iomode = args->iomode; __entry->offset = args->offset; @@ -1957,7 +1957,7 @@ TRACE_EVENT(pnfs_update_layout, ), TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->pos = pos; __entry->count = count; @@ -2012,7 +2012,7 @@ DECLARE_EVENT_CLASS(pnfs_layout_event, ), TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->pos = pos; __entry->count = count; @@ -2194,7 +2194,7 @@ DECLARE_EVENT_CLASS(nfs4_flexfiles_io_event, __entry->error = -error; __entry->nfs_error = hdr->res.op_status; __entry->fhandle = nfs_fhandle_hash(hdr->args.fh); - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->dev = inode->i_sb->s_dev; __entry->offset = hdr->args.offset; __entry->count = hdr->args.count; @@ -2258,7 +2258,7 @@ TRACE_EVENT(ff_layout_commit_error, __entry->error = -error; __entry->nfs_error = data->res.op_status; __entry->fhandle = nfs_fhandle_hash(data->args.fh); - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->dev = inode->i_sb->s_dev; __entry->offset = data->args.offset; __entry->count = data->args.count; @@ -2423,7 +2423,7 @@ TRACE_EVENT(nfs4_llseek, TP_STRUCT__entry( __field(unsigned long, error) __field(u32, fhandle) - __field(u32, fileid) + __field(u64, fileid) __field(dev_t, dev) __field(int, stateid_seq) __field(u32, stateid_hash) @@ -2434,10 +2434,9 @@ TRACE_EVENT(nfs4_llseek, ), TP_fast_assign( - const struct nfs_inode *nfsi = NFS_I(inode); const struct nfs_fh *fh = args->sa_fh; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->dev = inode->i_sb->s_dev; __entry->fhandle = nfs_fhandle_hash(fh); __entry->offset_s = args->sa_offset; @@ -2499,7 +2498,7 @@ DECLARE_EVENT_CLASS(nfs4_sparse_event, __entry->offset = args->falloc_offset; __entry->len = args->falloc_length; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __entry->stateid_seq = be32_to_cpu(args->falloc_stateid.seqid); @@ -2568,14 +2567,11 @@ TRACE_EVENT(nfs4_copy, ), TP_fast_assign( - const struct nfs_inode *src_nfsi = NFS_I(src_inode); - const struct nfs_inode *dst_nfsi = NFS_I(dst_inode); - - __entry->src_fileid = src_nfsi->fileid; + __entry->src_fileid = src_inode->i_ino; __entry->src_dev = src_inode->i_sb->s_dev; __entry->src_fhandle = nfs_fhandle_hash(args->src_fh); __entry->src_offset = args->src_pos; - __entry->dst_fileid = dst_nfsi->fileid; + __entry->dst_fileid = dst_inode->i_ino; __entry->dst_dev = dst_inode->i_sb->s_dev; __entry->dst_fhandle = nfs_fhandle_hash(args->dst_fh); __entry->dst_offset = args->dst_pos; @@ -2666,14 +2662,11 @@ TRACE_EVENT(nfs4_clone, ), TP_fast_assign( - const struct nfs_inode *src_nfsi = NFS_I(src_inode); - const struct nfs_inode *dst_nfsi = NFS_I(dst_inode); - - __entry->src_fileid = src_nfsi->fileid; + __entry->src_fileid = src_inode->i_ino; __entry->src_dev = src_inode->i_sb->s_dev; __entry->src_fhandle = nfs_fhandle_hash(args->src_fh); __entry->src_offset = args->src_offset; - __entry->dst_fileid = dst_nfsi->fileid; + __entry->dst_fileid = dst_inode->i_ino; __entry->dst_dev = dst_inode->i_sb->s_dev; __entry->dst_fhandle = nfs_fhandle_hash(args->dst_fh); __entry->dst_offset = args->dst_offset; @@ -2724,7 +2717,7 @@ TRACE_EVENT(nfs4_copy_notify, TP_STRUCT__entry( __field(unsigned long, error) __field(u32, fhandle) - __field(u32, fileid) + __field(u64, fileid) __field(dev_t, dev) __field(int, stateid_seq) __field(u32, stateid_hash) @@ -2733,9 +2726,7 @@ TRACE_EVENT(nfs4_copy_notify, ), TP_fast_assign( - const struct nfs_inode *nfsi = NFS_I(inode); - - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->dev = inode->i_sb->s_dev; __entry->fhandle = nfs_fhandle_hash(args->cna_src_fh); __entry->stateid_seq = @@ -2830,7 +2821,7 @@ DECLARE_EVENT_CLASS(nfs4_xattr_event, TP_fast_assign( __entry->error = error < 0 ? -error : 0; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); __assign_str(name); ), diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index ff467959f733..4ada21f4eebd 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -80,7 +80,7 @@ DECLARE_EVENT_CLASS(nfs_inode_event, TP_fast_assign( const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(inode); __entry->cache_validity = nfsi->cache_validity; @@ -121,7 +121,7 @@ DECLARE_EVENT_CLASS(nfs_inode_event_done, const struct nfs_inode *nfsi = NFS_I(inode); __entry->error = error < 0 ? -error : 0; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->type = nfs_umode_to_dtype(inode->i_mode); __entry->version = inode_peek_iversion_raw(inode); @@ -211,7 +211,7 @@ TRACE_EVENT(nfs_access_exit, const struct nfs_inode *nfsi = NFS_I(inode); __entry->error = error < 0 ? -error : 0; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->type = nfs_umode_to_dtype(inode->i_mode); __entry->version = inode_peek_iversion_raw(inode); @@ -265,7 +265,7 @@ DECLARE_EVENT_CLASS(nfs_update_size_class, __entry->dev = inode->i_sb->s_dev; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->version = inode_peek_iversion_raw(inode); __entry->cur_size = i_size_read(inode); __entry->new_size = new_size; @@ -317,7 +317,7 @@ DECLARE_EVENT_CLASS(nfs_inode_range_event, __entry->dev = inode->i_sb->s_dev; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->version = inode_peek_iversion_raw(inode); __entry->range_start = range_start; __entry->range_end = range_end; @@ -371,7 +371,7 @@ DECLARE_EVENT_CLASS(nfs_readdir_event, const struct nfs_inode *nfsi = NFS_I(dir); __entry->dev = dir->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = dir->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(dir); if (cookie != 0) @@ -429,9 +429,9 @@ DECLARE_EVENT_CLASS(nfs_lookup_event, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->flags = flags; - __entry->fileid = d_is_negative(dentry) ? 0 : NFS_FILEID(d_inode(dentry)); + __entry->fileid = d_is_negative(dentry) ? 0 : d_inode(dentry)->i_ino; __assign_str(name); ), @@ -476,10 +476,10 @@ DECLARE_EVENT_CLASS(nfs_lookup_event_done, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->error = error < 0 ? -error : 0; __entry->flags = flags; - __entry->fileid = d_is_negative(dentry) ? 0 : NFS_FILEID(d_inode(dentry)); + __entry->fileid = d_is_negative(dentry) ? 0 : d_inode(dentry)->i_ino; __assign_str(name); ), @@ -532,7 +532,7 @@ TRACE_EVENT(nfs_atomic_open_enter, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->flags = flags; __entry->fmode = (__force unsigned long)ctx->mode; __assign_str(name); @@ -571,7 +571,7 @@ TRACE_EVENT(nfs_atomic_open_exit, TP_fast_assign( __entry->error = -error; __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->flags = flags; __entry->fmode = (__force unsigned long)ctx->mode; __assign_str(name); @@ -608,7 +608,7 @@ TRACE_EVENT(nfs_create_enter, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->flags = flags; __assign_str(name); ), @@ -644,7 +644,7 @@ TRACE_EVENT(nfs_create_exit, TP_fast_assign( __entry->error = -error; __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->flags = flags; __assign_str(name); ), @@ -676,7 +676,7 @@ DECLARE_EVENT_CLASS(nfs_directory_event, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __assign_str(name); ), @@ -714,7 +714,7 @@ DECLARE_EVENT_CLASS(nfs_directory_event_done, TP_fast_assign( __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->error = error < 0 ? -error : 0; __assign_str(name); ), @@ -768,8 +768,8 @@ TRACE_EVENT(nfs_link_enter, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); - __entry->dir = NFS_FILEID(dir); + __entry->fileid = inode->i_ino; + __entry->dir = dir->i_ino; __assign_str(name); ), @@ -803,8 +803,8 @@ TRACE_EVENT(nfs_link_exit, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; - __entry->fileid = NFS_FILEID(inode); - __entry->dir = NFS_FILEID(dir); + __entry->fileid = inode->i_ino; + __entry->dir = dir->i_ino; __entry->error = error < 0 ? -error : 0; __assign_str(name); ), @@ -840,8 +840,8 @@ DECLARE_EVENT_CLASS(nfs_rename_event, TP_fast_assign( __entry->dev = old_dir->i_sb->s_dev; - __entry->old_dir = NFS_FILEID(old_dir); - __entry->new_dir = NFS_FILEID(new_dir); + __entry->old_dir = old_dir->i_ino; + __entry->new_dir = new_dir->i_ino; __assign_str(old_name); __assign_str(new_name); ), @@ -889,8 +889,8 @@ DECLARE_EVENT_CLASS(nfs_rename_event_done, TP_fast_assign( __entry->dev = old_dir->i_sb->s_dev; __entry->error = -error; - __entry->old_dir = NFS_FILEID(old_dir); - __entry->new_dir = NFS_FILEID(new_dir); + __entry->old_dir = old_dir->i_ino; + __entry->new_dir = new_dir->i_ino; __assign_str(old_name); __assign_str(new_name); ), @@ -943,7 +943,7 @@ TRACE_EVENT(nfs_sillyrename_unlink, struct inode *dir = d_inode(data->dentry->d_parent); size_t len = data->args.name.len; __entry->dev = dir->i_sb->s_dev; - __entry->dir = NFS_FILEID(dir); + __entry->dir = dir->i_ino; __entry->error = -error; memcpy(__get_str(name), data->args.name.name, len); @@ -981,7 +981,7 @@ DECLARE_EVENT_CLASS(nfs_folio_event, const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(inode); __entry->offset = offset; @@ -1031,7 +1031,7 @@ DECLARE_EVENT_CLASS(nfs_folio_event_done, const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(inode); __entry->offset = offset; @@ -1109,7 +1109,7 @@ DECLARE_EVENT_CLASS(nfs_kiocb_event, const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(inode); __entry->offset = iocb->ki_pos; @@ -1160,7 +1160,7 @@ TRACE_EVENT(nfs_aop_readahead, const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(inode); __entry->offset = pos; @@ -1199,7 +1199,7 @@ TRACE_EVENT(nfs_aop_readahead_done, const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(inode); __entry->nr_pages = nr_pages; @@ -1239,7 +1239,7 @@ TRACE_EVENT(nfs_initiate_read, __entry->offset = hdr->args.offset; __entry->count = hdr->args.count; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1284,7 +1284,7 @@ TRACE_EVENT(nfs_readpage_done, __entry->res_count = hdr->res.count; __entry->eof = hdr->res.eof; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1330,7 +1330,7 @@ TRACE_EVENT(nfs_readpage_short, __entry->res_count = hdr->res.count; __entry->eof = hdr->res.eof; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1377,7 +1377,7 @@ TRACE_EVENT(nfs_pgio_error, __entry->arg_count = hdr->args.count; __entry->res_count = hdr->res.count; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1416,7 +1416,7 @@ TRACE_EVENT(nfs_initiate_write, __entry->count = hdr->args.count; __entry->stable = hdr->args.stable; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1467,7 +1467,7 @@ TRACE_EVENT(nfs_writeback_done, &verf->verifier, NFS4_VERIFIER_SIZE); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1507,7 +1507,7 @@ DECLARE_EVENT_CLASS(nfs_page_class, const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->req = req; __entry->offset = req_offset(req); @@ -1555,7 +1555,7 @@ DECLARE_EVENT_CLASS(nfs_page_error_class, TP_fast_assign( const struct nfs_inode *nfsi = NFS_I(inode); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->offset = req_offset(req); __entry->count = req->wb_bytes; @@ -1609,7 +1609,7 @@ TRACE_EVENT(nfs_initiate_commit, __entry->offset = data->args.offset; __entry->count = data->args.count; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1655,7 +1655,7 @@ TRACE_EVENT(nfs_commit_done, &verf->verifier, NFS4_VERIFIER_SIZE); __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); ), @@ -1701,7 +1701,7 @@ DECLARE_EVENT_CLASS(nfs_direct_req_class, const struct nfs_fh *fh = &nfsi->fh; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); __entry->offset = dreq->io_start; __entry->count = dreq->count; @@ -1765,7 +1765,7 @@ DECLARE_EVENT_CLASS(nfs_local_dio_class, const struct nfs_fh *fh = &nfsi->fh; __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; + __entry->fileid = inode->i_ino; __entry->fhandle = nfs_fhandle_hash(fh); __entry->offset = offset; __entry->count = count; diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 4a87b2fdb2e6..7dd478ffc2fa 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -759,7 +759,7 @@ int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, dprintk("NFS: initiated pgio call " "(req %s/%llu, %u bytes @ offset %llu)\n", hdr->inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(hdr->inode), + (unsigned long long)hdr->inode->i_ino, hdr->args.count, (unsigned long long)hdr->args.offset); diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 743467e9ba20..7715e2bd5871 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1463,8 +1463,6 @@ _pnfs_return_layout(struct inode *ino) pnfs_clear_layoutcommit(ino, &tmp_list); pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); - if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) - NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); /* Don't send a LAYOUTRETURN if list was initially empty */ if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || @@ -1476,6 +1474,8 @@ _pnfs_return_layout(struct inode *ino) send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); spin_unlock(&ino->i_lock); + if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) + NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); if (send) status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, 0); @@ -2229,11 +2229,11 @@ lookup_again: dprintk("%s wait for layoutreturn\n", __func__); lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); if (!IS_ERR(lseg)) { - pnfs_put_layout_hdr(lo); dprintk("%s retrying\n", __func__); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); + pnfs_put_layout_hdr(lo); goto lookup_again; } trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, @@ -2373,7 +2373,7 @@ out: dprintk("%s: inode %s/%llu pNFS layout segment %s for " "(%s, offset: %llu, length: %llu)\n", __func__, ino->i_sb->s_id, - (unsigned long long)NFS_FILEID(ino), + (unsigned long long)ino->i_ino, IS_ERR_OR_NULL(lseg) ? "not found" : "found", iomode==IOMODE_RW ? "read/write" : "read-only", (unsigned long long)pos, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 12632a706da8..648c95b78eea 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -1075,14 +1075,14 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags) /* r_netid */ nlen = xdr_stream_decode_string_dup(xdr, &netid, XDR_MAX_NETOBJ, gfp_flags); - if (unlikely(nlen < 0)) + if (unlikely(nlen <= 0)) goto out_err; /* r_addr: ip/ip6addr with port in dec octets - see RFC 5665 */ /* port is ".ABC.DEF", 8 chars max */ rlen = xdr_stream_decode_string_dup(xdr, &buf, INET6_ADDRSTRLEN + IPV6_SCOPE_ID_LEN + 8, gfp_flags); - if (unlikely(rlen < 0)) + if (unlikely(rlen <= 0)) goto out_free_netid; /* replace port '.' with '-' */ @@ -1199,7 +1199,7 @@ pnfs_layout_mark_request_commit(struct nfs_page *req, nfs_request_add_commit_list_locked(req, list, cinfo); mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); - nfs_folio_mark_unstable(nfs_page_to_folio(req), cinfo); + nfs_folio_mark_unstable(req, cinfo); return; out_resched: mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); diff --git a/fs/nfs/proc.c b/fs/nfs/proc.c index 70795684b8e8..03c2c1f31be9 100644 --- a/fs/nfs/proc.c +++ b/fs/nfs/proc.c @@ -598,6 +598,7 @@ nfs_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle, { info->max_link = 0; info->max_namelen = NFS2_MAXNAMLEN; + info->case_preserving = true; return 0; } @@ -718,12 +719,14 @@ static const struct inode_operations nfs_dir_inode_operations = { .permission = nfs_permission, .getattr = nfs_getattr, .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, }; static const struct inode_operations nfs_file_inode_operations = { .permission = nfs_permission, .getattr = nfs_getattr, .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, }; const struct nfs_rpc_ops nfs_v2_clientops = { diff --git a/fs/nfs/read.c b/fs/nfs/read.c index e1fe78d7b8d0..2b70bd2b934b 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -132,10 +132,32 @@ static void nfs_readpage_release(struct nfs_page *req, int error) static void nfs_page_group_set_uptodate(struct nfs_page *req) { - if (nfs_page_group_sync_on_bit(req, PG_UPTODATE)) + bool uptodate = false; + + nfs_page_group_lock(req); + if (!test_bit(PG_READ_FAILED, &req->wb_head->wb_flags) && + nfs_page_group_sync_on_bit_locked(req, PG_UPTODATE)) + uptodate = true; + nfs_page_group_unlock(req); + + if (uptodate) folio_mark_uptodate(nfs_page_to_folio(req)); } +static void nfs_page_group_mark_read_failed(struct nfs_page *req) +{ + struct nfs_page *tmp; + + nfs_page_group_lock(req); + set_bit(PG_READ_FAILED, &req->wb_head->wb_flags); + tmp = req; + do { + clear_bit(PG_UPTODATE, &tmp->wb_flags); + tmp = tmp->wb_this_page; + } while (tmp != req); + nfs_page_group_unlock(req); +} + static void nfs_read_completion(struct nfs_pgio_header *hdr) { unsigned long bytes = 0; @@ -172,6 +194,7 @@ static void nfs_read_completion(struct nfs_pgio_header *hdr) if (bytes <= hdr->good_bytes) nfs_page_group_set_uptodate(req); else { + nfs_page_group_mark_read_failed(req); error = hdr->error; xchg(&nfs_req_openctx(req)->error, error); } diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 4cd420b14ce3..cb19f1540d98 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -509,6 +509,10 @@ static void nfs_show_mount_options(struct seq_file *m, struct nfs_server *nfss, default: break; } + if (clp->cl_xprtsec.cert_serial) + seq_puts(m, ",cert_serial=<redacted>"); + if (clp->cl_xprtsec.privkey_serial) + seq_puts(m, ",privkey_serial=<redacted>"); if (version != 4) nfs_show_mountd_options(m, nfss, showdefaults); @@ -623,7 +627,7 @@ static void show_implementation_id(struct seq_file *m, struct nfs_server *nfss) int nfs_show_devname(struct seq_file *m, struct dentry *root) { - char *page = (char *) __get_free_page(GFP_KERNEL); + char *page = kmalloc(PAGE_SIZE, GFP_KERNEL); char *devname, *dummy; int err = 0; if (!page) @@ -633,7 +637,7 @@ int nfs_show_devname(struct seq_file *m, struct dentry *root) err = PTR_ERR(devname); else seq_escape(m, devname, " \t\n\\"); - free_page((unsigned long)page); + kfree(page); return err; } EXPORT_SYMBOL_GPL(nfs_show_devname); diff --git a/fs/nfs/symlink.c b/fs/nfs/symlink.c index 58146e935402..74a072896f8d 100644 --- a/fs/nfs/symlink.c +++ b/fs/nfs/symlink.c @@ -22,6 +22,8 @@ #include <linux/mm.h> #include <linux/string.h> +#include "internal.h" + /* Symlink caching in the page cache is even more simplistic * and straight-forward than readdir caching. */ @@ -74,4 +76,5 @@ const struct inode_operations nfs_symlink_inode_operations = { .get_link = nfs_get_link, .getattr = nfs_getattr, .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, }; diff --git a/fs/nfs/unlink.c b/fs/nfs/unlink.c index df3ca4669df6..b57cfaa4d516 100644 --- a/fs/nfs/unlink.c +++ b/fs/nfs/unlink.c @@ -124,7 +124,7 @@ static int nfs_call_unlink(struct dentry *dentry, struct inode *inode, struct nf struct dentry *alias; down_read_non_owner(&NFS_I(dir)->rmdir_sem); - alias = d_alloc_parallel(dentry->d_parent, &data->args.name, &data->wq); + alias = d_alloc_parallel(dentry->d_parent, &data->args.name); if (IS_ERR(alias)) { up_read_non_owner(&NFS_I(dir)->rmdir_sem); return 0; @@ -185,7 +185,6 @@ nfs_async_unlink(struct dentry *dentry, const struct qstr *name) data->cred = get_current_cred(); data->res.dir_attr = &data->dir_attr; - init_waitqueue_head(&data->wq); status = -EBUSY; spin_lock(&dentry->d_lock); @@ -461,7 +460,7 @@ nfs_sillyrename(struct inode *dir, struct dentry *dentry) if (dentry->d_flags & DCACHE_NFSFS_RENAMED) goto out; - fileid = NFS_FILEID(d_inode(dentry)); + fileid = d_inode(dentry)->i_ino; sdentry = NULL; do { diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 3134bb17f3e3..d2b03ceaeb4f 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -807,7 +807,7 @@ nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo) mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo); mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); - nfs_folio_mark_unstable(nfs_page_to_folio(req), cinfo); + nfs_folio_mark_unstable(req, cinfo); } EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); @@ -866,10 +866,12 @@ nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, nfs_request_add_commit_list(req, cinfo); } -static void nfs_folio_clear_commit(struct folio *folio) +static void nfs_folio_clear_commit(struct nfs_page *req) { + struct folio *folio = nfs_page_to_folio(req); + if (folio) { - long nr = folio_nr_pages(folio); + long nr = DIV_ROUND_UP(req->wb_bytes, PAGE_SIZE); node_stat_mod_folio(folio, NR_WRITEBACK, -nr); bdi_wb_stat_mod(folio->mapping->host, WB_WRITEBACK, -nr); @@ -889,7 +891,7 @@ static void nfs_clear_request_commit(struct nfs_commit_info *cinfo, nfs_request_remove_commit_list(req, cinfo); } mutex_unlock(&NFS_I(inode)->commit_mutex); - nfs_folio_clear_commit(nfs_page_to_folio(req)); + nfs_folio_clear_commit(req); } } @@ -927,7 +929,7 @@ static void nfs_write_completion(struct nfs_pgio_header *hdr) } if (nfs_write_need_commit(hdr)) { struct nfs_open_context *ctx = - hdr->req->wb_lock_context->open_context; + req->wb_lock_context->open_context; /* Reset wb_nio, since the write was successful. */ req->wb_nio = 0; @@ -1741,7 +1743,7 @@ void nfs_retry_commit(struct list_head *page_list, req = nfs_list_entry(page_list->next); nfs_list_remove_request(req); nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx); - nfs_folio_clear_commit(nfs_page_to_folio(req)); + nfs_folio_clear_commit(req); nfs_unlock_and_release_request(req); } } @@ -1813,11 +1815,11 @@ static void nfs_commit_release_pages(struct nfs_commit_data *data) req = nfs_list_entry(data->pages.next); nfs_list_remove_request(req); folio = nfs_page_to_folio(req); - nfs_folio_clear_commit(folio); + nfs_folio_clear_commit(req); dprintk("NFS: commit (%s/%llu %d@%lld)", nfs_req_openctx(req)->dentry->d_sb->s_id, - (unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req)->dentry)), + (unsigned long long)d_inode(nfs_req_openctx(req)->dentry)->i_ino, req->wb_bytes, (long long)req_offset(req)); if (status < 0) { diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c index 9d829c84f374..5be7721c22c2 100644 --- a/fs/nfsd/blocklayout.c +++ b/fs/nfsd/blocklayout.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2014-2016 Christoph Hellwig. */ -#include <linux/exportfs.h> +#include <linux/exportfs_block.h> #include <linux/iomap.h> #include <linux/slab.h> #include <linux/pr.h> @@ -32,8 +32,8 @@ nfsd4_block_map_extent(struct inode *inode, const struct svc_fh *fhp, u32 device_generation = 0; int error; - error = sb->s_export_op->map_blocks(inode, offset, length, &iomap, - iomode != IOMODE_READ, &device_generation); + error = sb->s_export_op->block_ops->map_blocks(inode, offset, length, + &iomap, iomode != IOMODE_READ, &device_generation); if (error) { if (error == -ENXIO) return nfserr_layoutunavailable; @@ -179,23 +179,20 @@ static __be32 nfsd4_block_commit_blocks(struct inode *inode, struct nfsd4_layoutcommit *lcp, struct iomap *iomaps, int nr_iomaps) { - struct timespec64 mtime = inode_get_mtime(inode); - struct iattr iattr = { .ia_valid = 0 }; int error; - if (lcp->lc_mtime.tv_nsec == UTIME_NOW || - timespec64_compare(&lcp->lc_mtime, &mtime) < 0) - lcp->lc_mtime = current_time(inode); - iattr.ia_valid |= ATTR_ATIME | ATTR_CTIME | ATTR_MTIME; - iattr.ia_atime = iattr.ia_ctime = iattr.ia_mtime = lcp->lc_mtime; - - if (lcp->lc_size_chg) { - iattr.ia_valid |= ATTR_SIZE; - iattr.ia_size = lcp->lc_newsize; - } - - error = inode->i_sb->s_export_op->commit_blocks(inode, iomaps, - nr_iomaps, &iattr); + /* + * This ignores the client provided mtime in loca_time_modify, as a + * fully client specified mtime doesn't really fit into the Linux + * multi-grain timestamp architecture. + * + * RFC 8881 Section 18.42 makes it clear that the client provided + * timestamp is a "may" condition, and clients that want to force a + * specific timestamp should send a separate SETATTR in the compound. + */ + error = inode->i_sb->s_export_op->block_ops->commit_blocks(inode, + iomaps, nr_iomaps, + lcp->lc_size_chg ? lcp->lc_newsize : 0); kfree(iomaps); return nfserrno(error); } @@ -218,8 +215,8 @@ nfsd4_block_get_device_info_simple(struct super_block *sb, b->type = PNFS_BLOCK_VOLUME_SIMPLE; b->simple.sig_len = PNFS_BLOCK_UUID_LEN; - return sb->s_export_op->get_uuid(sb, b->simple.sig, &b->simple.sig_len, - &b->simple.offset); + return sb->s_export_op->block_ops->get_uuid(sb, b->simple.sig, + &b->simple.sig_len, &b->simple.offset); } static __be32 diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 665153f1720e..a47c90f40422 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -17,6 +17,8 @@ #include <linux/module.h> #include <linux/exportfs.h> #include <linux/sunrpc/svc_xprt.h> +#include <net/genetlink.h> +#include <uapi/linux/nfsd_netlink.h> #include "nfsd.h" #include "nfsfh.h" @@ -24,6 +26,7 @@ #include "pnfs.h" #include "filecache.h" #include "trace.h" +#include "netlink.h" #define NFSDDBG_FACILITY NFSDDBG_EXPORT @@ -36,35 +39,24 @@ * second map contains a reference to the entry in the first map. */ -static struct workqueue_struct *nfsd_export_wq; - #define EXPKEY_HASHBITS 8 #define EXPKEY_HASHMAX (1 << EXPKEY_HASHBITS) #define EXPKEY_HASHMASK (EXPKEY_HASHMAX -1) -static void expkey_release(struct work_struct *work) +static void expkey_put(struct kref *ref) { - struct svc_expkey *key = container_of(to_rcu_work(work), - struct svc_expkey, ek_rwork); + struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); if (test_bit(CACHE_VALID, &key->h.flags) && !test_bit(CACHE_NEGATIVE, &key->h.flags)) path_put(&key->ek_path); auth_domain_put(key->ek_client); - kfree(key); -} - -static void expkey_put(struct kref *ref) -{ - struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); - - INIT_RCU_WORK(&key->ek_rwork, expkey_release); - queue_rcu_work(nfsd_export_wq, &key->ek_rwork); + kfree_rcu(key, ek_rcu); } static int expkey_upcall(struct cache_detail *cd, struct cache_head *h) { - return sunrpc_cache_pipe_upcall(cd, h); + return sunrpc_cache_upcall(cd, h); } static void expkey_request(struct cache_detail *cd, @@ -263,12 +255,18 @@ static void expkey_flush(void) mutex_unlock(&nfsd_mutex); } +static int expkey_notify(struct cache_detail *cd, struct cache_head *h) +{ + return nfsd_cache_notify(cd, h, NFSD_CACHE_TYPE_EXPKEY); +} + static const struct cache_detail svc_expkey_cache_template = { .owner = THIS_MODULE, .hash_size = EXPKEY_HASHMAX, .name = "nfsd.fh", .cache_put = expkey_put, .cache_upcall = expkey_upcall, + .cache_notify = expkey_notify, .cache_request = expkey_request, .cache_parse = expkey_parse, .cache_show = expkey_show, @@ -319,6 +317,266 @@ svc_expkey_update(struct cache_detail *cd, struct svc_expkey *new, return NULL; } +/** + * nfsd_nl_expkey_get_reqs_dumpit - dump pending expkey requests + * @skb: reply buffer + * @cb: netlink metadata and command arguments + * + * Walk the expkey cache's pending request list and create a netlink + * message with a nested entry for each cache_request, containing the + * seqno, client string, fsidtype and fsid. + * + * Uses cb->args[0] as a seqno cursor for dump continuation across + * multiple netlink messages. + * + * Returns the size of the reply or a negative errno. + */ +int nfsd_nl_expkey_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + struct cache_head **items; + u64 *seqnos; + int cnt, i, emitted; + void *hdr; + int ret; + + nn = net_generic(sock_net(skb->sk), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_expkey_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + cnt = sunrpc_cache_requests_count(cd); + if (!cnt) { + ret = 0; + goto out_unlock; + } + + items = kcalloc(cnt, sizeof(*items), GFP_KERNEL); + seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL); + if (!items || !seqnos) { + ret = -ENOMEM; + goto out_alloc; + } + + cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt, + cb->args[0]); + if (!cnt) { + ret = 0; + goto out_alloc; + } + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, + NLM_F_MULTI, NFSD_CMD_EXPKEY_GET_REQS); + if (!hdr) { + ret = -ENOBUFS; + goto out_put; + } + + emitted = 0; + for (i = 0; i < cnt; i++) { + struct svc_expkey *ek; + struct nlattr *nest; + + ek = container_of(items[i], struct svc_expkey, h); + + nest = nla_nest_start(skb, NFSD_A_EXPKEY_REQS_REQUESTS); + if (!nest) + break; + + if (nla_put_u64_64bit(skb, NFSD_A_EXPKEY_SEQNO, + seqnos[i], 0) || + nla_put_string(skb, NFSD_A_EXPKEY_CLIENT, + ek->ek_client->name) || + nla_put_u8(skb, NFSD_A_EXPKEY_FSIDTYPE, + ek->ek_fsidtype) || + nla_put(skb, NFSD_A_EXPKEY_FSID, + key_len(ek->ek_fsidtype), ek->ek_fsid)) { + nla_nest_cancel(skb, nest); + break; + } + + nla_nest_end(skb, nest); + cb->args[0] = seqnos[i]; + emitted++; + } + + if (!emitted) { + genlmsg_cancel(skb, hdr); + ret = -EMSGSIZE; + goto out_put; + } + + genlmsg_end(skb, hdr); + ret = skb->len; +out_put: + for (i = 0; i < cnt; i++) + cache_put(items[i], cd); +out_alloc: + kfree(seqnos); + kfree(items); +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; +} + +/** + * nfsd_nl_parse_one_expkey - parse one expkey entry from netlink + * @cd: cache_detail for the expkey cache + * @attr: nested attribute containing expkey fields + * + * Parses one expkey entry from a netlink message and updates the + * cache. Mirrors the logic in expkey_parse(). + * + * Returns 0 on success or a negative errno. + */ +static int nfsd_nl_parse_one_expkey(struct cache_detail *cd, + struct nlattr *attr) +{ + struct nlattr *tb[NFSD_A_EXPKEY_PATH + 1]; + struct auth_domain *dom = NULL; + struct svc_expkey key; + struct svc_expkey *ek = NULL; + struct timespec64 boot; + int err; + u8 fsidtype; + int fsid_len; + + err = nla_parse_nested(tb, NFSD_A_EXPKEY_PATH, attr, + nfsd_expkey_nl_policy, NULL); + if (err) + return err; + + /* client (required) */ + if (!tb[NFSD_A_EXPKEY_CLIENT]) + return -EINVAL; + + dom = auth_domain_find(nla_data(tb[NFSD_A_EXPKEY_CLIENT])); + if (!dom) + return -ENOENT; + + /* fsidtype (required) */ + if (!tb[NFSD_A_EXPKEY_FSIDTYPE]) { + err = -EINVAL; + goto out_dom; + } + fsidtype = nla_get_u8(tb[NFSD_A_EXPKEY_FSIDTYPE]); + if (key_len(fsidtype) == 0) { + err = -EINVAL; + goto out_dom; + } + + /* fsid (required) */ + if (!tb[NFSD_A_EXPKEY_FSID]) { + err = -EINVAL; + goto out_dom; + } + fsid_len = nla_len(tb[NFSD_A_EXPKEY_FSID]); + if (fsid_len != key_len(fsidtype)) { + err = -EINVAL; + goto out_dom; + } + + /* expiry (required, wallclock seconds) */ + if (!tb[NFSD_A_EXPKEY_EXPIRY]) { + err = -EINVAL; + goto out_dom; + } + + key.h.flags = 0; + getboottime64(&boot); + key.h.expiry_time = nla_get_u64(tb[NFSD_A_EXPKEY_EXPIRY]) - + boot.tv_sec; + key.ek_client = dom; + key.ek_fsidtype = fsidtype; + memcpy(key.ek_fsid, nla_data(tb[NFSD_A_EXPKEY_FSID]), fsid_len); + + ek = svc_expkey_lookup(cd, &key); + if (!ek) { + err = -ENOMEM; + goto out_dom; + } + + if (tb[NFSD_A_EXPKEY_NEGATIVE]) { + set_bit(CACHE_NEGATIVE, &key.h.flags); + ek = svc_expkey_update(cd, &key, ek); + if (ek) + trace_nfsd_expkey_update(ek, NULL); + else + err = -ENOMEM; + } else if (tb[NFSD_A_EXPKEY_PATH]) { + err = kern_path(nla_data(tb[NFSD_A_EXPKEY_PATH]), 0, + &key.ek_path); + if (err) + goto out_ek; + ek = svc_expkey_update(cd, &key, ek); + if (ek) + trace_nfsd_expkey_update(ek, + nla_data(tb[NFSD_A_EXPKEY_PATH])); + else + err = -ENOMEM; + path_put(&key.ek_path); + } else { + err = -EINVAL; + goto out_ek; + } + + cache_flush(); + +out_ek: + if (ek) + cache_put(&ek->h, cd); +out_dom: + auth_domain_put(dom); + return err; +} + +/** + * nfsd_nl_expkey_set_reqs_doit - respond to expkey requests + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Parse one or more expkey cache responses from userspace and + * update the expkey cache accordingly. + * + * Returns 0 on success or a negative errno. + */ +int nfsd_nl_expkey_set_reqs_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + const struct nlattr *attr; + int rem, ret = 0; + + nn = net_generic(genl_info_net(info), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_expkey_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + nlmsg_for_each_attr_type(attr, NFSD_A_EXPKEY_REQS_REQUESTS, + info->nlhdr, GENL_HDRLEN, rem) { + ret = nfsd_nl_parse_one_expkey(cd, (struct nlattr *)attr); + if (ret) + break; + } + +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; +} #define EXPORT_HASHBITS 8 #define EXPORT_HASHMAX (1<< EXPORT_HASHBITS) @@ -364,13 +622,11 @@ static void export_stats_destroy(struct export_stats *stats) EXP_STATS_COUNTERS_NUM); } -static void svc_export_release(struct work_struct *work) +static void svc_export_release(struct rcu_head *rcu_head) { - struct svc_export *exp = container_of(to_rcu_work(work), - struct svc_export, ex_rwork); + struct svc_export *exp = container_of(rcu_head, struct svc_export, + ex_rcu); - path_put(&exp->ex_path); - auth_domain_put(exp->ex_client); nfsd4_fslocs_free(&exp->ex_fslocs); export_stats_destroy(exp->ex_stats); kfree(exp->ex_stats); @@ -382,13 +638,450 @@ static void svc_export_put(struct kref *ref) { struct svc_export *exp = container_of(ref, struct svc_export, h.ref); - INIT_RCU_WORK(&exp->ex_rwork, svc_export_release); - queue_rcu_work(nfsd_export_wq, &exp->ex_rwork); + path_put(&exp->ex_path); + auth_domain_put(exp->ex_client); + call_rcu(&exp->ex_rcu, svc_export_release); +} + +/** + * nfsd_nl_svc_export_get_reqs_dumpit - dump pending svc_export requests + * @skb: reply buffer + * @cb: netlink metadata and command arguments + * + * Walk the svc_export cache's pending request list and create a netlink + * message with a nested entry for each cache_request, containing the + * seqno, client string, and path. + * + * Uses cb->args[0] as a seqno cursor for dump continuation across + * multiple netlink messages. + * + * Returns the size of the reply or a negative errno. + */ +int nfsd_nl_svc_export_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + struct cache_head **items; + u64 *seqnos; + int cnt, i, emitted; + char *pathbuf; + void *hdr; + int ret; + + nn = net_generic(sock_net(skb->sk), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_export_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + cnt = sunrpc_cache_requests_count(cd); + if (!cnt) { + ret = 0; + goto out_unlock; + } + + items = kcalloc(cnt, sizeof(*items), GFP_KERNEL); + seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL); + pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!items || !seqnos || !pathbuf) { + ret = -ENOMEM; + goto out_alloc; + } + + cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt, + cb->args[0]); + if (!cnt) { + ret = 0; + goto out_alloc; + } + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, + NLM_F_MULTI, NFSD_CMD_SVC_EXPORT_GET_REQS); + if (!hdr) { + ret = -ENOBUFS; + goto out_put; + } + + emitted = 0; + for (i = 0; i < cnt; i++) { + struct svc_export *exp; + struct nlattr *nest; + char *pth; + + exp = container_of(items[i], struct svc_export, h); + + pth = d_path(&exp->ex_path, pathbuf, PATH_MAX); + if (IS_ERR(pth)) + continue; + + nest = nla_nest_start(skb, + NFSD_A_SVC_EXPORT_REQS_REQUESTS); + if (!nest) + break; + + if (nla_put_u64_64bit(skb, NFSD_A_SVC_EXPORT_SEQNO, + seqnos[i], 0) || + nla_put_string(skb, NFSD_A_SVC_EXPORT_CLIENT, + exp->ex_client->name) || + nla_put_string(skb, NFSD_A_SVC_EXPORT_PATH, pth)) { + nla_nest_cancel(skb, nest); + break; + } + + nla_nest_end(skb, nest); + cb->args[0] = seqnos[i]; + emitted++; + } + + if (!emitted) { + genlmsg_cancel(skb, hdr); + ret = -EMSGSIZE; + goto out_put; + } + + genlmsg_end(skb, hdr); + ret = skb->len; +out_put: + for (i = 0; i < cnt; i++) + cache_put(items[i], cd); +out_alloc: + kfree(pathbuf); + kfree(seqnos); + kfree(items); +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; +} + +/** + * nfsd_nl_parse_fslocations - parse fslocations from netlink + * @attr: NFSD_A_SVC_EXPORT_FSLOCATIONS nested attribute + * @fsloc: fslocations struct to fill in + * + * Returns 0 on success or a negative errno. + */ +static int nfsd_nl_parse_fslocations(struct nlattr *attr, + struct nfsd4_fs_locations *fsloc) +{ + struct nlattr *loc_attr; + int rem, count = 0; + int err; + + if (fsloc->locations) + return -EINVAL; + + /* Count locations first */ + nla_for_each_nested_type(loc_attr, NFSD_A_FSLOCATIONS_LOCATION, + attr, rem) + count++; + + if (count > MAX_FS_LOCATIONS) + return -EINVAL; + if (!count) + return 0; + + fsloc->locations = kcalloc(count, sizeof(struct nfsd4_fs_location), + GFP_KERNEL); + if (!fsloc->locations) + return -ENOMEM; + + nla_for_each_nested_type(loc_attr, NFSD_A_FSLOCATIONS_LOCATION, + attr, rem) { + struct nlattr *tb[NFSD_A_FSLOCATION_PATH + 1]; + struct nfsd4_fs_location *loc; + + err = nla_parse_nested(tb, NFSD_A_FSLOCATION_PATH, loc_attr, + nfsd_fslocation_nl_policy, NULL); + if (err) + goto out_free; + + if (!tb[NFSD_A_FSLOCATION_HOST] || + !tb[NFSD_A_FSLOCATION_PATH]) { + err = -EINVAL; + goto out_free; + } + + loc = &fsloc->locations[fsloc->locations_count++]; + loc->hosts = kstrdup(nla_data(tb[NFSD_A_FSLOCATION_HOST]), + GFP_KERNEL); + loc->path = kstrdup(nla_data(tb[NFSD_A_FSLOCATION_PATH]), + GFP_KERNEL); + if (!loc->hosts || !loc->path) { + err = -ENOMEM; + goto out_free; + } + } + + return 0; +out_free: + nfsd4_fslocs_free(fsloc); + return err; +} + +static struct svc_export *svc_export_update(struct svc_export *new, + struct svc_export *old); +static struct svc_export *svc_export_lookup(struct svc_export *); +static int check_export(const struct path *path, int *flags, + unsigned char *uuid); + +/** + * nfsd_nl_parse_one_export - parse one svc_export entry from a netlink message + * @cd: cache_detail for the svc_export cache + * @attr: nested attribute containing svc-export fields + * + * Parses one svc-export entry from a netlink message and updates the + * cache. Mirrors the logic in svc_export_parse(). + * + * Returns 0 on success or a negative errno. + */ +static int nfsd_nl_parse_one_export(struct cache_detail *cd, + struct nlattr *attr) +{ + struct nlattr *tb[NFSD_A_SVC_EXPORT_FSID + 1]; + struct auth_domain *dom = NULL; + struct svc_export exp = {}, *expp; + struct nlattr *secinfo_attr; + struct timespec64 boot; + int err, rem; + + err = nla_parse_nested(tb, NFSD_A_SVC_EXPORT_FSID, attr, + nfsd_svc_export_nl_policy, NULL); + if (err) + return err; + + /* client (required) */ + if (!tb[NFSD_A_SVC_EXPORT_CLIENT]) + return -EINVAL; + + dom = auth_domain_find(nla_data(tb[NFSD_A_SVC_EXPORT_CLIENT])); + if (!dom) + return -ENOENT; + + /* path (required) */ + if (!tb[NFSD_A_SVC_EXPORT_PATH]) { + err = -EINVAL; + goto out_dom; + } + + err = kern_path(nla_data(tb[NFSD_A_SVC_EXPORT_PATH]), 0, + &exp.ex_path); + if (err) + goto out_dom; + + exp.ex_client = dom; + exp.cd = cd; + exp.ex_devid_map = NULL; + exp.ex_xprtsec_modes = NFSEXP_XPRTSEC_ALL; + + /* expiry (required, wallclock seconds) */ + if (!tb[NFSD_A_SVC_EXPORT_EXPIRY]) { + err = -EINVAL; + goto out_path; + } + getboottime64(&boot); + exp.h.expiry_time = nla_get_u64(tb[NFSD_A_SVC_EXPORT_EXPIRY]) - + boot.tv_sec; + + if (tb[NFSD_A_SVC_EXPORT_NEGATIVE]) { + set_bit(CACHE_NEGATIVE, &exp.h.flags); + } else { + /* flags */ + if (tb[NFSD_A_SVC_EXPORT_FLAGS]) + exp.ex_flags = nla_get_u32(tb[NFSD_A_SVC_EXPORT_FLAGS]); + + /* anon uid */ + if (tb[NFSD_A_SVC_EXPORT_ANON_UID]) { + u32 uid = nla_get_u32(tb[NFSD_A_SVC_EXPORT_ANON_UID]); + + exp.ex_anon_uid = make_kuid(current_user_ns(), uid); + } + + /* anon gid */ + if (tb[NFSD_A_SVC_EXPORT_ANON_GID]) { + u32 gid = nla_get_u32(tb[NFSD_A_SVC_EXPORT_ANON_GID]); + + exp.ex_anon_gid = make_kgid(current_user_ns(), gid); + } + + /* fsid */ + if (tb[NFSD_A_SVC_EXPORT_FSID]) + exp.ex_fsid = nla_get_s32(tb[NFSD_A_SVC_EXPORT_FSID]); + + /* fslocations */ + if (tb[NFSD_A_SVC_EXPORT_FSLOCATIONS]) { + struct nlattr *fsl = tb[NFSD_A_SVC_EXPORT_FSLOCATIONS]; + + err = nfsd_nl_parse_fslocations(fsl, + &exp.ex_fslocs); + if (err) + goto out_path; + } + + /* uuid */ + if (tb[NFSD_A_SVC_EXPORT_UUID]) { + if (nla_len(tb[NFSD_A_SVC_EXPORT_UUID]) != + EX_UUID_LEN) { + err = -EINVAL; + goto out_fslocs; + } + exp.ex_uuid = kmemdup(nla_data(tb[NFSD_A_SVC_EXPORT_UUID]), + EX_UUID_LEN, GFP_KERNEL); + if (!exp.ex_uuid) { + err = -ENOMEM; + goto out_fslocs; + } + } + + /* secinfo (multi-attr) */ + nla_for_each_nested_type(secinfo_attr, + NFSD_A_SVC_EXPORT_SECINFO, + attr, rem) { + struct nlattr *ftb[NFSD_A_AUTH_FLAVOR_FLAGS + 1]; + struct exp_flavor_info *f; + + if (exp.ex_nflavors >= MAX_SECINFO_LIST) { + err = -EINVAL; + goto out_uuid; + } + + err = nla_parse_nested(ftb, + NFSD_A_AUTH_FLAVOR_FLAGS, + secinfo_attr, + nfsd_auth_flavor_nl_policy, + NULL); + if (err) + goto out_uuid; + + f = &exp.ex_flavors[exp.ex_nflavors++]; + + if (ftb[NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR]) + f->pseudoflavor = nla_get_u32(ftb[NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR]); + + if (ftb[NFSD_A_AUTH_FLAVOR_FLAGS]) + f->flags = nla_get_u32(ftb[NFSD_A_AUTH_FLAVOR_FLAGS]); + + /* Only some flags are allowed to differ between flavors: */ + if (~NFSEXP_SECINFO_FLAGS & (f->flags ^ exp.ex_flags)) { + err = -EINVAL; + goto out_uuid; + } + } + + /* xprtsec (multi-attr u32) */ + if (tb[NFSD_A_SVC_EXPORT_XPRTSEC]) { + struct nlattr *xp_attr; + + exp.ex_xprtsec_modes = 0; + nla_for_each_nested_type(xp_attr, + NFSD_A_SVC_EXPORT_XPRTSEC, + attr, rem) { + u32 mode = nla_get_u32(xp_attr); + + if (mode > NFSEXP_XPRTSEC_MTLS) { + err = -EINVAL; + goto out_uuid; + } + exp.ex_xprtsec_modes |= mode; + } + } + + err = check_export(&exp.ex_path, &exp.ex_flags, + exp.ex_uuid); + if (err) + goto out_uuid; + + if (exp.h.expiry_time < seconds_since_boot()) + goto out_uuid; + + err = -EINVAL; + if (!uid_valid(exp.ex_anon_uid)) + goto out_uuid; + if (!gid_valid(exp.ex_anon_gid)) + goto out_uuid; + err = 0; + + nfsd4_setup_layout_type(&exp); + } + + expp = svc_export_lookup(&exp); + if (!expp) { + err = -ENOMEM; + goto out_uuid; + } + expp = svc_export_update(&exp, expp); + if (expp) { + trace_nfsd_export_update(expp); + cache_flush(); + exp_put(expp); + } else { + err = -ENOMEM; + } + +out_uuid: + kfree(exp.ex_uuid); +out_fslocs: + nfsd4_fslocs_free(&exp.ex_fslocs); +out_path: + path_put(&exp.ex_path); +out_dom: + auth_domain_put(dom); + return err; +} + +/** + * nfsd_nl_svc_export_set_reqs_doit - respond to svc_export requests + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Parse one or more svc_export cache responses from userspace and + * update the export cache accordingly. + * + * Returns 0 on success or a negative errno. + */ +int nfsd_nl_svc_export_set_reqs_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + const struct nlattr *attr; + int rem, ret = 0; + + nn = net_generic(genl_info_net(info), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_export_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + nlmsg_for_each_attr_type(attr, NFSD_A_SVC_EXPORT_REQS_REQUESTS, + info->nlhdr, GENL_HDRLEN, rem) { + ret = nfsd_nl_parse_one_export(cd, (struct nlattr *)attr); + if (ret) + break; + } + +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; } static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h) { - return sunrpc_cache_pipe_upcall(cd, h); + return sunrpc_cache_upcall(cd, h); +} + +static int svc_export_notify(struct cache_detail *cd, struct cache_head *h) +{ + return nfsd_cache_notify(cd, h, NFSD_CACHE_TYPE_SVC_EXPORT); } static void svc_export_request(struct cache_detail *cd, @@ -410,10 +1103,6 @@ static void svc_export_request(struct cache_detail *cd, (*bpp)[-1] = '\n'; } -static struct svc_export *svc_export_update(struct svc_export *new, - struct svc_export *old); -static struct svc_export *svc_export_lookup(struct svc_export *); - static int check_export(const struct path *path, int *flags, unsigned char *uuid) { struct inode *inode = d_inode(path->dentry); @@ -735,7 +1424,8 @@ static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen) goto out4; err = 0; - nfsd4_setup_layout_type(&exp); + if (exp.ex_flags & NFSEXP_PNFS) + nfsd4_setup_layout_type(&exp); } expp = svc_export_lookup(&exp); @@ -907,6 +1597,7 @@ static const struct cache_detail svc_export_cache_template = { .name = "nfsd.export", .cache_put = svc_export_put, .cache_upcall = svc_export_upcall, + .cache_notify = svc_export_notify, .cache_request = svc_export_request, .cache_parse = svc_export_parse, .cache_show = svc_export_show, @@ -1492,36 +2183,6 @@ const struct seq_operations nfs_exports_op = { .show = e_show, }; -/** - * nfsd_export_wq_init - allocate the export release workqueue - * - * Called once at module load. The workqueue runs deferred svc_export and - * svc_expkey release work scheduled by queue_rcu_work() in the cache put - * callbacks. - * - * Return values: - * %0: workqueue allocated - * %-ENOMEM: allocation failed - */ -int nfsd_export_wq_init(void) -{ - nfsd_export_wq = alloc_workqueue("nfsd_export", WQ_UNBOUND, 0); - if (!nfsd_export_wq) - return -ENOMEM; - return 0; -} - -/** - * nfsd_export_wq_shutdown - drain and free the export release workqueue - * - * Called once at module unload. Per-namespace teardown in - * nfsd_export_shutdown() has already drained all deferred work. - */ -void nfsd_export_wq_shutdown(void) -{ - destroy_workqueue(nfsd_export_wq); -} - /* * Initialize the exports module. */ @@ -1583,9 +2244,6 @@ nfsd_export_shutdown(struct net *net) cache_unregister_net(nn->svc_expkey_cache, net); cache_unregister_net(nn->svc_export_cache, net); - /* Drain deferred export and expkey release work. */ - rcu_barrier(); - flush_workqueue(nfsd_export_wq); cache_destroy_net(nn->svc_expkey_cache, net); cache_destroy_net(nn->svc_export_cache, net); svcauth_unix_purge(net); diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index b05399374574..d2b09cd76145 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -7,7 +7,6 @@ #include <linux/sunrpc/cache.h> #include <linux/percpu_counter.h> -#include <linux/workqueue.h> #include <uapi/linux/nfsd/export.h> #include <linux/nfs4.h> @@ -76,7 +75,7 @@ struct svc_export { u32 ex_layout_types; struct nfsd4_deviceid_map *ex_devid_map; struct cache_detail *cd; - struct rcu_work ex_rwork; + struct rcu_head ex_rcu; unsigned long ex_xprtsec_modes; struct export_stats *ex_stats; }; @@ -93,7 +92,7 @@ struct svc_expkey { u32 ek_fsid[6]; struct path ek_path; - struct rcu_work ek_rwork; + struct rcu_head ek_rcu; }; #define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC)) @@ -111,8 +110,6 @@ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, /* * Function declarations */ -int nfsd_export_wq_init(void); -void nfsd_export_wq_shutdown(void); int nfsd_export_init(struct net *); void nfsd_export_shutdown(struct net *); void nfsd_export_flush(struct net *); diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 1e2b38ed1d35..24511c3208db 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -724,6 +724,52 @@ nfsd_file_close_inode_sync(struct inode *inode) nfsd_file_dispose_list(&dispose); } +/** + * nfsd_file_close_export - close cached file handles for an export + * @net: net namespace in which to operate + * @path: export path whose cached files should be closed + * + * Close out GC-managed nfsd_file entries whose underlying file is on + * the same filesystem as, and a descendant of, @path. nfsd_file + * entries do not carry an export reference, so the check uses the + * file's dentry ancestry. False positives (closing a cached handle + * that did not originate from the target export) are harmless -- the + * handle is simply reopened on the next access. + * + * Called from the NFSD_CMD_UNLOCK_EXPORT handler before revoking + * NFSv4 state, to ensure that cached file handles do not hold the + * filesystem busy. + */ +void nfsd_file_close_export(struct net *net, const struct path *path) +{ + struct rhashtable_iter iter; + struct nfsd_file *nf; + LIST_HEAD(dispose); + + rhltable_walk_enter(&nfsd_file_rhltable, &iter); + do { + rhashtable_walk_start(&iter); + + nf = rhashtable_walk_next(&iter); + while (!IS_ERR_OR_NULL(nf)) { + if (nf->nf_net == net && + test_bit(NFSD_FILE_GC, &nf->nf_flags) && + nf->nf_file && + file_inode(nf->nf_file)->i_sb == + path->dentry->d_sb && + is_subdir(nf->nf_file->f_path.dentry, + path->dentry)) + nfsd_file_cond_queue(nf, &dispose); + nf = rhashtable_walk_next(&iter); + } + + rhashtable_walk_stop(&iter); + } while (nf == ERR_PTR(-EAGAIN)); + rhashtable_walk_exit(&iter); + + nfsd_file_dispose_list(&dispose); +} + static int nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg, void *data) diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h index b383dbc5b921..683b6437cacc 100644 --- a/fs/nfsd/filecache.h +++ b/fs/nfsd/filecache.h @@ -70,6 +70,7 @@ struct net *nfsd_file_put_local(struct nfsd_file __rcu **nf); struct nfsd_file *nfsd_file_get(struct nfsd_file *nf); struct file *nfsd_file_file(struct nfsd_file *nf); void nfsd_file_close_inode_sync(struct inode *inode); +void nfsd_file_close_export(struct net *net, const struct path *path); void nfsd_file_net_dispose(struct nfsd_net *nn); bool nfsd_file_is_cached(struct inode *inode); __be32 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp, diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c index 81c943345d13..fbee3676d253 100644 --- a/fs/nfsd/netlink.c +++ b/fs/nfsd/netlink.c @@ -12,11 +12,51 @@ #include <uapi/linux/nfsd_netlink.h> /* Common nested types */ +const struct nla_policy nfsd_auth_flavor_nl_policy[NFSD_A_AUTH_FLAVOR_FLAGS + 1] = { + [NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR] = { .type = NLA_U32, }, + [NFSD_A_AUTH_FLAVOR_FLAGS] = NLA_POLICY_MASK(NLA_U32, 0x3ffff), +}; + +const struct nla_policy nfsd_expkey_nl_policy[NFSD_A_EXPKEY_PATH + 1] = { + [NFSD_A_EXPKEY_SEQNO] = { .type = NLA_U64, }, + [NFSD_A_EXPKEY_CLIENT] = { .type = NLA_NUL_STRING, }, + [NFSD_A_EXPKEY_FSIDTYPE] = { .type = NLA_U8, }, + [NFSD_A_EXPKEY_FSID] = { .type = NLA_BINARY, }, + [NFSD_A_EXPKEY_NEGATIVE] = { .type = NLA_FLAG, }, + [NFSD_A_EXPKEY_EXPIRY] = { .type = NLA_U64, }, + [NFSD_A_EXPKEY_PATH] = { .type = NLA_NUL_STRING, }, +}; + +const struct nla_policy nfsd_fslocation_nl_policy[NFSD_A_FSLOCATION_PATH + 1] = { + [NFSD_A_FSLOCATION_HOST] = { .type = NLA_NUL_STRING, }, + [NFSD_A_FSLOCATION_PATH] = { .type = NLA_NUL_STRING, }, +}; + +const struct nla_policy nfsd_fslocations_nl_policy[NFSD_A_FSLOCATIONS_LOCATION + 1] = { + [NFSD_A_FSLOCATIONS_LOCATION] = NLA_POLICY_NESTED(nfsd_fslocation_nl_policy), +}; + const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1] = { [NFSD_A_SOCK_ADDR] = { .type = NLA_BINARY, }, [NFSD_A_SOCK_TRANSPORT_NAME] = { .type = NLA_NUL_STRING, }, }; +const struct nla_policy nfsd_svc_export_nl_policy[NFSD_A_SVC_EXPORT_FSID + 1] = { + [NFSD_A_SVC_EXPORT_SEQNO] = { .type = NLA_U64, }, + [NFSD_A_SVC_EXPORT_CLIENT] = { .type = NLA_NUL_STRING, }, + [NFSD_A_SVC_EXPORT_PATH] = { .type = NLA_NUL_STRING, }, + [NFSD_A_SVC_EXPORT_NEGATIVE] = { .type = NLA_FLAG, }, + [NFSD_A_SVC_EXPORT_EXPIRY] = { .type = NLA_U64, }, + [NFSD_A_SVC_EXPORT_ANON_UID] = { .type = NLA_U32, }, + [NFSD_A_SVC_EXPORT_ANON_GID] = { .type = NLA_U32, }, + [NFSD_A_SVC_EXPORT_FSLOCATIONS] = NLA_POLICY_NESTED(nfsd_fslocations_nl_policy), + [NFSD_A_SVC_EXPORT_UUID] = { .type = NLA_BINARY, }, + [NFSD_A_SVC_EXPORT_SECINFO] = NLA_POLICY_NESTED(nfsd_auth_flavor_nl_policy), + [NFSD_A_SVC_EXPORT_XPRTSEC] = NLA_POLICY_MASK(NLA_U32, 0x7), + [NFSD_A_SVC_EXPORT_FLAGS] = NLA_POLICY_MASK(NLA_U32, 0x3ffff), + [NFSD_A_SVC_EXPORT_FSID] = { .type = NLA_S32, }, +}; + const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1] = { [NFSD_A_VERSION_MAJOR] = { .type = NLA_U32, }, [NFSD_A_VERSION_MINOR] = { .type = NLA_U32, }, @@ -48,6 +88,36 @@ static const struct nla_policy nfsd_pool_mode_set_nl_policy[NFSD_A_POOL_MODE_MOD [NFSD_A_POOL_MODE_MODE] = { .type = NLA_NUL_STRING, }, }; +/* NFSD_CMD_SVC_EXPORT_SET_REQS - do */ +static const struct nla_policy nfsd_svc_export_set_reqs_nl_policy[NFSD_A_SVC_EXPORT_REQS_REQUESTS + 1] = { + [NFSD_A_SVC_EXPORT_REQS_REQUESTS] = NLA_POLICY_NESTED(nfsd_svc_export_nl_policy), +}; + +/* NFSD_CMD_EXPKEY_SET_REQS - do */ +static const struct nla_policy nfsd_expkey_set_reqs_nl_policy[NFSD_A_EXPKEY_REQS_REQUESTS + 1] = { + [NFSD_A_EXPKEY_REQS_REQUESTS] = NLA_POLICY_NESTED(nfsd_expkey_nl_policy), +}; + +/* NFSD_CMD_CACHE_FLUSH - do */ +static const struct nla_policy nfsd_cache_flush_nl_policy[NFSD_A_CACHE_FLUSH_MASK + 1] = { + [NFSD_A_CACHE_FLUSH_MASK] = NLA_POLICY_MASK(NLA_U32, 0x3), +}; + +/* NFSD_CMD_UNLOCK_IP - do */ +static const struct nla_policy nfsd_unlock_ip_nl_policy[NFSD_A_UNLOCK_IP_ADDRESS + 1] = { + [NFSD_A_UNLOCK_IP_ADDRESS] = NLA_POLICY_MIN_LEN(16), +}; + +/* NFSD_CMD_UNLOCK_FILESYSTEM - do */ +static const struct nla_policy nfsd_unlock_filesystem_nl_policy[NFSD_A_UNLOCK_FILESYSTEM_PATH + 1] = { + [NFSD_A_UNLOCK_FILESYSTEM_PATH] = { .type = NLA_NUL_STRING, }, +}; + +/* NFSD_CMD_UNLOCK_EXPORT - do */ +static const struct nla_policy nfsd_unlock_export_nl_policy[NFSD_A_UNLOCK_EXPORT_PATH + 1] = { + [NFSD_A_UNLOCK_EXPORT_PATH] = { .type = NLA_NUL_STRING, }, +}; + /* Ops table for nfsd */ static const struct genl_split_ops nfsd_nl_ops[] = { { @@ -103,6 +173,63 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .doit = nfsd_nl_pool_mode_get_doit, .flags = GENL_CMD_CAP_DO, }, + { + .cmd = NFSD_CMD_SVC_EXPORT_GET_REQS, + .dumpit = nfsd_nl_svc_export_get_reqs_dumpit, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP, + }, + { + .cmd = NFSD_CMD_SVC_EXPORT_SET_REQS, + .doit = nfsd_nl_svc_export_set_reqs_doit, + .policy = nfsd_svc_export_set_reqs_nl_policy, + .maxattr = NFSD_A_SVC_EXPORT_REQS_REQUESTS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_EXPKEY_GET_REQS, + .dumpit = nfsd_nl_expkey_get_reqs_dumpit, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP, + }, + { + .cmd = NFSD_CMD_EXPKEY_SET_REQS, + .doit = nfsd_nl_expkey_set_reqs_doit, + .policy = nfsd_expkey_set_reqs_nl_policy, + .maxattr = NFSD_A_EXPKEY_REQS_REQUESTS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_CACHE_FLUSH, + .doit = nfsd_nl_cache_flush_doit, + .policy = nfsd_cache_flush_nl_policy, + .maxattr = NFSD_A_CACHE_FLUSH_MASK, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_UNLOCK_IP, + .doit = nfsd_nl_unlock_ip_doit, + .policy = nfsd_unlock_ip_nl_policy, + .maxattr = NFSD_A_UNLOCK_IP_ADDRESS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_UNLOCK_FILESYSTEM, + .doit = nfsd_nl_unlock_filesystem_doit, + .policy = nfsd_unlock_filesystem_nl_policy, + .maxattr = NFSD_A_UNLOCK_FILESYSTEM_PATH, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_UNLOCK_EXPORT, + .doit = nfsd_nl_unlock_export_doit, + .policy = nfsd_unlock_export_nl_policy, + .maxattr = NFSD_A_UNLOCK_EXPORT_PATH, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, +}; + +static const struct genl_multicast_group nfsd_nl_mcgrps[] = { + [NFSD_NLGRP_NONE] = { "none", }, + [NFSD_NLGRP_EXPORTD] = { "exportd", }, }; struct genl_family nfsd_nl_family __ro_after_init = { @@ -113,4 +240,6 @@ struct genl_family nfsd_nl_family __ro_after_init = { .module = THIS_MODULE, .split_ops = nfsd_nl_ops, .n_split_ops = ARRAY_SIZE(nfsd_nl_ops), + .mcgrps = nfsd_nl_mcgrps, + .n_mcgrps = ARRAY_SIZE(nfsd_nl_mcgrps), }; diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h index 478117ff6b8c..af41aa0d4a65 100644 --- a/fs/nfsd/netlink.h +++ b/fs/nfsd/netlink.h @@ -13,7 +13,12 @@ #include <uapi/linux/nfsd_netlink.h> /* Common nested types */ +extern const struct nla_policy nfsd_auth_flavor_nl_policy[NFSD_A_AUTH_FLAVOR_FLAGS + 1]; +extern const struct nla_policy nfsd_expkey_nl_policy[NFSD_A_EXPKEY_PATH + 1]; +extern const struct nla_policy nfsd_fslocation_nl_policy[NFSD_A_FSLOCATION_PATH + 1]; +extern const struct nla_policy nfsd_fslocations_nl_policy[NFSD_A_FSLOCATIONS_LOCATION + 1]; extern const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1]; +extern const struct nla_policy nfsd_svc_export_nl_policy[NFSD_A_SVC_EXPORT_FSID + 1]; extern const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1]; int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, @@ -26,6 +31,22 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_listener_get_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_pool_mode_set_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_pool_mode_get_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_svc_export_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int nfsd_nl_svc_export_set_reqs_doit(struct sk_buff *skb, + struct genl_info *info); +int nfsd_nl_expkey_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int nfsd_nl_expkey_set_reqs_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_unlock_ip_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_unlock_export_doit(struct sk_buff *skb, struct genl_info *info); + +enum { + NFSD_NLGRP_NONE, + NFSD_NLGRP_EXPORTD, +}; extern struct genl_family nfsd_nl_family; diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 0ac538c76180..76305b86c1a9 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -131,10 +131,7 @@ static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp) resp->status = fh_getattr(fh, &resp->stat); out: - /* argp->acl_{access,default} may have been allocated in - nfssvc_decode_setaclargs. */ - posix_acl_release(argp->acl_access); - posix_acl_release(argp->acl_default); + /* argp->acl_{access,default} are released in nfsaclsvc_release_setacl. */ return rpc_success; out_drop_lock: @@ -310,6 +307,16 @@ static void nfsaclsvc_release_access(struct svc_rqst *rqstp) fh_put(&resp->fh); } +static void nfsaclsvc_release_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(argp->acl_access); + posix_acl_release(argp->acl_default); +} + #define ST 1 /* status*/ #define AT 21 /* attributes */ #define pAT (1+AT) /* post attributes - conditional */ @@ -343,7 +350,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_func = nfsacld_proc_setacl, .pc_decode = nfsaclsvc_decode_setaclargs, .pc_encode = nfssvc_encode_attrstatres, - .pc_release = nfssvc_release_attrstat, + .pc_release = nfsaclsvc_release_setacl, .pc_argsize = sizeof(struct nfsd3_setaclargs), .pc_argzero = sizeof(struct nfsd3_setaclargs), .pc_ressize = sizeof(struct nfsd_attrstat), diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index 7b5433bd3019..e87731380be8 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -118,10 +118,7 @@ out_drop_lock: out_errno: resp->status = nfserrno(error); out: - /* argp->acl_{access,default} may have been allocated in - nfs3svc_decode_setaclargs. */ - posix_acl_release(argp->acl_access); - posix_acl_release(argp->acl_default); + /* argp->acl_{access,default} are released in nfs3svc_release_setacl. */ return rpc_success; } @@ -223,6 +220,16 @@ static void nfs3svc_release_getacl(struct svc_rqst *rqstp) posix_acl_release(resp->acl_default); } +static void nfs3svc_release_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(argp->acl_access); + posix_acl_release(argp->acl_default); +} + #define ST 1 /* status*/ #define AT 21 /* attributes */ #define pAT (1+AT) /* post attributes - conditional */ @@ -256,7 +263,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_func = nfsd3_proc_setacl, .pc_decode = nfs3svc_decode_setaclargs, .pc_encode = nfs3svc_encode_setaclres, - .pc_release = nfs3svc_release_fhandle, + .pc_release = nfs3svc_release_setacl, .pc_argsize = sizeof(struct nfsd3_setaclargs), .pc_argzero = sizeof(struct nfsd3_setaclargs), .pc_ressize = sizeof(struct nfsd3_attrstat), diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index 42adc5461db0..aeda7a802bdf 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -710,23 +710,46 @@ nfsd3_proc_pathconf(struct svc_rqst *rqstp) resp->p_name_max = 255; /* at least */ resp->p_no_trunc = 0; resp->p_chown_restricted = 1; - resp->p_case_insensitive = 0; - resp->p_case_preserving = 1; + resp->p_case_insensitive = false; + resp->p_case_preserving = true; resp->status = fh_verify(rqstp, &argp->fh, 0, NFSD_MAY_NOP); if (resp->status == nfs_ok) { struct super_block *sb = argp->fh.fh_dentry->d_sb; + int err; - /* Note that we don't care for remote fs's here */ - switch (sb->s_magic) { - case EXT2_SUPER_MAGIC: + if (sb->s_magic == EXT2_SUPER_MAGIC) { resp->p_link_max = EXT2_LINK_MAX; resp->p_name_max = EXT2_NAME_LEN; + } + + err = nfsd_get_case_info(argp->fh.fh_dentry, + &resp->p_case_insensitive, + &resp->p_case_preserving); + /* + * RFC 1813 lists NFS3ERR_STALE, NFS3ERR_BADHANDLE, and + * NFS3ERR_SERVERFAULT as the only PATHCONF errors. + */ + switch (err) { + case 0: + case -EOPNOTSUPP: + /* Both arms leave the output booleans valid. */ + break; + case -EACCES: + case -EPERM: + /* + * Policy denied the query. Report STALE so the + * handle is unusable without implying a server + * malfunction. + */ + resp->status = nfserr_stale; + break; + case -ESTALE: + resp->status = nfserr_stale; break; - case MSDOS_SUPER_MAGIC: - resp->p_case_insensitive = 1; - resp->p_case_preserving = 0; + default: + resp->status = nfserr_serverfault; break; } } diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index ba06d3d3e6dd..71ba61b5d0a3 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -126,7 +126,7 @@ idtoname_hash(struct ent *ent) static int idtoname_upcall(struct cache_detail *cd, struct cache_head *h) { - return sunrpc_cache_pipe_upcall_timeout(cd, h); + return sunrpc_cache_upcall_warn(cd, h); } static void @@ -306,7 +306,7 @@ nametoid_hash(struct ent *ent) static int nametoid_upcall(struct cache_detail *cd, struct cache_head *h) { - return sunrpc_cache_pipe_upcall_timeout(cd, h); + return sunrpc_cache_upcall_warn(cd, h); } static void diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index 69e41105efdd..f34320e4c2f4 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2014 Christoph Hellwig. */ -#include <linux/blkdev.h> +#include <linux/exportfs_block.h> #include <linux/kmod.h> #include <linux/file.h> #include <linux/jhash.h> @@ -127,30 +127,17 @@ nfsd4_set_deviceid(struct nfsd4_deviceid *id, const struct svc_fh *fhp, void nfsd4_setup_layout_type(struct svc_export *exp) { -#if defined(CONFIG_NFSD_BLOCKLAYOUT) || defined(CONFIG_NFSD_SCSILAYOUT) struct super_block *sb = exp->ex_path.mnt->mnt_sb; -#endif - - if (!(exp->ex_flags & NFSEXP_PNFS)) - return; + expfs_block_layouts_t block_supported = exportfs_layouts_supported(sb); -#ifdef CONFIG_NFSD_FLEXFILELAYOUT - exp->ex_layout_types |= 1 << LAYOUT_FLEX_FILES; -#endif -#ifdef CONFIG_NFSD_BLOCKLAYOUT - if (sb->s_export_op->get_uuid && - sb->s_export_op->map_blocks && - sb->s_export_op->commit_blocks) + if (IS_ENABLED(CONFIG_NFSD_FLEXFILELAYOUT)) + exp->ex_layout_types |= 1 << LAYOUT_FLEX_FILES; + if (IS_ENABLED(CONFIG_NFSD_BLOCKLAYOUT) && + (block_supported & EXPFS_BLOCK_IN_BAND_ID)) exp->ex_layout_types |= 1 << LAYOUT_BLOCK_VOLUME; -#endif -#ifdef CONFIG_NFSD_SCSILAYOUT - if (sb->s_export_op->map_blocks && - sb->s_export_op->commit_blocks && - sb->s_bdev && - sb->s_bdev->bd_disk->fops->pr_ops && - sb->s_bdev->bd_disk->fops->get_unique_id) + if (IS_ENABLED(CONFIG_NFSD_SCSILAYOUT) && + (block_supported & EXPFS_BLOCK_OUT_OF_BAND_ID)) exp->ex_layout_types |= 1 << LAYOUT_SCSI; -#endif } void nfsd4_close_layout(struct nfs4_layout_stateid *ls) @@ -247,6 +234,8 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, get_nfs4_file(fp); stp->sc_file = fp; + if (parent->sc_export) + stp->sc_export = exp_get(parent->sc_export); ls = layoutstateid(stp); INIT_LIST_HEAD(&ls->ls_perclnt); @@ -264,10 +253,12 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, ls->ls_file = find_any_file(fp); BUG_ON(!ls->ls_file); + ls->ls_fenced = false; + ls->ls_fence_delay = 0; + INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker); + if (nfsd4_layout_setlease(ls)) { - nfsd_file_put(ls->ls_file); - put_nfs4_file(fp); - kmem_cache_free(nfs4_layout_stateid_cache, ls); + nfs4_put_stid(stp); return NULL; } @@ -280,10 +271,6 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, list_add(&ls->ls_perfile, &fp->fi_lo_states); spin_unlock(&fp->fi_lock); - ls->ls_fenced = false; - ls->ls_fence_delay = 0; - INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker); - trace_nfsd_layoutstate_alloc(&ls->ls_stid.sc_stateid); return ls; } diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 85e94c30285a..8561540ab2db 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -253,7 +253,7 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, .na_iattr = iap, .na_seclabel = &open->op_label, }; - struct dentry *parent, *child; + struct dentry *parent, *child = ERR_PTR(-EINVAL); __u32 v_mtime, v_atime; struct inode *inode; __be32 status; @@ -277,10 +277,14 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, if (open->op_acl) { if (open->op_dpacl || open->op_pacl) { status = nfserr_inval; - goto out_write; + goto out; + } + if (is_create_with_attrs(open)) { + status = nfsd4_acl_to_attr(NF4REG, open->op_acl, + &attrs); + if (status) + goto out; } - if (is_create_with_attrs(open)) - nfsd4_acl_to_attr(NF4REG, open->op_acl, &attrs); } else if (is_create_with_attrs(open)) { /* The dpacl and pacl will get released by nfsd_attrs_free(). */ attrs.na_dpacl = open->op_dpacl; @@ -293,7 +297,7 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, &QSTR_LEN(open->op_fname, open->op_fnamelen)); if (IS_ERR(child)) { status = nfserrno(PTR_ERR(child)); - goto out_write; + goto out; } if (d_really_is_negative(child)) { @@ -407,7 +411,6 @@ set_attr: out: end_creating(child); nfsd_attrs_free(&attrs); -out_write: fh_drop_write(fhp); return status; } @@ -837,7 +840,7 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out_aftermask; if (create->cr_acl) { - if (create->cr_dpacl || create->cr_pacl) { + if (attrs.na_dpacl || attrs.na_pacl) { status = nfserr_inval; goto out_aftermask; } @@ -1413,6 +1416,9 @@ nfsd4_clone(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, dst, clone->cl_dst_pos, clone->cl_count, EX_ISSYNC(cstate->current_fh.fh_export)); + if (!status && (READ_ONCE(dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0) + nfsd_update_cmtime_attr(dst->nf_file, 0); + nfsd_file_put(dst); nfsd_file_put(src); out: @@ -1467,7 +1473,7 @@ void nfsd4_async_copy_reaper(struct nfsd_net *nn) list_for_each_safe(pos, next, &clp->async_copies) { copy = list_entry(pos, struct nfsd4_copy, copies); if (test_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags)) { - if (--copy->cp_ttl) { + if (!--copy->cp_ttl) { list_del_init(©->copies); list_add(©->copies, &reaplist); } @@ -2118,8 +2124,10 @@ do_callback: set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); trace_nfsd_copy_async_done(copy); - nfsd4_send_cb_offload(copy); atomic_dec(©->cp_nn->pending_async_copies); + if (copy->cp_res.wr_bytes_written > 0 && copy->attr_update) + nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); + nfsd4_send_cb_offload(copy); return 0; } @@ -2179,6 +2187,9 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, memcpy(&result->cb_stateid, ©->cp_stateid.cs_stid, sizeof(result->cb_stateid)); dup_copy_fields(copy, async_copy); + if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & + FMODE_NOCMTIME) != 0) + async_copy->attr_update = true; memcpy(async_copy->cp_cb_offload.co_referring_sessionid.data, cstate->session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN); @@ -2197,6 +2208,10 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, } else { status = nfsd4_do_copy(copy, copy->nf_src->nf_file, copy->nf_dst->nf_file, true); + if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & + FMODE_NOCMTIME) != 0 && + copy->cp_res.wr_bytes_written > 0) + nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); } out: trace_nfsd_copy_done(copy, status); @@ -2535,10 +2550,6 @@ nfsd4_get_dir_delegation(struct svc_rqst *rqstp, dd = nfsd_get_dir_deleg(cstate, gdd, nf); nfsd_file_put(nf); if (IS_ERR(dd)) { - int err = PTR_ERR(dd); - - if (err != -EAGAIN) - return nfserrno(err); gdd->gddrnf_status = GDD4_UNAVAIL; return nfs_ok; } diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index b338473d6e52..6ea25a52d2f4 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -718,7 +718,8 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, return PTR_ERR(namecopy); name.data = namecopy; name.len = namelen; - get_user(princhashlen, &ci->cc_princhash.cp_len); + if (get_user(princhashlen, &ci->cc_princhash.cp_len)) + return -EFAULT; if (princhashlen > 0) { princhashcopy = memdup_user( &ci->cc_princhash.cp_data, diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index c2d13b26a687..a42f34842d77 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1168,6 +1168,7 @@ out_dec: void nfs4_put_stid(struct nfs4_stid *s) { + struct svc_export *exp = s->sc_export; struct nfs4_file *fp = s->sc_file; struct nfs4_client *clp = s->sc_client; @@ -1183,6 +1184,8 @@ nfs4_put_stid(struct nfs4_stid *s) nfs4_free_cpntf_statelist(clp->net, s); spin_unlock(&clp->cl_lock); s->sc_free(s); + if (exp) + exp_put(exp); if (fp) put_nfs4_file(fp); } @@ -1221,10 +1224,6 @@ static void put_deleg_file(struct nfs4_file *fp) static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct file *f) { - struct iattr ia = { .ia_valid = ATTR_ATIME | ATTR_CTIME | ATTR_MTIME | ATTR_DELEG }; - struct inode *inode = file_inode(f); - int ret; - /* don't do anything if FMODE_NOCMTIME isn't set */ if ((READ_ONCE(f->f_mode) & FMODE_NOCMTIME) == 0) return; @@ -1242,17 +1241,7 @@ static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct f return; /* Stamp everything to "now" */ - inode_lock(inode); - ret = notify_change(&nop_mnt_idmap, f->f_path.dentry, &ia, NULL); - inode_unlock(inode); - if (ret) { - struct inode *inode = file_inode(f); - - pr_notice_ratelimited("nfsd: Unable to update timestamps on inode %02x:%02x:%llu: %d\n", - MAJOR(inode->i_sb->s_dev), - MINOR(inode->i_sb->s_dev), - inode->i_ino, ret); - } + nfsd_update_cmtime_attr(f, ATTR_ATIME); } static void nfs4_unlock_deleg_lease(struct nfs4_delegation *dp) @@ -1390,7 +1379,8 @@ static void destroy_delegation(struct nfs4_delegation *dp) * stateid or it's called from a laundromat thread (nfsd4_landromat()) that * determined that this specific state has expired and needs to be revoked * (both mark state with the appropriate stid sc_status mode). It is also - * assumed that a reference was taken on the @dp state. + * assumed that a reference was taken on the @dp state. This function + * consumes that reference. * * If this function finds that the @dp state is SC_STATUS_FREED it means * that a FREE_STATEID operation for this stateid has been processed and @@ -1759,21 +1749,119 @@ static struct nfs4_stid *find_one_sb_stid(struct nfs4_client *clp, struct super_block *sb, unsigned int sc_types) { - unsigned long id, tmp; + unsigned long id = 0; struct nfs4_stid *stid; spin_lock(&clp->cl_lock); - idr_for_each_entry_ul(&clp->cl_stateids, stid, tmp, id) + while ((stid = idr_get_next_ul(&clp->cl_stateids, &id)) != NULL) { if ((stid->sc_type & sc_types) && stid->sc_status == 0 && stid->sc_file->fi_inode->i_sb == sb) { refcount_inc(&stid->sc_count); break; } + id++; + } spin_unlock(&clp->cl_lock); return stid; } +/* + * Release the export reference an admin-revoked stateid holds, + * so the svc_export (and its vfsmount) is not pinned until the + * client issues FREE_STATEID. sc_export is no longer consulted + * once SC_STATUS_ADMIN_REVOKED is set. + */ +static void drop_stid_export(struct nfs4_client *clp, + struct nfs4_stid *stid) +{ + struct svc_export *exp; + + spin_lock(&clp->cl_lock); + exp = stid->sc_export; + stid->sc_export = NULL; + spin_unlock(&clp->cl_lock); + if (exp) + exp_put(exp); +} + +static void revoke_ol_stid(struct nfs4_client *clp, + struct nfs4_ol_stateid *stp) +{ + struct nfs4_stid *stid = &stp->st_stid; + + lockdep_assert_held(&stp->st_mutex); + spin_lock(&clp->cl_lock); + if (stid->sc_status == 0) { + stid->sc_status |= SC_STATUS_ADMIN_REVOKED; + atomic_inc(&clp->cl_admin_revoked); + spin_unlock(&clp->cl_lock); + if (stid->sc_type == SC_TYPE_LOCK) { + struct nfs4_lockowner *lo = + lockowner(stp->st_stateowner); + struct nfsd_file *nf; + + nf = find_any_file(stp->st_stid.sc_file); + if (nf) { + get_file(nf->nf_file); + filp_close(nf->nf_file, (fl_owner_t)lo); + nfsd_file_put(nf); + } + } + release_all_access(stp); + drop_stid_export(clp, stid); + } else + spin_unlock(&clp->cl_lock); +} + +static void revoke_one_stid(struct nfsd_net *nn, struct nfs4_client *clp, + struct nfs4_stid *stid) +{ + struct nfs4_ol_stateid *stp; + struct nfs4_delegation *dp; + + switch (stid->sc_type) { + case SC_TYPE_OPEN: + stp = openlockstateid(stid); + mutex_lock_nested(&stp->st_mutex, OPEN_STATEID_MUTEX); + revoke_ol_stid(clp, stp); + mutex_unlock(&stp->st_mutex); + break; + case SC_TYPE_LOCK: + stp = openlockstateid(stid); + mutex_lock_nested(&stp->st_mutex, LOCK_STATEID_MUTEX); + revoke_ol_stid(clp, stp); + mutex_unlock(&stp->st_mutex); + break; + case SC_TYPE_DELEG: + /* + * Extra reference guards against concurrent FREE_STATEID. + */ + refcount_inc(&stid->sc_count); + dp = delegstateid(stid); + spin_lock(&nn->deleg_lock); + if (!unhash_delegation_locked(dp, SC_STATUS_ADMIN_REVOKED)) + dp = NULL; + spin_unlock(&nn->deleg_lock); + if (dp) { + revoke_delegation(dp); + drop_stid_export(clp, stid); + } else + nfs4_put_stid(stid); + break; + case SC_TYPE_LAYOUT: + spin_lock(&clp->cl_lock); + if (stid->sc_status == 0) { + stid->sc_status |= SC_STATUS_ADMIN_REVOKED; + atomic_inc(&clp->cl_admin_revoked); + } + spin_unlock(&clp->cl_lock); + nfsd4_close_layout(layoutstateid(stid)); + drop_stid_export(clp, stid); + break; + } +} + /** * nfsd4_revoke_states - revoke all nfsv4 states associated with given filesystem * @nn: used to identify instance of nfsd (there is one per net namespace) @@ -1804,70 +1892,8 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) struct nfs4_stid *stid = find_one_sb_stid(clp, sb, sc_types); if (stid) { - struct nfs4_ol_stateid *stp; - struct nfs4_delegation *dp; - struct nfs4_layout_stateid *ls; - spin_unlock(&nn->client_lock); - switch (stid->sc_type) { - case SC_TYPE_OPEN: - stp = openlockstateid(stid); - mutex_lock_nested(&stp->st_mutex, - OPEN_STATEID_MUTEX); - - spin_lock(&clp->cl_lock); - if (stid->sc_status == 0) { - stid->sc_status |= - SC_STATUS_ADMIN_REVOKED; - atomic_inc(&clp->cl_admin_revoked); - spin_unlock(&clp->cl_lock); - release_all_access(stp); - } else - spin_unlock(&clp->cl_lock); - mutex_unlock(&stp->st_mutex); - break; - case SC_TYPE_LOCK: - stp = openlockstateid(stid); - mutex_lock_nested(&stp->st_mutex, - LOCK_STATEID_MUTEX); - spin_lock(&clp->cl_lock); - if (stid->sc_status == 0) { - struct nfs4_lockowner *lo = - lockowner(stp->st_stateowner); - struct nfsd_file *nf; - - stid->sc_status |= - SC_STATUS_ADMIN_REVOKED; - atomic_inc(&clp->cl_admin_revoked); - spin_unlock(&clp->cl_lock); - nf = find_any_file(stp->st_stid.sc_file); - if (nf) { - get_file(nf->nf_file); - filp_close(nf->nf_file, - (fl_owner_t)lo); - nfsd_file_put(nf); - } - release_all_access(stp); - } else - spin_unlock(&clp->cl_lock); - mutex_unlock(&stp->st_mutex); - break; - case SC_TYPE_DELEG: - refcount_inc(&stid->sc_count); - dp = delegstateid(stid); - spin_lock(&nn->deleg_lock); - if (!unhash_delegation_locked( - dp, SC_STATUS_ADMIN_REVOKED)) - dp = NULL; - spin_unlock(&nn->deleg_lock); - if (dp) - revoke_delegation(dp); - break; - case SC_TYPE_LAYOUT: - ls = layoutstateid(stid); - nfsd4_close_layout(ls); - break; - } + revoke_one_stid(nn, clp, stid); nfs4_put_stid(stid); spin_lock(&nn->client_lock); if (clp->cl_minorversion == 0) @@ -1885,6 +1911,73 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) spin_unlock(&nn->client_lock); } +static struct nfs4_stid *find_one_export_stid(struct nfs4_client *clp, + const struct path *path, + unsigned int sc_types) +{ + unsigned long id = 0; + struct nfs4_stid *stid; + + spin_lock(&clp->cl_lock); + while ((stid = idr_get_next_ul(&clp->cl_stateids, &id)) != NULL) { + if ((stid->sc_type & sc_types) && + stid->sc_status == 0 && + stid->sc_export && + path_equal(&stid->sc_export->ex_path, path)) { + refcount_inc(&stid->sc_count); + break; + } + id++; + } + spin_unlock(&clp->cl_lock); + return stid; +} + +/** + * nfsd4_revoke_export_states - revoke nfsv4 states acquired through an export + * @nn: used to identify instance of nfsd (there is one per net namespace) + * @path: export path whose states should be revoked + * + * All nfs4 states (open, lock, delegation, layout) acquired through any + * export matching @path are revoked, regardless of which client holds + * them. Matching is by path identity (dentry + vfsmount), so multiple + * svc_export objects for the same path -- one per auth_domain -- are + * handled correctly. + * + * Userspace (exportfs -u) sends this after removing the last client + * for a path, enabling the underlying filesystem to be unmounted. + */ +void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) +{ + unsigned int idhashval; + unsigned int sc_types; + + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; + + spin_lock(&nn->client_lock); + for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) { + struct list_head *head = &nn->conf_id_hashtbl[idhashval]; + struct nfs4_client *clp; + retry: + list_for_each_entry(clp, head, cl_idhash) { + struct nfs4_stid *stid = find_one_export_stid( + clp, path, + sc_types); + if (stid) { + spin_unlock(&nn->client_lock); + revoke_one_stid(nn, clp, stid); + nfs4_put_stid(stid); + spin_lock(&nn->client_lock); + if (clp->cl_minorversion == 0) + nn->nfs40_last_revoke = + ktime_get_boottime_seconds(); + goto retry; + } + } + } + spin_unlock(&nn->client_lock); +} + static inline int hash_sessionid(struct nfs4_sessionid *sessionid) { @@ -5056,6 +5149,7 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) { struct nfs4_client *cl = s->sc_client; LIST_HEAD(reaplist); + struct nfs4_layout_stateid *ls; struct nfs4_ol_stateid *stp; struct nfs4_delegation *dp; bool unhashed; @@ -5081,6 +5175,12 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) spin_unlock(&cl->cl_lock); nfs4_put_stid(s); break; + case SC_TYPE_LAYOUT: + ls = layoutstateid(s); + list_del_init(&ls->ls_perclnt); + spin_unlock(&cl->cl_lock); + nfs4_put_stid(s); + break; default: spin_unlock(&cl->cl_lock); } @@ -5176,6 +5276,7 @@ retry: /* Replace unconfirmed owners without checking for replay. */ release_openowner(oo); oo = NULL; + goto retry; } if (oo) { if (new) @@ -6164,6 +6265,8 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, dp = alloc_init_deleg(clp, fp, odstate, dl_type); if (!dp) goto out_delegees; + if (stp->st_stid.sc_export) + dp->dl_stid.sc_export = exp_get(stp->st_stid.sc_export); fl = nfs4_alloc_init_lease(dp); if (!fl) @@ -6378,7 +6481,6 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, } open->op_delegate_type = deleg_ts ? OPEN_DELEGATE_WRITE_ATTRS_DELEG : OPEN_DELEGATE_WRITE; - dp->dl_cb_fattr.ncf_cur_fsize = stat.size; dp->dl_cb_fattr.ncf_initial_cinfo = nfsd4_change_attribute(&stat); dp->dl_atime = stat.atime; dp->dl_ctime = stat.ctime; @@ -6498,8 +6600,11 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf goto out; } - if (!open->op_stp) + if (!open->op_stp) { new_stp = true; + stp->st_stid.sc_export = + exp_get(current_fh->fh_export); + } } /* @@ -8195,6 +8300,9 @@ retry: stp->st_stateowner = nfs4_get_stateowner(&lo->lo_owner); get_nfs4_file(fp); stp->st_stid.sc_file = fp; + if (open_stp->st_stid.sc_export) + stp->st_stid.sc_export = + exp_get(open_stp->st_stid.sc_export); stp->st_access_bmap = 0; stp->st_deny_bmap = open_stp->st_deny_bmap; stp->st_openstp = open_stp; @@ -9429,11 +9537,15 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, if (status != nfserr_jukebox || !nfsd_wait_for_delegreturn(rqstp, inode)) goto out_status; + status = nfs_ok; + goto out_status; + } + if (!ncf->ncf_file_modified) { + if (ncf->ncf_initial_cinfo != ncf->ncf_cb_change) + ncf->ncf_file_modified = true; + else if (i_size_read(inode) != ncf->ncf_cb_fsize) + ncf->ncf_file_modified = true; } - if (!ncf->ncf_file_modified && - (ncf->ncf_initial_cinfo != ncf->ncf_cb_change || - ncf->ncf_cur_fsize != ncf->ncf_cb_fsize)) - ncf->ncf_file_modified = true; if (ncf->ncf_file_modified) { int err; @@ -9521,6 +9633,9 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, dp = alloc_init_deleg(clp, fp, NULL, NFS4_OPEN_DELEGATE_READ); if (!dp) goto out_delegees; + if (cstate->current_fh.fh_export) + dp->dl_stid.sc_export = + exp_get(cstate->current_fh.fh_export); fl = nfs4_alloc_init_lease(dp); if (!fl) @@ -9560,3 +9675,31 @@ out_delegees: put_nfs4_file(fp); return ERR_PTR(status); } + +/** + * nfsd_update_cmtime_attr - update file's delegated ctime/mtime, + * and optionally other attributes (ie ATTR_ATIME). + * @f: pointer to an opened file + * @flags: any additional flags that should be updated + * + * Given upon opening a file delegated attributes were issues, update + * @f attributes to current times. + */ +void nfsd_update_cmtime_attr(struct file *f, unsigned int flags) +{ + int ret; + struct inode *inode = file_inode(f); + struct iattr attr = { + .ia_valid = ATTR_CTIME | ATTR_MTIME | ATTR_DELEG | flags, + }; + + inode_lock(inode); + ret = notify_change(&nop_mnt_idmap, f->f_path.dentry, &attr, NULL); + inode_unlock(inode); + if (ret) + pr_notice_ratelimited("nfsd: Unable to update timestamps on " + "inode %02x:%02x:%llu: %d\n", + MAJOR(inode->i_sb->s_dev), + MINOR(inode->i_sb->s_dev), + inode->i_ino, ret); +} diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 2a0946c630e1..e17488a911f7 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -2008,10 +2008,11 @@ static __be32 nfsd4_decode_secinfo_no_name(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) { struct nfsd4_secinfo_no_name *sin = &u->secinfo_no_name; + + sin->sin_exp = NULL; if (xdr_stream_decode_u32(argp->xdr, &sin->sin_style) < 0) return nfserr_bad_xdr; - sin->sin_exp = NULL; return nfs_ok; } @@ -3158,6 +3159,8 @@ struct nfsd4_fattr_args { u32 rdattr_err; bool contextsupport; bool ignore_crossmnt; + bool case_insensitive; + bool case_preserving; }; typedef __be32(*nfsd4_enc_attr)(struct xdr_stream *xdr, @@ -3356,6 +3359,33 @@ static __be32 nfsd4_encode_fattr4_acl(struct xdr_stream *xdr, return nfs_ok; } +static __be32 nfsd4_encode_fattr4_case_insensitive(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_bool(xdr, args->case_insensitive); +} + +static __be32 nfsd4_encode_fattr4_case_preserving(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_bool(xdr, args->case_preserving); +} + +static __be32 nfsd4_encode_fattr4_homogeneous(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + /* + * Casefold-capable filesystems (e.g. ext4 or f2fs with the + * casefold feature) attach a Unicode encoding at mount time + * but apply case folding per directory. The per-file-system + * case_insensitive and case_preserving values can therefore + * legitimately differ across objects that share the same fsid. + * Report FATTR4_HOMOGENEOUS = FALSE on such filesystems to + * keep that variation consistent with RFC 8881 Section 5.8.2.16. + */ + return nfsd4_encode_bool(xdr, !sb_has_encoding(args->dentry->d_sb)); +} + static __be32 nfsd4_encode_fattr4_filehandle(struct xdr_stream *xdr, const struct nfsd4_fattr_args *args) { @@ -3748,8 +3778,8 @@ static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = { [FATTR4_ACLSUPPORT] = nfsd4_encode_fattr4_aclsupport, [FATTR4_ARCHIVE] = nfsd4_encode_fattr4__noop, [FATTR4_CANSETTIME] = nfsd4_encode_fattr4__true, - [FATTR4_CASE_INSENSITIVE] = nfsd4_encode_fattr4__false, - [FATTR4_CASE_PRESERVING] = nfsd4_encode_fattr4__true, + [FATTR4_CASE_INSENSITIVE] = nfsd4_encode_fattr4_case_insensitive, + [FATTR4_CASE_PRESERVING] = nfsd4_encode_fattr4_case_preserving, [FATTR4_CHOWN_RESTRICTED] = nfsd4_encode_fattr4__true, [FATTR4_FILEHANDLE] = nfsd4_encode_fattr4_filehandle, [FATTR4_FILEID] = nfsd4_encode_fattr4_fileid, @@ -3758,7 +3788,7 @@ static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = { [FATTR4_FILES_TOTAL] = nfsd4_encode_fattr4_files_total, [FATTR4_FS_LOCATIONS] = nfsd4_encode_fattr4_fs_locations, [FATTR4_HIDDEN] = nfsd4_encode_fattr4__noop, - [FATTR4_HOMOGENEOUS] = nfsd4_encode_fattr4__true, + [FATTR4_HOMOGENEOUS] = nfsd4_encode_fattr4_homogeneous, [FATTR4_MAXFILESIZE] = nfsd4_encode_fattr4_maxfilesize, [FATTR4_MAXLINK] = nfsd4_encode_fattr4_maxlink, [FATTR4_MAXNAME] = nfsd4_encode_fattr4_maxname, @@ -3854,13 +3884,16 @@ static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = { /* * Note: @fhp can be NULL; in this case, we might have to compose the filehandle - * ourselves. + * ourselves. @case_cache is NULL for callers that encode a single dentry + * (GETATTR, the buffer wrapper); READDIR passes a per-request cache so + * non-directory children share the parent's case-folding probe result. */ static __be32 nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, struct svc_fh *fhp, struct svc_export *exp, struct dentry *dentry, const u32 *bmval, - int ignore_crossmnt) + int ignore_crossmnt, + struct nfsd_case_attrs_cache *case_cache) { DECLARE_BITMAP(attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); struct nfs4_delegation *dp = NULL; @@ -3968,6 +4001,47 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, args.fhp = tempfh; } else args.fhp = fhp; + if (attrmask[0] & (FATTR4_WORD0_CASE_INSENSITIVE | + FATTR4_WORD0_CASE_PRESERVING)) { + /* + * In a batched encoder (READDIR) every non-directory + * child shares the same case-folding answer, so the + * directory being read is probed once and the result is + * cached. The probe targets case_cache->dir, the held + * readdir filehandle's dentry, instead of the child's + * locklessly-acquired dentry, which a concurrent rename + * could move under an unrelated parent. Directory + * entries are queried directly because casefold-capable + * filesystems answer per directory. + * + * Per RFC 8881 Section 18.7.3, an attribute advertised + * in SUPPORTED_ATTRS must come back with a value or the + * GETATTR must fail. nfsd_get_case_info() fills POSIX + * defaults and returns -EOPNOTSUPP when the underlying + * filesystem does not expose case state; encode those + * defaults so the reply agrees with what SUPPORTED_ATTRS + * advertises. Other errors fail the operation as the + * spec requires. + */ + if (case_cache && !d_is_dir(dentry)) { + if (!case_cache->valid) { + err = nfsd_get_case_info(case_cache->dir, + &case_cache->insensitive, + &case_cache->preserving); + if (err && err != -EOPNOTSUPP) + goto out_nfserr; + case_cache->valid = true; + } + args.case_insensitive = case_cache->insensitive; + args.case_preserving = case_cache->preserving; + } else { + err = nfsd_get_case_info(dentry, + &args.case_insensitive, + &args.case_preserving); + if (err && err != -EOPNOTSUPP) + goto out_nfserr; + } + } if (attrmask[0] & FATTR4_WORD0_ACL) { err = nfsd4_get_nfs4_acl(rqstp, dentry, &args.acl); @@ -4124,7 +4198,7 @@ __be32 nfsd4_encode_fattr_to_buf(__be32 **p, int words, svcxdr_init_encode_from_buffer(&xdr, &dummy, *p, words << 2); ret = nfsd4_encode_fattr4(rqstp, &xdr, fhp, exp, dentry, bmval, - ignore_crossmnt); + ignore_crossmnt, NULL); *p = xdr.p; return ret; } @@ -4162,6 +4236,7 @@ nfsd4_encode_entry4_fattr(struct nfsd4_readdir *cd, const char *name, struct dentry *dentry; __be32 nfserr; int ignore_crossmnt = 0; + bool crossed = false; dentry = lookup_one_positive_unlocked(&nop_mnt_idmap, &QSTR_LEN(name, namlen), @@ -4198,11 +4273,18 @@ nfsd4_encode_entry4_fattr(struct nfsd4_readdir *cd, const char *name, nfserr = check_nfsd_access(exp, cd->rd_rqstp, false); if (nfserr) goto out_put; + crossed = true; } out_encode: + /* + * A crossed entry no longer shares a parent with the directory + * being read, so it must neither consume nor populate the + * per-readdir case-folding cache. + */ nfserr = nfsd4_encode_fattr4(cd->rd_rqstp, cd->xdr, NULL, exp, dentry, - cd->rd_bmval, ignore_crossmnt); + cd->rd_bmval, ignore_crossmnt, + crossed ? NULL : &cd->rd_case_cache); out_put: dput(dentry); exp_put(exp); @@ -4449,7 +4531,7 @@ nfsd4_encode_getattr(struct nfsd4_compoundres *resp, __be32 nfserr, /* obj_attributes */ return nfsd4_encode_fattr4(resp->rqstp, xdr, fhp, fhp->fh_export, - fhp->fh_dentry, getattr->ga_bmval, 0); + fhp->fh_dentry, getattr->ga_bmval, 0, NULL); } static __be32 @@ -4976,6 +5058,8 @@ static __be32 nfsd4_encode_dirlist4(struct xdr_stream *xdr, readdir->rd_maxcount = maxcount; readdir->common.err = 0; readdir->cookie_offset = 0; + readdir->rd_case_cache.dir = readdir->rd_fhp->fh_dentry; + readdir->rd_case_cache.valid = false; offset = readdir->rd_cookie; status = nfsd_readdir(readdir->rd_rqstp, readdir->rd_fhp, &offset, &readdir->common, nfsd4_encode_entry4); diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 39e7012a60d8..fa92e31d19d6 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -10,6 +10,7 @@ #include <linux/ctype.h> #include <linux/fs_context.h> +#include <linux/sunrpc/cache.h> #include <linux/sunrpc/svcsock.h> #include <linux/lockd/bind.h> #include <linux/sunrpc/addr.h> @@ -246,7 +247,7 @@ static ssize_t write_unlock_ip(struct file *file, char *buf, size_t size) if (rpc_pton(net, fo_path, size, sap, salen) == 0) return -EINVAL; - trace_nfsd_ctl_unlock_ip(net, buf); + trace_nfsd_ctl_unlock_ip(net, sap, svc_addr_len(sap)); return nlmsvc_unlock_all_by_ip(sap); } @@ -1412,6 +1413,21 @@ static int create_proc_exports_entry(void) unsigned int nfsd_net_id; +struct nfsd_genl_rqstp { + struct sockaddr rq_daddr; + struct sockaddr rq_saddr; + unsigned long rq_flags; + ktime_t rq_stime; + __be32 rq_xid; + u32 rq_vers; + u32 rq_prog; + u32 rq_proc; + + /* NFSv4 compound */ + u32 rq_opcnt; + u32 rq_opnum[16]; +}; + static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, struct netlink_callback *cb, struct nfsd_genl_rqstp *genl_rqstp) @@ -1594,16 +1610,27 @@ out_unlock: static int nfsd_nl_fh_key_set(const struct nlattr *attr, struct nfsd_net *nn) { siphash_key_t *fh_key = nn->fh_key; + u64 k0, k1; + bool changed; + + k0 = get_unaligned_le64(nla_data(attr)); + k1 = get_unaligned_le64(nla_data(attr) + 8); if (!fh_key) { fh_key = kmalloc(sizeof(siphash_key_t), GFP_KERNEL); - if (!fh_key) + if (!fh_key) { + trace_nfsd_ctl_fh_key_set(false, -ENOMEM); return -ENOMEM; + } nn->fh_key = fh_key; + changed = true; + } else { + changed = fh_key->key[0] != k0 || fh_key->key[1] != k1; } - fh_key->key[0] = get_unaligned_le64(nla_data(attr)); - fh_key->key[1] = get_unaligned_le64(nla_data(attr) + 8); + fh_key->key[0] = k0; + fh_key->key[1] = k1; + trace_nfsd_ctl_fh_key_set(changed, 0); return 0; } @@ -1682,7 +1709,6 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) attr = info->attrs[NFSD_A_SERVER_FH_KEY]; if (attr) { ret = nfsd_nl_fh_key_set(attr, nn); - trace_nfsd_ctl_fh_key_set((const char *)nn->fh_key, ret); if (ret) goto out_unlock; } @@ -2188,6 +2214,194 @@ err_free_msg: } /** + * nfsd_nl_cache_flush_doit - flush nfsd caches via netlink + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Flush the svc_export and/or expkey caches. If NFSD_A_CACHE_FLUSH_MASK + * is provided, only flush the caches indicated by the bitmask (bit 0 = + * svc_export, bit 1 = expkey). If omitted, flush both. + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + u32 mask = ~0U; + + if (info->attrs[NFSD_A_CACHE_FLUSH_MASK]) + mask = nla_get_u32(info->attrs[NFSD_A_CACHE_FLUSH_MASK]); + + mutex_lock(&nfsd_mutex); + + if ((mask & NFSD_CACHE_TYPE_SVC_EXPORT) && + nn->svc_export_cache) + cache_purge(nn->svc_export_cache); + + if ((mask & NFSD_CACHE_TYPE_EXPKEY) && + nn->svc_expkey_cache) + cache_purge(nn->svc_expkey_cache); + + mutex_unlock(&nfsd_mutex); + + return 0; +} + +int nfsd_cache_notify(struct cache_detail *cd, struct cache_head *h, u32 cache_type) +{ + struct genlmsghdr *hdr; + struct sk_buff *msg; + + if (!genl_has_listeners(&nfsd_nl_family, cd->net, NFSD_NLGRP_EXPORTD)) + return -ENOLINK; + + msg = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + hdr = genlmsg_put(msg, 0, 0, &nfsd_nl_family, 0, NFSD_CMD_CACHE_NOTIFY); + if (!hdr) { + nlmsg_free(msg); + return -ENOMEM; + } + + if (nla_put_u32(msg, NFSD_A_CACHE_NOTIFY_CACHE_TYPE, cache_type)) { + nlmsg_free(msg); + return -ENOMEM; + } + + genlmsg_end(msg, hdr); + return genlmsg_multicast_netns(&nfsd_nl_family, cd->net, msg, 0, + NFSD_NLGRP_EXPORTD, GFP_KERNEL); +} + +/** + * nfsd_nl_unlock_ip_doit - release NLM locks held by an IP address + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return: 0 on success or a negative errno. + */ +int nfsd_nl_unlock_ip_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct sockaddr *sap; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_UNLOCK_IP_ADDRESS)) + return -EINVAL; + sap = nla_data(info->attrs[NFSD_A_UNLOCK_IP_ADDRESS]); + switch (sap->sa_family) { + case AF_INET: + if (nla_len(info->attrs[NFSD_A_UNLOCK_IP_ADDRESS]) < + sizeof(struct sockaddr_in)) + return -EINVAL; + break; + case AF_INET6: + if (nla_len(info->attrs[NFSD_A_UNLOCK_IP_ADDRESS]) < + sizeof(struct sockaddr_in6)) + return -EINVAL; + break; + default: + return -EAFNOSUPPORT; + } + /* + * nlmsvc_unlock_all_by_ip() releases matching locks + * across all network namespaces because lockd operates + * a single global instance. + */ + trace_nfsd_ctl_unlock_ip(genl_info_net(info), sap, + svc_addr_len(sap)); + return nlmsvc_unlock_all_by_ip(sap); +} + +/** + * nfsd_nl_unlock_filesystem_doit - revoke NFS state under a filesystem path + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return: 0 on success or a negative errno. + */ +int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct path path; + int error; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_UNLOCK_FILESYSTEM_PATH)) + return -EINVAL; + + trace_nfsd_ctl_unlock_fs(net, + nla_data(info->attrs[NFSD_A_UNLOCK_FILESYSTEM_PATH])); + error = kern_path( + nla_data(info->attrs[NFSD_A_UNLOCK_FILESYSTEM_PATH]), + 0, &path); + if (error) + return error; + + error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); + + mutex_lock(&nfsd_mutex); + if (nn->nfsd_serv) { + nfsd4_cancel_copy_by_sb(net, path.dentry->d_sb); + nfsd4_revoke_states(nn, path.dentry->d_sb); + } else { + error = -EINVAL; + } + mutex_unlock(&nfsd_mutex); + + path_put(&path); + return error; +} + +/** + * nfsd_nl_unlock_export_doit - revoke NFSv4 state for an export path + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Revokes all NFSv4 state (opens, locks, delegations, layouts) acquired + * through any export of the given path, regardless of which client holds + * the state. Userspace (exportfs -u) sends this after removing the last + * client for a path so the underlying filesystem can be unmounted. + * + * Unlike NFSD_CMD_UNLOCK_FILESYSTEM, which operates at superblock + * granularity, this command revokes only the state associated with + * exports of a specific path. + * + * Return: 0 on success or a negative errno. + */ +int nfsd_nl_unlock_export_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct path path; + int error; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_UNLOCK_EXPORT_PATH)) + return -EINVAL; + + trace_nfsd_ctl_unlock_export(net, + nla_data(info->attrs[NFSD_A_UNLOCK_EXPORT_PATH])); + error = kern_path( + nla_data(info->attrs[NFSD_A_UNLOCK_EXPORT_PATH]), + 0, &path); + if (error) + return error; + + mutex_lock(&nfsd_mutex); + if (nn->nfsd_serv) { + nfsd_file_close_export(net, &path); + nfsd4_revoke_export_states(nn, &path); + } else + error = -EINVAL; + mutex_unlock(&nfsd_mutex); + + path_put(&path); + return error; +} + +/** * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace * @net: a freshly-created network namespace * @@ -2310,12 +2524,9 @@ static int __init init_nfsd(void) if (retval) goto out_free_pnfs; nfsd_lockd_init(); /* lockd->nfsd callbacks */ - retval = nfsd_export_wq_init(); - if (retval) - goto out_free_lockd; retval = register_pernet_subsys(&nfsd_net_ops); if (retval < 0) - goto out_free_export_wq; + goto out_free_lockd; retval = register_cld_notifier(); if (retval) goto out_free_subsys; @@ -2344,8 +2555,6 @@ out_free_cld: unregister_cld_notifier(); out_free_subsys: unregister_pernet_subsys(&nfsd_net_ops); -out_free_export_wq: - nfsd_export_wq_shutdown(); out_free_lockd: nfsd_lockd_shutdown(); nfsd_drc_slab_free(); @@ -2366,7 +2575,6 @@ static void __exit exit_nfsd(void) nfsd4_destroy_laundry_wq(); unregister_cld_notifier(); unregister_pernet_subsys(&nfsd_net_ops); - nfsd_export_wq_shutdown(); nfsd_drc_slab_free(); nfsd_lockd_shutdown(); nfsd4_free_slabs(); diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 7c009f07c90b..11bce03b9031 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -45,11 +45,10 @@ bool nfsd_support_version(int vers); /* * Default and maximum payload size (NFS READ or WRITE), in bytes. - * The default is historical, and the maximum is an implementation - * limit. + * The maximum is an implementation limit. */ enum { - NFSSVC_DEFBLKSIZE = 1 * 1024 * 1024, + NFSSVC_DEFBLKSIZE = 4 * 1024 * 1024, NFSSVC_MAXBLKSIZE = RPCSVC_MAXPAYLOAD, }; @@ -60,21 +59,6 @@ struct readdir_cd { /* Maximum number of operations per session compound */ #define NFSD_MAX_OPS_PER_COMPOUND 200 -struct nfsd_genl_rqstp { - struct sockaddr rq_daddr; - struct sockaddr rq_saddr; - unsigned long rq_flags; - ktime_t rq_stime; - __be32 rq_xid; - u32 rq_vers; - u32 rq_prog; - u32 rq_proc; - - /* NFSv4 compound */ - u32 rq_opcnt; - u32 rq_opnum[16]; -}; - extern struct svc_program nfsd_programs[]; extern const struct svc_version nfsd_version2, nfsd_version3, nfsd_version4; extern struct mutex nfsd_mutex; @@ -123,7 +107,7 @@ struct dentry *nfsd_client_mkdir(struct nfsd_net *nn, const struct tree_descr *, struct dentry **fdentries); void nfsd_client_rmdir(struct dentry *dentry); - +int nfsd_cache_notify(struct cache_detail *cd, struct cache_head *h, u32 cache_type); #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) #ifdef CONFIG_NFSD_V2_ACL diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 953675eba5c3..dec83e92650d 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -145,6 +145,7 @@ struct nfs4_stid { spinlock_t sc_lock; struct nfs4_client *sc_client; struct nfs4_file *sc_file; + struct svc_export *sc_export; void (*sc_free)(struct nfs4_stid *); }; @@ -843,6 +844,7 @@ extern void nfsd4_shutdown_copy(struct nfs4_client *clp); void nfsd4_put_client(struct nfs4_client *clp); void nfsd4_async_copy_reaper(struct nfsd_net *nn); bool nfsd4_has_active_async_copies(struct nfs4_client *clp); +void nfsd_update_cmtime_attr(struct file *f, unsigned int flags); extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, struct nfsd_net *nn); extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn); @@ -861,6 +863,7 @@ struct nfsd_file *find_any_file(struct nfs4_file *f); #ifdef CONFIG_NFSD_V4 void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb); +void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path); void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb); int nfsd_net_cb_init(struct nfsd_net *nn); void nfsd_net_cb_shutdown(struct nfsd_net *nn); @@ -868,6 +871,10 @@ void nfsd_net_cb_shutdown(struct nfsd_net *nn); static inline void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) { } +static inline void nfsd4_revoke_export_states(struct nfsd_net *nn, + const struct path *path) +{ +} static inline void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) { } diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 5ad38f50836d..1c5a1e50f946 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -1985,19 +1985,20 @@ TRACE_EVENT(nfsd_cb_recall_any_done, TRACE_EVENT(nfsd_ctl_unlock_ip, TP_PROTO( const struct net *net, - const char *address + const struct sockaddr *addr, + const unsigned int addrlen ), - TP_ARGS(net, address), + TP_ARGS(net, addr, addrlen), TP_STRUCT__entry( __field(unsigned int, netns_ino) - __string(address, address) + __sockaddr(addr, addrlen) ), TP_fast_assign( __entry->netns_ino = net->ns.inum; - __assign_str(address); + __assign_sockaddr(addr, addr, addrlen); ), - TP_printk("address=%s", - __get_str(address) + TP_printk("addr=%pISpc", + __get_sockaddr(addr) ) ); @@ -2020,6 +2021,25 @@ TRACE_EVENT(nfsd_ctl_unlock_fs, ) ); +TRACE_EVENT(nfsd_ctl_unlock_export, + TP_PROTO( + const struct net *net, + const char *path + ), + TP_ARGS(net, path), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __string(path, path) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __assign_str(path); + ), + TP_printk("path=%s", + __get_str(path) + ) +); + TRACE_EVENT(nfsd_ctl_filehandle, TP_PROTO( const struct net *net, @@ -2243,23 +2263,21 @@ TRACE_EVENT(nfsd_end_grace, TRACE_EVENT(nfsd_ctl_fh_key_set, TP_PROTO( - const char *key, + bool changed, int result ), - TP_ARGS(key, result), + TP_ARGS(changed, result), TP_STRUCT__entry( - __field(u32, key_hash) + __field(bool, changed) __field(int, result) ), TP_fast_assign( - if (key) - __entry->key_hash = ~crc32_le(0xFFFFFFFF, key, 16); - else - __entry->key_hash = 0; + __entry->changed = changed; __entry->result = result; ), - TP_printk("key=0x%08x result=%d", - __entry->key_hash, __entry->result + TP_printk("key %s, result=%d", + __entry->changed ? "updated" : "unmodified", + __entry->result ) ); diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index eafdf7b7890f..1e89c7ff9493 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -32,6 +32,7 @@ #include <linux/writeback.h> #include <linux/security.h> #include <linux/sunrpc/xdr.h> +#include <linux/fileattr.h> #include "xdr3.h" @@ -1439,7 +1440,7 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned long exp_op_flags = 0; unsigned int pflags = current->flags; bool restore_flags = false; - unsigned int nvecs; + int nvecs; trace_nfsd_write_opened(rqstp, fhp, offset, *cnt); @@ -1479,6 +1480,10 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, } nvecs = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, payload); + if (nvecs < 0) { + host_err = nvecs; + goto out_nfserr; + } since = READ_ONCE(file->f_wb_err); if (verf) @@ -1508,8 +1513,10 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, nfsd_stats_io_write_add(nn, exp, *cnt); fsnotify_modify(file); host_err = filemap_check_wb_err(file->f_mapping, since); - if (host_err < 0) + if (host_err < 0) { + commit_reset_write_verifier(nn, rqstp, host_err); goto out_nfserr; + } if (stable && fhp->fh_use_wgather) { host_err = wait_for_concurrent_writes(file); @@ -1689,6 +1696,8 @@ nfsd_commit(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, nfsd_copy_write_verifier(verf, nn); err2 = filemap_check_wb_err(nf->nf_file->f_mapping, since); + if (err2 < 0) + commit_reset_write_verifier(nn, rqstp, err2); err = nfserrno(err2); break; case -EINVAL: @@ -2407,7 +2416,7 @@ static __be32 nfsd_buffered_readdir(struct file *file, struct svc_fh *fhp, loff_t offset; struct readdir_data buf = { .ctx.actor = nfsd_buffered_filldir, - .dirent = (void *)__get_free_page(GFP_KERNEL) + .dirent = kmalloc(PAGE_SIZE, GFP_KERNEL) }; if (!buf.dirent) @@ -2458,7 +2467,7 @@ static __be32 nfsd_buffered_readdir(struct file *file, struct svc_fh *fhp, offset = vfs_llseek(file, 0, SEEK_CUR); } - free_page((unsigned long)(buf.dirent)); + kfree((buf.dirent)); if (host_err) return nfserrno(host_err); @@ -2891,3 +2900,88 @@ nfsd_permission(struct svc_cred *cred, struct svc_export *exp, return err? nfserrno(err) : 0; } + +/** + * nfsd_get_case_info - get case sensitivity info for a dentry + * @dentry: dentry to query + * @case_insensitive: set to true if name comparison ignores case + * @case_preserving: set to true if case is preserved on disk + * + * On casefold-capable filesystems the flag lives on the directory, + * not on its entries, so for a non-directory @dentry the parent is + * queried instead. A directory (including an export root, whose + * parent lies outside the export) is queried as-is so its own + * contents' lookup behavior is reported. NFSD advertises + * fattr4_homogeneous as FALSE, so per-directory answers may differ + * within an export. + * + * The probe runs with kernel credentials. case_insensitive and + * case_preserving describe the directory's structural lookup + * behavior, not the caller's identity; running under the calling + * client's mapped credentials would let per-client MAC policy on + * the parent directory turn this query into NFS4ERR_ACCESS even + * though the underlying property is the same for every client. + * + * When the filesystem does not expose case-folding state (no + * ->fileattr_get, or the callback returns -EOPNOTSUPP / + * -ENOIOCTLCMD / -ENOTTY / -EINVAL), the outputs are filled with + * POSIX defaults (case-sensitive, case-preserving) on the premise + * that a filesystem with case-folding support wires up + * fileattr_get. + * + * Return: 0 with outputs filled, -EOPNOTSUPP with outputs filled + * to POSIX defaults, or a negative errno (e.g., -EIO, + * -ESTALE, -ENOMEM) with outputs unmodified. + */ +int +nfsd_get_case_info(struct dentry *dentry, bool *case_insensitive, + bool *case_preserving) +{ + struct file_kattr fa = {}; + const struct cred *saved; + struct cred *probe; + struct dentry *cd; + bool put = false; + int err; + + if (d_is_dir(dentry)) { + cd = dentry; + } else { + cd = dget_parent(dentry); + put = true; + } + + probe = prepare_kernel_cred(&init_task); + if (!probe) { + err = -ENOMEM; + goto out; + } + saved = override_creds(probe); + + err = vfs_fileattr_get(cd, &fa); + + put_cred(revert_creds(saved)); +out: + if (put) + dput(cd); + switch (err) { + case 0: + *case_insensitive = fa.fsx_xflags & FS_XFLAG_CASEFOLD; + *case_preserving = + !(fa.fsx_xflags & FS_XFLAG_CASENONPRESERVING); + return 0; + case -EINVAL: + case -ENOTTY: + case -ENOIOCTLCMD: + case -EOPNOTSUPP: + /* + * Filesystem does not expose case state. + * Report POSIX defaults. + */ + *case_insensitive = false; + *case_preserving = true; + return -EOPNOTSUPP; + default: + return err; + } +} diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index 702a844f2106..e09ea04a51b9 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -156,6 +156,9 @@ __be32 nfsd_readdir(struct svc_rqst *, struct svc_fh *, loff_t *, struct readdir_cd *, nfsd_filldir_t); __be32 nfsd_statfs(struct svc_rqst *, struct svc_fh *, struct kstatfs *, int access); +int nfsd_get_case_info(struct dentry *dentry, + bool *case_insensitive, + bool *case_preserving); __be32 nfsd_permission(struct svc_cred *cred, struct svc_export *exp, struct dentry *dentry, int acc); diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 522067b7fd75..a7c9714b0b0e 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -209,8 +209,8 @@ struct nfsd3_pathconfres { __u32 p_name_max; __u32 p_no_trunc; __u32 p_chown_restricted; - __u32 p_case_insensitive; - __u32 p_case_preserving; + bool p_case_insensitive; + bool p_case_preserving; }; struct nfsd3_commitres { diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 417e9ad9fbb3..85574b2a139a 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -432,6 +432,19 @@ struct nfsd4_read { u32 rd_eof; /* response */ }; +/* + * Cache the case-folding properties of @dir so a batched encoder + * (e.g., READDIR) does not re-probe per child. @dir is the + * directory being read, held by the request, so it is stable + * against rename for the duration of the cache's lifetime. + */ +struct nfsd_case_attrs_cache { + struct dentry *dir; + bool valid; + bool insensitive; + bool preserving; +}; + struct nfsd4_readdir { u64 rd_cookie; /* request */ nfs4_verifier rd_verf; /* request */ @@ -444,6 +457,7 @@ struct nfsd4_readdir { struct readdir_cd common; struct xdr_stream *xdr; int cookie_offset; + struct nfsd_case_attrs_cache rd_case_cache; }; struct nfsd4_release_lockowner { @@ -752,6 +766,7 @@ struct nfsd4_copy { struct nfsd_file *nf_src; struct nfsd_file *nf_dst; + bool attr_update; copy_stateid_t cp_stateid; diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index 2e553d698d0f..680e4009a30d 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c @@ -134,9 +134,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, } set_buffer_mapped(bh); bh->b_blocknr = pblocknr; /* set block address for read */ - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(opf, bh); + bh_submit(bh, opf, bh_end_read); bh->b_blocknr = blocknr; /* set back to the given block address */ *submit_ptr = pblocknr; err = 0; diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c index 62d4c1b787e9..85379ac23ae2 100644 --- a/fs/nilfs2/gcinode.c +++ b/fs/nilfs2/gcinode.c @@ -83,9 +83,7 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff, if (!buffer_mapped(bh)) set_buffer_mapped(bh); bh->b_blocknr = pbn; - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); if (vbn) bh->b_blocknr = vbn; out: diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c index e0a606643e87..b73f2c5d10f0 100644 --- a/fs/nilfs2/ioctl.c +++ b/fs/nilfs2/ioctl.c @@ -69,7 +69,7 @@ static int nilfs_ioctl_wrap_copy(struct the_nilfs *nilfs, if (argv->v_index > ~(__u64)0 - argv->v_nmembs) return -EINVAL; - buf = (void *)get_zeroed_page(GFP_NOFS); + buf = kzalloc(PAGE_SIZE, GFP_NOFS); if (unlikely(!buf)) return -ENOMEM; maxmembs = PAGE_SIZE / argv->v_size; @@ -107,7 +107,7 @@ static int nilfs_ioctl_wrap_copy(struct the_nilfs *nilfs, } argv->v_nmembs = total; - free_pages((unsigned long)buf, 0); + kfree(buf); return ret; } diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c index 09adb40c65e5..2a435349fd21 100644 --- a/fs/nilfs2/mdt.c +++ b/fs/nilfs2/mdt.c @@ -148,9 +148,7 @@ nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, blk_opf_t opf, } map_bh(bh, inode->i_sb, (sector_t)blknum); - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(opf, bh); + bh_submit(bh, opf, bh_end_read); ret = 0; trace_nilfs2_mdt_submit_block(inode, inode->i_ino, blkoff, diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c index 40ac679ec56e..e2fe95de3d71 100644 --- a/fs/nilfs2/namei.c +++ b/fs/nilfs2/namei.c @@ -258,7 +258,7 @@ out: else nilfs_transaction_abort(dir->i_sb); - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; out_fail: drop_nlink(inode); diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 1491a4d4b1e1..9332f5ac6083 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -2512,12 +2512,33 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, struct nilfs_sc_info *sci = nilfs->ns_writer; struct nilfs_transaction_info ti; int err; + size_t i, nfreesegs = argv[4].v_nmembs; + __u64 *segnumv = kbufs[4]; if (unlikely(!sci)) return -EROFS; nilfs_transaction_lock(sb, &ti, 1); + /* + * Validate segment numbers under ns_segctor_sem (held for write + * by nilfs_transaction_lock above) so the check is serialized + * against nilfs_ioctl_resize(), which can modify ns_nsegments. + * Rejecting bad input here, before any segment-cleaning work + * begins, avoids the per-element diagnostic path inside + * nilfs_sufile_updatev() that would otherwise run under this + * same lock and stall concurrent readers. + */ + for (i = 0; i < nfreesegs; i++) { + if (segnumv[i] >= nilfs->ns_nsegments) { + nilfs_err(sb, + "Segment number %llu to be freed is out of range", + (unsigned long long)segnumv[i]); + err = -EINVAL; + goto bail_unlock; + } + } + err = nilfs_mdt_save_to_shadow_map(nilfs->ns_dat); if (unlikely(err)) goto out_unlock; @@ -2558,6 +2579,7 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, sci->sc_freesegs = NULL; sci->sc_nfreesegs = 0; nilfs_mdt_clear_shadow_map(nilfs->ns_dat); + bail_unlock: nilfs_transaction_unlock(sb); return err; } diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index 7aa5ef8606cd..893a504cb80c 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -1070,8 +1070,6 @@ nilfs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_time_gran = 1; sb->s_max_links = NILFS_LINK_MAX; - sb->s_bdi = bdi_get(sb->s_bdev->bd_disk->bdi); - err = load_nilfs(nilfs, sb); if (err) goto failed_nilfs; diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index 38290b9c07f7..a3555bebad63 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -14,6 +14,7 @@ #include <linux/sched/mm.h> #include <linux/statfs.h> #include <linux/stringhash.h> +#include <linux/pidfs.h> #include "fanotify.h" @@ -120,10 +121,7 @@ static bool fanotify_error_event_equal(struct fanotify_error_event *fee1, struct fanotify_error_event *fee2) { /* Error events against the same file system are always merged. */ - if (!fanotify_fsid_equal(&fee1->fsid, &fee2->fsid)) - return false; - - return true; + return fanotify_fsid_equal(&fee1->fsid, &fee2->fsid); } static bool fanotify_should_merge(struct fanotify_event *old, @@ -842,6 +840,15 @@ static struct fanotify_event *fanotify_alloc_event( /* Whoever is interested in the event, pays for the allocation. */ old_memcg = set_active_memcg(group->memcg); + if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) + pid = task_pid(current); + else + pid = task_tgid(current); + + if (FAN_GROUP_FLAG(group, FAN_REPORT_PIDFD) && + pidfs_register_pid_gfp(pid, gfp)) + goto out; + if (fanotify_is_perm_event(mask)) { event = fanotify_alloc_perm_event(data, data_type, gfp); } else if (fanotify_is_error_event(mask)) { @@ -863,15 +870,10 @@ static struct fanotify_event *fanotify_alloc_event( if (!event) goto out; - if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) - pid = get_pid(task_pid(current)); - else - pid = get_pid(task_tgid(current)); - /* Mix event info, FAN_ONDIR flag and pid into event merge key */ hash ^= hash_long((unsigned long)pid | ondir, FANOTIFY_EVENT_HASH_BITS); fanotify_init_event(event, hash, mask); - event->pid = pid; + event->pid = get_pid(pid); out: set_active_memcg(old_memcg); diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index ae904451dfc0..b604e3da58ad 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -19,6 +19,7 @@ #include <linux/memcontrol.h> #include <linux/statfs.h> #include <linux/exportfs.h> +#include <linux/pidfd.h> #include <asm/ioctls.h> @@ -903,25 +904,13 @@ static ssize_t copy_event_to_user(struct fsnotify_group *group, metadata.fd = fd >= 0 ? fd : FAN_NOFD; if (pidfd_mode) { - /* - * Complain if the FAN_REPORT_PIDFD and FAN_REPORT_TID mutual - * exclusion is ever lifted. At the time of incoporating pidfd - * support within fanotify, the pidfd API only supported the - * creation of pidfds for thread-group leaders. - */ - WARN_ON_ONCE(FAN_GROUP_FLAG(group, FAN_REPORT_TID)); + unsigned int pidfd_flags = PIDFD_STALE; - /* - * The PIDTYPE_TGID check for an event->pid is performed - * preemptively in an attempt to catch out cases where the event - * listener reads events after the event generating process has - * already terminated. Depending on flag FAN_REPORT_FD_ERROR, - * report either -ESRCH or FAN_NOPIDFD to the event listener in - * those cases with all other pidfd creation errors reported as - * the error code itself or as FAN_EPIDFD. - */ - if (metadata.pid && pid_has_task(event->pid, PIDTYPE_TGID)) - pidfd = pidfd_prepare(event->pid, 0, &pidfd_file); + if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) + pidfd_flags |= PIDFD_THREAD; + + if (metadata.pid) + pidfd = pidfd_prepare(event->pid, pidfd_flags, &pidfd_file); if (!FAN_GROUP_FLAG(group, FAN_REPORT_FD_ERROR) && pidfd < 0) pidfd = pidfd == -ESRCH ? FAN_NOPIDFD : FAN_EPIDFD; @@ -1628,14 +1617,6 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) #endif return -EINVAL; - /* - * A pidfd can only be returned for a thread-group leader; thus - * FAN_REPORT_PIDFD and FAN_REPORT_TID need to remain mutually - * exclusive. - */ - if ((flags & FAN_REPORT_PIDFD) && (flags & FAN_REPORT_TID)) - return -EINVAL; - /* Don't allow mixing mnt events with inode events for now */ if (flags & FAN_REPORT_MNT) { if (class != FAN_CLASS_NOTIF) diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 2dac70b99b0d..7e2f330fd283 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -14,6 +14,9 @@ #include <linux/fsnotify_backend.h> #include "fsnotify.h" +#define CREATE_TRACE_POINTS +#include <trace/events/fsnotify.h> + /* * Clear all of the marks on an inode when it is being evicted from core */ @@ -504,6 +507,8 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, int ret = 0; __u32 test_mask, marks_mask = 0; + trace_fsnotify(mask, data, data_type, dir, file_name, inode, cookie); + if (path) mnt = real_mount(path->mnt); diff --git a/fs/notify/mark.c b/fs/notify/mark.c index e256b420100d..b2640d836a71 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -343,6 +343,35 @@ void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) fsnotify_conn_set_children_dentry_flags(conn); } +/** + * fsnotify_modify_mark_mask - set and/or clear flags in a mark's mask + * @mark: mark to be modified + * @set: bits to be set in mask + * @clear: bits to be cleared in mask + * + * Modify a fsnotify_mark mask as directed, and update its associated conn. + * The caller is expected to hold a reference to the mark. + */ +void fsnotify_modify_mark_mask(struct fsnotify_mark *mark, u32 set, u32 clear) +{ + bool recalc = false; + u32 mask; + + WARN_ON_ONCE(clear & set); + + spin_lock(&mark->lock); + mask = mark->mask; + mark->mask |= set; + mark->mask &= ~clear; + if (mark->mask != mask) + recalc = true; + spin_unlock(&mark->lock); + + if (recalc) + fsnotify_recalc_mask(mark->connector); +} +EXPORT_SYMBOL_GPL(fsnotify_modify_mark_mask); + /* Free all connectors queued for freeing once SRCU period ends */ static void fsnotify_connector_destroy_workfn(struct work_struct *work) { diff --git a/fs/nsfs.c b/fs/nsfs.c index 51e8c9430477..c3b6ae76594a 100644 --- a/fs/nsfs.c +++ b/fs/nsfs.c @@ -266,7 +266,7 @@ static long ns_ioctl(struct file *filp, unsigned int ioctl, else tsk = find_task_by_pid_ns(arg, pid_ns); if (!tsk) - break; + return ret; switch (ioctl) { case NS_GET_PID_FROM_PIDNS: @@ -664,7 +664,6 @@ static int nsfs_init_fs_context(struct fs_context *fc) struct pseudo_fs_context *ctx = init_pseudo(fc, NSFS_MAGIC); if (!ctx) return -ENOMEM; - fc->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; ctx->s_d_flags |= DCACHE_DONTCACHE; ctx->ops = &nsfs_ops; ctx->eops = &nsfs_export_operations; diff --git a/fs/ntfs/Makefile b/fs/ntfs/Makefile index 0ce4d9a9388a..e120c2e69862 100644 --- a/fs/ntfs/Makefile +++ b/fs/ntfs/Makefile @@ -5,6 +5,6 @@ obj-$(CONFIG_NTFS_FS) += ntfs.o ntfs-y := aops.o attrib.o collate.o dir.o file.o index.o inode.o \ mft.o mst.o namei.o runlist.o super.o unistr.o attrlist.o ea.o \ upcase.o bitmap.o lcnalloc.o logfile.o reparse.o compress.o \ - iomap.o debug.o sysctl.o quota.o object_id.o bdev-io.o + iomap.o debug.o sysctl.o object_id.o bdev-io.o ccflags-$(CONFIG_NTFS_DEBUG) += -DDEBUG diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index 1fbf832ad165..173de4cbee0f 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c @@ -38,11 +38,9 @@ static void ntfs_iomap_read_end_io(struct bio *bio) } static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter, - struct iomap_read_folio_ctx *ctx) + struct iomap_read_folio_ctx *ctx) { - struct bio *bio = ctx->read_ctx; - bio->bi_end_io = ntfs_iomap_read_end_io; - submit_bio(bio); + iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io); } static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { @@ -251,6 +249,8 @@ static int ntfs_writepages(struct address_space *mapping, .wbc = wbc, .ops = &ntfs_writeback_ops, }; + bool need_iput = false; + int ret; if (NVolShutdown(ni->vol)) return -EIO; @@ -267,7 +267,20 @@ static int ntfs_writepages(struct address_space *mapping, return -EOPNOTSUPP; } - return iomap_writepages(&wpc); + /* + * Prevent eviction in writeback to avoid deadlock in + * ntfs_drop_big_inode(). + */ + if ((ni->type == AT_DATA || ni->type == AT_INDEX_ALLOCATION) && + igrab(inode)) + need_iput = true; + + ret = iomap_writepages(&wpc); + + if (need_iput) + iput(inode); + + return ret; } static int ntfs_swap_activate(struct swap_info_struct *sis, diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c index 97b660eaa00c..239b7bcbaedf 100644 --- a/fs/ntfs/attrib.c +++ b/fs/ntfs/attrib.c @@ -16,6 +16,7 @@ * Copyright (c) 2010 Erik Larsson */ +#include <linux/string_choices.h> #include <linux/writeback.h> #include <linux/iomap.h> @@ -174,7 +175,10 @@ int ntfs_map_runlist_nolock(struct ntfs_inode *ni, s64 vcn, struct ntfs_attr_sea err = -EIO; goto err_out; } - WARN_ON(!ctx->attr->non_resident); + if (unlikely(!ctx->attr->non_resident)) { + err = -EIO; + goto err_out; + } } a = ctx->attr; /* @@ -583,29 +587,168 @@ static u32 ntfs_resident_attr_min_value_length(const __le32 type) case AT_STANDARD_INFORMATION: return offsetof(struct standard_information, ver) + sizeof(((struct standard_information *)0)->ver.v1.reserved12); - case AT_ATTRIBUTE_LIST: - return offsetof(struct attr_list_entry, name); case AT_FILE_NAME: - return offsetof(struct file_name_attr, file_name); - case AT_OBJECT_ID: - return sizeof(struct guid); - case AT_SECURITY_DESCRIPTOR: - return sizeof(struct security_descriptor_relative); + return offsetof(struct file_name_attr, file_name) + + sizeof(__le16) * 1; case AT_VOLUME_INFORMATION: return sizeof(struct volume_information); case AT_INDEX_ROOT: return sizeof(struct index_root); - case AT_REPARSE_POINT: - return offsetof(struct reparse_point, reparse_data); case AT_EA_INFORMATION: return sizeof(struct ea_information); - case AT_EA: - return offsetof(struct ea_attr, ea_name) + 1; default: return 0; } } +static bool ntfs_attr_type_is_resident_only(const __le32 type) +{ + switch (type) { + case AT_STANDARD_INFORMATION: + case AT_FILE_NAME: + case AT_OBJECT_ID: + case AT_VOLUME_NAME: + case AT_VOLUME_INFORMATION: + case AT_INDEX_ROOT: + case AT_EA_INFORMATION: + return true; + default: + return false; + } +} + +static bool ntfs_file_name_attr_value_is_valid(const u8 *value, const u32 value_length) +{ + const struct file_name_attr *fn; + u32 file_name_size; + + fn = (const struct file_name_attr *)value; + file_name_size = fn->file_name_length * sizeof(__le16); + + return file_name_size <= + value_length - offsetof(struct file_name_attr, file_name); +} + +static bool ntfs_volume_name_attr_value_is_valid(const u32 value_length) +{ + if (value_length & 1) + return false; + + return value_length <= NTFS_MAX_LABEL_LEN * sizeof(__le16); +} + +static bool ntfs_index_root_attr_value_is_valid(const u8 *value, const u32 value_length) +{ + const struct index_root *ir; + u32 index_size; + u32 entries_offset; + u32 index_length; + u32 allocated_size; + + ir = (const struct index_root *)value; + index_size = value_length - offsetof(struct index_root, index); + entries_offset = le32_to_cpu(ir->index.entries_offset); + index_length = le32_to_cpu(ir->index.index_length); + allocated_size = le32_to_cpu(ir->index.allocated_size); + + if ((entries_offset | index_length | allocated_size) & 7 || + entries_offset < sizeof(struct index_header) || + entries_offset > index_length || + index_length > allocated_size || + allocated_size > index_size || + index_length - entries_offset < sizeof(struct index_entry_header)) + return false; + + return true; +} + +struct ntfs_resident_attr_value { + const u8 *data; + u32 len; +}; + +static bool ntfs_resident_attr_value_get(const struct attr_record *a, + struct ntfs_resident_attr_value *value) +{ + u32 attr_len; + u16 value_offset; + + attr_len = le32_to_cpu(a->length); + if (attr_len < offsetof(struct attr_record, data.resident.reserved) + + sizeof(a->data.resident.reserved)) + return false; + + value->len = le32_to_cpu(a->data.resident.value_length); + value_offset = le16_to_cpu(a->data.resident.value_offset); + + if (value->len > attr_len || value_offset > attr_len - value->len) + return false; + + value->data = (const u8 *)a + value_offset; + return true; +} + +static bool ntfs_non_resident_attr_value_is_valid(const struct attr_record *a) +{ + u32 attr_len; + u32 min_len; + u16 mp_offset; + + attr_len = le32_to_cpu(a->length); + min_len = offsetof(struct attr_record, data.non_resident.initialized_size) + + sizeof(a->data.non_resident.initialized_size); + if (attr_len < min_len) + return false; + + mp_offset = le16_to_cpu(a->data.non_resident.mapping_pairs_offset); + return mp_offset >= min_len && mp_offset <= attr_len; +} + +static bool ntfs_attr_value_is_valid(struct ntfs_volume *vol, + const struct attr_record *a, + const u64 mft_no) +{ + struct ntfs_resident_attr_value value; + u32 min_len; + + if (a->non_resident) { + if (ntfs_attr_type_is_resident_only(a->type)) + goto corrupt; + if (!ntfs_non_resident_attr_value_is_valid(a)) + goto corrupt; + return true; + } + + if (!ntfs_resident_attr_value_get(a, &value)) + goto corrupt; + + min_len = ntfs_resident_attr_min_value_length(a->type); + if (min_len && value.len < min_len) + goto corrupt; + + switch (a->type) { + case AT_FILE_NAME: + if (!ntfs_file_name_attr_value_is_valid(value.data, value.len)) + goto corrupt; + break; + case AT_VOLUME_NAME: + if (!ntfs_volume_name_attr_value_is_valid(value.len)) + goto corrupt; + break; + case AT_INDEX_ROOT: + if (!ntfs_index_root_attr_value_is_valid(value.data, value.len)) + goto corrupt; + break; + } + return true; + +corrupt: + ntfs_error(vol->sb, + "Corrupt %#x attribute in MFT record %llu\n", + le32_to_cpu(a->type), mft_no); + return false; +} + /* * ntfs_attr_find - find (next) attribute in mft record * @type: attribute type to find @@ -672,6 +815,9 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name, __le16 *upcase = vol->upcase; u32 upcase_len = vol->upcase_len; unsigned int space; + u16 name_offset; + u32 attr_len; + u32 name_size; /* * Iterate over attributes in mft record starting at @ctx->attr, or the @@ -699,8 +845,25 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name, return -ENOENT; if (unlikely(!a->length)) break; - if (type == AT_UNUSED) + if (a->name_length) { + name_offset = le16_to_cpu(a->name_offset); + attr_len = le32_to_cpu(a->length); + name_size = a->name_length * sizeof(__le16); + + if (name_offset > attr_len || + attr_len - name_offset < name_size) { + ntfs_error(vol->sb, + "Corrupt attribute name in MFT record %llu\n", + ctx->ntfs_ino->mft_no); + break; + } + } + + if (type == AT_UNUSED) { + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; return 0; + } if (a->type != type) continue; /* @@ -712,14 +875,6 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name, if (a->name_length) return -ENOENT; } else { - if (a->name_length && ((le16_to_cpu(a->name_offset) + - a->name_length * sizeof(__le16)) > - le32_to_cpu(a->length))) { - ntfs_error(vol->sb, "Corrupt attribute name in MFT record %llu\n", - ctx->ntfs_ino->mft_no); - break; - } - if (!ntfs_are_names_equal(name, name_len, (__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)), a->name_length, ic, upcase, upcase_len)) { @@ -749,37 +904,8 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name, } } - /* Validate attribute's value offset/length */ - if (!a->non_resident) { - u32 min_len; - u32 value_length = le32_to_cpu(a->data.resident.value_length); - u16 value_offset = le16_to_cpu(a->data.resident.value_offset); - - if (value_length > le32_to_cpu(a->length) || - value_offset > le32_to_cpu(a->length) - value_length) - break; - - min_len = ntfs_resident_attr_min_value_length(a->type); - if (min_len && value_length < min_len) { - ntfs_error(vol->sb, - "Too small %#x resident attribute value in MFT record %lld\n", - le32_to_cpu(a->type), (long long)ctx->ntfs_ino->mft_no); - break; - } - } else { - u32 min_len; - u16 mp_offset; - - min_len = offsetof(struct attr_record, data.non_resident.initialized_size) + - sizeof(a->data.non_resident.initialized_size); - if (le32_to_cpu(a->length) < min_len) - break; - - mp_offset = le16_to_cpu(a->data.non_resident.mapping_pairs_offset); - if (mp_offset < min_len || - mp_offset > le32_to_cpu(a->length)) - break; - } + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; /* * The names match or @name not present and attribute is @@ -845,11 +971,71 @@ char *ntfs_attr_name_get(const struct ntfs_volume *vol, const __le16 *uname, return NULL; } +/* + * ntfs_attr_list_entry_is_valid - sanity check one $ATTRIBUTE_LIST entry + * @ale: the attribute-list entry to check + * @al_end: end of the attribute-list buffer @ale lives in + * + * Verify that @ale is a well-formed attr_list_entry wholly contained in + * [.., @al_end): its fixed header must lie in range before any field is + * dereferenced, its length must be a multiple of 8 that covers the fixed + * header plus the name, the name must lie within the buffer, the entry must + * be in use and carry a live MFT reference. Return true if valid. + */ +bool ntfs_attr_list_entry_is_valid(const struct attr_list_entry *ale, + const u8 *al_end) +{ + const u8 *al = (const u8 *)ale; + u16 ale_len; + + /* The fixed header must be in bounds before it is parsed. */ + if (al + offsetof(struct attr_list_entry, name) > al_end) + return false; + ale_len = le16_to_cpu(ale->length); + /* On-disk entries are 8-byte aligned (see struct attr_list_entry). */ + if (ale_len & 7) + return false; + if (ale->name_offset != sizeof(struct attr_list_entry)) + return false; + if ((u32)ale->name_offset + + (u32)ale->name_length * sizeof(__le16) > ale_len || + al + ale_len > al_end) + return false; + if (ale->type == AT_UNUSED) + return false; + if (MSEQNO_LE(ale->mft_reference) == 0) + return false; + return true; +} + +/* + * ntfs_attr_list_is_valid - sanity check an in-memory $ATTRIBUTE_LIST + * @al_start: start of the attribute list buffer + * @size: length of the attribute list in bytes + * + * Verify that [@al_start, @al_start + @size) is a sequence of valid + * attr_list_entry records (see ntfs_attr_list_entry_is_valid()) that tile the + * buffer exactly. Return true if valid, false otherwise. + */ +bool ntfs_attr_list_is_valid(const u8 *al_start, s64 size) +{ + const u8 *al = al_start; + const u8 *al_end = al_start + size; + + while (al < al_end) { + const struct attr_list_entry *ale = + (const struct attr_list_entry *)al; + + if (!ntfs_attr_list_entry_is_valid(ale, al_end)) + return false; + al += le16_to_cpu(ale->length); + } + return al == al_end; +} + int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size) { struct inode *attr_vi = NULL; - u8 *al; - struct attr_list_entry *ale; if (!al_start || size <= 0) return -EINVAL; @@ -871,19 +1057,7 @@ int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size } iput(attr_vi); - for (al = al_start; al < al_start + size; al += le16_to_cpu(ale->length)) { - ale = (struct attr_list_entry *)al; - if (ale->name_offset != sizeof(struct attr_list_entry)) - break; - if (le16_to_cpu(ale->length) <= ale->name_offset + ale->name_length || - al + le16_to_cpu(ale->length) > al_start + size) - break; - if (ale->type == AT_UNUSED) - break; - if (MSEQNO_LE(ale->mft_reference) == 0) - break; - } - if (al != al_start + size) { + if (!ntfs_attr_list_is_valid(al_start, size)) { ntfs_error(base_ni->vol->sb, "Corrupt attribute list, mft = %llu", base_ni->mft_no); return -EIO; @@ -1139,9 +1313,8 @@ find_attr_list_attr: * we have reached the right one or the search has failed. */ if (lowest_vcn && (u8 *)next_al_entry >= al_start && - (u8 *)next_al_entry + 6 < al_end && - (u8 *)next_al_entry + le16_to_cpu( - next_al_entry->length) <= al_end && + ntfs_attr_list_entry_is_valid(next_al_entry, + al_end) && le64_to_cpu(next_al_entry->lowest_vcn) <= lowest_vcn && next_al_entry->type == al_entry->type && @@ -1254,22 +1427,8 @@ do_next_attr_loop: ctx->attr = a; - if (a->non_resident) { - u32 min_len; - u16 mp_offset; - - min_len = offsetof(struct attr_record, - data.non_resident.initialized_size) + - sizeof(a->data.non_resident.initialized_size); - - if (le32_to_cpu(a->length) < min_len) - break; - - mp_offset = - le16_to_cpu(a->data.non_resident.mapping_pairs_offset); - if (mp_offset < min_len || mp_offset > attr_len) - break; - } + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; /* * If no @val specified or @val specified and it matches, we @@ -1281,19 +1440,6 @@ do_next_attr_loop: u32 value_length = le32_to_cpu(a->data.resident.value_length); u16 value_offset = le16_to_cpu(a->data.resident.value_offset); - if (attr_len < offsetof(struct attr_record, data.resident.reserved) + - sizeof(a->data.resident.reserved)) - break; - if (value_length > attr_len || value_offset > attr_len - value_length) - break; - - value_length = ntfs_resident_attr_min_value_length(a->type); - if (value_length && le32_to_cpu(a->data.resident.value_length) < - value_length) { - pr_err("Too small resident attribute value in MFT record %lld, type %#x\n", - (long long)ctx->ntfs_ino->mft_no, a->type); - break; - } if (value_length == val_len && !memcmp((u8 *)a + value_offset, val, val_len)) { attr_found: @@ -1857,7 +2003,7 @@ int ntfs_attr_make_non_resident(struct ntfs_inode *ni, const u32 data_size) if (IS_ERR(rl)) { err = PTR_ERR(rl); ntfs_debug("Failed to allocate cluster%s, error code %i.", - ntfs_bytes_to_cluster(vol, new_size) > 1 ? "s" : "", + str_plural(ntfs_bytes_to_cluster(vol, new_size)), err); goto folio_err_out; } @@ -2924,12 +3070,12 @@ int ntfs_attr_open(struct ntfs_inode *ni, const __le32 type, struct ntfs_inode *base_ni; int err; - ntfs_debug("Entering for inode %lld, attr 0x%x.\n", - (unsigned long long)ni->mft_no, type); - if (!ni || !ni->vol) return -EINVAL; + ntfs_debug("Entering for inode %lld, attr 0x%x.\n", + ni->mft_no, type); + if (NInoAttr(ni)) base_ni = ni->ext.base_ntfs_ino; else @@ -4538,10 +4684,12 @@ attr_resize_again: while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) { struct inode *tvi; struct attr_record *a; + u32 value_len; a = ctx->attr; if (a->non_resident || a->type == AT_ATTRIBUTE_LIST) continue; + value_len = le32_to_cpu(a->data.resident.value_length); if (ntfs_attr_can_be_non_resident(vol, a->type)) continue; @@ -4553,6 +4701,8 @@ attr_resize_again: if (le32_to_cpu(a->length) <= (sizeof(struct attr_record) - sizeof(s64)) + ((a->name_length * sizeof(__le16) + 7) & ~7) + 8) continue; + if (a->type == AT_DATA && !value_len) + continue; if (a->type == AT_DATA) tvi = ntfs_iget(sb, base_ni->mft_no); @@ -4565,8 +4715,7 @@ attr_resize_again: continue; } - if (ntfs_attr_make_non_resident(NTFS_I(tvi), - le32_to_cpu(ctx->attr->data.resident.value_length))) { + if (ntfs_attr_make_non_resident(NTFS_I(tvi), value_len)) { iput(tvi); continue; } @@ -5179,6 +5328,7 @@ int ntfs_non_resident_attr_insert_range(struct ntfs_inode *ni, s64 start_vcn, s6 ret = ntfs_attr_map_whole_runlist(ni); if (ret) { up_write(&ni->runlist.lock); + kfree(hole_rl); return ret; } @@ -5390,6 +5540,7 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo s64 old_data_size; s64 vcn_start, vcn_end, vcn_uninit, vcn, try_alloc_cnt; s64 lcn, alloc_cnt; + s64 rl_lcn, rl_length, rl_vcn; int err = 0; struct runlist_element *rl; bool balloc; @@ -5469,19 +5620,23 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo while (vcn < vcn_uninit) { down_read(&ni->runlist.lock); rl = ntfs_attr_find_vcn_nolock(ni, vcn, NULL); - up_read(&ni->runlist.lock); if (IS_ERR(rl)) { + up_read(&ni->runlist.lock); err = PTR_ERR(rl); goto out; } + rl_lcn = rl->lcn; + rl_length = rl->length; + rl_vcn = rl->vcn; + up_read(&ni->runlist.lock); - if (rl->lcn > 0) { - vcn += rl->length - (vcn - rl->vcn); - } else if (rl->lcn == LCN_DELALLOC || rl->lcn == LCN_HOLE) { - try_alloc_cnt = min(rl->length - (vcn - rl->vcn), + if (rl_lcn > 0) { + vcn += rl_length - (vcn - rl_vcn); + } else if (rl_lcn == LCN_DELALLOC || rl_lcn == LCN_HOLE) { + try_alloc_cnt = min(rl_length - (vcn - rl_vcn), vcn_uninit - vcn); - if (rl->lcn == LCN_DELALLOC) { + if (rl_lcn == LCN_DELALLOC) { vcn += try_alloc_cnt; continue; } @@ -5496,11 +5651,14 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo if (err) goto out; - err = ntfs_dio_zero_range(VFS_I(ni), - lcn << vol->cluster_size_bits, - alloc_cnt << vol->cluster_size_bits); - if (err > 0) - goto out; + if (balloc) { + err = ntfs_dio_zero_range(VFS_I(ni), + lcn << vol->cluster_size_bits, + alloc_cnt << + vol->cluster_size_bits); + if (err > 0) + goto out; + } if (signal_pending(current)) goto out; diff --git a/fs/ntfs/attrib.h b/fs/ntfs/attrib.h index f7acc7986b09..e2224fbfaabe 100644 --- a/fs/ntfs/attrib.h +++ b/fs/ntfs/attrib.h @@ -71,6 +71,10 @@ int ntfs_attr_lookup(const __le32 type, const __le16 *name, const u32 name_len, const u32 ic, const s64 lowest_vcn, const u8 *val, const u32 val_len, struct ntfs_attr_search_ctx *ctx); +bool ntfs_attr_list_entry_is_valid(const struct attr_list_entry *ale, + const u8 *al_end); +bool ntfs_attr_list_is_valid(const u8 *al_start, s64 size); + int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size); diff --git a/fs/ntfs/attrlist.c b/fs/ntfs/attrlist.c index bd501e8a628c..be3086d34338 100644 --- a/fs/ntfs/attrlist.c +++ b/fs/ntfs/attrlist.c @@ -57,6 +57,15 @@ int ntfs_attrlist_update(struct ntfs_inode *base_ni) struct ntfs_inode *attr_ni; int err; + /* + * generic_shutdown_super() clears SB_ACTIVE before evicting cached + * inodes. Do not look up the attribute-list inode after SB_ACTIVE has + * been cleared; it may already be I_FREEING, and waiting on it can + * self-deadlock. + */ + if (!(VFS_I(base_ni)->i_sb->s_flags & SB_ACTIVE)) + return -EIO; + attr_vi = ntfs_attr_iget(VFS_I(base_ni), AT_ATTRIBUTE_LIST, AT_UNNAMED, 0); if (IS_ERR(attr_vi)) { err = PTR_ERR(attr_vi); @@ -118,16 +127,16 @@ int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr) int entry_len, entry_offset, err; struct mft_record *ni_mrec; u8 *old_al; - - ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n", - (long long) ni->mft_no, - (unsigned int) le32_to_cpu(attr->type)); + __le64 lowest_vcn; if (!ni || !attr) { ntfs_debug("Invalid arguments.\n"); return -EINVAL; } + ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n", + ni->mft_no, (unsigned int) le32_to_cpu(attr->type)); + ni_mrec = map_mft_record(ni); if (IS_ERR(ni_mrec)) { ntfs_debug("Invalid arguments.\n"); @@ -159,17 +168,21 @@ int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr) ntfs_error(ni->vol->sb, "Failed to get search context"); goto err_out; } + if (attr->non_resident) + lowest_vcn = attr->data.non_resident.lowest_vcn; + else + lowest_vcn = 0; err = ntfs_attr_lookup(attr->type, (attr->name_length) ? (__le16 *) ((u8 *)attr + le16_to_cpu(attr->name_offset)) : AT_UNNAMED, attr->name_length, CASE_SENSITIVE, - (attr->non_resident) ? le64_to_cpu(attr->data.non_resident.lowest_vcn) : - 0, (attr->non_resident) ? NULL : ((u8 *)attr + + le64_to_cpu(lowest_vcn), + (attr->non_resident) ? NULL : ((u8 *)attr + le16_to_cpu(attr->data.resident.value_offset)), (attr->non_resident) ? 0 : le32_to_cpu(attr->data.resident.value_length), ctx); if (!err) { /* Found some extent, check it to be before new extent. */ - if (ctx->al_entry->lowest_vcn == attr->data.non_resident.lowest_vcn) { + if (ctx->al_entry->lowest_vcn == lowest_vcn) { err = -EEXIST; ntfs_debug("Such attribute already present in the attribute list.\n"); ntfs_attr_put_search_ctx(ctx); diff --git a/fs/ntfs/bdev-io.c b/fs/ntfs/bdev-io.c index 67e65c88d681..27d7c2767a33 100644 --- a/fs/ntfs/bdev-io.c +++ b/fs/ntfs/bdev-io.c @@ -97,6 +97,8 @@ int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size idx_end++; for (; idx < idx_end; idx++, from = 0) { + u32 len; + folio = read_mapping_folio(sb->s_bdev->bd_mapping, idx, NULL); if (IS_ERR(folio)) { ntfs_error(sb, "Unable to read %ld page", idx); @@ -105,9 +107,10 @@ int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size offset = (loff_t)idx << PAGE_SHIFT; to = min_t(u32, end - offset, PAGE_SIZE); + len = to - from; - memcpy_to_folio(folio, from, buf + buf_off, to); - buf_off += to; + memcpy_to_folio(folio, from, buf + buf_off, len); + buf_off += len; folio_mark_uptodate(folio); folio_mark_dirty(folio); folio_put(folio); diff --git a/fs/ntfs/dir.c b/fs/ntfs/dir.c index 20f5c7074bdd..6fa9ae3377cb 100644 --- a/fs/ntfs/dir.c +++ b/fs/ntfs/dir.c @@ -23,6 +23,13 @@ __le16 I30[5] = { cpu_to_le16('$'), cpu_to_le16('I'), cpu_to_le16('3'), cpu_to_le16('0'), 0 }; +static inline u64 ntfs_check_mref(u64 mref) +{ + if (IS_ERR_MREF(mref)) + return ERR_MREF(-EIO); + return mref; +} + /* * ntfs_lookup_inode_by_name - find an inode in a directory given its name * @dir_ni: ntfs inode of the directory in which to search for the name @@ -135,10 +142,6 @@ u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, /* Key length should not be zero if it is not last entry. */ if (!ie->key_length) goto dir_err_out; - /* Check the consistency of an index entry */ - if (ntfs_index_entry_inconsistent(NULL, vol, ie, COLLATION_FILE_NAME, - dir_ni->mft_no)) - goto dir_err_out; /* * We perform a case sensitive comparison and if that matches * we are done and return the mft reference of the inode (i.e. @@ -182,7 +185,7 @@ found_it: mref = le64_to_cpu(ie->data.dir.indexed_file); ntfs_attr_put_search_ctx(ctx); unmap_mft_record(dir_ni); - return mref; + return ntfs_check_mref(mref); } /* * For a case insensitive mount, we also perform a case @@ -277,7 +280,7 @@ found_it: if (name) { ntfs_attr_put_search_ctx(ctx); unmap_mft_record(dir_ni); - return name->mref; + return ntfs_check_mref(name->mref); } ntfs_debug("Entry not found."); err = -ENOENT; @@ -342,43 +345,20 @@ fast_descend_into_child_node: dir_ni->mft_no); goto unm_err_out; } - /* Catch multi sector transfer fixup errors. */ - if (unlikely(!ntfs_is_indx_record(ia->magic))) { - ntfs_error(sb, - "Directory index record with vcn 0x%llx is corrupt. Corrupt inode 0x%llx. Run chkdsk.", - vcn, dir_ni->mft_no); - goto unm_err_out; - } - if (le64_to_cpu(ia->index_block_vcn) != vcn) { - ntfs_error(sb, - "Actual VCN (0x%llx) of index buffer is different from expected VCN (0x%llx). Directory inode 0x%llx is corrupt or driver bug.", - le64_to_cpu(ia->index_block_vcn), - vcn, dir_ni->mft_no); - goto unm_err_out; - } - if (le32_to_cpu(ia->index.allocated_size) + 0x18 != - dir_ni->itype.index.block_size) { - ntfs_error(sb, - "Index buffer (VCN 0x%llx) of directory inode 0x%llx has a size (%u) differing from the directory specified size (%u). Directory inode is corrupt or driver bug.", - vcn, dir_ni->mft_no, - le32_to_cpu(ia->index.allocated_size) + 0x18, - dir_ni->itype.index.block_size); - goto unm_err_out; - } index_end = (u8 *)ia + dir_ni->itype.index.block_size; if (index_end > kaddr + PAGE_SIZE) { ntfs_error(sb, - "Index buffer (VCN 0x%llx) of directory inode 0x%llx crosses page boundary. Impossible! Cannot access! This is probably a bug in the driver.", - vcn, dir_ni->mft_no); + "Index buffer (VCN 0x%llx) of directory inode 0x%llx crosses page boundary. Impossible! Cannot access! This is probably a bug in the driver.", + vcn, dir_ni->mft_no); goto unm_err_out; } - index_end = (u8 *)&ia->index + le32_to_cpu(ia->index.index_length); - if (index_end > (u8 *)ia + dir_ni->itype.index.block_size) { - ntfs_error(sb, - "Size of index buffer (VCN 0x%llx) of directory inode 0x%llx exceeds maximum size.", - vcn, dir_ni->mft_no); + err = ntfs_index_block_inconsistent(vol, ia, + dir_ni->itype.index.block_size, + vcn, COLLATION_FILE_NAME, + dir_ni->mft_no); + if (err) goto unm_err_out; - } + index_end = (u8 *)&ia->index + le32_to_cpu(ia->index.index_length); /* The first index entry. */ ie = (struct index_entry *)((u8 *)&ia->index + le32_to_cpu(ia->index.entries_offset)); @@ -388,15 +368,6 @@ fast_descend_into_child_node: * reach the last entry. */ for (;; ie = (struct index_entry *)((u8 *)ie + le16_to_cpu(ie->length))) { - /* Bounds checks. */ - if ((u8 *)ie < (u8 *)ia || - (u8 *)ie + sizeof(struct index_entry_header) > index_end || - (u8 *)ie + sizeof(struct index_entry_header) + le16_to_cpu(ie->key_length) > - index_end || (u8 *)ie + le16_to_cpu(ie->length) > index_end) { - ntfs_error(sb, "Index entry out of bounds in directory inode 0x%llx.", - dir_ni->mft_no); - goto unm_err_out; - } /* * The last entry cannot contain a name. It can however contain * a pointer to a child node in the B+tree so we just break out. @@ -406,10 +377,6 @@ fast_descend_into_child_node: /* Key length should not be zero if it is not last entry. */ if (!ie->key_length) goto unm_err_out; - /* Check the consistency of an index entry */ - if (ntfs_index_entry_inconsistent(NULL, vol, ie, COLLATION_FILE_NAME, - dir_ni->mft_no)) - goto unm_err_out; /* * We perform a case sensitive comparison and if that matches * we are done and return the mft reference of the inode (i.e. @@ -453,7 +420,7 @@ found_it2: mref = le64_to_cpu(ie->data.dir.indexed_file); kfree(kaddr); iput(ia_vi); - return mref; + return ntfs_check_mref(mref); } /* * For a case insensitive mount, we also perform a case @@ -578,7 +545,7 @@ found_it2: if (name) { kfree(kaddr); iput(ia_vi); - return name->mref; + return ntfs_check_mref(name->mref); } ntfs_debug("Entry not found."); err = -ENOENT; @@ -892,6 +859,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *actor) ictx->vcn_size_bits = vol->cluster_size_bits; else ictx->vcn_size_bits = NTFS_BLOCK_SIZE_BITS; + ictx->cr = ir->collation_rule; /* The first index entry. */ next = (struct index_entry *)((u8 *)&ir->index + @@ -929,13 +897,6 @@ static int ntfs_readdir(struct file *file, struct dir_context *actor) if (!next) break; nextdir: - /* Check the consistency of an index entry */ - if (ntfs_index_entry_inconsistent(ictx, vol, next, COLLATION_FILE_NAME, - ndir->mft_no)) { - err = -EIO; - goto out; - } - if (ie_pos < actor->pos) { ie_pos += le16_to_cpu(next->length); continue; diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c index c4a4a3e3e599..0cd192752b7c 100644 --- a/fs/ntfs/ea.c +++ b/fs/ntfs/ea.c @@ -53,11 +53,11 @@ static int ntfs_ea_lookup(char *ea_buf, s64 ea_buf_size, const char *name, loff_t offset, p_ea_size; unsigned int next; - if (ea_buf_size < sizeof(struct ea_attr)) - goto out; - offset = 0; do { + if (ea_buf_size - offset < sizeof(struct ea_attr)) + break; + p_ea = (const struct ea_attr *)&ea_buf[offset]; next = le32_to_cpu(p_ea->next_entry_offset); p_ea_size = next ? next : (ea_buf_size - offset); @@ -479,13 +479,13 @@ ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size) if (ea_info_qsize > ea_buf_size || ea_info_qsize == 0) goto out; - if (ea_info_qsize < sizeof(struct ea_attr)) { - err = -EIO; - goto out; - } - offset = 0; do { + if (ea_info_qsize - offset < sizeof(struct ea_attr)) { + err = -EIO; + goto out; + } + p_ea = (const struct ea_attr *)&ea_buf[offset]; next = le32_to_cpu(p_ea->next_entry_offset); ea_size = next ? next : (ea_info_qsize - offset); diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index e8bea22b81a7..6a7b638e523d 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -22,6 +22,7 @@ #include "ea.h" #include "iomap.h" #include "bitmap.h" +#include "volume.h" #include <linux/filelock.h> @@ -675,10 +676,29 @@ static int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, static const char *ntfs_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) { - if (!NTFS_I(inode)->target) + struct ntfs_inode *ni = NTFS_I(inode); + char *target; + int err; + + if (!dentry) + return ERR_PTR(-ECHILD); + + if (!ni->target) return ERR_PTR(-EINVAL); - return NTFS_I(inode)->target; + if (ni->reparse_tag == IO_REPARSE_TAG_MOUNT_POINT || + (ni->reparse_tag == IO_REPARSE_TAG_SYMLINK && + !(ni->reparse_flags & cpu_to_le32(SYMLINK_FLAG_RELATIVE)))) { + if (NVolNativeSymlinkRel(ni->vol)) { + err = ntfs_translate_symlink_path(dentry, ni->target, &target); + if (err < 0) + return ERR_PTR(err); + set_delayed_call(done, kfree_link, target); + return target; + } + } + + return ni->target; } static ssize_t ntfs_file_splice_read(struct file *in, loff_t *ppos, @@ -707,12 +727,21 @@ static int ntfs_ioctl_get_volume_label(struct file *filp, unsigned long arg) { struct ntfs_volume *vol = NTFS_SB(file_inode(filp)->i_sb); char __user *buf = (char __user *)arg; + char label[FSLABEL_MAX]; + ssize_t len; + mutex_lock(&vol->volume_label_lock); if (!vol->volume_label) { - if (copy_to_user(buf, "", 1)) - return -EFAULT; - } else if (copy_to_user(buf, vol->volume_label, - MIN(FSLABEL_MAX, strlen(vol->volume_label) + 1))) + label[0] = '\0'; + len = 0; + } else { + len = strscpy(label, vol->volume_label, sizeof(label)); + if (len == -E2BIG) + len = FSLABEL_MAX - 1; + } + mutex_unlock(&vol->volume_label_lock); + + if (copy_to_user(buf, label, len + 1)) return -EFAULT; return 0; } diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c index a547bdcfa456..faa7ee920a3a 100644 --- a/fs/ntfs/index.c +++ b/fs/ntfs/index.c @@ -28,41 +28,10 @@ * length must have been checked beforehand to not overflow from the * index record. */ -int ntfs_index_entry_inconsistent(struct ntfs_index_context *icx, - struct ntfs_volume *vol, const struct index_entry *ie, - __le32 collation_rule, u64 inum) +static int ntfs_index_entry_inconsistent(const struct ntfs_volume *vol, + const struct index_entry *ie, + __le32 collation_rule, u64 inum) { - if (icx) { - struct index_header *ih; - u8 *ie_start, *ie_end; - - if (icx->is_in_root) - ih = &icx->ir->index; - else - ih = &icx->ib->index; - - if ((le32_to_cpu(ih->index_length) > le32_to_cpu(ih->allocated_size)) || - (le32_to_cpu(ih->index_length) > icx->block_size)) { - ntfs_error(vol->sb, "%s Index entry(0x%p)'s length is too big.", - icx->is_in_root ? "Index root" : "Index block", - (u8 *)icx->entry); - return -EINVAL; - } - - ie_start = (u8 *)ih + le32_to_cpu(ih->entries_offset); - ie_end = (u8 *)ih + le32_to_cpu(ih->index_length); - - if (ie_start > (u8 *)ie || - ie_end <= (u8 *)ie + le16_to_cpu(ie->length) || - le16_to_cpu(ie->length) > le32_to_cpu(ih->allocated_size) || - le16_to_cpu(ie->length) > icx->block_size) { - ntfs_error(vol->sb, "Index entry(0x%p) is out of range from %s", - (u8 *)icx->entry, - icx->is_in_root ? "index root" : "index block"); - return -EIO; - } - } - if (ie->key_length && ((le16_to_cpu(ie->key_length) + offsetof(struct index_entry, key)) > le16_to_cpu(ie->length))) { @@ -141,6 +110,10 @@ static int ntfs_ib_write(struct ntfs_index_context *icx, struct index_block *ib) ret = ntfs_inode_attr_pwrite(VFS_I(icx->ia_ni), ntfs_ib_vcn_to_pos(icx, vcn), icx->block_size, (u8 *)ib, icx->sync_write); + + /* Perform data restoration before returning */ + post_write_mst_fixup((struct ntfs_record *)ib); + if (ret != icx->block_size) { ntfs_debug("Failed to write index block %lld, inode %llu", vcn, (unsigned long long)icx->idx_ni->mft_no); @@ -178,7 +151,6 @@ int ntfs_icx_ib_sync_write(struct ntfs_index_context *icx) icx->ib = NULL; icx->ib_dirty = false; } else { - post_write_mst_fixup((struct ntfs_record *)icx->ib); icx->sync_write = false; } @@ -303,6 +275,93 @@ static int ntfs_ie_end(struct index_entry *ie) return ie->flags & INDEX_ENTRY_END || !ie->length; } +static int ntfs_index_header_inconsistent(struct ntfs_volume *vol, + const struct index_header *ih, + u32 bytes_available, u64 inum) +{ + u32 entries_offset, index_length, allocated_size; + + if (bytes_available < sizeof(struct index_header)) { + ntfs_error(vol->sb, + "index block in inode %llu is smaller than an index header.", + (unsigned long long)inum); + return -EIO; + } + + entries_offset = le32_to_cpu(ih->entries_offset); + index_length = le32_to_cpu(ih->index_length); + allocated_size = le32_to_cpu(ih->allocated_size); + + if (entries_offset < sizeof(struct index_header) || + entries_offset > bytes_available) { + ntfs_error(vol->sb, + "Invalid index entry offset in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (index_length <= entries_offset) { + ntfs_error(vol->sb, + "No space for index entries in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (allocated_size < index_length) { + ntfs_error(vol->sb, + "Index entries overflow in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (allocated_size > bytes_available || index_length > bytes_available) { + ntfs_error(vol->sb, + "Index entries in inode %llu exceed the available buffer.", + (unsigned long long)inum); + return -EIO; + } + + return 0; +} + +int ntfs_index_entries_inconsistent(const struct ntfs_volume *vol, + const struct index_header *ih, + __le32 collation_rule, u64 inum) +{ + struct index_entry *ie; + u8 *index_end = (u8 *)ih + le32_to_cpu(ih->index_length); + + for (ie = ntfs_ie_get_first((struct index_header *)ih); + ; ie = ntfs_ie_get_next(ie)) { + if ((u8 *)ie + sizeof(struct index_entry_header) > index_end || + (u8 *)ie + le16_to_cpu(ie->length) > index_end) { + ntfs_error(vol->sb, + "Index entry out of bounds in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (le16_to_cpu(ie->length) < sizeof(struct index_entry_header)) { + ntfs_error(vol->sb, + "Index entry too small in inode %llu.", + inum); + return -EIO; + } + + if (ntfs_ie_end(ie)) + break; + + if (!ie->key_length) + return -EIO; + + if (ntfs_index_entry_inconsistent(vol, ie, + collation_rule, inum)) + return -EIO; + } + + return 0; +} + /* * Find the last entry in the index block */ @@ -437,7 +496,7 @@ static struct index_entry *ntfs_ie_dup_novcn(struct index_entry *ie) * The size of block is assumed to have been checked to be what is * defined in the index root. * - * Returns 0 if no error was found -1 otherwise (with errno unchanged) + * Returns 0 if no error was found, -EIO otherwise * * |<--->| offsetof(struct index_block, index) * | |<--->| sizeof(struct index_header) @@ -452,21 +511,21 @@ static struct index_entry *ntfs_ie_dup_novcn(struct index_entry *ie) * * size(struct index_header) <= ent_offset < ind_length <= alloc_size < bk_size */ -static int ntfs_index_block_inconsistent(struct ntfs_index_context *icx, - struct index_block *ib, s64 vcn) +int ntfs_index_block_inconsistent(struct ntfs_volume *vol, + const struct index_block *ib, + u32 block_size, s64 vcn, __le32 cr, + u64 inum) { u32 ib_size = (unsigned int)le32_to_cpu(ib->index.allocated_size) + offsetof(struct index_block, index); - struct super_block *sb = icx->idx_ni->vol->sb; - unsigned long long inum = icx->idx_ni->mft_no; + struct super_block *sb = vol->sb; ntfs_debug("Entering\n"); if (!ntfs_is_indx_record(ib->magic)) { - ntfs_error(sb, "Corrupt index block signature: vcn %lld inode %llu\n", - vcn, (unsigned long long)icx->idx_ni->mft_no); - return -1; + vcn, (unsigned long long)inum); + return -EIO; } if (le64_to_cpu(ib->index_block_vcn) != vcn) { @@ -474,32 +533,42 @@ static int ntfs_index_block_inconsistent(struct ntfs_index_context *icx, "Corrupt index block: s64 (%lld) is different from expected s64 (%lld) in inode %llu\n", (long long)le64_to_cpu(ib->index_block_vcn), vcn, inum); - return -1; + return -EIO; } - if (ib_size != icx->block_size) { + if (ib_size != block_size) { ntfs_error(sb, - "Corrupt index block : s64 (%lld) of inode %llu has a size (%u) differing from the index specified size (%u)\n", - vcn, inum, ib_size, icx->block_size); - return -1; + "Corrupt index block : s64 (%lld) of inode %llu has a size (%u) differing from the index specified size (%u)\n", + vcn, inum, ib_size, block_size); + return -EIO; } - if (le32_to_cpu(ib->index.entries_offset) < sizeof(struct index_header)) { - ntfs_error(sb, "Invalid index entry offset in inode %lld\n", inum); - return -1; - } - if (le32_to_cpu(ib->index.index_length) <= - le32_to_cpu(ib->index.entries_offset)) { - ntfs_error(sb, "No space for index entries in inode %lld\n", inum); - return -1; - } - if (le32_to_cpu(ib->index.allocated_size) < - le32_to_cpu(ib->index.index_length)) { - ntfs_error(sb, "Index entries overflow in inode %lld\n", inum); - return -1; + if (ntfs_index_header_inconsistent(vol, &ib->index, + block_size - + offsetof(struct index_block, index), + inum)) + return -EIO; + if (ntfs_index_entries_inconsistent(vol, &ib->index, cr, inum)) + return -EIO; + return 0; +} + +int ntfs_index_root_inconsistent(struct ntfs_volume *vol, + const struct attr_record *a, + const struct index_root *ir, u64 inum) +{ + u32 value_length = le32_to_cpu(a->data.resident.value_length); + + if (value_length < offsetof(struct index_root, index)) { + ntfs_error(vol->sb, "$INDEX_ROOT in inode %llu is too small.", + (unsigned long long)inum); + return -EIO; } - return 0; + return ntfs_index_header_inconsistent(vol, &ir->index, + value_length - + offsetof(struct index_root, index), + inum); } static struct index_root *ntfs_ir_lookup(struct ntfs_inode *ni, __le16 *name, @@ -665,23 +734,24 @@ static int ntfs_ib_read(struct ntfs_index_context *icx, s64 vcn, struct index_bl else ntfs_error(icx->idx_ni->vol->sb, "Failed to read full index block at %lld\n", pos); - return -1; + return -EIO; } post_read_mst_fixup((struct ntfs_record *)((u8 *)dst), icx->block_size); - if (ntfs_index_block_inconsistent(icx, dst, vcn)) - return -1; - + if (ntfs_index_block_inconsistent(icx->idx_ni->vol, dst, + icx->block_size, vcn, icx->cr, + icx->idx_ni->mft_no)) + return -EIO; return 0; } static int ntfs_icx_parent_inc(struct ntfs_index_context *icx) { - icx->pindex++; - if (icx->pindex >= MAX_PARENT_VCN) { + if (icx->pindex >= MAX_PARENT_VCN - 1) { ntfs_error(icx->idx_ni->vol->sb, "Index is over %d level deep", MAX_PARENT_VCN); return -EOPNOTSUPP; } + icx->pindex++; return 0; } @@ -1173,6 +1243,8 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) struct index_entry *ie; struct index_block *ib = NULL; s64 new_ib_vcn; + u32 index_length; + u32 old_value_length; int ix_root_size; int ret = 0; @@ -1220,6 +1292,21 @@ retry: goto clear_bmp; } + old_value_length = le32_to_cpu(ctx->attr->data.resident.value_length); + index_length = le32_to_cpu(ir->index.entries_offset) + + sizeof(struct index_entry_header) + sizeof(s64); + ix_root_size = offsetof(struct index_root, index) + index_length; + /* Grow the resident value before publishing the larger root header. */ + if (ix_root_size > old_value_length) { + ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); + if (ret) + goto resize_failed; + + icx->idx_ni->data_size = ix_root_size; + icx->idx_ni->initialized_size = ix_root_size; + icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; + } + ntfs_ir_nill(ir); ie = ntfs_ie_get_first(&ir->index); @@ -1228,48 +1315,49 @@ retry: ir->index.flags = LARGE_INDEX; NInoSetIndexAllocPresent(icx->idx_ni); - ir->index.index_length = cpu_to_le32(le32_to_cpu(ir->index.entries_offset) + - le16_to_cpu(ie->length)); + ir->index.index_length = cpu_to_le32(index_length); ir->index.allocated_size = ir->index.index_length; - ix_root_size = sizeof(struct index_root) - sizeof(struct index_header) + - le32_to_cpu(ir->index.allocated_size); - ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); - if (ret) { - /* - * When there is no space to build a non-resident - * index, we may have to move the root to an extent - */ - if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) { + if (ix_root_size <= old_value_length) { + ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); + if (ret) + goto resize_failed; + + icx->idx_ni->data_size = ix_root_size; + icx->idx_ni->initialized_size = ix_root_size; + icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; + } + ntfs_ie_set_vcn(ie, new_ib_vcn); + goto err_out; + +resize_failed: + /* + * When there is no space to build a non-resident + * index, we may have to move the root to an extent + */ + if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) { + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); + if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size - + le32_to_cpu(ctx->attr->data.resident.value_length))) { + if (ntfs_attrlist_update(ctx->base_ntfs_ino ? + ctx->base_ntfs_ino : ctx->ntfs_ino)) + goto clear_bmp; ntfs_attr_put_search_ctx(ctx); ctx = NULL; - ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); - if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size - - le32_to_cpu(ctx->attr->data.resident.value_length))) { - if (ntfs_attrlist_update(ctx->base_ntfs_ino ? - ctx->base_ntfs_ino : ctx->ntfs_ino)) - goto clear_bmp; - ntfs_attr_put_search_ctx(ctx); - ctx = NULL; - goto retry; - } + goto retry; } - goto clear_bmp; - } else { - icx->idx_ni->data_size = icx->idx_ni->initialized_size = ix_root_size; - icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; } - ntfs_ie_set_vcn(ie, new_ib_vcn); - +clear_bmp: + ntfs_ibm_clear(icx, new_ib_vcn); + goto err_out; err_out: kvfree(ib); if (ctx) ntfs_attr_put_search_ctx(ctx); out: return ret; -clear_bmp: - ntfs_ibm_clear(icx, new_ib_vcn); - goto err_out; } /* @@ -1280,9 +1368,16 @@ clear_bmp: static int ntfs_ir_truncate(struct ntfs_index_context *icx, int data_size) { int ret; + u32 old_allocated_size; + bool shrink; ntfs_debug("Entering\n"); + old_allocated_size = le32_to_cpu(icx->ir->index.allocated_size); + shrink = data_size < old_allocated_size; + if (shrink) + icx->ir->index.allocated_size = cpu_to_le32(data_size); + /* * INDEX_ROOT must be resident and its entries can be moved to * struct index_block, so ENOSPC isn't a real error. @@ -1294,9 +1389,14 @@ static int ntfs_ir_truncate(struct ntfs_index_context *icx, int data_size) if (!icx->ir) return -ENOENT; - icx->ir->index.allocated_size = cpu_to_le32(data_size); - } else if (ret != -ENOSPC) - ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate INDEX_ROOT"); + if (!shrink) + icx->ir->index.allocated_size = cpu_to_le32(data_size); + } else { + if (shrink) + icx->ir->index.allocated_size = cpu_to_le32(old_allocated_size); + if (ret != -ENOSPC) + ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate INDEX_ROOT"); + } return ret; } @@ -1970,6 +2070,7 @@ struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_ind { struct index_entry *entry; struct index_block *ib; + int err; s64 vcn; entry = ie; @@ -1979,14 +2080,20 @@ struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_ind ib = kvzalloc(ictx->block_size, GFP_NOFS); if (!ib) return ERR_PTR(-ENOMEM); - /* down from level zero */ + /* + * Descending from root index (level 0) to the first + * child level. is_in_root == true implies pindex == 0, + * so advance to level 1. + */ + ictx->pindex = 1; ictx->ir = NULL; ictx->ib = ib; - ictx->pindex = 1; ictx->is_in_root = false; } else { /* down from non-zero level */ - ictx->pindex++; + err = ntfs_icx_parent_inc(ictx); + if (err) + return ERR_PTR(err); } ictx->parent_pos[ictx->pindex] = 0; diff --git a/fs/ntfs/index.h b/fs/ntfs/index.h index e68d6fabaf9f..9a03f53bba47 100644 --- a/fs/ntfs/index.h +++ b/fs/ntfs/index.h @@ -89,8 +89,16 @@ struct ntfs_index_context { bool sync_write; }; -int ntfs_index_entry_inconsistent(struct ntfs_index_context *icx, struct ntfs_volume *vol, - const struct index_entry *ie, __le32 collation_rule, u64 inum); +int ntfs_index_root_inconsistent(struct ntfs_volume *vol, + const struct attr_record *a, + const struct index_root *ir, u64 inum); +int ntfs_index_block_inconsistent(struct ntfs_volume *vol, + const struct index_block *ib, + u32 block_size, s64 vcn, + __le32 cr, u64 inum); +int ntfs_index_entries_inconsistent(const struct ntfs_volume *vol, + const struct index_header *ih, + __le32 collation_rule, u64 inum); struct ntfs_index_context *ntfs_index_ctx_get(struct ntfs_inode *ni, __le16 *name, u32 name_len); void ntfs_index_ctx_put(struct ntfs_index_context *ictx); diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 16890d411194..7381a18cfadd 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -488,6 +488,8 @@ void __ntfs_init_inode(struct super_block *sb, struct ntfs_inode *ni) ni->flags = 0; ni->mft_lcn[0] = LCN_RL_NOT_MAPPED; ni->mft_lcn_count = 0; + ni->reparse_tag = 0; + ni->reparse_flags = 0; ni->target = NULL; ni->i_dealloc_clusters = 0; } @@ -848,6 +850,12 @@ static int ntfs_read_locked_inode(struct inode *vi) a->data.resident.value_offset), le32_to_cpu( a->data.resident.value_length)); + /* A resident list is not validated on load; check it now. */ + if (!ntfs_attr_list_is_valid(ni->attr_list, + ni->attr_list_size)) { + ntfs_error(vi->i_sb, "Corrupt attribute list."); + goto unm_err_out; + } } } skip_attr_list_load: @@ -857,8 +865,26 @@ skip_attr_list_load: ntfs_ea_get_wsl_inode(vi, &dev, flags); } - if (m->flags & MFT_RECORD_IS_DIRECTORY) { + if (ni->flags & FILE_ATTR_REPARSE_POINT) { + unsigned int mode; + + mode = ntfs_make_symlink(ni); + if (mode) + vi->i_mode |= mode; + else { + vi->i_mode &= ~S_IFLNK; + if (m->flags & MFT_RECORD_IS_DIRECTORY) + vi->i_mode |= S_IFDIR; + else + vi->i_mode |= S_IFREG; + } + } else if (m->flags & MFT_RECORD_IS_DIRECTORY) { vi->i_mode |= S_IFDIR; + } else { + vi->i_mode |= S_IFREG; + } + + if (S_ISDIR(vi->i_mode)) { /* * Apply the directory permissions mask set in the mount * options. @@ -868,18 +894,6 @@ skip_attr_list_load: if (vi->i_nlink > 1) set_nlink(vi, 1); } else { - if (ni->flags & FILE_ATTR_REPARSE_POINT) { - unsigned int mode; - - mode = ntfs_make_symlink(ni); - if (mode) - vi->i_mode |= mode; - else { - vi->i_mode &= ~S_IFLNK; - vi->i_mode |= S_IFREG; - } - } else - vi->i_mode |= S_IFREG; /* Apply the file permissions mask set in the mount options. */ vi->i_mode &= ~vol->fmask; } @@ -888,9 +902,8 @@ skip_attr_list_load: * If an attribute list is present we now have the attribute list value * in ntfs_ino->attr_list and it is ntfs_ino->attr_list_size bytes. */ - if (S_ISDIR(vi->i_mode)) { + if (m->flags & MFT_RECORD_IS_DIRECTORY) { struct index_root *ir; - u8 *ir_end, *index_end; view_index_meta: /* It is a directory, find index root attribute. */ @@ -940,10 +953,9 @@ view_index_meta: } ir = (struct index_root *)((u8 *)a + le16_to_cpu(a->data.resident.value_offset)); - ir_end = (u8 *)ir + le32_to_cpu(a->data.resident.value_length); - index_end = (u8 *)&ir->index + - le32_to_cpu(ir->index.index_length); - if (index_end > ir_end) { + if (ntfs_index_root_inconsistent(ni->vol, a, ir, ni->mft_no) || + ntfs_index_entries_inconsistent(ni->vol, &ir->index, + ir->collation_rule, ni->mft_no)) { ntfs_error(vi->i_sb, "Directory index is corrupt."); goto unm_err_out; } @@ -1014,7 +1026,7 @@ view_index_meta: m = NULL; ctx = NULL; /* Setup the operations for this inode. */ - ntfs_set_vfs_operations(vi, S_IFDIR, 0); + ntfs_set_vfs_operations(vi, vi->i_mode, 0); if (ir->index.flags & LARGE_INDEX) NInoSetIndexAllocPresent(ni); } else { @@ -1180,6 +1192,15 @@ no_data_attr_special_case: vi->i_flags |= S_IMMUTABLE; /* + * System files such as $Bitmap and $MFT are maintained by the driver + * itself, and writing them from userspace corrupts the volume. + * Always make them immutable regardless of the sys_immutable option. + * Directories are skipped so the root and $Extend stay usable. + */ + if (ni->mft_no < FILE_first_user && S_ISREG(vi->i_mode)) + vi->i_flags |= S_IMMUTABLE; + + /* * The number of 512-byte blocks used on disk (for stat). This is in so * far inaccurate as it doesn't account for any named streams or other * special non-resident attributes, but that is how Windows works, too, @@ -1195,6 +1216,9 @@ no_data_attr_special_case: else vi->i_blocks = ni->allocated_size >> 9; + if (S_ISLNK(vi->i_mode) && ni->target) + vi->i_size = strlen(ni->target); + ntfs_debug("Done."); return 0; unm_err_out: @@ -1483,7 +1507,6 @@ static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi) struct attr_record *a; struct ntfs_attr_search_ctx *ctx; struct index_root *ir; - u8 *ir_end, *index_end; int err = 0; ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no); @@ -1534,9 +1557,9 @@ static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi) } ir = (struct index_root *)((u8 *)a + le16_to_cpu(a->data.resident.value_offset)); - ir_end = (u8 *)ir + le32_to_cpu(a->data.resident.value_length); - index_end = (u8 *)&ir->index + le32_to_cpu(ir->index.index_length); - if (index_end > ir_end) { + if (ntfs_index_root_inconsistent(vol, a, ir, ni->mft_no) || + ntfs_index_entries_inconsistent(vol, &ir->index, + ir->collation_rule, ni->mft_no)) { ntfs_error(vi->i_sb, "Index is corrupt."); goto unm_err_out; } @@ -1994,10 +2017,7 @@ int ntfs_read_inode_mount(struct inode *vi) /* Catch the end of the attribute list. */ if ((u8 *)al_entry == al_end) goto em_put_err_out; - if (!al_entry->length) - goto em_put_err_out; - if ((u8 *)al_entry + 6 > al_end || - (u8 *)al_entry + le16_to_cpu(al_entry->length) > al_end) + if (!ntfs_attr_list_entry_is_valid(al_entry, al_end)) goto em_put_err_out; next_al_entry = (struct attr_list_entry *)((u8 *)al_entry + le16_to_cpu(al_entry->length)); @@ -2367,6 +2387,14 @@ int ntfs_show_options(struct seq_file *sf, struct dentry *root) seq_puts(sf, ",discard"); if (NVolDisableSparse(vol)) seq_puts(sf, ",disable_sparse"); + if (NVolNativeSymlinkRel(vol)) + seq_puts(sf, ",native_symlink=rel"); + else + seq_puts(sf, ",native_symlink=raw"); + if (NVolSymlinkNative(vol)) + seq_puts(sf, ",symlink=native"); + else + seq_puts(sf, ",symlink=wsl"); if (vol->sb->s_flags & SB_POSIXACL) seq_puts(sf, ",acl"); return 0; @@ -2582,8 +2610,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni) mutex_lock_nested(&index_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); if (NInoBeingDeleted(ni)) { - iput(index_vi); mutex_unlock(&index_ni->mrec_lock); + iput(index_vi); continue; } @@ -2591,8 +2619,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni) if (!ictx) { ntfs_error(sb, "Failed to get index ctx, inode %llu", index_ni->mft_no); - iput(index_vi); mutex_unlock(&index_ni->mrec_lock); + iput(index_vi); continue; } @@ -2601,8 +2629,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni) ntfs_debug("Index lookup failed, inode %llu", index_ni->mft_no); ntfs_index_ctx_put(ictx); - iput(index_vi); mutex_unlock(&index_ni->mrec_lock); + iput(index_vi); continue; } /* Update flags and file size. */ diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h index 67942b97fac6..9aacd5787ffe 100644 --- a/fs/ntfs/inode.h +++ b/fs/ntfs/inode.h @@ -142,6 +142,8 @@ struct ntfs_inode { struct ntfs_inode *base_ntfs_ino; } ext; unsigned int i_dealloc_clusters; + __le32 reparse_tag; + __le32 reparse_flags; char *target; }; diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c index dc7d8c893a69..52eecf5cb256 100644 --- a/fs/ntfs/iomap.c +++ b/fs/ntfs/iomap.c @@ -89,7 +89,6 @@ static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, lo u32 attr_len; int err = 0; char *kattr; - struct page *ipage; if (NInoAttr(ni)) base_ni = ni->ext.base_ntfs_ino; @@ -130,18 +129,10 @@ static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, lo kattr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset); - ipage = alloc_page(GFP_NOFS | __GFP_ZERO); - if (!ipage) { - err = -ENOMEM; - goto out; - } - - memcpy(page_address(ipage), kattr, attr_len); iomap->type = IOMAP_INLINE; - iomap->inline_data = page_address(ipage); + iomap->inline_data = kattr; iomap->offset = 0; iomap->length = attr_len; - iomap->private = ipage; out: if (ctx) @@ -286,21 +277,8 @@ static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t leng srcmap, true); } -static int ntfs_read_iomap_end(struct inode *inode, loff_t pos, loff_t length, - ssize_t written, unsigned int flags, struct iomap *iomap) -{ - if (iomap->type == IOMAP_INLINE) { - struct page *ipage = iomap->private; - - put_page(ipage); - } - - return written; -} - const struct iomap_ops ntfs_read_iomap_ops = { .iomap_begin = ntfs_read_iomap_begin, - .iomap_end = ntfs_read_iomap_end, }; /* @@ -358,7 +336,6 @@ static const struct iomap_ops ntfs_zero_read_iomap_ops = { const struct iomap_ops ntfs_seek_iomap_ops = { .iomap_begin = ntfs_seek_iomap_begin, - .iomap_end = ntfs_read_iomap_end, }; int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length) @@ -659,7 +636,6 @@ static int ntfs_write_iomap_begin_resident(struct inode *inode, loff_t offset, u32 attr_len; int err = 0; char *kattr; - struct page *ipage; ctx = ntfs_attr_get_search_ctx(ni, NULL); if (!ctx) { @@ -680,24 +656,18 @@ static int ntfs_write_iomap_begin_resident(struct inode *inode, loff_t offset, attr_len = le32_to_cpu(a->data.resident.value_length); kattr = (u8 *)a + le16_to_cpu(a->data.resident.value_offset); - ipage = alloc_page(GFP_NOFS | __GFP_ZERO); - if (!ipage) { - err = -ENOMEM; - goto out; - } - - memcpy(page_address(ipage), kattr, attr_len); iomap->type = IOMAP_INLINE; - iomap->inline_data = page_address(ipage); + iomap->inline_data = kattr; iomap->offset = 0; - /* iomap requires there is only one INLINE_DATA extent */ iomap->length = attr_len; - iomap->private = ipage; out: if (ctx) ntfs_attr_put_search_ctx(ctx); - mutex_unlock(&ni->mrec_lock); + + if (err) + mutex_unlock(&ni->mrec_lock); + return err; } @@ -778,43 +748,10 @@ static int ntfs_write_iomap_end_resident(struct inode *inode, loff_t pos, unsigned int flags, struct iomap *iomap) { struct ntfs_inode *ni = NTFS_I(inode); - struct ntfs_attr_search_ctx *ctx; - u32 attr_len; - int err; - char *kattr; - struct page *ipage = iomap->private; - mutex_lock(&ni->mrec_lock); - ctx = ntfs_attr_get_search_ctx(ni, NULL); - if (!ctx) { - written = -ENOMEM; - goto err_out; - } - - err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, - CASE_SENSITIVE, 0, NULL, 0, ctx); - if (err) { - if (err == -ENOENT) - err = -EIO; - written = err; - goto err_out; - } - - /* The total length of the attribute value. */ - attr_len = le32_to_cpu(ctx->attr->data.resident.value_length); - if (pos >= attr_len || pos + written > attr_len) - goto err_out; - - kattr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset); - memcpy(kattr + pos, iomap_inline_data(iomap, pos), written); - mark_mft_record_dirty(ctx->ntfs_ino); -err_out: - if (ctx) - ntfs_attr_put_search_ctx(ctx); - put_page(ipage); + mark_mft_record_dirty(ni); mutex_unlock(&ni->mrec_lock); return written; - } static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, diff --git a/fs/ntfs/layout.h b/fs/ntfs/layout.h index d94f914e830f..9438fd9b668e 100644 --- a/fs/ntfs/layout.h +++ b/fs/ntfs/layout.h @@ -2267,6 +2267,8 @@ enum { IO_REPARSE_PLUGIN_SELECT = cpu_to_le32(0xffff0fff), }; +#define SYMLINK_FLAG_RELATIVE 1 + /* * struct reparse_point - $REPARSE_POINT attribute content (0xc0)\ * @@ -2287,6 +2289,23 @@ struct reparse_point { u8 reparse_data[]; } __packed; +struct mount_point_reparse_data { + __le16 substitute_name_offset; + __le16 substitute_name_length; + __le16 print_name_offset; + __le16 print_name_length; + __le16 path_buffer[]; +} __packed; + +struct symlink_reparse_data { + __le16 substitute_name_offset; + __le16 substitute_name_length; + __le16 print_name_offset; + __le16 print_name_length; + __le32 flags; + __le16 path_buffer[]; +} __packed; + /* * struct ea_information - $EA_INFORMATION attribute content (0xd0) * diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c index 3f8d1640f1d5..024ddee42dc8 100644 --- a/fs/ntfs/logfile.c +++ b/fs/ntfs/logfile.c @@ -132,7 +132,7 @@ static bool ntfs_check_restart_area(struct inode *vi, struct restart_page_header { u64 file_size; struct restart_area *ra; - u16 ra_ofs, ra_len, ca_ofs; + u32 ra_ofs, ra_len, ca_ofs; u8 fs_bits; ntfs_debug("Entering."); @@ -622,8 +622,7 @@ is_empty: ntfs_debug("Done."); return true; err_out: - if (rstr1_ph) - kvfree(rstr1_ph); + kvfree(rstr1_ph); return false; } @@ -710,6 +709,9 @@ map_vcn: if (unlikely(lcn == LCN_RL_NOT_MAPPED)) { vcn = rl->vcn; kvfree(empty_buf); + empty_buf = NULL; + kfree(ra); + ra = NULL; goto map_vcn; } /* If this run is not valid abort with an error. */ @@ -753,7 +755,7 @@ map_vcn: } while (start < end); } while ((++rl)->vcn < end_vcn); up_write(&log_ni->runlist.lock); - kfree(empty_buf); + kvfree(empty_buf); kfree(ra); truncate_inode_pages(log_vi->i_mapping, 0); /* Set the flag so we do not have to do it again on remount. */ diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c index 7d989267a82b..fd20d7abd6f5 100644 --- a/fs/ntfs/mft.c +++ b/fs/ntfs/mft.c @@ -30,6 +30,8 @@ int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m, { struct attr_record *a; struct super_block *sb = vol->sb; + u16 attrs_offset; + u32 bytes_in_use; if (!ntfs_is_file_record(m->magic)) { ntfs_error(sb, "Record %llu has no FILE magic (0x%x)\n", @@ -65,7 +67,16 @@ int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m, goto err_out; } - a = (struct attr_record *)((char *)m + le16_to_cpu(m->attrs_offset)); + attrs_offset = le16_to_cpu(m->attrs_offset); + bytes_in_use = le32_to_cpu(m->bytes_in_use); + + if (attrs_offset > bytes_in_use || + bytes_in_use - attrs_offset < sizeof_field(struct attr_record, type)) { + ntfs_error(sb, "Record %llu has corrupt attribute offset\n", mft_no); + goto err_out; + } + + a = (struct attr_record *)((char *)m + attrs_offset); if ((char *)a < (char *)m || (char *)a > (char *)m + vol->mft_record_size) { ntfs_error(sb, "Record %llu is corrupt\n", mft_no); goto err_out; @@ -449,7 +460,7 @@ static void ntfs_bio_end_io(struct bio *bio) int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no, struct mft_record *m) { - u8 *kmirr = NULL; + u8 *kmirr; struct folio *folio; unsigned int folio_ofs, lcn_folio_off = 0; int err = 0; @@ -479,6 +490,7 @@ int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no, kmirr = kmap_local_folio(folio, 0) + folio_ofs; /* Copy the mst protected mft record to the mirror. */ memcpy(kmirr, m, vol->mft_record_size); + kunmap_local(kmirr); if (vol->cluster_size_bits > PAGE_SHIFT) { lcn_folio_off = folio->index << PAGE_SHIFT; @@ -490,20 +502,22 @@ int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no, NTFS_B_TO_SECTOR(vol, NTFS_CLU_TO_B(vol, vol->mftmirr_lcn) + lcn_folio_off + folio_ofs); - if (!bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs)) { + if (bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs)) + err = submit_bio_wait(bio); + else err = -EIO; - bio_put(bio); - goto unlock_folio; - } + bio_put(bio); - bio->bi_end_io = ntfs_bio_end_io; - submit_bio(bio); - /* Current state: all buffers are clean, unlocked, and uptodate. */ + /* + * The in-memory mirror is now valid because we just memcpy()'d the + * mst-protected mft record into it. Mark the folio uptodate even on + * write error so a subsequent read_mapping_folio() does not refetch + * the stale on-disk mirror and overwrite this copy. The error is + * propagated to the caller via @err. + */ folio_mark_uptodate(folio); -unlock_folio: folio_unlock(folio); - kunmap_local(kmirr); folio_put(folio); if (likely(!err)) { ntfs_debug("Done."); @@ -588,20 +602,36 @@ int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int syn } /* Synchronize the mft mirror now if not @sync. */ - if (!sync && ni->mft_no < vol->mftmirr_size) - ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m); + if (!sync && ni->mft_no < vol->mftmirr_size) { + int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no, + fixup_m); + if (unlikely(sub_err) && !err) + err = sub_err; + } - folio_get(folio); - bio->bi_private = folio; - bio->bi_end_io = ntfs_bio_end_io; - submit_bio(bio); + if (sync) { + int sub_err = submit_bio_wait(bio); + + bio_put(bio); + if (unlikely(sub_err) && !err) + err = sub_err; + } else { + folio_get(folio); + bio->bi_private = folio; + bio->bi_end_io = ntfs_bio_end_io; + submit_bio(bio); + } offset += vol->cluster_size; i++; } /* If @sync, now synchronize the mft mirror. */ - if (sync && ni->mft_no < vol->mftmirr_size) - ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m); + if (sync && ni->mft_no < vol->mftmirr_size) { + int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m); + + if (unlikely(sub_err) && !err) + err = sub_err; + } kunmap_local(kaddr); if (unlikely(err)) { /* I/O error during writing. This is really bad! */ @@ -617,10 +647,10 @@ put_bio_out: bio_put(bio); err_out: /* - * Current state: all buffers are clean, unlocked, and uptodate. - * The caller should mark the base inode as bad so that no more i/o - * happens. ->drop_inode() will still be invoked so all extent inodes - * and other allocated memory will be freed. + * The caller should mark the base inode as bad so no more I/O + * happens. ->drop_inode() will still be invoked so all extent inodes + * and other allocated memory will be freed. ENOMEM is retried by + * redirtying the mft record below. */ if (err == -ENOMEM) { ntfs_error(vol->sb, @@ -713,23 +743,6 @@ static int ntfs_test_inode_wb(struct inode *vi, u64 ino, void *data) * * If the mft record is not a FILE record or it is a base mft record, we can * safely write it and return 'true'. - * - * We now know the mft record is an extent mft record. We check if the inode - * corresponding to its base mft record is in icache. If it is not, we cannot - * safely determine the state of the extent inode, so we return 'false'. - * - * We now have the base inode for the extent mft record. We check if it has an - * ntfs inode for the extent mft record attached. If not, it is safe to write - * the extent mft record and we return 'true'. - * - * If the extent inode is attached, we check if it is dirty. If so, we return - * 'false' (letting the standard write_inode path handle it). - * - * If it is not dirty, we attempt to lock the extent mft record. If the lock - * was already taken, it is not safe to write and we return 'false'. - * - * If we manage to obtain the lock we have exclusive access to the extent mft - * record. We set @locked_ni to the now locked ntfs inode and return 'true'. */ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, const struct mft_record *m, struct ntfs_inode **locked_ni, @@ -738,8 +751,7 @@ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, struct super_block *sb = vol->sb; struct inode *mft_vi = vol->mft_ino; struct inode *vi; - struct ntfs_inode *ni, *eni, **extent_nis; - int i; + struct ntfs_inode *ni; struct ntfs_attr na = {0}; ntfs_debug("Entering for inode 0x%llx.", mft_no); @@ -819,100 +831,10 @@ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, mft_no); return true; } - /* - * This is an extent mft record. Check if the inode corresponding to - * its base mft record is in icache and obtain a reference to it if it - * is. - */ - na.mft_no = MREF_LE(m->base_mft_record); - na.state = 0; - ntfs_debug("Mft record 0x%llx is an extent record. Looking for base inode 0x%llx in icache.", - mft_no, na.mft_no); - if (!na.mft_no) { - /* Balance the below iput(). */ - vi = igrab(mft_vi); - WARN_ON(vi != mft_vi); - } else { - vi = find_inode_nowait(sb, mft_no, ntfs_test_inode_wb, &na); - if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated) - return false; - } - - if (!vi) - return false; - ntfs_debug("Base inode 0x%llx is in icache.", na.mft_no); - /* - * The base inode is in icache. Check if it has the extent inode - * corresponding to this extent mft record attached. - */ - ni = NTFS_I(vi); - mutex_lock(&ni->extent_lock); - if (ni->nr_extents <= 0) { - /* - * The base inode has no attached extent inodes, write this - * extent mft record. - */ - mutex_unlock(&ni->extent_lock); - *ref_vi = vi; - ntfs_debug("Base inode 0x%llx has no attached extent inodes, write the extent record.", - na.mft_no); - return true; - } - /* Iterate over the attached extent inodes. */ - extent_nis = ni->ext.extent_ntfs_inos; - for (eni = NULL, i = 0; i < ni->nr_extents; ++i) { - if (mft_no == extent_nis[i]->mft_no) { - /* - * Found the extent inode corresponding to this extent - * mft record. - */ - eni = extent_nis[i]; - break; - } - } - /* - * If the extent inode was not attached to the base inode, write this - * extent mft record. - */ - if (!eni) { - mutex_unlock(&ni->extent_lock); - *ref_vi = vi; - ntfs_debug("Extent inode 0x%llx is not attached to its base inode 0x%llx, write the extent record.", - mft_no, na.mft_no); - return true; - } - ntfs_debug("Extent inode 0x%llx is attached to its base inode 0x%llx.", - mft_no, na.mft_no); - /* Take a reference to the extent ntfs inode. */ - atomic_inc(&eni->count); - mutex_unlock(&ni->extent_lock); - /* if extent inode is dirty, write_inode will write it */ - if (NInoDirty(eni)) { - atomic_dec(&eni->count); - *ref_vi = vi; - return false; - } - - /* - * Found the extent inode coresponding to this extent mft record. - * Try to take the mft record lock. - */ - if (unlikely(!mutex_trylock(&eni->mrec_lock))) { - atomic_dec(&eni->count); - *ref_vi = vi; - ntfs_debug("Extent mft record 0x%llx is already locked, do not write it.", - mft_no); - return false; - } - ntfs_debug("Managed to lock extent mft record 0x%llx, write it.", - mft_no); - /* - * The write has to occur while we hold the mft record lock so return - * the locked extent ntfs inode. - */ - *locked_ni = eni; - return true; + ntfs_debug("Mft record 0x%llx is an extent record, skip it.", + mft_no); + return false; } static const char *es = " Leaving inconsistent metadata. Unmount and run chkdsk."; @@ -1034,7 +956,7 @@ static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vo b = ffz((unsigned long)*byte); if (b < 8 && b >= (bit & 7)) { ll = data_pos + (bit & ~7ull) + b; - if (unlikely(ll > (1ll << 32))) { + if (unlikely(ll >= (1ll << 32))) { folio_unlock(folio); kunmap_local(buf); folio_put(folio); @@ -2715,14 +2637,16 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w s64 vcn = ntfs_pidx_to_cluster(vol, folio->index); s64 end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size); unsigned int folio_sz; - struct runlist_element *rl = NULL; loff_t i_size = i_size_read(vi); ntfs_debug("Entering for inode 0x%llx, attribute type 0x%x, folio index 0x%lx.", ni->mft_no, ni->type, folio->index); - if (!locked_nis || !ref_inos) + if (!locked_nis || !ref_inos) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); return -ENOMEM; + } /* We have to zero every time due to mmap-at-end-of-file. */ if (folio->index >= (i_size >> folio_shift(folio))) @@ -2757,19 +2681,7 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w &tni, &ref_inos[nr_ref_inos])) { unsigned int mft_record_off = 0; s64 vcn_off = vcn; - - /* - * Skip $MFT extent mft records and let them being written - * by writeback to avioid deadlocks. the $MFT runlist - * lock must be taken before $MFT extent mrec_lock is taken. - */ - if (tni && tni->nr_extents < 0 && - tni->ext.base_ntfs_ino == NTFS_I(vol->mft_ino)) { - mutex_unlock(&tni->mrec_lock); - atomic_dec(&tni->count); - iput(vol->mft_ino); - continue; - } + s64 rl_len = 0; /* * The record should be written. If a locked ntfs @@ -2789,8 +2701,12 @@ flush_bio: } if (vol->cluster_size < folio_size(folio)) { + struct runlist_element *rl; + down_write(&ni->runlist.lock); rl = ntfs_attr_vcn_to_rl(ni, vcn_off, &lcn); + if (!IS_ERR(rl)) + rl_len = rl->length - (vcn_off - rl->vcn); up_write(&ni->runlist.lock); if (IS_ERR(rl) || lcn < 0) { err = -EIO; @@ -2821,7 +2737,7 @@ flush_bio: if (vol->cluster_size == NTFS_BLOCK_SIZE && (mft_record_off || - (rl && rl->length - (vcn_off - rl->vcn) == 1) || + rl_len == 1 || mft_ofs + NTFS_BLOCK_SIZE >= PAGE_SIZE)) folio_sz = NTFS_BLOCK_SIZE; else @@ -2840,9 +2756,13 @@ flush_bio: } prev_mft_ofs = mft_ofs; - if (mft_no < vol->mftmirr_size) - ntfs_sync_mft_mirror(vol, mft_no, + if (mft_no < vol->mftmirr_size) { + int sub_err = ntfs_sync_mft_mirror(vol, mft_no, (struct mft_record *)(kaddr + mft_ofs)); + + if (unlikely(sub_err) && !err) + err = sub_err; + } } else if (ref_inos[nr_ref_inos]) nr_ref_inos++; } diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c index 96c450e62efc..5ff25e9aaa32 100644 --- a/fs/ntfs/namei.c +++ b/fs/ntfs/namei.c @@ -230,9 +230,8 @@ static struct dentry *ntfs_lookup(struct inode *dir_ino, struct dentry *dent, if (MREF_ERR(mref) == -ENOENT) { ntfs_debug("Entry was not found, adding negative dentry."); /* The dcache will handle negative entries. */ - d_add(dent, NULL); ntfs_debug("Done."); - return NULL; + return d_splice_alias(NULL, dent); } ntfs_error(vol->sb, "ntfs_lookup_ino_by_name() failed with error code %i.", -MREF_ERR(mref)); @@ -344,9 +343,9 @@ static int ntfs_sd_add_everyone(struct ntfs_inode *ni) sd_len = sizeof(struct security_descriptor_relative) + 2 * (sizeof(struct ntfs_sid) + 8) + sizeof(struct ntfs_acl) + sizeof(struct ntfs_ace) + 4; - sd = kmalloc(sd_len, GFP_NOFS); + sd = kzalloc(sd_len, GFP_NOFS); if (!sd) - return -1; + return -ENOMEM; sd->revision = 1; sd->control = SE_DACL_PRESENT | SE_SELF_RELATIVE; @@ -394,7 +393,7 @@ static int ntfs_sd_add_everyone(struct ntfs_inode *ni) static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *dir, __le16 *name, u8 name_len, mode_t mode, dev_t dev, - __le16 *target, int target_len) + const char *target, int target_len) { struct ntfs_inode *dir_ni = NTFS_I(dir); struct ntfs_volume *vol = dir_ni->vol; @@ -608,7 +607,10 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d goto err_out; if (S_ISLNK(mode)) { - err = ntfs_reparse_set_wsl_symlink(ni, target, target_len); + if (NVolSymlinkNative(vol)) + err = ntfs_reparse_set_native_symlink(ni, target, target_len); + else + err = ntfs_reparse_set_wsl_symlink(ni, target, target_len); if (!err) rollback_reparse = true; } else if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode) || @@ -1264,6 +1266,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, struct ntfs_volume *vol = NTFS_SB(sb); struct ntfs_inode *old_ni, *new_ni = NULL; struct ntfs_inode *old_dir_ni = NTFS_I(old_dir), *new_dir_ni = NTFS_I(new_dir); + bool new_dir_first = false; if (NVolShutdown(old_dir_ni->vol)) return -EIO; @@ -1299,36 +1302,39 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, old_inode = old_dentry->d_inode; new_inode = new_dentry->d_inode; old_ni = NTFS_I(old_inode); + if (new_inode) + new_ni = NTFS_I(new_inode); + if (old_dir != new_dir) + new_dir_first = is_subdir(new_dentry->d_parent, + old_dentry->d_parent); if (!(vol->vol_flags & VOLUME_IS_DIRTY)) ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); mutex_lock_nested(&old_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); - mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + if (new_ni) + mutex_lock_nested(&new_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_2); - if (NInoBeingDeleted(old_ni) || NInoBeingDeleted(old_dir_ni)) { + if (old_dir == new_dir) { + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + } else if (new_dir_first) { + mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); + } else { + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); + } + + if (NInoBeingDeleted(old_ni) || NInoBeingDeleted(old_dir_ni) || + (new_ni && NInoBeingDeleted(new_ni)) || + (old_dir != new_dir && NInoBeingDeleted(new_dir_ni))) { err = -ENOENT; - goto unlock_old; + goto err_out; } is_dir = S_ISDIR(old_inode->i_mode); if (new_inode) { - new_ni = NTFS_I(new_inode); - mutex_lock_nested(&new_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_2); - if (old_dir != new_dir) { - mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); - if (NInoBeingDeleted(new_dir_ni)) { - err = -ENOENT; - goto err_out; - } - } - - if (NInoBeingDeleted(new_ni)) { - err = -ENOENT; - goto err_out; - } - if (is_dir) { struct mft_record *ni_mrec; @@ -1346,14 +1352,6 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, err = ntfs_delete(new_ni, new_dir_ni, uname_new, new_name_len, false); if (err) goto err_out; - } else { - if (old_dir != new_dir) { - mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); - if (NInoBeingDeleted(new_dir_ni)) { - err = -ENOENT; - goto err_out; - } - } } err = __ntfs_link(old_ni, new_dir_ni, uname_new, new_name_len); @@ -1384,13 +1382,17 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, inode_inc_iversion(new_dir); err_out: - if (old_dir != new_dir) + if (old_dir == new_dir) { + mutex_unlock(&old_dir_ni->mrec_lock); + } else if (new_dir_first) { + mutex_unlock(&old_dir_ni->mrec_lock); mutex_unlock(&new_dir_ni->mrec_lock); - if (new_inode) + } else { + mutex_unlock(&new_dir_ni->mrec_lock); + mutex_unlock(&old_dir_ni->mrec_lock); + } + if (new_ni) mutex_unlock(&new_ni->mrec_lock); - -unlock_old: - mutex_unlock(&old_dir_ni->mrec_lock); mutex_unlock(&old_ni->mrec_lock); if (uname_new) kmem_cache_free(ntfs_name_cache, uname_new); @@ -1409,9 +1411,7 @@ static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir, int err = 0; struct ntfs_inode *ni; __le16 *usrc; - __le16 *utarget; int usrc_len; - int utarget_len; int symlen = strlen(symname); if (NVolShutdown(vol)) @@ -1432,23 +1432,12 @@ static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir, goto out; } - utarget_len = ntfs_nlstoucs(vol, symname, symlen, &utarget, - PATH_MAX); - if (utarget_len < 0) { - if (utarget_len != -ENAMETOOLONG) - ntfs_error(sb, "Failed to convert target name to Unicode."); - err = -ENOMEM; - kmem_cache_free(ntfs_name_cache, usrc); - goto out; - } - if (!(vol->vol_flags & VOLUME_IS_DIRTY)) ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); ni = __ntfs_create(idmap, dir, usrc, usrc_len, S_IFLNK | 0777, 0, - utarget, utarget_len); + symname, symlen); kmem_cache_free(ntfs_name_cache, usrc); - kvfree(utarget); if (IS_ERR(ni)) { err = PTR_ERR(ni); goto out; @@ -1532,8 +1521,7 @@ static int ntfs_link(struct dentry *old_dentry, struct inode *dir, if (uname_len < 0) { if (uname_len != -ENAMETOOLONG) ntfs_error(sb, "Failed to convert name to unicode."); - err = -ENOMEM; - goto out; + return -ENOMEM; } if (!(vol->vol_flags & VOLUME_IS_DIRTY)) @@ -1563,7 +1551,7 @@ static int ntfs_link(struct dentry *old_dentry, struct inode *dir, mutex_unlock(&ni->mrec_lock); out: - kfree(uname); + kmem_cache_free(ntfs_name_cache, uname); return err; } diff --git a/fs/ntfs/quota.c b/fs/ntfs/quota.c deleted file mode 100644 index b443243b58fb..000000000000 --- a/fs/ntfs/quota.c +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * NTFS kernel quota ($Quota) handling. - * - * Copyright (c) 2004 Anton Altaparmakov - */ - -#include "index.h" -#include "quota.h" -#include "debug.h" -#include "ntfs.h" - -/* - * ntfs_mark_quotas_out_of_date - mark the quotas out of date on an ntfs volume - * @vol: ntfs volume on which to mark the quotas out of date - * - * Mark the quotas out of date on the ntfs volume @vol and return 'true' on - * success and 'false' on error. - */ -bool ntfs_mark_quotas_out_of_date(struct ntfs_volume *vol) -{ - struct ntfs_index_context *ictx; - struct quota_control_entry *qce; - const __le32 qid = QUOTA_DEFAULTS_ID; - int err; - - ntfs_debug("Entering."); - if (NVolQuotaOutOfDate(vol)) - goto done; - if (!vol->quota_ino || !vol->quota_q_ino) { - ntfs_error(vol->sb, "Quota inodes are not open."); - return false; - } - inode_lock(vol->quota_q_ino); - ictx = ntfs_index_ctx_get(NTFS_I(vol->quota_q_ino), I30, 4); - if (!ictx) { - ntfs_error(vol->sb, "Failed to get index context."); - goto err_out; - } - err = ntfs_index_lookup(&qid, sizeof(qid), ictx); - if (err) { - if (err == -ENOENT) - ntfs_error(vol->sb, "Quota defaults entry is not present."); - else - ntfs_error(vol->sb, "Lookup of quota defaults entry failed."); - goto err_out; - } - if (ictx->data_len < offsetof(struct quota_control_entry, sid)) { - ntfs_error(vol->sb, "Quota defaults entry size is invalid. Run chkdsk."); - goto err_out; - } - qce = (struct quota_control_entry *)ictx->data; - if (le32_to_cpu(qce->version) != QUOTA_VERSION) { - ntfs_error(vol->sb, - "Quota defaults entry version 0x%x is not supported.", - le32_to_cpu(qce->version)); - goto err_out; - } - ntfs_debug("Quota defaults flags = 0x%x.", le32_to_cpu(qce->flags)); - /* If quotas are already marked out of date, no need to do anything. */ - if (qce->flags & QUOTA_FLAG_OUT_OF_DATE) - goto set_done; - /* - * If quota tracking is neither requested, nor enabled and there are no - * pending deletes, no need to mark the quotas out of date. - */ - if (!(qce->flags & (QUOTA_FLAG_TRACKING_ENABLED | - QUOTA_FLAG_TRACKING_REQUESTED | - QUOTA_FLAG_PENDING_DELETES))) - goto set_done; - /* - * Set the QUOTA_FLAG_OUT_OF_DATE bit thus marking quotas out of date. - * This is verified on WinXP to be sufficient to cause windows to - * rescan the volume on boot and update all quota entries. - */ - qce->flags |= QUOTA_FLAG_OUT_OF_DATE; - /* Ensure the modified flags are written to disk. */ - ntfs_index_entry_mark_dirty(ictx); -set_done: - ntfs_index_ctx_put(ictx); - inode_unlock(vol->quota_q_ino); - /* - * We set the flag so we do not try to mark the quotas out of date - * again on remount. - */ - NVolSetQuotaOutOfDate(vol); -done: - ntfs_debug("Done."); - return true; -err_out: - if (ictx) - ntfs_index_ctx_put(ictx); - inode_unlock(vol->quota_q_ino); - return false; -} diff --git a/fs/ntfs/quota.h b/fs/ntfs/quota.h deleted file mode 100644 index 4b7322661a32..000000000000 --- a/fs/ntfs/quota.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Defines for NTFS kernel quota ($Quota) handling. - * - * Copyright (c) 2004 Anton Altaparmakov - */ - -#ifndef _LINUX_NTFS_QUOTA_H -#define _LINUX_NTFS_QUOTA_H - -#include "volume.h" - -bool ntfs_mark_quotas_out_of_date(struct ntfs_volume *vol); - -#endif /* _LINUX_NTFS_QUOTA_H */ diff --git a/fs/ntfs/reparse.c b/fs/ntfs/reparse.c index 74713716813f..fa523dc3691e 100644 --- a/fs/ntfs/reparse.c +++ b/fs/ntfs/reparse.c @@ -24,6 +24,47 @@ struct wsl_link_reparse_data { char link[]; }; +static bool reparse_name_is_valid(size_t size, size_t name_off, u16 len) +{ + if ((name_off | len) & 1) + return false; + + return name_off + len <= size; +} + +/* + * Windows-native reparse payloads store pathnames as UTF-16 strings with '\\' + * separators. Convert the on-disk UTF-16 target into the mount's NLS and + * normalize path separators. + */ +static int ntfs_reparse_target_to_nls(struct ntfs_volume *vol, + const __le16 *uname, u16 ulen, + char **target) +{ + int err, i; + + *target = NULL; + ulen >>= 1; + if (!ulen) + return -EINVAL; + + if (!uname[ulen - 1]) + ulen--; + + err = ntfs_ucstonls(vol, uname, ulen, (unsigned char **)target, 0); + if (err < 0) { + ntfs_attr_name_free((unsigned char **)target); + return err; + } + + for (i = 0; i < err; i++) { + if ((*target)[i] == '\\') + (*target)[i] = '/'; + } + + return 0; +} + /* Index entry in $Extend/$Reparse */ struct reparse_index { struct index_entry_header header; @@ -38,8 +79,10 @@ __le16 reparse_index_name[] = {cpu_to_le16('$'), cpu_to_le16('R'), 0}; * Check if the reparse point attribute buffer is valid. * Returns true if valid, false otherwise. */ -static bool ntfs_is_valid_reparse_buffer(struct ntfs_inode *ni, - const struct reparse_point *reparse_attr, size_t size) +static bool valid_reparse_buffer(struct ntfs_inode *ni, + const struct reparse_point *reparse_attr, + size_t size, + size_t payload_min_len) { size_t expected; @@ -50,6 +93,11 @@ static bool ntfs_is_valid_reparse_buffer(struct ntfs_inode *ni, if (size < sizeof(struct reparse_point)) return false; + /* The payload must contain the fixed fields for the current tag. */ + if (payload_min_len && + le16_to_cpu(reparse_attr->reparse_data_length) < payload_min_len) + return false; + /* Reserved zero tag is invalid */ if (reparse_attr->reparse_tag == IO_REPARSE_TAG_RESERVED_ZERO) return false; @@ -79,35 +127,97 @@ static bool ntfs_is_valid_reparse_buffer(struct ntfs_inode *ni, static bool valid_reparse_data(struct ntfs_inode *ni, const struct reparse_point *reparse_attr, size_t size) { - const struct wsl_link_reparse_data *wsl_reparse_data = - (const struct wsl_link_reparse_data *)reparse_attr->reparse_data; - unsigned int data_len = le16_to_cpu(reparse_attr->reparse_data_length); - - if (ntfs_is_valid_reparse_buffer(ni, reparse_attr, size) == false) + if (size < sizeof(*reparse_attr)) return false; switch (reparse_attr->reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: + { + struct mount_point_reparse_data *data; + size_t data_offs; + + if (!valid_reparse_buffer(ni, reparse_attr, size, sizeof(*data))) + return false; + + data = (struct mount_point_reparse_data *)reparse_attr->reparse_data; + data_offs = offsetof(struct reparse_point, reparse_data) + + offsetof(struct mount_point_reparse_data, path_buffer); + + if (!reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->substitute_name_offset), + le16_to_cpu(data->substitute_name_length)) || + !reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->print_name_offset), + le16_to_cpu(data->print_name_length))) + return false; + break; + } + case IO_REPARSE_TAG_SYMLINK: + { + struct symlink_reparse_data *data; + size_t data_offs; + + if (!valid_reparse_buffer(ni, reparse_attr, size, + sizeof(*data))) + return false; + + data = (struct symlink_reparse_data *)reparse_attr->reparse_data; + data_offs = offsetof(struct reparse_point, reparse_data) + + offsetof(struct symlink_reparse_data, path_buffer); + + if (!reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->substitute_name_offset), + le16_to_cpu(data->substitute_name_length)) || + !reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->print_name_offset), + le16_to_cpu(data->print_name_length))) + return false; + break; + } case IO_REPARSE_TAG_LX_SYMLINK: - if (data_len <= sizeof(wsl_reparse_data->type) || - wsl_reparse_data->type != cpu_to_le32(2)) + { + struct wsl_link_reparse_data *data; + + if (!valid_reparse_buffer(ni, reparse_attr, size, + sizeof(*data))) + return false; + + data = (struct wsl_link_reparse_data *)reparse_attr->reparse_data; + + if (le16_to_cpu(reparse_attr->reparse_data_length) <= sizeof(data->type) || + data->type != cpu_to_le32(2)) return false; break; + } case IO_REPARSE_TAG_AF_UNIX: case IO_REPARSE_TAG_LX_FIFO: case IO_REPARSE_TAG_LX_CHR: case IO_REPARSE_TAG_LX_BLK: - if (data_len || !(ni->flags & FILE_ATTRIBUTE_RECALL_ON_OPEN)) + if (!valid_reparse_buffer(ni, reparse_attr, size, 0)) return false; + if (le16_to_cpu(reparse_attr->reparse_data_length) || + !(ni->flags & FILE_ATTRIBUTE_RECALL_ON_OPEN)) + return false; + break; + default: + if (!valid_reparse_buffer(ni, reparse_attr, size, 0)) + return false; + break; } return true; } -static unsigned int ntfs_reparse_tag_mode(struct reparse_point *reparse_attr) +static unsigned int ntfs_reparse_tag_mode(__le32 reparse_tag) { unsigned int mode = 0; - switch (reparse_attr->reparse_tag) { + switch (reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: case IO_REPARSE_TAG_SYMLINK: case IO_REPARSE_TAG_LX_SYMLINK: mode = S_IFLNK; @@ -134,32 +244,75 @@ static unsigned int ntfs_reparse_tag_mode(struct reparse_point *reparse_attr) unsigned int ntfs_make_symlink(struct ntfs_inode *ni) { s64 attr_size = 0; + int err; unsigned int lth; struct reparse_point *reparse_attr; - struct wsl_link_reparse_data *wsl_link_data; unsigned int mode = 0; + kvfree(ni->target); + ni->target = NULL; + ni->reparse_tag = 0; + ni->reparse_flags = 0; + reparse_attr = ntfs_attr_readall(ni, AT_REPARSE_POINT, NULL, 0, &attr_size); - if (reparse_attr && attr_size && + if (reparse_attr && valid_reparse_data(ni, reparse_attr, attr_size)) { + err = -EINVAL; + switch (reparse_attr->reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: + { + struct mount_point_reparse_data *data = + (struct mount_point_reparse_data *)reparse_attr->reparse_data; + const __le16 *name = (const __le16 *)((u8 *)data->path_buffer + + le16_to_cpu(data->substitute_name_offset)); + + err = ntfs_reparse_target_to_nls(ni->vol, + name, + le16_to_cpu(data->substitute_name_length), + &ni->target); + break; + } + case IO_REPARSE_TAG_SYMLINK: + { + struct symlink_reparse_data *data = + (struct symlink_reparse_data *)reparse_attr->reparse_data; + const __le16 *name = (const __le16 *)((u8 *)data->path_buffer + + le16_to_cpu(data->substitute_name_offset)); + + err = ntfs_reparse_target_to_nls(ni->vol, + name, + le16_to_cpu(data->substitute_name_length), + &ni->target); + if (!err) + ni->reparse_flags = data->flags; + break; + } case IO_REPARSE_TAG_LX_SYMLINK: - wsl_link_data = + { + struct wsl_link_reparse_data *wsl_link_data = (struct wsl_link_reparse_data *)reparse_attr->reparse_data; + if (wsl_link_data->type == cpu_to_le32(2)) { lth = le16_to_cpu(reparse_attr->reparse_data_length) - - sizeof(wsl_link_data->type); + sizeof(wsl_link_data->type); ni->target = kvzalloc(lth + 1, GFP_NOFS); if (ni->target) { memcpy(ni->target, wsl_link_data->link, lth); ni->target[lth] = 0; - mode = ntfs_reparse_tag_mode(reparse_attr); + err = 0; } } break; + } default: - mode = ntfs_reparse_tag_mode(reparse_attr); + err = 0; + } + + if (!err) { + mode = ntfs_reparse_tag_mode(reparse_attr->reparse_tag); + ni->reparse_tag = reparse_attr->reparse_tag; } } else ni->flags &= ~FILE_ATTR_REPARSE_POINT; @@ -184,8 +337,9 @@ unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mr reparse_attr = (struct reparse_point *)ntfs_attr_readall(NTFS_I(vi), AT_REPARSE_POINT, NULL, 0, &attr_size); - if (reparse_attr && attr_size) { + if (reparse_attr && attr_size >= sizeof(*reparse_attr)) { switch (reparse_attr->reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: case IO_REPARSE_TAG_SYMLINK: case IO_REPARSE_TAG_LX_SYMLINK: dt_type = DT_LNK; @@ -211,6 +365,115 @@ unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mr return dt_type; } +static bool ntfs_is_drive_letter(const char *target) +{ + return ((target[0] >= 'A' && target[0] <= 'Z') || + (target[0] >= 'a' && target[0] <= 'z')) && + target[1] == ':'; +} + +/* + * ntfs_translate_symlink_path + * + * @dentry: dentry of the symlink/junction being resolved + * @target: NUL-terminated NLS target string with '\\' already normalized to '/' + * @translated: out parameter, set to a newly kmalloc'd relative path on success + * + * Windows junctions (IO_REPARSE_TAG_MOUNT_POINT) and non-relative symlinks + * (IO_REPARSE_TAG_SYMLINK without SYMLINK_FLAG_RELATIVE) store substitute + * names such as "/??/C:/foo", "//?/C:/foo", "/foo", or "C:/foo". Linux + * cannot continue pathname lookup from those syntaxes, so rewrite them as a + * path relative to the symlink's containing directory on this NTFS volume, + * anchored at the volume root via "../". + * + * Note: bind-mounted subtrees of the volume may resolve to unexpected + * locations because the computed "../" depth is relative to the NTFS volume + * root, not the bind-mounted subtree root. + * + * Return: 0 on success with *translated set to a newly allocated string the + * caller must kfree(); negative errno on failure. + */ +int ntfs_translate_symlink_path(struct dentry *dentry, const char *target, + char **translated) +{ + char *buf, *link_path, *out, *p; + const char *path, *tail; + unsigned int up_levels = 0; + size_t tail_len, out_len; + int err; + + if (!dentry || !target || !translated) + return -EINVAL; + + path = target; + /* reject UNC path. */ + if (path[0] == '/' && path[1] == '/' && + !(path[2] == '?' && path[3] == '/')) + return -EOPNOTSUPP; + + /* target starts with "/??/" or "//?/"? */ + if ((path[0] == '/' && path[1] == '?' && path[2] == '?' && path[3] == '/') || + (path[0] == '/' && path[1] == '/' && path[2] == '?' && path[3] == '/')) + path += 4; + + /* target must start with a drive character or '/'. */ + if (ntfs_is_drive_letter(path)) { + if (path[2] && path[2] != '/') + return -EOPNOTSUPP; + tail = path + 2; + if (*tail == '/') + tail++; + } else if (*path == '/') { + tail = path + 1; + } else { + return -EOPNOTSUPP; + } + + tail_len = strlen(tail); + + buf = kmalloc(PATH_MAX, GFP_NOFS); + if (!buf) + return -ENOMEM; + + link_path = dentry_path_raw(dentry, buf, PATH_MAX); + if (IS_ERR(link_path)) { + err = PTR_ERR(link_path); + goto out; + } + + /* count '/' after the leading slash. */ + for (p = link_path + 1; *p; p++) + if (*p == '/') + up_levels++; + + /* build "./" + ("../" * up_levels) + tail. */ + out_len = 2 + up_levels * 3 + tail_len; + if (out_len >= PATH_MAX) { + err = -ENAMETOOLONG; + goto out; + } + + out = kmalloc(out_len + 1, GFP_NOFS); + if (!out) { + err = -ENOMEM; + goto out; + } + + memcpy(out, "./", 2); + p = out + 2; + while (up_levels--) { + memcpy(p, "../", 3); + p += 3; + } + memcpy(p, tail, tail_len + 1); + + *translated = out; + err = 0; +out: + kfree(buf); + return err; +} + /* * Set the index for new reparse data */ @@ -496,40 +759,163 @@ out: * Set reparse data for a WSL type symlink */ int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni, - const __le16 *target, int target_len) + const char *target, int target_len) { int err = 0; - int len; int reparse_len; - unsigned char *utarget = NULL; struct reparse_point *reparse; struct wsl_link_reparse_data *data; - len = ntfs_ucstonls(ni->vol, target, target_len, &utarget, 0); - if (len <= 0) - return -EINVAL; - - reparse_len = sizeof(struct reparse_point) + sizeof(data->type) + len; + reparse_len = sizeof(struct reparse_point) + sizeof(data->type) + + target_len; reparse = kvzalloc(reparse_len, GFP_NOFS); + if (!reparse) + return -ENOMEM; + + ni->target = kstrdup(target, GFP_NOFS); + if (!ni->target) { + kvfree(reparse); + return -ENOMEM; + } + + data = (struct wsl_link_reparse_data *)reparse->reparse_data; + reparse->reparse_tag = IO_REPARSE_TAG_LX_SYMLINK; + reparse->reparse_data_length = + cpu_to_le16(sizeof(data->type) + target_len); + reparse->reserved = 0; + data->type = cpu_to_le32(2); + memcpy(data->link, target, target_len); + err = ntfs_set_ntfs_reparse_data(ni, + (char *)reparse, reparse_len); + kvfree(reparse); + if (err) { + kfree(ni->target); + ni->target = NULL; + } else { + ni->reparse_tag = IO_REPARSE_TAG_LX_SYMLINK; + ni->reparse_flags = 0; + } + return err; +} + +int ntfs_reparse_set_native_symlink(struct ntfs_inode *ni, + const char *target, int target_len) +{ + int err = 0; + bool is_absolute, prt_sub_shared = true; + char *sub_name = NULL; + char *prt_name = NULL; + __le16 *sub_name_utf16 = NULL; + __le16 *prt_name_utf16 = NULL; + int sub_len, prt_len; + int total_data_len, total_reparse_len; + struct reparse_point *reparse = NULL; + struct symlink_reparse_data *data; + int i; + + /* Determine if target is absolute (starts with drive letter like C:/ or C:\) */ + is_absolute = target_len > 2 && + ntfs_is_drive_letter(target) && + (target[2] == '/' || target[2] == '\\'); + + + /* Normalize and prepare NLS paths */ + prt_name = kstrdup(target, GFP_NOFS); + if (!prt_name) + return -ENOMEM; + + /* Replace '/' with '\' */ + for (i = 0; i < target_len; i++) { + if (prt_name[i] == '/') + prt_name[i] = '\\'; + } + + if (is_absolute) { + /* Prepend '\??\' to Substitutename */ + sub_name = kmalloc(target_len + 5, GFP_NOFS); + if (!sub_name) { + err = -ENOMEM; + goto out; + } + snprintf(sub_name, target_len + 5, "\\??\\%s", prt_name); + prt_sub_shared = false; + } else { + /* For relative symlinks (including absolute paths without drive letters), + * SubstituteName and PrintName are identical. + */ + sub_name = prt_name; + } + + /* Convert NLS paths to UTF-16 */ + sub_len = ntfs_nlstoucs(ni->vol, sub_name, strlen(sub_name), + &sub_name_utf16, PATH_MAX); + if (sub_len < 0) { + err = sub_len; + goto out; + } + + prt_len = ntfs_nlstoucs(ni->vol, prt_name, strlen(prt_name), + &prt_name_utf16, PATH_MAX); + if (prt_len < 0) { + err = prt_len; + goto out; + } + + /* Check for buffer size limits */ + total_data_len = sizeof(struct symlink_reparse_data) + + (sub_len + prt_len) * sizeof(__le16); + if (total_data_len > 16384) { /* 16KB max reparse tag size */ + err = -EFBIG; + goto out; + } + + total_reparse_len = sizeof(struct reparse_point) + total_data_len; + reparse = kvzalloc(total_reparse_len, GFP_NOFS); if (!reparse) { err = -ENOMEM; - kfree(utarget); - } else { - data = (struct wsl_link_reparse_data *)reparse->reparse_data; - reparse->reparse_tag = IO_REPARSE_TAG_LX_SYMLINK; - reparse->reparse_data_length = - cpu_to_le16(sizeof(data->type) + len); - reparse->reserved = 0; - data->type = cpu_to_le32(2); - memcpy(data->link, utarget, len); - err = ntfs_set_ntfs_reparse_data(ni, - (char *)reparse, reparse_len); - kvfree(reparse); - if (!err) - ni->target = utarget; - else - kfree(utarget); + goto out; + } + + /* Pack fields in reparse buffer */ + reparse->reparse_tag = IO_REPARSE_TAG_SYMLINK; + reparse->reparse_data_length = cpu_to_le16(total_data_len); + reparse->reserved = 0; + + data = (struct symlink_reparse_data *)reparse->reparse_data; + data->substitute_name_offset = 0; + data->substitute_name_length = cpu_to_le16(sub_len * sizeof(__le16)); + data->print_name_offset = data->substitute_name_length; + data->print_name_length = cpu_to_le16(prt_len * sizeof(__le16)); + data->flags = is_absolute ? 0 : cpu_to_le32(SYMLINK_FLAG_RELATIVE); + + /* Copy names to path_buffer */ + memcpy(data->path_buffer, sub_name_utf16, sub_len * sizeof(__le16)); + memcpy(data->path_buffer + sub_len, prt_name_utf16, prt_len * sizeof(__le16)); + + err = ntfs_set_ntfs_reparse_data(ni, (char *)reparse, total_reparse_len); + if (!err) { + int len = strlen(sub_name); + + for (i = 0; i < len; i++) { + if (sub_name[i] == '\\') + sub_name[i] = '/'; + } + ni->target = sub_name; + sub_name = NULL; + if (prt_sub_shared) + prt_name = NULL; + ni->reparse_tag = IO_REPARSE_TAG_SYMLINK; + ni->reparse_flags = is_absolute ? 0 : + cpu_to_le32(SYMLINK_FLAG_RELATIVE); } + +out: + kfree(prt_name); + if (!prt_sub_shared) + kfree(sub_name); + kvfree(sub_name_utf16); + kvfree(prt_name_utf16); + kvfree(reparse); return err; } @@ -568,6 +954,10 @@ int ntfs_reparse_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode) err = ntfs_set_ntfs_reparse_data(ni, (char *)reparse, reparse_len); kvfree(reparse); + if (!err) { + ni->reparse_tag = reparse_tag; + ni->reparse_flags = 0; + } } return err; diff --git a/fs/ntfs/reparse.h b/fs/ntfs/reparse.h index 28da40257f2a..c11a5bb7e6a5 100644 --- a/fs/ntfs/reparse.h +++ b/fs/ntfs/reparse.h @@ -11,8 +11,12 @@ extern __le16 reparse_index_name[]; unsigned int ntfs_make_symlink(struct ntfs_inode *ni); unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mref); +int ntfs_translate_symlink_path(struct dentry *dentry, const char *target, + char **translated); int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni, - const __le16 *target, int target_len); + const char *target, int target_len); +int ntfs_reparse_set_native_symlink(struct ntfs_inode *ni, + const char *symname, int symlen); int ntfs_reparse_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode); int ntfs_delete_reparse_index(struct ntfs_inode *ni); int ntfs_remove_ntfs_reparse_data(struct ntfs_inode *ni); diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c index da21dbeaaf66..cbb6576cf725 100644 --- a/fs/ntfs/runlist.c +++ b/fs/ntfs/runlist.c @@ -763,7 +763,7 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * buf = (u8 *)attr + le16_to_cpu(attr->data.non_resident.mapping_pairs_offset); attr_end = (u8 *)attr + le32_to_cpu(attr->length); - if (unlikely(buf < (u8 *)attr || buf > attr_end)) { + if (unlikely(buf < (u8 *)attr || buf >= attr_end)) { ntfs_error(vol->sb, "Corrupt attribute."); return ERR_PTR(-EIO); } @@ -811,7 +811,7 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * */ b = *buf & 0xf; if (b) { - if (unlikely(buf + b > attr_end)) + if (unlikely(buf + b >= attr_end)) goto io_error; for (deltaxcn = (s8)buf[b--]; b; b--) deltaxcn = (deltaxcn << 8) + buf[b]; @@ -855,12 +855,16 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * u8 b2 = *buf & 0xf; b = b2 + ((*buf >> 4) & 0xf); - if (buf + b > attr_end) + if (buf + b >= attr_end) goto io_error; for (deltaxcn = (s8)buf[b--]; b > b2; b--) deltaxcn = (deltaxcn << 8) + buf[b]; /* Change the current lcn to its new value. */ - lcn += deltaxcn; + if (unlikely(check_add_overflow(lcn, deltaxcn, &lcn))) { + ntfs_error(vol->sb, + "LCN overflow in mapping pairs array."); + goto err_out; + } #ifdef DEBUG /* * On NTFS 1.2-, apparently can have lcn == -1 to @@ -1817,7 +1821,7 @@ struct runlist_element *ntfs_rl_punch_hole(struct runlist_element *dst_rl, int d !ntfs_rle_contain(s_rl, start_vcn)) return ERR_PTR(-EINVAL); - begin_split = s_rl->vcn != start_vcn ? true : false; + begin_split = s_rl->vcn != start_vcn; e_rl = ntfs_rl_find_vcn_nolock(dst_rl, end_vcn); if (!e_rl || @@ -1825,10 +1829,10 @@ struct runlist_element *ntfs_rl_punch_hole(struct runlist_element *dst_rl, int d !ntfs_rle_contain(e_rl, end_vcn)) return ERR_PTR(-EINVAL); - end_split = e_rl->vcn + e_rl->length - 1 != end_vcn ? true : false; + end_split = e_rl->vcn + e_rl->length - 1 != end_vcn; /* @s_rl has to be split into left, punched hole, and right */ - one_split_3 = e_rl == s_rl && begin_split && end_split ? true : false; + one_split_3 = e_rl == s_rl && begin_split && end_split; punch_cnt = (int)(e_rl - s_rl) + 1; @@ -1968,7 +1972,7 @@ struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, i !ntfs_rle_contain(s_rl, start_vcn)) return ERR_PTR(-EINVAL); - begin_split = s_rl->vcn != start_vcn ? true : false; + begin_split = s_rl->vcn != start_vcn; e_rl = ntfs_rl_find_vcn_nolock(dst_rl, end_vcn); if (!e_rl || @@ -1976,10 +1980,10 @@ struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, i !ntfs_rle_contain(e_rl, end_vcn)) return ERR_PTR(-EINVAL); - end_split = e_rl->vcn + e_rl->length - 1 != end_vcn ? true : false; + end_split = e_rl->vcn + e_rl->length - 1 != end_vcn; /* @s_rl has to be split into left, collapsed, and right */ - one_split_3 = e_rl == s_rl && begin_split && end_split ? true : false; + one_split_3 = e_rl == s_rl && begin_split && end_split; punch_cnt = (int)(e_rl - s_rl) + 1; *punch_rl = kvcalloc(punch_cnt + 1, sizeof(struct runlist_element), @@ -2056,10 +2060,11 @@ struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, i * consists of holes. */ merge_cnt = 0; - i = new_1st_cnt == 0 ? 1 : new_1st_cnt; - if (ntfs_rle_lcn_contiguous(&new_rl[i - 1], &new_rl[i])) { - /* Merge right and left */ - s_rl = &new_rl[new_1st_cnt - 1]; + if (new_1st_cnt > 0 && + ntfs_rle_lcn_contiguous(&new_rl[new_1st_cnt - 1], + &new_rl[new_1st_cnt])) { + /* Merge right and left. */ + s_rl = &new_rl[new_1st_cnt - 1]; s_rl->length += s_rl[1].length; merge_cnt = 1; } diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c index 22dc7865eca7..8abe7bee4c0d 100644 --- a/fs/ntfs/super.c +++ b/fs/ntfs/super.c @@ -17,7 +17,6 @@ #include "sysctl.h" #include "logfile.h" -#include "quota.h" #include "index.h" #include "ntfs.h" #include "ea.h" @@ -45,6 +44,28 @@ static const struct constant_table ntfs_param_enums[] = { }; enum { + NATIVE_SYMLINK_RAW, + NATIVE_SYMLINK_REL, +}; + +static const struct constant_table ntfs_native_symlink_enums[] = { + { "raw", NATIVE_SYMLINK_RAW }, + { "rel", NATIVE_SYMLINK_REL }, + {} +}; + +enum { + SYMLINK_WSL, + SYMLINK_NATIVE, +}; + +static const struct constant_table ntfs_symlink_enums[] = { + { "wsl", SYMLINK_WSL }, + { "native", SYMLINK_NATIVE }, + {} +}; + +enum { Opt_uid, Opt_gid, Opt_umask, @@ -67,6 +88,8 @@ enum { Opt_acl, Opt_discard, Opt_nocase, + Opt_native_symlink, + Opt_symlink, }; static const struct fs_parameter_spec ntfs_parameters[] = { @@ -92,6 +115,8 @@ static const struct fs_parameter_spec ntfs_parameters[] = { fsparam_flag("discard", Opt_discard), fsparam_flag("sparse", Opt_sparse), fsparam_flag("nocase", Opt_nocase), + fsparam_enum("native_symlink", Opt_native_symlink, ntfs_native_symlink_enums), + fsparam_enum("symlink", Opt_symlink, ntfs_symlink_enums), {} }; @@ -216,6 +241,18 @@ static int ntfs_parse_param(struct fs_context *fc, struct fs_parameter *param) else NVolClearDisableSparse(vol); break; + case Opt_native_symlink: + if (result.uint_32 == NATIVE_SYMLINK_REL) + NVolSetNativeSymlinkRel(vol); + else + NVolClearNativeSymlinkRel(vol); + break; + case Opt_symlink: + if (result.uint_32 == SYMLINK_NATIVE) + NVolSetSymlinkNative(vol); + else + NVolClearSymlinkNative(vol); + break; case Opt_sparse: break; default: @@ -240,8 +277,7 @@ static int ntfs_reconfigure(struct fs_context *fc) * flags are set. Also, empty the logfile journal as it would become * stale as soon as something is written to the volume and mark the * volume dirty so that chkdsk is run if the volume is not umounted - * cleanly. Finally, mark the quotas out of date so Windows rescans - * the volume on boot and updates them. + * cleanly. * * When remounting read-only, mark the volume clean if no volume errors * have occurred. @@ -274,12 +310,6 @@ static int ntfs_reconfigure(struct fs_context *fc) NVolSetErrors(vol); return -EROFS; } - if (!ntfs_mark_quotas_out_of_date(vol)) { - ntfs_error(sb, "Failed to mark quotas out of date%s", - es); - NVolSetErrors(vol); - return -EROFS; - } } else if (!sb_rdonly(sb) && (fc->sb_flags & SB_RDONLY)) { /* Remounting read-only. */ if (!NVolErrors(vol)) { @@ -413,6 +443,7 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label) { struct ntfs_inode *vol_ni = NTFS_I(vol->vol_ino); struct ntfs_attr_search_ctx *ctx; + char *new_label; __le16 *uname; int uname_len, ret; @@ -425,7 +456,7 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label) return uname_len; } - if (uname_len > NTFS_MAX_LABEL_LEN) { + if (uname_len > NTFS_MAX_LABEL_LEN) { ntfs_error(vol->sb, "Volume label is too long (max %d characters).", NTFS_MAX_LABEL_LEN); @@ -433,30 +464,54 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label) return -EINVAL; } + /* + * Allocate the in-memory label copy up front. If kstrdup() fails we + * bail out before touching on-disk metadata, so the in-memory label + * and the on-disk label stay in sync. + */ + new_label = kstrdup(label, GFP_KERNEL); + if (!new_label) { + kvfree(uname); + return -ENOMEM; + } + mutex_lock(&vol_ni->mrec_lock); ctx = ntfs_attr_get_search_ctx(vol_ni, NULL); if (!ctx) { ret = -ENOMEM; - goto out; + goto out; } - if (!ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0, - ctx)) - ntfs_attr_record_rm(ctx); + ret = ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0, + ctx); + if (!ret) + ret = ntfs_attr_record_rm(ctx); + else if (ret == -ENOENT) + ret = 0; ntfs_attr_put_search_ctx(ctx); + if (ret) + goto out; ret = ntfs_resident_attr_record_add(vol_ni, AT_VOLUME_NAME, AT_UNNAMED, 0, (u8 *)uname, uname_len * sizeof(__le16), 0); out: - mutex_unlock(&vol_ni->mrec_lock); - kvfree(uname); - mark_inode_dirty_sync(vol->vol_ino); - if (ret >= 0) { - kfree(vol->volume_label); - vol->volume_label = kstrdup(label, GFP_KERNEL); + char *old_label; + + mutex_lock(&vol->volume_label_lock); + old_label = vol->volume_label; + vol->volume_label = new_label; + mutex_unlock(&vol->volume_label_lock); + + kfree(old_label); + mark_inode_dirty_sync(vol->vol_ino); ret = 0; } + mutex_unlock(&vol_ni->mrec_lock); + kvfree(uname); + + if (ret < 0) + kfree(new_label); return ret; } @@ -979,6 +1034,13 @@ mft_unmap_out: ntfs_is_baad_recordp((__le32 *)kmirr)) bytes = vol->mft_record_size; } + /* Compare the two records. */ + if (memcmp(kmft, kmirr, bytes)) { + ntfs_error(sb, + "$MFT and $MFTMirr record %i do not match. Run chkdsk.", + i); + goto mm_unmap_out; + } kmft += vol->mft_record_size; kmirr += vol->mft_record_size; } while (++i < vol->mftmirr_size); @@ -1155,73 +1217,6 @@ iput_out: } /* - * load_and_init_quota - load and setup the quota file for a volume if present - * @vol: ntfs super block describing device whose quota file to load - * - * Return 'true' on success or 'false' on error. If $Quota is not present, we - * leave vol->quota_ino as NULL and return success. - */ -static bool load_and_init_quota(struct ntfs_volume *vol) -{ - static const __le16 Quota[7] = { cpu_to_le16('$'), - cpu_to_le16('Q'), cpu_to_le16('u'), - cpu_to_le16('o'), cpu_to_le16('t'), - cpu_to_le16('a'), 0 }; - static __le16 Q[3] = { cpu_to_le16('$'), - cpu_to_le16('Q'), 0 }; - struct ntfs_name *name = NULL; - u64 mref; - struct inode *tmp_ino; - - ntfs_debug("Entering."); - /* - * Find the inode number for the quota file by looking up the filename - * $Quota in the extended system files directory $Extend. - */ - inode_lock(vol->extend_ino); - mref = ntfs_lookup_inode_by_name(NTFS_I(vol->extend_ino), Quota, 6, - &name); - inode_unlock(vol->extend_ino); - kfree(name); - if (IS_ERR_MREF(mref)) { - /* - * If the file does not exist, quotas are disabled and have - * never been enabled on this volume, just return success. - */ - if (MREF_ERR(mref) == -ENOENT) { - ntfs_debug("$Quota not present. Volume does not have quotas enabled."); - /* - * No need to try to set quotas out of date if they are - * not enabled. - */ - NVolSetQuotaOutOfDate(vol); - return true; - } - /* A real error occurred. */ - ntfs_error(vol->sb, "Failed to find inode number for $Quota."); - return false; - } - /* Get the inode. */ - tmp_ino = ntfs_iget(vol->sb, MREF(mref)); - if (IS_ERR(tmp_ino)) { - if (!IS_ERR(tmp_ino)) - iput(tmp_ino); - ntfs_error(vol->sb, "Failed to load $Quota."); - return false; - } - vol->quota_ino = tmp_ino; - /* Get the $Q index allocation attribute. */ - tmp_ino = ntfs_index_iget(vol->quota_ino, Q, 2); - if (IS_ERR(tmp_ino)) { - ntfs_error(vol->sb, "Failed to load $Quota/$Q index."); - return false; - } - vol->quota_q_ino = tmp_ino; - ntfs_debug("Done."); - return true; -} - -/* * load_and_init_attrdef - load the attribute definitions table for a volume * @vol: ntfs super block describing device whose attrdef to load * @@ -1302,7 +1297,6 @@ static bool load_and_init_upcase(struct ntfs_volume *vol) u8 *addr; pgoff_t index, max_index; unsigned int size; - int i, max; ntfs_debug("Entering."); /* Read upcase table and setup vol->upcase and vol->upcase_len. */ @@ -1353,16 +1347,11 @@ read_partial_upcase_page: mutex_unlock(&ntfs_lock); return true; } - max = default_upcase_len; - if (max > vol->upcase_len) - max = vol->upcase_len; - for (i = 0; i < max; i++) - if (vol->upcase[i] != default_upcase[i]) - break; - if (i == max) { + if (default_upcase_len == vol->upcase_len && + !memcmp(vol->upcase, default_upcase, + default_upcase_len * sizeof(*default_upcase))) { kvfree(vol->upcase); vol->upcase = default_upcase; - vol->upcase_len = max; ntfs_nr_upcase_users++; mutex_unlock(&ntfs_lock); ntfs_debug("Volume specified $UpCase matches default. Using default."); @@ -1510,6 +1499,7 @@ iput_volume_failed: vol->volume_label = NULL; } + ntfs_attr_reinit_search_ctx(ctx); if (ntfs_attr_lookup(AT_VOLUME_INFORMATION, NULL, 0, 0, 0, NULL, 0, ctx) || ctx->attr->non_resident || ctx->attr->flags) { ntfs_attr_put_search_ctx(ctx); @@ -1638,18 +1628,6 @@ get_ctx_vol_failed: ntfs_error(sb, "Failed to load $Extend."); goto iput_sec_err_out; } - /* Find the quota file, load it if present, and set it up. */ - if (!load_and_init_quota(vol) && - vol->on_errors == ON_ERRORS_REMOUNT_RO) { - static const char *es1 = "Failed to load $Quota"; - static const char *es2 = ". Run chkdsk."; - - sb->s_flags |= SB_RDONLY; - ntfs_error(sb, "%s. Mounting read-only%s", es1, es2); - /* This will prevent a read-write remount. */ - NVolSetErrors(vol); - } - return true; iput_sec_err_out: @@ -1671,7 +1649,7 @@ iput_attrdef_err_out: iput_upcase_err_out: vol->upcase_len = 0; mutex_lock(&ntfs_lock); - if (vol->upcase == default_upcase) { + if (vol->upcase && vol->upcase == default_upcase) { ntfs_nr_upcase_users--; vol->upcase = NULL; } @@ -1701,12 +1679,12 @@ static void ntfs_volume_free(struct ntfs_volume *vol) * the number of upcase users if we are a user. */ mutex_lock(&ntfs_lock); - if (vol->upcase == default_upcase) { + if (vol->upcase && vol->upcase == default_upcase) { ntfs_nr_upcase_users--; vol->upcase = NULL; } - if (!ntfs_nr_upcase_users && default_upcase) { + if (!ntfs_nr_upcase_users) { kvfree(default_upcase); default_upcase = NULL; } @@ -1721,8 +1699,7 @@ static void ntfs_volume_free(struct ntfs_volume *vol) unload_nls(vol->nls_map); - if (vol->lcn_empty_bits_per_page) - kvfree(vol->lcn_empty_bits_per_page); + kvfree(vol->lcn_empty_bits_per_page); kfree(vol->volume_label); kfree(vol); } @@ -1747,10 +1724,6 @@ static void ntfs_put_super(struct super_block *sb) /* NTFS 3.0+ specific. */ if (vol->major_ver >= 3) { - if (vol->quota_q_ino) - ntfs_commit_inode(vol->quota_q_ino); - if (vol->quota_ino) - ntfs_commit_inode(vol->quota_ino); if (vol->extend_ino) ntfs_commit_inode(vol->extend_ino); if (vol->secure_ino) @@ -1799,14 +1772,6 @@ static void ntfs_put_super(struct super_block *sb) /* NTFS 3.0+ specific clean up. */ if (vol->major_ver >= 3) { - if (vol->quota_q_ino) { - iput(vol->quota_q_ino); - vol->quota_q_ino = NULL; - } - if (vol->quota_ino) { - iput(vol->quota_ino); - vol->quota_ino = NULL; - } if (vol->extend_ino) { iput(vol->extend_ino); vol->extend_ino = NULL; @@ -1933,7 +1898,7 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) struct address_space *mapping = vol->lcnbmp_ino->i_mapping; struct folio *folio; pgoff_t index, max_index; - struct file_ra_state *ra; + struct file_ra_state ra = { 0 }; ntfs_debug("Entering."); /* Serialize accesses to the cluster bitmap. */ @@ -1941,11 +1906,7 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) if (NVolFreeClusterKnown(vol)) return atomic64_read(&vol->free_clusters); - ra = kzalloc(sizeof(*ra), GFP_NOFS); - if (!ra) - return 0; - - file_ra_state_init(ra, mapping); + file_ra_state_init(&ra, mapping); /* * Convert the number of bits into bytes rounded up, then convert into @@ -1964,7 +1925,7 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) * Get folio from page cache, getting it from backing store * if necessary, and increment the use count. */ - folio = ntfs_get_locked_folio(mapping, index, max_index, ra); + folio = ntfs_get_locked_folio(mapping, index, max_index, &ra); /* Ignore pages which errored synchronously. */ if (IS_ERR(folio)) { @@ -2003,7 +1964,6 @@ s64 get_nr_free_clusters(struct ntfs_volume *vol) else atomic64_set(&vol->free_clusters, nr_free); - kfree(ra); NVolSetFreeClusterKnown(vol); wake_up_all(&vol->free_waitq); ntfs_debug("Exiting."); @@ -2058,15 +2018,11 @@ static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, struct address_space *mapping = vol->mftbmp_ino->i_mapping; struct folio *folio; pgoff_t index; - struct file_ra_state *ra; + struct file_ra_state ra = { 0 }; ntfs_debug("Entering."); - ra = kzalloc(sizeof(*ra), GFP_NOFS); - if (!ra) - return 0; - - file_ra_state_init(ra, mapping); + file_ra_state_init(&ra, mapping); /* Use multiples of 4 bytes, thus max_size is PAGE_SIZE / 4. */ ntfs_debug("Reading $MFT/$BITMAP, max_index = 0x%lx, max_size = 0x%lx.", @@ -2078,7 +2034,7 @@ static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, * Get folio from page cache, getting it from backing store * if necessary, and increment the use count. */ - folio = ntfs_get_locked_folio(mapping, index, max_index, ra); + folio = ntfs_get_locked_folio(mapping, index, max_index, &ra); /* Ignore pages which errored synchronously. */ if (IS_ERR(folio)) { @@ -2110,7 +2066,6 @@ static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, else atomic64_set(&vol->free_mft_records, nr_free); - kfree(ra); ntfs_debug("Exiting."); return nr_free; } @@ -2455,14 +2410,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) vol->vol_ino = NULL; /* NTFS 3.0+ specific clean up. */ if (vol->major_ver >= 3) { - if (vol->quota_q_ino) { - iput(vol->quota_q_ino); - vol->quota_q_ino = NULL; - } - if (vol->quota_ino) { - iput(vol->quota_ino); - vol->quota_ino = NULL; - } if (vol->extend_ino) { iput(vol->extend_ino); vol->extend_ino = NULL; @@ -2494,7 +2441,7 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) } vol->upcase_len = 0; mutex_lock(&ntfs_lock); - if (vol->upcase == default_upcase) { + if (vol->upcase && vol->upcase == default_upcase) { ntfs_nr_upcase_users--; vol->upcase = NULL; } @@ -2509,8 +2456,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) } /* Error exit code path. */ unl_upcase_iput_tmp_ino_err_out_now: - if (vol->lcn_empty_bits_per_page) - kvfree(vol->lcn_empty_bits_per_page); /* * Decrease the number of upcase users and destroy the global default * upcase table if necessary. @@ -2530,6 +2475,9 @@ iput_tmp_ino_err_out_now: /* Errors at this stage are irrelevant. */ err_out_now: sb->s_fs_info = NULL; + kvfree(vol->lcn_empty_bits_per_page); + kfree(vol->volume_label); + unload_nls(vol->nls_map); kfree(vol); ntfs_debug("Failed, returning -EINVAL."); lockdep_on(); @@ -2610,6 +2558,7 @@ static int ntfs_init_fs_context(struct fs_context *fc) NVolSetCaseSensitive(vol); init_rwsem(&vol->mftbmp_lock); init_rwsem(&vol->lcnbmp_lock); + mutex_init(&vol->volume_label_lock); fc->s_fs_info = vol; fc->ops = &ntfs_context_ops; @@ -2628,7 +2577,7 @@ MODULE_ALIAS_FS("ntfs"); static int ntfs_workqueue_init(void) { - ntfs_wq = alloc_workqueue("ntfs-bg-io", 0, 0); + ntfs_wq = alloc_workqueue("ntfs-bg-io", WQ_PERCPU, 0); if (!ntfs_wq) return -ENOMEM; return 0; diff --git a/fs/ntfs/volume.h b/fs/ntfs/volume.h index af41427ec622..65fd3908af26 100644 --- a/fs/ntfs/volume.h +++ b/fs/ntfs/volume.h @@ -72,12 +72,11 @@ * @vol_flags: Volume flags. * @major_ver: Ntfs major version of volume. * @minor_ver: Ntfs minor version of volume. + * @volume_label_lock: protects @volume_label. * @volume_label: volume label. * @root_ino: The VFS inode of the root directory. * @secure_ino: The VFS inode of $Secure (NTFS3.0+ only, otherwise NULL). * @extend_ino: The VFS inode of $Extend (NTFS3.0+ only, otherwise NULL). - * @quota_ino: The VFS inode of $Quota. - * @quota_q_ino: Attribute inode for $Quota/$Q. * @nls_map: NLS (National Language Support) table. * @nls_utf8: NLS table for UTF-8. * @free_waitq: Wait queue for threads waiting for free clusters or MFT records. @@ -133,6 +132,7 @@ struct ntfs_volume { struct inode *logfile_ino; struct inode *lcnbmp_ino; struct rw_semaphore lcnbmp_lock; + struct mutex volume_label_lock; struct inode *vol_ino; __le16 vol_flags; u8 major_ver; @@ -141,8 +141,6 @@ struct ntfs_volume { struct inode *root_ino; struct inode *secure_ino; struct inode *extend_ino; - struct inode *quota_ino; - struct inode *quota_q_ino; struct nls_table *nls_map; bool nls_utf8; wait_queue_head_t free_waitq; @@ -165,7 +163,6 @@ struct ntfs_volume { * Otherwise be case insensitive but still * create file names in POSIX namespace. * NV_LogFileEmpty LogFile journal is empty. - * NV_QuotaOutOfDate Quota is out of date. * NV_UsnJrnlStamped UsnJrnl has been stamped. * NV_ReadOnly Volume is mounted read-only. * NV_Compression Volume supports compression. @@ -180,13 +177,13 @@ struct ntfs_volume { * * NV_Discard Issue discard/TRIM commands for freed clusters. * NV_DisableSparse Disable creation of sparse regions. + * NV_NativeSymlinkRel Translate absolute Windows reparse targets (native_symlink=rel). */ enum { NV_Errors, NV_ShowSystemFiles, NV_CaseSensitive, NV_LogFileEmpty, - NV_QuotaOutOfDate, NV_UsnJrnlStamped, NV_ReadOnly, NV_Compression, @@ -198,6 +195,8 @@ enum { NV_CheckWindowsNames, NV_Discard, NV_DisableSparse, + NV_NativeSymlinkRel, + NV_SymlinkNative, }; /* @@ -223,7 +222,6 @@ DEFINE_NVOL_BIT_OPS(Errors) DEFINE_NVOL_BIT_OPS(ShowSystemFiles) DEFINE_NVOL_BIT_OPS(CaseSensitive) DEFINE_NVOL_BIT_OPS(LogFileEmpty) -DEFINE_NVOL_BIT_OPS(QuotaOutOfDate) DEFINE_NVOL_BIT_OPS(UsnJrnlStamped) DEFINE_NVOL_BIT_OPS(ReadOnly) DEFINE_NVOL_BIT_OPS(Compression) @@ -235,6 +233,8 @@ DEFINE_NVOL_BIT_OPS(HideDotFiles) DEFINE_NVOL_BIT_OPS(CheckWindowsNames) DEFINE_NVOL_BIT_OPS(Discard) DEFINE_NVOL_BIT_OPS(DisableSparse) +DEFINE_NVOL_BIT_OPS(NativeSymlinkRel) +DEFINE_NVOL_BIT_OPS(SymlinkNative) static inline void ntfs_inc_free_clusters(struct ntfs_volume *vol, s64 nr) { diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index e61c5bf7e27e..c621a4c582f9 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -962,11 +962,8 @@ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, /* Try to find in cache. */ down_read(&ni->file.run_lock); - if (!no_da && run_lookup_entry(&ni->file.run_da, vcn, lcn, len, NULL)) { - /* The requested vcn is delay allocated. */ - *lcn = DELALLOC_LCN; - } else if (run_lookup_entry(&ni->file.run, vcn, lcn, len, NULL)) { - /* The requested vcn is known in current run. */ + if (run_lookup_entry_da(&ni->file.run, !no_da ? &ni->file.run_da : NULL, + vcn, lcn, len)) { } else { *len = 0; } @@ -1004,6 +1001,7 @@ int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, struct ATTRIB *attr, *attr_b; struct ATTR_LIST_ENTRY *le, *le_b; struct mft_inode *mi, *mi_b; + struct page *page; CLST hint, svcn, to_alloc, evcn1, next_svcn, asize, end, vcn0; CLST alloc, evcn; unsigned fr; @@ -1011,11 +1009,8 @@ int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, int step; again: - if (da && run_lookup_entry(run_da, vcn, lcn, len, NULL)) { - /* The requested vcn is delay allocated. */ - *lcn = DELALLOC_LCN; - } else if (run_lookup_entry(run, vcn, lcn, len, NULL)) { - /* The requested vcn is known in current run. */ + if (run_lookup_entry_da(run, da ? &ni->file.run_da : NULL, vcn, lcn, + len)) { } else { *len = 0; } @@ -1042,10 +1037,13 @@ again: *lcn = RESIDENT_LCN; *len = data_size; if (res && data_size) { - *res = kmemdup(resident_data(attr_b), data_size, - GFP_KERNEL); - if (!*res) + page = alloc_page(GFP_KERNEL); + if (!page) { err = -ENOMEM; + } else { + *res = page_address(page); + memcpy(*res, resident_data(attr_b), data_size); + } } goto out; } @@ -1100,7 +1098,8 @@ again: } if (!*len) { - if (run_lookup_entry(run, vcn, lcn, len, NULL)) { + if (run_lookup_entry_da(run, da ? run_da : NULL, vcn, lcn, + len)) { if (*lcn != SPARSE_LCN || !new) goto ok; /* Slow normal way without allocation. */ @@ -1157,7 +1156,7 @@ again: struct ATTRIB *attr2; attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, NULL, - 0, &vcn0, &mi); + 0, &vcn0, &mi); if (!attr2) { err = -EINVAL; goto out; diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index d99ab086ef6f..873d52233003 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -305,7 +305,9 @@ static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi, if (sbi->options->nohidden && (fname->dup.fa & FILE_ATTRIBUTE_HIDDEN)) return true; - if (fname->name_len + sizeof(struct NTFS_DE) > le16_to_cpu(e->size)) + if (sizeof(struct NTFS_DE) + + offsetof(struct ATTR_FILE_NAME, name) + + fname->name_len * sizeof(short) > le16_to_cpu(e->size)) return true; name_len = ntfs_utf16_to_nls(sbi, fname->name, fname->name_len, name, @@ -489,10 +491,17 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) goto out; } + /* + * Keep directory metadata stable for the whole walk. Loading subrecords + * once is not enough if concurrent writeback can still compact ATTR_LIST + * entries and free the record that ntfs_read_hdr() is currently walking. + */ + ni_lock(ni); + root = indx_get_root(&ni->dir, ni, NULL, NULL); if (!root) { err = -EINVAL; - goto out; + goto out_unlock; } if (pos >= sbi->record_size) { @@ -503,7 +512,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) */ err = ntfs_read_hdr(sbi, ni, &root->ihdr, 0, pos, name, ctx); if (err) - goto out; + goto out_unlock; bit = 0; } @@ -514,7 +523,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) /* Get the next used index. */ err = indx_used_bit(&ni->dir, ni, &bit); if (err) - goto out; + goto out_unlock; if (bit == MINUS_ONE_T) { /* no more used indexes. end of dir. */ @@ -524,13 +533,13 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) if (bit >= max_bit) { /* Corrupted directory. */ err = -EINVAL; - goto out; + goto out_unlock; } err = indx_read_ra(&ni->dir, ni, bit << ni->dir.idx2vbn_bits, &node, &file->f_ra); if (err) - goto out; + goto out_unlock; /* * Add each name from index in 'ctx'. @@ -539,9 +548,12 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) ((u64)bit << index_bits) + sbi->record_size, pos, name, ctx); if (err) - goto out; + goto out_unlock; } +out_unlock: + ni_unlock(ni); + out: kfree(name); put_indx_node(node); diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index b041639ab406..d601f088618c 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -89,6 +89,84 @@ static int ntfs_ioctl_fitrim(struct ntfs_sb_info *sbi, unsigned long arg) return 0; } +/* + * ntfs_fileattr_get - inode_operations::fileattr_get + */ +int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + u32 flags = 0; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (sbi->options->nocase) + flags |= FS_CASEFOLD_FL; + + if (inode->i_flags & S_IMMUTABLE) + flags |= FS_IMMUTABLE_FL; + + if (inode->i_flags & S_APPEND) + flags |= FS_APPEND_FL; + + if (is_compressed(ni)) + flags |= FS_COMPR_FL; + + if (is_encrypted(ni)) + flags |= FS_ENCRYPT_FL; + + if (ni->nodump) + flags |= FS_NODUMP_FL; + + fileattr_fill_flags(fa, flags); + + return 0; +} + +/* + * ntfs_fileattr_set - inode_operations::fileattr_set + */ +int ntfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry, + struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + u32 flags = fa->flags; + unsigned int new_fl = 0; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + if (flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL)) + return -EOPNOTSUPP; + + if (flags & FS_IMMUTABLE_FL) + new_fl |= S_IMMUTABLE; + + if (flags & FS_APPEND_FL) + new_fl |= S_APPEND; + + inode_set_flags(inode, new_fl, S_IMMUTABLE | S_APPEND); + + /* Save nodump flag to return in ntfs_getattr. */ + if (flags & FS_NODUMP_FL) + ni->nodump = 1; + else + ni->nodump = 0; + + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + + return 0; +} + static int ntfs_ioctl_get_volume_label(struct ntfs_sb_info *sbi, u8 __user *buf) { if (copy_to_user(buf, sbi->volume.label, FSLABEL_MAX)) @@ -203,6 +281,9 @@ int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path, if (inode->i_flags & S_APPEND) stat->attributes |= STATX_ATTR_APPEND; + if (ni->nodump) + stat->attributes |= STATX_ATTR_NODUMP; + if (is_compressed(ni)) stat->attributes |= STATX_ATTR_COMPRESSED; @@ -246,18 +327,44 @@ static int ntfs_extend_initialized_size(struct file *file, return 0; } +/* Zero pagecache after 'from'. */ +static void ntfs_zero_tail(struct address_space *mapping, loff_t from) +{ + struct folio_batch fbatch; + pgoff_t index = from >> PAGE_SHIFT; + unsigned nr, i; + + folio_batch_init(&fbatch); + + nr = filemap_get_folios(mapping, &index, -1, &fbatch); + + for (i = 0; i < nr; i++) { + struct folio *folio = fbatch.folios[i]; + u32 st = folio_pos(folio) < from ? + offset_in_folio(folio, from) : + 0; + + folio_lock(folio); + folio_zero_segment(folio, st, folio_size(folio)); + + folio_unlock(folio); + } + folio_batch_release(&fbatch); +} + static void ntfs_filemap_close(struct vm_area_struct *vma) { struct inode *inode = file_inode(vma->vm_file); struct ntfs_inode *ni = ntfs_i(inode); + u64 i_size = i_size_read(inode); u64 from = (u64)vma->vm_pgoff << PAGE_SHIFT; - u64 to = min_t(u64, i_size_read(inode), - from + vma->vm_end - vma->vm_start); + u64 to = min(i_size, from + vma->vm_end - vma->vm_start); if (ni->i_valid < to) { ni->i_valid = to; mark_inode_dirty(inode); } + ntfs_zero_tail(inode->i_mapping, ni->i_valid); } /* Copy of generic_file_vm_ops. */ @@ -347,93 +454,6 @@ out: return err; } -static int ntfs_extend(struct inode *inode, loff_t pos, size_t count, - struct file *file) -{ - struct ntfs_inode *ni = ntfs_i(inode); - struct address_space *mapping = inode->i_mapping; - loff_t end = pos + count; - bool extend_init = file && pos > ni->i_valid; - int err; - - if (end <= inode->i_size && !extend_init) - return 0; - - /* Mark rw ntfs as dirty. It will be cleared at umount. */ - ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_DIRTY); - - if (end > inode->i_size) { - /* - * Normal files: increase file size, allocate space. - * Sparse/Compressed: increase file size. No space allocated. - */ - err = ntfs_set_size(inode, end); - if (err) - goto out; - } - - if (extend_init && !is_compressed(ni)) { - err = ntfs_extend_initialized_size(file, ni, pos); - if (err) - goto out; - } else { - err = 0; - } - - inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - mark_inode_dirty(inode); - - if (IS_SYNC(inode)) { - int err2; - - err = filemap_fdatawrite_range(mapping, pos, end - 1); - err2 = write_inode_now(inode, 1); - if (!err) - err = err2; - if (!err) - err = filemap_fdatawait_range(mapping, pos, end - 1); - } - -out: - return err; -} - -static int ntfs_truncate(struct inode *inode, loff_t new_size) -{ - int err; - struct ntfs_inode *ni = ntfs_i(inode); - u64 new_valid = min_t(u64, ni->i_valid, new_size); - - truncate_setsize(inode, new_size); - - ni_lock(ni); - - down_write(&ni->file.run_lock); - err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, - &new_valid, ni->mi.sbi->options->prealloc, NULL, - false); - up_write(&ni->file.run_lock); - - ni->i_valid = new_valid; - - ni_unlock(ni); - - if (err) - return err; - - ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; - inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - if (!IS_DIRSYNC(inode)) { - mark_inode_dirty(inode); - } else { - err = ntfs_sync_inode(inode); - if (err) - return err; - } - - return 0; -} - /* * ntfs_fallocate - file_operations::ntfs_fallocate * @@ -640,57 +660,25 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) if (is_supported_holes) { CLST vcn = vbo >> cluster_bits; CLST cend = bytes_to_cluster(sbi, end); - CLST cend_v = bytes_to_cluster(sbi, ni->i_valid); CLST lcn, clen; bool new; - if (cend_v > cend) - cend_v = cend; - /* * Allocate and zero new clusters. - * Zeroing these clusters may be too long. - */ - for (; vcn < cend_v; vcn += clen) { - err = attr_data_get_block(ni, vcn, cend_v - vcn, - &lcn, &clen, &new, - true, NULL, false); - if (err) - goto out; - } - - /* - * Moving up 'valid size'. - */ - err = ntfs_extend_initialized_size( - file, ni, (u64)cend_v << cluster_bits); - if (err) - goto out; - - /* - * Allocate but not zero new clusters. */ for (; vcn < cend; vcn += clen) { err = attr_data_get_block(ni, vcn, cend - vcn, &lcn, &clen, &new, - false, NULL, false); + true, NULL, false); if (err) goto out; } } if (mode & FALLOC_FL_KEEP_SIZE) { - ni_lock(ni); - /* True - Keep preallocated. */ - err = attr_set_size(ni, ATTR_DATA, NULL, 0, - &ni->file.run, i_size, &ni->i_valid, - true); - ni_unlock(ni); + err = ntfs_set_size(inode, i_size); if (err) goto out; - i_size_write(inode, i_size); - } else if (new_size > i_size) { - i_size_write(inode, new_size); } } @@ -747,16 +735,20 @@ int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, oldsize = i_size_read(inode); newsize = attr->ia_size; - if (newsize <= oldsize) - err = ntfs_truncate(inode, newsize); - else - err = ntfs_extend(inode, newsize, 0, NULL); + if (newsize != oldsize) { + truncate_setsize(inode, newsize); - if (err) - goto out; + err = ntfs_set_size(inode, newsize); + if (err) { + i_size_write(inode, oldsize); + goto out; + } - ni->ni_flags |= NI_FLAG_UPDATE_PARENT; - i_size_write(inode, newsize); + ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + inode_set_mtime_to_ts(inode, + inode_set_ctime_current(inode)); + } } setattr_copy(idmap, inode, attr); @@ -1020,7 +1012,7 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) CLST lcn, clen; frame = valid >> frame_bits; - frame_vbo = valid & ~(frame_size - 1); + frame_vbo = valid & ~(u64)(frame_size - 1); off = valid & (frame_size - 1); err = attr_data_get_block(ni, frame << NTFS_LZNT_CUNIT, 1, &lcn, @@ -1089,7 +1081,7 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) if (bytes > count) bytes = count; - frame_vbo = pos & ~(frame_size - 1); + frame_vbo = pos & ~(u64)(frame_size - 1); index = frame_vbo >> PAGE_SHIFT; if (unlikely(fault_in_iov_iter_readable(from, bytes))) { @@ -1230,6 +1222,7 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct ntfs_inode *ni = ntfs_i(inode); + loff_t vbo, endbyte; ssize_t ret, err; if (!inode_trylock(inode)) { @@ -1264,15 +1257,30 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } - ret = ntfs_extend(inode, iocb->ki_pos, ret, file); - if (ret) - goto out; + vbo = iocb->ki_pos; + endbyte = vbo + ret; + + if (endbyte > inode->i_size) { + /* + * Normal files: increase file size, allocate space. + * Sparse/Compressed: increase file size. No space allocated. + */ + ret = ntfs_set_size(inode, endbyte); + if (ret) + goto out; + } if (is_compressed(ni)) { ret = ntfs_compress_write(iocb, from); goto out; } + if (vbo > ni->i_valid) { + ret = ntfs_extend_initialized_size(file, ni, vbo); + if (ret) + goto out; + } + /* Fallback to buffered I/O if the inode does not support direct I/O. */ if (!(iocb->ki_flags & IOCB_DIRECT) || !ntfs_should_use_dio(iocb, from)) { @@ -1295,7 +1303,8 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } - ret = iomap_dio_rw(iocb, from, &ntfs_iomap_ops, NULL, 0, NULL, 0); + ret = iomap_dio_rw(iocb, from, &ntfs_iomap_ops, NULL, + IOMAP_DIO_FORCE_WAIT, NULL, 0); if (ret == -ENOTBLK) { /* Returns -ENOTBLK in case of a page invalidation failure for writes.*/ @@ -1304,7 +1313,7 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) } if (ret >= 0 && iov_iter_count(from)) { - loff_t offset = iocb->ki_pos, endbyte; + vbo = iocb->ki_pos; iocb->ki_flags &= ~IOCB_DIRECT; err = iomap_file_buffered_write(iocb, from, &ntfs_iomap_ops, @@ -1322,15 +1331,15 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) * to complete off the I/O request. */ ret += err; - endbyte = offset + err - 1; - err = filemap_write_and_wait_range(inode->i_mapping, offset, + endbyte = vbo + err - 1; + err = filemap_write_and_wait_range(inode->i_mapping, vbo, endbyte); if (err) { ret = err; goto out; } - invalidate_mapping_pages(inode->i_mapping, offset >> PAGE_SHIFT, + invalidate_mapping_pages(inode->i_mapping, vbo >> PAGE_SHIFT, endbyte >> PAGE_SHIFT); } @@ -1411,8 +1420,9 @@ static int ntfs_file_release(struct inode *inode, struct file *file) down_write(&ni->file.run_lock); /* Deallocate preallocated. */ - err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, - inode->i_size, &ni->i_valid, false); + err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, + inode->i_size, &ni->i_valid, false, NULL, + true); up_write(&ni->file.run_lock); ni_unlock(ni); @@ -1524,7 +1534,12 @@ static loff_t ntfs_llseek(struct file *file, loff_t offset, int whence) loff_t maxbytes = ntfs_get_maxbytes(ni); loff_t ret; - if (whence == SEEK_DATA || whence == SEEK_HOLE) { + if (whence != SEEK_DATA && whence != SEEK_HOLE) { + ret = generic_file_llseek_size(file, offset, whence, maxbytes, + i_size_read(inode)); + } else if ((unsigned long long)offset >= i_size_read(inode)) { + ret = -ENXIO; + } else { inode_lock_shared(inode); /* Scan file for hole or data. */ ret = ni_seek_data_or_hole(ni, offset, whence == SEEK_DATA); @@ -1532,9 +1547,6 @@ static loff_t ntfs_llseek(struct file *file, loff_t offset, int whence) if (ret >= 0) ret = vfs_setpos(file, ret, maxbytes); - } else { - ret = generic_file_llseek_size(file, offset, whence, maxbytes, - i_size_read(inode)); } return ret; } @@ -1547,6 +1559,8 @@ const struct inode_operations ntfs_file_inode_operations = { .get_acl = ntfs_get_acl, .set_acl = ntfs_set_acl, .fiemap = ntfs_fiemap, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, }; const struct file_operations ntfs_file_operations = { diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 7b035da63c12..2b49bc077558 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -1330,7 +1330,7 @@ int ni_expand_list(struct ntfs_inode *ni) { int err = 0; u32 asize, done = 0; - struct ATTRIB *attr, *ins_attr; + struct ATTRIB *attr, *ins_attr = NULL; struct ATTR_LIST_ENTRY *le; bool is_mft = ni->mi.rno == MFT_REC_MFT; struct MFT_REF ref; @@ -1363,7 +1363,7 @@ int ni_expand_list(struct ntfs_inode *ni) le16_to_cpu(attr->name_off), true, &ins_attr, NULL, NULL); - if (err) + if (err || !ins_attr) goto out; memcpy(ins_attr, attr, asize); @@ -1855,8 +1855,8 @@ enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, static struct folio *ntfs_lock_new_page(struct address_space *mapping, pgoff_t index, gfp_t gfp) { - struct folio *folio = __filemap_get_folio(mapping, index, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + struct folio *folio = __filemap_get_folio( + mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); if (IS_ERR(folio)) return folio; @@ -2800,8 +2800,8 @@ int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, err = ni_add_name(new_dir_ni, ni, new_de); if (!err) { err = ni_remove_name(dir_ni, ni, de, &de2, &undo); - WARN_ON(err && - ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)); + if (err && ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)) + _ntfs_bad_inode(&ni->vfs_inode); } /* @@ -2859,6 +2859,11 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) return err; } + if (!clen) { + /* Corrupted file. */ + return -EINVAL; + } + if (lcn == RESIDENT_LCN) { /* clen - resident size in bytes. clen == ni->vfs_inode.i_size */ if (offset >= clen) { @@ -2889,8 +2894,14 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) * the file offset is set to offset. */ if (lcn != SPARSE_LCN) { - vbo = (u64)vcn << cluster_bits; - return max(vbo, offset); + /* Normal cluster. */ + break; + } + + if ((ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) && + (vcn & (NTFS_LZNT_CLUSTERS - 1))) { + /* Compressed cluster in compressed frame. */ + break; } } else { /* @@ -2904,16 +2915,15 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) /* native compression hole begins at aligned vcn. */ (!(ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) || !(vcn & (NTFS_LZNT_CLUSTERS - 1)))) { - vbo = (u64)vcn << cluster_bits; - return max(vbo, offset); + /* Hole in sparsed or compressed file frame. */ + break; } } - if (!clen) { - /* Corrupted file. */ - return -EINVAL; - } } + + vbo = (u64)vcn << cluster_bits; + return max(vbo, offset); } /* diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index acfa18b84401..f038c799e7ac 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -764,8 +764,19 @@ static bool check_rstbl(const struct RESTART_TABLE *rt, size_t bytes) /* * Walk through the list headed by the first entry to make * sure none of the entries are currently being used. + * + * Bound traversal by ne (rt->used) to defeat a crafted on-disk + * cycle in the free chain. Each entry in a legitimate free + * list is unique, so a chain that visits more than ne slots + * is malformed. Without this guard, an attacker-controlled + * RESTART_TABLE with a self-loop or A->B->A cycle whose + * offsets satisfy the existing alignment + in-bounds guards + * spins forever at mount time. */ - for (off = ff; off;) { + for (off = ff, i = 0; off; i++) { + if (i > ne) + return false; + if (off == RESTART_ENTRY_ALLOCATED) return false; @@ -1172,7 +1183,7 @@ static int read_log_page(struct ntfs_log *log, u32 vbo, goto out; if (page_buf->rhdr.sign != NTFS_FFFF_SIGNATURE) - ntfs_fix_post_read(&page_buf->rhdr, PAGE_SIZE, false); + ntfs_fix_post_read(&page_buf->rhdr, log->page_size, false); if (page_buf != *buffer) memcpy(*buffer, Add2Ptr(page_buf, page_off), bytes); @@ -2599,11 +2610,12 @@ static int read_next_log_rec(struct ntfs_log *log, struct lcb *lcb, u64 *lsn) bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) { + const bool has_subnode = hdr_has_subnode(hdr); __le16 mask; u32 min_de, de_off, used, total; const struct NTFS_DE *e; - if (hdr_has_subnode(hdr)) { + if (has_subnode) { min_de = sizeof(struct NTFS_DE) + sizeof(u64); mask = NTFS_IE_HAS_SUBNODES; } else { @@ -2620,20 +2632,33 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) return false; } - e = Add2Ptr(hdr, de_off); + e = (const struct NTFS_DE *)((const u8 *)hdr + de_off); for (;;) { u16 esize = le16_to_cpu(e->size); - struct NTFS_DE *next = Add2Ptr(e, esize); + u16 key_size = le16_to_cpu(e->key_size); + u16 data_size; - if (esize < min_de || PtrOffset(hdr, next) > used || + if (!IS_ALIGNED(esize, 8) || esize < min_de || (e->flags & NTFS_IE_HAS_SUBNODES) != mask) { return false; } - if (de_is_last(e)) + if (size_add(de_off, esize) > used) + return false; + + if (de_is_last(e)) { + if (key_size) + return false; + break; + } - e = next; + data_size = esize - min_de; + if (key_size > data_size) + return false; + + de_off += esize; + e = (const struct NTFS_DE *)((const u8 *)hdr + de_off); } return true; @@ -3325,6 +3350,17 @@ skip_load_parent: nsize = ALIGN(nsize, 8); data_off = le16_to_cpu(attr->res.data_off); + /* + * aoff comes from the on-disk lrh->attr_off. Forbid + * writes that begin below the resident attribute's + * data_off (which would overwrite the resident header), + * and forbid aoff + dlen < data_off, which would make + * the data_size assignment below underflow to ~4 GiB. + */ + if (aoff < data_off || aoff + dlen < data_off || + aoff + dlen > asize) + goto dirty_vol; + if (nsize < asize) { memmove(Add2Ptr(attr, aoff), data, dlen); data = NULL; // To skip below memmove(). @@ -3368,7 +3404,10 @@ move_data: memmove(Add2Ptr(attr, aoff), data, dlen); if (run_get_highest_vcn(le64_to_cpu(attr->nres.svcn), - attr_run(attr), &t64)) { + attr_run(attr), + le32_to_cpu(attr->size) - + le16_to_cpu(attr->nres.run_off), + &t64)) { goto dirty_vol; } @@ -3497,6 +3536,18 @@ move_data: e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + /* + * e->view.data_off and dlen come from the on-disk + * INDEX_ROOT entry / LRH. The neighbouring read sites + * (e.g. fs/ntfs3/index.c) check that + * view.data_off + view.data_size <= e->size; mirror that + * bound here so the memmove cannot reach past the entry. + */ + if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || + le16_to_cpu(e->view.data_off) + dlen > + le16_to_cpu(e->size)) + goto dirty_vol; + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); mi->dirty = true; @@ -3570,9 +3621,23 @@ move_data: } e1 = Add2Ptr(e, esize); - nsize = esize; used = le32_to_cpu(hdr->used); + /* + * Reject crafted entries whose e->size makes e + esize + * point past the INDEX_HDR's used boundary. Without this, + * PtrOffset(e1, hdr + used) underflows to a quasi-infinite + * size_t when fed to the memmove() below. + * + * Also reject esize == 0: memmove(e, e, ...) is a no-op and + * leaves hdr->used unchanged, masking the crafted entry. + */ + if (!esize || Add2Ptr(e, esize) > Add2Ptr(hdr, used) || + PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + + nsize = esize; + memmove(e, e1, PtrOffset(e1, Add2Ptr(hdr, used))); hdr->used = cpu_to_le32(used - nsize); @@ -3689,6 +3754,12 @@ move_data: goto dirty_vol; } + /* See UpdateRecordDataRoot for the rationale. */ + if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || + le16_to_cpu(e->view.data_off) + dlen > + le16_to_cpu(e->size)) + goto dirty_vol; + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); a_dirty = true; @@ -3796,11 +3867,7 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) log->l_size = log->orig_file_size = ni->vfs_inode.i_size; /* Get the size of page. NOTE: To replay we can use default page. */ -#if PAGE_SIZE >= DefaultLogPageSize && PAGE_SIZE <= DefaultLogPageSize * 2 log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, true); -#else - log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, false); -#endif if (!log->page_size) { err = -EINVAL; goto out; @@ -3938,9 +4005,28 @@ check_restart_area: */ t32 = le32_to_cpu(log->rst_info.r_page->sys_page_size); if (log->page_size != t32) { + u32 old_page_size = log->page_size; + log->l_size = log->orig_file_size; log->page_size = norm_file_page(t32, &log->l_size, t32 == DefaultLogPageSize); + + /* + * If the adopted on-disk page size is larger than the size used + * to allocate one_page_buf above, grow the scratch buffer so a + * later read_log_page() cannot overflow it. + */ + if (log->page_size > old_page_size) { + void *buf; + + buf = krealloc(log->one_page_buf, log->page_size, + GFP_NOFS); + if (!buf) { + err = -ENOMEM; + goto out; + } + log->one_page_buf = buf; + } } if (log->page_size != t32 || @@ -4219,13 +4305,26 @@ check_dirty_page_table: if (rst->major_ver) goto end_conv_1; /* reduce tab pressure. */ + t16 = le16_to_cpu(dptbl->size); + if (t16 < sizeof(struct DIR_PAGE_ENTRY)) { + log->set_dirty = true; + goto out; + } + + t32 = (t16 - sizeof(struct DIR_PAGE_ENTRY)) / sizeof(u64); + dp = NULL; while ((dp = enum_rstbl(dptbl, dp))) { struct DIR_PAGE_ENTRY_32 *dp0 = (struct DIR_PAGE_ENTRY_32 *)dp; - // NOTE: Danger. Check for of boundary. + u32 lcns = le32_to_cpu(dp->lcns_follow); + + if (lcns > t32) { + log->set_dirty = true; + goto out; + } + memmove(&dp->vcn, &dp0->vcn_low, - 2 * sizeof(u64) + - le32_to_cpu(dp->lcns_follow) * sizeof(u64)); + 2 * sizeof(u64) + lcns * sizeof(u64)); } end_conv_1: @@ -4547,12 +4646,34 @@ copy_lcns: * whole routine a loop, case Lcns do not fit below. */ t16 = le16_to_cpu(lrh->lcns_follow); + t32 = le32_to_cpu(dp->lcns_follow); + if (le64_to_cpu(lrh->target_vcn) < le64_to_cpu(dp->vcn)) { + err = -EINVAL; + goto out; + } + + /* + * find_dp() only validates that target_vcn is the first + * cluster covered by dp. The walk through lrh->lcns_follow + * further entries must stay within the allocated + * dp->page_lcns[] array, which is sized by dp->lcns_follow. + */ + if (le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn) + t16 > + le32_to_cpu(dp->lcns_follow)) { + err = -EINVAL; + log->set_dirty = true; + goto out; + } + for (i = 0; i < t16; i++) { size_t j = (size_t)(le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn)); + if (j >= t32 || i >= t32 - j) { + err = -EINVAL; + goto out; + } dp->page_lcns[j + i] = lrh->page_lcns[i]; } - goto next_log_record_analyze; case DeleteDirtyClusters: { diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index d0434756029b..bc7469d0a34d 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -2654,7 +2654,6 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) struct ATTRIB *attr; u32 uni_bytes; struct ntfs_inode *ni = sbi->volume.ni; - /* Allocate PATH_MAX bytes. */ struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); if (!uni) diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 5344b29b0577..2b439ac04356 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -611,16 +611,51 @@ static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr, */ static bool index_hdr_check(const struct INDEX_HDR *hdr, u32 bytes) { + const bool has_subnode = hdr_has_subnode(hdr); + const u16 min_size = sizeof(struct NTFS_DE) + + (has_subnode ? sizeof(u64) : 0); u32 end = le32_to_cpu(hdr->used); u32 tot = le32_to_cpu(hdr->total); u32 off = le32_to_cpu(hdr->de_off); + const struct NTFS_DE *e; if (!IS_ALIGNED(off, 8) || tot > bytes || end > tot || - size_add(off, sizeof(struct NTFS_DE)) > end) { + size_add(off, min_size) > end) { /* incorrect index buffer. */ return false; } + /* Ensure every key stays inside its entry before lookup walks it. */ + e = (const struct NTFS_DE *)((const u8 *)hdr + off); + for (;;) { + u16 e_size = le16_to_cpu(e->size); + u16 key_size = le16_to_cpu(e->key_size); + u16 data_size; + + if (!IS_ALIGNED(e_size, 8) || e_size < min_size || + de_has_vcn(e) != has_subnode) { + /* incorrect index entry. */ + return false; + } + + if (size_add(off, e_size) > end) + return false; + + if (de_is_last(e)) { + if (key_size) + return false; + + break; + } + + data_size = e_size - min_size; + if (key_size > data_size) + return false; + + off += e_size; + e = (const struct NTFS_DE *)((const u8 *)hdr + off); + } + return true; } @@ -754,6 +789,10 @@ fill_table: binary_search: e_key_len = le16_to_cpu(e->key_size); + /* Validate key_size fits within the entry data area. */ + if (e_key_len > le16_to_cpu(e->size) - sizeof(struct NTFS_DE)) + return NULL; + diff2 = (*cmp)(key, key_len, e + 1, e_key_len, ctx); if (diff2 > 0) { if (found) { @@ -1506,6 +1545,7 @@ static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, if (bit != MINUS_ONE_T) { bmp = NULL; + bmp_size = bmp_size_v = 0; } else { if (bmp->non_res) { bmp_size = le64_to_cpu(bmp->nres.data_size); @@ -1587,7 +1627,8 @@ out1: static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni, const struct NTFS_DE *new_de, struct NTFS_DE *root_de, const void *ctx, - struct ntfs_fnd *fnd, bool undo, NTFS_CMP_FUNC cmp) + struct ntfs_fnd *fnd, bool undo, + NTFS_CMP_FUNC cmp) { int err = 0; struct NTFS_DE *e, *e0, *re; @@ -1742,6 +1783,22 @@ static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni, hdr_used = le32_to_cpu(hdr->used); hdr_total = le32_to_cpu(hdr->total); + /* + * The destination INDEX_BUFFER has 'hdr_total' bytes of payload + * available after the header, of which 'hdr_used' are already + * consumed by the single terminal END entry installed by + * indx_new(). A crafted image can present a resident root whose + * non-last entries (summing to 'to_move') exceed what fits in + * this buffer; copying them unchecked would overrun the + * kmalloc(1u << indx->index_bits) allocation backing the new + * buffer. Reject the copy in that case. + */ + if (to_move > hdr_total - hdr_used) { + err = -EINVAL; + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + goto out_put_n; + } + /* Copy root entries into new buffer. */ hdr_insert_head(hdr, re, to_move); @@ -1796,13 +1853,15 @@ out_free_root: * Attempt to insert an entry into an Index Allocation Buffer. * If necessary, it will split the buffer. */ -static int -indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, - struct INDEX_ROOT *root, const struct NTFS_DE *new_de, - const void *ctx, int level, struct ntfs_fnd *fnd, NTFS_CMP_FUNC cmp) +static int indx_insert_into_buffer(struct ntfs_index *indx, + struct ntfs_inode *ni, + struct INDEX_ROOT *root, + const struct NTFS_DE *new_de, + const void *ctx, int level, + struct ntfs_fnd *fnd, NTFS_CMP_FUNC cmp) { int err; - const struct NTFS_DE *sp; + const struct NTFS_DE *sp; /* split_point */ struct NTFS_DE *e, *de_t, *up_e; struct indx_node *n2; struct indx_node *n1 = fnd->nodes[level]; @@ -1828,10 +1887,9 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, * No space to insert into buffer. Split it. * To split we: * - Save split point ('cause index buffers will be changed) - * - Allocate NewBuffer and copy all entries <= sp into new buffer - * - Remove all entries (sp including) from TargetBuffer - * - Insert NewEntry into left or right buffer (depending on sp <=> - * NewEntry) + * - Allocate new buffer (up_e) and copy all entries <= sp into new buffer + * - Remove all entries (sp including) from hdr1 + * - Insert new_de into left or right buffer (depending on sp <=> new_de) * - Insert sp into parent buffer (or root) * - Make sp a parent for new buffer */ @@ -1845,7 +1903,22 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, return -ENOMEM; memcpy(up_e, sp, sp_size); + /* Make a copy for undo. */ used1 = le32_to_cpu(hdr1->used); + + /* + * hdr_find_split does not validate per-entry sizes, so a crafted + * NTFS_DE whose le16 size field is out of range can place sp such + * that (PtrOffset(hdr1, sp) + sp_size) exceeds used1. Without this + * guard the u32 'used = used1 - to_copy - sp_size' underflows and + * the subsequent memmove count becomes a near-4-GiB value, + * triggering an out-of-bounds kernel write. + */ + if (PtrOffset(hdr1, sp) + sp_size > used1) { + err = -EINVAL; + goto out; + } + hdr1_saved = kmemdup(hdr1, used1, GFP_NOFS); if (!hdr1_saved) { err = -ENOMEM; @@ -1894,8 +1967,7 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, */ hdr_insert_de(indx, (*cmp)(new_de + 1, le16_to_cpu(new_de->key_size), - up_e + 1, le16_to_cpu(up_e->key_size), - ctx) < 0 ? + up_e + 1, le16_to_cpu(up_e->key_size), ctx) < 0 ? hdr2 : hdr1, new_de, NULL, ctx, cmp); @@ -1912,11 +1984,13 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, * insert the promoted entry into the parent. */ if (!level) { - /* Insert in root. */ - err = indx_insert_into_root(indx, ni, up_e, NULL, ctx, fnd, 0, cmp); + /* Insert split_point in root. */ + err = indx_insert_into_root(indx, ni, up_e, NULL, ctx, fnd, 0, + cmp); } else { /* * The target buffer's parent is another index buffer. + * Insert split_point in parent index ( call itself recursively ) * TODO: Remove recursion. */ err = indx_insert_into_buffer(indx, ni, root, up_e, ctx, @@ -2022,13 +2096,21 @@ out1: static struct indx_node *indx_find_buffer(struct ntfs_index *indx, struct ntfs_inode *ni, const struct INDEX_ROOT *root, - __le64 vbn, struct indx_node *n) + __le64 vbn, struct indx_node *n, + int depth) { int err; const struct NTFS_DE *e; struct indx_node *r; const struct INDEX_HDR *hdr = n ? &n->index->ihdr : &root->ihdr; + /* + * Limit recursion depth to prevent stack overflow from crafted + * images. Use the same bound as the fnd->nodes array (20). + */ + if (depth > ARRAY_SIZE(((struct ntfs_fnd *)NULL)->nodes)) + return ERR_PTR(-EINVAL); + /* Step 1: Scan one level. */ for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) { if (!e) @@ -2049,7 +2131,8 @@ static struct indx_node *indx_find_buffer(struct ntfs_index *indx, if (err) return ERR_PTR(err); - r = indx_find_buffer(indx, ni, root, vbn, n); + r = indx_find_buffer(indx, ni, root, vbn, n, + depth + 1); if (r) return r; } @@ -2462,7 +2545,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, fnd_clear(fnd); - in = indx_find_buffer(indx, ni, root, sub_vbn, NULL); + in = indx_find_buffer(indx, ni, root, sub_vbn, NULL, 0); if (IS_ERR(in)) { err = PTR_ERR(in); goto out; diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 42af1abe17f8..0c9bd669117d 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -592,7 +592,6 @@ static void ntfs_iomap_read_end_io(struct bio *bio) u32 f_size = folio_size(folio); loff_t f_pos = folio_pos(folio); - if (valid < f_pos + f_size) { u32 z_from = valid <= f_pos ? 0 : @@ -609,10 +608,7 @@ static void ntfs_iomap_read_end_io(struct bio *bio) static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter, struct iomap_read_folio_ctx *ctx) { - struct bio *bio = ctx->read_ctx; - - bio->bi_end_io = ntfs_iomap_read_end_io; - submit_bio(bio); + iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io); } static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { @@ -693,17 +689,18 @@ int ntfs_set_size(struct inode *inode, u64 new_size) return -EFBIG; } + /* Mark rw ntfs as dirty. It will be cleared at umount. */ + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + ni_lock(ni); down_write(&ni->file.run_lock); + if (new_size < ni->i_valid) + ni->i_valid = new_size; + /* last 'true' means keep preallocated. */ err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, &ni->i_valid, true); - if (!err) { - i_size_write(inode, new_size); - mark_inode_dirty(inode); - } - up_write(&ni->file.run_lock); ni_unlock(ni); @@ -764,7 +761,7 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, clen_max = bytes_to_cluster(sbi, endbyte) - vcn; } - /* + /* * Force to allocate clusters if directIO(write) or writeback_range. * NOTE: attr_data_get_block allocates clusters only for sparse file. * Normal file allocates clusters in attr_set_size. @@ -778,11 +775,6 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return err; } - if (!clen) { - /* broken file? */ - return -EINVAL; - } - if (lcn == EOF_LCN) { /* request out of file. */ if (flags & IOMAP_REPORT) { @@ -801,7 +793,7 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (lcn == RESIDENT_LCN) { if (offset >= clen) { - kfree(res); + __free_page(virt_to_page(res)); if (flags & IOMAP_REPORT) { /* special code for report. */ return -ENOENT; @@ -816,6 +808,11 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } + if (!clen) { + /* broken file? */ + return -EINVAL; + } + iomap->bdev = inode->i_sb->s_bdev; iomap->offset = offset; iomap->length = ((loff_t)clen << cluster_bits) - off; @@ -829,7 +826,6 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, iomap->type = IOMAP_DELALLOC; iomap->addr = IOMAP_NULL_ADDR; } else { - /* Translate clusters into bytes. */ iomap->addr = ((loff_t)lcn << cluster_bits) + off; if (length && iomap->length > length) @@ -921,7 +917,7 @@ static int ntfs_iomap_end(struct inode *inode, loff_t pos, loff_t length, out: if (iomap->type == IOMAP_INLINE) { - kfree(iomap->private); + __free_page(virt_to_page(iomap->private)); iomap->private = NULL; } @@ -986,37 +982,11 @@ static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); } - static const struct iomap_writeback_ops ntfs_writeback_ops = { .writeback_range = ntfs_writeback_range, .writeback_submit = iomap_ioend_writeback_submit, }; -static int ntfs_resident_writepage(struct folio *folio, - struct writeback_control *wbc) -{ - struct address_space *mapping = folio->mapping; - struct inode *inode = mapping->host; - struct ntfs_inode *ni = ntfs_i(inode); - int ret; - - /* Avoid any operation if inode is bad. */ - if (unlikely(is_bad_ni(ni))) - return -EINVAL; - - if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) - return -EIO; - - ni_lock(ni); - ret = attr_data_write_resident(ni, folio); - ni_unlock(ni); - - if (ret != E_NTFS_NONRESIDENT) - folio_unlock(folio); - mapping_set_error(mapping, ret); - return ret; -} - static int ntfs_writepages(struct address_space *mapping, struct writeback_control *wbc) { @@ -1024,7 +994,7 @@ static int ntfs_writepages(struct address_space *mapping, struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); struct iomap_writepage_ctx wpc = { - .inode = mapping->host, + .inode = inode, .wbc = wbc, .ops = &ntfs_writeback_ops, }; @@ -1038,9 +1008,22 @@ static int ntfs_writepages(struct address_space *mapping, if (is_resident(ni)) { struct folio *folio = NULL; + err = 0; + + while ((folio = writeback_iter(mapping, wbc, folio, &err))) { + int err2; + + ni_lock(ni); + err2 = attr_data_write_resident(ni, folio); + ni_unlock(ni); - while ((folio = writeback_iter(mapping, wbc, folio, &err))) - err = ntfs_resident_writepage(folio, wbc); + folio_unlock(folio); + if (err2) { + mapping_set_error(mapping, err2); + if (!err) + err = err2; + } + } return err; } @@ -1291,7 +1274,6 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, if (!(mode & 0222)) fa |= FILE_ATTRIBUTE_READONLY; - /* Allocate PATH_MAX bytes. */ new_de = kzalloc(PATH_MAX, GFP_KERNEL); if (!new_de) { err = -ENOMEM; @@ -1730,7 +1712,6 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; struct NTFS_DE *de; - /* Allocate PATH_MAX bytes. */ de = kzalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -2095,6 +2076,8 @@ const struct inode_operations ntfs_link_inode_operations = { .get_link = ntfs_get_link, .setattr = ntfs_setattr, .listxattr = ntfs_listxattr, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, }; const struct address_space_operations ntfs_aops = { diff --git a/fs/ntfs3/lznt.c b/fs/ntfs3/lznt.c index fdc9b2ebf341..f818d9785004 100644 --- a/fs/ntfs3/lznt.c +++ b/fs/ntfs3/lznt.c @@ -240,7 +240,7 @@ static inline ssize_t decompress_chunk(u8 *unc, u8 *unc_end, const u8 *cmpr, if (up - unc > LZNT_CHUNK_SIZE) return -EINVAL; /* Correct index */ - while (unc + s_max_off[index] < up) + while (index < ARRAY_SIZE(s_max_off) - 1 && unc + s_max_off[index] < up) index += 1; /* Check the current flag for zero. */ diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c index b2af8f695e60..c59de5f2fa97 100644 --- a/fs/ntfs3/namei.c +++ b/fs/ntfs3/namei.c @@ -340,7 +340,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, ntfs_sync_inode(dir); if (IS_DIRSYNC(new_dir)) - ntfs_sync_inode(inode); + ntfs_sync_inode(new_dir); } if (dir_ni != new_dir_ni) @@ -518,6 +518,8 @@ const struct inode_operations ntfs_dir_inode_operations = { .getattr = ntfs_getattr, .listxattr = ntfs_listxattr, .fiemap = ntfs_fiemap, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, }; const struct inode_operations ntfs_special_inode_operations = { @@ -526,6 +528,8 @@ const struct inode_operations ntfs_special_inode_operations = { .listxattr = ntfs_listxattr, .get_acl = ntfs_get_acl, .set_acl = ntfs_set_acl, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, }; const struct dentry_operations ntfs_dentry_ops = { diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index bbf3b6a1dcbe..d98d7e474476 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -392,6 +392,9 @@ struct ntfs_inode { */ u8 ni_bad; + /* Keep track of FS_NODUMP_FL. */ + u8 nodump; + union { struct ntfs_index dir; struct { @@ -529,6 +532,9 @@ bool dir_is_empty(struct inode *dir); extern const struct file_operations ntfs_dir_operations; /* Globals from file.c */ +int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int ntfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry, + struct file_kattr *fa); int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask, u32 flags); int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, @@ -852,6 +858,9 @@ static inline void mi_get_ref(const struct mft_inode *mi, struct MFT_REF *ref) /* Globals from run.c */ bool run_lookup_entry(const struct runs_tree *run, CLST vcn, CLST *lcn, CLST *len, size_t *index); +bool run_lookup_entry_da(const struct runs_tree *run, + const struct runs_tree *run_da, CLST vcn, CLST *lcn, + CLST *len); void run_truncate(struct runs_tree *run, CLST vcn); void run_truncate_head(struct runs_tree *run, CLST vcn); void run_truncate_around(struct runs_tree *run, CLST vcn); @@ -877,7 +886,8 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, #else #define run_unpack_ex run_unpack #endif -int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn); +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn); int run_clone(const struct runs_tree *run, struct runs_tree *new_run); bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done); CLST run_len(const struct runs_tree *run); diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 1ce7d92fb274..3ebf0154eda3 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -224,6 +224,66 @@ bool run_lookup_entry(const struct runs_tree *run, CLST vcn, CLST *lcn, } /* + * run_overlaps + * + * true if run overlaps with range [svcn, svcn + len) + */ +static bool run_overlaps(const struct runs_tree *run, CLST svcn, CLST len, + CLST *vcn, CLST *clen) +{ + size_t i; + const struct ntfs_run *r = run->runs; + CLST end = svcn + len; + + for (i = 0; i < run->count; i++, r++) { + /* Check if [r->vcn, r->vcn+r->len) overlaps [svcn, end). */ + if (r->vcn < end && svcn < r->vcn + r->len) { + if (vcn) + *vcn = r->vcn; + if (clen) + *clen = r->len; + return true; + } + } + + return false; +} + +/* + * run_lookup_entry_da + * + * - lookup vcn in delalloc run + * - lookup vcn in real run + * - correct result if real run overlaps with delalloc + */ +bool run_lookup_entry_da(const struct runs_tree *run, + const struct runs_tree *run_da, CLST vcn, CLST *lcn, + CLST *len) +{ + CLST vcn1, len1; + + if (run_da && run_lookup_entry(run_da, vcn, lcn, len, NULL)) { + *lcn = DELALLOC_LCN; + return true; + } + + if (!run_lookup_entry(run, vcn, lcn, len, NULL)) + return false; + + if (run_da && run_overlaps(run_da, vcn, *len, &vcn1, &len1)) { + /* Correct return value. */ + if (vcn1 > vcn) { + *len = vcn1 - vcn; + } else { + *lcn = DELALLOC_LCN; + *len = len1; + } + } + + return true; +} + +/* * run_truncate_head - Decommit the range before vcn. */ void run_truncate_head(struct runs_tree *run, CLST vcn) @@ -1205,18 +1265,23 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, * Return the highest vcn from a mapping pairs array * it used while replaying log file. */ -int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn) +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn) { + const u8 *run_last = run_buf + run_buf_size; u64 vcn64 = vcn; u8 size_size; - while ((size_size = *run_buf & 0xF)) { + while (run_buf < run_last && (size_size = *run_buf & 0xF)) { u8 offset_size = *run_buf++ >> 4; u64 len; if (size_size > 8 || offset_size > 8) return -EINVAL; + if (run_buf + size_size + offset_size > run_last) + return -EINVAL; + len = run_unpack_s64(run_buf, size_size, 0); if (!len) return -EINVAL; @@ -1281,7 +1346,6 @@ bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done) return true; } - e = run->runs + run->count; r = run->runs + index; end = vcn + len; @@ -1292,9 +1356,12 @@ bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done) if (r_end > end) { /* Remove a middle part, split. */ + CLST tail_lcn = r->lcn == SPARSE_LCN ? + SPARSE_LCN : (r->lcn + (end - r->vcn)); + *done += len; r->len = d; - return run_add_entry(run, end, r->lcn, r_end - end, + return run_add_entry(run, end, tail_lcn, r_end - end, false); } /* Remove tail of run .*/ diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 004f59937559..3305fe406cb2 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -1174,7 +1174,10 @@ read_boot: rec->total = cpu_to_le32(sbi->record_size); ((struct ATTRIB *)Add2Ptr(rec, ao))->type = ATTR_END; - sb_set_blocksize(sb, min_t(u32, sbi->cluster_size, PAGE_SIZE)); + if (!sb_set_blocksize(sb, min_t(u32, sbi->cluster_size, PAGE_SIZE))) { + err = -EINVAL; + goto out; + } sbi->block_mask = sb->s_blocksize - 1; sbi->blocks_per_cluster = sbi->cluster_size >> sb->s_blocksize_bits; @@ -1225,7 +1228,8 @@ out: /* * Try alternative boot (last sector) */ - sb_set_blocksize(sb, block_size); + if (!sb_set_blocksize(sb, block_size)) + return -EINVAL; hint = "Alternative boot"; dev_size = dev_size0; /* restore original size. */ goto read_boot; diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index 9eeac0ab2b71..04814dd29375 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -851,6 +851,12 @@ out: return err; } +static bool ntfs_is_reserved_lxattr(const char *name) +{ + return !strcmp(name, "$LXUID") || !strcmp(name, "$LXGID") || + !strcmp(name, "$LXMOD") || !strcmp(name, "$LXDEV"); +} + /* * ntfs_setxattr - inode_operations::setxattr */ @@ -867,7 +873,9 @@ static noinline int ntfs_setxattr(const struct xattr_handler *handler, if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { if (sizeof(u8) != size) goto out; - new_fa = cpu_to_le32(*(u8 *)value); + /* system.dos_attrib only covers the low DOS attribute byte. */ + new_fa = (ni->std_fa & ~cpu_to_le32(0xff)) | + cpu_to_le32(*(u8 *)value); goto set_new_fa; } @@ -955,6 +963,12 @@ set_new_fa: goto out; } + /* Do not allow non privileged users to change $LXUID/$LXGID... */ + if (ntfs_is_reserved_lxattr(name) && !capable(CAP_SYS_ADMIN)) { + err = -EPERM; + goto out; + } + /* Deal with NTFS extended attribute. */ err = ntfs_set_ea(inode, name, strlen(name), value, size, flags, 0, NULL); diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c index 6e5fd3f12a84..be09e766ac1f 100644 --- a/fs/ocfs2/alloc.c +++ b/fs/ocfs2/alloc.c @@ -7576,7 +7576,7 @@ int ocfs2_trim_mainbm(struct super_block *sb, struct fstrim_range *range) len = range->len >> osb->s_clustersize_bits; minlen = range->minlen >> osb->s_clustersize_bits; - if (minlen >= osb->bitmap_cpg || range->len < sb->s_blocksize) + if (minlen >= osb->bitmap_cpg || range->len < osb->s_clustersize) return -EINVAL; trace_ocfs2_trim_mainbm(start, len, minlen); diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 6ec198bdab12..4acdbb70882c 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -2372,6 +2372,15 @@ commit: unlock: up_write(&oi->ip_alloc_sem); + if (data_ac) { + ocfs2_free_alloc_context(data_ac); + data_ac = NULL; + } + if (meta_ac) { + ocfs2_free_alloc_context(meta_ac); + meta_ac = NULL; + } + /* everything looks good, let's start the cleanup */ if (!ret && dwc->dw_orphaned) { BUG_ON(dwc->dw_writer_pid != task_pid_nr(current)); @@ -2383,10 +2392,6 @@ unlock: ocfs2_inode_unlock(inode, 1); brelse(di_bh); out: - if (data_ac) - ocfs2_free_alloc_context(data_ac); - if (meta_ac) - ocfs2_free_alloc_context(meta_ac); ocfs2_run_deallocs(osb, &dealloc); ocfs2_dio_free_write_ctx(inode, dwc); diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c index 701d27d908d4..7bfe377af2df 100644 --- a/fs/ocfs2/buffer_head_io.c +++ b/fs/ocfs2/buffer_head_io.c @@ -62,9 +62,7 @@ int ocfs2_write_block(struct ocfs2_super *osb, struct buffer_head *bh, /* remove from dirty list before I/O. */ clear_buffer_dirty(bh); - get_bh(bh); /* for end_buffer_write_sync() */ - bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE, bh); + bh_submit(bh, REQ_OP_WRITE, bh_end_write); wait_on_buffer(bh); @@ -145,9 +143,7 @@ int ocfs2_read_blocks_sync(struct ocfs2_super *osb, u64 block, #endif } - get_bh(bh); /* for end_buffer_read_sync() */ - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); } read_failure: @@ -323,11 +319,9 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, continue; } - get_bh(bh); /* for end_buffer_read_sync() */ if (validate) set_buffer_needs_validate(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); continue; } } @@ -350,8 +344,6 @@ read_failure: wait_on_buffer(bh); put_bh(bh); bhs[i] = NULL; - } else if (bh && buffer_uptodate(bh)) { - clear_buffer_uptodate(bh); } continue; } @@ -380,8 +372,11 @@ read_failure: BUG_ON(buffer_jbd(bh)); clear_buffer_needs_validate(bh); status = validate(sb, bh); - if (status) + if (status) { + if (buffer_uptodate(bh)) + clear_buffer_uptodate(bh); goto read_failure; + } } } @@ -446,10 +441,8 @@ int ocfs2_write_super_or_backup(struct ocfs2_super *osb, /* remove from dirty list before I/O. */ clear_buffer_dirty(bh); - get_bh(bh); /* for end_buffer_write_sync() */ - bh->b_end_io = end_buffer_write_sync; ocfs2_compute_meta_ecc(osb->sb, bh->b_data, &di->i_check); - submit_bh(REQ_OP_WRITE, bh); + bh_submit(bh, REQ_OP_WRITE, bh_end_write); wait_on_buffer(bh); diff --git a/fs/ocfs2/dlm/dlmdebug.c b/fs/ocfs2/dlm/dlmdebug.c index fe4fdd09bae3..ce6d301cb942 100644 --- a/fs/ocfs2/dlm/dlmdebug.c +++ b/fs/ocfs2/dlm/dlmdebug.c @@ -260,10 +260,10 @@ void dlm_print_one_mle(struct dlm_master_list_entry *mle) { char *buf; - buf = (char *) get_zeroed_page(GFP_ATOMIC); + buf = kzalloc(PAGE_SIZE, GFP_ATOMIC); if (buf) { dump_mle(mle, buf, PAGE_SIZE - 1); - free_page((unsigned long)buf); + kfree(buf); } } @@ -280,7 +280,7 @@ static struct dentry *dlm_debugfs_root; /* begin - utils funcs */ static int debug_release(struct inode *inode, struct file *file) { - free_page((unsigned long)file->private_data); + kfree(file->private_data); return 0; } @@ -327,17 +327,15 @@ static int debug_purgelist_open(struct inode *inode, struct file *file) struct dlm_ctxt *dlm = inode->i_private; char *buf = NULL; - buf = (char *) get_zeroed_page(GFP_NOFS); + buf = kzalloc(PAGE_SIZE, GFP_NOFS); if (!buf) - goto bail; + return -ENOMEM; i_size_write(inode, debug_purgelist_print(dlm, buf, PAGE_SIZE - 1)); file->private_data = buf; return 0; -bail: - return -ENOMEM; } static const struct file_operations debug_purgelist_fops = { @@ -384,17 +382,15 @@ static int debug_mle_open(struct inode *inode, struct file *file) struct dlm_ctxt *dlm = inode->i_private; char *buf = NULL; - buf = (char *) get_zeroed_page(GFP_NOFS); + buf = kzalloc(PAGE_SIZE, GFP_NOFS); if (!buf) - goto bail; + return -ENOMEM; i_size_write(inode, debug_mle_print(dlm, buf, PAGE_SIZE - 1)); file->private_data = buf; return 0; -bail: - return -ENOMEM; } static const struct file_operations debug_mle_fops = { @@ -560,6 +556,7 @@ static int debug_lockres_open(struct inode *inode, struct file *file) struct dlm_ctxt *dlm = inode->i_private; struct debug_lockres *dl; void *buf; + int status = -ENOMEM; buf = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!buf) @@ -572,16 +569,23 @@ static int debug_lockres_open(struct inode *inode, struct file *file) dl->dl_len = PAGE_SIZE; dl->dl_buf = buf; - dlm_grab(dlm); - dl->dl_ctxt = dlm; + /* ->release uses dl_ctxt after open, so it needs a real pin. */ + dl->dl_ctxt = dlm_grab(dlm); + if (!dl->dl_ctxt) { + status = -ENOENT; + goto bailseq; + } return 0; +bailseq: + seq_release_private(inode, file); bailfree: kfree(buf); bail: - mlog_errno(-ENOMEM); - return -ENOMEM; + if (status != -ENOENT) + mlog_errno(status); + return status; } static int debug_lockres_release(struct inode *inode, struct file *file) @@ -775,17 +779,15 @@ static int debug_state_open(struct inode *inode, struct file *file) struct dlm_ctxt *dlm = inode->i_private; char *buf = NULL; - buf = (char *) get_zeroed_page(GFP_NOFS); + buf = kzalloc(PAGE_SIZE, GFP_NOFS); if (!buf) - goto bail; + return -ENOMEM; i_size_write(inode, debug_state_print(dlm, buf, PAGE_SIZE - 1)); file->private_data = buf; return 0; -bail: - return -ENOMEM; } static const struct file_operations debug_state_fops = { diff --git a/fs/ocfs2/dlm/dlmdomain.c b/fs/ocfs2/dlm/dlmdomain.c index dc9da9133c8e..97bb9400e24b 100644 --- a/fs/ocfs2/dlm/dlmdomain.c +++ b/fs/ocfs2/dlm/dlmdomain.c @@ -63,7 +63,7 @@ static inline void byte_copymap(u8 dmap[], unsigned long smap[], static void dlm_free_pagevec(void **vec, int pages) { while (pages--) - free_page((unsigned long)vec[pages]); + kfree(vec[pages]); kfree(vec); } @@ -75,9 +75,11 @@ static void **dlm_alloc_pagevec(int pages) if (!vec) return NULL; - for (i = 0; i < pages; i++) - if (!(vec[i] = (void *)__get_free_page(GFP_KERNEL))) + for (i = 0; i < pages; i++) { + vec[i] = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!vec[i]) goto out_free; + } mlog(0, "Allocated DLM hash pagevec; %d pages (%lu expected), %lu buckets per page\n", pages, (unsigned long)DLM_HASH_PAGES, diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index 93eff38fdadd..aee3b4c56dcc 100644 --- a/fs/ocfs2/dlm/dlmmaster.c +++ b/fs/ocfs2/dlm/dlmmaster.c @@ -2548,7 +2548,7 @@ static int dlm_migrate_lockres(struct dlm_ctxt *dlm, /* preallocate up front. if this fails, abort */ ret = -ENOMEM; - mres = (struct dlm_migratable_lockres *) __get_free_page(GFP_NOFS); + mres = kmalloc(PAGE_SIZE, GFP_NOFS); if (!mres) { mlog_errno(ret); goto leave; @@ -2725,8 +2725,7 @@ leave: if (wake) wake_up(&res->wq); - if (mres) - free_page((unsigned long)mres); + kfree(mres); dlm_put(dlm); diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c index 128872bd945d..9b97bf73df22 100644 --- a/fs/ocfs2/dlm/dlmrecovery.c +++ b/fs/ocfs2/dlm/dlmrecovery.c @@ -837,7 +837,7 @@ int dlm_request_all_locks_handler(struct o2net_msg *msg, u32 len, void *data, } /* this will get freed by dlm_request_all_locks_worker */ - buf = (char *) __get_free_page(GFP_NOFS); + buf = kmalloc(PAGE_SIZE, GFP_NOFS); if (!buf) { kfree(item); dlm_put(dlm); @@ -933,7 +933,7 @@ static void dlm_request_all_locks_worker(struct dlm_work_item *item, void *data) } } leave: - free_page((unsigned long)data); + kfree(data); } diff --git a/fs/ocfs2/dlmglue.c b/fs/ocfs2/dlmglue.c index 7283bb2c5a31..a23dd8f86c89 100644 --- a/fs/ocfs2/dlmglue.c +++ b/fs/ocfs2/dlmglue.c @@ -3134,6 +3134,22 @@ static void *ocfs2_dlm_seq_next(struct seq_file *m, void *v, loff_t *pos) * - Add last pr/ex unlock times and first lock wait time in usecs */ #define OCFS2_DLM_DEBUG_STR_VERSION 4 + +/* + * The debug iterator snapshots lockres by value, so a userspace-stack LVB + * pointer copied from the original lockres must be rebased to the copied + * lksb before the dump walks the raw bytes. + */ +static void ocfs2_dlm_seq_rebase_lvb(struct ocfs2_lock_res *lockres) +{ + if (!ocfs2_stack_supports_plocks()) + return; + + if (lockres->l_lksb.lksb_fsdlm.sb_lvbptr) + lockres->l_lksb.lksb_fsdlm.sb_lvbptr = + (char *)&lockres->l_lksb + sizeof(struct dlm_lksb); +} + static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) { int i; @@ -3191,6 +3207,7 @@ static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) lockres->l_blocking); /* Dump the raw LVB */ + ocfs2_dlm_seq_rebase_lvb(lockres); lvb = ocfs2_dlm_lvb(&lockres->l_lksb); for(i = 0; i < DLM_LVB_LEN; i++) seq_printf(m, "0x%x\t", lvb[i]); diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 7df9921c1a38..d6e977ba6565 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -444,21 +444,26 @@ int ocfs2_truncate_file(struct inode *inode, struct ocfs2_dinode *fe = NULL; struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - /* We trust di_bh because it comes from ocfs2_inode_lock(), which - * already validated it */ + /* + * On local mounts ocfs2_inode_lock_update() skips the inode + * refresh path, so truncation still needs to reject an inode + * state that no longer matches di_bh. + */ fe = (struct ocfs2_dinode *) di_bh->b_data; trace_ocfs2_truncate_file((unsigned long long)OCFS2_I(inode)->ip_blkno, (unsigned long long)le64_to_cpu(fe->i_size), (unsigned long long)new_i_size); - mlog_bug_on_msg(le64_to_cpu(fe->i_size) != i_size_read(inode), - "Inode %llu, inode i_size = %lld != di " - "i_size = %llu, i_flags = 0x%x\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno, - i_size_read(inode), - (unsigned long long)le64_to_cpu(fe->i_size), - le32_to_cpu(fe->i_flags)); + if (unlikely(le64_to_cpu(fe->i_size) != i_size_read(inode))) { + status = ocfs2_error(inode->i_sb, + "Inode %llu has inconsistent i_size: inode = %lld, dinode = %llu, i_flags = 0x%x\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + i_size_read(inode), + (unsigned long long)le64_to_cpu(fe->i_size), + le32_to_cpu(fe->i_flags)); + goto bail; + } if (new_i_size > le64_to_cpu(fe->i_size)) { trace_ocfs2_truncate_file_error( diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index a510a0eb1adc..662dbc845b8b 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -13,6 +13,7 @@ #include <linux/pagemap.h> #include <linux/quotaops.h> #include <linux/iversion.h> +#include <linux/fs_dirent.h> #include <asm/byteorder.h> @@ -64,7 +65,40 @@ static int ocfs2_filecheck_read_inode_block_full(struct inode *inode, static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, struct buffer_head *bh); static int ocfs2_filecheck_repair_inode_block(struct super_block *sb, - struct buffer_head *bh); + struct buffer_head *bh); + +static bool ocfs2_valid_inode_mode(umode_t mode) +{ + return fs_umode_to_ftype(mode) != FT_UNKNOWN; +} + +static bool ocfs2_dinode_has_unexpected_rdev(struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + + return !S_ISCHR(mode) && !S_ISBLK(mode) && di->id1.dev1.i_rdev != 0; +} + +static bool ocfs2_dinode_has_size_without_clusters(struct super_block *sb, + struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) + return false; + if (!le64_to_cpu(di->i_size) || le32_to_cpu(di->i_clusters)) + return false; + + if (S_ISDIR(mode)) + return true; + + return !ocfs2_sparse_alloc(OCFS2_SB(sb)) && S_ISREG(mode); +} void ocfs2_set_inode_flags(struct inode *inode) { @@ -1494,6 +1528,86 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + /* + * Reject dinodes whose i_mode does not name one of the seven + * canonical POSIX file types. ocfs2_populate_inode() copies + * i_mode verbatim into inode->i_mode and then dispatches via + * switch (mode & S_IFMT) to file/dir/symlink/special_file iops; + * an unrecognised type falls into ocfs2_special_file_iops with + * init_special_inode(), which interprets i_rdev. Constrain the + * type here so the dispatch only ever sees a value mkfs.ocfs2 / + * VFS can produce. + */ + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + goto bail; + } + + /* + * id1.dev1.i_rdev is the device-number arm of the id1 union and + * is only meaningful for character and block device inodes. For + * any other regular user-visible file type the on-disk value + * must be zero. ocfs2_populate_inode() currently runs + * + * inode->i_rdev = huge_decode_dev(le64_to_cpu(fe->id1.dev1.i_rdev)); + * + * unconditionally, before the S_IFMT switch decides whether the + * inode is a special file. As a result, an i_rdev value present + * on a non-device inode is silently published into the in-core + * inode; a subsequent forced re-read or in-core mode mutation + * (cluster peer with raw write access to the shared LUN, + * on-disk corruption, or a separately forged dinode) can then + * expose the attacker-controlled device number to + * init_special_inode() without ever showing an unusual i_mode + * at validation time. + * + * System inodes (OCFS2_SYSTEM_FL) legitimately use the bitmap1 + * and journal1 arms of the same union (allocator i_used / + * i_total counters and the journal ij_flags / + * ij_recovery_generation pair); those bytes are not an i_rdev + * and must not be checked here. Restrict the cross-check to + * non-system inodes, which is the full attacker-controllable + * surface. + */ + if (ocfs2_dinode_has_unexpected_rdev(di)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + goto bail; + } + + /* + * Non-inline directories must not have i_size without allocated + * clusters: directory growth adds storage before advancing i_size, + * and readdir walks i_size block-by-block. A forged directory + * with zero clusters and a huge i_size would repeatedly fault on + * holes while advancing through the claimed size. + * + * Non-inline regular files have the same invariant on non-sparse + * volumes. Sparse regular files are different: truncate can + * legitimately grow i_size without allocating clusters, so keep + * the sparse-alloc carveout for S_IFREG only. System inodes and + * inline-data dinodes have their own storage rules. + */ + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + rc = ocfs2_error(sb, + "Invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + rc = ocfs2_error(sb, + "Invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + goto bail; + } + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { struct ocfs2_inline_data *data = &di->id2.i_data; @@ -1525,6 +1639,29 @@ int ocfs2_validate_inode_block(struct super_block *sb, } } + if (S_ISLNK(le16_to_cpu(di->i_mode)) && + !le32_to_cpu(di->i_clusters)) { + int max_inline = ocfs2_fast_symlink_chars(sb); + u64 i_size = le64_to_cpu(di->i_size); + + if (i_size >= max_inline) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: fast symlink i_size %llu exceeds max %d\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)i_size, + max_inline - 1); + goto bail; + } + + if (strnlen((char *)di->id2.i_symlink, i_size + 1) != i_size) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: fast symlink is not NUL-terminated at i_size %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)i_size); + goto bail; + } + } + if (le32_to_cpu(di->i_flags) & OCFS2_CHAIN_FL) { struct ocfs2_chain_list *cl = &di->id2.i_chain; u16 bpc = 1 << (OCFS2_SB(sb)->s_clustersize_bits - @@ -1559,6 +1696,38 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + if (ocfs2_dinode_has_extents(di)) { + struct ocfs2_extent_list *el = &di->id2.i_list; + u16 count = le16_to_cpu(el->l_count); + u16 next_free = le16_to_cpu(el->l_next_free_rec); + + if (count == 0) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_count is zero\n", + (unsigned long long)bh->b_blocknr); + goto bail; + } + /* + * The exact capacity depends on i_xattr_inline_size, another + * unvalidated on-disk field. Inline xattrs only shrink the + * list, so the no-xattr maximum is a safe upper bound that a + * valid l_count never exceeds. + */ + if (count > ocfs2_extent_recs_per_inode(sb)) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_count %u exceeds max %u\n", + (unsigned long long)bh->b_blocknr, count, + ocfs2_extent_recs_per_inode(sb)); + goto bail; + } + if (next_free > count) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_next_free_rec %u exceeds l_count %u\n", + (unsigned long long)bh->b_blocknr, next_free, count); + goto bail; + } + } + rc = 0; bail: @@ -1624,6 +1793,40 @@ static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, (unsigned long long)bh->b_blocknr, le32_to_cpu(di->i_fs_generation)); rc = -OCFS2_FILECHECK_ERR_GENERATION; + goto bail; + } + + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_unexpected_rdev(di)) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; } bail: @@ -1812,4 +2015,3 @@ const struct ocfs2_caching_operations ocfs2_inode_caching_ops = { .co_io_lock = ocfs2_inode_cache_io_lock, .co_io_unlock = ocfs2_inode_cache_io_unlock, }; - diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c index f9bf3bac085d..d8afbc1a76bb 100644 --- a/fs/ocfs2/journal.c +++ b/fs/ocfs2/journal.c @@ -473,8 +473,12 @@ bail: */ int ocfs2_assure_trans_credits(handle_t *handle, int nblocks) { - int old_nblks = jbd2_handle_buffer_credits(handle); + int old_nblks; + if (is_handle_aborted(handle)) + return -EROFS; + + old_nblks = jbd2_handle_buffer_credits(handle); trace_ocfs2_assure_trans_credits(old_nblks); if (old_nblks >= nblocks) return 0; @@ -1022,11 +1026,8 @@ static int ocfs2_journal_toggle_dirty(struct ocfs2_super *osb, struct ocfs2_dinode *fe; fe = (struct ocfs2_dinode *)bh->b_data; - - /* The journal bh on the osb always comes from ocfs2_journal_init() - * and was validated there inside ocfs2_inode_lock_full(). It's a - * code bug if we mess it up. */ - BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + if (WARN_ON(!OCFS2_IS_VALID_DINODE(fe))) + return -EIO; flags = le32_to_cpu(fe->id1.journal1.ij_flags); if (dirty) diff --git a/fs/ocfs2/journal.h b/fs/ocfs2/journal.h index 6397170f302f..f8b3b2a3d630 100644 --- a/fs/ocfs2/journal.h +++ b/fs/ocfs2/journal.h @@ -196,6 +196,9 @@ static inline void ocfs2_checkpoint_inode(struct inode *inode) if (ocfs2_mount_local(osb)) return; + if (!osb->journal) + return; + if (!ocfs2_ci_fully_checkpointed(INODE_CACHE(inode))) { /* WARNING: This only kicks off a single * checkpoint. If someone races you and adds more diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c index c53de4439d93..ad1678ee7cc4 100644 --- a/fs/ocfs2/move_extents.c +++ b/fs/ocfs2/move_extents.c @@ -534,6 +534,8 @@ static void ocfs2_probe_alloc_group(struct inode *inode, struct buffer_head *bh, u32 base_cpos = ocfs2_blocks_to_clusters(inode->i_sb, le64_to_cpu(gd->bg_blkno)); + *phys_cpos = 0; + for (i = base_bit; i < le16_to_cpu(gd->bg_bits); i++) { used = ocfs2_test_bit(i, (unsigned long *)gd->bg_bitmap); @@ -555,7 +557,7 @@ static void ocfs2_probe_alloc_group(struct inode *inode, struct buffer_head *bh, last_free_bits++; if (last_free_bits == move_len) { - i -= move_len; + i = i - move_len + 1; *goal_bit = i; *phys_cpos = base_cpos + i; break; diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h index 7b50e03dfa66..62cad6522c7a 100644 --- a/fs/ocfs2/ocfs2.h +++ b/fs/ocfs2/ocfs2.h @@ -494,8 +494,6 @@ struct ocfs2_super struct rb_root osb_rf_lock_tree; struct ocfs2_refcount_tree *osb_ref_tree_lru; - struct mutex system_file_mutex; - /* * OCFS2 needs to schedule several different types of work which * require cluster locking, disk I/O, recovery waits, etc. Since these diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c index 12cbb4fccda0..f55810c59b1b 100644 --- a/fs/ocfs2/quota_local.c +++ b/fs/ocfs2/quota_local.c @@ -302,7 +302,7 @@ static int ocfs2_add_recovery_chunk(struct super_block *sb, if (!rc) return -ENOMEM; rc->rc_chunk = chunk; - rc->rc_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); + rc->rc_bitmap = kzalloc(sb->s_blocksize, GFP_NOFS); if (!rc->rc_bitmap) { kfree(rc); return -ENOMEM; diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 8eee5be4d1ed..7323bde70caa 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -2131,10 +2131,15 @@ static int ocfs2_remove_refcount_extent(handle_t *handle, rb->rf_flags = 0; rb->rf_parent = 0; rb->rf_cpos = 0; - memset(&rb->rf_records, 0, sb->s_blocksize - - offsetof(struct ocfs2_refcount_block, rf_records)); + rb->rf_records.rl_used = 0; + rb->rf_records.rl_reserved2 = 0; + rb->rf_records.rl_reserved1 = 0; + /* rl_count determines the memset size and fortify object size. */ rb->rf_records.rl_count = cpu_to_le16(ocfs2_refcount_recs_per_rb(sb)); + memset(rb->rf_records.rl_recs, 0, + le16_to_cpu(rb->rf_records.rl_count) * + sizeof(*rb->rf_records.rl_recs)); } ocfs2_journal_dirty(handle, ref_root_bh); diff --git a/fs/ocfs2/stack_user.c b/fs/ocfs2/stack_user.c index 5803f1dee679..91e19d33847c 100644 --- a/fs/ocfs2/stack_user.c +++ b/fs/ocfs2/stack_user.c @@ -327,18 +327,14 @@ static int ocfs2_control_install_private(struct file *file) ocfs2_control_this_node = p->op_this_node; running_proto.pv_major = p->op_proto.pv_major; running_proto.pv_minor = p->op_proto.pv_minor; - } - -out_unlock: - mutex_unlock(&ocfs2_control_lock); - - if (!rc && set_p) { - /* We set the global values successfully */ atomic_inc(&ocfs2_control_opened); ocfs2_control_set_handshake_state(file, OCFS2_CONTROL_HANDSHAKE_VALID); } +out_unlock: + mutex_unlock(&ocfs2_control_lock); + return rc; } diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index d284e0e37252..a4a2b87a45fe 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -231,8 +231,16 @@ static int ocfs2_validate_gd_parent(struct super_block *sb, int resize) { unsigned int max_bits; + unsigned int max_bitmap_bits; + unsigned int max_bitmap_size; + int suballocator; struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data; + suballocator = le64_to_cpu(di->i_blkno) != OCFS2_SB(sb)->bitmap_blkno; + max_bitmap_size = ocfs2_group_bitmap_size(sb, suballocator, + OCFS2_SB(sb)->s_feature_incompat); + max_bitmap_bits = max_bitmap_size * 8; + if (di->i_blkno != gd->bg_parent_dinode) { do_error("Group descriptor #%llu has bad parent pointer (%llu, expected %llu)\n", (unsigned long long)bh->b_blocknr, @@ -240,6 +248,20 @@ static int ocfs2_validate_gd_parent(struct super_block *sb, (unsigned long long)le64_to_cpu(di->i_blkno)); } + if (le16_to_cpu(gd->bg_size) > max_bitmap_size) { + do_error("Group descriptor #%llu has bitmap size %u but physical max of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_size), + max_bitmap_size); + } + + if (le16_to_cpu(gd->bg_bits) > max_bitmap_bits) { + do_error("Group descriptor #%llu has bit count %u but physical max of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_bits), + max_bitmap_bits); + } + max_bits = le16_to_cpu(di->id2.i_chain.cl_cpg) * le16_to_cpu(di->id2.i_chain.cl_bpc); if (le16_to_cpu(gd->bg_bits) > max_bits) { do_error("Group descriptor #%llu has bit count of %u\n", diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index b875f01c9756..bcac614ff02a 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1224,7 +1224,6 @@ static struct file_system_type ocfs2_fs_type = { .name = "ocfs2", .kill_sb = kill_block_super, .fs_flags = FS_REQUIRES_DEV|FS_RENAME_DOES_D_MOVE, - .next = NULL, .init_fs_context = ocfs2_init_fs_context, .parameters = ocfs2_param_spec, }; @@ -1997,8 +1996,6 @@ static int ocfs2_initialize_super(struct super_block *sb, spin_lock_init(&osb->osb_xattr_lock); ocfs2_init_steal_slots(osb); - mutex_init(&osb->system_file_mutex); - atomic_set(&osb->alloc_stats.moves, 0); atomic_set(&osb->alloc_stats.local_data, 0); atomic_set(&osb->alloc_stats.bitmap_data, 0); diff --git a/fs/ocfs2/sysfile.c b/fs/ocfs2/sysfile.c index d53a6cc866be..67e492f4b828 100644 --- a/fs/ocfs2/sysfile.c +++ b/fs/ocfs2/sysfile.c @@ -98,11 +98,9 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb, } else arr = get_local_system_inode(osb, type, slot); - mutex_lock(&osb->system_file_mutex); if (arr && ((inode = *arr) != NULL)) { /* get a ref in addition to the array ref */ inode = igrab(inode); - mutex_unlock(&osb->system_file_mutex); BUG_ON(!inode); return inode; @@ -112,11 +110,10 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb, inode = _ocfs2_get_system_file_inode(osb, type, slot); /* add one more if putting into array for first time */ - if (arr && inode) { - *arr = igrab(inode); - BUG_ON(!*arr); + if (inode && arr && !*arr && !cmpxchg(&(*arr), NULL, inode)) { + inode = igrab(inode); + BUG_ON(!inode); } - mutex_unlock(&osb->system_file_mutex); return inode; } diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 86cfd4c2adf9..fcddd3c13acd 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -950,15 +950,51 @@ static int ocfs2_xattr_list_entries(struct inode *inode, return result; } +static int ocfs2_xattr_ibody_lookup_header(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header) +{ + u16 xattr_count; + size_t max_entries; + u16 inline_size = le16_to_cpu(di->i_xattr_inline_size); + + if (inline_size > inode->i_sb->s_blocksize || + inline_size < sizeof(struct ocfs2_xattr_header)) { + ocfs2_error(inode->i_sb, + "Invalid xattr inline size %u in inode %llu\n", + inline_size, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + return -EFSCORRUPTED; + } + + *header = (struct ocfs2_xattr_header *) + ((void *)di + inode->i_sb->s_blocksize - inline_size); + + xattr_count = le16_to_cpu((*header)->xh_count); + max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) { + ocfs2_error(inode->i_sb, + "xattr entry count %u exceeds maximum %zu in inode %llu\n", + xattr_count, max_entries, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + return -EFSCORRUPTED; + } + + return 0; +} + int ocfs2_has_inline_xattr_value_outside(struct inode *inode, struct ocfs2_dinode *di) { struct ocfs2_xattr_header *xh; + int ret; int i; - xh = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - - le16_to_cpu(di->i_xattr_inline_size)); + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xh); + if (ret) + return 1; for (i = 0; i < le16_to_cpu(xh->xh_count); i++) if (!ocfs2_xattr_is_local(&xh->xh_entries[i])) @@ -975,39 +1011,13 @@ static int ocfs2_xattr_ibody_list(struct inode *inode, struct ocfs2_xattr_header *header = NULL; struct ocfs2_inode_info *oi = OCFS2_I(inode); int ret = 0; - u16 xattr_count; - size_t max_entries; - u16 inline_size; if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) return ret; - inline_size = le16_to_cpu(di->i_xattr_inline_size); - - /* Validate inline size is reasonable */ - if (inline_size > inode->i_sb->s_blocksize || - inline_size < sizeof(struct ocfs2_xattr_header)) { - ocfs2_error(inode->i_sb, - "Invalid xattr inline size %u in inode %llu\n", - inline_size, - (unsigned long long)OCFS2_I(inode)->ip_blkno); - return -EFSCORRUPTED; - } - - header = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - inline_size); - - xattr_count = le16_to_cpu(header->xh_count); - max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) / - sizeof(struct ocfs2_xattr_entry); - - if (xattr_count > max_entries) { - ocfs2_error(inode->i_sb, - "xattr entry count %u exceeds maximum %zu in inode %llu\n", - xattr_count, max_entries, - (unsigned long long)OCFS2_I(inode)->ip_blkno); - return -EFSCORRUPTED; - } + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; ret = ocfs2_xattr_list_entries(inode, header, buffer, buffer_size); @@ -1200,8 +1210,9 @@ static int ocfs2_xattr_ibody_get(struct inode *inode, return -ENODATA; xs->end = (void *)di + inode->i_sb->s_blocksize; - xs->header = (struct ocfs2_xattr_header *) - (xs->end - le16_to_cpu(di->i_xattr_inline_size)); + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header); + if (ret) + return ret; xs->base = (void *)xs->header; xs->here = xs->header->xh_entries; @@ -2465,9 +2476,9 @@ static int ocfs2_xattr_ibody_remove(struct inode *inode, .vb_access = ocfs2_journal_access_di, }; - header = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - - le16_to_cpu(di->i_xattr_inline_size)); + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; ret = ocfs2_remove_value_outside(inode, &vb, header, ref_ci, ref_root_bh); @@ -2726,12 +2737,14 @@ static int ocfs2_xattr_ibody_find(struct inode *inode, xs->xattr_bh = xs->inode_bh; xs->end = (void *)di + inode->i_sb->s_blocksize; - if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) - xs->header = (struct ocfs2_xattr_header *) - (xs->end - le16_to_cpu(di->i_xattr_inline_size)); - else + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) { + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header); + if (ret) + return ret; + } else { xs->header = (struct ocfs2_xattr_header *) (xs->end - OCFS2_SB(inode->i_sb)->s_xattr_inline_size); + } xs->base = (void *)xs->header; xs->here = xs->header->xh_entries; @@ -6003,14 +6016,17 @@ static int ocfs2_xattr_inline_attach_refcount(struct inode *inode, struct ocfs2_cached_dealloc_ctxt *dealloc) { struct ocfs2_dinode *di = (struct ocfs2_dinode *)fe_bh->b_data; - struct ocfs2_xattr_header *header = (struct ocfs2_xattr_header *) - (fe_bh->b_data + inode->i_sb->s_blocksize - - le16_to_cpu(di->i_xattr_inline_size)); + struct ocfs2_xattr_header *header; + int ret; struct ocfs2_xattr_value_buf vb = { .vb_bh = fe_bh, .vb_access = ocfs2_journal_access_di, }; + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; + return ocfs2_xattr_attach_refcount_normal(inode, &vb, header, ref_ci, ref_root_bh, dealloc); } @@ -6495,12 +6511,10 @@ static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args) handle_t *handle; struct ocfs2_super *osb = OCFS2_SB(args->old_inode->i_sb); struct ocfs2_dinode *di = (struct ocfs2_dinode *)args->old_bh->b_data; - int inline_size = le16_to_cpu(di->i_xattr_inline_size); - int header_off = osb->sb->s_blocksize - inline_size; - struct ocfs2_xattr_header *xh = (struct ocfs2_xattr_header *) - (args->old_bh->b_data + header_off); - struct ocfs2_xattr_header *new_xh = (struct ocfs2_xattr_header *) - (args->new_bh->b_data + header_off); + int inline_size; + int header_off; + struct ocfs2_xattr_header *xh; + struct ocfs2_xattr_header *new_xh; struct ocfs2_alloc_context *meta_ac = NULL; struct ocfs2_inode_info *new_oi; struct ocfs2_dinode *new_di; @@ -6509,6 +6523,15 @@ static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args) .vb_access = ocfs2_journal_access_di, }; + ret = ocfs2_xattr_ibody_lookup_header(args->old_inode, di, &xh); + if (ret) + goto out; + + inline_size = le16_to_cpu(di->i_xattr_inline_size); + header_off = osb->sb->s_blocksize - inline_size; + new_xh = (struct ocfs2_xattr_header *) + (args->new_bh->b_data + header_off); + ret = ocfs2_reflink_lock_xattr_allocators(osb, xh, args->ref_root_bh, &credits, &meta_ac); if (ret) { diff --git a/fs/omfs/inode.c b/fs/omfs/inode.c index 834cae1e6223..1d915ef72119 100644 --- a/fs/omfs/inode.c +++ b/fs/omfs/inode.c @@ -478,7 +478,8 @@ static int omfs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_time_min = 0; sb->s_time_max = U64_MAX / MSEC_PER_SEC; - sb_set_blocksize(sb, 0x200); + if (!sb_set_blocksize(sb, 0x200)) + goto end; bh = sb_bread(sb, 0); if (!bh) @@ -530,7 +531,8 @@ static int omfs_fill_super(struct super_block *sb, struct fs_context *fc) * Use sys_blocksize as the fs block since it is smaller than a * page while the fs blocksize can be larger. */ - sb_set_blocksize(sb, sbi->s_sys_blocksize); + if (!sb_set_blocksize(sb, sbi->s_sys_blocksize)) + goto out_brelse_bh; /* * ...and the difference goes into a shift. sys_blocksize is always diff --git a/fs/open.c b/fs/open.c index 681d405bc61e..408925d7bd0b 100644 --- a/fs/open.c +++ b/fs/open.c @@ -960,7 +960,7 @@ static int do_dentry_open(struct file *f, if (f->f_mapping->a_ops && f->f_mapping->a_ops->direct_IO) f->f_mode |= FMODE_CAN_ODIRECT; - f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC); + f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC | __O_REGULAR); f->f_iocb_flags = iocb_flags(f); file_ra_state_init(&f->f_ra, f->f_mapping->host->i_mapping); @@ -968,33 +968,6 @@ static int do_dentry_open(struct file *f, if ((f->f_flags & O_DIRECT) && !(f->f_mode & FMODE_CAN_ODIRECT)) return -EINVAL; - /* - * XXX: Huge page cache doesn't support writing yet. Drop all page - * cache for this file before processing writes. - */ - if (f->f_mode & FMODE_WRITE) { - /* - * Depends on full fence from get_write_access() to synchronize - * against collapse_file() regarding i_writecount and nr_thps - * updates. Ensures subsequent insertion of THPs into the page - * cache will fail. - */ - if (filemap_nr_thps(inode->i_mapping)) { - struct address_space *mapping = inode->i_mapping; - - filemap_invalidate_lock(inode->i_mapping); - /* - * unmap_mapping_range just need to be called once - * here, because the private pages is not need to be - * unmapped mapping (e.g. data segment of dynamic - * shared libraries here). - */ - unmap_mapping_range(mapping, 0, 0, 0); - truncate_inode_pages(mapping, 0); - filemap_invalidate_unlock(inode->i_mapping); - } - } - return 0; cleanup_all: @@ -1158,7 +1131,7 @@ struct file *kernel_file_open(const struct path *path, int flags, EXPORT_SYMBOL_GPL(kernel_file_open); #define WILL_CREATE(flags) (flags & (O_CREAT | __O_TMPFILE)) -#define O_PATH_FLAGS (O_DIRECTORY | O_NOFOLLOW | O_PATH | O_CLOEXEC) +#define O_PATH_FLAGS (O_DIRECTORY | O_NOFOLLOW | O_PATH | O_CLOEXEC | O_EMPTYPATH) inline struct open_how build_open_how(int flags, umode_t mode) { @@ -1184,7 +1157,15 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op) int acc_mode = ACC_MODE(flags); BUILD_BUG_ON_MSG(upper_32_bits(VALID_OPEN_FLAGS), - "struct open_flags doesn't yet handle flags > 32 bits"); + "VALID_OPEN_FLAGS must fit in 32 bits"); + /* The whole point: OPENAT2_REGULAR must be unrepresentable in int. */ + BUILD_BUG_ON_MSG(!upper_32_bits(OPENAT2_REGULAR), + "OPENAT2_REGULAR must live in the upper 32 bits of open_how::flags"); + /* Prevent a future bit collision between UAPI and internal carrier. */ + BUILD_BUG_ON_MSG(OPENAT2_REGULAR & VALID_OPEN_FLAGS, + "OPENAT2_REGULAR must not alias any open()/openat() flag"); + BUILD_BUG_ON_MSG(__O_REGULAR & VALID_OPENAT2_FLAGS, + "__O_REGULAR must not alias any user-visible flag"); /* * Strip flags that aren't relevant in determining struct open_flags. @@ -1196,7 +1177,7 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op) * values before calling build_open_flags(), but openat2(2) checks all * of its arguments. */ - if (flags & ~VALID_OPEN_FLAGS) + if (flags & ~VALID_OPENAT2_FLAGS) return -EINVAL; if (how->resolve & ~VALID_RESOLVE_FLAGS) return -EINVAL; @@ -1236,6 +1217,14 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op) if (!(acc_mode & MAY_WRITE)) return -EINVAL; } + /* + * Asking to open a directory and a regular file at the same time is + * contradictory. + */ + if ((flags & (O_DIRECTORY | OPENAT2_REGULAR)) == + (O_DIRECTORY | OPENAT2_REGULAR)) + return -EINVAL; + if (flags & O_PATH) { /* O_PATH only permits certain other flags to be set. */ if (flags & ~O_PATH_FLAGS) @@ -1252,6 +1241,19 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op) if (flags & __O_SYNC) flags |= O_DSYNC; + /* + * Translate the upper-32-bit UAPI bit OPENAT2_REGULAR into the + * kernel-internal lower-32-bit __O_REGULAR carrier so the bit + * survives the assignment to op->open_flag (an int) below and the + * subsequent flow through f->f_flags (unsigned int) and the + * i_op->atomic_open() callback (unsigned). do_dentry_open() strips + * __O_REGULAR before the file becomes visible to userspace. + */ + if (flags & OPENAT2_REGULAR) { + flags &= ~OPENAT2_REGULAR; + flags |= __O_REGULAR; + } + op->open_flag = flags; /* O_TRUNC implies we need access checks for write permissions */ @@ -1279,6 +1281,8 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op) lookup_flags |= LOOKUP_DIRECTORY; if (!(flags & O_NOFOLLOW)) lookup_flags |= LOOKUP_FOLLOW; + if (flags & O_EMPTYPATH) + lookup_flags |= LOOKUP_EMPTY; if (how->resolve & RESOLVE_NO_XDEV) lookup_flags |= LOOKUP_NO_XDEV; @@ -1360,7 +1364,7 @@ static int do_sys_openat2(int dfd, const char __user *filename, if (unlikely(err)) return err; - CLASS(filename, name)(filename); + CLASS(filename_flags, name)(filename, op.lookup_flags); return FD_ADD(how->flags, do_file_open(dfd, name, &op)); } diff --git a/fs/orangefs/dir.c b/fs/orangefs/dir.c index 6e2ebc8b9867..115b2c2f5269 100644 --- a/fs/orangefs/dir.c +++ b/fs/orangefs/dir.c @@ -191,7 +191,8 @@ static int fill_from_part(struct orangefs_dir_part *part, { const int offset = sizeof(struct orangefs_readdir_response_s); struct orangefs_khandle *khandle; - __u32 *len, padlen; + __u32 *len; + u64 padlen; loff_t i; char *s; i = ctx->pos & ~PART_MASK; @@ -215,8 +216,8 @@ static int fill_from_part(struct orangefs_dir_part *part, * len is the size of the string itself. padlen is the * total size of the encoded string. */ - padlen = (sizeof *len + *len + 1) + - (8 - (sizeof *len + *len + 1)%8)%8; + padlen = (u64)sizeof *len + *len + 1; + padlen += (8 - padlen % 8) % 8; if (part->len < i + padlen + sizeof *khandle) goto next; s = (void *)part + offset + i + sizeof *len; diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c index bec5475de094..75e65e72c2d6 100644 --- a/fs/orangefs/namei.c +++ b/fs/orangefs/namei.c @@ -362,7 +362,7 @@ static struct dentry *orangefs_mkdir(struct mnt_idmap *idmap, struct inode *dir, __orangefs_setattr(dir, &iattr); out: op_release(new_op); - return ERR_PTR(ret); + return ret ? ERR_PTR(ret) : NULL; } static int orangefs_rename(struct mnt_idmap *idmap, diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 13cb60b52bd6..e963701b4c87 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -853,7 +853,7 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) { struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); struct inode *udir = d_inode(c->destdir); - struct dentry *temp, *upper; + struct dentry *temp, *upper, *newdentry = NULL; struct file *tmpfile; int err; @@ -889,6 +889,14 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) err = PTR_ERR(upper); if (!IS_ERR(upper)) { err = ovl_do_link(ofs, temp, udir, upper); + if (!err) { + /* + * Record the linked dentry -- not the disconnected + * O_TMPFILE dentry -- so that ->d_revalidate() on + * the upper fs sees the real parent/name. + */ + newdentry = dget(upper); + } end_creating(upper); } @@ -903,7 +911,7 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) if (!c->metacopy) ovl_set_upperdata(d_inode(c->dentry)); - ovl_inode_update(d_inode(c->dentry), dget(temp)); + ovl_inode_update(d_inode(c->dentry), newdentry); out: ovl_end_write(c->dentry); diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 00c69707bda9..bc71231cad53 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -783,8 +783,8 @@ static const struct address_space_operations ovl_aops = { * * This chain is valid: * - inode->i_rwsem (inode_lock[2]) - * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) * - OVL_I(inode)->lock (ovl_inode_lock[2]) + * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) * - OVL_I(lowerinode)->lock (ovl_inode_lock[1]) * * And this chain is valid: @@ -797,8 +797,8 @@ static const struct address_space_operations ovl_aops = { * held, because it is in reverse order of the non-nested case using the same * upper fs: * - inode->i_rwsem (inode_lock[1]) - * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) * - OVL_I(inode)->lock (ovl_inode_lock[1]) + * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) */ #define OVL_MAX_NESTING FILESYSTEM_MAX_STACK_DEPTH diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 1dcc75b3a90f..e7fe29cb6028 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -838,15 +838,14 @@ static int ovl_iterate_merged(struct file *file, struct dir_context *ctx) struct ovl_dir_file *od = file->private_data; struct dentry *dentry = file->f_path.dentry; struct ovl_cache_entry *p; - int err = 0; + int err; if (!od->cache) { struct ovl_dir_cache *cache; cache = ovl_cache_get(dentry); - err = PTR_ERR(cache); if (IS_ERR(cache)) - return err; + return PTR_ERR(cache); od->cache = cache; ovl_seek_cursor(od, ctx->pos); @@ -869,7 +868,7 @@ static int ovl_iterate_merged(struct file *file, struct dir_context *ctx) od->cursor = p->l_node.next; ctx->pos++; } - return err; + return 0; } static bool ovl_need_adjust_d_ino(struct file *file) diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 7b86a6bac644..b41f4788e4f0 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -1354,7 +1354,7 @@ int ovl_ensure_verity_loaded(const struct path *datapath) struct inode *inode = d_inode(datapath->dentry); struct file *filp; - if (!fsverity_active(inode) && IS_VERITY(inode)) { + if (IS_VERITY(inode) && fsverity_get_info(inode) == NULL) { /* * If this inode was not yet opened, the verity info hasn't been * loaded yet, so we need to do that here to force it into memory. diff --git a/fs/pidfs.c b/fs/pidfs.c index 1cce4f34a051..aaa609ddab04 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -37,6 +37,8 @@ static struct kmem_cache *pidfs_attr_cachep __ro_after_init; static struct path pidfs_root_path = {}; +static struct simple_xattr_cache pidfs_xa_cache; + void pidfs_get_root(struct path *path) { *path = pidfs_root_path; @@ -96,7 +98,7 @@ static const struct rhashtable_params pidfs_ino_ht_params = { * use file handles. */ struct pidfs_attr { - struct simple_xattrs *xattrs; + struct list_head xattrs; union { struct pidfs_anon_attr; struct llist_node pidfs_llist; @@ -196,12 +198,7 @@ static void pidfs_free_attr_work(struct work_struct *work) head = llist_del_all(&pidfs_free_list); llist_for_each_entry_safe(attr, next, head, pidfs_llist) { - struct simple_xattrs *xattrs = attr->xattrs; - - if (xattrs) { - simple_xattrs_free(xattrs, NULL); - kfree(xattrs); - } + simple_xattrs_free(&pidfs_xa_cache, &attr->xattrs, NULL); kfree(attr); } } @@ -229,7 +226,7 @@ void pidfs_free_pid(struct pid *pid) if (IS_ERR(attr)) return; - if (likely(!attr->xattrs)) + if (likely(list_empty(&attr->xattrs))) kfree(attr); else if (llist_add(&attr->pidfs_llist, &pidfs_free_list)) schedule_work(&pidfs_free_work); @@ -338,14 +335,14 @@ static inline bool pid_in_current_pidns(const struct pid *pid) return false; } -static __u32 pidfs_coredump_mask(unsigned long mm_flags) +static __u32 pidfs_coredump_mask(enum task_dumpable dumpable) { - switch (__get_dumpable(mm_flags)) { - case SUID_DUMP_USER: + switch (dumpable) { + case TASK_DUMPABLE_OWNER: return PIDFD_COREDUMP_USER; - case SUID_DUMP_ROOT: + case TASK_DUMPABLE_ROOT: return PIDFD_COREDUMP_ROOT; - case SUID_DUMP_DISABLE: + case TASK_DUMPABLE_OFF: return PIDFD_COREDUMP_SKIP; default: WARN_ON_ONCE(true); @@ -433,14 +430,9 @@ static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg) return -ESRCH; if ((mask & PIDFD_INFO_COREDUMP) && !kinfo.coredump_mask) { - guard(task_lock)(task); - if (task->mm) { - unsigned long flags = __mm_flags_get_dumpable(task->mm); - - kinfo.coredump_mask = pidfs_coredump_mask(flags); - kinfo.mask |= PIDFD_INFO_COREDUMP; - /* No coredump actually took place, so no coredump signal. */ - } + kinfo.coredump_mask = pidfs_coredump_mask(task_exec_state_get_dumpable(task)); + kinfo.mask |= PIDFD_INFO_COREDUMP; + /* No coredump actually took place, so no coredump signal. */ } /* Unconditionally return identifiers and credentials, the rest only on request */ @@ -779,7 +771,7 @@ void pidfs_coredump(const struct coredump_params *cprm) VFS_WARN_ON_ONCE(attr == PIDFS_PID_DEAD); /* Note how we were coredumped and that we coredumped. */ - attr->coredump_mask = pidfs_coredump_mask(cprm->mm_flags) | + attr->coredump_mask = pidfs_coredump_mask(cprm->dumpable) | PIDFD_COREDUMPED; /* If coredumping is set to skip we should never end up here. */ VFS_WARN_ON_ONCE(attr->coredump_mask & PIDFD_COREDUMP_SKIP); @@ -815,14 +807,8 @@ static ssize_t pidfs_listxattr(struct dentry *dentry, char *buf, size_t size) { struct inode *inode = d_inode(dentry); struct pid *pid = inode->i_private; - struct pidfs_attr *attr = pid->attr; - struct simple_xattrs *xattrs; - xattrs = READ_ONCE(attr->xattrs); - if (!xattrs) - return 0; - - return simple_xattr_list(inode, xattrs, buf, size); + return simple_xattr_list(inode, &pid->attr->xattrs, buf, size); } static const struct inode_operations pidfs_inode_operations = { @@ -991,14 +977,16 @@ static void pidfs_put_data(void *data) } /** - * pidfs_register_pid - register a struct pid in pidfs + * pidfs_register_pid_gfp - register a struct pid in pidfs with custom GFP + * flags * @pid: pid to pin + * @gfp: GFP flags for memory allocation * - * Register a struct pid in pidfs. + * Register a struct pid in pidfs with custom GFP flags. * * Return: On success zero, on error a negative error code is returned. */ -int pidfs_register_pid(struct pid *pid) +int pidfs_register_pid_gfp(struct pid *pid, gfp_t gfp) { struct pidfs_attr *new_attr __free(kfree) = NULL; struct pidfs_attr *attr; @@ -1014,10 +1002,12 @@ int pidfs_register_pid(struct pid *pid) if (attr) return 0; - new_attr = kmem_cache_zalloc(pidfs_attr_cachep, GFP_KERNEL); + new_attr = kmem_cache_zalloc(pidfs_attr_cachep, gfp); if (!new_attr) return -ENOMEM; + INIT_LIST_HEAD_RCU(&new_attr->xattrs); + /* Synchronize with pidfs_exit(). */ guard(spinlock_irq)(&pid->wait_pidfd.lock); @@ -1057,16 +1047,9 @@ static int pidfs_xattr_get(const struct xattr_handler *handler, const char *suffix, void *value, size_t size) { struct pid *pid = inode->i_private; - struct pidfs_attr *attr = pid->attr; - const char *name; - struct simple_xattrs *xattrs; + const char *name = xattr_full_name(handler, suffix); - xattrs = READ_ONCE(attr->xattrs); - if (!xattrs) - return -ENODATA; - - name = xattr_full_name(handler, suffix); - return simple_xattr_get(xattrs, name, value, size); + return simple_xattr_get(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size); } static int pidfs_xattr_set(const struct xattr_handler *handler, @@ -1075,20 +1058,13 @@ static int pidfs_xattr_set(const struct xattr_handler *handler, const void *value, size_t size, int flags) { struct pid *pid = inode->i_private; - struct pidfs_attr *attr = pid->attr; - const char *name; - struct simple_xattrs *xattrs; + const char *name = xattr_full_name(handler, suffix); struct simple_xattr *old_xattr; /* Ensure we're the only one to set @attr->xattrs. */ WARN_ON_ONCE(!inode_is_locked(inode)); - xattrs = simple_xattrs_lazy_alloc(&attr->xattrs, value, flags); - if (IS_ERR_OR_NULL(xattrs)) - return PTR_ERR(xattrs); - - name = xattr_full_name(handler, suffix); - old_xattr = simple_xattr_set(xattrs, name, value, size, flags); + old_xattr = simple_xattr_set(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size, flags); if (IS_ERR(old_xattr)) return PTR_ERR(old_xattr); @@ -1115,8 +1091,6 @@ static int pidfs_init_fs_context(struct fs_context *fc) if (!ctx) return -ENOMEM; - fc->s_iflags |= SB_I_NOEXEC; - fc->s_iflags |= SB_I_NODEV; ctx->s_d_flags |= DCACHE_DONTCACHE; ctx->ops = &pidfs_sops; ctx->eops = &pidfs_export_operations; diff --git a/fs/pipe.c b/fs/pipe.c index 9841648c9cf3..429b0714ec57 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -111,16 +111,76 @@ void pipe_double_lock(struct pipe_inode_info *pipe1, pipe_lock(pipe2); } -static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe) +#define PIPE_PREALLOC_MAX 8 + +struct anon_pipe_prealloc { + struct page *pages[PIPE_PREALLOC_MAX]; + unsigned int count; +}; + +/* + * Pre-allocate pages outside pipe->mutex for multi-page writes. + * alloc_page() with GFP_HIGHUSER can sleep in reclaim and runs memcg + * charging; doing it under the mutex stalls a concurrent reader. + * + * Loop alloc_page() instead of alloc_pages_bulk_*(): the bulk path refuses + * __GFP_ACCOUNT under memcg (see commit 8dcb3060d81d "memcg: page_alloc: + * skip bulk allocator for __GFP_ACCOUNT") and silently degrades to a single + * page. A per-page loop keeps memcg accounting and the task NUMA mempolicy + * honoured for every page; the per-call overhead is small compared to the + * pipe->mutex hold-time being shrunk. Any shortfall is covered by the + * in-lock alloc_page() fallback in anon_pipe_get_page(). + */ +static void anon_pipe_get_page_prealloc(struct anon_pipe_prealloc *prealloc, + size_t total_len) +{ + unsigned int want, i; + struct page *page; + + prealloc->count = 0; + if (total_len <= PAGE_SIZE) + return; + + want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE), + PIPE_PREALLOC_MAX); + + for (i = 0; i < want; i++) { + page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT); + if (!page) + break; + prealloc->pages[prealloc->count++] = page; + } +} + +static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc) +{ + if (!prealloc->count) + return NULL; + + prealloc->count--; + + return prealloc->pages[prealloc->count]; +} + +static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe, + struct anon_pipe_prealloc *prealloc) { + struct page *page; + + /* Drain prealloc first to keep tmp_page[] hot for later small writes. */ + page = anon_pipe_prealloc_pop(prealloc); + if (page) + return page; + for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { if (pipe->tmp_page[i]) { - struct page *page = pipe->tmp_page[i]; + page = pipe->tmp_page[i]; pipe->tmp_page[i] = NULL; return page; } } + /* FWIW: This is called with pipe->mutex held */ return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT); } @@ -139,6 +199,38 @@ static void anon_pipe_put_page(struct pipe_inode_info *pipe, put_page(page); } +/* + * Stash leftover prealloc pages in tmp_page[] so the next write to this + * pipe gets a hot page without entering the allocator. + */ +static void anon_pipe_refill_tmp_pages(struct pipe_inode_info *pipe, + struct anon_pipe_prealloc *prealloc) +{ + int i, idx; + + if (!prealloc->count) + return; + + for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { + if (pipe->tmp_page[i]) + continue; + if (!prealloc->count) + return; + idx = --prealloc->count; + pipe->tmp_page[i] = prealloc->pages[idx]; + prealloc->pages[idx] = NULL; + } +} + +/* Runs after mutex_unlock() to keep put_page() out of the critical section. */ +static void anon_pipe_free_pages(struct anon_pipe_prealloc *prealloc) +{ + while (prealloc->count) { + prealloc->count--; + put_page(prealloc->pages[prealloc->count]); + } +} + static void anon_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { @@ -432,6 +524,7 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from) { struct file *filp = iocb->ki_filp; struct pipe_inode_info *pipe = filp->private_data; + struct anon_pipe_prealloc prealloc; unsigned int head; ssize_t ret = 0; size_t total_len = iov_iter_count(from); @@ -455,6 +548,8 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from) if (unlikely(total_len == 0)) return 0; + anon_pipe_get_page_prealloc(&prealloc, total_len); + mutex_lock(&pipe->mutex); if (!pipe->readers) { @@ -512,7 +607,7 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from) struct page *page; int copied; - page = anon_pipe_get_page(pipe); + page = anon_pipe_get_page(pipe, &prealloc); if (unlikely(!page)) { if (!ret) ret = -ENOMEM; @@ -576,9 +671,11 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from) wake_next_writer = true; } out: + anon_pipe_refill_tmp_pages(pipe, &prealloc); if (pipe_is_full(pipe)) wake_next_writer = false; mutex_unlock(&pipe->mutex); + anon_pipe_free_pages(&prealloc); /* * If we do do a wakeup event, we do a 'sync' wakeup, because we @@ -664,7 +761,8 @@ pipe_poll(struct file *filp, poll_table *wait) union pipe_index idx; /* Epoll has some historical nasty semantics, this enables them */ - WRITE_ONCE(pipe->poll_usage, true); + if (unlikely(!READ_ONCE(pipe->poll_usage))) + WRITE_ONCE(pipe->poll_usage, true); /* * Reading pipe state only -- no need for acquiring the semaphore. diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 12591c95c925..b4bfe4ddf64e 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -1126,7 +1126,7 @@ retry_deleg: if (error) goto out_inode_unlock; - error = try_break_deleg(inode, &delegated_inode); + error = try_break_deleg(inode, 0, &delegated_inode); if (error) goto out_inode_unlock; @@ -1234,7 +1234,7 @@ retry_deleg: if (error) goto out_inode_unlock; - error = try_break_deleg(inode, &delegated_inode); + error = try_break_deleg(inode, 0, &delegated_inode); if (error) goto out_inode_unlock; diff --git a/fs/proc/Makefile b/fs/proc/Makefile index 7b4db9c56e6a..8bc615ff84e5 100644 --- a/fs/proc/Makefile +++ b/fs/proc/Makefile @@ -16,7 +16,9 @@ proc-y += cmdline.o proc-y += consoles.o proc-y += cpuinfo.o proc-y += devices.o -proc-y += interrupts.o +ifneq ($(CONFIG_GENERIC_IRQ_SHOW),y) +proc-y += interrupts.o +endif proc-y += loadavg.o proc-y += meminfo.o proc-y += stat.o diff --git a/fs/proc/array.c b/fs/proc/array.c index 90fb0c6b5f99..479ea8cb4ef4 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -482,6 +482,11 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, unsigned long flags; int exit_code = task->exit_code; struct signal_struct *sig = task->signal; + int ret; + + ret = down_read_killable(&task->signal->exec_update_lock); + if (ret) + return ret; state = *get_task_state(task); vsize = eip = esp = 0; @@ -657,6 +662,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, seq_puts(m, " 0"); seq_putc(m, '\n'); + up_read(&task->signal->exec_update_lock); if (mm) mmput(mm); return 0; diff --git a/fs/proc/base.c b/fs/proc/base.c index d9acfa89c894..780f81259052 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -91,6 +91,7 @@ #include <linux/sched/mm.h> #include <linux/sched/coredump.h> #include <linux/sched/debug.h> +#include <linux/sched/exec_state.h> #include <linux/sched/stat.h> #include <linux/posix-timers.h> #include <linux/time_namespace.h> @@ -218,33 +219,24 @@ static int get_task_root(struct task_struct *task, struct path *root) return result; } -static int proc_cwd_link(struct dentry *dentry, struct path *path) +static int proc_cwd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task = get_proc_task(d_inode(dentry)); int result = -ENOENT; - if (task) { - task_lock(task); - if (task->fs) { - get_fs_pwd(task->fs, path); - result = 0; - } - task_unlock(task); - put_task_struct(task); + task_lock(task); + if (task->fs) { + get_fs_pwd(task->fs, path); + result = 0; } + task_unlock(task); return result; } -static int proc_root_link(struct dentry *dentry, struct path *path) +static int proc_root_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task = get_proc_task(d_inode(dentry)); - int result = -ENOENT; - - if (task) { - result = get_task_root(task, path); - put_task_struct(task); - } - return result; + return get_task_root(task, path); } /* @@ -261,7 +253,7 @@ static ssize_t get_mm_proctitle(struct mm_struct *mm, char __user *buf, if (pos >= PAGE_SIZE) return 0; - page = (char *)__get_free_page(GFP_KERNEL); + page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page) return -ENOMEM; @@ -284,7 +276,7 @@ static ssize_t get_mm_proctitle(struct mm_struct *mm, char __user *buf, ret = len; } } - free_page((unsigned long)page); + kfree(page); return ret; } @@ -347,7 +339,7 @@ static ssize_t get_mm_cmdline(struct mm_struct *mm, char __user *buf, if (count > arg_end - pos) count = arg_end - pos; - page = (char *)__get_free_page(GFP_KERNEL); + page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page) return -ENOMEM; @@ -371,7 +363,7 @@ static ssize_t get_mm_cmdline(struct mm_struct *mm, char __user *buf, count -= got; } - free_page((unsigned long)page); + kfree(page); return len; } @@ -423,18 +415,24 @@ static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns, { unsigned long wchan; char symname[KSYM_NAME_LEN]; + int err; + err = down_read_killable(&task->signal->exec_update_lock); + if (err) + return err; if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) goto print0; wchan = get_wchan(task); if (wchan && !lookup_symbol_name(wchan, symname)) { seq_puts(m, symname); + up_read(&task->signal->exec_update_lock); return 0; } print0: seq_putc(m, '0'); + up_read(&task->signal->exec_update_lock); return 0; } #endif /* CONFIG_KALLSYMS */ @@ -704,23 +702,6 @@ static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns, /* Here the fs part begins */ /************************************************************************/ -/* permission checks */ -static bool proc_fd_access_allowed(struct inode *inode) -{ - struct task_struct *task; - bool allowed = false; - /* Allow access to a task's file descriptors if it is us or we - * may use ptrace attach to the process and find out that - * information. - */ - task = get_proc_task(inode); - if (task) { - allowed = ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS); - put_task_struct(task); - } - return allowed; -} - int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -908,7 +889,7 @@ static ssize_t mem_rw(struct file *file, char __user *buf, if (!mm) return 0; - page = (char *)__get_free_page(GFP_KERNEL); + page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page) return -ENOMEM; @@ -949,7 +930,7 @@ static ssize_t mem_rw(struct file *file, char __user *buf, mmput(mm); free: - free_page((unsigned long) page); + kfree(page); return copied; } @@ -1016,7 +997,7 @@ static ssize_t environ_read(struct file *file, char __user *buf, if (!mm || !mm->env_end) return 0; - page = (char *)__get_free_page(GFP_KERNEL); + page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page) return -ENOMEM; @@ -1062,7 +1043,7 @@ static ssize_t environ_read(struct file *file, char __user *buf, mmput(mm); free: - free_page((unsigned long) page); + kfree(page); return ret; } @@ -1777,16 +1758,12 @@ static const struct file_operations proc_pid_set_comm_operations = { .release = single_release, }; -static int proc_exe_link(struct dentry *dentry, struct path *exe_path) +static int proc_exe_link(struct dentry *dentry, struct path *exe_path, + struct task_struct *task) { - struct task_struct *task; struct file *exe_file; - task = get_proc_task(d_inode(dentry)); - if (!task) - return -ENOENT; exe_file = get_task_exe_file(task); - put_task_struct(task); if (exe_file) { *exe_path = exe_file->f_path; path_get(&exe_file->f_path); @@ -1796,26 +1773,42 @@ static int proc_exe_link(struct dentry *dentry, struct path *exe_path) return -ENOENT; } +static int call_proc_get_link(struct dentry *dentry, struct inode *inode, struct path *path_out) +{ + struct task_struct *task; + int ret; + + task = get_proc_task(inode); + if (!task) + return -ENOENT; + ret = down_read_killable(&task->signal->exec_update_lock); + if (ret) + goto out_put_task; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + ret = -EACCES; + goto out; + } + ret = PROC_I(inode)->op.proc_get_link(dentry, path_out, task); + +out: + up_read(&task->signal->exec_update_lock); +out_put_task: + put_task_struct(task); + return ret; +} + static const char *proc_pid_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) { struct path path; - int error = -EACCES; + int error; if (!dentry) return ERR_PTR(-ECHILD); - - /* Are we allowed to snoop on the tasks file descriptors? */ - if (!proc_fd_access_allowed(inode)) - goto out; - - error = PROC_I(inode)->op.proc_get_link(dentry, &path); - if (error) - goto out; - - error = nd_jump_link(&path); -out: + error = call_proc_get_link(dentry, inode, &path); + if (!error) + error = nd_jump_link(&path); return ERR_PTR(error); } @@ -1849,17 +1842,11 @@ static int proc_pid_readlink(struct dentry * dentry, char __user * buffer, int b struct inode *inode = d_inode(dentry); struct path path; - /* Are we allowed to snoop on the tasks file descriptors? */ - if (!proc_fd_access_allowed(inode)) - goto out; - - error = PROC_I(inode)->op.proc_get_link(dentry, &path); - if (error) - goto out; - - error = do_proc_readlink(&path, buffer, buflen); - path_put(&path); -out: + error = call_proc_get_link(dentry, inode, &path); + if (!error) { + error = do_proc_readlink(&path, buffer, buflen); + path_put(&path); + } return error; } @@ -1893,7 +1880,6 @@ void task_dump_owner(struct task_struct *task, umode_t mode, cred = __task_cred(task); uid = cred->euid; gid = cred->egid; - rcu_read_unlock(); /* * Before the /proc/pid/status file was created the only way to read @@ -1903,29 +1889,22 @@ void task_dump_owner(struct task_struct *task, umode_t mode, * made this apply to all per process world readable and executable * directories. */ - if (mode != (S_IFDIR|S_IRUGO|S_IXUGO)) { - struct mm_struct *mm; - task_lock(task); - mm = task->mm; - /* Make non-dumpable tasks owned by some root */ - if (mm) { - if (get_dumpable(mm) != SUID_DUMP_USER) { - struct user_namespace *user_ns = mm->user_ns; - - uid = make_kuid(user_ns, 0); - if (!uid_valid(uid)) - uid = GLOBAL_ROOT_UID; - - gid = make_kgid(user_ns, 0); - if (!gid_valid(gid)) - gid = GLOBAL_ROOT_GID; - } - } else { - uid = GLOBAL_ROOT_UID; - gid = GLOBAL_ROOT_GID; + if (mode != (S_IFDIR | S_IRUGO | S_IXUGO)) { + struct task_exec_state *exec_state; + + exec_state = task_exec_state_rcu(task); + if (READ_ONCE(exec_state->dumpable) != TASK_DUMPABLE_OWNER) { + uid = make_kuid(exec_state->user_ns, 0); + if (!uid_valid(uid)) + uid = GLOBAL_ROOT_UID; + + gid = make_kgid(exec_state->user_ns, 0); + if (!gid_valid(gid)) + gid = GLOBAL_ROOT_GID; } - task_unlock(task); } + rcu_read_unlock(); + *ruid = uid; *rgid = gid; } @@ -2132,8 +2111,7 @@ bool proc_fill_cache(struct file *file, struct dir_context *ctx, goto end_instantiate; if (!child) { - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); - child = d_alloc_parallel(dir, &qname, &wq); + child = d_alloc_parallel(dir, &qname); if (IS_ERR(child)) goto end_instantiate; if (d_in_lookup(child)) { @@ -2250,21 +2228,16 @@ static const struct dentry_operations tid_map_files_dentry_operations = { .d_delete = pid_delete_dentry, }; -static int map_files_get_link(struct dentry *dentry, struct path *path) +static int map_files_get_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { unsigned long vm_start, vm_end; struct vm_area_struct *vma; - struct task_struct *task; struct mm_struct *mm; int rc; rc = -ENOENT; - task = get_proc_task(d_inode(dentry)); - if (!task) - goto out; - mm = get_task_mm(task); - put_task_struct(task); if (!mm) goto out; @@ -2360,17 +2333,15 @@ static struct dentry *proc_map_files_lookup(struct inode *dir, if (!task) goto out; - result = ERR_PTR(-EACCES); - if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) - goto out_put_task; - result = ERR_PTR(-ENOENT); if (dname_to_vma_addr(dentry, &vm_start, &vm_end)) goto out_put_task; - mm = get_task_mm(task); - if (!mm) + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR(mm)) { + result = ERR_CAST(mm); goto out_put_task; + } result = ERR_PTR(-EINTR); if (mmap_read_lock_killable(mm)) @@ -2420,23 +2391,22 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) if (!task) goto out; - ret = -EACCES; - if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) - goto out_put_task; - ret = 0; if (!dir_emit_dots(file, ctx)) goto out_put_task; - mm = get_task_mm(task); - if (!mm) + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR(mm)) { + ret = PTR_ERR(mm); + /* if the task has no mm, the directory should just be empty */ + if (ret == -ESRCH) + ret = 0; goto out_put_task; + } ret = mmap_read_lock_killable(mm); - if (ret) { - mmput(mm); - goto out_put_task; - } + if (ret) + goto out_put_mm; nr_files = 0; @@ -2462,8 +2432,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) if (!p) { ret = -ENOMEM; mmap_read_unlock(mm); - mmput(mm); - goto out_put_task; + goto out_put_mm; } p->start = vma->vm_start; @@ -2471,7 +2440,6 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) p->mode = vma->vm_file->f_mode; } mmap_read_unlock(mm); - mmput(mm); for (i = 0; i < nr_files; i++) { char buf[4 * sizeof(long) + 2]; /* max: %lx-%lx\0 */ @@ -2488,6 +2456,8 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) ctx->pos++; } +out_put_mm: + mmput(mm); out_put_task: put_task_struct(task); out: @@ -2965,7 +2935,7 @@ static ssize_t proc_coredump_filter_read(struct file *file, char __user *buf, ret = 0; mm = get_task_mm(task); if (mm) { - unsigned long flags = __mm_flags_get_dumpable(mm); + unsigned long flags = __mm_flags_get_word(mm); len = snprintf(buffer, sizeof(buffer), "%08lx\n", ((flags & MMF_DUMP_FILTER_MASK) >> @@ -3543,28 +3513,42 @@ out: struct tgid_iter { unsigned int tgid; struct task_struct *task; + struct pid_namespace *const pid_ns; }; -static struct tgid_iter next_tgid(struct pid_namespace *ns, struct tgid_iter iter) + +static struct tgid_iter +make_tgid_iter(unsigned int init_tgid, struct pid_namespace *pid_ns) { - struct pid *pid; + return (struct tgid_iter){ + .tgid = init_tgid - 1, + .pid_ns = pid_ns, + }; +} + +static bool next_tgid(struct tgid_iter *it) +{ + if (it->task) { + put_task_struct(it->task); + it->task = NULL; + } - if (iter.task) - put_task_struct(iter.task); rcu_read_lock(); -retry: - iter.task = NULL; - pid = find_ge_pid(iter.tgid, ns); - if (pid) { - iter.tgid = pid_nr_ns(pid, ns); - iter.task = pid_task(pid, PIDTYPE_TGID); - if (!iter.task) { - iter.tgid += 1; - goto retry; + while (1) { + it->tgid += 1; + const auto pid = find_ge_pid(it->tgid, it->pid_ns); + if (pid) { + it->tgid = pid_nr_ns(pid, it->pid_ns); + it->task = pid_task(pid, PIDTYPE_TGID); + if (it->task) { + get_task_struct(it->task); + rcu_read_unlock(); + return true; + } + } else { + rcu_read_unlock(); + return false; } - get_task_struct(iter.task); } - rcu_read_unlock(); - return iter; } #define TGID_OFFSET (FIRST_PROCESS_ENTRY + 2) @@ -3572,9 +3556,8 @@ retry: /* for the /proc/ directory itself, after non-process stuff has been done */ int proc_pid_readdir(struct file *file, struct dir_context *ctx) { - struct tgid_iter iter; struct proc_fs_info *fs_info = proc_sb_info(file_inode(file)->i_sb); - struct pid_namespace *ns = proc_pid_ns(file_inode(file)->i_sb); + struct pid_namespace *pid_ns = proc_pid_ns(file_inode(file)->i_sb); loff_t pos = ctx->pos; if (pos >= PID_MAX_LIMIT + TGID_OFFSET) @@ -3590,11 +3573,9 @@ int proc_pid_readdir(struct file *file, struct dir_context *ctx) return 0; ctx->pos = pos = pos + 1; } - iter.tgid = pos - TGID_OFFSET; - iter.task = NULL; - for (iter = next_tgid(ns, iter); - iter.task; - iter.tgid += 1, iter = next_tgid(ns, iter)) { + + auto iter = make_tgid_iter(pos - TGID_OFFSET, pid_ns); + while (next_tgid(&iter)) { char name[10 + 1]; unsigned int len; diff --git a/fs/proc/fd.c b/fs/proc/fd.c index 05c7513e77c7..0f9a1556f2a3 100644 --- a/fs/proc/fd.c +++ b/fs/proc/fd.c @@ -171,24 +171,19 @@ static const struct dentry_operations tid_fd_dentry_operations = { .d_delete = pid_delete_dentry, }; -static int proc_fd_link(struct dentry *dentry, struct path *path) +static int proc_fd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task; int ret = -ENOENT; - - task = get_proc_task(d_inode(dentry)); - if (task) { - unsigned int fd = proc_fd(d_inode(dentry)); - struct file *fd_file; - - fd_file = fget_task(task, fd); - if (fd_file) { - *path = fd_file->f_path; - path_get(&fd_file->f_path); - ret = 0; - fput(fd_file); - } - put_task_struct(task); + unsigned int fd = proc_fd(d_inode(dentry)); + struct file *fd_file; + + fd_file = fget_task(task, fd); + if (fd_file) { + *path = fd_file->f_path; + path_get(&fd_file->f_path); + ret = 0; + fput(fd_file); } return ret; diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 8bb81e58c9d8..26086a283672 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -112,6 +112,8 @@ static bool pde_subdir_insert(struct proc_dir_entry *dir, /* Add new node and rebalance tree. */ rb_link_node(&de->subdir_node, parent, new); rb_insert_color(&de->subdir_node, root); + if (S_ISDIR(de->mode)) + dir->nlink++; return true; } @@ -404,7 +406,6 @@ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, write_unlock(&proc_subdir_lock); goto out_free_inum; } - dir->nlink++; write_unlock(&proc_subdir_lock); return dp; @@ -427,9 +428,13 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, if (xlate_proc_name(name, parent, &fn) != 0) goto out; qstr.name = fn; - qstr.len = strlen(fn); - if (qstr.len == 0 || qstr.len >= 256) { - WARN(1, "name len %u\n", qstr.len); + qstr.len = strnlen(fn, NAME_MAX + 1); + if (qstr.len == 0) { + WARN(1, "empty name\n"); + return NULL; + } + if (qstr.len > NAME_MAX) { + WARN(1, "name too long\n"); return NULL; } if (qstr.len == 1 && fn[0] == '.') { @@ -702,6 +707,8 @@ static void pde_erase(struct proc_dir_entry *pde, struct proc_dir_entry *parent) { rb_erase(&pde->subdir_node, &parent->subdir); RB_CLEAR_NODE(&pde->subdir_node); + if (S_ISDIR(pde->mode)) + parent->nlink--; } /* @@ -727,8 +734,6 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent) de = NULL; } else { pde_erase(de, parent); - if (S_ISDIR(de->mode)) - parent->nlink--; } } write_unlock(&proc_subdir_lock); @@ -787,8 +792,6 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent) continue; } next = de->parent; - if (S_ISDIR(de->mode)) - next->nlink--; write_unlock(&proc_subdir_lock); proc_entry_rundown(de); @@ -841,3 +844,13 @@ ssize_t proc_simple_write(struct file *f, const char __user *ubuf, size_t size, kfree(buf); return ret == 0 ? size : ret; } + +/* + * Not exported to modules: + * modules' /proc files aren't permanent because modules aren't permanent. + */ +void impl_proc_make_permanent(struct proc_dir_entry *pde) +{ + if (pde) + pde_make_permanent(pde); +} diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 64dc44832808..b232e1098117 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -79,8 +79,11 @@ static inline bool pde_is_permanent(const struct proc_dir_entry *pde) return pde->flags & PROC_ENTRY_PERMANENT; } +/* This is for builtin code, not even for modules which are compiled in. */ static inline void pde_make_permanent(struct proc_dir_entry *pde) { + /* Ensure magic flag does something. */ + static_assert(PROC_ENTRY_PERMANENT != 0); pde->flags |= PROC_ENTRY_PERMANENT; } @@ -107,7 +110,7 @@ extern struct kmem_cache *proc_dir_entry_cache; void pde_free(struct proc_dir_entry *pde); union proc_op { - int (*proc_get_link)(struct dentry *, struct path *); + int (*proc_get_link)(struct dentry *, struct path *, struct task_struct *); int (*proc_show)(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task); diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c index 39f4169f669f..2f46f1396744 100644 --- a/fs/proc/namespaces.c +++ b/fs/proc/namespaces.c @@ -55,6 +55,10 @@ static const char *proc_ns_get_link(struct dentry *dentry, if (!task) return ERR_PTR(-EACCES); + error = down_read_killable(&task->signal->exec_update_lock); + if (error) + goto out_put_task; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) goto out; @@ -64,6 +68,8 @@ static const char *proc_ns_get_link(struct dentry *dentry, error = nd_jump_link(&ns_path); out: + up_read(&task->signal->exec_update_lock); +out_put_task: put_task_struct(task); return ERR_PTR(error); } @@ -80,11 +86,17 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl if (!task) return res; + res = down_read_killable(&task->signal->exec_update_lock); + if (res) + goto out_put_task; + if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { res = ns_get_name(name, sizeof(name), task, ns_ops); if (res >= 0) res = readlink_copy(buffer, buflen, name, strlen(name)); } + up_read(&task->signal->exec_update_lock); +out_put_task: put_task_struct(task); return res; } diff --git a/fs/proc/page.c b/fs/proc/page.c index f9b2c2c906cd..7d9387143435 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -173,7 +173,7 @@ u64 stable_page_flags(const struct page *page) u |= 1 << KPF_MMAP; if (is_anon) { u |= 1 << KPF_ANON; - if (mapping & FOLIO_MAPPING_KSM) + if ((mapping & FOLIO_MAPPING_FLAGS) == FOLIO_MAPPING_KSM) u |= 1 << KPF_KSM; } diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c index 184cddeb8215..00cc385bce21 100644 --- a/fs/proc/proc_net.c +++ b/fs/proc/proc_net.c @@ -23,6 +23,7 @@ #include <linux/uidgid.h> #include <net/net_namespace.h> #include <linux/seq_file.h> +#include <linux/security.h> #include "internal.h" @@ -270,6 +271,7 @@ static struct net *get_proc_task_net(struct inode *dir) struct task_struct *task; struct nsproxy *ns; struct net *net = NULL; + struct proc_fs_info *fs_info = proc_sb_info(dir->i_sb); rcu_read_lock(); task = pid_task(proc_pid(dir), PIDTYPE_PID); @@ -282,6 +284,12 @@ static struct net *get_proc_task_net(struct inode *dir) } rcu_read_unlock(); + if (net && (fs_info->pidonly == PROC_PIDONLY_ON) && + security_capable(fs_info->mounter_cred, net->user_ns, CAP_NET_ADMIN, CAP_OPT_NONE) < 0) { + put_net(net); + net = NULL; + } + return net; } diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 49ab74e0bfde..04a382178c65 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -692,8 +692,7 @@ static bool proc_sys_fill_cache(struct file *file, child = d_lookup(dir, &qname); if (!child) { - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); - child = d_alloc_parallel(dir, &qname, &wq); + child = d_alloc_parallel(dir, &qname); if (IS_ERR(child)) return false; if (d_in_lookup(child)) { diff --git a/fs/proc/root.c b/fs/proc/root.c index 0f9100559471..99adddfeb4a4 100644 --- a/fs/proc/root.c +++ b/fs/proc/root.c @@ -223,12 +223,17 @@ static int proc_parse_param(struct fs_context *fc, struct fs_parameter *param) return 0; } -static void proc_apply_options(struct proc_fs_info *fs_info, +static int proc_apply_options(struct proc_fs_info *fs_info, struct fs_context *fc, struct user_namespace *user_ns) { struct proc_fs_context *ctx = fc->fs_private; + if ((ctx->mask & (1 << Opt_subset)) && + fc->purpose == FS_CONTEXT_FOR_RECONFIGURE && + ctx->pidonly != fs_info->pidonly) + return invalf(fc, "proc: subset=pid cannot be changed\n"); + if (ctx->mask & (1 << Opt_gid)) fs_info->pid_gid = make_kgid(user_ns, ctx->gid); if (ctx->mask & (1 << Opt_hidepid)) @@ -240,6 +245,7 @@ static void proc_apply_options(struct proc_fs_info *fs_info, put_pid_ns(fs_info->pid_ns); fs_info->pid_ns = get_pid_ns(ctx->pid_ns); } + return 0; } static int proc_fill_super(struct super_block *s, struct fs_context *fc) @@ -254,10 +260,13 @@ static int proc_fill_super(struct super_block *s, struct fs_context *fc) return -ENOMEM; fs_info->pid_ns = get_pid_ns(ctx->pid_ns); - proc_apply_options(fs_info, fc, current_user_ns()); + fs_info->mounter_cred = get_cred(fc->cred); + ret = proc_apply_options(fs_info, fc, current_user_ns()); + if (ret) + return ret; /* User space would break if executables or devices appear on proc */ - s->s_iflags |= SB_I_USERNS_VISIBLE | SB_I_NOEXEC | SB_I_NODEV; + s->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; s->s_flags |= SB_NODIRATIME | SB_NOSUID | SB_NOEXEC; s->s_blocksize = 1024; s->s_blocksize_bits = 10; @@ -266,6 +275,9 @@ static int proc_fill_super(struct super_block *s, struct fs_context *fc) s->s_time_gran = 1; s->s_fs_info = fs_info; + if (fs_info->pidonly == PROC_PIDONLY_ON) + s->s_iflags |= SB_I_RESTRICTED_VARIANT; + /* * procfs isn't actually a stacking filesystem; however, there is * too much magic going on inside it to permit stacking things on @@ -303,8 +315,7 @@ static int proc_reconfigure(struct fs_context *fc) sync_filesystem(sb); - proc_apply_options(fs_info, fc, current_user_ns()); - return 0; + return proc_apply_options(fs_info, fc, current_user_ns()); } static int proc_get_tree(struct fs_context *fc) @@ -350,6 +361,7 @@ static void proc_kill_sb(struct super_block *sb) kill_anon_super(sb); if (fs_info) { put_pid_ns(fs_info->pid_ns); + put_cred(fs_info->mounter_cred); kfree_rcu(fs_info, rcu); } } @@ -359,7 +371,7 @@ static struct file_system_type proc_fs_type = { .init_fs_context = proc_init_fs_context, .parameters = proc_fs_parameters, .kill_sb = proc_kill_sb, - .fs_flags = FS_USERNS_MOUNT | FS_DISALLOW_NOTIFY_PERM, + .fs_flags = FS_USERNS_MOUNT | FS_USERNS_MOUNT_RESTRICTED | FS_DISALLOW_NOTIFY_PERM, }; void __init proc_root_init(void) diff --git a/fs/proc/stat.c b/fs/proc/stat.c index 8b444e862319..79b42f2bfd00 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c @@ -18,41 +18,6 @@ #ifndef arch_irq_stat_cpu #define arch_irq_stat_cpu(cpu) 0 #endif -#ifndef arch_irq_stat -#define arch_irq_stat() 0 -#endif - -u64 get_idle_time(struct kernel_cpustat *kcs, int cpu) -{ - u64 idle, idle_usecs = -1ULL; - - if (cpu_online(cpu)) - idle_usecs = get_cpu_idle_time_us(cpu, NULL); - - if (idle_usecs == -1ULL) - /* !NO_HZ or cpu offline so we can rely on cpustat.idle */ - idle = kcs->cpustat[CPUTIME_IDLE]; - else - idle = idle_usecs * NSEC_PER_USEC; - - return idle; -} - -static u64 get_iowait_time(struct kernel_cpustat *kcs, int cpu) -{ - u64 iowait, iowait_usecs = -1ULL; - - if (cpu_online(cpu)) - iowait_usecs = get_cpu_iowait_time_us(cpu, NULL); - - if (iowait_usecs == -1ULL) - /* !NO_HZ or cpu offline so we can rely on cpustat.iowait */ - iowait = kcs->cpustat[CPUTIME_IOWAIT]; - else - iowait = iowait_usecs * NSEC_PER_USEC; - - return iowait; -} static void show_irq_gap(struct seq_file *p, unsigned int gap) { @@ -105,8 +70,8 @@ static int show_stat(struct seq_file *p, void *v) user += cpustat[CPUTIME_USER]; nice += cpustat[CPUTIME_NICE]; system += cpustat[CPUTIME_SYSTEM]; - idle += get_idle_time(&kcpustat, i); - iowait += get_iowait_time(&kcpustat, i); + idle += cpustat[CPUTIME_IDLE]; + iowait += cpustat[CPUTIME_IOWAIT]; irq += cpustat[CPUTIME_IRQ]; softirq += cpustat[CPUTIME_SOFTIRQ]; steal += cpustat[CPUTIME_STEAL]; @@ -122,7 +87,6 @@ static int show_stat(struct seq_file *p, void *v) sum_softirq += softirq_stat; } } - sum += arch_irq_stat(); seq_put_decimal_ull(p, "cpu ", nsec_to_clock_t(user)); seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); @@ -146,8 +110,8 @@ static int show_stat(struct seq_file *p, void *v) user = cpustat[CPUTIME_USER]; nice = cpustat[CPUTIME_NICE]; system = cpustat[CPUTIME_SYSTEM]; - idle = get_idle_time(&kcpustat, i); - iowait = get_iowait_time(&kcpustat, i); + idle = cpustat[CPUTIME_IDLE]; + iowait = cpustat[CPUTIME_IOWAIT]; irq = cpustat[CPUTIME_IRQ]; softirq = cpustat[CPUTIME_SOFTIRQ]; steal = cpustat[CPUTIME_STEAL]; diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 751b9ba160fb..d32408f7cd5e 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -132,6 +132,22 @@ static void release_task_mempolicy(struct proc_maps_private *priv) #ifdef CONFIG_PER_VMA_LOCK +static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + int ret = mmap_read_lock_killable(lock_ctx->mm); + + if (!ret) + lock_ctx->mmap_locked = true; + + return ret; +} + +static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + mmap_read_unlock(lock_ctx->mm); + lock_ctx->mmap_locked = false; +} + static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) { lock_ctx->locked_vma = NULL; @@ -146,25 +162,11 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static const struct seq_operations proc_pid_maps_op; - static inline bool lock_vma_range(struct seq_file *m, struct proc_maps_locking_ctx *lock_ctx) { - /* - * smaps and numa_maps perform page table walk, therefore require - * mmap_lock but maps can be read with locking just the vma and - * walking the vma tree under rcu read protection. - */ - if (m->op != &proc_pid_maps_op) { - if (mmap_read_lock_killable(lock_ctx->mm)) - return false; - - lock_ctx->mmap_locked = true; - } else { - rcu_read_lock(); - reset_lock_ctx(lock_ctx); - } + rcu_read_lock(); + reset_lock_ctx(lock_ctx); return true; } @@ -172,7 +174,7 @@ static inline bool lock_vma_range(struct seq_file *m, static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) { if (lock_ctx->mmap_locked) { - mmap_read_unlock(lock_ctx->mm); + unlock_ctx_mm(lock_ctx); } else { unlock_ctx_vma(lock_ctx); rcu_read_unlock(); @@ -213,17 +215,45 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return true; } +static inline void drop_rcu(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return; + + rcu_read_unlock(); +} + +static inline void reacquire_rcu(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return; + + rcu_read_lock(); + /* Reinitialize the iterator. */ + vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); +} + #else /* CONFIG_PER_VMA_LOCK */ +static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + return mmap_read_lock_killable(lock_ctx->mm); +} + +static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + mmap_read_unlock(lock_ctx->mm); +} + static inline bool lock_vma_range(struct seq_file *m, struct proc_maps_locking_ctx *lock_ctx) { - return mmap_read_lock_killable(lock_ctx->mm) == 0; + return lock_ctx_mm(lock_ctx) == 0; } static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) { - mmap_read_unlock(lock_ctx->mm); + unlock_ctx_mm(lock_ctx); } static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, @@ -238,6 +268,9 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return false; } +static inline void drop_rcu(struct proc_maps_private *priv) {} +static inline void reacquire_rcu(struct proc_maps_private *priv) {} + #endif /* CONFIG_PER_VMA_LOCK */ static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) @@ -538,12 +571,10 @@ static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) { - if (lock_ctx->mmap_locked) { - mmap_read_unlock(lock_ctx->mm); - lock_ctx->mmap_locked = false; - } else { + if (lock_ctx->mmap_locked) + unlock_ctx_mm(lock_ctx); + else unlock_ctx_vma(lock_ctx); - } } static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, @@ -1280,21 +1311,75 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK + +static const struct mm_walk_ops smaps_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, +}; + +static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, +}; + +static inline const struct mm_walk_ops * +get_smaps_walk_ops(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return &smaps_walk_ops; + return &smaps_walk_vma_lock_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return &smaps_shmem_walk_ops; + return &smaps_shmem_walk_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_smaps_walk_ops(struct proc_maps_private *priv) +{ + return &smaps_walk_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops(struct proc_maps_private *priv) +{ + return &smaps_shmem_walk_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ + /* * Gather mem stats from @vma with the indicated beginning * address @start, and keep them in @mss. * * Use vm_start of @vma as the beginning address if @start is 0. */ -static void smap_gather_stats(struct vm_area_struct *vma, - struct mem_size_stats *mss, unsigned long start) +static void smap_gather_stats(struct proc_maps_private *priv, + struct vm_area_struct *vma, + struct mem_size_stats *mss, unsigned long start) { - const struct mm_walk_ops *ops = &smaps_walk_ops; + const struct mm_walk_ops *ops = get_smaps_walk_ops(priv); /* Invalid start */ if (start >= vma->vm_end) return; + if (vma == get_gate_vma(priv->lock_ctx.mm)) + return; + + /* Might sleep. Drop RCU read lock but keep the VMA locked. */ + drop_rcu(priv); + if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { /* * For shared or readonly shmem mappings we know that all @@ -1312,15 +1397,16 @@ static void smap_gather_stats(struct vm_area_struct *vma, !(vma->vm_flags & VM_WRITE))) { mss->swap += shmem_swapped; } else { - ops = &smaps_shmem_walk_ops; + ops = get_smaps_shmem_walk_ops(priv); } } - /* mmap_lock is held in m_start */ if (!start) walk_page_vma(vma, ops, mss); else walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); + + reacquire_rcu(priv); } #define SEQ_PUT_DEC(str, val) \ @@ -1369,10 +1455,11 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, static int show_smap(struct seq_file *m, void *v) { + struct proc_maps_private *priv = m->private; struct vm_area_struct *vma = v; struct mem_size_stats mss = {}; - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); show_map_vma(m, vma); @@ -1413,7 +1500,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret = mmap_read_lock_killable(mm); + ret = lock_ctx_mm(&priv->lock_ctx); if (ret) goto out_put_mm; @@ -1425,7 +1512,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) vma_start = vma->vm_start; do { - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); last_vma_end = vma->vm_end; /* @@ -1434,8 +1521,8 @@ static int show_smaps_rollup(struct seq_file *m, void *v) */ if (mmap_lock_is_contended(mm)) { vma_iter_invalidate(&vmi); - mmap_read_unlock(mm); - ret = mmap_read_lock_killable(mm); + unlock_ctx_mm(&priv->lock_ctx); + ret = lock_ctx_mm(&priv->lock_ctx); if (ret) { release_task_mempolicy(priv); goto out_put_mm; @@ -1484,14 +1571,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v) /* Case 1 and 2 above */ if (vma->vm_start >= last_vma_end) { - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); last_vma_end = vma->vm_end; continue; } /* Case 4 above */ if (vma->vm_end > last_vma_end) { - smap_gather_stats(vma, &mss, last_vma_end); + smap_gather_stats(priv, vma, &mss, last_vma_end); last_vma_end = vma->vm_end; } } @@ -1505,7 +1592,7 @@ empty_set: __show_smap(m, &mss, true); release_task_mempolicy(priv); - mmap_read_unlock(mm); + unlock_ctx_mm(&priv->lock_ctx); out_put_mm: mmput(mm); @@ -2042,7 +2129,6 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_SOFT_DIRTY; if (pmd_swp_uffd_wp(pmd)) flags |= PM_UFFD_WP; - VM_WARN_ON_ONCE(!pmd_is_migration_entry(pmd)); page = softleaf_to_page(entry); } @@ -2523,12 +2609,16 @@ static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry)) return; - if (softleaf_is_migration(entry)) + if (softleaf_is_migration(entry)) { set_huge_pte_at(vma->vm_mm, addr, ptep, pte_swp_mkuffd_wp(ptent), psize); - else - huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, - huge_pte_mkuffd_wp(ptent)); + } else { + pte_t old_pte, new_pte; + + old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); + new_pte = huge_pte_mkuffd_wp(old_pte); + huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); + } } #endif /* CONFIG_HUGETLB_PAGE */ @@ -2869,7 +2959,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, if (~categories & PAGE_IS_WRITTEN) goto out_unlock; - if (end != start + HPAGE_SIZE) { + if (end != start + huge_page_size(hstate_vma(vma))) { /* Partial HugeTLB page WP isn't possible. */ pagemap_scan_backout_range(p, start, end); p->arg.walk_end = start; @@ -2886,8 +2976,62 @@ out_unlock: return ret; } + +/* + * Write-protect the unpopulated hugetlb entries covering [addr, end) by + * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() + * does for populated entries. + * + * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the + * loop normally runs a single iteration; it is written to cover the full range + * in case the walker ever coalesces adjacent holes. + * + * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- + * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table + * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the + * same VMA, so the scanning thread would deadlock against itself. PMD sharing + * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so + * the vma lock guards nothing that matters for these entries anyway. + */ +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + struct hstate *h = hstate_vma(vma); + unsigned long psize = huge_page_size(h); + struct mm_struct *mm = vma->vm_mm; + spinlock_t *ptl; + pte_t *ptep; + pte_t pte; + + for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { + ptep = huge_pte_alloc(mm, vma, addr, psize); + if (!ptep) + return -ENOMEM; + + i_mmap_lock_write(vma->vm_file->f_mapping); + ptl = huge_pte_lock(h, mm, ptep); + pte = huge_ptep_get(mm, addr, ptep); + make_uffd_wp_huge_pte(vma, addr, ptep, pte); + /* + * A none entry has no cached translation, so installing the + * marker needs no TLB flush. Flush only if a fault populated + * the entry between huge_pte_alloc() and the page table lock. + */ + if (!huge_pte_none(pte)) + flush_hugetlb_tlb_range(vma, addr, addr + psize); + spin_unlock(ptl); + i_mmap_unlock_write(vma->vm_file->f_mapping); + } + + return 0; +} #else #define pagemap_scan_hugetlb_entry NULL +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + return 0; +} #endif static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, @@ -2907,7 +3051,10 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, if (~p->arg.flags & PM_SCAN_WP_MATCHING) return ret; - err = uffd_wp_range(vma, addr, end - addr, true); + if (is_vm_hugetlb_page(vma)) + err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); + else + err = uffd_wp_range(vma, addr, end - addr, true); if (err < 0) ret = err; @@ -3291,6 +3438,31 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK +static const struct mm_walk_ops show_numa_vma_lock_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, +}; + +static inline const struct mm_walk_ops * +get_show_numa_ops(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return &show_numa_ops; + return &show_numa_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_show_numa_ops(struct proc_maps_private *priv) +{ + return &show_numa_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ + /* * Display pages allocated per node and memory policy via /proc. */ @@ -3335,8 +3507,13 @@ static int show_numa_map(struct seq_file *m, void *v) if (is_vm_hugetlb_page(vma)) seq_puts(m, " huge"); - /* mmap_lock is held by m_start */ - walk_page_vma(vma, &show_numa_ops, md); + /* Skip walking pages if gate VMA */ + if (vma != get_gate_vma(proc_priv->lock_ctx.mm)) { + /* Might sleep. Drop RCU read lock but keep the VMA locked. */ + drop_rcu(proc_priv); + walk_page_vma(vma, get_show_numa_ops(proc_priv), md); + reacquire_rcu(proc_priv); + } if (!md->pages) goto out; diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c index b5343d209381..433aa947cd57 100644 --- a/fs/proc/uptime.c +++ b/fs/proc/uptime.c @@ -18,12 +18,8 @@ static int uptime_proc_show(struct seq_file *m, void *v) int i; idle_nsec = 0; - for_each_possible_cpu(i) { - struct kernel_cpustat kcs; - - kcpustat_cpu_fetch(&kcs, i); - idle_nsec += get_idle_time(&kcs, i); - } + for_each_possible_cpu(i) + idle_nsec += kcpustat_field(CPUTIME_IDLE, i); ktime_get_boottime_ts64(&uptime); timens_add_boottime(&uptime); diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c index 4deb0eeadbde..42fcd500fad2 100644 --- a/fs/qnx4/inode.c +++ b/fs/qnx4/inode.c @@ -202,7 +202,8 @@ static int qnx4_fill_super(struct super_block *s, struct fs_context *fc) return -ENOMEM; s->s_fs_info = qs; - sb_set_blocksize(s, QNX4_BLOCK_SIZE); + if (!sb_set_blocksize(s, QNX4_BLOCK_SIZE)) + return -EINVAL; s->s_op = &qnx4_sops; s->s_magic = QNX4_SUPER_MAGIC; diff --git a/fs/qnx6/dir.c b/fs/qnx6/dir.c index 135fb42f6936..56bbaffe4b44 100644 --- a/fs/qnx6/dir.c +++ b/fs/qnx6/dir.c @@ -132,16 +132,16 @@ static int qnx6_readdir(struct file *file, struct dir_context *ctx) struct qnx6_dir_entry *de; struct folio *folio; char *kaddr = qnx6_get_folio(inode, n, &folio); - char *limit; + struct qnx6_dir_entry *limit; if (IS_ERR(kaddr)) { pr_err("%s(): read failed\n", __func__); ctx->pos = (n + 1) << PAGE_SHIFT; return PTR_ERR(kaddr); } - de = (struct qnx6_dir_entry *)(kaddr + offset); - limit = kaddr + last_entry(inode, n); - for (; (char *)de < limit; de++, ctx->pos += QNX6_DIR_ENTRY_SIZE) { + de = (struct qnx6_dir_entry *)kaddr + offset; + limit = (struct qnx6_dir_entry *)kaddr + last_entry(inode, n); + for (; de < limit; de++, ctx->pos += QNX6_DIR_ENTRY_SIZE) { int size = de->de_size; u32 no_inode = fs32_to_cpu(sbi, de->de_inode); diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 64cf42721496..9850de3955d3 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -3022,7 +3022,7 @@ static const struct ctl_table fs_dqstats_table[] = { static int __init dquot_init(void) { int i, ret; - unsigned long nr_hash, order; + unsigned long nr_hash; struct shrinker *dqcache_shrinker; printk(KERN_NOTICE "VFS: Disk quotas %s\n", __DQUOT_VERSION__); @@ -3035,8 +3035,7 @@ static int __init dquot_init(void) SLAB_PANIC), NULL); - order = 0; - dquot_hash = (struct hlist_head *)__get_free_pages(GFP_KERNEL, order); + dquot_hash = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!dquot_hash) panic("Cannot create dquot hash table"); @@ -3046,7 +3045,7 @@ static int __init dquot_init(void) panic("Cannot create dquot stat counters"); /* Find power-of-two hlist_heads which can fit into allocation */ - nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head); + nr_hash = PAGE_SIZE / sizeof(struct hlist_head); dq_hash_bits = ilog2(nr_hash); nr_hash = 1UL << dq_hash_bits; @@ -3054,8 +3053,8 @@ static int __init dquot_init(void) for (i = 0; i < nr_hash; i++) INIT_HLIST_HEAD(dquot_hash + i); - pr_info("VFS: Dquot-cache hash table entries: %ld (order %ld," - " %ld bytes)\n", nr_hash, order, (PAGE_SIZE << order)); + pr_info("VFS: Dquot-cache hash table entries: %ld (%ld bytes)\n", + nr_hash, PAGE_SIZE); dqcache_shrinker = shrinker_alloc(0, "dquota-cache"); if (!dqcache_shrinker) diff --git a/fs/read_write.c b/fs/read_write.c index 50bff7edc91f..e8c14e2760b2 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -641,13 +641,12 @@ ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t return __kernel_write_iter(file, &iter, pos); } /* - * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()", - * but autofs is one of the few internal kernel users that actually + * autofs is one of the few internal kernel users that actually * wants this _and_ can be built as a module. So we need to export * this symbol for autofs, even though it really isn't appropriate * for any other kernel modules. */ -EXPORT_SYMBOL_GPL(__kernel_write); +EXPORT_SYMBOL_FOR_MODULES(__kernel_write, "autofs4"); ssize_t kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index 1a9b29119f88..48af75b9dc85 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -408,6 +408,8 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); void resctrl_file_fflags_init(const char *config, unsigned long fflags); +void resctrl_file_mode_init(const char *config, umode_t mode); + void rdt_staged_configs_clear(void); bool closid_allocated(unsigned int closid); diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 9fd901c78dc6..a932a1fea818 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -135,10 +135,10 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); struct rmid_entry *entry; + bool rmid_dirty = true; u32 idx, cur_idx = 1; void *arch_mon_ctx; void *arch_priv; - bool rmid_dirty; u64 val = 0; arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv; @@ -161,22 +161,27 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) break; entry = __rmid_entry(idx); - if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, entry->rmid, - QOS_L3_OCCUP_EVENT_ID, arch_priv, &val, - arch_mon_ctx)) { - rmid_dirty = true; - } else { - rmid_dirty = (val >= resctrl_rmid_realloc_threshold); - - /* - * x86's CLOSID and RMID are independent numbers, so the entry's - * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the - * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't - * used to select the configuration. It is thus necessary to track both - * CLOSID and RMID because there may be dependencies between them - * on some architectures. - */ - trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val); + if (!force_free) { + if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, + entry->rmid, QOS_L3_OCCUP_EVENT_ID, + arch_priv, &val, arch_mon_ctx)) { + rmid_dirty = true; + } else { + rmid_dirty = (val >= resctrl_rmid_realloc_threshold); + + /* + * x86's CLOSID and RMID are independent numbers, + * so the entry's CLOSID is an empty CLOSID + * (X86_RESCTRL_EMPTY_CLOSID). On Arm the RMID + * (PMG) extends the CLOSID (PARTID) space with + * bits that aren't used to select the configuration. + * It is thus necessary to track both CLOSID and + * RMID because there may be dependencies between + * them on some architectures. + */ + trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, + d->hdr.id, val); + } } if (force_free || !rmid_dirty) { @@ -1211,9 +1216,10 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_ * NULL; otherwise, assign the counter to the specified domain @d. * * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr() - * will fail. The assignment process will abort at the first failure encountered - * during domain traversal, which may result in the event being only partially - * assigned. + * will fail. When attempting to assign counters to all domains, carry on trying + * to assign counters after a failure since only some domains may have counters + * and the goal is to assign counters where possible. If any counter assignment + * fails, return the error from the last failing assignment. * * Return: * 0 on success, < 0 on failure. @@ -1226,9 +1232,11 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro if (!d) { list_for_each_entry(d, &r->mon_domains, hdr.list) { - ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); - if (ret) - return ret; + int err; + + err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); + if (err) + ret = err; } } else { ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); @@ -1422,6 +1430,11 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes ret = -EINVAL; goto out_unlock; } + if (!r->mon.mbm_cntr_configurable) { + rdt_last_cmd_puts("event_filter is not configurable\n"); + ret = -EPERM; + goto out_unlock; + } ret = resctrl_parse_mem_transactions(buf, &evt_cfg); if (!ret && mevt->evt_cfg != evt_cfg) { @@ -1451,7 +1464,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, else seq_puts(s, "[default]\n"); - if (!IS_ENABLED(CONFIG_RESCTRL_ASSIGN_FIXED)) { + if (!r->mon.mbm_cntr_assign_fixed) { if (enabled) seq_puts(s, "default\n"); else @@ -1502,6 +1515,12 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, } if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) { + if (r->mon.mbm_cntr_assign_fixed) { + ret = -EINVAL; + rdt_last_cmd_puts("Counter assignment mode is not configurable\n"); + goto out_unlock; + } + ret = resctrl_arch_mbm_cntr_assign_set(r, enable); if (ret) goto out_unlock; @@ -1886,6 +1905,8 @@ int resctrl_l3_mon_resource_init(void) resctrl_file_fflags_init("available_mbm_cntrs", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG); + if (r->mon.mbm_cntr_configurable) + resctrl_file_mode_init("event_filter", 0644); resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE); diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 5dfdaa6f9d8f..cc9966ff6cdf 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -74,6 +74,8 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx); static void rdtgroup_destroy_root(void); +static void mon_put_kn_priv(void); + struct dentry *debugfs_resctrl; /* @@ -585,14 +587,20 @@ unlock: * * On resource group creation via a mkdir, an extra kernfs_node reference is * taken to ensure that the rdtgroup structure remains accessible for the - * rdtgroup_kn_unlock() calls where it is removed. + * rdtgroup_kn_unlock() calls where it is removed. The default group is + * statically allocated: it does not have an extra reference but will have + * RDT_DELETED set on unmount to support safe access to its associated files + * via rdtgroup_kn_lock_live/rdtgroup_kn_unlock(). * - * Drop the extra reference here, then free the rdtgroup structure. + * For all but the default group: drop the extra reference, then free the + * rdtgroup structure. * * Return: void */ static void rdtgroup_remove(struct rdtgroup *rdtgrp) { + if (rdtgrp == &rdtgroup_default) + return; kernfs_put(rdtgrp->kn); kfree(rdtgrp); } @@ -2022,7 +2030,7 @@ static struct rftype res_common_files[] = { }, { .name = "event_filter", - .mode = 0644, + .mode = 0444, .kf_ops = &rdtgroup_kf_single_ops, .seq_show = event_filter_show, .write = event_filter_write, @@ -2215,6 +2223,15 @@ void resctrl_file_fflags_init(const char *config, unsigned long fflags) rft->fflags = fflags; } +void resctrl_file_mode_init(const char *config, umode_t mode) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->mode = mode; +} + /** * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file * @r: The resource group with which the file is associated. @@ -2331,22 +2348,19 @@ static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_nod continue; kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt); - if (IS_ERR(kn_subdir2)) { - ret = PTR_ERR(kn_subdir2); - goto out; - } + if (IS_ERR(kn_subdir2)) + return PTR_ERR(kn_subdir2); ret = rdtgroup_kn_set_ugid(kn_subdir2); if (ret) - goto out; + return ret; ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG); if (ret) - break; + return ret; } -out: - return ret; + return 0; } static int rdtgroup_mkdir_info_resdir(void *priv, char *name, @@ -2510,10 +2524,13 @@ static void mba_sc_domain_destroy(struct rdt_resource *r, } /* - * MBA software controller is supported only if - * MBM is supported and MBA is in linear scale, - * and the MBM monitor scope is the same as MBA - * control scope. + * The MBA software controller is supported only if MBM is supported and MBA is + * in linear scale, and the MBM monitor scope is the same as MBA control scope. + * + * The software controller cannot be supported when the MBM counters are + * assignable. There is no guarantee that MBM counters are assigned to the + * event backing the software controller in all monitoring domains of all + * monitoring groups. */ static bool supports_mba_mbps(void) { @@ -2522,7 +2539,8 @@ static bool supports_mba_mbps(void) return (resctrl_is_mbm_enabled() && r->alloc_capable && is_mba_linear() && - r->ctrl_scope == rmbm->mon_scope); + r->ctrl_scope == rmbm->mon_scope && + !rmbm->mon.mbm_cntr_assignable); } /* @@ -2802,6 +2820,12 @@ static int rdt_get_tree(struct fs_context *fc) goto out; } + /* Avoid races from pending operations from a previous mount */ + if (atomic_read(&rdtgroup_default.waitcount) != 0) { + ret = -EBUSY; + goto out; + } + ret = setup_rmid_lru_list(); if (ret) goto out; @@ -2883,6 +2907,7 @@ out_mondata: kernfs_remove(kn_mondata); out_mongrp: if (resctrl_arch_mon_capable()) { + mon_put_kn_priv(); rdtgroup_unassign_cntrs(&rdtgroup_default); kernfs_remove(kn_mongrp); } @@ -2937,7 +2962,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) ctx->enable_cdpl2 = true; return 0; case Opt_mba_mbps: - msg = "mba_MBps requires MBM and linear scale MBA at L3 scope"; + msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope"; if (!supports_mba_mbps()) return invalfc(fc, msg); ctx->enable_mba_mbps = true; @@ -3059,10 +3084,6 @@ static void rmdir_all_sub(void) if (rdtgrp == &rdtgroup_default) continue; - if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || - rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) - rdtgroup_pseudo_lock_remove(rdtgrp); - /* * Give any CPUs back to the default group. We cannot copy * cpu_online_mask because a CPU might have executed the @@ -3073,7 +3094,13 @@ static void rmdir_all_sub(void) rdtgroup_unassign_cntrs(rdtgrp); - free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + rdtgroup_pseudo_lock_remove(rdtgrp); + } else { + /* Pseudo-locked group's RMID is freed during setup. */ + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + } kernfs_remove(rdtgrp->kn); list_del(&rdtgrp->rdtgroup_list); @@ -3164,6 +3191,7 @@ static void resctrl_fs_teardown(void) mon_put_kn_priv(); rdt_pseudo_lock_release(); rdtgroup_default.mode = RDT_MODE_SHAREABLE; + rdtgroup_default.flags = RDT_DELETED; closid_exit(); schemata_list_destroy(); rdtgroup_destroy_root(); @@ -4264,6 +4292,7 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx) ctx->kfc.root = rdt_root; rdtgroup_default.kn = kernfs_root_to_node(rdt_root); + rdtgroup_default.flags = 0; return 0; } diff --git a/fs/select.c b/fs/select.c index 75978b18f48f..95d76531015a 100644 --- a/fs/select.c +++ b/fs/select.c @@ -150,7 +150,7 @@ void poll_freewait(struct poll_wqueues *pwq) } while (entry > p->entries); old = p; p = p->next; - free_page((unsigned long) old); + kfree(old); } } EXPORT_SYMBOL(poll_freewait); @@ -165,7 +165,7 @@ static struct poll_table_entry *poll_get_entry(struct poll_wqueues *p) if (!table || POLL_TABLE_FULL(table)) { struct poll_table_page *new_table; - new_table = (struct poll_table_page *) __get_free_page(GFP_KERNEL); + new_table = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!new_table) { p->error = -ENOMEM; return NULL; @@ -708,6 +708,17 @@ static int kern_select(int n, fd_set __user *inp, fd_set __user *outp, if (copy_from_user(&tv, tvp, sizeof(tv))) return -EFAULT; + /* + * Reject negative components before normalisation. The seconds + * sum below is performed in signed long and a crafted negative + * timeval can wrap to a positive value that passes + * timespec64_valid() and turns into an effectively-infinite + * deadline via timespec64_add_safe()'s saturation, instead of + * the -EINVAL POSIX requires for negative timeouts. + */ + if (tv.tv_sec < 0 || tv.tv_usec < 0) + return -EINVAL; + to = &end_time; if (poll_select_set_timeout(to, tv.tv_sec + (tv.tv_usec / USEC_PER_SEC), diff --git a/fs/smb/client/Makefile b/fs/smb/client/Makefile index 6e83b5204699..fc6b9d35c962 100644 --- a/fs/smb/client/Makefile +++ b/fs/smb/client/Makefile @@ -42,7 +42,7 @@ cifs-$(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) += \ smb1session.o \ smb1transport.o -cifs-$(CONFIG_CIFS_COMPRESSION) += compress.o compress/lz77.o +cifs-$(CONFIG_CIFS_COMPRESSION) += compress.o ifneq ($(CONFIG_CIFS_ALLOW_INSECURE_LEGACY),) # diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index 02791ec3c5a1..88d5e9a32f28 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -286,6 +286,14 @@ replay_again: &rqst[0], &oplock, &oparms, utf16_path); if (rc) goto oshr_free; + + if (oplock != SMB2_OPLOCK_LEVEL_II) { + rc = -EINVAL; + cifs_dbg(FYI, "%s: Oplock level %d not suitable for cached directory\n", + __func__, oplock); + goto oshr_free; + } + smb2_set_next_command(tcon, &rqst[0]); memset(&qi_iov, 0, sizeof(qi_iov)); diff --git a/fs/smb/client/cifs_fs_sb.h b/fs/smb/client/cifs_fs_sb.h index 84e7e366b0ff..d6494e1d93cc 100644 --- a/fs/smb/client/cifs_fs_sb.h +++ b/fs/smb/client/cifs_fs_sb.h @@ -56,6 +56,7 @@ struct cifs_sb_info { struct smb3_fs_context *ctx; atomic_t active; atomic_t mnt_cifs_flags; + atomic_t outstanding_rreq; /* nr of rreqs not yet fully deinitialized */ struct delayed_work prune_tlinks; struct rcu_head rcu; diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c index 3a41bbada04c..44c407275680 100644 --- a/fs/smb/client/cifs_spnego.c +++ b/fs/smb/client/cifs_spnego.c @@ -8,6 +8,7 @@ */ #include <linux/list.h> +#include <linux/cred.h> #include <linux/slab.h> #include <linux/string.h> #include <keys/user-type.h> @@ -40,12 +41,27 @@ cifs_spnego_key_destroy(struct key *key) kfree(key->payload.data[0]); } +static int +cifs_spnego_key_vet_description(const char *description) +{ + /* + * cifs.spnego descriptions are authority-bearing inputs to cifs.upcall. + * They are only valid when produced by CIFS while using the private + * spnego_cred installed below. Do not let userspace create this type + * of key through request_key(2)/add_key(2), since the helper treats + * pid/uid/creduid/upcall_target as kernel-originating fields. + */ + if (current_cred() != spnego_cred) + return -EPERM; + return 0; +} /* * keytype for CIFS spnego keys */ struct key_type cifs_spnego_key_type = { .name = "cifs.spnego", + .vet_description = cifs_spnego_key_vet_description, .instantiate = cifs_spnego_key_instantiate, .destroy = cifs_spnego_key_destroy, .describe = user_describe, diff --git a/fs/smb/client/cifs_swn.c b/fs/smb/client/cifs_swn.c index 9753a432d099..9951817d0d7f 100644 --- a/fs/smb/client/cifs_swn.c +++ b/fs/smb/client/cifs_swn.c @@ -28,10 +28,54 @@ struct cifs_swn_reg { bool net_name_notify; bool share_name_notify; bool ip_notify; +}; - struct cifs_tcon *tcon; +struct cifs_swn_reg_info { + int id; + unsigned int ref_count; + const char *net_name; + const char *share_name; + bool net_name_notify; + bool share_name_notify; + bool ip_notify; }; +static void cifs_swn_snapshot_reg(struct cifs_swn_reg *swnreg, + struct cifs_swn_reg_info *info) +{ + info->id = swnreg->id; + info->ref_count = kref_read(&swnreg->ref_count); + info->net_name = swnreg->net_name; + info->share_name = swnreg->share_name; + info->net_name_notify = swnreg->net_name_notify; + info->share_name_notify = swnreg->share_name_notify; + info->ip_notify = swnreg->ip_notify; +} + +static int cifs_swn_dup_reg(struct cifs_swn_reg *swnreg, + struct cifs_swn_reg_info *info) +{ + cifs_swn_snapshot_reg(swnreg, info); + + info->net_name = kstrdup(swnreg->net_name, GFP_KERNEL); + if (!info->net_name) + return -ENOMEM; + + info->share_name = kstrdup(swnreg->share_name, GFP_KERNEL); + if (!info->share_name) { + kfree(info->net_name); + return -ENOMEM; + } + + return 0; +} + +static void cifs_swn_free_reg_info(struct cifs_swn_reg_info *info) +{ + kfree(info->net_name); + kfree(info->share_name); +} + static int cifs_swn_auth_info_krb(struct cifs_tcon *tcon, struct sk_buff *skb) { int ret; @@ -73,7 +117,8 @@ static int cifs_swn_auth_info_ntlm(struct cifs_tcon *tcon, struct sk_buff *skb) * The authentication information to connect to the witness service is bundled * into the message. */ -static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) +static int cifs_swn_send_register_message(struct cifs_swn_reg_info *swnreg, + struct cifs_tcon *tcon) { struct sk_buff *skb; struct genlmsghdr *hdr; @@ -109,10 +154,10 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) * told to switch to it (client move message). In these cases we unregister from the * server address and register to the new address when we receive the notification. */ - if (swnreg->tcon->ses->server->use_swn_dstaddr) - addr = &swnreg->tcon->ses->server->swn_dstaddr; + if (tcon->ses->server->use_swn_dstaddr) + addr = &tcon->ses->server->swn_dstaddr; else - addr = &swnreg->tcon->ses->server->dstaddr; + addr = &tcon->ses->server->dstaddr; ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage), addr); if (ret < 0) @@ -136,10 +181,10 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) goto nlmsg_fail; } - authtype = cifs_select_sectype(swnreg->tcon->ses->server, swnreg->tcon->ses->sectype); + authtype = cifs_select_sectype(tcon->ses->server, tcon->ses->sectype); switch (authtype) { case Kerberos: - ret = cifs_swn_auth_info_krb(swnreg->tcon, skb); + ret = cifs_swn_auth_info_krb(tcon, skb); if (ret < 0) { cifs_dbg(VFS, "%s: Failed to get kerberos auth info: %d\n", __func__, ret); goto nlmsg_fail; @@ -147,7 +192,7 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg) break; case NTLMv2: case RawNTLMSSP: - ret = cifs_swn_auth_info_ntlm(swnreg->tcon, skb); + ret = cifs_swn_auth_info_ntlm(tcon, skb); if (ret < 0) { cifs_dbg(VFS, "%s: Failed to get NTLM auth info: %d\n", __func__, ret); goto nlmsg_fail; @@ -176,7 +221,8 @@ nlmsg_fail: /* * Sends an uregister message to the userspace daemon based on the registration */ -static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg) +static int cifs_swn_send_unregister_message(struct cifs_swn_reg_info *swnreg, + struct cifs_tcon *tcon) { struct sk_buff *skb; struct genlmsghdr *hdr; @@ -205,7 +251,7 @@ static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg) goto nlmsg_fail; ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage), - &swnreg->tcon->ses->server->dstaddr); + &tcon->ses->server->dstaddr); if (ret < 0) goto nlmsg_fail; @@ -242,6 +288,88 @@ nlmsg_fail: } /* + * Allocation-free mirror of extract_hostname() + extract_sharename() from + * fs/smb/client/unc.c. Those helpers kmalloc(GFP_KERNEL); this runs under + * cifs_tcp_ses_lock and tcon->tc_lock, both spinlocks, so we mirror their + * parsing in place against the caller's stable net_name/share_name strings. + * Keep in sync with unc.c. + */ +static bool cifs_swn_tcon_matches(struct cifs_tcon *tcon, + const char *net_name, + const char *share_name) +{ + const char *unc = tcon->tree_name; + const char *host, *share, *delim; + size_t host_len, share_len; + + if (!tcon->use_witness) + return false; + + /* extract_hostname: require strlen(unc) >= 3 */ + if (strnlen(unc, 3) < 3) + return false; + /* extract_hostname: skip all leading '\' characters */ + for (host = unc; *host == '\\'; host++) + ; + if (!*host) + return false; + delim = strchr(host, '\\'); + if (!delim) + return false; + host_len = delim - host; + if (strlen(net_name) != host_len || + strncasecmp(host, net_name, host_len)) + return false; + + /* extract_sharename: start at unc + 2, then first '\' onward */ + share = unc + 2; + delim = strchr(share, '\\'); + if (!delim) + return false; + share = delim + 1; + share_len = strlen(share); + + return strlen(share_name) == share_len && + !strncasecmp(share, share_name, share_len); +} + +/* + * One SWN registration id represents one net/share name pair. Multiple + * mounted tcons can therefore share the id. Pick a live representative at + * use time instead of caching the first tcon pointer in the registration. + */ +static struct cifs_tcon *cifs_swn_get_tcon(struct cifs_swn_reg_info *swnreg) +{ + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_EXITING || + !cifs_swn_tcon_matches(tcon, swnreg->net_name, + swnreg->share_name)) { + spin_unlock(&tcon->tc_lock); + continue; + } + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, + tcon->tc_count, + netfs_trace_tcon_ref_get_swn_notify); + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + return tcon; + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + return NULL; +} + +/* * Try to find a matching registration for the tcon's server name and share name. * Calls to this function must be protected by cifs_swnreg_idr_mutex. * TODO Try to avoid memory allocations @@ -347,8 +475,6 @@ static struct cifs_swn_reg *cifs_get_swn_reg(struct cifs_tcon *tcon) reg->net_name_notify = true; reg->share_name_notify = true; reg->ip_notify = (tcon->capabilities & SMB2_SHARE_CAP_SCALEOUT); - - reg->tcon = tcon; unlock: mutex_unlock(&cifs_swnreg_idr_mutex); @@ -368,11 +494,6 @@ fail_unlock: static void cifs_swn_reg_release(struct kref *ref) { struct cifs_swn_reg *swnreg = container_of(ref, struct cifs_swn_reg, ref_count); - int ret; - - ret = cifs_swn_send_unregister_message(swnreg); - if (ret < 0) - cifs_dbg(VFS, "%s: Failed to send unregister message: %d\n", __func__, ret); idr_remove(&cifs_swnreg_idr, swnreg->id); kfree(swnreg->net_name); @@ -380,23 +501,33 @@ static void cifs_swn_reg_release(struct kref *ref) kfree(swnreg); } -static void cifs_put_swn_reg(struct cifs_swn_reg *swnreg) +static void cifs_put_swn_reg_locked(struct cifs_swn_reg *swnreg, + struct cifs_tcon *tcon) { - mutex_lock(&cifs_swnreg_idr_mutex); + if (kref_read(&swnreg->ref_count) == 1) { + struct cifs_swn_reg_info swnreg_info; + int ret; + + cifs_swn_snapshot_reg(swnreg, &swnreg_info); + ret = cifs_swn_send_unregister_message(&swnreg_info, tcon); + if (ret < 0) + cifs_dbg(VFS, "%s: Failed to send unregister message: %d\n", + __func__, ret); + } + kref_put(&swnreg->ref_count, cifs_swn_reg_release); - mutex_unlock(&cifs_swnreg_idr_mutex); } -static int cifs_swn_resource_state_changed(struct cifs_swn_reg *swnreg, const char *name, int state) +static int cifs_swn_resource_state_changed(struct cifs_tcon *tcon, const char *name, int state) { switch (state) { case CIFS_SWN_RESOURCE_STATE_UNAVAILABLE: cifs_dbg(FYI, "%s: resource name '%s' become unavailable\n", __func__, name); - cifs_signal_cifsd_for_reconnect(swnreg->tcon->ses->server, true); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); break; case CIFS_SWN_RESOURCE_STATE_AVAILABLE: cifs_dbg(FYI, "%s: resource name '%s' become available\n", __func__, name); - cifs_signal_cifsd_for_reconnect(swnreg->tcon->ses->server, true); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); break; case CIFS_SWN_RESOURCE_STATE_UNKNOWN: cifs_dbg(FYI, "%s: resource name '%s' changed to unknown state\n", __func__, name); @@ -502,7 +633,7 @@ unlock: return ret; } -static int cifs_swn_client_move(struct cifs_swn_reg *swnreg, struct sockaddr_storage *addr) +static int cifs_swn_client_move(struct cifs_tcon *tcon, struct sockaddr_storage *addr) { struct sockaddr_in *ipv4 = (struct sockaddr_in *)addr; struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)addr; @@ -512,14 +643,17 @@ static int cifs_swn_client_move(struct cifs_swn_reg *swnreg, struct sockaddr_sto else if (addr->ss_family == AF_INET6) cifs_dbg(FYI, "%s: move to %pI6\n", __func__, &ipv6->sin6_addr); - return cifs_swn_reconnect(swnreg->tcon, addr); + return cifs_swn_reconnect(tcon, addr); } int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) { struct cifs_swn_reg *swnreg; + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; char name[256]; int type; + int ret = 0; if (info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]) { int swnreg_id; @@ -527,21 +661,34 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) swnreg_id = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]); mutex_lock(&cifs_swnreg_idr_mutex); swnreg = idr_find(&cifs_swnreg_idr, swnreg_id); - mutex_unlock(&cifs_swnreg_idr_mutex); if (swnreg == NULL) { + mutex_unlock(&cifs_swnreg_idr_mutex); cifs_dbg(FYI, "%s: registration id %d not found\n", __func__, swnreg_id); return -EINVAL; } + ret = cifs_swn_dup_reg(swnreg, &swnreg_info); + mutex_unlock(&cifs_swnreg_idr_mutex); + if (ret) + return ret; } else { cifs_dbg(FYI, "%s: missing registration id attribute\n", __func__); return -EINVAL; } + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + cifs_dbg(FYI, "%s: registration id %d has no live tcon\n", + __func__, swnreg_info.id); + ret = -ENODEV; + goto free_info; + } + if (info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]) { type = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]); } else { cifs_dbg(FYI, "%s: missing notification type attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } switch (type) { @@ -553,15 +700,18 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) sizeof(name)); } else { cifs_dbg(FYI, "%s: missing resource name attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } if (info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]) { state = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]); } else { cifs_dbg(FYI, "%s: missing resource state attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } - return cifs_swn_resource_state_changed(swnreg, name, state); + ret = cifs_swn_resource_state_changed(tcon, name, state); + break; } case CIFS_SWN_NOTIFICATION_CLIENT_MOVE: { struct sockaddr_storage addr; @@ -570,28 +720,36 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) nla_memcpy(&addr, info->attrs[CIFS_GENL_ATTR_SWN_IP], sizeof(addr)); } else { cifs_dbg(FYI, "%s: missing IP address attribute\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto out; } - return cifs_swn_client_move(swnreg, &addr); + ret = cifs_swn_client_move(tcon, &addr); + break; } default: cifs_dbg(FYI, "%s: unknown notification type %d\n", __func__, type); break; } - return 0; +out: + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +free_info: + cifs_swn_free_reg_info(&swnreg_info); + return ret; } int cifs_swn_register(struct cifs_tcon *tcon) { struct cifs_swn_reg *swnreg; + struct cifs_swn_reg_info swnreg_info; int ret; swnreg = cifs_get_swn_reg(tcon); if (IS_ERR(swnreg)) return PTR_ERR(swnreg); - ret = cifs_swn_send_register_message(swnreg); + cifs_swn_snapshot_reg(swnreg, &swnreg_info); + ret = cifs_swn_send_register_message(&swnreg_info, tcon); if (ret < 0) { cifs_dbg(VFS, "%s: Failed to send swn register message: %d\n", __func__, ret); /* Do not put the swnreg or return error, the echo task will retry */ @@ -612,35 +770,68 @@ int cifs_swn_unregister(struct cifs_tcon *tcon) return PTR_ERR(swnreg); } + cifs_put_swn_reg_locked(swnreg, tcon); mutex_unlock(&cifs_swnreg_idr_mutex); - cifs_put_swn_reg(swnreg); - return 0; } -void cifs_swn_dump(struct seq_file *m) +/* + * Snapshot one registration under cifs_swnreg_idr_mutex and return. Callers + * intentionally do the per-registration network/genlmsg work without the + * mutex held, both to keep the critical section short and to avoid nesting + * cifs_swnreg_idr_mutex inside the higher tc_lock when a live tcon is then + * pinned for the send. + */ +static int cifs_swn_get_next_reg_info(int *id, struct cifs_swn_reg_info *info) { struct cifs_swn_reg *swnreg; + int ret = 0; + + mutex_lock(&cifs_swnreg_idr_mutex); + swnreg = idr_get_next(&cifs_swnreg_idr, id); + if (swnreg) { + ret = cifs_swn_dup_reg(swnreg, info); + if (!ret) { + *id = swnreg->id + 1; + ret = 1; + } + } + mutex_unlock(&cifs_swnreg_idr_mutex); + + return ret; +} + +void cifs_swn_dump(struct seq_file *m) +{ + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; struct sockaddr_in *sa; struct sockaddr_in6 *sa6; - int id; + int id = 0; + int ret; seq_puts(m, "Witness registrations:"); - mutex_lock(&cifs_swnreg_idr_mutex); - idr_for_each_entry(&cifs_swnreg_idr, swnreg, id) { + while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) { seq_printf(m, "\nId: %u Refs: %u Network name: '%s'%s Share name: '%s'%s Ip address: ", - id, kref_read(&swnreg->ref_count), - swnreg->net_name, swnreg->net_name_notify ? "(y)" : "(n)", - swnreg->share_name, swnreg->share_name_notify ? "(y)" : "(n)"); - switch (swnreg->tcon->ses->server->dstaddr.ss_family) { + swnreg_info.id, swnreg_info.ref_count, + swnreg_info.net_name, swnreg_info.net_name_notify ? "(y)" : "(n)", + swnreg_info.share_name, swnreg_info.share_name_notify ? "(y)" : "(n)"); + + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + seq_puts(m, "(no live tcon)"); + goto next; + } + + switch (tcon->ses->server->dstaddr.ss_family) { case AF_INET: - sa = (struct sockaddr_in *) &swnreg->tcon->ses->server->dstaddr; + sa = (struct sockaddr_in *)&tcon->ses->server->dstaddr; seq_printf(m, "%pI4", &sa->sin_addr.s_addr); break; case AF_INET6: - sa6 = (struct sockaddr_in6 *) &swnreg->tcon->ses->server->dstaddr; + sa6 = (struct sockaddr_in6 *)&tcon->ses->server->dstaddr; seq_printf(m, "%pI6", &sa6->sin6_addr.s6_addr); if (sa6->sin6_scope_id) seq_printf(m, "%%%u", sa6->sin6_scope_id); @@ -648,23 +839,38 @@ void cifs_swn_dump(struct seq_file *m) default: seq_puts(m, "(unknown)"); } - seq_printf(m, "%s", swnreg->ip_notify ? "(y)" : "(n)"); + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +next: + seq_printf(m, "%s", swnreg_info.ip_notify ? "(y)" : "(n)"); + cifs_swn_free_reg_info(&swnreg_info); } - mutex_unlock(&cifs_swnreg_idr_mutex); + if (ret < 0) + seq_printf(m, "\nFailed to snapshot witness registration: %d", ret); seq_puts(m, "\n"); } void cifs_swn_check(void) { - struct cifs_swn_reg *swnreg; - int id; + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; + int id = 0; int ret; - mutex_lock(&cifs_swnreg_idr_mutex); - idr_for_each_entry(&cifs_swnreg_idr, swnreg, id) { - ret = cifs_swn_send_register_message(swnreg); + while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) { + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + cifs_dbg(FYI, "%s: registration id %d has no live tcon\n", + __func__, swnreg_info.id); + goto free_info; + } + + ret = cifs_swn_send_register_message(&swnreg_info, tcon); if (ret < 0) cifs_dbg(FYI, "%s: Failed to send register message: %d\n", __func__, ret); + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +free_info: + cifs_swn_free_reg_info(&swnreg_info); } - mutex_unlock(&cifs_swnreg_idr_mutex); + if (ret < 0) + cifs_dbg(FYI, "%s: Failed to snapshot registration: %d\n", __func__, ret); } diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index ec5d47779304..9424281a7674 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -275,6 +275,73 @@ cifs_copy_sid(struct smb_sid *dst, const struct smb_sid *src) return size; } +static int parse_sid(const struct smb_sid *psid, const char *end_of_acl) +{ + unsigned int sid_len; + + /* SID must contain the fixed header before num_subauth is trusted. */ + if (end_of_acl < (const char *)psid + CIFS_SID_BASE_SIZE) { + cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid); + return -EINVAL; + } + if (psid->num_subauth > SID_MAX_SUB_AUTHORITIES) { + cifs_dbg(VFS, "SID contains too many subauthorities %u\n", + psid->num_subauth); + return -EINVAL; + } + + sid_len = CIFS_SID_BASE_SIZE + psid->num_subauth * sizeof(__le32); + if (end_of_acl < (const char *)psid + sid_len) { + cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid); + return -EINVAL; + } + +#ifdef CONFIG_CIFS_DEBUG2 + if (psid->num_subauth) { + int i; + + cifs_dbg(FYI, "SID revision %d num_auth %d\n", + psid->revision, psid->num_subauth); + + for (i = 0; i < psid->num_subauth; i++) { + cifs_dbg(FYI, "SID sub_auth[%d]: 0x%x\n", + i, le32_to_cpu(psid->sub_auth[i])); + } + + cifs_dbg(FYI, "RID 0x%x\n", + le32_to_cpu(psid->sub_auth[psid->num_subauth - 1])); + } +#endif + + return 0; +} + +static int sid_from_sd(const struct smb_ntsd *pntsd, __u32 secdesclen, + __u32 sid_offset, struct smb_sid **sid) +{ + struct smb_sid *psid; + char *end_of_acl; + + if (secdesclen < sizeof(struct smb_ntsd)) { + cifs_dbg(VFS, "ACL too small to parse security descriptor\n"); + return -EINVAL; + } + end_of_acl = (char *)pntsd + secdesclen; + + if (sid_offset < sizeof(struct smb_ntsd) || + sid_offset > secdesclen - CIFS_SID_BASE_SIZE) { + cifs_dbg(VFS, "Server returned illegal SID offset\n"); + return -EINVAL; + } + + psid = (struct smb_sid *)((char *)pntsd + sid_offset); + if (parse_sid(psid, end_of_acl)) + return -EINVAL; + + *sid = psid; + return 0; +} + static int id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid) { @@ -515,14 +582,14 @@ exit_cifs_idmap(void) } /* copy ntsd, owner sid, and group sid from a security descriptor to another */ -static __u32 copy_sec_desc(const struct smb_ntsd *pntsd, - struct smb_ntsd *pnntsd, - __u32 sidsoffset, - struct smb_sid *pownersid, - struct smb_sid *pgrpsid) +static int copy_sec_desc(const struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, + __u32 sidsoffset, __u32 secdesclen, + __u32 *pnsecdesclen, struct smb_sid *pownersid, + struct smb_sid *pgrpsid) { struct smb_sid *owner_sid_ptr, *group_sid_ptr; struct smb_sid *nowner_sid_ptr, *ngroup_sid_ptr; + int rc; /* copy security descriptor control portion */ pnntsd->revision = pntsd->revision; @@ -533,28 +600,34 @@ static __u32 copy_sec_desc(const struct smb_ntsd *pntsd, pnntsd->gsidoffset = cpu_to_le32(sidsoffset + sizeof(struct smb_sid)); /* copy owner sid */ - if (pownersid) + if (pownersid) { owner_sid_ptr = pownersid; - else - owner_sid_ptr = (struct smb_sid *)((char *)pntsd + - le32_to_cpu(pntsd->osidoffset)); + } else { + rc = sid_from_sd(pntsd, secdesclen, + le32_to_cpu(pntsd->osidoffset), &owner_sid_ptr); + if (rc) + return rc; + } nowner_sid_ptr = (struct smb_sid *)((char *)pnntsd + sidsoffset); cifs_copy_sid(nowner_sid_ptr, owner_sid_ptr); /* copy group sid */ - if (pgrpsid) + if (pgrpsid) { group_sid_ptr = pgrpsid; - else - group_sid_ptr = (struct smb_sid *)((char *)pntsd + - le32_to_cpu(pntsd->gsidoffset)); + } else { + rc = sid_from_sd(pntsd, secdesclen, + le32_to_cpu(pntsd->gsidoffset), &group_sid_ptr); + if (rc) + return rc; + } ngroup_sid_ptr = (struct smb_sid *)((char *)pnntsd + sidsoffset + sizeof(struct smb_sid)); cifs_copy_sid(ngroup_sid_ptr, group_sid_ptr); - return sidsoffset + (2 * sizeof(struct smb_sid)); + *pnsecdesclen = sidsoffset + (2 * sizeof(struct smb_sid)); + return 0; } - /* change posix mode to reflect permissions pmode is the existing mode (we only want to overwrite part of this @@ -889,7 +962,7 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, */ fattr->cf_mode &= ~07777; fattr->cf_mode |= - le32_to_cpu(ppace[i]->sid.sub_auth[2]); + le32_to_cpu(ppace[i]->sid.sub_auth[2]) & 07777; break; } else { if (compare_sids(&(ppace[i]->sid), pownersid) == 0) { @@ -1112,7 +1185,8 @@ set_size: static __u16 replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *pndacl, struct smb_sid *pownersid, struct smb_sid *pgrpsid, - struct smb_sid *pnownersid, struct smb_sid *pngrpsid) + struct smb_sid *pnownersid, struct smb_sid *pngrpsid, + int *aclflag) { int i; u16 size = 0; @@ -1136,12 +1210,15 @@ static __u16 replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *p pntace = (struct smb_ace *) (acl_base + size); pnntace = (struct smb_ace *) (nacl_base + nsize); - if (pnownersid && compare_sids(&pntace->sid, pownersid) == 0) + if (pnownersid && compare_sids(&pntace->sid, pownersid) == 0) { ace_size = cifs_copy_ace(pnntace, pntace, pnownersid); - else if (pngrpsid && compare_sids(&pntace->sid, pgrpsid) == 0) + *aclflag |= CIFS_ACL_DACL; + } else if (pngrpsid && compare_sids(&pntace->sid, pgrpsid) == 0) { ace_size = cifs_copy_ace(pnntace, pntace, pngrpsid); - else + *aclflag |= CIFS_ACL_DACL; + } else { ace_size = cifs_copy_ace(pnntace, pntace, NULL); + } size += le16_to_cpu(pntace->size); nsize += ace_size; @@ -1232,36 +1309,15 @@ finalize_dacl: return 0; } -static int parse_sid(struct smb_sid *psid, char *end_of_acl) +static bool dacl_offset_valid(unsigned int acl_len, __u32 dacloffset) { - /* BB need to add parm so we can store the SID BB */ - - /* validate that we do not go past end of ACL - sid must be at least 8 - bytes long (assuming no sub-auths - e.g. the null SID */ - if (end_of_acl < (char *)psid + 8) { - cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid); - return -EINVAL; - } - -#ifdef CONFIG_CIFS_DEBUG2 - if (psid->num_subauth) { - int i; - cifs_dbg(FYI, "SID revision %d num_auth %d\n", - psid->revision, psid->num_subauth); - - for (i = 0; i < psid->num_subauth; i++) { - cifs_dbg(FYI, "SID sub_auth[%d]: 0x%x\n", - i, le32_to_cpu(psid->sub_auth[i])); - } + if (acl_len < sizeof(struct smb_acl)) + return false; - /* BB add length check to make sure that we do not have huge - num auths and therefore go off the end */ - cifs_dbg(FYI, "RID 0x%x\n", - le32_to_cpu(psid->sub_auth[psid->num_subauth-1])); - } -#endif + if (dacloffset < sizeof(struct smb_ntsd)) + return false; - return 0; + return dacloffset <= acl_len - sizeof(struct smb_acl); } @@ -1273,24 +1329,25 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, int rc = 0; struct smb_sid *owner_sid_ptr, *group_sid_ptr; struct smb_acl *dacl_ptr; /* no need for SACL ptr */ - char *end_of_acl = ((char *)pntsd) + acl_len; - __u32 dacloffset; + char *end_of_acl; + __u32 dacloffset, osidoffset, gsidoffset; if (pntsd == NULL) return smb_EIO(smb_eio_trace_null_pointers); + if (acl_len < (int)sizeof(struct smb_ntsd)) { + cifs_dbg(VFS, "ACL too small to parse security descriptor\n"); + return -EINVAL; + } + end_of_acl = ((char *)pntsd) + acl_len; - owner_sid_ptr = (struct smb_sid *)((char *)pntsd + - le32_to_cpu(pntsd->osidoffset)); - group_sid_ptr = (struct smb_sid *)((char *)pntsd + - le32_to_cpu(pntsd->gsidoffset)); + osidoffset = le32_to_cpu(pntsd->osidoffset); + gsidoffset = le32_to_cpu(pntsd->gsidoffset); dacloffset = le32_to_cpu(pntsd->dacloffset); - dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n", - pntsd->revision, pntsd->type, le32_to_cpu(pntsd->osidoffset), - le32_to_cpu(pntsd->gsidoffset), + pntsd->revision, pntsd->type, osidoffset, gsidoffset, le32_to_cpu(pntsd->sacloffset), dacloffset); /* cifs_dump_mem("owner_sid: ", owner_sid_ptr, 64); */ - rc = parse_sid(owner_sid_ptr, end_of_acl); + rc = sid_from_sd(pntsd, acl_len, osidoffset, &owner_sid_ptr); if (rc) { cifs_dbg(FYI, "%s: Error %d parsing Owner SID\n", __func__, rc); return rc; @@ -1302,9 +1359,9 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, return rc; } - rc = parse_sid(group_sid_ptr, end_of_acl); + rc = sid_from_sd(pntsd, acl_len, gsidoffset, &group_sid_ptr); if (rc) { - cifs_dbg(FYI, "%s: Error %d mapping Owner SID to gid\n", + cifs_dbg(FYI, "%s: Error %d parsing Group SID\n", __func__, rc); return rc; } @@ -1315,11 +1372,18 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, return rc; } - if (dacloffset) + if (dacloffset) { + if (!dacl_offset_valid(acl_len, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); parse_dacl(dacl_ptr, end_of_acl, owner_sid_ptr, group_sid_ptr, fattr, get_mode_from_special_sid); - else + } else { cifs_dbg(FYI, "no ACL\n"); /* BB grant all or default perms? */ + } return rc; } @@ -1337,21 +1401,41 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, struct smb_sid *nowner_sid_ptr = NULL, *ngroup_sid_ptr = NULL; struct smb_acl *dacl_ptr = NULL; /* no need for SACL ptr */ struct smb_acl *ndacl_ptr = NULL; /* no need for SACL ptr */ - char *end_of_acl = ((char *)pntsd) + secdesclen; + char *end_of_acl; u16 size = 0; + __u32 osidoffset, gsidoffset; + + if (secdesclen < sizeof(struct smb_ntsd)) { + cifs_dbg(VFS, "ACL too small to parse security descriptor\n"); + return -EINVAL; + } + end_of_acl = ((char *)pntsd) + secdesclen; dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); rc = validate_dacl(dacl_ptr, end_of_acl); if (rc) return rc; } - owner_sid_ptr = (struct smb_sid *)((char *)pntsd + - le32_to_cpu(pntsd->osidoffset)); - group_sid_ptr = (struct smb_sid *)((char *)pntsd + - le32_to_cpu(pntsd->gsidoffset)); + osidoffset = le32_to_cpu(pntsd->osidoffset); + gsidoffset = le32_to_cpu(pntsd->gsidoffset); + rc = sid_from_sd(pntsd, secdesclen, osidoffset, &owner_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Error %d parsing Owner SID\n", __func__, rc); + return rc; + } + rc = sid_from_sd(pntsd, secdesclen, gsidoffset, &group_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Error %d parsing Group SID\n", __func__, rc); + return rc; + } if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */ ndacloffset = sizeof(struct smb_ntsd); @@ -1367,8 +1451,10 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); /* copy the non-dacl portion of secdesc */ - *pnsecdesclen = copy_sec_desc(pntsd, pnntsd, sidsoffset, - NULL, NULL); + rc = copy_sec_desc(pntsd, pnntsd, sidsoffset, secdesclen, + pnsecdesclen, NULL, NULL); + if (rc) + return rc; *aclflag |= CIFS_ACL_DACL; } else { @@ -1439,14 +1525,17 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, /* Replace ACEs for old owner with new one */ size = replace_sids_and_copy_aces(dacl_ptr, ndacl_ptr, owner_sid_ptr, group_sid_ptr, - nowner_sid_ptr, ngroup_sid_ptr); + nowner_sid_ptr, ngroup_sid_ptr, + aclflag); ndacl_ptr->size = cpu_to_le16(size); } sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); /* copy the non-dacl portion of secdesc */ - *pnsecdesclen = copy_sec_desc(pntsd, pnntsd, sidsoffset, - nowner_sid_ptr, ngroup_sid_ptr); + rc = copy_sec_desc(pntsd, pnntsd, sidsoffset, secdesclen, + pnsecdesclen, nowner_sid_ptr, ngroup_sid_ptr); + if (rc) + goto chown_chgrp_exit; chown_chgrp_exit: /* errors could jump here. So make sure we return soon after this */ @@ -1654,7 +1743,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, kuid_t uid, kgid_t gid) { int rc = 0; - int aclflag = CIFS_ACL_DACL; /* default flag to set */ + int aclflag = 0; __u32 secdesclen = 0; __u32 nsecdesclen = 0; __u32 dacloffset = 0; @@ -1710,6 +1799,12 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, nsecdesclen = sizeof(struct smb_ntsd) + (sizeof(struct smb_sid) * 2); dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + rc = -EINVAL; + goto id_mode_to_cifs_acl_exit; + } + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); rc = validate_dacl(dacl_ptr, (char *)pntsd + secdesclen); if (rc) { @@ -1732,7 +1827,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, * descriptor parameters, and security descriptor itself */ nsecdesclen = max_t(u32, nsecdesclen, DEFAULT_SEC_DESC_LEN); - pnntsd = kmalloc(nsecdesclen, GFP_KERNEL); + pnntsd = kzalloc(nsecdesclen, GFP_KERNEL); if (!pnntsd) { kfree(pntsd); cifs_put_tlink(tlink); @@ -1744,14 +1839,24 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, cifs_dbg(NOISY, "build_sec_desc rc: %d\n", rc); - if (ops->set_acl == NULL) - rc = -EOPNOTSUPP; + if (rc != 0) + goto id_mode_to_cifs_acl_exit; - if (!rc) { - /* Set the security descriptor */ - rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); - cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); + if (aclflag == 0) { + cifs_dbg(FYI, "set_cifs_acl aclflag=0, no change mapped\n"); + goto id_mode_to_cifs_acl_exit; } + + if (ops->set_acl == NULL) { + rc = -EOPNOTSUPP; + goto id_mode_to_cifs_acl_exit; + } + + /* Set the security descriptor */ + rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); + cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); + +id_mode_to_cifs_acl_exit: cifs_put_tlink(tlink); kfree(pnntsd); diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index 9f76b0347fa9..66b9104e7ca2 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -12,6 +12,7 @@ #include <linux/module.h> #include <linux/fs.h> +#include <linux/fs_context.h> #include <linux/filelock.h> #include <linux/mount.h> #include <linux/slab.h> @@ -30,6 +31,7 @@ #include <linux/xattr.h> #include <linux/mm.h> #include <linux/key-type.h> +#include <linux/fileattr.h> #include <uapi/linux/magic.h> #include <net/ipv6.h> #include "cifsfs.h" @@ -306,6 +308,20 @@ static void cifs_kill_sb(struct super_block *sb) /* Wait for all pending oplock breaks to complete */ flush_workqueue(cifsoplockd_wq); + /* Wait for all opened files to release */ + flush_workqueue(deferredclose_wq); + + /* + * Wait for all in-flight netfs I/O requests to finish their + * cleanup_work so that any cifsFileInfo final puts they queue + * to fileinfo_put_wq/serverclose_wq have been queued, then + * drain the workqueue so the cfile dentry refs are dropped to + * avoid the busy dentry warning. + */ + wait_var_event(&cifs_sb->outstanding_rreq, + !atomic_read(&cifs_sb->outstanding_rreq)); + flush_workqueue(serverclose_wq); + flush_workqueue(fileinfo_put_wq); /* finally release root dentry */ dput(cifs_sb->root); @@ -434,7 +450,8 @@ cifs_alloc_inode(struct super_block *sb) spin_lock_init(&cifs_inode->writers_lock); cifs_inode->writers = 0; cifs_inode->netfs.inode.i_blkbits = 14; /* 2**14 = CIFS_MAX_MSGSIZE */ - cifs_inode->netfs.remote_i_size = 0; + cifs_inode->netfs._remote_i_size = 0; + cifs_inode->netfs._zero_point = 0; cifs_inode->uniqueid = 0; cifs_inode->createtime = 0; cifs_inode->epoch = 0; @@ -966,26 +983,19 @@ cifs_get_root(struct smb3_fs_context *ctx, struct super_block *sb) return dentry; } -static int cifs_set_super(struct super_block *sb, void *data) -{ - struct cifs_mnt_data *mnt_data = data; - sb->s_fs_info = mnt_data->cifs_sb; - return set_anon_super(sb, NULL); -} - struct dentry * -cifs_smb3_do_mount(struct file_system_type *fs_type, - int flags, struct smb3_fs_context *old_ctx) +cifs_smb3_do_mount(struct fs_context *fc, struct smb3_fs_context *old_ctx) { struct cifs_mnt_data mnt_data; struct cifs_sb_info *cifs_sb; struct super_block *sb; struct dentry *root; + unsigned int saved_sb_flags; int rc; if (cifsFYI) { - cifs_dbg(FYI, "%s: devname=%s flags=0x%x\n", __func__, - old_ctx->source, flags); + cifs_dbg(FYI, "%s: devname=%s sb_flags=0x%x\n", __func__, + old_ctx->source, fc->sb_flags); } else { cifs_info("Attempting to mount %s\n", old_ctx->source); } @@ -1012,7 +1022,7 @@ cifs_smb3_do_mount(struct file_system_type *fs_type, rc = cifs_mount(cifs_sb, cifs_sb->ctx); if (rc) { - if (!(flags & SB_SILENT)) + if (!(fc->sb_flags & SB_SILENT)) cifs_dbg(VFS, "cifs_mount failed w/return code = %d\n", rc); root = ERR_PTR(rc); @@ -1021,12 +1031,27 @@ cifs_smb3_do_mount(struct file_system_type *fs_type, mnt_data.ctx = cifs_sb->ctx; mnt_data.cifs_sb = cifs_sb; - mnt_data.flags = flags; - - /* BB should we make this contingent on mount parm? */ - flags |= SB_NODIRATIME | SB_NOATIME; + mnt_data.flags = 0; - sb = sget(fs_type, cifs_match_super, cifs_set_super, flags, &mnt_data); + /* + * sb->s_flags is set from fc->sb_flags by alloc_super(). CIFS has + * historically forced SB_NODIRATIME | SB_NOATIME on every mount and + * ignored the caller-supplied SB_* flags. Preserve that behaviour by + * overriding fc->sb_flags around the sget_fc() call. + * + * Hand cifs_sb to sget_fc() via fc->s_fs_info; sget_fc() copies it + * onto sb->s_fs_info before running set() and clears fc->s_fs_info + * on successful publish. Pass the rest of the per-mount context to + * cifs_match_super() through fc->sget_key. + */ + saved_sb_flags = fc->sb_flags; + fc->sb_flags = SB_NODIRATIME | SB_NOATIME; + fc->s_fs_info = cifs_sb; + fc->sget_key = &mnt_data; + sb = sget_fc(fc, cifs_match_super, set_anon_super_fc); + fc->sget_key = NULL; + fc->s_fs_info = NULL; + fc->sb_flags = saved_sb_flags; if (IS_ERR(sb)) { cifs_umount(cifs_sb); return ERR_CAST(sb); @@ -1162,6 +1187,56 @@ struct file_system_type smb3_fs_type = { MODULE_ALIAS_FS("smb3"); MODULE_ALIAS("smb3"); +int cifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(dentry->d_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct inode *inode = d_inode(dentry); + u32 attrs; + + /* Preserve FS_COMPR_FL previously reported by cifs_ioctl(). */ + if (CIFS_I(inode)->cifsAttrs & ATTR_COMPRESSED) + fa->flags |= FS_COMPR_FL; + + /* + * FS_CASEFOLD_FL is defined by UAPI as a folder attribute, + * and userspace tools (e.g., lsattr) display it only on + * directories. Confine the case-handling bits to directories + * to match that convention; for non-directories the share's + * case semantics are still discoverable through the parent. + */ + if (!S_ISDIR(inode->i_mode)) + return 0; + + /* + * The server's FS_ATTRIBUTE_INFORMATION response, cached on + * the tcon at mount, reflects the share's case-handling + * semantics after any POSIX extensions negotiation. Prefer + * it over the client-local nocase mount option, which only + * governs dentry comparison on this superblock. + * + * QueryFSInfo is best-effort at mount; when it did not + * populate fsAttrInfo, MaxPathNameComponentLength remains + * zero. In that case fall back to nocase so the reporting + * matches the comparison behavior installed on the sb. + */ + if (le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength) == 0) { + if (tcon->nocase) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + return 0; + } + attrs = le32_to_cpu(tcon->fsAttrInfo.Attributes); + if (!(attrs & FILE_CASE_SENSITIVE_SEARCH)) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + if (!(attrs & FILE_CASE_PRESERVED_NAMES)) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + return 0; +} + const struct inode_operations cifs_dir_inode_ops = { .create = cifs_create, .atomic_open = cifs_atomic_open, @@ -1180,6 +1255,7 @@ const struct inode_operations cifs_dir_inode_ops = { .listxattr = cifs_listxattr, .get_acl = cifs_get_acl, .set_acl = cifs_set_acl, + .fileattr_get = cifs_fileattr_get, }; const struct inode_operations cifs_file_inode_ops = { @@ -1190,6 +1266,7 @@ const struct inode_operations cifs_file_inode_ops = { .fiemap = cifs_fiemap, .get_acl = cifs_get_acl, .set_acl = cifs_set_acl, + .fileattr_get = cifs_fileattr_get, }; const char *cifs_get_link(struct dentry *dentry, struct inode *inode, @@ -1303,7 +1380,8 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, struct cifsFileInfo *smb_file_src = src_file->private_data; struct cifsFileInfo *smb_file_target = dst_file->private_data; struct cifs_tcon *target_tcon, *src_tcon; - unsigned long long destend, fstart, fend, old_size, new_size; + unsigned long long i_size, new_size; + unsigned long long destend, fstart, fend; unsigned int xid; int rc; @@ -1347,7 +1425,7 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, * Advance the EOF marker after the flush above to the end of the range * if it's short of that. */ - if (src_cifsi->netfs.remote_i_size < off + len) { + if (netfs_read_remote_i_size(src_inode) < off + len) { rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len); if (rc < 0) goto unlock; @@ -1368,22 +1446,24 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, rc = cifs_flush_folio(target_inode, destend, &fstart, &fend, false); if (rc) goto unlock; - if (fend > target_cifsi->netfs.zero_point) - target_cifsi->netfs.zero_point = fend + 1; - old_size = target_cifsi->netfs.remote_i_size; + + spin_lock(&target_inode->i_lock); + if (fend > target_cifsi->netfs._zero_point) + netfs_write_zero_point(target_inode, fend + 1); + i_size = target_inode->i_size; + spin_unlock(&target_inode->i_lock); /* Discard all the folios that overlap the destination region. */ cifs_dbg(FYI, "about to discard pages %llx-%llx\n", fstart, fend); truncate_inode_pages_range(&target_inode->i_data, fstart, fend); - fscache_invalidate(cifs_inode_cookie(target_inode), NULL, - i_size_read(target_inode), 0); + fscache_invalidate(cifs_inode_cookie(target_inode), NULL, i_size, 0); rc = -EOPNOTSUPP; if (target_tcon->ses->server->ops->duplicate_extents) { rc = target_tcon->ses->server->ops->duplicate_extents(xid, smb_file_src, smb_file_target, off, len, destoff); - if (rc == 0 && new_size > old_size) { + if (rc == 0 && new_size > i_size) { truncate_setsize(target_inode, new_size); fscache_resize_cookie(cifs_inode_cookie(target_inode), new_size); @@ -1402,13 +1482,21 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, rc = -EINVAL; } } - if (rc == 0 && new_size > target_cifsi->netfs.zero_point) - target_cifsi->netfs.zero_point = new_size; + if (rc == 0) { + spin_lock(&target_inode->i_lock); + if (new_size > target_cifsi->netfs._zero_point) + netfs_write_zero_point(target_inode, new_size); + spin_unlock(&target_inode->i_lock); + } } - /* force revalidate of size and timestamps of target file now - that target is updated on the server */ - CIFS_I(target_inode)->time = 0; + /* + * On success, duplicate_extents already updated the target inode attrs + * or marked them stale if the refresh failed. On failure, mark attrs + * stale because EOF may have changed before the clone failed. + */ + if (rc) + CIFS_I(target_inode)->time = 0; unlock: /* although unlocking in the reverse order from locking is not strictly necessary here it is a little cleaner to be consistent */ @@ -1474,7 +1562,7 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, * Advance the EOF marker after the flush above to the end of the range * if it's short of that. */ - if (src_cifsi->netfs.remote_i_size < off + len) { + if (netfs_read_remote_i_size(src_inode) < off + len) { rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len); if (rc < 0) goto unlock; @@ -1502,8 +1590,12 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, fscache_resize_cookie(cifs_inode_cookie(target_inode), i_size_read(target_inode)); } - if (rc > 0 && destoff + rc > target_cifsi->netfs.zero_point) - target_cifsi->netfs.zero_point = destoff + rc; + if (rc > 0) { + spin_lock(&target_inode->i_lock); + if (destoff + rc > target_cifsi->netfs._zero_point) + netfs_write_zero_point(target_inode, destoff + rc); + spin_unlock(&target_inode->i_lock); + } } file_accessed(src_file); @@ -1601,7 +1693,7 @@ const struct file_operations cifs_file_strict_ops = { const struct file_operations cifs_file_direct_ops = { .read_iter = netfs_unbuffered_read_iter, - .write_iter = netfs_file_write_iter, + .write_iter = cifs_direct_write_iter, .open = cifs_open, .release = cifs_close, .lock = cifs_lock, @@ -1657,7 +1749,7 @@ const struct file_operations cifs_file_strict_nobrl_ops = { const struct file_operations cifs_file_direct_nobrl_ops = { .read_iter = netfs_unbuffered_read_iter, - .write_iter = netfs_file_write_iter, + .write_iter = cifs_direct_write_iter, .open = cifs_open, .release = cifs_close, .fsync = cifs_fsync, diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h index c455b15f2778..854e672a4e37 100644 --- a/fs/smb/client/cifsfs.h +++ b/fs/smb/client/cifsfs.h @@ -89,6 +89,9 @@ extern const struct inode_operations cifs_file_inode_ops; extern const struct inode_operations cifs_symlink_inode_ops; extern const struct inode_operations cifs_namespace_inode_operations; +struct file_kattr; +int cifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + /* Functions related to files and directories */ extern const struct netfs_request_ops cifs_req_ops; @@ -104,6 +107,7 @@ int cifs_closedir(struct inode *inode, struct file *file); ssize_t cifs_strict_readv(struct kiocb *iocb, struct iov_iter *to); ssize_t cifs_strict_writev(struct kiocb *iocb, struct iov_iter *from); ssize_t cifs_file_write_iter(struct kiocb *iocb, struct iov_iter *from); +ssize_t cifs_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); ssize_t cifs_loose_read_iter(struct kiocb *iocb, struct iov_iter *iter); int cifs_flock(struct file *file, int cmd, struct file_lock *fl); int cifs_lock(struct file *file, int cmd, struct file_lock *flock); @@ -144,8 +148,9 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, struct file *src_file, long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg); void cifs_setsize(struct inode *inode, loff_t offset); +struct fs_context; struct smb3_fs_context; -struct dentry *cifs_smb3_do_mount(struct file_system_type *fs_type, int flags, +struct dentry *cifs_smb3_do_mount(struct fs_context *fc, struct smb3_fs_context *old_ctx); char *cifs_silly_fullpath(struct dentry *dentry); @@ -161,6 +166,6 @@ extern const struct export_operations cifs_export_ops; #endif /* CONFIG_CIFS_NFSD_EXPORT */ /* when changing internal version - update following two lines at same time */ -#define SMB3_PRODUCT_BUILD 60 -#define CIFS_VERSION "2.60" +#define SMB3_PRODUCT_BUILD 61 +#define CIFS_VERSION "2.61" #endif /* _CIFSFS_H */ diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h index 82e0adc1dabd..08e94633a9c1 100644 --- a/fs/smb/client/cifsglob.h +++ b/fs/smb/client/cifsglob.h @@ -250,6 +250,7 @@ struct cifs_open_info_data { bool adjust_tz; bool reparse_point; bool contains_posix_file_info; + bool unknown_nlink; struct { /* ioctl response buffer */ struct { @@ -425,7 +426,7 @@ struct smb_version_operations { int (*set_file_info)(struct inode *, const char *, FILE_BASIC_INFO *, const unsigned int); int (*set_compression)(const unsigned int, struct cifs_tcon *, - struct cifsFileInfo *); + struct cifsFileInfo *, __u16); /* check if we can send an echo or nor */ bool (*can_echo)(struct TCP_Server_Info *); /* send echo request */ @@ -789,6 +790,8 @@ struct TCP_Server_Info { struct { bool requested; /* "compress" mount option set*/ bool enabled; /* actually negotiated with server */ + bool chained; /* chained transforms were negotiated */ + bool pattern; /* Pattern_V1 chained payloads were negotiated */ __le16 alg; /* preferred alg negotiated with server */ } compression; __u16 signing_algorithm; @@ -1391,6 +1394,7 @@ struct cifs_search_info { bool emptyDir:1; bool unicode:1; bool smallBuf:1; /* so we know which buf_release function to call */ + bool is_dynamic_buf:1; /* dynamically allocated buffer - can be variable size */ }; #define ACL_NO_MODE ((umode_t)(-1)) @@ -1904,6 +1908,7 @@ enum cifs_find_flags { #define CIFS_NO_BUFFER 0 /* Response buffer not returned */ #define CIFS_SMALL_BUFFER 1 #define CIFS_LARGE_BUFFER 2 +#define CIFS_DYNAMIC_BUFFER 3 /* Dynamically allocated buffer */ #define CIFS_IOVEC 4 /* array of response buffers */ /* Type of Request to SendReceive2 */ @@ -2387,9 +2392,12 @@ static inline int cifs_open_create_options(unsigned int oflags, int opts) } /* - * The number of blocks is not related to (i_size / i_blksize), but instead - * 512 byte (2**9) size is required for calculating num blocks. + * inode->i_blocks is counted in 512-byte units, independent of + * inode->i_blksize. */ -#define CIFS_INO_BLOCKS(size) DIV_ROUND_UP_ULL((u64)(size), 512) +#define CIFS_INO_BLOCK_SIZE 512ULL +#define CIFS_INO_BLOCKS(size) \ + DIV_ROUND_UP_ULL((u64)(size), CIFS_INO_BLOCK_SIZE) +#define CIFS_INO_BYTES(blocks) ((u64)(blocks) * CIFS_INO_BLOCK_SIZE) #endif /* _CIFS_GLOB_H */ diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h index 79d891f7df1a..00168839c123 100644 --- a/fs/smb/client/cifsproto.h +++ b/fs/smb/client/cifsproto.h @@ -19,6 +19,7 @@ struct statfs; struct smb_rqst; struct smb3_fs_context; +struct fs_context; /* ***************************************************************** @@ -235,7 +236,7 @@ void cifs_mount_put_conns(struct cifs_mount_ctx *mnt_ctx); int cifs_mount_get_session(struct cifs_mount_ctx *mnt_ctx); int cifs_is_path_remote(struct cifs_mount_ctx *mnt_ctx); int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx); -int cifs_match_super(struct super_block *sb, void *data); +int cifs_match_super(struct super_block *sb, struct fs_context *fc); int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx); void cifs_umount(struct cifs_sb_info *cifs_sb); void cifs_mark_open_files_invalid(struct cifs_tcon *tcon); @@ -316,7 +317,7 @@ int generate_smb311signingkey(struct cifs_ses *ses, #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ -void cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb); +void cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb, const char *reason, int rc); bool couldbe_mf_symlink(const struct cifs_fattr *fattr); int check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, struct cifs_fattr *fattr, diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c index 3990a9012264..40162d5554ea 100644 --- a/fs/smb/client/cifssmb.c +++ b/fs/smb/client/cifssmb.c @@ -1465,6 +1465,7 @@ cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) struct cifs_io_subrequest *rdata = mid->callback_data; struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode); struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink); + struct inode *inode = &ictx->inode; struct smb_rqst rqst = { .rq_iov = rdata->iov, .rq_nvec = 1, .rq_iter = rdata->subreq.io_iter }; @@ -1538,7 +1539,7 @@ do_retry: } else { size_t trans = rdata->subreq.transferred + rdata->got_bytes; if (trans < rdata->subreq.len && - rdata->subreq.start + trans >= ictx->remote_i_size) { + rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) { rdata->result = 0; __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); } else if (rdata->got_bytes > 0) { @@ -3206,7 +3207,7 @@ out_close: int CIFSSMB_set_compression(const unsigned int xid, struct cifs_tcon *tcon, - __u16 fid) + __u16 fid, __u16 compression_state) { int rc = 0; int bytes_returned; @@ -3221,7 +3222,7 @@ CIFSSMB_set_compression(const unsigned int xid, struct cifs_tcon *tcon, return rc; in_len = rc; - pSMB->compression_state = cpu_to_le16(COMPRESSION_FORMAT_DEFAULT); + pSMB->compression_state = cpu_to_le16(compression_state); pSMB->TotalParameterCount = 0; pSMB->TotalDataCount = cpu_to_le32(2); @@ -5828,38 +5829,6 @@ CIFSSMBSetFileDisposition(const unsigned int xid, struct cifs_tcon *tcon, return rc; } -static int -CIFSSMBSetPathInfoFB(const unsigned int xid, struct cifs_tcon *tcon, - const char *fileName, const FILE_BASIC_INFO *data, - const struct nls_table *nls_codepage, - struct cifs_sb_info *cifs_sb) -{ - int oplock = 0; - struct cifs_open_parms oparms; - struct cifs_fid fid; - int rc; - - oparms = (struct cifs_open_parms) { - .tcon = tcon, - .cifs_sb = cifs_sb, - .desired_access = GENERIC_WRITE, - .create_options = cifs_create_options(cifs_sb, 0), - .disposition = FILE_OPEN, - .path = fileName, - .fid = &fid, - }; - - rc = CIFS_open(xid, &oparms, &oplock, NULL); - if (rc) - goto out; - - rc = CIFSSMBSetFileInfo(xid, tcon, data, fid.netfid, current->tgid); - CIFSSMBClose(xid, tcon, fid.netfid); -out: - - return rc; -} - int CIFSSMBSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon, const char *fileName, const FILE_BASIC_INFO *data, @@ -5938,10 +5907,6 @@ SetTimesRetry: if (rc == -EAGAIN) goto SetTimesRetry; - if (rc == -EOPNOTSUPP) - return CIFSSMBSetPathInfoFB(xid, tcon, fileName, data, - nls_codepage, cifs_sb); - return rc; } diff --git a/fs/smb/client/compress.c b/fs/smb/client/compress.c index be9023f841e6..8f0860970741 100644 --- a/fs/smb/client/compress.c +++ b/fs/smb/client/compress.c @@ -22,7 +22,7 @@ #include "cifsproto.h" #include "smb2proto.h" -#include "compress/lz77.h" +#include "../common/compress/lz77.h" #include "compress.h" /* @@ -44,6 +44,11 @@ struct bucket { unsigned int count; }; +static inline size_t pow4(size_t n) +{ + return n * n * n * n; +} + /* * has_low_entropy() - Compute Shannon entropy of the sampled data. * @bkt: Bytes counts of the sample. @@ -65,7 +70,6 @@ static bool has_low_entropy(struct bucket *bkt, size_t slen) const size_t threshold = 65, max_entropy = 8 * ilog2(16); size_t i, p, p2, len, sum = 0; -#define pow4(n) (n * n * n * n) len = ilog2(pow4(slen)); for (i = 0; i < 256 && bkt[i].count > 0; i++) { @@ -329,14 +333,14 @@ int smb_compress(struct TCP_Server_Info *server, struct smb_rqst *rq, compress_s goto err_free; } - dlen = lz77_compressed_alloc_size(slen); + dlen = smb_lz77_compressed_alloc_size(slen); dst = kvzalloc(dlen, GFP_KERNEL); if (!dst) { ret = -ENOMEM; goto err_free; } - ret = lz77_compress(src, slen, dst, &dlen); + ret = smb_lz77_compress(src, slen, dst, &dlen); if (!ret) { struct smb2_compression_hdr hdr = { 0 }; struct smb_rqst comp_rq = { .rq_nvec = 3, }; diff --git a/fs/smb/client/compress.h b/fs/smb/client/compress.h index 2679baca129b..e08e6d339d21 100644 --- a/fs/smb/client/compress.h +++ b/fs/smb/client/compress.h @@ -18,6 +18,7 @@ #include <linux/uio.h> #include <linux/kernel.h> #include "../common/smb2pdu.h" +#include "../common/compress/compress.h" #include "cifsglob.h" /* sizeof(smb2_compression_hdr) - sizeof(OriginalPayloadSize) */ @@ -34,29 +35,6 @@ int smb_compress(struct TCP_Server_Info *server, struct smb_rqst *rq, compress_send_fn send_fn); bool should_compress(const struct cifs_tcon *tcon, const struct smb_rqst *rq); -/* - * smb_compress_alg_valid() - Validate a compression algorithm. - * @alg: Compression algorithm to check. - * @valid_none: Conditional check whether NONE algorithm should be - * considered valid or not. - * - * If @alg is SMB3_COMPRESS_NONE, this function returns @valid_none. - * - * Note that 'NONE' (0) compressor type is considered invalid in protocol - * negotiation, as it's never requested to/returned from the server. - * - * Return: true if @alg is valid/supported, false otherwise. - */ -static __always_inline int smb_compress_alg_valid(__le16 alg, bool valid_none) -{ - if (alg == SMB3_COMPRESS_NONE) - return valid_none; - - if (alg == SMB3_COMPRESS_LZ77 || alg == SMB3_COMPRESS_PATTERN) - return true; - - return false; -} #else /* !CONFIG_CIFS_COMPRESSION */ static inline int smb_compress(void *unused1, void *unused2, void *unused3) { @@ -68,9 +46,5 @@ static inline bool should_compress(void *unused1, void *unused2) return false; } -static inline int smb_compress_alg_valid(__le16 unused1, bool unused2) -{ - return -EOPNOTSUPP; -} #endif /* !CONFIG_CIFS_COMPRESSION */ #endif /* _SMB_COMPRESS_H */ diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index dcde25da468d..ba749ec25a59 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -6,6 +6,7 @@ * */ #include <linux/fs.h> +#include <linux/fs_context.h> #include <linux/net.h> #include <linux/string.h> #include <linux/sched/mm.h> @@ -456,8 +457,8 @@ static int __reconnect_target_locked(struct TCP_Server_Info *server, server->hostname = hostname; spin_unlock(&server->srv_lock); } else { - cifs_dbg(FYI, "%s: couldn't extract hostname or address from dfs target: %ld\n", - __func__, PTR_ERR(hostname)); + cifs_dbg(FYI, "%s: couldn't extract hostname or address from dfs target: %pe\n", + __func__, hostname); cifs_dbg(FYI, "%s: default to last target server: %s\n", __func__, server->hostname); } @@ -1871,14 +1872,6 @@ smbd_connected: * this will succeed. No need for try_module_get(). */ __module_get(THIS_MODULE); - tcp_ses->tsk = kthread_run(cifs_demultiplex_thread, - tcp_ses, "cifsd"); - if (IS_ERR(tcp_ses->tsk)) { - rc = PTR_ERR(tcp_ses->tsk); - cifs_dbg(VFS, "error %d create cifsd thread\n", rc); - module_put(THIS_MODULE); - goto out_err_crypto_release; - } tcp_ses->min_offload = ctx->min_offload; tcp_ses->retrans = ctx->retrans; /* @@ -1886,9 +1879,7 @@ smbd_connected: * to the struct since the kernel thread not created yet * no need to spinlock this update of tcpStatus */ - spin_lock(&tcp_ses->srv_lock); tcp_ses->tcpStatus = CifsNeedNegotiate; - spin_unlock(&tcp_ses->srv_lock); if ((ctx->max_credits < 20) || (ctx->max_credits > 60000)) tcp_ses->max_credits = SMB2_MAX_CREDITS_AVAILABLE; @@ -1897,7 +1888,16 @@ smbd_connected: tcp_ses->nr_targets = 1; tcp_ses->ignore_signature = ctx->ignore_signature; - /* thread spawned, put it on the list */ + + tcp_ses->tsk = kthread_create(cifs_demultiplex_thread, + tcp_ses, "cifsd"); + if (IS_ERR(tcp_ses->tsk)) { + rc = PTR_ERR(tcp_ses->tsk); + cifs_dbg(VFS, "error %d create cifsd thread\n", rc); + module_put(THIS_MODULE); + goto out_err_crypto_release; + } + /* thread created, put it on the list */ spin_lock(&cifs_tcp_ses_lock); list_add(&tcp_ses->tcp_ses_list, &cifs_tcp_ses_list); spin_unlock(&cifs_tcp_ses_lock); @@ -1905,6 +1905,12 @@ smbd_connected: /* queue echo request delayed work */ queue_delayed_work(cifsiod_wq, &tcp_ses->echo, tcp_ses->echo_interval); + /* + * Use split create/wake logic to ensure that tcp_ses is fully populated + * and tcp_ses->tsk is valid + */ + wake_up_process(tcp_ses->tsk); + return tcp_ses; out_err_crypto_release: @@ -2991,9 +2997,9 @@ static int match_prepath(struct super_block *sb, } int -cifs_match_super(struct super_block *sb, void *data) +cifs_match_super(struct super_block *sb, struct fs_context *fc) { - struct cifs_mnt_data *mnt_data = data; + struct cifs_mnt_data *mnt_data = fc->sget_key; struct smb3_fs_context *ctx; struct cifs_sb_info *cifs_sb; struct TCP_Server_Info *tcp_srv; @@ -3479,6 +3485,7 @@ int cifs_setup_cifs_sb(struct cifs_sb_info *cifs_sb) spin_lock_init(&cifs_sb->tlink_tree_lock); cifs_sb->tlink_tree = RB_ROOT; + atomic_set(&cifs_sb->outstanding_rreq, 0); cifs_dbg(FYI, "file mode: %04ho dir mode: %04ho\n", ctx->file_mode, ctx->dir_mode); @@ -3869,7 +3876,7 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) * After reconnecting to a different server, unique ids won't match anymore, so we disable * serverino. This prevents dentry revalidation to think the dentry are stale (ESTALE). */ - cifs_autodisable_serverino(cifs_sb); + cifs_autodisable_serverino(cifs_sb, "DFS failover may potentially connect to a different server, inode numbers won't match anymore", 0); /* * Force the use of prefix path to support failover on DFS paths that resolve to targets * that have different prefix paths. @@ -3996,6 +4003,9 @@ cifs_umount(struct cifs_sb_info *cifs_sb) } spin_unlock(&cifs_sb->tlink_tree_lock); + flush_workqueue(serverclose_wq); + flush_workqueue(fileinfo_put_wq); + kfree(cifs_sb->prepath); call_rcu(&cifs_sb->rcu, delayed_free); } diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c index 83f8cf2f8d2b..8cd93cd2f00f 100644 --- a/fs/smb/client/dfs_cache.c +++ b/fs/smb/client/dfs_cache.c @@ -363,10 +363,10 @@ static struct cache_dfs_tgt *alloc_target(const char *name, int path_consumed) { struct cache_dfs_tgt *t; - t = kmalloc_obj(*t, GFP_ATOMIC); + t = kmalloc_obj(*t, GFP_KERNEL); if (!t) return ERR_PTR(-ENOMEM); - t->name = kstrdup(name, GFP_ATOMIC); + t->name = kstrdup(name, GFP_KERNEL); if (!t->name) { kfree(t); return ERR_PTR(-ENOMEM); @@ -626,7 +626,7 @@ static int update_cache_entry_locked(struct cache_entry *ce, const struct dfs_in target = READ_ONCE(ce->tgthint); if (target) { - th = kstrdup(target->name, GFP_ATOMIC); + th = kstrdup(target->name, GFP_KERNEL); if (!th) return -ENOMEM; } @@ -760,11 +760,11 @@ static int setup_referral(const char *path, struct cache_entry *ce, memset(ref, 0, sizeof(*ref)); - ref->path_name = kstrdup(path, GFP_ATOMIC); + ref->path_name = kstrdup(path, GFP_KERNEL); if (!ref->path_name) return -ENOMEM; - ref->node_name = kstrdup(target, GFP_ATOMIC); + ref->node_name = kstrdup(target, GFP_KERNEL); if (!ref->node_name) { rc = -ENOMEM; goto err_free_path; @@ -1328,7 +1328,7 @@ int dfs_cache_remount_fs(struct cifs_sb_info *cifs_sb) * After reconnecting to a different server, unique ids won't match anymore, so we disable * serverino. This prevents dentry revalidation to think the dentry are stale (ESTALE). */ - cifs_autodisable_serverino(cifs_sb); + cifs_autodisable_serverino(cifs_sb, "DFS failover may potentially connect to a different server, inode numbers won't match anymore", 0); /* * Force the use of prefix path to support failover on DFS paths that resolve to targets * that have different prefix paths. diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c index e4295a5b55b3..88a4a1787ff0 100644 --- a/fs/smb/client/dir.c +++ b/fs/smb/client/dir.c @@ -241,6 +241,12 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, goto cifs_create_get_file_info; } + if ((oflags & __O_REGULAR) && !S_ISREG(newinode->i_mode)) { + CIFSSMBClose(xid, tcon, fid->netfid); + iput(newinode); + return -EFTYPE; + } + if (S_ISDIR(newinode->i_mode)) { CIFSSMBClose(xid, tcon, fid->netfid); iput(newinode); @@ -458,9 +464,15 @@ cifs_create_set_dentry: goto out_err; } - if (newinode && S_ISDIR(newinode->i_mode)) { - rc = -EISDIR; - goto out_err; + if (newinode) { + if ((oflags & __O_REGULAR) && !S_ISREG(newinode->i_mode)) { + rc = -EFTYPE; + goto out_err; + } + if (S_ISDIR(newinode->i_mode)) { + rc = -EISDIR; + goto out_err; + } } *inode = newinode; diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index 664a2c223089..968740e7c9c3 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -241,6 +241,7 @@ static void cifs_issue_read(struct netfs_io_subrequest *subreq) return; failed: + add_credits_and_wake_if(rdata->server, &rdata->credits, 0); subreq->error = rc; netfs_read_subreq_terminated(subreq); } @@ -287,6 +288,7 @@ static int cifs_init_request(struct netfs_io_request *rreq, struct file *file) return smb_EIO1(smb_eio_trace_not_netfs_writeback, rreq->origin); } + atomic_inc(&cifs_sb->outstanding_rreq); return 0; } @@ -301,16 +303,20 @@ static void cifs_rreq_done(struct netfs_io_request *rreq) /* we do not want atime to be less than mtime, it broke some apps */ atime = inode_set_atime_to_ts(inode, current_time(inode)); mtime = inode_get_mtime(inode); - if (timespec64_compare(&atime, &mtime)) + if (timespec64_compare(&atime, &mtime) < 0) inode_set_atime_to_ts(inode, inode_get_mtime(inode)); } static void cifs_free_request(struct netfs_io_request *rreq) { struct cifs_io_request *req = container_of(rreq, struct cifs_io_request, rreq); + struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); if (req->cfile) cifsFileInfo_put(req->cfile); + + if (atomic_dec_and_test(&cifs_sb->outstanding_rreq)) + wake_up_var(&cifs_sb->outstanding_rreq); } static void cifs_free_subrequest(struct netfs_io_subrequest *subreq) @@ -1563,6 +1569,8 @@ int cifs_closedir(struct inode *inode, struct file *file) cfile->srch_inf.ntwrk_buf_start = NULL; if (cfile->srch_inf.smallBuf) cifs_small_buf_release(buf); + else if (cfile->srch_inf.is_dynamic_buf) + kfree(buf); else cifs_buf_release(buf); } @@ -2514,21 +2522,64 @@ int cifs_lock(struct file *file, int cmd, struct file_lock *flock) return rc; } +static void cifs_update_i_blocks_for_write(struct inode *inode, loff_t start, + loff_t end) +{ + struct cifsInodeInfo *cinode = CIFS_I(inode); + u64 allocated_end = CIFS_INO_BYTES(inode->i_blocks); + u64 blocks; + + if (cinode->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) + return; + + /* + * Grow the local estimate only across the currently known allocated + * prefix. A write beyond that may leave a hole. + */ + if ((u64)start > allocated_end) + return; + + blocks = CIFS_INO_BLOCKS(end); + if ((u64)inode->i_blocks < blocks) + inode->i_blocks = blocks; +} + +static void cifs_update_i_blocks_after_write(struct kiocb *iocb, + ssize_t written) +{ + struct inode *inode = file_inode(iocb->ki_filp); + loff_t end = iocb->ki_pos; + + if (written <= 0) + return; + + spin_lock(&inode->i_lock); + cifs_update_i_blocks_for_write(inode, end - written, end); + spin_unlock(&inode->i_lock); +} + void cifs_write_subrequest_terminated(struct cifs_io_subrequest *wdata, ssize_t result) { struct netfs_io_request *wreq = wdata->rreq; - struct netfs_inode *ictx = netfs_inode(wreq->inode); + struct inode *inode = wreq->inode; + struct netfs_inode *ictx = netfs_inode(inode); loff_t wrend; if (result > 0) { + spin_lock(&inode->i_lock); + wrend = wdata->subreq.start + wdata->subreq.transferred + result; - if (wrend > ictx->zero_point && + if (wrend > ictx->_zero_point && (wdata->rreq->origin == NETFS_UNBUFFERED_WRITE || wdata->rreq->origin == NETFS_DIO_WRITE)) - ictx->zero_point = wrend; - if (wrend > ictx->remote_i_size) + netfs_write_zero_point(inode, wrend); + if (wrend > ictx->_remote_i_size) netfs_resize_file(ictx, wrend, true); + cifs_update_i_blocks_for_write(inode, wdata->subreq.start, + wrend); + + spin_unlock(&inode->i_lock); } netfs_write_subrequest_terminated(&wdata->subreq, result); @@ -2915,6 +2966,7 @@ cifs_writev(struct kiocb *iocb, struct iov_iter *from) } rc = netfs_buffered_write_iter_locked(iocb, from, NULL); + cifs_update_i_blocks_after_write(iocb, rc); out: up_read(&cinode->lock_sem); @@ -2944,6 +2996,7 @@ cifs_strict_writev(struct kiocb *iocb, struct iov_iter *from) (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) { written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); goto out; } written = cifs_writev(iocb, from); @@ -2956,6 +3009,7 @@ cifs_strict_writev(struct kiocb *iocb, struct iov_iter *from) * these pages but not on the region from pos to ppos+len-1. */ written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); if (CIFS_CACHE_READ(cinode)) { /* * We have read level caching and we have just sent a write @@ -2974,6 +3028,15 @@ out: return written; } +ssize_t cifs_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t written; + + written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); + return written; +} + ssize_t cifs_loose_read_iter(struct kiocb *iocb, struct iov_iter *iter) { ssize_t rc; @@ -2998,6 +3061,7 @@ ssize_t cifs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (iocb->ki_filp->f_flags & O_DIRECT) { written = netfs_unbuffered_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); if (written > 0 && CIFS_CACHE_READ(cinode)) { cifs_zap_mapping(inode); cifs_dbg(FYI, @@ -3013,6 +3077,7 @@ ssize_t cifs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return written; written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); if (!CIFS_CACHE_WRITE(CIFS_I(inode))) { rc = filemap_fdatawrite(inode->i_mapping); diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index b63ec7ab6e51..abc29500a664 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -420,7 +420,7 @@ static int parse_symlink_flavor(struct fs_context *fc, char *value, #define DUP_CTX_STR(field) \ do { \ if (ctx->field) { \ - new_ctx->field = kstrdup(ctx->field, GFP_ATOMIC); \ + new_ctx->field = kstrdup(ctx->field, GFP_KERNEL); \ if (new_ctx->field == NULL) { \ smb3_cleanup_fs_context_contents(new_ctx); \ return -ENOMEM; \ @@ -693,6 +693,41 @@ static int smb3_handle_conflicting_options(struct fs_context *fc) { struct smb3_fs_context *ctx = smb3_fc2context(fc); + if (ctx->rdma && ctx->vals->protocol_id < SMB30_PROT_ID) { + cifs_errorf(fc, "SMB Direct requires Version >=3.0\n"); + return -EOPNOTSUPP; + } + + if (ctx->multiuser && !IS_ENABLED(CONFIG_KEYS)) { + cifs_errorf(fc, "Multiuser mounts require kernels with CONFIG_KEYS enabled\n"); + return -EOPNOTSUPP; + } + + if (ctx->multiuser && ctx->upcall_target == UPTARGET_MOUNT) { + cifs_errorf(fc, "multiuser mount option not supported with upcalltarget set as 'mount'\n"); + return -EINVAL; + } + + if (ctx->uid_specified && !ctx->forceuid_specified) { + ctx->override_uid = 1; + pr_notice("enabling forceuid mount option implicitly because uid= option is specified\n"); + } + + if (ctx->gid_specified && !ctx->forcegid_specified) { + ctx->override_gid = 1; + pr_notice("enabling forcegid mount option implicitly because gid= option is specified\n"); + } + + if (ctx->override_uid && !ctx->uid_specified) { + ctx->override_uid = 0; + pr_notice("ignoring forceuid mount option specified with no uid= option\n"); + } + + if (ctx->override_gid && !ctx->gid_specified) { + ctx->override_gid = 0; + pr_notice("ignoring forcegid mount option specified with no gid= option\n"); + } + if (ctx->multichannel_specified) { if (ctx->multichannel) { if (!ctx->max_channels_specified) { @@ -711,19 +746,14 @@ static int smb3_handle_conflicting_options(struct fs_context *fc) return -EINVAL; } } - } else { - if (ctx->max_channels_specified) { - if (ctx->max_channels > 1) - ctx->multichannel = true; - else - ctx->multichannel = false; - } else { + } else if (ctx->max_channels_specified) { + if (ctx->max_channels > 1) + ctx->multichannel = true; + else ctx->multichannel = false; - ctx->max_channels = 1; - } } - //resetting default values as remount doesn't initialize fs_context again + /* clear parse-time latches so they don't persist across remounts */ ctx->multichannel_specified = false; ctx->max_channels_specified = false; @@ -736,7 +766,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, static int smb3_fs_context_parse_monolithic(struct fs_context *fc, void *data); static int smb3_get_tree(struct fs_context *fc); -static void smb3_sync_ses_chan_max(struct cifs_ses *ses, unsigned int max_channels); +static void smb3_sync_ses_chan_max(struct cifs_ses *ses, size_t max_channels); static int smb3_reconfigure(struct fs_context *fc); static const struct fs_context_operations smb3_fs_context_ops = { @@ -804,28 +834,23 @@ static int smb3_fs_context_parse_monolithic(struct fs_context *fc, if (ret < 0) break; } - return ret ?: smb3_handle_conflicting_options(fc); + return ret; } /* - * Validate the preparsed information in the config. + * smb3_fs_context_validate - check initial-mount-only constraints: + * UNC presence, address resolution, dialect warnings + * + * @fc: generic mount context */ static int smb3_fs_context_validate(struct fs_context *fc) { struct smb3_fs_context *ctx = smb3_fc2context(fc); + int rc; - if (ctx->rdma && ctx->vals->protocol_id < SMB30_PROT_ID) { - cifs_errorf(fc, "SMB Direct requires Version >=3.0\n"); - return -EOPNOTSUPP; - } - -#ifndef CONFIG_KEYS - /* Muliuser mounts require CONFIG_KEYS support */ - if (ctx->multiuser) { - cifs_errorf(fc, "Multiuser mounts require kernels with CONFIG_KEYS enabled\n"); - return -1; - } -#endif + rc = smb3_handle_conflicting_options(fc); + if (rc) + return rc; if (ctx->got_version == false) pr_warn_once("No dialect specified on mount. Default has changed to a more secure dialect, SMB2.1 or later (e.g. SMB3.1.1), from CIFS (SMB1). To use the less secure SMB1 dialect to access old servers which do not support SMB3.1.1 (or even SMB3 or SMB2.1) specify vers=1.0 on mount.\n"); @@ -860,26 +885,6 @@ static int smb3_fs_context_validate(struct fs_context *fc) /* set the port that we got earlier */ cifs_set_port((struct sockaddr *)&ctx->dstaddr, ctx->port); - if (ctx->uid_specified && !ctx->forceuid_specified) { - ctx->override_uid = 1; - pr_notice("enabling forceuid mount option implicitly because uid= option is specified\n"); - } - - if (ctx->gid_specified && !ctx->forcegid_specified) { - ctx->override_gid = 1; - pr_notice("enabling forcegid mount option implicitly because gid= option is specified\n"); - } - - if (ctx->override_uid && !ctx->uid_specified) { - ctx->override_uid = 0; - pr_notice("ignoring forceuid mount option specified with no uid= option\n"); - } - - if (ctx->override_gid && !ctx->gid_specified) { - ctx->override_gid = 0; - pr_notice("ignoring forcegid mount option specified with no gid= option\n"); - } - return 0; } @@ -889,7 +894,7 @@ static int smb3_get_tree_common(struct fs_context *fc) struct dentry *root; int rc = 0; - root = cifs_smb3_do_mount(fc->fs_type, 0, ctx); + root = cifs_smb3_do_mount(fc, ctx); if (IS_ERR(root)) return PTR_ERR(root); @@ -1010,25 +1015,34 @@ do { \ int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses) { + char *password = NULL, *password2 = NULL; + if (ses->password && cifs_sb->ctx->password && strcmp(ses->password, cifs_sb->ctx->password)) { - kfree_sensitive(cifs_sb->ctx->password); - cifs_sb->ctx->password = kstrdup(ses->password, GFP_KERNEL); - if (!cifs_sb->ctx->password) + password = kstrdup(ses->password, GFP_KERNEL); + if (!password) return -ENOMEM; } if (ses->password2 && cifs_sb->ctx->password2 && strcmp(ses->password2, cifs_sb->ctx->password2)) { - kfree_sensitive(cifs_sb->ctx->password2); - cifs_sb->ctx->password2 = kstrdup(ses->password2, GFP_KERNEL); - if (!cifs_sb->ctx->password2) { - kfree_sensitive(cifs_sb->ctx->password); - cifs_sb->ctx->password = NULL; + password2 = kstrdup(ses->password2, GFP_KERNEL); + if (!password2) { + kfree_sensitive(password); return -ENOMEM; } } + + if (password) { + kfree_sensitive(cifs_sb->ctx->password); + cifs_sb->ctx->password = password; + } + if (password2) { + kfree_sensitive(cifs_sb->ctx->password2); + cifs_sb->ctx->password2 = password2; + } + return 0; } @@ -1041,7 +1055,7 @@ int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, struct cifs_se * with the session's channel lock. This should be called whenever the maximum * allowed channels for a session changes (e.g., after a remount or reconfigure). */ -static void smb3_sync_ses_chan_max(struct cifs_ses *ses, unsigned int max_channels) +static void smb3_sync_ses_chan_max(struct cifs_ses *ses, size_t max_channels) { spin_lock(&ses->chan_lock); ses->chan_max = max_channels; @@ -1051,12 +1065,15 @@ static void smb3_sync_ses_chan_max(struct cifs_ses *ses, unsigned int max_channe static int smb3_reconfigure(struct fs_context *fc) { struct smb3_fs_context *ctx = smb3_fc2context(fc); + struct smb3_fs_context *new_ctx = NULL; + struct smb3_fs_context *old_ctx = NULL; struct dentry *root = fc->root; struct cifs_sb_info *cifs_sb = CIFS_SB(root->d_sb); struct cifs_ses *ses = cifs_sb_master_tcon(cifs_sb)->ses; unsigned int rsize = ctx->rsize, wsize = ctx->wsize; char *new_password = NULL, *new_password2 = NULL; bool need_recon = false; + bool need_mchan_update; int rc; if (ses->expired_pwd) @@ -1066,6 +1083,18 @@ static int smb3_reconfigure(struct fs_context *fc) if (rc) return rc; + rc = smb3_handle_conflicting_options(fc); + if (rc) + return rc; + + old_ctx = kzalloc_obj(*old_ctx); + if (!old_ctx) + return -ENOMEM; + + rc = smb3_fs_context_dup(old_ctx, cifs_sb->ctx); + if (rc) + goto free_old_ctx; + /* * We can not change UNC/username/password/domainname/ * workstation_name/nodename/iocharset @@ -1075,16 +1104,22 @@ static int smb3_reconfigure(struct fs_context *fc) STEAL_STRING(cifs_sb, ctx, UNC); STEAL_STRING(cifs_sb, ctx, source); STEAL_STRING(cifs_sb, ctx, username); + STEAL_STRING(cifs_sb, ctx, domainname); + STEAL_STRING(cifs_sb, ctx, nodename); + STEAL_STRING(cifs_sb, ctx, iocharset); - if (need_recon == false) + if (!need_recon) { STEAL_STRING_SENSITIVE(cifs_sb, ctx, password); - else { + } else { if (ctx->password) { new_password = kstrdup(ctx->password, GFP_KERNEL); - if (!new_password) - return -ENOMEM; - } else + if (!new_password) { + rc = -ENOMEM; + goto restore_ctx; + } + } else { STEAL_STRING_SENSITIVE(cifs_sb, ctx, password); + } } /* @@ -1094,11 +1129,29 @@ static int smb3_reconfigure(struct fs_context *fc) if (ctx->password2) { new_password2 = kstrdup(ctx->password2, GFP_KERNEL); if (!new_password2) { - kfree_sensitive(new_password); - return -ENOMEM; + rc = -ENOMEM; + goto restore_ctx; } - } else + } else { STEAL_STRING_SENSITIVE(cifs_sb, ctx, password2); + } + + /* if rsize or wsize not passed in on remount, use previous values */ + ctx->rsize = rsize ? CIFS_ALIGN_RSIZE(fc, rsize) : cifs_sb->ctx->rsize; + ctx->wsize = wsize ? CIFS_ALIGN_WSIZE(fc, wsize) : cifs_sb->ctx->wsize; + + new_ctx = kzalloc_obj(*new_ctx); + if (!new_ctx) { + rc = -ENOMEM; + goto restore_ctx; + } + + rc = smb3_fs_context_dup(new_ctx, ctx); + if (rc) + goto restore_ctx; + + need_mchan_update = ctx->multichannel != cifs_sb->ctx->multichannel || + ctx->max_channels != cifs_sb->ctx->max_channels; /* * we may update the passwords in the ses struct below. Make sure we do @@ -1109,54 +1162,55 @@ static int smb3_reconfigure(struct fs_context *fc) /* * smb2_reconnect may swap password and password2 in case session setup * failed. First get ctx passwords in sync with ses passwords. It should - * be okay to do this even if this function were to return an error at a - * later stage + * be done before committing new passwords. */ rc = smb3_sync_session_ctx_passwords(cifs_sb, ses); if (rc) { mutex_unlock(&ses->session_mutex); - kfree_sensitive(new_password); - kfree_sensitive(new_password2); - return rc; + goto cleanup_new_ctx; } /* - * now that allocations for passwords are done, commit them + * If multichannel or max_channels has changed, update the session's channels accordingly. + * This may add or remove channels to match the new configuration. + */ + if (need_mchan_update) { + /* Prevent concurrent scaling operations */ + spin_lock(&ses->ses_lock); + if (ses->flags & CIFS_SES_FLAG_SCALE_CHANNELS) { + spin_unlock(&ses->ses_lock); + mutex_unlock(&ses->session_mutex); + rc = -EINVAL; + goto cleanup_new_ctx; + } + ses->flags |= CIFS_SES_FLAG_SCALE_CHANNELS; + spin_unlock(&ses->ses_lock); + } + + /* + * Commit session passwords before any channel work so newly added + * channels authenticate with the new credentials. */ if (new_password) { kfree_sensitive(ses->password); ses->password = new_password; + new_password = NULL; } if (new_password2) { kfree_sensitive(ses->password2); ses->password2 = new_password2; + new_password2 = NULL; } - /* - * If multichannel or max_channels has changed, update the session's channels accordingly. - * This may add or remove channels to match the new configuration. - */ - if ((ctx->multichannel != cifs_sb->ctx->multichannel) || - (ctx->max_channels != cifs_sb->ctx->max_channels)) { - + if (need_mchan_update) { /* Synchronize ses->chan_max with the new mount context */ smb3_sync_ses_chan_max(ses, ctx->max_channels); - /* Now update the session's channels to match the new configuration */ - /* Prevent concurrent scaling operations */ - spin_lock(&ses->ses_lock); - if (ses->flags & CIFS_SES_FLAG_SCALE_CHANNELS) { - spin_unlock(&ses->ses_lock); - mutex_unlock(&ses->session_mutex); - return -EINVAL; - } - ses->flags |= CIFS_SES_FLAG_SCALE_CHANNELS; - spin_unlock(&ses->ses_lock); mutex_unlock(&ses->session_mutex); - rc = smb3_update_ses_channels(ses, ses->server, - false /* from_reconnect */, - false /* disable_mchan */); + smb3_update_ses_channels(ses, ses->server, + false /* from_reconnect */, + false /* disable_mchan */); /* Clear scaling flag after operation */ spin_lock(&ses->ses_lock); @@ -1166,16 +1220,12 @@ static int smb3_reconfigure(struct fs_context *fc) mutex_unlock(&ses->session_mutex); } - STEAL_STRING(cifs_sb, ctx, domainname); - STEAL_STRING(cifs_sb, ctx, nodename); - STEAL_STRING(cifs_sb, ctx, iocharset); - - /* if rsize or wsize not passed in on remount, use previous values */ - ctx->rsize = rsize ? CIFS_ALIGN_RSIZE(fc, rsize) : cifs_sb->ctx->rsize; - ctx->wsize = wsize ? CIFS_ALIGN_WSIZE(fc, wsize) : cifs_sb->ctx->wsize; - smb3_cleanup_fs_context_contents(cifs_sb->ctx); - rc = smb3_fs_context_dup(cifs_sb->ctx, ctx); + memcpy(cifs_sb->ctx, new_ctx, sizeof(*new_ctx)); + kfree(new_ctx); + new_ctx = NULL; + smb3_cleanup_fs_context(old_ctx); + old_ctx = NULL; smb3_update_mnt_flags(cifs_sb); #ifdef CONFIG_CIFS_DFS_UPCALL if (!rc) @@ -1183,6 +1233,19 @@ static int smb3_reconfigure(struct fs_context *fc) #endif return rc; + +cleanup_new_ctx: + smb3_cleanup_fs_context_contents(new_ctx); +restore_ctx: + kfree(new_ctx); + kfree_sensitive(new_password); + kfree_sensitive(new_password2); + smb3_cleanup_fs_context_contents(cifs_sb->ctx); + memcpy(cifs_sb->ctx, old_ctx, sizeof(*old_ctx)); +free_old_ctx: + kfree(old_ctx); + + return rc; } static int smb3_fs_context_parse_param(struct fs_context *fc, @@ -1878,11 +1941,6 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, } /* case Opt_ignore: - is ignored as expected ... */ - if (ctx->multiuser && ctx->upcall_target == UPTARGET_MOUNT) { - cifs_errorf(fc, "multiuser mount option not supported with upcalltarget set as 'mount'\n"); - goto cifs_parse_mount_err; - } - return 0; cifs_parse_mount_err: diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index 16a5310155d5..deed04dd9b91 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -28,6 +28,23 @@ #include "cached_dir.h" #include "reparse.h" +static void cifs_invalidate_cached_dir(struct cifs_tcon *tcon, + struct dentry *parent) +{ + struct cached_fid *parent_cfid = NULL; + + if (!tcon || !parent) + return; + + if (!open_cached_dir_by_dentry(tcon, parent, &parent_cfid)) { + mutex_lock(&parent_cfid->dirents.de_mutex); + parent_cfid->dirents.is_valid = false; + parent_cfid->dirents.is_failed = true; + mutex_unlock(&parent_cfid->dirents.de_mutex); + close_cached_dir(parent_cfid); + } +} + /* * Set parameters for the netfs library */ @@ -119,7 +136,7 @@ cifs_revalidate_cache(struct inode *inode, struct cifs_fattr *fattr) fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode); mtime = inode_get_mtime(inode); if (timespec64_equal(&mtime, &fattr->cf_mtime) && - cifs_i->netfs.remote_i_size == fattr->cf_eof) { + netfs_read_remote_i_size(inode) == fattr->cf_eof) { cifs_dbg(FYI, "%s: inode %llu is unchanged\n", __func__, cifs_i->uniqueid); return; @@ -173,12 +190,12 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr, CIFS_I(inode)->time = 0; /* force reval */ return -ESTALE; } - if (inode_state_read_once(inode) & I_NEW) - CIFS_I(inode)->netfs.zero_point = fattr->cf_eof; - cifs_revalidate_cache(inode, fattr); spin_lock(&inode->i_lock); + if (inode_state_read_once(inode) & I_NEW) + netfs_write_zero_point(inode, fattr->cf_eof); + fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode); fattr->cf_atime = timestamp_truncate(fattr->cf_atime, inode); fattr->cf_ctime = timestamp_truncate(fattr->cf_ctime, inode); @@ -212,7 +229,7 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr, else clear_bit(CIFS_INO_DELETE_PENDING, &cifs_i->flags); - cifs_i->netfs.remote_i_size = fattr->cf_eof; + netfs_write_remote_i_size(inode, fattr->cf_eof); /* * Can't safely change the file size here if the client is writing to * it due to potential races. @@ -892,6 +909,8 @@ static void cifs_open_info_to_fattr(struct cifs_fattr *fattr, struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); memset(fattr, 0, sizeof(*fattr)); + if (data->unknown_nlink) + fattr->cf_flags |= CIFS_FATTR_UNKNOWN_NLINK; fattr->cf_cifsattrs = le32_to_cpu(info->Attributes); if (info->DeletePending) fattr->cf_flags |= CIFS_FATTR_DELETE_PENDING; @@ -1128,7 +1147,7 @@ static void cifs_set_fattr_ino(int xid, struct cifs_tcon *tcon, struct super_blo fattr->cf_uniqueid = CIFS_I(*inode)->uniqueid; else { fattr->cf_uniqueid = iunique(sb, ROOT_I); - cifs_autodisable_serverino(cifs_sb); + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve inode number via get_srv_inum", rc); } return; } @@ -1625,7 +1644,7 @@ retry_iget5_locked: fattr->cf_flags &= ~CIFS_FATTR_INO_COLLISION; if (inode_has_hashed_dentries(inode)) { - cifs_autodisable_serverino(CIFS_SB(sb)); + cifs_autodisable_serverino(CIFS_SB(sb), "Inode number collision detected", 0); iput(inode); fattr->cf_uniqueid = iunique(sb, ROOT_I); goto retry_iget5_locked; @@ -1691,8 +1710,9 @@ struct inode *cifs_root_iget(struct super_block *sb) iget_root: if (!rc) { if (fattr.cf_flags & CIFS_FATTR_JUNCTION) { + cifs_dbg(VFS, "Removing junction mark and disabling 'serverino' to prevent inode collisions\n"); fattr.cf_flags &= ~CIFS_FATTR_JUNCTION; - cifs_autodisable_serverino(cifs_sb); + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve attributes for junction point", rc); } inode = cifs_iget(sb, &fattr); } @@ -2067,6 +2087,9 @@ psx_del_no_retry: cifs_set_file_info(inode, attrs, xid, full_path, origattr); out_reval: + if (!rc && dentry->d_parent) + cifs_invalidate_cached_dir(tcon, dentry->d_parent); + if (inode) { cifs_inode = CIFS_I(inode); cifs_inode->time = 0; /* will force revalidate to get info @@ -2378,7 +2401,6 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry) } rc = server->ops->rmdir(xid, tcon, full_path, cifs_sb); - cifs_put_tlink(tlink); cifsInode = CIFS_I(d_inode(direntry)); @@ -2388,6 +2410,8 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry) i_size_write(d_inode(direntry), 0); clear_nlink(d_inode(direntry)); spin_unlock(&d_inode(direntry)->i_lock); + if (direntry->d_parent) + cifs_invalidate_cached_dir(tcon, direntry->d_parent); } /* force revalidate to go get info when needed */ @@ -2402,6 +2426,7 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry) inode_set_ctime_current(d_inode(direntry)); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + cifs_put_tlink(tlink); rmdir_exit: free_dentry_path(page); @@ -2668,6 +2693,12 @@ unlink_target: } /* force revalidate to go get info when needed */ + if (!rc) { + cifs_invalidate_cached_dir(tcon, source_dentry->d_parent); + if (target_dentry->d_parent != source_dentry->d_parent) + cifs_invalidate_cached_dir(tcon, target_dentry->d_parent); + } + CIFS_I(source_dir)->time = CIFS_I(target_dir)->time = 0; cifs_rename_exit: @@ -2772,7 +2803,9 @@ cifs_revalidate_mapping(struct inode *inode) if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RW_CACHE) goto skip_invalidate; - cifs_inode->netfs.zero_point = cifs_inode->netfs.remote_i_size; + spin_lock(&inode->i_lock); + netfs_write_zero_point(inode, netfs_inode(inode)->_remote_i_size); + spin_unlock(&inode->i_lock); rc = filemap_invalidate_inode(inode, true, 0, LLONG_MAX); if (rc) { cifs_dbg(VFS, "%s: invalidate inode %p failed with rc %d\n", @@ -2782,9 +2815,7 @@ cifs_revalidate_mapping(struct inode *inode) } skip_invalidate: - clear_bit_unlock(CIFS_INO_LOCK, flags); - smp_mb__after_atomic(); - wake_up_bit(flags, CIFS_INO_LOCK); + clear_and_wake_up_bit(CIFS_INO_LOCK, flags); return rc; } @@ -2843,7 +2874,7 @@ int cifs_revalidate_dentry_attr(struct dentry *dentry) } cifs_dbg(FYI, "Update attributes: %s inode 0x%p count %d dentry: 0x%p d_time %ld jiffies %ld\n", - full_path, inode, icount_read(inode), + full_path, inode, icount_read_once(inode), dentry, cifs_get_time(dentry), jiffies); again: @@ -3008,13 +3039,20 @@ int cifs_fiemap(struct inode *inode, struct fiemap_extent_info *fei, u64 start, void cifs_setsize(struct inode *inode, loff_t offset) { + loff_t old_size; + u64 blocks = CIFS_INO_BLOCKS(offset); + spin_lock(&inode->i_lock); + old_size = i_size_read(inode); i_size_write(inode, offset); + /* - * Until we can query the server for actual allocation size, - * this is best estimate we have for blocks allocated for a file. + * Extending EOF does not allocate the intervening range. Only clamp + * i_blocks on shrink; allocation growth comes from writes or from the + * server-reported AllocationSize. */ - inode->i_blocks = CIFS_INO_BLOCKS(offset); + if (offset < old_size && (u64)inode->i_blocks > blocks) + inode->i_blocks = blocks; spin_unlock(&inode->i_lock); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); truncate_pagecache(inode, offset); @@ -3346,7 +3384,8 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (attrs->ia_valid & ATTR_GID) gid = attrs->ia_gid; - if (sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) { + if ((sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) || + cifs_sb_master_tcon(cifs_sb)->posix_extensions) { if (uid_valid(uid) || gid_valid(gid)) { mode = NO_CHANGE_64; rc = id_mode_to_cifs_acl(inode, full_path, &mode, diff --git a/fs/smb/client/ioctl.c b/fs/smb/client/ioctl.c index 9afab3237e54..9fa743be3652 100644 --- a/fs/smb/client/ioctl.c +++ b/fs/smb/client/ioctl.c @@ -67,6 +67,116 @@ static long cifs_ioctl_query_info(unsigned int xid, struct file *filep, return rc; } +static int cifs_set_compression_by_path(unsigned int xid, struct file *filep, + struct cifs_tcon *tcon, + __u16 compression_state) +{ + struct inode *inode = file_inode(filep); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct TCP_Server_Info *server = tcon->ses->server; + struct cifs_open_parms oparms; + struct cifs_open_info_data data = {}; + struct cifsFileInfo *tmp_cfile = NULL; + struct cifs_fid fid = {}; + const char *full_path; + __u32 oplock = 0; + u64 uniqueid; + void *page; + int rc; + + if (!server->ops->open || !server->ops->close || + !server->ops->query_file_info) + return -EOPNOTSUPP; + + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) || + cifs_sb->mnt_cifs_serverino_autodisabled) + return -EOPNOTSUPP; + + if (d_unhashed(filep->f_path.dentry)) + return -ESTALE; + + page = alloc_dentry_path(); + full_path = build_path_from_dentry(filep->f_path.dentry, page); + if (IS_ERR(full_path)) { + free_dentry_path(page); + return PTR_ERR(full_path); + } + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, FILE_WRITE_DATA | + FILE_READ_ATTRIBUTES, + FILE_OPEN, 0, ACL_NO_MODE); + oparms.fid = &fid; + + rc = server->ops->open(xid, &oparms, &oplock, NULL); + if (rc) + goto out; + + tmp_cfile = kzalloc_obj(*tmp_cfile); + if (!tmp_cfile) { + rc = -ENOMEM; + goto close; + } + + tmp_cfile->fid = fid; + rc = server->ops->query_file_info(xid, tcon, tmp_cfile, &data); + if (rc) + goto close; + + uniqueid = le64_to_cpu(data.fi.IndexNumber); + if (uniqueid != CIFS_I(inode)->uniqueid) { + rc = -ESTALE; + goto close; + } + + rc = server->ops->set_compression(xid, tcon, tmp_cfile, + compression_state); + +close: + server->ops->close(xid, tcon, &fid); + if (tmp_cfile) + kfree(tmp_cfile); + cifs_free_open_info(&data); +out: + free_dentry_path(page); + return rc; +} + +static int cifs_ioctl_set_compression(unsigned int xid, struct file *filep, + struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, + __u16 compression_state) +{ + struct cifsFileInfo *wfile; + struct cifs_tcon *wtcon; + struct inode *inode = file_inode(filep); + int rc; + + if (!tcon->ses->server->ops->set_compression) + return -EOPNOTSUPP; + + if (cfile && (cfile->fid.access & FILE_WRITE_DATA)) { + rc = tcon->ses->server->ops->set_compression(xid, tcon, cfile, + compression_state); + if (rc != -EACCES) + return rc; + } + + rc = cifs_get_writable_file(CIFS_I(inode), FIND_FSUID_ONLY, &wfile); + if (!rc) { + wtcon = tlink_tcon(wfile->tlink); + rc = wtcon->ses->server->ops->set_compression(xid, wtcon, wfile, + compression_state); + cifsFileInfo_put(wfile); + if (rc != -EACCES) + return rc; + } else if (rc != -EBADF) { + return rc; + } + + return cifs_set_compression_by_path(xid, filep, tcon, + compression_state); +} + static long cifs_ioctl_copychunk(unsigned int xid, struct file *dst_file, unsigned long srcfd) { @@ -296,7 +406,7 @@ search_end: break; case SMB2_ENCRYPTION_AES256_CCM: case SMB2_ENCRYPTION_AES256_GCM: - out.session_key_length = CIFS_SESS_KEY_SIZE; + out.session_key_length = ses->auth_key.len; out.server_in_key_length = out.server_out_key_length = SMB3_GCM256_CRYPTKEY_SIZE; break; default: @@ -356,6 +466,8 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) struct tcon_link *tlink; struct cifs_sb_info *cifs_sb; __u64 ExtAttrBits = 0; + bool enable_compression; + __u16 compression_state; #ifdef CONFIG_CIFS_POSIX #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY __u64 caps; @@ -392,13 +504,11 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) } #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ #endif /* CONFIG_CIFS_POSIX */ - rc = 0; - if (CIFS_I(inode)->cifsAttrs & ATTR_COMPRESSED) { - /* add in the compressed bit */ - ExtAttrBits = FS_COMPR_FL; - rc = put_user(ExtAttrBits & FS_FL_USER_VISIBLE, - (int __user *)arg); - } + if (CIFS_I(inode)->cifsAttrs & FILE_ATTRIBUTE_COMPRESSED) + ExtAttrBits |= FS_COMPR_FL; + + rc = put_user(ExtAttrBits & FS_FL_USER_VISIBLE, + (int __user *)arg); break; case FS_IOC_SETFLAGS: if (pSMBFile == NULL) @@ -421,16 +531,31 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) * break; */ - /* Currently only flag we can set is compressed flag */ - if ((ExtAttrBits & FS_COMPR_FL) == 0) + /* Currently only flag we can set or clear is compressed. */ + if (ExtAttrBits & ~FS_COMPR_FL) { + rc = -EOPNOTSUPP; break; + } - /* Try to set compress flag */ - if (tcon->ses->server->ops->set_compression) { - rc = tcon->ses->server->ops->set_compression( - xid, tcon, pSMBFile); - cifs_dbg(FYI, "set compress flag rc %d\n", rc); + enable_compression = ExtAttrBits & FS_COMPR_FL; + compression_state = enable_compression ? + COMPRESSION_FORMAT_DEFAULT : + COMPRESSION_FORMAT_NONE; + + rc = cifs_ioctl_set_compression(xid, filep, tcon, + pSMBFile, + compression_state); + if (rc == 0) { + spin_lock(&inode->i_lock); + if (enable_compression) + CIFS_I(inode)->cifsAttrs |= + FILE_ATTRIBUTE_COMPRESSED; + else + CIFS_I(inode)->cifsAttrs &= + ~FILE_ATTRIBUTE_COMPRESSED; + spin_unlock(&inode->i_lock); } + cifs_dbg(FYI, "set compress flag rc %d\n", rc); break; case CIFS_IOC_COPYCHUNK_FILE: rc = cifs_ioctl_copychunk(xid, filep, arg); diff --git a/fs/smb/client/link.c b/fs/smb/client/link.c index dd127917a340..8d5d6aca742a 100644 --- a/fs/smb/client/link.c +++ b/fs/smb/client/link.c @@ -234,7 +234,7 @@ cifs_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, struct cifs_open_parms oparms; struct cifs_io_parms io_parms = {0}; int buf_type = CIFS_NO_BUFFER; - struct cifs_open_info_data query_data; + struct cifs_open_info_data query_data = {}; oparms = (struct cifs_open_parms) { .tcon = tcon, @@ -320,7 +320,7 @@ smb3_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, int buf_type = CIFS_NO_BUFFER; __le16 *utf16_path; __u8 oplock = SMB2_OPLOCK_LEVEL_NONE; - struct smb2_file_all_info *pfile_info = NULL; + struct cifs_open_info_data data = {}; oparms = (struct cifs_open_parms) { .tcon = tcon, @@ -336,20 +336,12 @@ smb3_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, if (utf16_path == NULL) return -ENOMEM; - pfile_info = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2, - GFP_KERNEL); - - if (pfile_info == NULL) { - kfree(utf16_path); - return -ENOMEM; - } - - rc = SMB2_open(xid, &oparms, utf16_path, &oplock, pfile_info, NULL, + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, &data, NULL, NULL, NULL); if (rc) goto qmf_out_open_fail; - if (pfile_info->EndOfFile != cpu_to_le64(CIFS_MF_SYMLINK_FILE_SIZE)) { + if (data.fi.EndOfFile != cpu_to_le64(CIFS_MF_SYMLINK_FILE_SIZE)) { /* it's not a symlink */ rc = -ENOENT; /* Is there a better rc to return? */ goto qmf_out; @@ -367,7 +359,6 @@ qmf_out: SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); qmf_out_open_fail: kfree(utf16_path); - kfree(pfile_info); return rc; } diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c index 0c54b9b79a2c..e4bac2a0b85d 100644 --- a/fs/smb/client/misc.c +++ b/fs/smb/client/misc.c @@ -278,7 +278,7 @@ dump_smb(void *buf, int smb_buf_length) } void -cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb) +cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb, const char *reason, int rc) { unsigned int sbflags = cifs_sb_flags(cifs_sb); @@ -290,6 +290,10 @@ cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb) atomic_andnot(CIFS_MOUNT_SERVER_INUM, &cifs_sb->mnt_cifs_flags); cifs_sb->mnt_cifs_serverino_autodisabled = true; + if (rc) + cifs_dbg(VFS, "%s: %d\n", reason, rc); + else + cifs_dbg(VFS, "%s\n", reason); cifs_dbg(VFS, "Autodisabling the use of server inode numbers on %s\n", tcon ? tcon->tree_name : "new server"); cifs_dbg(VFS, "The server doesn't seem to support them properly or the files might be on different servers (DFS)\n"); @@ -752,6 +756,10 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, node->ref_flag = le16_to_cpu(ref->ReferralEntryFlags); /* copy DfsPath */ + if (le16_to_cpu(ref->DfsPathOffset) > data_end - (char *)ref) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } temp = (char *)ref + le16_to_cpu(ref->DfsPathOffset); max_len = data_end - temp; node->path_name = cifs_strndup_from_utf16(temp, max_len, @@ -762,6 +770,10 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, } /* copy link target UNC */ + if (le16_to_cpu(ref->NetworkAddressOffset) > data_end - (char *)ref) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } temp = (char *)ref + le16_to_cpu(ref->NetworkAddressOffset); max_len = data_end - temp; node->node_name = cifs_strndup_from_utf16(temp, max_len, diff --git a/fs/smb/client/namespace.c b/fs/smb/client/namespace.c index 52a520349cb7..52a51b032fae 100644 --- a/fs/smb/client/namespace.c +++ b/fs/smb/client/namespace.c @@ -294,4 +294,5 @@ struct vfsmount *cifs_d_automount(struct path *path) } const struct inode_operations cifs_namespace_inode_operations = { + .fileattr_get = cifs_fileattr_get, }; diff --git a/fs/smb/client/netlink.c b/fs/smb/client/netlink.c index 147d9409252c..0dd10913c37a 100644 --- a/fs/smb/client/netlink.c +++ b/fs/smb/client/netlink.c @@ -33,13 +33,17 @@ static const struct nla_policy cifs_genl_policy[CIFS_GENL_ATTR_MAX + 1] = { static const struct genl_ops cifs_genl_ops[] = { { .cmd = CIFS_GENL_CMD_SWN_NOTIFY, + .flags = GENL_ADMIN_PERM, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, .doit = cifs_swn_notify, }, }; static const struct genl_multicast_group cifs_genl_mcgrps[] = { - [CIFS_GENL_MCGRP_SWN] = { .name = CIFS_GENL_MCGRP_SWN_NAME }, + [CIFS_GENL_MCGRP_SWN] = { + .name = CIFS_GENL_MCGRP_SWN_NAME, + .flags = GENL_MCAST_CAP_NET_ADMIN, + }, }; struct genl_family cifs_genl_family = { diff --git a/fs/smb/client/readdir.c b/fs/smb/client/readdir.c index be22bbc4a65a..ee5996e6d7d8 100644 --- a/fs/smb/client/readdir.c +++ b/fs/smb/client/readdir.c @@ -73,7 +73,6 @@ cifs_prime_dcache(struct dentry *parent, struct qstr *name, struct cifs_sb_info *cifs_sb = CIFS_SB(sb); bool posix = cifs_sb_master_tcon(cifs_sb)->posix_extensions; bool reparse_need_reval = false; - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); int rc; cifs_dbg(FYI, "%s: for %s\n", __func__, name->name); @@ -105,7 +104,7 @@ retry: (fattr->cf_flags & CIFS_FATTR_NEED_REVAL)) return; - dentry = d_alloc_parallel(parent, name, &wq); + dentry = d_alloc_parallel(parent, name); } if (IS_ERR(dentry)) return; @@ -143,7 +142,8 @@ retry: fattr->cf_rdev = inode->i_rdev; fattr->cf_uid = inode->i_uid; fattr->cf_gid = inode->i_gid; - fattr->cf_eof = CIFS_I(inode)->netfs.remote_i_size; + fattr->cf_eof = + netfs_read_remote_i_size(inode); fattr->cf_symlink_target = NULL; } else { CIFS_I(inode)->time = 0; @@ -415,7 +415,7 @@ ffirst_retry: if (rc == 0) { cifsFile->invalidHandle = false; } else if (rc == -EOPNOTSUPP && (sbflags & CIFS_MOUNT_SERVER_INUM)) { - cifs_autodisable_serverino(cifs_sb); + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve inode number via query_dir_first", rc); goto ffirst_retry; } error_exit: @@ -732,6 +732,8 @@ find_cifs_entry(const unsigned int xid, struct cifs_tcon *tcon, loff_t pos, if (cfile->srch_inf.smallBuf) cifs_small_buf_release(cfile->srch_inf. ntwrk_buf_start); + else if (cfile->srch_inf.is_dynamic_buf) + kfree(cfile->srch_inf.ntwrk_buf_start); else cifs_buf_release(cfile->srch_inf. ntwrk_buf_start); @@ -1027,7 +1029,7 @@ static int cifs_filldir(char *find_entry, struct file *file, fattr.cf_uniqueid = de.ino; } else { fattr.cf_uniqueid = iunique(sb, ROOT_I); - cifs_autodisable_serverino(cifs_sb); + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve inode number from readdir", 0); } if ((sbflags & CIFS_MOUNT_MF_SYMLINKS) && couldbe_mf_symlink(&fattr)) diff --git a/fs/smb/client/reparse.c b/fs/smb/client/reparse.c index cd1e1eaee67a..5cc5b0410d48 100644 --- a/fs/smb/client/reparse.c +++ b/fs/smb/client/reparse.c @@ -67,6 +67,7 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, char *sym = NULL; struct kvec iov; bool directory; + int path_len; int rc = 0; if (strlen(symname) > REPARSE_SYM_PATH_MAX) @@ -168,7 +169,21 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') sym[0] = sym[1] = sym[2] = sym[5] = '_'; - path = cifs_convert_path_to_utf16(sym, cifs_sb); + /* + * On a POSIX paths mount the symlink target is stored verbatim, so + * convert it with cifs_strndup_to_utf16(). cifs_convert_path_to_utf16() + * must not be used here: it strips a leading path separator (it is + * meant for share-relative SMB paths), which would corrupt an absolute + * POSIX symlink target such as "/foo/bar". Using NO_MAP_UNI_RSVD also + * matches the readback path in smb2_parse_native_symlink(). + */ + if (sbflags & CIFS_MOUNT_POSIX_PATHS) + path = cifs_strndup_to_utf16(sym, strlen(sym), &path_len, + cifs_sb->local_nls, + NO_MAP_UNI_RSVD); + else + path = cifs_convert_path_to_utf16(sym, cifs_sb); + if (!path) { rc = -ENOMEM; goto out; diff --git a/fs/smb/client/smb1maperror.c b/fs/smb/client/smb1maperror.c index 74530088d17d..ab3d09613c91 100644 --- a/fs/smb/client/smb1maperror.c +++ b/fs/smb/client/smb1maperror.c @@ -10,6 +10,9 @@ */ #include <linux/bsearch.h> + +#include <kunit/visibility.h> + #include "cifsproto.h" #include "smb1proto.h" #include "smberr.h" @@ -239,48 +242,45 @@ int __init smb1_init_maperror(void) } #if IS_ENABLED(CONFIG_SMB1_KUNIT_TESTS) -#define EXPORT_SYMBOL_FOR_SMB_TEST(sym) \ - EXPORT_SYMBOL_FOR_MODULES(sym, "smb1maperror_test") - const struct ntstatus_to_dos_err * search_ntstatus_to_dos_map_test(__u32 ntstatus) { return search_ntstatus_to_dos_map(ntstatus); } -EXPORT_SYMBOL_FOR_SMB_TEST(search_ntstatus_to_dos_map_test); +EXPORT_SYMBOL_IF_KUNIT(search_ntstatus_to_dos_map_test); const struct ntstatus_to_dos_err * ntstatus_to_dos_map_test = ntstatus_to_dos_map; -EXPORT_SYMBOL_FOR_SMB_TEST(ntstatus_to_dos_map_test); +EXPORT_SYMBOL_IF_KUNIT(ntstatus_to_dos_map_test); unsigned int ntstatus_to_dos_num = ARRAY_SIZE(ntstatus_to_dos_map); -EXPORT_SYMBOL_FOR_SMB_TEST(ntstatus_to_dos_num); +EXPORT_SYMBOL_IF_KUNIT(ntstatus_to_dos_num); const struct smb_to_posix_error * search_mapping_table_ERRDOS_test(__u16 smb_err) { return search_mapping_table_ERRDOS(smb_err); } -EXPORT_SYMBOL_FOR_SMB_TEST(search_mapping_table_ERRDOS_test); +EXPORT_SYMBOL_IF_KUNIT(search_mapping_table_ERRDOS_test); const struct smb_to_posix_error * mapping_table_ERRDOS_test = mapping_table_ERRDOS; -EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRDOS_test); +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRDOS_test); unsigned int mapping_table_ERRDOS_num = ARRAY_SIZE(mapping_table_ERRDOS); -EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRDOS_num); +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRDOS_num); const struct smb_to_posix_error * search_mapping_table_ERRSRV_test(__u16 smb_err) { return search_mapping_table_ERRSRV(smb_err); } -EXPORT_SYMBOL_FOR_SMB_TEST(search_mapping_table_ERRSRV_test); +EXPORT_SYMBOL_IF_KUNIT(search_mapping_table_ERRSRV_test); const struct smb_to_posix_error * mapping_table_ERRSRV_test = mapping_table_ERRSRV; -EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRSRV_test); +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRSRV_test); unsigned int mapping_table_ERRSRV_num = ARRAY_SIZE(mapping_table_ERRSRV); -EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRSRV_num); +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRSRV_num); #endif diff --git a/fs/smb/client/smb1maperror_test.c b/fs/smb/client/smb1maperror_test.c index 2caaf11228ef..903c46f71291 100644 --- a/fs/smb/client/smb1maperror_test.c +++ b/fs/smb/client/smb1maperror_test.c @@ -75,3 +75,4 @@ kunit_test_suite(maperror_suite); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("KUnit tests of SMB1 maperror"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/fs/smb/client/smb1ops.c b/fs/smb/client/smb1ops.c index e198e3dda917..dc5a8c1da623 100644 --- a/fs/smb/client/smb1ops.c +++ b/fs/smb/client/smb1ops.c @@ -505,21 +505,27 @@ static int cifs_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, const char *full_path) { - int rc; - FILE_ALL_INFO *file_info; + int rc = -EOPNOTSUPP; + FILE_ALL_INFO file_info; - file_info = kmalloc_obj(FILE_ALL_INFO); - if (file_info == NULL) - return -ENOMEM; + if (tcon->ses->capabilities & CAP_NT_SMBS) + rc = CIFSSMBQPathInfo(xid, tcon, full_path, &file_info, + 0 /* not legacy */, cifs_sb->local_nls, + cifs_remap(cifs_sb)); - rc = CIFSSMBQPathInfo(xid, tcon, full_path, file_info, - 0 /* not legacy */, cifs_sb->local_nls, - cifs_remap(cifs_sb)); + /* + * Non-UNICODE variant of fallback functions below expands wildcards, + * so they cannot be used for querying paths with wildcard characters. + * Therefore for such paths returns -ENOENT as they cannot exist. + */ + if ((rc == -EOPNOTSUPP || rc == -EINVAL) && + !(tcon->ses->capabilities & CAP_UNICODE) && + strpbrk(full_path, "*?\"><")) + rc = -ENOENT; if (rc == -EOPNOTSUPP || rc == -EINVAL) - rc = SMBQueryInformation(xid, tcon, full_path, file_info, + rc = SMBQueryInformation(xid, tcon, full_path, &file_info, cifs_sb->local_nls, cifs_remap(cifs_sb)); - kfree(file_info); return rc; } @@ -949,7 +955,7 @@ smb_set_file_info(struct inode *inode, const char *full_path, struct cifs_open_parms oparms; struct cifsFileInfo *open_file; FILE_BASIC_INFO new_buf; - struct cifs_open_info_data query_data; + struct cifs_open_info_data query_data = {}; __le64 write_time = buf->LastWriteTime; struct cifsInodeInfo *cinode = CIFS_I(inode); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); @@ -1110,9 +1116,10 @@ out: static int cifs_set_compression(const unsigned int xid, struct cifs_tcon *tcon, - struct cifsFileInfo *cfile) + struct cifsFileInfo *cfile, __u16 compression_state) { - return CIFSSMB_set_compression(xid, tcon, cfile->fid.netfid); + return CIFSSMB_set_compression(xid, tcon, cfile->fid.netfid, + compression_state); } static int diff --git a/fs/smb/client/smb1pdu.h b/fs/smb/client/smb1pdu.h index 7584e94d9b2b..0870949144ab 100644 --- a/fs/smb/client/smb1pdu.h +++ b/fs/smb/client/smb1pdu.h @@ -1211,11 +1211,6 @@ typedef struct smb_com_transaction_compr_ioctl_req { __le16 compression_state; /* See below for valid flags */ } __packed TRANSACT_COMPR_IOCTL_REQ; -/* compression state flags */ -#define COMPRESSION_FORMAT_NONE 0x0000 -#define COMPRESSION_FORMAT_DEFAULT 0x0001 -#define COMPRESSION_FORMAT_LZNT1 0x0002 - typedef struct smb_com_transaction_ioctl_rsp { struct smb_hdr hdr; /* wct = 19 */ __u8 Reserved[3]; diff --git a/fs/smb/client/smb1proto.h b/fs/smb/client/smb1proto.h index 5f522d359952..80eaeb3dd2ec 100644 --- a/fs/smb/client/smb1proto.h +++ b/fs/smb/client/smb1proto.h @@ -117,7 +117,7 @@ struct inode *cifs_create_reparse_inode(struct cifs_open_info_data *data, struct kvec *reparse_iov, struct kvec *xattr_iov); int CIFSSMB_set_compression(const unsigned int xid, struct cifs_tcon *tcon, - __u16 fid); + __u16 fid, __u16 compression_state); int cifs_do_get_acl(const unsigned int xid, struct cifs_tcon *tcon, const unsigned char *searchName, struct posix_acl **acl, const int acl_type, const struct nls_table *nls_codepage, diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index b292aa94a593..5ef919bce52d 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -49,6 +49,9 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) __func__, le32_to_cpu(p->ErrorId)); len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); + if (len > end - ((u8 *)p + sizeof(*p))) + return ERR_PTR(-EINVAL); + p = (struct smb2_error_context_rsp *)(p->ErrorContextData + len); } } else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) && @@ -151,8 +154,6 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *smb2_path; __u8 smb2_oplock; struct cifs_open_info_data *data = buf; - struct smb2_file_all_info file_info = {}; - struct smb2_file_all_info *smb2_data = data ? &file_info : NULL; struct kvec err_iov = {}; int err_buftype = CIFS_NO_BUFFER; struct cifs_fid *fid = oparms->fid; @@ -179,14 +180,14 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, } smb2_oplock = SMB2_OPLOCK_LEVEL_BATCH; - rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, data, NULL, &err_iov, &err_buftype); if (rc == -EACCES && retry_without_read_attributes) { free_rsp_buf(err_buftype, err_iov.iov_base); memset(&err_iov, 0, sizeof(err_iov)); err_buftype = CIFS_NO_BUFFER; oparms->desired_access &= ~FILE_READ_ATTRIBUTES; - rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, data, NULL, &err_iov, &err_buftype); } if (rc && data) { @@ -199,9 +200,9 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, oparms->path, &data->symlink_target); if (!rc) { - memset(smb2_data, 0, sizeof(*smb2_data)); + memset(&data->fi, 0, sizeof(data->fi)); oparms->create_options |= OPEN_REPARSE_POINT; - rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, data, NULL, NULL, NULL); oparms->create_options &= ~OPEN_REPARSE_POINT; } @@ -235,23 +236,22 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, rc = 0; } - if (smb2_data) { + if (data) { /* if open response does not have IndexNumber field - get it */ - if (smb2_data->IndexNumber == 0) { + if (data->fi.IndexNumber == 0) { rc = SMB2_get_srv_num(xid, oparms->tcon, fid->persistent_fid, fid->volatile_fid, - &smb2_data->IndexNumber); + &data->fi.IndexNumber); if (rc) { /* * let get_inode_info disable server inode * numbers */ - smb2_data->IndexNumber = 0; + data->fi.IndexNumber = 0; rc = 0; } } - memcpy(&data->fi, smb2_data, sizeof(data->fi)); } *oplock = smb2_oplock; diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 286912616c73..6c9c229b91f6 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -111,7 +111,7 @@ static int check_wsl_eas(struct kvec *rsp_iov) u32 outlen, next; u16 vlen; u8 nlen; - u8 *end; + u8 *ea_end, *iov_end; outlen = le32_to_cpu(rsp->OutputBufferLength); if (outlen < SMB2_WSL_MIN_QUERY_EA_RESP_SIZE || @@ -120,15 +120,19 @@ static int check_wsl_eas(struct kvec *rsp_iov) ea = (void *)((u8 *)rsp_iov->iov_base + le16_to_cpu(rsp->OutputBufferOffset)); - end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + ea_end = (u8 *)ea + outlen; + iov_end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + if (ea_end > iov_end) + return -EINVAL; + for (;;) { - if ((u8 *)ea > end - sizeof(*ea)) + if ((u8 *)ea > ea_end - sizeof(*ea)) return -EINVAL; nlen = ea->ea_name_length; vlen = le16_to_cpu(ea->ea_value_length); if (nlen != SMB2_WSL_XATTR_NAME_LEN || - (u8 *)ea->ea_data + nlen + 1 + vlen > end) + (u8 *)ea->ea_data + nlen + 1 + vlen > ea_end) return -EINVAL; switch (vlen) { diff --git a/fs/smb/client/smb2maperror.c b/fs/smb/client/smb2maperror.c index 9ed21f7b618c..d86f2460d0e5 100644 --- a/fs/smb/client/smb2maperror.c +++ b/fs/smb/client/smb2maperror.c @@ -8,6 +8,9 @@ * */ #include <linux/errno.h> + +#include <kunit/visibility.h> + #include "cifsproto.h" #include "cifs_debug.h" #include "smb2proto.h" @@ -109,18 +112,15 @@ int __init smb2_init_maperror(void) } #if IS_ENABLED(CONFIG_SMB_KUNIT_TESTS) -#define EXPORT_SYMBOL_FOR_SMB_TEST(sym) \ - EXPORT_SYMBOL_FOR_MODULES(sym, "smb2maperror_test") - const struct status_to_posix_error *smb2_get_err_map_test(__u32 smb2_status) { return smb2_get_err_map(smb2_status); } -EXPORT_SYMBOL_FOR_SMB_TEST(smb2_get_err_map_test); +EXPORT_SYMBOL_IF_KUNIT(smb2_get_err_map_test); const struct status_to_posix_error *smb2_error_map_table_test = smb2_error_map_table; -EXPORT_SYMBOL_FOR_SMB_TEST(smb2_error_map_table_test); +EXPORT_SYMBOL_IF_KUNIT(smb2_error_map_table_test); unsigned int smb2_error_map_num = ARRAY_SIZE(smb2_error_map_table); -EXPORT_SYMBOL_FOR_SMB_TEST(smb2_error_map_num); +EXPORT_SYMBOL_IF_KUNIT(smb2_error_map_num); #endif diff --git a/fs/smb/client/smb2maperror_test.c b/fs/smb/client/smb2maperror_test.c index 0f8a44a5ed3c..44dc5e899cad 100644 --- a/fs/smb/client/smb2maperror_test.c +++ b/fs/smb/client/smb2maperror_test.c @@ -47,3 +47,4 @@ kunit_test_suite(maperror_suite); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("KUnit tests of SMB2 maperror"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 973fce3c959c..9068175e57cd 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -19,6 +19,9 @@ #include "nterr.h" #include "cached_dir.h" +static unsigned int __smb2_calc_size(void *buf, bool *have_data, + bool *data_area_overlap); + static int check_smb2_hdr(struct smb2_hdr *shdr, __u64 mid) { @@ -145,6 +148,8 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, int command; __u32 calc_len; /* calculated length */ __u64 mid; + bool have_data; + bool data_area_overlap; /* If server is a channel, select the primary channel */ pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; @@ -228,7 +233,13 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, } } - calc_len = smb2_calc_size(buf); + have_data = false; + data_area_overlap = false; + calc_len = __smb2_calc_size(buf, &have_data, &data_area_overlap); + + /* Reject responses whose data area overlaps the fixed area. */ + if (data_area_overlap) + return 1; /* For SMB2_IOCTL, OutputOffset and OutputLength are optional, so might * be 0, and not a real miscalculation */ @@ -241,13 +252,19 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, if (len != calc_len) { /* create failed on symlink */ if (command == SMB2_CREATE_HE && - shdr->Status == STATUS_STOPPED_ON_SYMLINK) + shdr->Status == STATUS_STOPPED_ON_SYMLINK && + len > calc_len) return 0; /* Windows 7 server returns 24 bytes more */ if (calc_len + 24 == len && command == SMB2_OPLOCK_BREAK_HE) return 0; - /* server can return one byte more due to implied bcc[0] */ - if (calc_len == len + 1) + /* + * Server can return one byte more due to implied bcc[0]. + * Allow it only when there is no data area; if data_length > 0 + * the +1 gap indicates an overreported data length rather than + * the bcc[0] omission. + */ + if (calc_len == len + 1 && !have_data) return 0; /* @@ -406,19 +423,28 @@ smb2_get_data_area_len(int *off, int *len, struct smb2_hdr *shdr) } /* - * Calculate the size of the SMB message based on the fixed header - * portion, the number of word parameters and the data portion of the message. + * Calculate the size of the SMB message based on the fixed header, fixed + * parameter area, and variable data area. + * + * If have_data is not NULL, it is set when a non-empty data area is found. + * If data_area_overlap is not NULL, it is set when the data area overlaps + * the fixed area. */ -unsigned int -smb2_calc_size(void *buf) +static unsigned int +__smb2_calc_size(void *buf, bool *have_data, bool *data_area_overlap) { struct smb2_pdu *pdu = buf; struct smb2_hdr *shdr = &pdu->hdr; int offset; /* the offset from the beginning of SMB to data area */ - int data_length; /* the length of the variable length data area */ + int data_length = 0; /* the length of the variable length data area */ /* Structure Size has already been checked to make sure it is 64 */ int len = le16_to_cpu(shdr->StructureSize); + if (have_data) + *have_data = false; + if (data_area_overlap) + *data_area_overlap = false; + /* * StructureSize2, ie length of fixed parameter area has already * been checked to make sure it is the correct length. @@ -441,16 +467,27 @@ smb2_calc_size(void *buf) if (offset + 1 < len) { cifs_dbg(VFS, "data area offset %d overlaps SMB2 header %d\n", offset + 1, len); + if (data_area_overlap) + *data_area_overlap = true; data_length = 0; + goto calc_size_exit; } else { len = offset + data_length; } } calc_size_exit: cifs_dbg(FYI, "SMB2 len %d\n", len); + if (have_data) + *have_data = (data_length > 0); return len; } +unsigned int +smb2_calc_size(void *buf) +{ + return __smb2_calc_size(buf, NULL, NULL); +} + /* Note: caller must free return buffer */ __le16 * cifs_convert_path_to_utf16(const char *from, struct cifs_sb_info *cifs_sb) diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index e6cb9b144530..cbd51a08e97e 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -1772,8 +1772,8 @@ replay_again: if (le32_to_cpu(io_rsp->OutputCount) < qi.input_buffer_length) qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount); if (qi.input_buffer_length > 0 && - le32_to_cpu(io_rsp->OutputOffset) + qi.input_buffer_length - > rsp_iov[1].iov_len) { + size_add(le32_to_cpu(io_rsp->OutputOffset), + qi.input_buffer_length) > rsp_iov[1].iov_len) { rc = -EFAULT; goto out; } @@ -2117,8 +2117,9 @@ smb2_sync_write(const unsigned int xid, struct cifs_fid *pfid, } /* Set or clear the SPARSE_FILE attribute based on value passed in setsparse */ -static bool smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, - struct cifsFileInfo *cfile, struct inode *inode, __u8 setsparse) +static int smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct inode *inode, + __u8 setsparse) { struct cifsInodeInfo *cifsi; int rc; @@ -2127,31 +2128,31 @@ static bool smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, /* if file already sparse don't bother setting sparse again */ if ((cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) && setsparse) - return true; /* already sparse */ + return 0; /* already sparse */ if (!(cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) && !setsparse) - return true; /* already not sparse */ + return 0; /* already not sparse */ /* * Can't check for sparse support on share the usual way via the * FS attribute info (FILE_SUPPORTS_SPARSE_FILES) on the share * since Samba server doesn't set the flag on the share, yet * supports the set sparse FSCTL and returns sparse correctly - * in the file attributes. If we fail setting sparse though we - * mark that server does not support sparse files for this share - * to avoid repeatedly sending the unsupported fsctl to server - * if the file is repeatedly extended. + * in the file attributes. If the server returns EOPNOTSUPP, mark + * that sparse files are not supported on this share to avoid + * repeatedly sending the unsupported FSCTL. */ if (tcon->broken_sparse_sup) - return false; + return -EOPNOTSUPP; rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, FSCTL_SET_SPARSE, &setsparse, 1, CIFSMaxBufSize, NULL, NULL); if (rc) { - tcon->broken_sparse_sup = true; + if (rc == -EOPNOTSUPP) + tcon->broken_sparse_sup = true; cifs_dbg(FYI, "set sparse rc = %d\n", rc); - return false; + return rc; } if (setsparse) @@ -2159,7 +2160,7 @@ static bool smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, else cifsi->cifsAttrs &= (~FILE_ATTRIBUTE_SPARSE_FILE); - return true; + return 0; } static int @@ -2192,10 +2193,14 @@ smb2_duplicate_extents(const unsigned int xid, u64 len, u64 dest_off) { int rc; + int qrc; unsigned int ret_data_len; struct inode *inode; + struct smb2_file_all_info file_inf; struct duplicate_extents_to_file dup_ext_buf; + struct timespec64 ts; struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink); + u64 asize; /* server fileays advertise duplicate extent support with this flag */ if ((le32_to_cpu(tcon->fsAttrInfo.Attributes) & @@ -2231,6 +2236,32 @@ smb2_duplicate_extents(const unsigned int xid, if (ret_data_len > 0) cifs_dbg(FYI, "Non-zero response length in duplicate extents\n"); + if (rc == 0) { + qrc = SMB2_query_info(xid, tcon, trgtfile->fid.persistent_fid, + trgtfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + CIFS_I(inode)->time = jiffies; + if (file_inf.LastWriteTime) { + ts = cifs_NTtimeToUnix(file_inf.LastWriteTime); + inode_set_mtime_to_ts(inode, ts); + } + if (file_inf.ChangeTime) { + ts = cifs_NTtimeToUnix(file_inf.ChangeTime); + inode_set_ctime_to_ts(inode, ts); + } + if (file_inf.LastAccessTime) { + ts = cifs_NTtimeToUnix(file_inf.LastAccessTime); + inode_set_atime_to_ts(inode, ts); + } + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + CIFS_I(inode)->time = 0; /* force reval */ + } + spin_unlock(&inode->i_lock); + } + duplicate_extents_out: if (rc) trace_smb3_clone_err(xid, srcfile->fid.volatile_fid, @@ -2246,10 +2277,10 @@ duplicate_extents_out: static int smb2_set_compression(const unsigned int xid, struct cifs_tcon *tcon, - struct cifsFileInfo *cfile) + struct cifsFileInfo *cfile, __u16 compression_state) { return SMB2_set_compression(xid, tcon, cfile->fid.persistent_fid, - cfile->fid.volatile_fid); + cfile->fid.volatile_fid, compression_state); } static int @@ -3402,8 +3433,7 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, struct inode *inode = file_inode(file); struct cifsInodeInfo *cifsi = CIFS_I(inode); struct cifsFileInfo *cfile = file->private_data; - struct netfs_inode *ictx = netfs_inode(inode); - unsigned long long i_size, new_size, remote_size; + unsigned long long i_size, new_size, remote_i_size, zero_point; long rc; unsigned int xid; @@ -3414,9 +3444,8 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, filemap_invalidate_lock(inode->i_mapping); - i_size = i_size_read(inode); - remote_size = ictx->remote_i_size; - if (offset + len >= remote_size && offset < i_size) { + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); + if (offset + len >= remote_i_size && offset < i_size) { unsigned long long top = umin(offset + len, i_size); rc = filemap_write_and_wait_range(inode->i_mapping, offset, top - 1); @@ -3449,9 +3478,11 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, cfile->fid.volatile_fid, cfile->pid, new_size); if (rc >= 0) { truncate_setsize(inode, new_size); + spin_lock(&inode->i_lock); netfs_resize_file(&cifsi->netfs, new_size, true); - if (offset < cifsi->netfs.zero_point) - cifsi->netfs.zero_point = offset; + if (offset < cifsi->netfs._zero_point) + netfs_write_zero_point(inode, offset); + spin_unlock(&inode->i_lock); fscache_resize_cookie(cifs_inode_cookie(inode), new_size); } } @@ -3474,7 +3505,7 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, struct inode *inode = file_inode(file); struct cifsFileInfo *cfile = file->private_data; struct file_zero_data_information fsctl_buf; - unsigned long long end = offset + len, i_size, remote_i_size; + unsigned long long end = offset + len, i_size, remote_i_size, zero_point; long rc; unsigned int xid; __u8 set_sparse = 1; @@ -3483,13 +3514,21 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, /* Need to make file sparse, if not already, before freeing range. */ /* Consider adding equivalent for compressed since it could also work */ - if (!smb2_set_sparse(xid, tcon, cfile, inode, set_sparse)) { - rc = -EOPNOTSUPP; + rc = smb2_set_sparse(xid, tcon, cfile, inode, set_sparse); + if (rc) goto out; - } filemap_invalidate_lock(inode->i_mapping); /* + * Flush dirty data first, otherwise a dirty folio spanning the punched + * range may be written back after the ioctl and refill the hole. + */ + rc = filemap_write_and_wait_range(inode->i_mapping, offset, + offset + len - 1); + if (rc < 0) + goto unlock; + + /* * We implement the punch hole through ioctl, so we need remove the page * caches first, otherwise the data may be inconsistent with the server. */ @@ -3516,14 +3555,17 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, * that we locally hole-punch the tail of the dirty data, the proposed * EOF update will end up in the wrong place. */ - i_size = i_size_read(inode); - remote_i_size = netfs_inode(inode)->remote_i_size; + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); + if (end > remote_i_size && i_size > remote_i_size) { unsigned long long extend_to = umin(end, i_size); rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, extend_to); - if (rc >= 0) - netfs_inode(inode)->remote_i_size = extend_to; + if (rc >= 0) { + spin_lock(&inode->i_lock); + netfs_write_remote_i_size(inode, extend_to); + spin_unlock(&inode->i_lock); + } } unlock: @@ -3540,7 +3582,7 @@ static int smb3_simple_fallocate_write_range(unsigned int xid, char *buf) { struct cifs_io_parms io_parms = {0}; - int nbytes; + unsigned int nbytes; int rc = 0; struct kvec iov[2]; @@ -3561,9 +3603,10 @@ static int smb3_simple_fallocate_write_range(unsigned int xid, rc = SMB2_write(xid, &io_parms, &nbytes, iov, 1); if (rc) break; + if (!nbytes) + return -EIO; if (nbytes > len) return -EINVAL; - buf += nbytes; off += nbytes; len -= nbytes; } @@ -3576,11 +3619,25 @@ static int smb3_simple_fallocate_range(unsigned int xid, loff_t off, loff_t len) { struct file_allocated_range_buffer in_data, *out_data = NULL, *tmp_data; + struct inode *inode = d_inode(cfile->dentry); u32 out_data_len; char *buf = NULL; + u64 range_start, range_len, range_end; loff_t l; int rc; + buf = kvzalloc(min_t(loff_t, len, SMB2_MAX_BUFFER_SIZE), GFP_KERNEL); + if (!buf) { + rc = -ENOMEM; + goto out; + } + + if (off >= i_size_read(inode)) { + rc = smb3_simple_fallocate_write_range(xid, tcon, cfile, + off, len, buf); + goto out; + } + in_data.file_offset = cpu_to_le64(off); in_data.length = cpu_to_le64(len); rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, @@ -3592,12 +3649,6 @@ static int smb3_simple_fallocate_range(unsigned int xid, if (rc) goto out; - buf = kzalloc(1024 * 1024, GFP_KERNEL); - if (buf == NULL) { - rc = -ENOMEM; - goto out; - } - tmp_data = out_data; while (len) { /* @@ -3614,13 +3665,21 @@ static int smb3_simple_fallocate_range(unsigned int xid, goto out; } - if (off < le64_to_cpu(tmp_data->file_offset)) { + range_start = le64_to_cpu(tmp_data->file_offset); + range_len = le64_to_cpu(tmp_data->length); + if (check_add_overflow(range_start, range_len, &range_end) || + range_end > S64_MAX) { + rc = -EINVAL; + goto out; + } + + if (off < range_start) { /* * We are at a hole. Write until the end of the region * or until the next allocated data, * whichever comes next. */ - l = le64_to_cpu(tmp_data->file_offset) - off; + l = range_start - off; if (len < l) l = len; rc = smb3_simple_fallocate_write_range(xid, tcon, @@ -3637,11 +3696,13 @@ static int smb3_simple_fallocate_range(unsigned int xid, * until the end of the data or the end of the region * we are supposed to fallocate, whichever comes first. */ - l = le64_to_cpu(tmp_data->length); - if (len < l) - l = len; - off += l; - len -= l; + if (off < range_end) { + l = range_end - off; + if (len < l) + l = len; + off += l; + len -= l; + } tmp_data = &tmp_data[1]; out_data_len -= sizeof(struct file_allocated_range_buffer); @@ -3649,7 +3710,7 @@ static int smb3_simple_fallocate_range(unsigned int xid, out: kfree(out_data); - kfree(buf); + kvfree(buf); return rc; } @@ -3662,18 +3723,22 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, struct cifsFileInfo *cfile = file->private_data; long rc = -EOPNOTSUPP; unsigned int xid; - loff_t new_eof; + loff_t old_eof, new_eof; + struct smb2_file_all_info file_inf; + u64 asize; + int qrc; xid = get_xid(); inode = d_inode(cfile->dentry); cifsi = CIFS_I(inode); + old_eof = i_size_read(inode); trace_smb3_falloc_enter(xid, cfile->fid.persistent_fid, tcon->tid, tcon->ses->Suid, off, len); /* if file not oplocked can't be sure whether asking to extend size */ if (!CIFS_CACHE_READ(cifsi)) - if (keep_size == false) { + if (!keep_size) { trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, tcon->tid, tcon->ses->Suid, off, len, rc); free_xid(xid); @@ -3683,21 +3748,98 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, /* * Extending the file */ - if ((keep_size == false) && i_size_read(inode) < off + len) { + if (!keep_size && old_eof < off + len) { rc = inode_newsize_ok(inode, off + len); if (rc) goto out; + /* + * A small range at or beyond EOF can be allocated by writing + * zeroes. For off > old_eof, this preserves the intervening + * hole instead of allocating from offset 0. + */ + if (off > old_eof || + (off == old_eof && old_eof != 0 && + (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE))) { + if (len > 1024 * 1024) { + rc = -EOPNOTSUPP; + goto out; + } + + rc = smb3_simple_fallocate_range(xid, tcon, cfile, + off, len); + if (rc) { + spin_lock(&inode->i_lock); + cifsi->time = 0; + spin_unlock(&inode->i_lock); + goto out; + } + + new_eof = off + len; + netfs_resize_file(&cifsi->netfs, new_eof, true); + cifs_setsize(inode, new_eof); + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + cifsi->time = 0; + } + spin_unlock(&inode->i_lock); + goto out; + } + if (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) smb2_set_sparse(xid, tcon, cfile, inode, false); new_eof = off + len; + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + if (qrc == 0) + asize = le64_to_cpu(file_inf.AllocationSize); + + /* + * FILE_ALLOCATION_INFORMATION can only describe allocation up to + * new_eof. Some servers may accept it without allocating blocks, + * so refresh AllocationSize before updating i_blocks. + */ + if (off == 0 || off == old_eof) { + if (qrc || asize < new_eof) { + rc = SMB2_set_allocation(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + cfile->pid, new_eof); + if (rc) + goto out; + } + } + rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, new_eof); - if (rc == 0) { - netfs_resize_file(&cifsi->netfs, new_eof, true); - cifs_setsize(inode, new_eof); + if (rc) + goto out; + + netfs_resize_file(&cifsi->netfs, new_eof, true); + cifs_setsize(inode, new_eof); + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + if (asize >= new_eof) + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + cifsi->time = 0; } + spin_unlock(&inode->i_lock); goto out; } @@ -3787,7 +3929,6 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, struct inode *inode = file_inode(file); struct cifsInodeInfo *cifsi = CIFS_I(inode); struct cifsFileInfo *cfile = file->private_data; - struct netfs_inode *ictx = &cifsi->netfs; loff_t old_eof, new_eof; xid = get_xid(); @@ -3805,7 +3946,9 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, goto out_2; truncate_pagecache_range(inode, off, old_eof); - ictx->zero_point = old_eof; + spin_lock(&inode->i_lock); + netfs_write_zero_point(inode, old_eof); + spin_unlock(&inode->i_lock); netfs_wait_for_outstanding_io(inode); rc = smb2_copychunk_range(xid, cfile, cfile, off + len, @@ -3822,8 +3965,10 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, rc = 0; truncate_setsize(inode, new_eof); + spin_lock(&inode->i_lock); netfs_resize_file(&cifsi->netfs, new_eof, true); - ictx->zero_point = new_eof; + netfs_write_zero_point(inode, new_eof); + spin_unlock(&inode->i_lock); fscache_resize_cookie(cifs_inode_cookie(inode), new_eof); out_2: filemap_invalidate_unlock(inode->i_mapping); @@ -3866,13 +4011,17 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, goto out_2; truncate_setsize(inode, new_eof); + spin_lock(&inode->i_lock); netfs_resize_file(&cifsi->netfs, i_size_read(inode), true); + spin_unlock(&inode->i_lock); fscache_resize_cookie(cifs_inode_cookie(inode), i_size_read(inode)); rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len); if (rc < 0) goto out_2; - cifsi->netfs.zero_point = new_eof; + spin_lock(&inode->i_lock); + netfs_write_zero_point(inode, new_eof); + spin_unlock(&inode->i_lock); rc = smb3_zero_data(file, tcon, off, len, xid); if (rc < 0) @@ -4061,7 +4210,7 @@ static long smb3_fallocate(struct file *file, struct cifs_tcon *tcon, int mode, return smb3_collapse_range(file, tcon, off, len); else if (mode == FALLOC_FL_INSERT_RANGE) return smb3_insert_range(file, tcon, off, len); - else if (mode == 0) + else if (mode == FALLOC_FL_ALLOCATE_RANGE) return smb3_simple_falloc(file, tcon, off, len, false); return -EOPNOTSUPP; @@ -4349,11 +4498,13 @@ static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst unsigned int req_size = sizeof(**req) + crypto_aead_reqsize(tfm); unsigned int iv_size = crypto_aead_ivsize(tfm); unsigned int len; + int ret; u8 *p; - *num_sgs = cifs_get_num_sgs(rqst, num_rqst, sig); - if (IS_ERR_VALUE((long)(int)*num_sgs)) - return ERR_PTR(*num_sgs); + ret = cifs_get_num_sgs(rqst, num_rqst, sig); + if (ret < 0) + return ERR_PTR(ret); + *num_sgs = ret; len = iv_size; len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1); @@ -4696,9 +4847,15 @@ cifs_copy_folioq_to_iter(struct folio_queue *folioq, size_t data_size, { for (; folioq; folioq = folioq->next) { for (int s = 0; s < folioq_count(folioq); s++) { - struct folio *folio = folioq_folio(folioq, s); - size_t fsize = folio_size(folio); - size_t n, len = umin(fsize - skip, data_size); + struct folio *folio; + size_t fsize, n, len; + + if (data_size == 0) + return 0; + + folio = folioq_folio(folioq, s); + fsize = folio_size(folio); + len = umin(fsize - skip, data_size); n = copy_folio_to_iter(folio, skip, len, iter); if (n != len) { @@ -4711,6 +4868,12 @@ cifs_copy_folioq_to_iter(struct folio_queue *folioq, size_t data_size, } } + if (data_size != 0) { + cifs_dbg(VFS, "%s: short copy, %zu bytes missing\n", + __func__, data_size); + return smb_EIO2(smb_eio_trace_rx_copy_to_iter, 0, data_size); + } + return 0; } @@ -4721,6 +4884,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid, { unsigned int data_offset; unsigned int data_len; + unsigned int end_off; unsigned int cur_off; unsigned int cur_page_idx; unsigned int pad_len; @@ -4825,7 +4989,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid, } /* Copy the data to the output I/O iterator. */ - rdata->result = cifs_copy_folioq_to_iter(buffer, buffer_len, + rdata->result = cifs_copy_folioq_to_iter(buffer, data_len, cur_off, &rdata->subreq.io_iter); if (rdata->result != 0) { if (is_offloaded) @@ -4834,9 +4998,10 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid, dequeue_mid(server, mid, rdata->result); return 0; } - rdata->got_bytes = buffer_len; + rdata->got_bytes = data_len; - } else if (buf_len >= data_offset + data_len) { + } else if (!check_add_overflow(data_offset, data_len, &end_off) && + buf_len >= end_off) { /* read response payload is in buf */ WARN_ONCE(buffer, "read data can be either in buf or in buffer"); copied = copy_to_iter(buf + data_offset, data_len, &rdata->subreq.io_iter); @@ -5085,6 +5250,12 @@ receive_encrypted_standard(struct TCP_Server_Info *server, one_more: shdr = (struct smb2_hdr *)buf; next_cmd = le32_to_cpu(shdr->NextCommand); + + if (*num_mids >= MAX_COMPOUND) { + cifs_server_dbg(VFS, "too many PDUs in compound\n"); + return -1; + } + if (next_cmd) { if (WARN_ON_ONCE(next_cmd > pdu_length)) return -1; @@ -5108,10 +5279,6 @@ one_more: mid_entry->resp_buf_size = server->pdu_size; } - if (*num_mids >= MAX_COMPOUND) { - cifs_server_dbg(VFS, "too many PDUs in compound\n"); - return -1; - } bufs[*num_mids] = buf; mids[(*num_mids)++] = mid_entry; @@ -5209,7 +5376,7 @@ int __cifs_sfu_make_node(unsigned int xid, struct inode *inode, { struct TCP_Server_Info *server = tcon->ses->server; struct cifs_open_parms oparms; - struct cifs_open_info_data idata; + struct cifs_open_info_data idata = {}; struct cifs_io_parms io_parms = {}; struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); struct cifs_fid fid; @@ -5258,16 +5425,13 @@ int __cifs_sfu_make_node(unsigned int xid, struct inode *inode, data = (u8 *)symname_utf16; break; case S_IFSOCK: - type_len = 8; - strscpy(type, "LnxSOCK"); - data = (u8 *)&pdev; - data_len = sizeof(pdev); + /* SFU socket is system file with one zero byte */ + type_len = 1; + type[0] = '\0'; break; case S_IFIFO: - type_len = 8; - strscpy(type, "LnxFIFO"); - data = (u8 *)&pdev; - data_len = sizeof(pdev); + /* SFU fifo is system file which is empty */ + type_len = 0; break; default: rc = -EPERM; diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index cb61051f9af3..4ce165e40657 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -636,10 +636,16 @@ build_compression_ctxt(struct smb2_compression_capabilities_context *pneg_ctxt) pneg_ctxt->DataLength = cpu_to_le16(sizeof(struct smb2_compression_capabilities_context) - sizeof(struct smb2_neg_context)); - pneg_ctxt->CompressionAlgorithmCount = cpu_to_le16(3); + /* + * Pattern_V1 is useful only as part of a chained transform. LZ77 remains + * the preferred general-purpose algorithm selected by this client. + */ + pneg_ctxt->CompressionAlgorithmCount = cpu_to_le16(4); + pneg_ctxt->Flags = SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED; pneg_ctxt->CompressionAlgorithms[0] = SMB3_COMPRESS_LZ77; pneg_ctxt->CompressionAlgorithms[1] = SMB3_COMPRESS_LZ77_HUFF; pneg_ctxt->CompressionAlgorithms[2] = SMB3_COMPRESS_LZNT1; + pneg_ctxt->CompressionAlgorithms[3] = SMB3_COMPRESS_PATTERN; } static unsigned int @@ -827,9 +833,12 @@ static void decode_compress_ctx(struct TCP_Server_Info *server, struct smb2_compression_capabilities_context *ctxt) { unsigned int len = le16_to_cpu(ctxt->DataLength); - __le16 alg; + unsigned int count, i; server->compression.enabled = false; + server->compression.chained = false; + server->compression.pattern = false; + server->compression.alg = SMB3_COMPRESS_NONE; /* * Caller checked that DataLength remains within SMB boundary. We still @@ -841,20 +850,37 @@ static void decode_compress_ctx(struct TCP_Server_Info *server, return; } - if (le16_to_cpu(ctxt->CompressionAlgorithmCount) != 1) { + count = le16_to_cpu(ctxt->CompressionAlgorithmCount); + if (!count || count > ARRAY_SIZE(ctxt->CompressionAlgorithms) || + len < 8 + count * sizeof(__le16)) { pr_warn_once("invalid SMB3 compress algorithm count\n"); return; } - alg = ctxt->CompressionAlgorithms[0]; - - /* 'NONE' (0) compressor type is never negotiated */ - if (alg == 0 || le16_to_cpu(alg) > 3) { - pr_warn_once("invalid compression algorithm '%u'\n", alg); + if (ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_NONE && + ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED) { + pr_warn_once("invalid SMB3 compression flags\n"); return; } - server->compression.alg = alg; + for (i = 0; i < count; i++) { + /* Record the intersection supported by the shared SMB codec. */ + if (ctxt->CompressionAlgorithms[i] == SMB3_COMPRESS_LZ77) + server->compression.alg = SMB3_COMPRESS_LZ77; + else if (ctxt->CompressionAlgorithms[i] == SMB3_COMPRESS_PATTERN) + server->compression.pattern = true; + } + if (server->compression.alg != SMB3_COMPRESS_LZ77) + return; + + /* + * Pattern_V1 cannot appear in an unchained transform even if a broken + * peer lists it in the algorithm array. + */ + server->compression.chained = + ctxt->Flags == SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED; + if (!server->compression.chained) + server->compression.pattern = false; server->compression.enabled = true; } @@ -1713,17 +1739,30 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) is_binding = (ses->ses_status == SES_GOOD); spin_unlock(&ses->ses_lock); + /* + * Per MS-SMB2 3.2.5.3, Session.SessionKey is the first 16 bytes of the + * GSS cryptographic key, right-padded with zero bytes if shorter. + * Allocate at least SMB2_NTLMV2_SESSKEY_SIZE bytes (zeroed) so the KDF + * input buffer is always valid for HMAC-SHA256 even with deprecated + * Kerberos enctypes that return a short session key. + */ + if (unlikely(msg->sesskey_len < SMB2_NTLMV2_SESSKEY_SIZE)) + cifs_dbg(VFS, + "short GSS session key (%u bytes); zero-padding per MS-SMB2 3.2.5.3\n", + msg->sesskey_len); + kfree_sensitive(ses->auth_key.response); - ses->auth_key.response = kmemdup(msg->data, - msg->sesskey_len, - GFP_KERNEL); + ses->auth_key.len = max_t(unsigned int, msg->sesskey_len, + SMB2_NTLMV2_SESSKEY_SIZE); + ses->auth_key.response = kzalloc(ses->auth_key.len, GFP_KERNEL); if (!ses->auth_key.response) { cifs_dbg(VFS, "%s: can't allocate (%u bytes) memory\n", - __func__, msg->sesskey_len); + __func__, ses->auth_key.len); + ses->auth_key.len = 0; rc = -ENOMEM; goto out_put_spnego_key; } - ses->auth_key.len = msg->sesskey_len; + memcpy(ses->auth_key.response, msg->data, msg->sesskey_len); sess_data->iov[1].iov_base = msg->data + msg->sesskey_len; sess_data->iov[1].iov_len = msg->secblob_len; @@ -2116,8 +2155,8 @@ SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, unc_path_len = cifs_strtoUTF16(unc_path, tree, strlen(tree), cp); if (unc_path_len <= 0) { - kfree(unc_path); - return -EINVAL; + rc = -EINVAL; + goto free_unc_path; } unc_path_len *= 2; @@ -2126,10 +2165,8 @@ SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, atomic_set(&tcon->num_remote_opens, 0); rc = smb2_plain_req_init(SMB2_TREE_CONNECT, tcon, server, (void **) &req, &total_len); - if (rc) { - kfree(unc_path); - return rc; - } + if (rc) + goto free_unc_path; if (smb3_encryption_required(tcon)) flags |= CIFS_TRANSFORM_REQ; @@ -2220,6 +2257,7 @@ SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, tcon_exit: free_rsp_buf(resp_buftype, rsp); +free_unc_path: kfree(unc_path); return rc; @@ -2358,9 +2396,9 @@ parse_posix_ctxt(struct create_context *cc, struct smb2_file_all_info *info, memset(posix, 0, sizeof(*posix)); - posix->nlink = le32_to_cpu(*(__le32 *)(beg + 0)); - posix->reparse_tag = le32_to_cpu(*(__le32 *)(beg + 4)); - posix->mode = le32_to_cpu(*(__le32 *)(beg + 8)); + posix->nlink = get_unaligned_le32(beg); + posix->reparse_tag = get_unaligned_le32(beg + 4); + posix->mode = get_unaligned_le32(beg + 8); sid = beg + 12; sid_len = posix_info_sid_size(sid, end); @@ -3249,7 +3287,7 @@ SMB2_open_free(struct smb_rqst *rqst) int SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, - __u8 *oplock, struct smb2_file_all_info *buf, + __u8 *oplock, struct cifs_open_info_data *buf, struct create_posix_rsp *posix, struct kvec *err_iov, int *buftype) { @@ -3264,9 +3302,12 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, int rc = 0; int flags = 0; int retries = 0, cur_sleep = 0; + struct smb2_file_all_info *file_info = buf ? &buf->fi : NULL; replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); oparms->replay = !!(retries); @@ -3330,21 +3371,22 @@ replay_again: oparms->fid->mid = le64_to_cpu(rsp->hdr.MessageId); #endif /* CIFS_DEBUG2 */ - if (buf) { - buf->CreationTime = rsp->CreationTime; - buf->LastAccessTime = rsp->LastAccessTime; - buf->LastWriteTime = rsp->LastWriteTime; - buf->ChangeTime = rsp->ChangeTime; - buf->AllocationSize = rsp->AllocationSize; - buf->EndOfFile = rsp->EndofFile; - buf->Attributes = rsp->FileAttributes; - buf->NumberOfLinks = cpu_to_le32(1); - buf->DeletePending = 0; /* successful open = not delete pending */ + if (file_info) { + file_info->CreationTime = rsp->CreationTime; + file_info->LastAccessTime = rsp->LastAccessTime; + file_info->LastWriteTime = rsp->LastWriteTime; + file_info->ChangeTime = rsp->ChangeTime; + file_info->AllocationSize = rsp->AllocationSize; + file_info->EndOfFile = rsp->EndofFile; + file_info->Attributes = rsp->FileAttributes; + file_info->NumberOfLinks = cpu_to_le32(1); + buf->unknown_nlink = true; + file_info->DeletePending = 0; /* successful open = not delete pending */ } rc = smb2_parse_contexts(server, &rsp_iov, &oparms->fid->epoch, - oparms->fid->lease_key, oplock, buf, posix); + oparms->fid->lease_key, oplock, file_info, posix); trace_smb3_open_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid, oparms->create_options, oparms->desired_access, @@ -3492,6 +3534,8 @@ SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); @@ -3613,14 +3657,14 @@ ioctl_exit: int SMB2_set_compression(const unsigned int xid, struct cifs_tcon *tcon, - u64 persistent_fid, u64 volatile_fid) + u64 persistent_fid, u64 volatile_fid, + __u16 compression_state) { int rc; struct compress_ioctl fsctl_input; char *ret_data = NULL; - fsctl_input.CompressionState = - cpu_to_le16(COMPRESSION_FORMAT_DEFAULT); + fsctl_input.CompressionState = cpu_to_le16(compression_state); rc = SMB2_ioctl(xid, tcon, persistent_fid, volatile_fid, FSCTL_SET_COMPRESSION, @@ -3686,6 +3730,8 @@ __SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; query_attrs = false; server = cifs_pick_channel(ses); @@ -3898,6 +3944,8 @@ query_info(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; allocated = false; server = cifs_pick_channel(ses); @@ -4070,6 +4118,8 @@ SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); @@ -4412,6 +4462,8 @@ SMB2_flush(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); @@ -4595,6 +4647,7 @@ smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode); struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink); struct smb2_hdr *shdr = (struct smb2_hdr *)rdata->iov[0].iov_base; + struct inode *inode = &ictx->inode; struct cifs_credits credits = { .value = 0, .instance = 0, @@ -4708,7 +4761,7 @@ do_retry: } else { size_t trans = rdata->subreq.transferred + rdata->got_bytes; if (trans < rdata->subreq.len && - rdata->subreq.start + trans >= ictx->remote_i_size) { + rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) { __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); rdata->result = 0; } @@ -4941,7 +4994,7 @@ smb2_writev_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) unsigned int rreq_debug_id = wdata->rreq->debug_id; unsigned int subreq_debug_index = wdata->subreq.debug_index; ssize_t result = 0; - size_t written; + size_t written = 0; WARN_ONCE(wdata->server != server, "wdata server %p != mid server %p", @@ -5354,7 +5407,7 @@ int posix_info_sid_size(const void *beg, const void *end) size_t subauth; int total; - if (beg + 1 > end) + if (beg + 2 > end) return -1; subauth = *(u8 *)(beg+1); @@ -5622,6 +5675,8 @@ smb2_parse_query_directory(struct cifs_tcon *tcon, if (srch_inf->ntwrk_buf_start) { if (srch_inf->smallBuf) cifs_small_buf_release(srch_inf->ntwrk_buf_start); + else if (srch_inf->is_dynamic_buf) + kfree(srch_inf->ntwrk_buf_start); else cifs_buf_release(srch_inf->ntwrk_buf_start); } @@ -5641,12 +5696,18 @@ smb2_parse_query_directory(struct cifs_tcon *tcon, cifs_dbg(FYI, "num entries %d last_index %lld srch start %p srch end %p\n", srch_inf->entries_in_buffer, srch_inf->index_of_last_entry, srch_inf->srch_entries_start, srch_inf->last_entry); - if (resp_buftype == CIFS_LARGE_BUFFER) + if (resp_buftype == CIFS_LARGE_BUFFER) { srch_inf->smallBuf = false; - else if (resp_buftype == CIFS_SMALL_BUFFER) + srch_inf->is_dynamic_buf = false; + } else if (resp_buftype == CIFS_SMALL_BUFFER) { srch_inf->smallBuf = true; - else + srch_inf->is_dynamic_buf = false; + } else if (resp_buftype == CIFS_DYNAMIC_BUFFER) { + srch_inf->smallBuf = false; + srch_inf->is_dynamic_buf = true; + } else { cifs_tcon_dbg(VFS, "Invalid search buffer type\n"); + } return 0; } @@ -5669,6 +5730,8 @@ SMB2_query_directory(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); flags = 0; server = cifs_pick_channel(ses); @@ -5887,6 +5950,25 @@ SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, } int +SMB2_set_allocation(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u32 pid, + loff_t allocation_size) +{ + struct smb2_file_alloc_info info; + void *data; + unsigned int size; + + info.AllocationSize = cpu_to_le64(allocation_size); + + data = &info; + size = sizeof(struct smb2_file_alloc_info); + + return send_set_info(xid, tcon, persistent_fid, volatile_fid, + pid, FILE_ALLOCATION_INFORMATION, SMB2_O_INFO_FILE, + 0, 1, &data, &size); +} + +int SMB2_set_acl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, struct smb_ntsd *pnntsd, int pacllen, int aclflag) diff --git a/fs/smb/client/smb2pdu.h b/fs/smb/client/smb2pdu.h index 30d70097fe2f..b9bf2fa989d5 100644 --- a/fs/smb/client/smb2pdu.h +++ b/fs/smb/client/smb2pdu.h @@ -195,10 +195,6 @@ struct network_resiliency_req { #define NO_FILE_ID 0xFFFFFFFFFFFFFFFFULL /* general ioctls to srv not to file */ -struct compress_ioctl { - __le16 CompressionState; /* See cifspdu.h for possible flag values */ -} __packed; - /* * Maximum number of iovs we need for an ioctl request. * [0] : struct smb2_ioctl_req diff --git a/fs/smb/client/smb2proto.h b/fs/smb/client/smb2proto.h index 1ceb95b907e6..2e9f70096825 100644 --- a/fs/smb/client/smb2proto.h +++ b/fs/smb/client/smb2proto.h @@ -136,7 +136,7 @@ int SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, struct cifs_tcon *tcon, const struct nls_table *cp); int SMB2_tdis(const unsigned int xid, struct cifs_tcon *tcon); int SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, - __le16 *path, __u8 *oplock, struct smb2_file_all_info *buf, + __le16 *path, __u8 *oplock, struct cifs_open_info_data *buf, struct create_posix_rsp *posix, struct kvec *err_iov, int *buftype); int SMB2_open_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, @@ -204,6 +204,9 @@ void SMB2_query_directory_free(struct smb_rqst *rqst); int SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, u32 pid, loff_t new_eof); +int SMB2_set_allocation(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u32 pid, + loff_t allocation_size); int SMB2_set_info_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, struct smb_rqst *rqst, u64 persistent_fid, u64 volatile_fid, u32 pid, u8 info_class, u8 info_type, @@ -216,7 +219,8 @@ int SMB2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, struct smb2_file_full_ea_info *buf, int len); int SMB2_set_compression(const unsigned int xid, struct cifs_tcon *tcon, - u64 persistent_fid, u64 volatile_fid); + u64 persistent_fid, u64 volatile_fid, + __u16 compression_state); int SMB2_oplock_break(const unsigned int xid, struct cifs_tcon *tcon, const u64 persistent_fid, const u64 volatile_fid, __u8 oplock_level); diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c index 41009039b4cb..1143ee52470a 100644 --- a/fs/smb/client/smb2transport.c +++ b/fs/smb/client/smb2transport.c @@ -169,7 +169,9 @@ smb2_find_smb_sess_tcon_unlocked(struct cifs_ses *ses, __u32 tid) list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { if (tcon->tid != tid) continue; + spin_lock(&tcon->tc_lock); ++tcon->tc_count; + spin_unlock(&tcon->tc_lock); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_find_sess_tcon); return tcon; @@ -251,7 +253,8 @@ smb2_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server) } static void generate_key(struct cifs_ses *ses, struct kvec label, - struct kvec context, __u8 *key, unsigned int key_size) + struct kvec context, __u8 *key, unsigned int key_size, + unsigned int full_key_size) { unsigned char zero = 0x0; __u8 i[4] = {0, 0, 0, 1}; @@ -265,7 +268,7 @@ static void generate_key(struct cifs_ses *ses, struct kvec label, memset(key, 0x0, key_size); hmac_sha256_init_usingrawkey(&hmac_ctx, ses->auth_key.response, - SMB2_NTLMV2_SESSKEY_SIZE); + full_key_size); hmac_sha256_update(&hmac_ctx, i, 4); hmac_sha256_update(&hmac_ctx, label.iov_base, label.iov_len); hmac_sha256_update(&hmac_ctx, &zero, 1); @@ -298,6 +301,7 @@ generate_smb3signingkey(struct cifs_ses *ses, struct TCP_Server_Info *server, const struct derivation_triplet *ptriplet) { + unsigned int full_key_size = SMB2_NTLMV2_SESSKEY_SIZE; bool is_binding = false; int chan_index = 0; @@ -330,12 +334,24 @@ generate_smb3signingkey(struct cifs_ses *ses, if (is_binding) { generate_key(ses, ptriplet->signing.label, ptriplet->signing.context, - ses->chans[chan_index].signkey, - SMB3_SIGN_KEY_SIZE); + ses->chans[chan_index].signkey, SMB3_SIGN_KEY_SIZE, + SMB2_NTLMV2_SESSKEY_SIZE); } else { generate_key(ses, ptriplet->signing.label, - ptriplet->signing.context, - ses->smb3signingkey, SMB3_SIGN_KEY_SIZE); + ptriplet->signing.context, ses->smb3signingkey, + SMB3_SIGN_KEY_SIZE, SMB2_NTLMV2_SESSKEY_SIZE); + + /* + * Per MS-SMB2 3.2.5.3.1, signing key always uses Session.SessionKey + * (first 16 bytes). Encryption/decryption keys use + * Session.FullSessionKey when dialect is 3.1.1 and cipher is + * AES-256-CCM or AES-256-GCM, otherwise Session.SessionKey. + */ + + if (server->dialect == SMB311_PROT_ID && + (server->cipher_type == SMB2_ENCRYPTION_AES256_CCM || + server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + full_key_size = ses->auth_key.len; /* safe to access primary channel, since it will never go away */ spin_lock(&ses->chan_lock); @@ -345,10 +361,13 @@ generate_smb3signingkey(struct cifs_ses *ses, generate_key(ses, ptriplet->encryption.label, ptriplet->encryption.context, - ses->smb3encryptionkey, SMB3_ENC_DEC_KEY_SIZE); + ses->smb3encryptionkey, SMB3_ENC_DEC_KEY_SIZE, + full_key_size); + generate_key(ses, ptriplet->decryption.label, ptriplet->decryption.context, - ses->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE); + ses->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE, + full_key_size); } #ifdef CONFIG_CIFS_DEBUG_DUMP_KEYS @@ -361,7 +380,7 @@ generate_smb3signingkey(struct cifs_ses *ses, &ses->Suid); cifs_dbg(VFS, "Cipher type %d\n", server->cipher_type); cifs_dbg(VFS, "Session Key %*ph\n", - SMB2_NTLMV2_SESSKEY_SIZE, ses->auth_key.response); + (int)ses->auth_key.len, ses->auth_key.response); cifs_dbg(VFS, "Signing Key %*ph\n", SMB3_SIGN_KEY_SIZE, ses->smb3signingkey); if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index 75f9f91a7ec9..563ef488a225 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -9,7 +9,6 @@ #include "cifs_debug.h" #include "cifsproto.h" #include "smb2proto.h" -#include "../smbdirect/public.h" /* Port numbers for SMBD transport */ #define SMB_PORT 445 @@ -558,3 +557,5 @@ void smbd_debug_proc_show(struct TCP_Server_Info *server, struct seq_file *m) server->rdma_readwrite_threshold, m); } + +MODULE_IMPORT_NS("SMBDIRECT"); diff --git a/fs/smb/client/smbdirect.h b/fs/smb/client/smbdirect.h index 287ac849213d..be205ec02077 100644 --- a/fs/smb/client/smbdirect.h +++ b/fs/smb/client/smbdirect.h @@ -12,7 +12,7 @@ #include "cifsglob.h" -#include "../smbdirect/smbdirect.h" +#include <linux/smbdirect.h> extern int rdma_readwrite_threshold; extern int smbd_max_frmr_depth; diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index b99ec5a417fa..5b21ad3c15fb 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -181,6 +181,7 @@ EM(netfs_trace_tcon_ref_get_find, "GET Find ") \ EM(netfs_trace_tcon_ref_get_find_sess_tcon, "GET FndSes") \ EM(netfs_trace_tcon_ref_get_reconnect_server, "GET Reconn") \ + EM(netfs_trace_tcon_ref_get_swn_notify, "GET SwnNot") \ EM(netfs_trace_tcon_ref_new, "NEW ") \ EM(netfs_trace_tcon_ref_new_ipc, "NEW Ipc ") \ EM(netfs_trace_tcon_ref_new_reconnect_server, "NEW Reconn") \ @@ -192,6 +193,7 @@ EM(netfs_trace_tcon_ref_put_mnt_ctx, "PUT MntCtx") \ EM(netfs_trace_tcon_ref_put_dfs_refer, "PUT DfsRfr") \ EM(netfs_trace_tcon_ref_put_reconnect_server, "PUT Reconn") \ + EM(netfs_trace_tcon_ref_put_swn_notify, "PUT SwnNot") \ EM(netfs_trace_tcon_ref_put_tlink, "PUT Tlink ") \ EM(netfs_trace_tcon_ref_see_cancelled_close, "SEE Cn-Cls") \ EM(netfs_trace_tcon_ref_see_fscache_collision, "SEE FV-CO!") \ diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index 05f8099047e1..fdf4e50c27ce 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -1158,7 +1158,7 @@ int cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid) { int length, len; - unsigned int data_offset, data_len; + unsigned int data_offset, data_len, end_off; struct cifs_io_subrequest *rdata = mid->callback_data; char *buf = server->smallbuf; unsigned int buflen = server->pdu_size; @@ -1256,11 +1256,14 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid) use_rdma_mr = rdata->mr; #endif data_len = server->ops->read_data_length(buf, use_rdma_mr); - if (!use_rdma_mr && (data_offset + data_len > buflen)) { - /* data_len is corrupt -- discard frame */ - rdata->result = smb_EIO2(smb_eio_trace_read_rsp_malformed, - data_offset + data_len, buflen); - return cifs_readv_discard(server, mid); + if (!use_rdma_mr) { + if (check_add_overflow(data_offset, data_len, &end_off) || + end_off > buflen) { + /* data_len is corrupt -- discard frame */ + rdata->result = smb_EIO2(smb_eio_trace_read_rsp_malformed, + end_off, buflen); + return cifs_readv_discard(server, mid); + } } #ifdef CONFIG_CIFS_SMB_DIRECT diff --git a/fs/smb/client/xattr.c b/fs/smb/client/xattr.c index 23227f2f9428..5091f6c0d7fe 100644 --- a/fs/smb/client/xattr.c +++ b/fs/smb/client/xattr.c @@ -272,7 +272,7 @@ static int cifs_xattr_get(const struct xattr_handler *handler, struct dentry *dentry, struct inode *inode, const char *name, void *value, size_t size) { - ssize_t rc = -EOPNOTSUPP; + int rc = -EOPNOTSUPP; unsigned int xid; struct super_block *sb = dentry->d_sb; struct cifs_sb_info *cifs_sb = CIFS_SB(sb); @@ -354,7 +354,7 @@ static int cifs_xattr_get(const struct xattr_handler *handler, inode, full_path, &acllen, extra_info); if (IS_ERR(pacl)) { rc = PTR_ERR(pacl); - cifs_dbg(VFS, "%s: error %zd getting sec desc\n", + cifs_dbg(VFS, "%s: error %d getting sec desc\n", __func__, rc); } else { if (value) { diff --git a/fs/smb/common/Makefile b/fs/smb/common/Makefile index 9e0730a385fb..f2c6e09d4e77 100644 --- a/fs/smb/common/Makefile +++ b/fs/smb/common/Makefile @@ -4,3 +4,6 @@ # obj-$(CONFIG_SMBFS) += cifs_md4.o +obj-$(CONFIG_SMBFS) += smb_compress.o + +smb_compress-y := compress/compress.o compress/lz77.o diff --git a/fs/smb/common/compress/compress.c b/fs/smb/common/compress/compress.c new file mode 100644 index 000000000000..b07a317597a4 --- /dev/null +++ b/fs/smb/common/compress/compress.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SMB2 compression transform helpers. + * + * Copyright (C) 2026 Namjae Jeon <linkinjeon@kernel.org> + */ +#include <linux/module.h> +#include <linux/overflow.h> +#include <linux/string.h> +#include <linux/unaligned.h> + +#include "compress.h" +#include "lz77.h" + +#define SMB2_COMPRESSION_CHAINED_HDR_LEN \ + offsetof(struct smb2_compression_hdr, CompressionAlgorithm) +#define SMB2_COMPRESSION_PAYLOAD_BASE_LEN \ + (sizeof(struct smb2_compression_payload_hdr) - sizeof(__le32)) + +/* + * A NONE payload carries bytes verbatim. Keep both cursors and remaining + * lengths together so every chained payload handler applies identical bounds + * accounting. + */ +static int smb_decompress_none(const u8 **src, u32 *slen, u8 **dst, u32 *dlen, + u32 len) +{ + if (len > *slen || len > *dlen) + return -EINVAL; + + memcpy(*dst, *src, len); + *src += len; + *slen -= len; + *dst += len; + *dlen -= len; + return 0; +} + +/* + * Pattern_V1 represents a run of one byte. Its wire payload is always the + * fixed-size smb2_compression_pattern_v1 structure. + */ +static int smb_decompress_pattern(const u8 **src, u32 *slen, u8 **dst, + u32 *dlen, u32 len) +{ + const struct smb2_compression_pattern_v1 *pattern; + u32 repetitions; + + if (len != sizeof(*pattern) || len > *slen) + return -EINVAL; + + pattern = (const struct smb2_compression_pattern_v1 *)*src; + repetitions = le32_to_cpu(pattern->Repetitions); + if (repetitions > *dlen) + return -EINVAL; + + memset(*dst, pattern->Pattern, repetitions); + *src += len; + *slen -= len; + *dst += repetitions; + *dlen -= repetitions; + return 0; +} + +/* + * LZ77 payload Length includes the four-byte OriginalPayloadSize field. + * Consume that field before passing the compressed stream to the raw codec. + */ +static int smb_decompress_lz77_payload(const u8 **src, u32 *slen, u8 **dst, + u32 *dlen, u32 len) +{ + u32 orig_size; + int rc; + + if (len < sizeof(__le32) || len > *slen) + return -EINVAL; + + orig_size = get_unaligned_le32(*src); + if (orig_size > *dlen) + return -EINVAL; + + *src += sizeof(__le32); + *slen -= sizeof(__le32); + len -= sizeof(__le32); + + rc = smb_lz77_decompress(*src, len, *dst, orig_size); + if (rc) + return rc; + + *src += len; + *slen -= len; + *dst += orig_size; + *dlen -= orig_size; + return 0; +} + +static int smb_decompress_chained(__le16 alg, bool allow_chained, + const struct smb2_compression_hdr *hdr, + u32 slen, void *dst, u32 dlen) +{ + const struct smb2_compression_payload_hdr *payload; + const u8 *src = (const u8 *)hdr + SMB2_COMPRESSION_CHAINED_HDR_LEN; + u32 orig_size = le32_to_cpu(hdr->OriginalCompressedSegmentSize); + u32 remaining = slen - SMB2_COMPRESSION_CHAINED_HDR_LEN; + u8 *out = dst; + u32 out_remaining = dlen; + bool first = true; + int rc; + + if (!allow_chained || orig_size != dlen) + return -EINVAL; + + /* + * The chained transform has an eight-byte top-level header. The next + * bytes are a sequence of payload headers whose Length fields account + * for payload data, including OriginalPayloadSize where applicable. + */ + while (remaining) { + __le16 payload_alg; + __le16 flags; + u32 len; + + if (remaining < SMB2_COMPRESSION_PAYLOAD_BASE_LEN) + return -EINVAL; + + payload = (const struct smb2_compression_payload_hdr *)src; + payload_alg = payload->CompressionAlgorithm; + flags = payload->Flags; + len = le32_to_cpu(payload->Length); + + /* + * CHAINED marks only the first payload. Requiring NONE on every + * later payload rejects ambiguous or independently chained data. + */ + if ((first && flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED)) || + (!first && flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE))) + return -EINVAL; + + src += SMB2_COMPRESSION_PAYLOAD_BASE_LEN; + remaining -= SMB2_COMPRESSION_PAYLOAD_BASE_LEN; + + if (payload_alg == SMB3_COMPRESS_NONE) { + rc = smb_decompress_none(&src, &remaining, &out, + &out_remaining, len); + } else if (payload_alg == SMB3_COMPRESS_PATTERN) { + rc = smb_decompress_pattern(&src, &remaining, &out, + &out_remaining, len); + } else if (payload_alg == alg && alg == SMB3_COMPRESS_LZ77) { + rc = smb_decompress_lz77_payload(&src, &remaining, &out, + &out_remaining, len); + } else { + return -EINVAL; + } + if (rc) + return rc; + first = false; + } + + return out_remaining ? -EINVAL : 0; +} + +static int smb_decompress_unchained(__le16 alg, + const struct smb2_compression_hdr *hdr, + u32 slen, void *dst, u32 dlen) +{ + u32 orig_size, offset, comp_size; + + if (hdr->CompressionAlgorithm != alg || + !smb_compress_alg_valid(hdr->CompressionAlgorithm, false)) + return -EINVAL; + + orig_size = le32_to_cpu(hdr->OriginalCompressedSegmentSize); + offset = le32_to_cpu(hdr->Offset); + if (offset > slen - sizeof(*hdr) || offset > dlen || + orig_size > dlen - offset || orig_size + offset != dlen) + return -EINVAL; + + memcpy(dst, (const u8 *)hdr + sizeof(*hdr), offset); + comp_size = slen - sizeof(*hdr) - offset; + return smb_lz77_decompress((const u8 *)hdr + sizeof(*hdr) + offset, + comp_size, (u8 *)dst + offset, orig_size); +} + +/** + * smb_compression_decompress() - decode an SMB2 compression transform + * @alg: negotiated general-purpose compression algorithm + * @allow_chained: whether chained transforms were negotiated + * @src: transform header followed by compressed payload data + * @slen: total number of bytes available at @src + * @dst: output buffer for the reconstructed SMB2 message + * @dlen: exact expected size of the reconstructed SMB2 message + * + * Validate the transform type and negotiated capabilities before dispatching + * to the chained or unchained decoder. The caller supplies the expected output + * size after applying its transport-specific message size limits. + * + * Return: 0 on success, otherwise a negative errno. + */ +int smb_compression_decompress(__le16 alg, bool allow_chained, + const void *src, u32 slen, void *dst, u32 dlen) +{ + const struct smb2_compression_hdr *hdr = src; + + if (!src || !dst || slen < sizeof(*hdr) || + hdr->ProtocolId != SMB2_COMPRESSION_TRANSFORM_ID || + alg == SMB3_COMPRESS_NONE) + return -EINVAL; + + if (hdr->Flags == cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED)) + return smb_decompress_chained(alg, allow_chained, hdr, slen, + dst, dlen); + + if (hdr->Flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE)) + return -EINVAL; + + return smb_decompress_unchained(alg, hdr, slen, dst, dlen); +} +EXPORT_SYMBOL_GPL(smb_compression_decompress); + +struct smb_compression_builder { + u8 *pos; + u32 remaining; + bool first; +}; + +/* + * Reserve one chained payload header and initialize its common fields. + * OriginalPayloadSize is present only for LZNT1/LZ77/LZ77+Huffman payloads. + */ +static struct smb2_compression_payload_hdr * +smb_compression_add_payload(struct smb_compression_builder *builder, + __le16 alg, u32 payload_len, bool orig_size) +{ + struct smb2_compression_payload_hdr *payload; + u32 hdr_len = SMB2_COMPRESSION_PAYLOAD_BASE_LEN; + u32 total_len; + + if (orig_size) + hdr_len += sizeof(payload->OriginalPayloadSize); + if (check_add_overflow(hdr_len, payload_len, &total_len) || + total_len > builder->remaining) + return NULL; + + payload = (struct smb2_compression_payload_hdr *)builder->pos; + payload->CompressionAlgorithm = alg; + payload->Flags = cpu_to_le16(builder->first ? + SMB2_COMPRESSION_FLAG_CHAINED : SMB2_COMPRESSION_FLAG_NONE); + payload->Length = cpu_to_le32(payload_len + + (orig_size ? sizeof(payload->OriginalPayloadSize) : 0)); + + builder->pos += hdr_len; + builder->remaining -= hdr_len; + builder->first = false; + return payload; +} + +static int smb_compression_add_pattern(struct smb_compression_builder *builder, + u8 pattern, u32 repetitions) +{ + struct smb2_compression_pattern_v1 *payload; + + if (!smb_compression_add_payload(builder, SMB3_COMPRESS_PATTERN, + sizeof(*payload), false)) + return -ENOSPC; + + payload = (struct smb2_compression_pattern_v1 *)builder->pos; + payload->Pattern = pattern; + payload->Reserved1 = 0; + payload->Reserved2 = 0; + payload->Repetitions = cpu_to_le32(repetitions); + builder->pos += sizeof(*payload); + builder->remaining -= sizeof(*payload); + return 0; +} + +static int smb_compression_add_none(struct smb_compression_builder *builder, + const u8 *src, u32 len) +{ + if (!smb_compression_add_payload(builder, SMB3_COMPRESS_NONE, len, false)) + return -ENOSPC; + + memcpy(builder->pos, src, len); + builder->pos += len; + builder->remaining -= len; + return 0; +} + +static int smb_compression_add_lz77(struct smb_compression_builder *builder, + const u8 *src, u32 len) +{ + struct smb2_compression_payload_hdr *payload; + u32 comp_len; + int rc; + + if (builder->remaining <= sizeof(*payload)) + return -ENOSPC; + + comp_len = builder->remaining - sizeof(*payload); + payload = smb_compression_add_payload(builder, SMB3_COMPRESS_LZ77, + comp_len, true); + if (!payload) + return -ENOSPC; + + rc = smb_lz77_compress(src, len, builder->pos, &comp_len); + if (rc) + return rc; + + payload->Length = cpu_to_le32(comp_len + + sizeof(payload->OriginalPayloadSize)); + payload->OriginalPayloadSize = cpu_to_le32(len); + builder->pos += comp_len; + builder->remaining -= comp_len; + return 0; +} + +/** + * smb_compression_compress_chained() - build a chained SMB2 transform + * @alg: negotiated general-purpose compression algorithm + * @allow_pattern: whether Pattern_V1 was negotiated + * @src: complete uncompressed SMB2 message + * @slen: size of @src + * @dst: output buffer for the transform + * @dlen: input capacity of @dst and output transform size + * + * Following the algorithm in [MS-SMB2] 3.1.4.4, encode sufficiently long + * repeated runs at the front and back as Pattern_V1 payloads. Compress a + * middle region larger than 1 KiB with LZ77; smaller middle regions are + * represented by a chained NONE payload. + * + * This helper does not decide whether the final transform is smaller than the + * original message. The transport caller owns that policy decision. + * + * Return: 0 on success, otherwise a negative errno. + */ +int smb_compression_compress_chained(__le16 alg, bool allow_pattern, + const void *src, u32 slen, + void *dst, u32 *dlen) +{ + struct smb2_compression_hdr *hdr = dst; + struct smb_compression_builder builder; + const u8 *input = src; + u32 forward = 0, backward = 0, middle_len; + int rc; + + if (!src || !dst || !dlen || alg != SMB3_COMPRESS_LZ77 || + *dlen <= SMB2_COMPRESSION_CHAINED_HDR_LEN || !slen) + return -EINVAL; + + hdr->ProtocolId = SMB2_COMPRESSION_TRANSFORM_ID; + hdr->OriginalCompressedSegmentSize = cpu_to_le32(slen); + builder.pos = (u8 *)dst + SMB2_COMPRESSION_CHAINED_HDR_LEN; + builder.remaining = *dlen - SMB2_COMPRESSION_CHAINED_HDR_LEN; + builder.first = true; + + if (allow_pattern && slen > 32) { + for (forward = 1; forward < slen; forward++) { + if (input[forward] != input[0]) + break; + } + if (forward <= 32) + forward = 0; + + for (backward = 1; backward < slen - forward; backward++) { + if (input[slen - backward - 1] != input[slen - 1]) + break; + } + if (backward <= 32) + backward = 0; + } + + if (forward) { + rc = smb_compression_add_pattern(&builder, input[0], forward); + if (rc) + return rc; + } + + middle_len = slen - forward - backward; + if (middle_len > 1024) + rc = smb_compression_add_lz77(&builder, input + forward, + middle_len); + else if (middle_len) + rc = smb_compression_add_none(&builder, + input + forward, middle_len); + else + rc = 0; + if (rc) + return rc; + + if (backward) { + rc = smb_compression_add_pattern(&builder, input[slen - 1], + backward); + if (rc) + return rc; + } + + *dlen = builder.pos - (u8 *)dst; + return 0; +} +EXPORT_SYMBOL_GPL(smb_compression_compress_chained); diff --git a/fs/smb/common/compress/compress.h b/fs/smb/common/compress/compress.h new file mode 100644 index 000000000000..7ace3bf4b664 --- /dev/null +++ b/fs/smb/common/compress/compress.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 Namjae Jeon <linkinjeon@kernel.org> + */ +#ifndef _COMMON_SMB_COMPRESS_H +#define _COMMON_SMB_COMPRESS_H + +#include "../smb2pdu.h" + +/* + * SMB3_COMPRESS_NONE is valid only in chained payload headers. It is never + * negotiated as a compression algorithm. + */ +static __always_inline bool smb_compress_alg_valid(__le16 alg, bool valid_none) +{ + if (alg == SMB3_COMPRESS_NONE) + return valid_none; + + return alg == SMB3_COMPRESS_LZ77 || alg == SMB3_COMPRESS_PATTERN; +} + +int smb_compression_decompress(__le16 alg, bool allow_chained, + const void *src, u32 slen, void *dst, u32 dlen); +int smb_compression_compress_chained(__le16 alg, bool allow_pattern, + const void *src, u32 slen, + void *dst, u32 *dlen); + +#endif /* _COMMON_SMB_COMPRESS_H */ diff --git a/fs/smb/client/compress/lz77.c b/fs/smb/common/compress/lz77.c index 7365d0f97396..9216d973d876 100644 --- a/fs/smb/client/compress/lz77.c +++ b/fs/smb/common/compress/lz77.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2024-2026, SUSE LLC + * Copyright (C) 2026 Namjae Jeon <linkinjeon@kernel.org> * * Authors: Enzo Matsumiya <ematsumiya@suse.de> + * Namjae Jeon <linkinjeon@kernel.org> * * Implementation of the LZ77 "plain" compression algorithm, as per MS-XCA spec. */ @@ -10,6 +12,8 @@ #include <linux/sizes.h> #include <linux/count_zeros.h> #include <linux/unaligned.h> +#include <linux/module.h> +#include <linux/overflow.h> #include "lz77.h" @@ -32,7 +36,7 @@ */ #define LZ77_MATCH_MAX_DIST SZ_8K #define LZ77_HASH_LOG 15 -#define LZ77_HASH_SIZE (1 << LZ77_HASH_LOG) +#define LZ77_HASH_SIZE BIT(LZ77_HASH_LOG) #define LZ77_RSTEP_SIZE sizeof(u32) #define LZ77_MSTEP_SIZE sizeof(u64) #define LZ77_SKIP_TRIGGER 4 @@ -215,7 +219,8 @@ static __always_inline u32 lz77_hash(const u32 v) return ((v ^ 0x9E3779B9) * 0x85EBCA6B) >> (32 - LZ77_HASH_LOG); } -noinline int lz77_compress(const void *src, const u32 slen, void *dst, u32 *dlen) +noinline int smb_lz77_compress(const void *src, const u32 slen, + void *dst, u32 *dlen) { const void *srcp, *rlim, *end, *anchor; u32 *htable, hash, flag_count = 0; @@ -223,10 +228,11 @@ noinline int lz77_compress(const void *src, const u32 slen, void *dst, u32 *dlen long flag = 0; /* This is probably a bug, so throw a warning. */ - if (WARN_ON_ONCE(*dlen < lz77_compressed_alloc_size(slen))) + if (WARN_ON_ONCE(*dlen < smb_lz77_compressed_alloc_size(slen))) return -EINVAL; - srcp = anchor = src; + srcp = src; + anchor = src; end = srcp + slen; /* absolute end */ rlim = end - LZ77_MSTEP_SIZE; /* read limit (for lz77_match_len()) */ dstp = dst; @@ -251,7 +257,8 @@ noinline int lz77_compress(const void *src, const u32 slen, void *dst, u32 *dlen /* * Main loop. * - * @dlen is >= lz77_compressed_alloc_size(), so run without bound-checking @dstp. + * @dlen is >= smb_lz77_compressed_alloc_size(), so run without + * bound-checking @dstp. * * This code was crafted in a way to best utilise fetch-decode-execute CPU flow. * Any attempt to optimize it, or even organize it, can lead to huge performance loss. @@ -333,3 +340,122 @@ out: return -EMSGSIZE; } +EXPORT_SYMBOL_GPL(smb_lz77_compress); + +static int lz77_decode_match_len(const u8 **src, const u8 *end, u16 token, + u8 *nibble, bool *have_nibble, u32 *len) +{ + u8 extra; + + *len = (token & 0x7) + 3; + if ((token & 0x7) != 0x7) + return 0; + + if (!*have_nibble) { + if (*src >= end) + return -EINVAL; + *nibble = *(*src)++; + extra = *nibble & 0xf; + *have_nibble = true; + } else { + extra = *nibble >> 4; + *have_nibble = false; + } + + *len += extra; + if (extra == 0xf) { + u8 b; + + if (*src >= end) + return -EINVAL; + b = *(*src)++; + if (b != 0xff) { + *len += b; + } else { + u16 w; + + if (end - *src < 2) + return -EINVAL; + w = get_unaligned_le16(*src); + *src += 2; + if (w) { + *len = w + 3; + } else { + u32 long_len; + + if (end - *src < 4) + return -EINVAL; + long_len = get_unaligned_le32(*src); + *src += 4; + if (check_add_overflow(long_len, 3, len)) + return -EINVAL; + } + } + } + + return 0; +} + +int smb_lz77_decompress(const void *src, const u32 slen, void *dst, + const u32 dlen) +{ + const u8 *sp = src, *send = sp + slen; + u8 *dp = dst, *dend = dp + dlen; + u32 flags = 0; + int flag_count = 0; + u8 nibble = 0; + bool have_nibble = false; + + while (dp < dend) { + u32 len, dist; + u16 token; + + if (!flag_count) { + if (send - sp < 4) + return -EINVAL; + flags = get_unaligned_le32(sp); + sp += 4; + flag_count = 32; + } + + if (!(flags & 0x80000000)) { + if (sp >= send) + return -EINVAL; + *dp++ = *sp++; + flags <<= 1; + flag_count--; + continue; + } + + flags <<= 1; + flag_count--; + + if (send - sp < 2) + return -EINVAL; + + token = get_unaligned_le16(sp); + sp += 2; + + dist = (token >> 3) + 1; + if (dist > dp - (u8 *)dst) + return -EINVAL; + + if (lz77_decode_match_len(&sp, send, token, &nibble, + &have_nibble, &len)) + return -EINVAL; + + if (len > dend - dp) + return -EINVAL; + + while (len--) { + *dp = *(dp - dist); + dp++; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(smb_lz77_decompress); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SMB plain LZ77 compression"); diff --git a/fs/smb/client/compress/lz77.h b/fs/smb/common/compress/lz77.h index 4e570846aefa..e032c0f1b48d 100644 --- a/fs/smb/client/compress/lz77.h +++ b/fs/smb/common/compress/lz77.h @@ -12,7 +12,7 @@ #include <linux/kernel.h> /** - * lz77_compressed_alloc_size() - Compute compressed buffer size. + * smb_lz77_compressed_alloc_size() - Compute compressed buffer size. * @size: uncompressed (src) size * * Compute allocation size for the compressed buffer based on uncompressed size. @@ -26,18 +26,21 @@ * Worst case scenario is an all-literal compression, which means: * metadata bytes = 4 + ((@size / 32) * 4) + 4, or, simplified, (@size >> 3) + 8 * - * The worst case scenario rarely happens, but such overprovisioning also allows lz77_compress() - * main loop to run without ever bound checking dst, which is a huge perf improvement, while also - * being safe when compression goes bad. + * The worst case scenario rarely happens, but such overprovisioning also + * allows smb_lz77_compress() main loop to run without ever bound checking dst, + * which is a huge perf improvement, while also being safe when compression goes + * bad. * * Return: required (*) allocation size for compressed buffer. * - * (*) checked once in the beginning of lz77_compress() + * (*) checked once in the beginning of smb_lz77_compress() */ -static __always_inline u32 lz77_compressed_alloc_size(const u32 size) +static __always_inline u32 smb_lz77_compressed_alloc_size(const u32 size) { return size + (size >> 3) + 8; } -int lz77_compress(const void *src, const u32 slen, void *dst, u32 *dlen); +int smb_lz77_compress(const void *src, const u32 slen, void *dst, u32 *dlen); +int smb_lz77_decompress(const void *src, const u32 slen, void *dst, + const u32 dlen); #endif /* _SMB_COMPRESS_LZ77_H */ diff --git a/fs/smb/common/fscc.h b/fs/smb/common/fscc.h index b4ccddca9256..941db5a95564 100644 --- a/fs/smb/common/fscc.h +++ b/fs/smb/common/fscc.h @@ -100,6 +100,24 @@ struct duplicate_extents_to_file_ex { __le32 Reserved; } __packed; +/* + * compression state flags + * See MS-FSCC 2.3.18 + * MS-FSCC 2.3.67 + * MS-FSCC 2.4.9 + */ +#define COMPRESSION_FORMAT_NONE 0x0000 +#define COMPRESSION_FORMAT_DEFAULT 0x0001 +#define COMPRESSION_FORMAT_LZNT1 0x0002 + +/* + * See MS-FSCC 2.3.18 + * MS-FSCC 2.3.67 + */ +struct compress_ioctl { + __le16 CompressionState; +} __packed; + /* See MS-FSCC 2.3.20 */ struct fsctl_get_integrity_information_rsp { __le16 ChecksumAlgorithm; @@ -260,12 +278,17 @@ typedef struct { char FileName[]; } __packed FILE_DIRECTORY_INFO; /* level 0x101 FF resp data */ -/* See MS-FSCC 2.4.13 */ +/* See MS-FSCC 2.4.14 */ struct smb2_file_eof_info { /* encoding of request for level 10 */ __le64 EndOfFile; /* new end of file value */ } __packed; /* level 20 Set */ -/* See MS-FSCC 2.4.14 */ +/* See MS-FSCC 2.4.4 */ +struct smb2_file_alloc_info { /* encoding of request for level 19 */ + __le64 AllocationSize; +} __packed; + +/* See MS-FSCC 2.4.15 */ typedef struct { __le32 NextEntryOffset; __u32 FileIndex; diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h index a4b12eb8df81..e7ff52b8aba5 100644 --- a/fs/smb/common/smb2pdu.h +++ b/fs/smb/common/smb2pdu.h @@ -218,10 +218,9 @@ struct smb2_transform_hdr { * These are simplified versions from the spec, as we don't need a fully fledged * form of both unchained and chained structs. * - * Moreover, even in chained compressed payloads, the initial compression header - * has the form of the unchained one -- i.e. it never has the - * OriginalPayloadSize field and ::Offset field always represent an offset - * (instead of a length, as it is in the chained header). + * For chained payloads, only the first 8 bytes belong to the transform header. + * CompressionAlgorithm, Flags and Offset below overlay the first chained + * payload header, where Offset represents Length. * * See MS-SMB2 2.2.42 for more details. */ @@ -524,9 +523,7 @@ struct smb2_compression_capabilities_context { __le16 CompressionAlgorithmCount; __le16 Padding; __le32 Flags; - __le16 CompressionAlgorithms[3]; - __u16 Pad; /* Some servers require pad to DataLen multiple of 8 */ - /* Check if pad needed */ + __le16 CompressionAlgorithms[4]; } __packed; /* @@ -1247,6 +1244,30 @@ struct create_mxac_req { } __packed; /* + * AAPL flags. See Samba libcli/smb/smb2_create_ctx.h + */ + +/* "AAPL" Context Command Codes */ +#define SMB2_CRTCTX_AAPL_SERVER_QUERY 1 +#define SMB2_CRTCTX_AAPL_RESOLVE_ID 2 + +/* "AAPL" Server Query request/response bitmap */ +#define SMB2_CRTCTX_AAPL_SERVER_CAPS 1 +#define SMB2_CRTCTX_AAPL_VOLUME_CAPS 2 +#define SMB2_CRTCTX_AAPL_MODEL_INFO 4 + +/* "AAPL" Client/Server Capabilities bitmap */ +#define SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR 1 +#define SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE 2 +#define SMB2_CRTCTX_AAPL_UNIX_BASED 4 +#define SMB2_CRTCTX_AAPL_SUPPORTS_NFS_ACE 8 + +/* "AAPL" Volume Capabilities bitmap */ +#define SMB2_CRTCTX_AAPL_SUPPORT_RESOLVE_ID 1 +#define SMB2_CRTCTX_AAPL_CASE_SENSITIVE 2 +#define SMB2_CRTCTX_AAPL_FULL_SYNC 4 + +/* * Flags * See MS-SMB2 2.2.13.2.11 * MS-SMB2 2.2.13.2.12 @@ -1566,6 +1587,10 @@ struct validate_negotiate_info_rsp { #define FILE_STANDARD_LINK_INFORMATION 54 #define FILE_ID_INFORMATION 59 #define FILE_ID_EXTD_DIRECTORY_INFORMATION 60 /* also for QUERY_DIR */ +#define FileId64ExtdDirectoryInformation 78 /* also for QUERY_DIR */ +#define FileId64ExtdBothDirectoryInformation 79 /* also for QUERY_DIR */ +#define FileIdAllExtdDirectoryInformation 80 /* also for QUERY_DIR */ +#define FileIdAllExtdBothDirectoryInformation 81 /* also for QUERY_DIR */ /* Used for Query Info and Find File POSIX Info for SMB3.1.1 and SMB1 */ #define SMB_FIND_FILE_POSIX_INFO 0x064 diff --git a/fs/smb/common/smbfsctl.h b/fs/smb/common/smbfsctl.h index 3253a18ecb5c..d1fcb46a7cde 100644 --- a/fs/smb/common/smbfsctl.h +++ b/fs/smb/common/smbfsctl.h @@ -61,8 +61,8 @@ #define FSCTL_LOCK_VOLUME 0x00090018 #define FSCTL_UNLOCK_VOLUME 0x0009001C #define FSCTL_IS_PATHNAME_VALID 0x0009002C /* BB add struct */ -#define FSCTL_GET_COMPRESSION 0x0009003C /* BB add struct */ -#define FSCTL_SET_COMPRESSION 0x0009C040 /* BB add struct */ +#define FSCTL_GET_COMPRESSION 0x0009003C +#define FSCTL_SET_COMPRESSION 0x0009C040 #define FSCTL_QUERY_FAT_BPB 0x00090058 /* BB add struct */ /* Verify the next FSCTL number, we had it as 0x00090090 before */ #define FSCTL_FILESYSTEM_GET_STATS 0x00090060 /* BB add struct */ @@ -159,6 +159,12 @@ #define IO_REPARSE_TAG_LX_CHR 0x80000025 #define IO_REPARSE_TAG_LX_BLK 0x80000026 +#define IO_REPARSE_TAG_LX_SYMLINK_LE cpu_to_le32(IO_REPARSE_TAG_LX_SYMLINK) +#define IO_REPARSE_TAG_AF_UNIX_LE cpu_to_le32(IO_REPARSE_TAG_AF_UNIX) +#define IO_REPARSE_TAG_LX_FIFO_LE cpu_to_le32(IO_REPARSE_TAG_LX_FIFO) +#define IO_REPARSE_TAG_LX_CHR_LE cpu_to_le32(IO_REPARSE_TAG_LX_CHR) +#define IO_REPARSE_TAG_LX_BLK_LE cpu_to_le32(IO_REPARSE_TAG_LX_BLK) + /* If Name Surrogate Bit is set, the file or directory represents another named entity in the system. */ #define IS_REPARSE_TAG_NAME_SURROGATE(tag) (!!((tag) & 0x20000000)) diff --git a/fs/smb/server/Makefile b/fs/smb/server/Makefile index 6407ba6b9340..a3e9306055e8 100644 --- a/fs/smb/server/Makefile +++ b/fs/smb/server/Makefile @@ -10,7 +10,7 @@ ksmbd-y := unicode.o auth.o vfs.o vfs_cache.o server.o ndr.o \ mgmt/tree_connect.o mgmt/user_session.o smb_common.o \ transport_tcp.o transport_ipc.o smbacl.o smb2pdu.o \ smb2ops.o smb2misc.o ksmbd_spnego_negtokeninit.asn1.o \ - ksmbd_spnego_negtokentarg.asn1.o asn1.o + ksmbd_spnego_negtokentarg.asn1.o asn1.o compress.o $(obj)/asn1.o: $(obj)/ksmbd_spnego_negtokeninit.asn1.h $(obj)/ksmbd_spnego_negtokentarg.asn1.h diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index e99409fa721c..4e7b6f0e6b8c 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -133,15 +133,17 @@ out: * @blen: NTLMv2 blob length * @domain_name: domain name * @cryptkey: session crypto key + * @sess_key: derived session key output buffer * * Return: 0 on success, error number on error */ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, struct ntlmv2_resp *ntlmv2, int blen, char *domain_name, - char *cryptkey) + char *cryptkey, char *sess_key) { char ntlmv2_hash[CIFS_ENCPWD_SIZE]; char ntlmv2_rsp[CIFS_HMAC_MD5_HASH_SIZE]; + char base_key[SMB2_NTLMV2_SESSKEY_SIZE]; struct hmac_md5_ctx ctx; int rc; @@ -164,12 +166,21 @@ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, /* Generate the session key */ hmac_md5_usingrawkey(ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE, ntlmv2_rsp, CIFS_HMAC_MD5_HASH_SIZE, - sess->sess_key); + base_key); if (crypto_memneq(ntlmv2->ntlmv2_hash, ntlmv2_rsp, - CIFS_HMAC_MD5_HASH_SIZE)) - return -EINVAL; - return 0; + CIFS_HMAC_MD5_HASH_SIZE)) { + rc = -EINVAL; + goto out; + } + + memcpy(sess_key, base_key, sizeof(base_key)); + rc = 0; +out: + memzero_explicit(ntlmv2_hash, sizeof(ntlmv2_hash)); + memzero_explicit(ntlmv2_rsp, sizeof(ntlmv2_rsp)); + memzero_explicit(base_key, sizeof(base_key)); + return rc; } /** @@ -179,12 +190,13 @@ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, * @blob_len: length of the @authblob message * @conn: connection * @sess: session of connection + * @sess_key: derived session key output buffer * * Return: 0 on success, error number on error */ int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, int blob_len, struct ksmbd_conn *conn, - struct ksmbd_session *sess) + struct ksmbd_session *sess, char *sess_key) { char *domain_name; unsigned int nt_off, dn_off; @@ -224,8 +236,10 @@ int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, ret = ksmbd_auth_ntlmv2(conn, sess, (struct ntlmv2_resp *)((char *)authblob + nt_off), nt_len - CIFS_ENCPWD_SIZE, - domain_name, conn->ntlmssp.cryptkey); + domain_name, conn->ntlmssp.cryptkey, sess_key); kfree(domain_name); + if (ret) + return ret; /* The recovered secondary session key */ if (conn->ntlmssp.client_flags & NTLMSSP_NEGOTIATE_KEY_XCH) { @@ -245,8 +259,8 @@ int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, if (!ctx_arc4) return -ENOMEM; - arc4_setkey(ctx_arc4, sess->sess_key, SMB2_NTLMV2_SESSKEY_SIZE); - arc4_crypt(ctx_arc4, sess->sess_key, + arc4_setkey(ctx_arc4, sess_key, SMB2_NTLMV2_SESSKEY_SIZE); + arc4_crypt(ctx_arc4, sess_key, (char *)authblob + sess_key_off, sess_key_len); kfree_sensitive(ctx_arc4); } @@ -388,7 +402,8 @@ ksmbd_build_ntlmssp_challenge_blob(struct challenge_message *chgblob, #ifdef CONFIG_SMB_SERVER_KERBEROS5 int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, - int in_len, char *out_blob, int *out_len) + int in_len, char *out_blob, int *out_len, + char *sess_key) { struct ksmbd_spnego_authen_response *resp; struct ksmbd_login_response_ext *resp_ext = NULL; @@ -436,14 +451,14 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, } else { if (!ksmbd_compare_user(sess->user, user)) { ksmbd_debug(AUTH, "different user tried to reuse session\n"); - retval = -EPERM; + retval = -EKEYREJECTED; ksmbd_free_user(user); goto out; } ksmbd_free_user(user); } - memcpy(sess->sess_key, resp->payload, resp->session_key_len); + memcpy(sess_key, resp->payload, resp->session_key_len); memcpy(out_blob, resp->payload + resp->session_key_len, resp->spnego_blob_len); *out_len = resp->spnego_blob_len; @@ -454,7 +469,8 @@ out: } #else int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, - int in_len, char *out_blob, int *out_len) + int in_len, char *out_blob, int *out_len, + char *sess_key) { return -EOPNOTSUPP; } @@ -513,7 +529,7 @@ struct derivation { bool binding; }; -static void generate_key(struct ksmbd_conn *conn, struct ksmbd_session *sess, +static void generate_key(struct ksmbd_conn *conn, const char *sess_key, struct kvec label, struct kvec context, __u8 *key, unsigned int key_size) { @@ -524,7 +540,7 @@ static void generate_key(struct ksmbd_conn *conn, struct ksmbd_session *sess, unsigned char prfhash[SMB2_HMACSHA256_SIZE]; struct hmac_sha256_ctx ctx; - hmac_sha256_init_usingrawkey(&ctx, sess->sess_key, + hmac_sha256_init_usingrawkey(&ctx, sess_key, SMB2_NTLMV2_SESSKEY_SIZE); hmac_sha256_update(&ctx, i, 4); hmac_sha256_update(&ctx, label.iov_base, label.iov_len); @@ -547,18 +563,21 @@ static int generate_smb3signingkey(struct ksmbd_session *sess, const struct derivation *signing) { struct channel *chann; - char *key; + char *key, *sess_key; chann = lookup_chann_list(sess, conn); if (!chann) return 0; - if (conn->dialect >= SMB30_PROT_ID && signing->binding) + if (conn->dialect >= SMB30_PROT_ID && signing->binding) { key = chann->smb3signingkey; - else + sess_key = chann->sess_key; + } else { key = sess->smb3signingkey; + sess_key = sess->sess_key; + } - generate_key(conn, sess, signing->label, signing->context, key, + generate_key(conn, sess_key, signing->label, signing->context, key, SMB3_SIGN_KEY_SIZE); if (!(conn->dialect >= SMB30_PROT_ID && signing->binding)) @@ -615,11 +634,11 @@ static void generate_smb3encryptionkey(struct ksmbd_conn *conn, struct ksmbd_session *sess, const struct derivation_twin *ptwin) { - generate_key(conn, sess, ptwin->encryption.label, + generate_key(conn, sess->sess_key, ptwin->encryption.label, ptwin->encryption.context, sess->smb3encryptionkey, SMB3_ENC_DEC_KEY_SIZE); - generate_key(conn, sess, ptwin->decryption.label, + generate_key(conn, sess->sess_key, ptwin->decryption.label, ptwin->decryption.context, sess->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE); diff --git a/fs/smb/server/auth.h b/fs/smb/server/auth.h index 5767aabc63c9..f14b7c033264 100644 --- a/fs/smb/server/auth.h +++ b/fs/smb/server/auth.h @@ -41,17 +41,18 @@ int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, void ksmbd_copy_gss_neg_header(void *buf); int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, struct ntlmv2_resp *ntlmv2, int blen, char *domain_name, - char *cryptkey); + char *cryptkey, char *sess_key); int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, int blob_len, struct ksmbd_conn *conn, - struct ksmbd_session *sess); + struct ksmbd_session *sess, char *sess_key); int ksmbd_decode_ntlmssp_neg_blob(struct negotiate_message *negblob, int blob_len, struct ksmbd_conn *conn); unsigned int ksmbd_build_ntlmssp_challenge_blob(struct challenge_message *chgblob, struct ksmbd_conn *conn); int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, - int in_len, char *out_blob, int *out_len); + int in_len, char *out_blob, int *out_len, + char *sess_key); void ksmbd_sign_smb2_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, int n_vec, char *sig); void ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, diff --git a/fs/smb/server/compress.c b/fs/smb/server/compress.c new file mode 100644 index 000000000000..f8cf515b9c30 --- /dev/null +++ b/fs/smb/server/compress.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SMB2 compression support for ksmbd. + * + * Receive and send SMB 3.1.1 compression transforms using the common helpers. + * + * Copyright (C) 2026 Namjae Jeon <linkinjeon@kernel.org> + */ +#include <linux/slab.h> + +#include "compress.h" +#include "smb_common.h" +#include "../common/compress/lz77.h" + +#define SMB_COMPRESS_MIN_LEN PAGE_SIZE + +/** + * ksmbd_decompress_request() - replace a compressed request with its SMB2 PDU + * @conn: connection which owns the current RFC1002 request buffer + * + * Derive the uncompressed size from the transform variant, enforce ksmbd's + * normal message limits, and ask the common decoder to validate every payload. + * On success, replace conn->request_buf with a regular RFC1002-framed SMB2 + * message so the rest of the request path needs no compression awareness. + * + * Return: 0 on success, otherwise a negative errno. + */ +int ksmbd_decompress_request(struct ksmbd_conn *conn) +{ + struct smb2_compression_hdr *hdr; + unsigned int pdu_size = get_rfc1002_len(conn->request_buf); + u32 orig_size, offset, out_size; + u32 max_allowed_pdu_size; + char *buf, *out; + int rc; + + if (pdu_size < sizeof(struct smb2_compression_hdr)) + return -EINVAL; + + if (conn->dialect != SMB311_PROT_ID || + conn->compress_algorithm == SMB3_COMPRESS_NONE) + return -EINVAL; + + hdr = smb_get_msg(conn->request_buf); + if (hdr->ProtocolId != SMB2_COMPRESSION_TRANSFORM_ID) + return -EINVAL; + + orig_size = le32_to_cpu(hdr->OriginalCompressedSegmentSize); + if (hdr->Flags == cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED)) { + out_size = orig_size; + } else { + offset = le32_to_cpu(hdr->Offset); + if (offset > pdu_size - sizeof(*hdr) || + check_add_overflow(orig_size, offset, &out_size)) + return -EINVAL; + } + + max_allowed_pdu_size = SMB3_MAX_MSGSIZE + conn->vals->max_write_size; + if (out_size > max_allowed_pdu_size || + out_size > MAX_STREAM_PROT_LEN) + return -EINVAL; + + out = kvmalloc(out_size + 4 + 1, KSMBD_DEFAULT_GFP); + if (!out) + return -ENOMEM; + + buf = (char *)hdr; + *(__be32 *)out = cpu_to_be32(out_size); + rc = smb_compression_decompress(conn->compress_algorithm, + conn->compress_chained, + buf, pdu_size, out + 4, out_size); + if (rc) { + kvfree(out); + return rc; + } + + kvfree(conn->request_buf); + conn->request_buf = out; + return 0; +} + +/** + * ksmbd_compress_response() - compress an eligible ksmbd response + * @work: request work item containing the response iov + * + * Compression transforms describe one contiguous SMB2 message, while ksmbd + * builds responses from multiple iov entries. Flatten the response first, + * produce the negotiated transform, and replace the response iov only when the + * result is smaller than the original message. + * + * Encrypted and compound responses are intentionally left unchanged. The + * caller may still continue sending the original response when this function + * returns zero. + * + * Return: 1 if the response was replaced, 0 if compression was skipped, or a + * negative errno on failure. + */ +int ksmbd_compress_response(struct ksmbd_work *work) +{ + struct smb2_compression_hdr *chdr; + struct smb2_hdr *req_hdr; + u32 src_len, dst_len, compressed_pdu_len, max_dst_len; + u8 *src = NULL, *out = NULL, *p; + int i, rc; + + if (!work->compress_response || work->encrypted || + work->conn->compress_algorithm != SMB3_COMPRESS_LZ77) + return 0; + + req_hdr = smb_get_msg(work->request_buf); + if (req_hdr->NextCommand || work->next_smb2_rcv_hdr_off || + work->next_smb2_rsp_hdr_off) + return 0; + + src_len = get_rfc1002_len(work->iov[0].iov_base); + if (src_len < SMB_COMPRESS_MIN_LEN) + return 0; + + src = kvmalloc(src_len, KSMBD_DEFAULT_GFP); + if (!src) + return -ENOMEM; + + p = src; + /* iov[0] contains only the RFC1002 length; the SMB2 PDU starts at iov[1]. */ + for (i = 1; i < work->iov_cnt; i++) { + if (work->iov[i].iov_len > src + src_len - p) { + rc = -EINVAL; + goto out; + } + memcpy(p, work->iov[i].iov_base, work->iov[i].iov_len); + p += work->iov[i].iov_len; + } + if (p != src + src_len) { + rc = -EINVAL; + goto out; + } + + max_dst_len = smb_lz77_compressed_alloc_size(src_len) + + sizeof(struct smb2_compression_hdr) + + 3 * sizeof(struct smb2_compression_payload_hdr) + + 2 * sizeof(struct smb2_compression_pattern_v1); + out = kvzalloc(sizeof(__be32) + max_dst_len, + KSMBD_DEFAULT_GFP); + if (!out) { + rc = -ENOMEM; + goto out; + } + + if (work->conn->compress_chained) { + dst_len = max_dst_len; + rc = smb_compression_compress_chained(SMB3_COMPRESS_LZ77, + work->conn->compress_pattern, + src, src_len, + out + sizeof(__be32), + &dst_len); + if (rc == -EMSGSIZE || dst_len >= src_len) { + rc = 0; + goto out; + } + if (rc) + goto out; + compressed_pdu_len = dst_len; + } else { + /* + * Peers which did not negotiate chained compression still use + * the original 16-byte unchained transform format. + */ + dst_len = smb_lz77_compressed_alloc_size(src_len); + rc = smb_lz77_compress(src, src_len, + out + sizeof(__be32) + sizeof(*chdr), + &dst_len); + if (rc == -EMSGSIZE || + dst_len + sizeof(*chdr) >= src_len) { + rc = 0; + goto out; + } + if (rc) + goto out; + + compressed_pdu_len = sizeof(*chdr) + dst_len; + chdr = (struct smb2_compression_hdr *)(out + sizeof(__be32)); + chdr->ProtocolId = SMB2_COMPRESSION_TRANSFORM_ID; + chdr->OriginalCompressedSegmentSize = cpu_to_le32(src_len); + chdr->CompressionAlgorithm = SMB3_COMPRESS_LZ77; + chdr->Flags = cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE); + chdr->Offset = 0; + } + + *(__be32 *)out = cpu_to_be32(compressed_pdu_len); + + /* + * Keep the transform in work->compress_buf until send completion. + * Existing response iovs can then be replaced without changing their + * individual ownership rules. + */ + work->compress_buf = out; + work->iov[0].iov_base = out; + work->iov[0].iov_len = sizeof(__be32); + work->iov[1].iov_base = out + sizeof(__be32); + work->iov[1].iov_len = compressed_pdu_len; + work->iov_cnt = 2; + work->iov_idx = 1; + out = NULL; + rc = 1; +out: + kvfree(out); + kvfree(src); + return rc; +} diff --git a/fs/smb/server/compress.h b/fs/smb/server/compress.h new file mode 100644 index 000000000000..663c6f44f09b --- /dev/null +++ b/fs/smb/server/compress.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SMB2 compression support for ksmbd. + * + * Copyright (C) 2026 Namjae Jeon <linkinjeon@kernel.org> + */ +#ifndef __KSMBD_COMPRESS_H__ +#define __KSMBD_COMPRESS_H__ + +#include "connection.h" +#include "../common/compress/compress.h" + +int ksmbd_decompress_request(struct ksmbd_conn *conn); +int ksmbd_compress_response(struct ksmbd_work *work); + +#endif /* __KSMBD_COMPRESS_H__ */ diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index c5aac4946cbe..9e8fdb39e5a2 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -12,6 +12,7 @@ #include "smb_common.h" #include "mgmt/ksmbd_ida.h" #include "connection.h" +#include "compress.h" #include "transport_tcp.h" #include "transport_rdma.h" #include "misc.h" @@ -79,6 +80,85 @@ static int create_proc_clients(void) { return 0; } static void delete_proc_clients(void) {} #endif +static struct workqueue_struct *ksmbd_conn_wq; + +int ksmbd_conn_wq_init(void) +{ + ksmbd_conn_wq = alloc_workqueue("ksmbd-conn-release", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!ksmbd_conn_wq) + return -ENOMEM; + return 0; +} + +void ksmbd_conn_wq_destroy(void) +{ + if (ksmbd_conn_wq) { + destroy_workqueue(ksmbd_conn_wq); + ksmbd_conn_wq = NULL; + } +} + +/* + * __ksmbd_conn_release_work() - perform the final, once-per-struct cleanup + * of a ksmbd_conn whose refcount has just dropped to zero. + * + * This is the common release path used by ksmbd_conn_put() for the embedded + * state that outlives the connection thread: async_ida and the attached + * transport (which owns the socket and iov for TCP). Called from a workqueue + * so that sleep-allowed teardown (sock_release -> tcp_close -> + * lock_sock_nested) never runs from an RCU softirq callback (free_opinfo_rcu) + * or any other non-sleeping putter context. + */ +static void __ksmbd_conn_release_work(struct work_struct *work) +{ + struct ksmbd_conn *conn = + container_of(work, struct ksmbd_conn, release_work); + + ida_destroy(&conn->async_ida); + conn->transport->ops->free_transport(conn->transport); + kfree(conn); +} + +/** + * ksmbd_conn_get() - take a reference on @conn and return it. + * + * @conn: connection instance to get a reference to + * + * Returns @conn unchanged so callers can write + * "fp->conn = ksmbd_conn_get(work->conn);" in one expression. Returns NULL + * if @conn is NULL. + */ +struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn) +{ + if (!conn) + return NULL; + + atomic_inc(&conn->refcnt); + return conn; +} + +/** + * ksmbd_conn_put() - drop a reference and, if it was the last, queue the + * release onto ksmbd_conn_wq so it runs from process context. + * + * @conn: connection instance to put a reference to + * + * Callable from any context including RCU softirq callbacks and non-sleeping + * locks; the actual release is deferred to the workqueue. ksmbd_conn_wq is + * created in ksmbd_server_init() before any conn can be allocated and is + * destroyed in ksmbd_server_exit() after rcu_barrier(), so it is always + * non-NULL while a conn reference is held. + */ +void ksmbd_conn_put(struct ksmbd_conn *conn) +{ + if (!conn) + return; + + if (atomic_dec_and_test(&conn->refcnt)) + queue_work(ksmbd_conn_wq, &conn->release_work); +} + /** * ksmbd_conn_free() - free resources of the connection instance * @@ -93,23 +173,19 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) hash_del(&conn->hlist); up_write(&conn_list_lock); + /* + * request_buf / preauth_info / mechToken are only ever accessed by the + * connection handler thread that owns @conn. ksmbd_conn_free() is + * called from the transport free_transport() path when that thread is + * exiting, so it is safe to release them unconditionally even when + * ksmbd_conn_put() below is not the final putter (oplock / ksmbd_file + * holders only retain the conn pointer, not these per-thread buffers). + */ xa_destroy(&conn->sessions); kvfree(conn->request_buf); kfree(conn->preauth_info); kfree(conn->mechToken); - if (atomic_dec_and_test(&conn->refcnt)) { - /* - * async_ida is embedded in struct ksmbd_conn, so pair - * ida_destroy() with the final kfree() rather than with - * the unconditional field teardown above. This keeps - * the IDA valid for the entire lifetime of the struct, - * even while other refcount holders (oplock / vfs - * durable handles) still reference the connection. - */ - ida_destroy(&conn->async_ida); - conn->transport->ops->free_transport(conn->transport); - kfree(conn); - } + ksmbd_conn_put(conn); } /** @@ -136,6 +212,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void) conn->um = ERR_PTR(-EOPNOTSUPP); if (IS_ERR(conn->um)) conn->um = NULL; + INIT_WORK(&conn->release_work, __ksmbd_conn_release_work); atomic_set(&conn->req_running, 0); atomic_set(&conn->r_count, 0); atomic_set(&conn->refcnt, 1); @@ -455,6 +532,17 @@ recheck: continue; } + if (((struct smb2_hdr *)smb_get_msg(conn->request_buf))->ProtocolId == + SMB2_COMPRESSION_TRANSFORM_ID) { + /* + * Convert the transform into a normal RFC1002-framed SMB2 + * request before protocol validation and work allocation. + */ + if (ksmbd_decompress_request(conn)) + break; + pdu_size = get_rfc1002_len(conn->request_buf); + } + if (!ksmbd_smb_request(conn)) break; @@ -512,8 +600,7 @@ void ksmbd_conn_r_count_dec(struct ksmbd_conn *conn) if (!atomic_dec_return(&conn->r_count) && waitqueue_active(&conn->r_count_q)) wake_up(&conn->r_count_q); - if (atomic_dec_and_test(&conn->refcnt)) - kfree(conn); + ksmbd_conn_put(conn); } int ksmbd_conn_transport_init(void) diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h index de2d46941c93..ec75633b7da0 100644 --- a/fs/smb/server/connection.h +++ b/fs/smb/server/connection.h @@ -16,6 +16,7 @@ #include <linux/kthread.h> #include <linux/nls.h> #include <linux/unicode.h> +#include <linux/workqueue.h> #include "smb_common.h" #include "ksmbd_work.h" @@ -114,12 +115,16 @@ struct ksmbd_conn { __le16 cipher_type; __le16 compress_algorithm; + /* Negotiated SMB 3.1.1 compression capabilities. */ + bool compress_chained; + bool compress_pattern; bool posix_ext_supported; bool signing_negotiated; __le16 signing_algorithm; bool binding; atomic_t refcnt; bool is_aapl; + struct work_struct release_work; }; struct ksmbd_conn_ops { @@ -164,6 +169,10 @@ void ksmbd_conn_wait_idle(struct ksmbd_conn *conn); int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id); struct ksmbd_conn *ksmbd_conn_alloc(void); void ksmbd_conn_free(struct ksmbd_conn *conn); +struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn); +void ksmbd_conn_put(struct ksmbd_conn *conn); +int ksmbd_conn_wq_init(void); +void ksmbd_conn_wq_destroy(void); bool ksmbd_conn_lookup_dialect(struct ksmbd_conn *c); int ksmbd_conn_write(struct ksmbd_work *work); int ksmbd_conn_rdma_read(struct ksmbd_conn *conn, diff --git a/fs/smb/server/ksmbd_work.c b/fs/smb/server/ksmbd_work.c index ab4958dc3eb0..e2c2f45264be 100644 --- a/fs/smb/server/ksmbd_work.c +++ b/fs/smb/server/ksmbd_work.c @@ -16,6 +16,36 @@ static struct kmem_cache *work_cache; static struct workqueue_struct *ksmbd_wq; +static int ksmbd_reserve_iov(struct ksmbd_work *work, int need_iov_cnt) +{ + struct kvec *new; + int new_alloc_cnt = work->iov_alloc_cnt; + + if (work->iov_alloc_cnt >= work->iov_cnt + need_iov_cnt) + return 0; + + do { + new_alloc_cnt += KSMBD_WORK_INLINE_IOVS; + } while (new_alloc_cnt < work->iov_cnt + need_iov_cnt); + + if (work->iov == work->iov_inline) { + new = kcalloc(new_alloc_cnt, sizeof(*new), KSMBD_DEFAULT_GFP); + if (!new) + return -ENOMEM; + + memcpy(new, work->iov_inline, sizeof(work->iov_inline)); + } else { + new = krealloc(work->iov, sizeof(*new) * new_alloc_cnt, + KSMBD_DEFAULT_GFP | __GFP_ZERO); + if (!new) + return -ENOMEM; + } + + work->iov = new; + work->iov_alloc_cnt = new_alloc_cnt; + return 0; +} + struct ksmbd_work *ksmbd_alloc_work_struct(void) { struct ksmbd_work *work = kmem_cache_zalloc(work_cache, KSMBD_DEFAULT_GFP); @@ -27,13 +57,8 @@ struct ksmbd_work *ksmbd_alloc_work_struct(void) INIT_LIST_HEAD(&work->async_request_entry); INIT_LIST_HEAD(&work->fp_entry); INIT_LIST_HEAD(&work->aux_read_list); - work->iov_alloc_cnt = 4; - work->iov = kzalloc_objs(struct kvec, work->iov_alloc_cnt, - KSMBD_DEFAULT_GFP); - if (!work->iov) { - kmem_cache_free(work_cache, work); - work = NULL; - } + work->iov_alloc_cnt = ARRAY_SIZE(work->iov_inline); + work->iov = work->iov_inline; } return work; } @@ -53,8 +78,10 @@ void ksmbd_free_work_struct(struct ksmbd_work *work) } kfree(work->tr_buf); + kvfree(work->compress_buf); kvfree(work->request_buf); - kfree(work->iov); + if (work->iov != work->iov_inline) + kfree(work->iov); if (work->async_id) ksmbd_release_id(&work->conn->async_ida, work->async_id); @@ -116,19 +143,9 @@ static int __ksmbd_iov_pin_rsp(struct ksmbd_work *work, void *ib, int len, return -ENOMEM; } - if (work->iov_alloc_cnt < work->iov_cnt + need_iov_cnt) { - struct kvec *new; - - work->iov_alloc_cnt += 4; - new = krealloc(work->iov, - sizeof(struct kvec) * work->iov_alloc_cnt, - KSMBD_DEFAULT_GFP | __GFP_ZERO); - if (!new) { - kfree(ar); - work->iov_alloc_cnt -= 4; - return -ENOMEM; - } - work->iov = new; + if (ksmbd_reserve_iov(work, need_iov_cnt)) { + kfree(ar); + return -ENOMEM; } /* Plus rfc_length size on first iov */ diff --git a/fs/smb/server/ksmbd_work.h b/fs/smb/server/ksmbd_work.h index d36393ff8310..88104f0cf363 100644 --- a/fs/smb/server/ksmbd_work.h +++ b/fs/smb/server/ksmbd_work.h @@ -13,6 +13,8 @@ struct ksmbd_conn; struct ksmbd_session; struct ksmbd_tree_connect; +#define KSMBD_WORK_INLINE_IOVS 4 + enum { KSMBD_WORK_ACTIVE = 0, KSMBD_WORK_CANCELLED, @@ -42,6 +44,7 @@ struct ksmbd_work { int iov_alloc_cnt; int iov_cnt; int iov_idx; + struct kvec iov_inline[KSMBD_WORK_INLINE_IOVS]; /* Next cmd hdr in compound req buf*/ int next_smb2_rcv_hdr_off; @@ -57,22 +60,34 @@ struct ksmbd_work { u64 compound_fid; u64 compound_pfid; u64 compound_sid; + __le32 compound_status; const struct cred *saved_cred; /* Number of granted credits */ unsigned int credits_granted; + /* + * Credit charge added to conn->outstanding_credits at receive time + * for the SMB2 PDU currently being processed, pending release. Zero + * once the charge has been returned (on the response or error path). + */ + unsigned short credit_charge; + /* response smb header size */ unsigned int response_sz; void *tr_buf; + /* Contiguous SMB2 compression transform owned by this work item. */ + void *compress_buf; unsigned char state; /* No response for cancelled request */ bool send_no_response:1; /* Request is encrypted */ bool encrypted:1; + /* READ response should be wrapped in a compression transform. */ + bool compress_response:1; /* Is this SYNC or ASYNC ksmbd_work */ bool asynchronous:1; bool need_invalidate_rkey:1; diff --git a/fs/smb/server/mgmt/share_config.c b/fs/smb/server/mgmt/share_config.c index 53f44ff4d376..6f97f8d39657 100644 --- a/fs/smb/server/mgmt/share_config.c +++ b/fs/smb/server/mgmt/share_config.c @@ -167,7 +167,10 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->path = kstrndup(ksmbd_share_config_path(resp), path_len, KSMBD_DEFAULT_GFP); - if (share->path) { + if (!share->path) { + ret = -ENOMEM; + } else { + ret = 0; share->path_sz = strlen(share->path); while (share->path_sz > 1 && share->path[share->path_sz - 1] == '/') @@ -179,9 +182,10 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->force_directory_mode = resp->force_directory_mode; share->force_uid = resp->force_uid; share->force_gid = resp->force_gid; - ret = parse_veto_list(share, - KSMBD_SHARE_CONFIG_VETO_LIST(resp), - resp->veto_list_sz); + if (!ret) + ret = parse_veto_list(share, + KSMBD_SHARE_CONFIG_VETO_LIST(resp), + resp->veto_list_sz); if (!ret && share->path) { if (__ksmbd_override_fsids(work, share)) { kill_share(share); diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index de58aed76cb4..f99c86284ba3 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -255,7 +255,7 @@ static void free_channel_list(struct ksmbd_session *sess) down_write(&sess->chann_lock); xa_for_each(&sess->ksmbd_chann_list, index, chann) { xa_erase(&sess->ksmbd_chann_list, index); - kfree(chann); + kfree_sensitive(chann); } xa_destroy(&sess->ksmbd_chann_list); @@ -449,7 +449,7 @@ static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) if (!chann) return -ENOENT; - kfree(chann); + kfree_sensitive(chann); return 0; } @@ -457,22 +457,19 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn) { struct ksmbd_session *sess; unsigned long id; + struct hlist_node *tmp; + int bkt; down_write(&sessions_table_lock); - if (conn->binding) { - int bkt; - struct hlist_node *tmp; - - hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) { - if (!ksmbd_chann_del(conn, sess) && - xa_empty(&sess->ksmbd_chann_list)) { - hash_del(&sess->hlist); - down_write(&conn->session_lock); - xa_erase(&conn->sessions, sess->id); - up_write(&conn->session_lock); - if (atomic_dec_and_test(&sess->refcnt)) - ksmbd_session_destroy(sess); - } + hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) { + if (!ksmbd_chann_del(conn, sess) && + xa_empty(&sess->ksmbd_chann_list)) { + hash_del(&sess->hlist); + down_write(&conn->session_lock); + xa_erase(&conn->sessions, sess->id); + up_write(&conn->session_lock); + if (atomic_dec_and_test(&sess->refcnt)) + ksmbd_session_destroy(sess); } } diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index 6aebd385be84..4637a8c8436d 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -19,6 +19,7 @@ struct ksmbd_file_table; struct channel { + char sess_key[CIFS_KEY_SIZE]; __u8 smb3signingkey[SMB3_SIGN_KEY_SIZE]; struct ksmbd_conn *conn; }; diff --git a/fs/smb/server/misc.c b/fs/smb/server/misc.c index a543ec9d3581..2dd91c9e956a 100644 --- a/fs/smb/server/misc.c +++ b/fs/smb/server/misc.c @@ -121,7 +121,9 @@ int parse_stream_name(char *filename, char **stream_name, int *s_type) char *stream_type; char *s_name; int rc = 0; + bool has_stream_type = false; + *stream_name = NULL; s_name = filename; filename = strsep(&s_name, ":"); ksmbd_debug(SMB, "filename : %s, streams : %s\n", filename, s_name); @@ -137,14 +139,20 @@ int parse_stream_name(char *filename, char **stream_name, int *s_type) ksmbd_debug(SMB, "stream name : %s, stream type : %s\n", s_name, stream_type); - if (!strncasecmp("$data", stream_type, 5)) + if (!strncasecmp("$data", stream_type, 5)) { *s_type = DATA_STREAM; - else if (!strncasecmp("$index_allocation", stream_type, 17)) + has_stream_type = true; + } else if (!strncasecmp("$index_allocation", stream_type, 17)) { *s_type = DIR_STREAM; - else + has_stream_type = true; + } else { rc = -ENOENT; + } } + if (has_stream_type && !s_name[0] && *s_type == DATA_STREAM) + goto out; + *stream_name = s_name; out: return rc; @@ -283,39 +291,6 @@ char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename) return ksmbd_casefold_sharename(um, name); } -/** - * convert_to_unix_name() - convert windows name to unix format - * @share: ksmbd_share_config pointer - * @name: file name that is relative to share - * - * Return: converted name on success, otherwise NULL - */ -char *convert_to_unix_name(struct ksmbd_share_config *share, const char *name) -{ - int no_slash = 0, name_len, path_len; - char *new_name; - - if (name[0] == '/') - name++; - - path_len = share->path_sz; - name_len = strlen(name); - new_name = kmalloc(path_len + name_len + 2, KSMBD_DEFAULT_GFP); - if (!new_name) - return new_name; - - memcpy(new_name, share->path, path_len); - if (new_name[path_len - 1] != '/') { - new_name[path_len] = '/'; - no_slash = 1; - } - - memcpy(new_name + path_len + no_slash, name, name_len); - path_len += name_len + no_slash; - new_name[path_len] = 0x00; - return new_name; -} - char *ksmbd_convert_dir_info_name(struct ksmbd_dir_info *d_info, const struct nls_table *local_nls, int *conv_len) diff --git a/fs/smb/server/misc.h b/fs/smb/server/misc.h index 13423696ae8c..3909104e18ad 100644 --- a/fs/smb/server/misc.h +++ b/fs/smb/server/misc.h @@ -25,7 +25,6 @@ void ksmbd_strip_last_slash(char *path); void ksmbd_conv_path_to_windows(char *path); char *ksmbd_casefold_sharename(struct unicode_map *um, const char *name); char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename); -char *convert_to_unix_name(struct ksmbd_share_config *share, const char *name); #define KSMBD_DIR_INFO_ALIGNMENT 8 struct ksmbd_dir_info; diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index cd3f28b0e7cb..79787099afdc 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -5,6 +5,7 @@ */ #include <linux/moduleparam.h> +#include <linux/err.h> #include "glob.h" #include "oplock.h" @@ -19,6 +20,53 @@ static LIST_HEAD(lease_table_list); static DEFINE_RWLOCK(lease_list_lock); +#define SMB2_LEASE_STATE_MASK_LE (SMB2_LEASE_READ_CACHING_LE | \ + SMB2_LEASE_HANDLE_CACHING_LE | \ + SMB2_LEASE_WRITE_CACHING_LE) + +static bool lease_state_valid(__le32 state) +{ + return !(state & ~SMB2_LEASE_STATE_MASK_LE); +} + +static __le32 lease_state_grantable(__le32 state) +{ + if (state == SMB2_LEASE_READ_CACHING_LE || + state == (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE) || + state == (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_WRITE_CACHING_LE) || + state == SMB2_LEASE_STATE_MASK_LE) + return state; + + return 0; +} + +static bool lease_v2_flags_valid(__le32 flags) +{ + return !(flags & ~SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE); +} + +static bool lease_has_parent_key(struct lease *lease) +{ + return lease->flags & SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE; +} + +static bool lease_break_in_progress(struct lease *lease) +{ + struct oplock_info *opinfo; + bool ret = false; + + spin_lock(&lease->lock); + list_for_each_entry(opinfo, &lease->open_list, lease_entry) { + if (opinfo->op_state == OPLOCK_ACK_WAIT) { + ret = true; + break; + } + } + spin_unlock(&lease->lock); + + return ret; +} + /** * alloc_opinfo() - allocate a new opinfo object for oplock info * @work: smb work @@ -30,7 +78,6 @@ static DEFINE_RWLOCK(lease_list_lock); static struct oplock_info *alloc_opinfo(struct ksmbd_work *work, u64 id, __u16 Tid) { - struct ksmbd_conn *conn = work->conn; struct ksmbd_session *sess = work->sess; struct oplock_info *opinfo; @@ -39,47 +86,60 @@ static struct oplock_info *alloc_opinfo(struct ksmbd_work *work, return NULL; opinfo->sess = sess; - opinfo->conn = conn; + opinfo->conn = ksmbd_conn_get(work->conn); opinfo->level = SMB2_OPLOCK_LEVEL_NONE; opinfo->op_state = OPLOCK_STATE_NONE; opinfo->pending_break = 0; opinfo->fid = id; opinfo->Tid = Tid; INIT_LIST_HEAD(&opinfo->op_entry); + INIT_LIST_HEAD(&opinfo->lease_entry); init_waitqueue_head(&opinfo->oplock_q); init_waitqueue_head(&opinfo->oplock_brk); atomic_set(&opinfo->refcount, 1); atomic_set(&opinfo->breaking_cnt, 0); - atomic_inc(&opinfo->conn->refcnt); return opinfo; } -static void lease_add_list(struct oplock_info *opinfo) +static void lease_get(struct lease *lease) { - struct lease_table *lb = opinfo->o_lease->l_lb; + atomic_inc(&lease->refcount); +} +static void lease_put(struct lease *lease) +{ + if (lease && atomic_dec_and_test(&lease->refcount)) + kfree(lease); +} + +static void lease_add_table(struct lease *lease, struct lease_table *lb) +{ + lease_get(lease); + lease->l_lb = lb; spin_lock(&lb->lb_lock); - list_add_rcu(&opinfo->lease_entry, &lb->lease_list); + list_add_rcu(&lease->l_entry, &lb->lease_list); spin_unlock(&lb->lb_lock); } -static void lease_del_list(struct oplock_info *opinfo) +static void lease_del_table(struct lease *lease) { - struct lease_table *lb = opinfo->o_lease->l_lb; + struct lease_table *lb = lease->l_lb; if (!lb) return; spin_lock(&lb->lb_lock); - if (list_empty(&opinfo->lease_entry)) { + if (list_empty(&lease->l_entry)) { spin_unlock(&lb->lb_lock); return; } - list_del_init(&opinfo->lease_entry); - opinfo->o_lease->l_lb = NULL; + list_del_init(&lease->l_entry); + lease->l_lb = NULL; spin_unlock(&lb->lb_lock); + + lease_put(lease); } static struct lease_table *alloc_lease_table(struct oplock_info *opinfo) @@ -92,18 +152,29 @@ static struct lease_table *alloc_lease_table(struct oplock_info *opinfo) memcpy(lb->client_guid, opinfo->conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); + lb->conn = ksmbd_conn_get(opinfo->conn); INIT_LIST_HEAD(&lb->lease_list); spin_lock_init(&lb->lb_lock); return lb; } -static int alloc_lease(struct oplock_info *opinfo, struct lease_ctx_info *lctx) +static void free_lease_table(struct lease_table *lb) +{ + if (!lb) + return; + + ksmbd_conn_put(lb->conn); + kfree(lb); +} + +static struct lease *alloc_lease(struct lease_ctx_info *lctx, + struct ksmbd_inode *ci) { struct lease *lease; lease = kmalloc_obj(struct lease, KSMBD_DEFAULT_GFP); if (!lease) - return -ENOMEM; + return NULL; memcpy(lease->lease_key, lctx->lease_key, SMB2_LEASE_KEY_SIZE); lease->state = lctx->req_state; @@ -113,27 +184,57 @@ static int alloc_lease(struct oplock_info *opinfo, struct lease_ctx_info *lctx) lease->is_dir = lctx->is_dir; memcpy(lease->parent_lease_key, lctx->parent_lease_key, SMB2_LEASE_KEY_SIZE); lease->version = lctx->version; - lease->epoch = le16_to_cpu(lctx->epoch) + 1; - INIT_LIST_HEAD(&opinfo->lease_entry); - opinfo->o_lease = lease; + lease->epoch = lctx->version == 2 ? le16_to_cpu(lctx->epoch) + 1 : 0; + lease->ci = ci; + lease->reuse_epoch = false; + lease->l_lb = NULL; + INIT_LIST_HEAD(&lease->l_entry); + INIT_LIST_HEAD(&lease->open_list); + spin_lock_init(&lease->lock); + atomic_set(&lease->refcount, 1); + + return lease; +} - return 0; +static void lease_add_open(struct lease *lease, struct oplock_info *opinfo) +{ + spin_lock(&lease->lock); + list_add(&opinfo->lease_entry, &lease->open_list); + spin_unlock(&lease->lock); } -static void free_lease(struct oplock_info *opinfo) +static void lease_del_open(struct oplock_info *opinfo) { - struct lease *lease; + struct lease *lease = opinfo->o_lease; + bool remove_table = false; + + if (!lease) + return; - lease = opinfo->o_lease; - kfree(lease); + spin_lock(&lease->lock); + if (!list_empty(&opinfo->lease_entry)) { + list_del_init(&opinfo->lease_entry); + remove_table = list_empty(&lease->open_list); + } + spin_unlock(&lease->lock); + + if (remove_table) { + write_lock(&lease_list_lock); + lease_del_table(lease); + write_unlock(&lease_list_lock); + } +} + +static void free_lease(struct oplock_info *opinfo) +{ + lease_put(opinfo->o_lease); } static void __free_opinfo(struct oplock_info *opinfo) { if (opinfo->is_lease) free_lease(opinfo); - if (opinfo->conn && atomic_dec_and_test(&opinfo->conn->refcnt)) - kfree(opinfo->conn); + ksmbd_conn_put(opinfo->conn); kfree(opinfo); } @@ -149,6 +250,21 @@ static void free_opinfo(struct oplock_info *opinfo) call_rcu(&opinfo->rcu, free_opinfo_rcu); } +void lease_update_oplock_levels(struct lease *lease) +{ + struct oplock_info *opinfo; + __u8 level; + + if (!lease) + return; + + level = smb2_map_lease_to_oplock(lease->state); + spin_lock(&lease->lock); + list_for_each_entry(opinfo, &lease->open_list, lease_entry) + opinfo->level = level; + spin_unlock(&lease->lock); +} + struct oplock_info *opinfo_get(struct ksmbd_file *fp) { struct oplock_info *opinfo; @@ -162,10 +278,24 @@ struct oplock_info *opinfo_get(struct ksmbd_file *fp) return opinfo; } -static struct oplock_info *opinfo_get_list(struct ksmbd_inode *ci) +struct oplock_snapshot { + bool durable_open; + bool durable_detached; + unsigned long long fid; +}; + +static struct oplock_info *opinfo_get_list(struct ksmbd_inode *ci, + struct ksmbd_file *skip_fp, + struct oplock_snapshot *snapshot) { struct oplock_info *opinfo; + if (snapshot) { + snapshot->durable_open = false; + snapshot->durable_detached = false; + snapshot->fid = KSMBD_NO_FID; + } + down_read(&ci->m_lock); opinfo = list_first_entry_or_null(&ci->m_op_list, struct oplock_info, op_entry); @@ -179,6 +309,16 @@ static struct oplock_info *opinfo_get_list(struct ksmbd_inode *ci) opinfo = NULL; } } + + if (opinfo && snapshot && opinfo->o_fp && + opinfo->o_fp != skip_fp && + READ_ONCE(opinfo->o_fp->is_durable)) { + snapshot->durable_open = true; + snapshot->durable_detached = + !READ_ONCE(opinfo->o_fp->conn) || + !READ_ONCE(opinfo->o_fp->tcon); + snapshot->fid = opinfo->fid; + } } up_read(&ci->m_lock); @@ -196,6 +336,18 @@ void opinfo_put(struct oplock_info *opinfo) free_opinfo(opinfo); } +static bool ksmbd_inode_has_lease(struct ksmbd_inode *ci) +{ + struct oplock_info *opinfo = opinfo_get_list(ci, NULL, NULL); + bool is_lease; + + if (!opinfo) + return false; + is_lease = opinfo->is_lease; + opinfo_put(opinfo); + return is_lease; +} + static void opinfo_add(struct oplock_info *opinfo, struct ksmbd_file *fp) { struct ksmbd_inode *ci = fp->f_ci; @@ -209,11 +361,9 @@ static void opinfo_del(struct oplock_info *opinfo) { struct ksmbd_inode *ci = opinfo->o_fp->f_ci; - if (opinfo->is_lease) { - write_lock(&lease_list_lock); - lease_del_list(opinfo); - write_unlock(&lease_list_lock); - } + if (opinfo->is_lease) + lease_del_open(opinfo); + down_write(&ci->m_lock); list_del(&opinfo->op_entry); up_write(&ci->m_lock); @@ -262,8 +412,10 @@ int opinfo_write_to_read(struct oplock_info *opinfo) } opinfo->level = SMB2_OPLOCK_LEVEL_II; - if (opinfo->is_lease) + if (opinfo->is_lease) { lease->state = lease->new_state; + lease_update_oplock_levels(lease); + } return 0; } @@ -278,7 +430,7 @@ int opinfo_read_handle_to_read(struct oplock_info *opinfo) struct lease *lease = opinfo->o_lease; lease->state = lease->new_state; - opinfo->level = SMB2_OPLOCK_LEVEL_II; + lease_update_oplock_levels(lease); return 0; } @@ -300,8 +452,10 @@ int opinfo_write_to_none(struct oplock_info *opinfo) return -EINVAL; } opinfo->level = SMB2_OPLOCK_LEVEL_NONE; - if (opinfo->is_lease) + if (opinfo->is_lease) { lease->state = lease->new_state; + lease_update_oplock_levels(lease); + } return 0; } @@ -322,8 +476,10 @@ int opinfo_read_to_none(struct oplock_info *opinfo) return -EINVAL; } opinfo->level = SMB2_OPLOCK_LEVEL_NONE; - if (opinfo->is_lease) + if (opinfo->is_lease) { lease->state = lease->new_state; + lease_update_oplock_levels(lease); + } return 0; } @@ -344,10 +500,7 @@ int lease_read_to_write(struct oplock_info *opinfo) lease->new_state = SMB2_LEASE_NONE_LE; lease->state |= SMB2_LEASE_WRITE_CACHING_LE; - if (lease->state & SMB2_LEASE_HANDLE_CACHING_LE) - opinfo->level = SMB2_OPLOCK_LEVEL_BATCH; - else - opinfo->level = SMB2_OPLOCK_LEVEL_EXCLUSIVE; + lease_update_oplock_levels(lease); return 0; } @@ -369,15 +522,7 @@ static int lease_none_upgrade(struct oplock_info *opinfo, __le32 new_state) lease->new_state = SMB2_LEASE_NONE_LE; lease->state = new_state; - if (lease->state & SMB2_LEASE_HANDLE_CACHING_LE) - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - opinfo->level = SMB2_OPLOCK_LEVEL_BATCH; - else - opinfo->level = SMB2_OPLOCK_LEVEL_II; - else if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - opinfo->level = SMB2_OPLOCK_LEVEL_EXCLUSIVE; - else if (lease->state & SMB2_LEASE_READ_CACHING_LE) - opinfo->level = SMB2_OPLOCK_LEVEL_II; + lease_update_oplock_levels(lease); return 0; } @@ -484,8 +629,12 @@ static inline int compare_guid_key(struct oplock_info *opinfo, const char *guid1, const char *key1) { const char *guid2, *key2; + struct ksmbd_conn *conn; - guid2 = opinfo->conn->ClientGUID; + conn = READ_ONCE(opinfo->conn); + if (!conn) + return 0; + guid2 = conn->ClientGUID; key2 = opinfo->o_lease->lease_key; if (!memcmp(guid1, guid2, SMB2_CLIENT_GUID_SIZE) && !memcmp(key1, key2, SMB2_LEASE_KEY_SIZE)) @@ -504,7 +653,7 @@ static inline int compare_guid_key(struct oplock_info *opinfo, * Return: oplock(lease) object on success, otherwise NULL */ static struct oplock_info *same_client_has_lease(struct ksmbd_inode *ci, - char *client_guid, + const char *client_guid, struct lease_ctx_info *lctx) { int ret; @@ -527,7 +676,12 @@ static struct oplock_info *same_client_has_lease(struct ksmbd_inode *ci, ret = compare_guid_key(opinfo, client_guid, lctx->lease_key); if (ret) { + if (!atomic_inc_not_zero(&opinfo->refcount)) + continue; + if (m_opinfo) + opinfo_put(m_opinfo); m_opinfo = opinfo; + /* skip upgrading lease about breaking lease */ if (atomic_read(&opinfo->breaking_cnt)) continue; @@ -549,8 +703,11 @@ static struct oplock_info *same_client_has_lease(struct ksmbd_inode *ci, if (lctx->req_state == (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE)) { - lease->epoch++; - lease->state = lctx->req_state; + if (lease->state != lctx->req_state) { + lease->epoch++; + lease->state = lctx->req_state; + lease_update_oplock_levels(lease); + } } } @@ -566,7 +723,7 @@ static struct oplock_info *same_client_has_lease(struct ksmbd_inode *ci, return m_opinfo; } -static void wait_for_break_ack(struct oplock_info *opinfo) +static bool wait_for_break_ack(struct oplock_info *opinfo) { int rc = 0; @@ -577,11 +734,16 @@ static void wait_for_break_ack(struct oplock_info *opinfo) /* is this a timeout ? */ if (!rc) { - if (opinfo->is_lease) + if (opinfo->is_lease) { opinfo->o_lease->state = SMB2_LEASE_NONE_LE; + lease_update_oplock_levels(opinfo->o_lease); + } opinfo->level = SMB2_OPLOCK_LEVEL_NONE; opinfo->op_state = OPLOCK_STATE_NONE; + return true; } + + return false; } static void wake_up_oplock_break(struct oplock_info *opinfo) @@ -595,6 +757,9 @@ static void wake_up_oplock_break(struct oplock_info *opinfo) static int oplock_break_pending(struct oplock_info *opinfo, int req_op_level) { while (test_and_set_bit(0, &opinfo->pending_break)) { + if (opinfo->is_lease) + opinfo->o_lease->reuse_epoch = true; + wait_on_bit(&opinfo->pending_break, 0, TASK_UNINTERRUPTIBLE); /* Not immediately break to none. */ @@ -628,6 +793,17 @@ static int oplock_break_pending(struct oplock_info *opinfo, int req_op_level) return 0; } +static bool lease_break_needed(struct oplock_info *opinfo, int req_op_level, + bool open_trunc) +{ + struct lease *lease = opinfo->o_lease; + + if (open_trunc) + return lease->state != SMB2_LEASE_NONE_LE; + + return opinfo->level > req_op_level; +} + /** * __smb2_oplock_break_noti() - send smb2 oplock break cmd from conn * to client @@ -699,6 +875,7 @@ static void __smb2_oplock_break_noti(struct work_struct *wk) out: ksmbd_free_work_struct(work); ksmbd_conn_r_count_dec(conn); + ksmbd_conn_put(conn); } /** @@ -710,11 +887,16 @@ out: */ static int smb2_oplock_break_noti(struct oplock_info *opinfo) { - struct ksmbd_conn *conn = opinfo->conn; + struct ksmbd_conn *conn; struct oplock_break_info *br_info; int ret = 0; - struct ksmbd_work *work = ksmbd_alloc_work_struct(); + struct ksmbd_work *work; + conn = READ_ONCE(opinfo->conn); + if (!conn) + return ksmbd_invalidate_durable_fd(opinfo->fid); + + work = ksmbd_alloc_work_struct(); if (!work) return -ENOMEM; @@ -729,7 +911,7 @@ static int smb2_oplock_break_noti(struct oplock_info *opinfo) br_info->open_trunc = opinfo->open_trunc; work->request_buf = (char *)br_info; - work->conn = conn; + work->conn = ksmbd_conn_get(conn); work->sess = opinfo->sess; ksmbd_conn_r_count_inc(conn); @@ -737,7 +919,8 @@ static int smb2_oplock_break_noti(struct oplock_info *opinfo) INIT_WORK(&work->work, __smb2_oplock_break_noti); ksmbd_queue_work(work); - wait_for_break_ack(opinfo); + if (wait_for_break_ack(opinfo)) + ret = ksmbd_invalidate_durable_fd(opinfo->fid); } else { __smb2_oplock_break_noti(&work->work); if (opinfo->level == SMB2_OPLOCK_LEVEL_II) @@ -803,21 +986,33 @@ static void __smb2_lease_break_noti(struct work_struct *wk) out: ksmbd_free_work_struct(work); ksmbd_conn_r_count_dec(conn); + ksmbd_conn_put(conn); } /** * smb2_lease_break_noti() - break lease when a new client request * write lease * @opinfo: contains lease state information + * @wait_ack: wait for lease break acknowledgment from the client + * @inc_epoch: increment the lease epoch before sending the break * * Return: 0 on success, otherwise error */ -static int smb2_lease_break_noti(struct oplock_info *opinfo) +static int smb2_lease_break_noti(struct oplock_info *opinfo, bool wait_ack, + bool inc_epoch) { - struct ksmbd_conn *conn = opinfo->conn; + struct ksmbd_conn *conn; struct ksmbd_work *work; struct lease_break_info *br_info; struct lease *lease = opinfo->o_lease; + int ret = 0; + + conn = READ_ONCE(opinfo->conn); + if (lease->version == 2 && lease->l_lb && lease->l_lb->conn && + !ksmbd_conn_releasing(lease->l_lb->conn)) + conn = lease->l_lb->conn; + if (!conn) + return ksmbd_invalidate_durable_fd(opinfo->fid); work = ksmbd_alloc_work_struct(); if (!work) @@ -831,29 +1026,35 @@ static int smb2_lease_break_noti(struct oplock_info *opinfo) br_info->curr_state = lease->state; br_info->new_state = lease->new_state; - if (lease->version == 2) - br_info->epoch = cpu_to_le16(++lease->epoch); - else + if (lease->version == 2) { + if (inc_epoch) + lease->epoch++; + br_info->epoch = cpu_to_le16(lease->epoch); + } else { br_info->epoch = 0; + } memcpy(br_info->lease_key, lease->lease_key, SMB2_LEASE_KEY_SIZE); work->request_buf = (char *)br_info; - work->conn = conn; + work->conn = ksmbd_conn_get(conn); work->sess = opinfo->sess; ksmbd_conn_r_count_inc(conn); if (opinfo->op_state == OPLOCK_ACK_WAIT) { INIT_WORK(&work->work, __smb2_lease_break_noti); ksmbd_queue_work(work); - wait_for_break_ack(opinfo); + if (wait_ack) { + if (wait_for_break_ack(opinfo)) + ret = ksmbd_invalidate_durable_fd(opinfo->fid); + } } else { __smb2_lease_break_noti(&work->work); if (opinfo->o_lease->new_state == SMB2_LEASE_NONE_LE) { - opinfo->level = SMB2_OPLOCK_LEVEL_NONE; opinfo->o_lease->state = SMB2_LEASE_NONE_LE; + lease_update_oplock_levels(opinfo->o_lease); } } - return 0; + return ret; } static void wait_lease_breaking(struct oplock_info *opinfo) @@ -874,9 +1075,10 @@ static void wait_lease_breaking(struct oplock_info *opinfo) } static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, - struct ksmbd_work *in_work) + struct ksmbd_work *in_work, bool share_break) { int err = 0; + bool sent_interim = false; /* Need to break exclusive/batch oplock, write lease or overwrite_if */ ksmbd_debug(OPLOCK, @@ -885,18 +1087,36 @@ static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, if (brk_opinfo->is_lease) { struct lease *lease = brk_opinfo->o_lease; + bool open_trunc = brk_opinfo->open_trunc; + bool was_pending = test_bit(0, &brk_opinfo->pending_break); + bool wait_ack; + bool inc_epoch = true; + + if (in_work && was_pending) { + setup_async_work(in_work, NULL, NULL); + smb2_send_interim_resp(in_work, STATUS_PENDING); + release_async_work(in_work); + sent_interim = true; + } - atomic_inc(&brk_opinfo->breaking_cnt); err = oplock_break_pending(brk_opinfo, req_op_level); if (err) return err < 0 ? err : 0; + if (was_pending) + open_trunc = brk_opinfo->open_trunc; - if (brk_opinfo->open_trunc) { +again: + atomic_inc(&brk_opinfo->breaking_cnt); + if (open_trunc) { /* * Create overwrite break trigger the lease break to * none. */ lease->new_state = SMB2_LEASE_NONE_LE; + } else if (share_break && + lease->state & SMB2_LEASE_HANDLE_CACHING_LE) { + lease->new_state = + lease->state & ~SMB2_LEASE_HANDLE_CACHING_LE; } else { if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) { if (lease->state & SMB2_LEASE_HANDLE_CACHING_LE) @@ -916,17 +1136,56 @@ static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, } } + if (in_work && !sent_interim) { + setup_async_work(in_work, NULL, NULL); + smb2_send_interim_resp(in_work, STATUS_PENDING); + release_async_work(in_work); + sent_interim = true; + } + if (lease->state & (SMB2_LEASE_WRITE_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE)) { - if (in_work) { - setup_async_work(in_work, NULL, NULL); - smb2_send_interim_resp(in_work, STATUS_PENDING); - release_async_work(in_work); - } - brk_opinfo->op_state = OPLOCK_ACK_WAIT; } else atomic_dec(&brk_opinfo->breaking_cnt); + + wait_ack = !(open_trunc && + lease->state == (SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)); + if (lease->reuse_epoch) { + inc_epoch = false; + lease->reuse_epoch = false; + } + err = smb2_lease_break_noti(brk_opinfo, wait_ack, inc_epoch); + inc_epoch = false; + + ksmbd_debug(OPLOCK, "oplock granted = %d\n", brk_opinfo->level); + if (brk_opinfo->op_state == OPLOCK_CLOSING) + err = -ENOENT; + + if (wait_ack) + wait_lease_breaking(brk_opinfo); + /* + * A share-mode conflict break only drops the conflicting + * caching bit; the triggering open fails with a sharing + * violation, so keep it to a single break. + * + * Otherwise chain another break while the lease is still + * incompatible with this open (req_op_level), or while a + * truncating waiter that arrived during the break still needs + * the lease dropped to none. open_trunc snapshotted for this + * break stays cleared, so the next state is computed from the + * lease state and the cascade steps down (e.g. RH->R->none) + * instead of collapsing straight to none. + */ + if (wait_ack && !err && !share_break && + (lease_break_needed(brk_opinfo, req_op_level, open_trunc) || + (brk_opinfo->open_trunc && + lease->state != SMB2_LEASE_NONE_LE))) + goto again; + + wake_up_oplock_break(brk_opinfo); + return err; } else { err = oplock_break_pending(brk_opinfo, req_op_level); if (err) @@ -937,25 +1196,50 @@ static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, brk_opinfo->op_state = OPLOCK_ACK_WAIT; } - if (brk_opinfo->is_lease) - err = smb2_lease_break_noti(brk_opinfo); - else - err = smb2_oplock_break_noti(brk_opinfo); + err = smb2_oplock_break_noti(brk_opinfo); ksmbd_debug(OPLOCK, "oplock granted = %d\n", brk_opinfo->level); if (brk_opinfo->op_state == OPLOCK_CLOSING) - err = -ENOENT; + err = -EAGAIN; wake_up_oplock_break(brk_opinfo); - wait_lease_breaking(brk_opinfo); - return err; } +struct oplock_break_entry { + struct list_head list; + struct oplock_info *opinfo; +}; + +static int oplock_break_add(struct list_head *head, struct oplock_info *opinfo) +{ + struct oplock_break_entry *ent; + + ent = kmalloc_obj(struct oplock_break_entry, KSMBD_DEFAULT_GFP); + if (!ent) + return -ENOMEM; + + ent->opinfo = opinfo; + list_add_tail(&ent->list, head); + return 0; +} + +static void oplock_break_drain_none(struct list_head *head) +{ + struct oplock_break_entry *ent, *tmp; + + list_for_each_entry_safe(ent, tmp, head, list) { + oplock_break(ent->opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL, false); + list_del(&ent->list); + opinfo_put(ent->opinfo); + kfree(ent); + } +} + void destroy_lease_table(struct ksmbd_conn *conn) { struct lease_table *lb, *lbtmp; - struct oplock_info *opinfo; + struct lease *lease, *ltmp; write_lock(&lease_list_lock); if (list_empty(&lease_table_list)) { @@ -967,25 +1251,18 @@ void destroy_lease_table(struct ksmbd_conn *conn) if (conn && memcmp(lb->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) continue; -again: - rcu_read_lock(); - list_for_each_entry_rcu(opinfo, &lb->lease_list, - lease_entry) { - rcu_read_unlock(); - lease_del_list(opinfo); - goto again; - } - rcu_read_unlock(); + list_for_each_entry_safe(lease, ltmp, &lb->lease_list, l_entry) + lease_del_table(lease); list_del(&lb->l_entry); - kfree(lb); + free_lease_table(lb); } write_unlock(&lease_list_lock); } -int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, +int find_same_lease_key(struct ksmbd_conn *conn, struct ksmbd_inode *ci, struct lease_ctx_info *lctx) { - struct oplock_info *opinfo; + struct lease *lease; int err = 0; struct lease_table *lb; @@ -999,7 +1276,7 @@ int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, } list_for_each_entry(lb, &lease_table_list, l_entry) { - if (!memcmp(lb->client_guid, sess->ClientGUID, + if (!memcmp(lb->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) goto found; } @@ -1008,67 +1285,40 @@ int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, return 0; found: - rcu_read_lock(); - list_for_each_entry_rcu(opinfo, &lb->lease_list, lease_entry) { - if (!atomic_inc_not_zero(&opinfo->refcount)) + list_for_each_entry(lease, &lb->lease_list, l_entry) { + if (lease->ci == ci) continue; - rcu_read_unlock(); - if (opinfo->o_fp->f_ci == ci) - goto op_next; - err = compare_guid_key(opinfo, sess->ClientGUID, - lctx->lease_key); - if (err) { + if (!memcmp(lease->lease_key, lctx->lease_key, + SMB2_LEASE_KEY_SIZE)) { err = -EINVAL; ksmbd_debug(OPLOCK, "found same lease key is already used in other files\n"); - opinfo_put(opinfo); goto out; } -op_next: - opinfo_put(opinfo); - rcu_read_lock(); } - rcu_read_unlock(); out: read_unlock(&lease_list_lock); return err; } -static void copy_lease(struct oplock_info *op1, struct oplock_info *op2) -{ - struct lease *lease1 = op1->o_lease; - struct lease *lease2 = op2->o_lease; - - op2->level = op1->level; - lease2->state = lease1->state; - memcpy(lease2->lease_key, lease1->lease_key, - SMB2_LEASE_KEY_SIZE); - lease2->duration = lease1->duration; - lease2->flags = lease1->flags; - lease2->epoch = lease1->epoch; - lease2->version = lease1->version; -} - -static void add_lease_global_list(struct oplock_info *opinfo, +static void add_lease_global_list(struct lease *lease, struct ksmbd_conn *conn, struct lease_table *new_lb) { struct lease_table *lb; write_lock(&lease_list_lock); list_for_each_entry(lb, &lease_table_list, l_entry) { - if (!memcmp(lb->client_guid, opinfo->conn->ClientGUID, + if (!memcmp(lb->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) { - opinfo->o_lease->l_lb = lb; - lease_add_list(opinfo); + lease_add_table(lease, lb); write_unlock(&lease_list_lock); - kfree(new_lb); + free_lease_table(new_lb); return; } } - opinfo->o_lease->l_lb = new_lb; - lease_add_list(opinfo); + lease_add_table(lease, new_lb); list_add(&new_lb->l_entry, &lease_table_list); write_unlock(&lease_list_lock); } @@ -1095,6 +1345,7 @@ void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, { struct oplock_info *opinfo; struct ksmbd_inode *p_ci = NULL; + LIST_HEAD(brk_list); if (lctx->version != 2) return; @@ -1120,12 +1371,14 @@ void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, continue; } - oplock_break(opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL); - opinfo_put(opinfo); + if (oplock_break_add(&brk_list, opinfo)) + opinfo_put(opinfo); } } up_read(&p_ci->m_lock); + oplock_break_drain_none(&brk_list); + ksmbd_inode_put(p_ci); } @@ -1133,6 +1386,7 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) { struct oplock_info *opinfo; struct ksmbd_inode *p_ci = NULL; + LIST_HEAD(brk_list); rcu_read_lock(); opinfo = rcu_dereference(fp->f_opinfo); @@ -1161,12 +1415,14 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) continue; } - oplock_break(opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL); - opinfo_put(opinfo); + if (oplock_break_add(&brk_list, opinfo)) + opinfo_put(opinfo); } } up_read(&p_ci->m_lock); + oplock_break_drain_none(&brk_list); + ksmbd_inode_put(p_ci); } @@ -1186,19 +1442,22 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct ksmbd_file *fp, __u16 tid, struct lease_ctx_info *lctx, int share_ret) { - struct ksmbd_session *sess = work->sess; int err = 0; + int break_level = SMB2_OPLOCK_LEVEL_II; struct oplock_info *opinfo = NULL, *prev_opinfo = NULL; struct ksmbd_inode *ci = fp->f_ci; struct lease_table *new_lb = NULL; + struct oplock_snapshot prev_op_snapshot; bool prev_op_has_lease; + bool prev_durable_open = false; + bool prev_durable_detached = false; + unsigned long long prev_fid = KSMBD_NO_FID; + bool new_lease = false; __le32 prev_op_state = 0; /* Only v2 leases handle the directory */ if (S_ISDIR(file_inode(fp->filp)->i_mode)) { - if (!lctx || lctx->version != 2 || - (lctx->flags != SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE && - !lctx->epoch)) + if (!lctx || lctx->version != 2) return 0; } @@ -1207,20 +1466,35 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, return -ENOMEM; if (lctx) { - err = alloc_lease(opinfo, lctx); - if (err) + opinfo->o_lease = alloc_lease(lctx, ci); + if (!opinfo->o_lease) { + err = -ENOMEM; goto err_out; + } opinfo->is_lease = 1; + new_lease = true; } /* ci does not have any oplock */ if (!opinfo_count(fp)) goto set_lev; - /* grant none-oplock if second open is trunc */ - if (fp->attrib_only && fp->cdoption != FILE_OVERWRITE_IF_LE && + /* + * A stat open that only requests metadata access must not break the + * existing caching state. READ_CONTROL (reading the security + * descriptor) does not conflict with a lease, but it does conflict + * with an oplock, so only treat a read-control-only open as a stat + * open when the existing holder is a lease. + */ + if (fp->cdoption != FILE_OVERWRITE_IF_LE && fp->cdoption != FILE_OVERWRITE_LE && - fp->cdoption != FILE_SUPERSEDE_LE) { + fp->cdoption != FILE_SUPERSEDE_LE && + (fp->attrib_only || + (!(fp->daccess & ~(FILE_READ_ATTRIBUTES_LE | + FILE_WRITE_ATTRIBUTES_LE | + FILE_SYNCHRONIZE_LE | + FILE_READ_CONTROL_LE)) && + ksmbd_inode_has_lease(ci)))) { req_op_level = SMB2_OPLOCK_LEVEL_NONE; goto set_lev; } @@ -1229,17 +1503,19 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct oplock_info *m_opinfo; /* is lease already granted ? */ - m_opinfo = same_client_has_lease(ci, sess->ClientGUID, + m_opinfo = same_client_has_lease(ci, work->conn->ClientGUID, lctx); if (m_opinfo) { - copy_lease(m_opinfo, opinfo); - if (atomic_read(&m_opinfo->breaking_cnt)) - opinfo->o_lease->flags = - SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE; + lease_put(opinfo->o_lease); + lease_get(m_opinfo->o_lease); + opinfo->o_lease = m_opinfo->o_lease; + opinfo->level = m_opinfo->level; + new_lease = false; + opinfo_put(m_opinfo); goto out; } } - prev_opinfo = opinfo_get_list(ci); + prev_opinfo = opinfo_get_list(ci, fp, &prev_op_snapshot); if (!prev_opinfo || (prev_opinfo->level == SMB2_OPLOCK_LEVEL_NONE && lctx)) { opinfo_put(prev_opinfo); @@ -1248,7 +1524,6 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, prev_op_has_lease = prev_opinfo->is_lease; if (prev_op_has_lease) prev_op_state = prev_opinfo->o_lease->state; - if (share_ret < 0 && prev_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { err = share_ret; @@ -1262,10 +1537,28 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, goto op_break_not_needed; } - err = oplock_break(prev_opinfo, SMB2_OPLOCK_LEVEL_II, work); + prev_durable_open = prev_op_snapshot.durable_open; + prev_durable_detached = prev_op_snapshot.durable_detached; + prev_fid = prev_op_snapshot.fid; + + err = oplock_break(prev_opinfo, break_level, work, + share_ret < 0 && prev_opinfo->is_lease); + if (prev_durable_detached || (prev_durable_open && err == -ENOENT)) + ksmbd_invalidate_durable_fd(prev_fid); opinfo_put(prev_opinfo); - if (err == -ENOENT) + if (err == -EAGAIN) { + share_ret = ksmbd_smb_check_shared_mode(fp->filp, fp); + if (share_ret < 0) { + err = share_ret; + goto err_out; + } + goto set_lev; + } + if (err == -ENOENT) { + if (req_op_level != SMB2_OPLOCK_LEVEL_NONE) + req_op_level = SMB2_OPLOCK_LEVEL_II; goto set_lev; + } /* Check all oplock was freed by close */ else if (err < 0) goto err_out; @@ -1294,17 +1587,13 @@ set_lev: out: /* - * Set o_fp before any publication so that concurrent readers - * (e.g. find_same_lease_key() on the lease list) that - * dereference opinfo->o_fp don't hit a NULL pointer. - * * Keep the original publication order so concurrent opens can * still observe the in-flight grant via ci->m_op_list, but make * everything after opinfo_add() no-fail by preallocating any new * lease_table first. */ opinfo->o_fp = fp; - if (opinfo->is_lease) { + if (new_lease) { new_lb = alloc_lease_table(opinfo); if (!new_lb) { err = -ENOMEM; @@ -1315,8 +1604,10 @@ out: opinfo_count_inc(fp); opinfo_add(opinfo, fp); + if (new_lease) + add_lease_global_list(opinfo->o_lease, opinfo->conn, new_lb); if (opinfo->is_lease) - add_lease_global_list(opinfo, new_lb); + lease_add_open(opinfo->o_lease, opinfo); rcu_assign_pointer(fp->f_opinfo, opinfo); @@ -1333,38 +1624,48 @@ err_out: * @fp: ksmbd file pointer * @is_trunc: truncate on open */ -static void smb_break_all_write_oplock(struct ksmbd_work *work, +static bool smb_break_all_write_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, int is_trunc) { struct oplock_info *brk_opinfo; + bool sent_break = false; - brk_opinfo = opinfo_get_list(fp->f_ci); + brk_opinfo = opinfo_get_list(fp->f_ci, NULL, NULL); if (!brk_opinfo) - return; + return false; if (brk_opinfo->level != SMB2_OPLOCK_LEVEL_BATCH && brk_opinfo->level != SMB2_OPLOCK_LEVEL_EXCLUSIVE) { opinfo_put(brk_opinfo); - return; + return false; } brk_opinfo->open_trunc = is_trunc; - oplock_break(brk_opinfo, SMB2_OPLOCK_LEVEL_II, work); + oplock_break(brk_opinfo, SMB2_OPLOCK_LEVEL_II, work, false); + sent_break = true; opinfo_put(brk_opinfo); + + return sent_break; } /** - * smb_break_all_levII_oplock() - send level2 oplock or read lease break command + * __smb_break_all_levII_oplock() - send level2 oplock or read lease break command * from server to client - * @work: smb work - * @fp: ksmbd file pointer - * @is_trunc: truncate on open + * @work: smb work + * @fp: ksmbd file pointer + * @is_trunc: truncate on open + * @send_interim: send interim response to the client + * @send_oplock_break: send oplock break notification to the client */ -void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, - int is_trunc) +static void __smb_break_all_levII_oplock(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc, + bool send_interim, bool send_oplock_break) { struct oplock_info *op, *brk_op; + struct oplock_break_entry *ent, *tmp; struct ksmbd_inode *ci; struct ksmbd_conn *conn = work->conn; + bool sent_interim = false; + LIST_HEAD(brk_list); if (!test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_OPLOCKS)) @@ -1386,14 +1687,8 @@ void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, continue; } - if (brk_op->is_lease && (brk_op->o_lease->state & - (~(SMB2_LEASE_READ_CACHING_LE | - SMB2_LEASE_HANDLE_CACHING_LE)))) { - ksmbd_debug(OPLOCK, "unexpected lease state(0x%x)\n", - brk_op->o_lease->state); - goto next; - } else if (brk_op->level != - SMB2_OPLOCK_LEVEL_II) { + if (!brk_op->is_lease && + brk_op->level != SMB2_OPLOCK_LEVEL_II) { ksmbd_debug(OPLOCK, "unexpected oplock(0x%x)\n", brk_op->level); goto next; @@ -1412,16 +1707,60 @@ void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, SMB2_LEASE_KEY_SIZE)) goto next; brk_op->open_trunc = is_trunc; - oplock_break(brk_op, SMB2_OPLOCK_LEVEL_NONE, NULL); + + /* + * Defer the break until ci->m_lock is released: oplock_break() + * may block waiting for the lease break acknowledgment, and the + * close that wakes that wait needs ci->m_lock for write. + */ + if (!oplock_break_add(&brk_list, brk_op)) + continue; next: opinfo_put(brk_op); } up_read(&ci->m_lock); + list_for_each_entry_safe(ent, tmp, &brk_list, list) { + brk_op = ent->opinfo; + + if (!brk_op->is_lease && !send_oplock_break) { + brk_op->level = SMB2_OPLOCK_LEVEL_NONE; + brk_op->op_state = OPLOCK_STATE_NONE; + } else { + oplock_break(brk_op, + brk_op->is_lease && !is_trunc ? + SMB2_OPLOCK_LEVEL_II : SMB2_OPLOCK_LEVEL_NONE, + send_interim && !sent_interim ? work : NULL, + false); + } + sent_interim = true; + list_del(&ent->list); + opinfo_put(brk_op); + kfree(ent); + } + if (op) opinfo_put(op); } +void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, + int is_trunc) +{ + __smb_break_all_levII_oplock(work, fp, is_trunc, true, true); +} + +void smb_break_all_levII_oplock_no_interim(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc) +{ + __smb_break_all_levII_oplock(work, fp, is_trunc, false, true); +} + +void smb_break_all_levII_oplock_for_delete(struct ksmbd_work *work, + struct ksmbd_file *fp) +{ + __smb_break_all_levII_oplock(work, fp, 0, false, false); +} + /** * smb_break_all_oplock() - break both batch/exclusive and level2 oplock * @work: smb work @@ -1429,12 +1768,14 @@ next: */ void smb_break_all_oplock(struct ksmbd_work *work, struct ksmbd_file *fp) { + bool sent_break; + if (!test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_OPLOCKS)) return; - smb_break_all_write_oplock(work, fp, 1); - smb_break_all_levII_oplock(work, fp, 1); + sent_break = smb_break_all_write_oplock(work, fp, 1); + __smb_break_all_levII_oplock(work, fp, 1, !sent_break, true); } /** @@ -1445,15 +1786,13 @@ void smb_break_all_oplock(struct ksmbd_work *work, struct ksmbd_file *fp) */ __u8 smb2_map_lease_to_oplock(__le32 lease_state) { - if (lease_state == (SMB2_LEASE_HANDLE_CACHING_LE | - SMB2_LEASE_READ_CACHING_LE | - SMB2_LEASE_WRITE_CACHING_LE)) { + if ((lease_state & SMB2_LEASE_WRITE_CACHING_LE) && + (lease_state & SMB2_LEASE_HANDLE_CACHING_LE)) { return SMB2_OPLOCK_LEVEL_BATCH; - } else if (lease_state != SMB2_LEASE_WRITE_CACHING_LE && - lease_state & SMB2_LEASE_WRITE_CACHING_LE) { - if (!(lease_state & SMB2_LEASE_HANDLE_CACHING_LE)) - return SMB2_OPLOCK_LEVEL_EXCLUSIVE; - } else if (lease_state & SMB2_LEASE_READ_CACHING_LE) { + } else if (lease_state & SMB2_LEASE_WRITE_CACHING_LE) { + return SMB2_OPLOCK_LEVEL_EXCLUSIVE; + } else if (lease_state & (SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)) { return SMB2_OPLOCK_LEVEL_II; } return 0; @@ -1468,14 +1807,19 @@ void create_lease_buf(u8 *rbuf, struct lease *lease) { if (lease->version == 2) { struct create_lease_v2 *buf = (struct create_lease_v2 *)rbuf; + __le32 flags = 0; memset(buf, 0, sizeof(struct create_lease_v2)); memcpy(buf->lcontext.LeaseKey, lease->lease_key, SMB2_LEASE_KEY_SIZE); - buf->lcontext.LeaseFlags = lease->flags; + if (lease_has_parent_key(lease)) + flags |= SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE; + if (lease_break_in_progress(lease)) + flags |= SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE; + buf->lcontext.LeaseFlags = flags; buf->lcontext.Epoch = cpu_to_le16(lease->epoch); buf->lcontext.LeaseState = lease->state; - if (lease->flags == SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE) + if (lease_has_parent_key(lease)) memcpy(buf->lcontext.ParentLeaseKey, lease->parent_lease_key, SMB2_LEASE_KEY_SIZE); buf->ccontext.DataOffset = cpu_to_le16(offsetof @@ -1493,7 +1837,9 @@ void create_lease_buf(u8 *rbuf, struct lease *lease) memset(buf, 0, sizeof(struct create_lease)); memcpy(buf->lcontext.LeaseKey, lease->lease_key, SMB2_LEASE_KEY_SIZE); - buf->lcontext.LeaseFlags = lease->flags; + if (lease_break_in_progress(lease)) + buf->lcontext.LeaseFlags = + SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE; buf->lcontext.LeaseState = lease->state; buf->ccontext.DataOffset = cpu_to_le16(offsetof (struct create_lease, lcontext)); @@ -1521,12 +1867,14 @@ struct lease_ctx_info *parse_lease_state(void *open_req) struct lease_ctx_info *lreq; cc = smb2_find_context_vals(req, SMB2_CREATE_REQUEST_LEASE, 4); - if (IS_ERR_OR_NULL(cc)) + if (IS_ERR(cc)) + return ERR_CAST(cc); + if (!cc) return NULL; lreq = kzalloc_obj(struct lease_ctx_info, KSMBD_DEFAULT_GFP); if (!lreq) - return NULL; + return ERR_PTR(-ENOMEM); if (sizeof(struct lease_context_v2) == le32_to_cpu(cc->DataLength)) { struct create_lease_v2 *lc = (struct create_lease_v2 *)cc; @@ -1540,11 +1888,15 @@ struct lease_ctx_info *parse_lease_state(void *open_req) lreq->flags = lc->lcontext.LeaseFlags; lreq->epoch = lc->lcontext.Epoch; lreq->duration = lc->lcontext.LeaseDuration; + if (!lease_state_valid(lreq->req_state) || + !lease_v2_flags_valid(lreq->flags)) + goto err_out; + lreq->req_state = lease_state_grantable(lreq->req_state); if (lreq->flags == SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE) memcpy(lreq->parent_lease_key, lc->lcontext.ParentLeaseKey, SMB2_LEASE_KEY_SIZE); lreq->version = 2; - } else { + } else if (sizeof(struct lease_context) == le32_to_cpu(cc->DataLength)) { struct create_lease *lc = (struct create_lease *)cc; if (le16_to_cpu(cc->DataOffset) + le32_to_cpu(cc->DataLength) < @@ -1553,14 +1905,18 @@ struct lease_ctx_info *parse_lease_state(void *open_req) memcpy(lreq->lease_key, lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); lreq->req_state = lc->lcontext.LeaseState; - lreq->flags = lc->lcontext.LeaseFlags; + lreq->flags = 0; lreq->duration = lc->lcontext.LeaseDuration; + if (!lease_state_valid(lreq->req_state)) + goto err_out; + lreq->req_state = lease_state_grantable(lreq->req_state); lreq->version = 1; - } + } else + goto err_out; return lreq; err_out: kfree(lreq); - return NULL; + return ERR_PTR(-EINVAL); } /** @@ -1585,6 +1941,9 @@ struct create_context *smb2_find_context_vals(void *open_req, const char *tag, i * CreateContextsOffset and CreateContextsLength are guaranteed to * be valid because of ksmbd_smb2_check_message(). */ + if (!req->CreateContextsOffset || !req->CreateContextsLength) + return NULL; + cc = (struct create_context *)((char *)req + le32_to_cpu(req->CreateContextsOffset)); remain_len = le32_to_cpu(req->CreateContextsLength); @@ -1794,8 +2153,8 @@ struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, char *lease_key) { struct oplock_info *opinfo = NULL, *ret_op = NULL; + struct lease *lease; struct lease_table *lt; - int ret; read_lock(&lease_list_lock); list_for_each_entry(lt, &lease_table_list, l_entry) { @@ -1808,29 +2167,30 @@ struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, return NULL; found: - rcu_read_lock(); - list_for_each_entry_rcu(opinfo, <->lease_list, lease_entry) { - if (!atomic_inc_not_zero(&opinfo->refcount)) + list_for_each_entry(lease, <->lease_list, l_entry) { + if (memcmp(lease->lease_key, lease_key, SMB2_LEASE_KEY_SIZE)) continue; - rcu_read_unlock(); - if (!opinfo->op_state || opinfo->op_state == OPLOCK_CLOSING) - goto op_next; - if (!(opinfo->o_lease->state & - (SMB2_LEASE_HANDLE_CACHING_LE | - SMB2_LEASE_WRITE_CACHING_LE))) - goto op_next; - ret = compare_guid_key(opinfo, conn->ClientGUID, - lease_key); - if (ret) { - ksmbd_debug(OPLOCK, "found opinfo\n"); + if (!(lease->state & (SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE))) + break; + + spin_lock(&lease->lock); + list_for_each_entry(opinfo, &lease->open_list, lease_entry) { + if (!opinfo->op_state || + opinfo->op_state == OPLOCK_CLOSING) + continue; + if (!atomic_inc_not_zero(&opinfo->refcount)) + continue; ret_op = opinfo; + break; + } + spin_unlock(&lease->lock); + if (ret_op) { + ksmbd_debug(OPLOCK, "found opinfo\n"); goto out; } -op_next: - opinfo_put(opinfo); - rcu_read_lock(); + break; } - rcu_read_unlock(); out: read_unlock(&lease_list_lock); @@ -1850,6 +2210,12 @@ int smb2_check_durable_oplock(struct ksmbd_conn *conn, if (!opinfo) return 0; + if (ksmbd_has_other_active_fd(fp)) { + ksmbd_debug(SMB, "Durable handle reconnect failed: competing open\n"); + ret = -EBADF; + goto out; + } + if (ksmbd_vfs_compare_durable_owner(fp, user) == false) { ksmbd_debug(SMB, "Durable handle reconnect failed: owner mismatch\n"); ret = -EBADF; diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h index d91a8266e065..3f581d22bb67 100644 --- a/fs/smb/server/oplock.h +++ b/fs/smb/server/oplock.h @@ -34,6 +34,7 @@ struct lease_ctx_info { struct lease_table { char client_guid[SMB2_CLIENT_GUID_SIZE]; + struct ksmbd_conn *conn; struct list_head lease_list; struct list_head l_entry; spinlock_t lb_lock; @@ -49,7 +50,13 @@ struct lease { int version; unsigned short epoch; bool is_dir; + bool reuse_epoch; + struct ksmbd_inode *ci; struct lease_table *l_lb; + struct list_head l_entry; + struct list_head open_list; + spinlock_t lock; + atomic_t refcount; }; struct oplock_info { @@ -92,6 +99,10 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, struct lease_ctx_info *lctx, int share_ret); void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, int is_trunc); +void smb_break_all_levII_oplock_no_interim(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc); +void smb_break_all_levII_oplock_for_delete(struct ksmbd_work *work, + struct ksmbd_file *fp); int opinfo_write_to_read(struct oplock_info *opinfo); int opinfo_read_handle_to_read(struct oplock_info *opinfo); int opinfo_write_to_none(struct oplock_info *opinfo); @@ -105,6 +116,7 @@ void opinfo_put(struct oplock_info *opinfo); void create_lease_buf(u8 *rbuf, struct lease *lease); struct lease_ctx_info *parse_lease_state(void *open_req); __u8 smb2_map_lease_to_oplock(__le32 lease_state); +void lease_update_oplock_levels(struct lease *lease); int lease_read_to_write(struct oplock_info *opinfo); /* Durable related functions */ @@ -116,7 +128,7 @@ void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp); struct create_context *smb2_find_context_vals(void *open_req, const char *tag, int tag_len); struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, char *lease_key); -int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, +int find_same_lease_key(struct ksmbd_conn *conn, struct ksmbd_inode *ci, struct lease_ctx_info *lctx); void destroy_lease_table(struct ksmbd_conn *conn); void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index 58ef02c423fc..f5baba934840 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -22,6 +22,7 @@ #include "crypto_ctx.h" #include "auth.h" #include "stats.h" +#include "compress.h" int ksmbd_debug_types; @@ -198,6 +199,12 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, else conn->ops->set_rsp_status(work, STATUS_USER_SESSION_DELETED); + if (conn->ops->is_sign_req(work, conn->ops->get_cmd_val(work))) { + struct smb2_hdr *rsp_hdr; + + rsp_hdr = ksmbd_resp_buf_curr(work); + rsp_hdr->Flags |= SMB2_FLAGS_SIGNED; + } goto send; } else if (rc > 0) { rc = conn->ops->get_ksmbd_tcon(work); @@ -236,14 +243,43 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (work->sess && (work->sess->sign || smb3_11_final_sess_setup_resp(work) || - conn->ops->is_sign_req(work, command))) - conn->ops->set_sign_rsp(work); + conn->ops->is_sign_req(work, command))) { + if (command == SMB2_SESSION_SETUP_HE && + work->sess->dialect >= SMB30_PROT_ID && + conn->dialect < SMB30_PROT_ID) + smb3_set_sign_rsp(work); + else + conn->ops->set_sign_rsp(work); + } } while (is_chained == true); send: + /* + * Release any credit charge still outstanding for this request. On + * the normal path smb2_set_rsp_credits() already returned it, but the + * abort, error and send-no-response paths skip that call, so the + * charge would otherwise leak and eventually exhaust the connection's + * outstanding credit window. + */ + if (work->credit_charge) { + spin_lock(&conn->credits_lock); + conn->outstanding_credits -= work->credit_charge; + work->credit_charge = 0; + spin_unlock(&conn->credits_lock); + } + if (work->tcon) ksmbd_tree_connect_put(work->tcon); smb3_preauth_hash_rsp(work); + /* + * Preauthentication hashes cover the original SMB2 response. Apply the + * transport compression wrapper only after updating the hash. + */ + if (work->compress_response) { + rc = ksmbd_compress_response(work); + if (rc < 0) + ksmbd_debug(CONN, "Failed to compress response: %d\n", rc); + } if (work->sess && work->sess->enc && work->encrypted && conn->ops->encrypt_resp) { rc = conn->ops->encrypt_resp(work); @@ -596,8 +632,14 @@ static int __init ksmbd_server_init(void) if (ret) goto err_crypto_destroy; + ret = ksmbd_conn_wq_init(); + if (ret) + goto err_workqueue_destroy; + return 0; +err_workqueue_destroy: + ksmbd_workqueue_destroy(); err_crypto_destroy: ksmbd_crypto_destroy(); err_release_inode_hash: @@ -623,6 +665,12 @@ static void __exit ksmbd_server_exit(void) { ksmbd_server_shutdown(); rcu_barrier(); + /* + * ksmbd_conn_put() defers the final release onto ksmbd_conn_wq, + * so drain it after rcu_barrier() has fired any pending RCU + * callbacks that may have queued a release. + */ + ksmbd_conn_wq_destroy(); ksmbd_release_inode_hash(); } diff --git a/fs/smb/server/smb2misc.c b/fs/smb/server/smb2misc.c index a1ddca21c47b..9f3629c86291 100644 --- a/fs/smb/server/smb2misc.c +++ b/fs/smb/server/smb2misc.c @@ -261,8 +261,12 @@ calc_size_exit: static inline int smb2_query_info_req_len(struct smb2_query_info_req *h) { - return le32_to_cpu(h->InputBufferLength) + - le32_to_cpu(h->OutputBufferLength); + return le32_to_cpu(h->InputBufferLength); +} + +static inline int smb2_query_info_resp_len(struct smb2_query_info_req *h) +{ + return le32_to_cpu(h->OutputBufferLength); } static inline int smb2_set_info_req_len(struct smb2_set_info_req *h) @@ -297,9 +301,10 @@ static inline int smb2_ioctl_resp_len(struct smb2_ioctl_req *h) le32_to_cpu(h->MaxOutputResponse); } -static int smb2_validate_credit_charge(struct ksmbd_conn *conn, +static int smb2_validate_credit_charge(struct ksmbd_work *work, struct smb2_hdr *hdr) { + struct ksmbd_conn *conn = work->conn; unsigned int req_len = 0, expect_resp_len = 0, calc_credit_num, max_len; unsigned short credit_charge = le16_to_cpu(hdr->CreditCharge); void *__hdr = hdr; @@ -308,6 +313,7 @@ static int smb2_validate_credit_charge(struct ksmbd_conn *conn, switch (hdr->Command) { case SMB2_QUERY_INFO: req_len = smb2_query_info_req_len(__hdr); + expect_resp_len = smb2_query_info_resp_len(__hdr); break; case SMB2_SET_INFO: req_len = smb2_set_info_req_len(__hdr); @@ -356,8 +362,10 @@ static int smb2_validate_credit_charge(struct ksmbd_conn *conn, ksmbd_debug(SMB, "Limits exceeding the maximum allowable outstanding requests, given : %u, pending : %u\n", credit_charge, conn->outstanding_credits); ret = 1; - } else + } else { conn->outstanding_credits += credit_charge; + work->credit_charge = credit_charge; + } spin_unlock(&conn->credits_lock); @@ -399,6 +407,11 @@ int ksmbd_smb2_check_message(struct ksmbd_work *work) return 1; } + if (len < __SMB2_HEADER_STRUCTURE_SIZE + sizeof(__le16)) { + ksmbd_debug(SMB, "Message is too small for StructureSize2\n"); + return 1; + } + if (smb2_req_struct_sizes[command] != pdu->StructureSize2) { if (!(command == SMB2_OPLOCK_BREAK_HE && (le16_to_cpu(pdu->StructureSize2) == OP_BREAK_STRUCT_SIZE_20 || @@ -460,7 +473,7 @@ int ksmbd_smb2_check_message(struct ksmbd_work *work) validate_credit: if ((work->conn->vals->req_capabilities & SMB2_GLOBAL_CAP_LARGE_MTU) && - smb2_validate_credit_charge(work->conn, hdr)) + smb2_validate_credit_charge(work, hdr)) return 1; return 0; diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 47b7af631f7b..bec692bca1ca 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -14,9 +14,10 @@ #include <linux/falloc.h> #include <linux/mount.h> #include <linux/filelock.h> +#include <linux/fileattr.h> #include "glob.h" -#include "smbfsctl.h" +#include "../common/smbfsctl.h" #include "oplock.h" #include "smbacl.h" @@ -41,6 +42,7 @@ #include "ndr.h" #include "stats.h" #include "transport_tcp.h" +#include "compress.h" static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) { @@ -55,6 +57,13 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) #define WORK_BUFFERS(w, rq, rs) __wbuf((w), (void **)&(rq), (void **)&(rs)) +#define SMB2_CREATE_FILE_ATTRIBUTE_MASK \ + (FILE_ATTRIBUTE_MASK & ~(FILE_ATTRIBUTE_INTEGRITY_STREAM | \ + FILE_ATTRIBUTE_NO_SCRUB_DATA)) + +/* Windows reports automatic write-time updates at roughly 15 ms resolution. */ +#define KSMBD_WRITE_TIME_RESOLUTION (15ULL * 10000) + /** * check_session_id() - check for valid session id in smb header * @conn: connection instance @@ -89,6 +98,47 @@ struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn return chann; } +#define KSMBD_MAX_CHANNELS 32 + +static int register_session_channel(struct ksmbd_session *sess, + struct ksmbd_conn *conn, + const char *sess_key) +{ + struct channel *chann, *old; + unsigned long index; + unsigned int count = 0; + int rc = 0; + + down_write(&sess->chann_lock); + if (xa_load(&sess->ksmbd_chann_list, (long)conn)) + goto out; + + xa_for_each(&sess->ksmbd_chann_list, index, chann) + count++; + if (count >= KSMBD_MAX_CHANNELS) { + rc = -ENOSPC; + goto out; + } + + chann = kmalloc_obj(struct channel, KSMBD_DEFAULT_GFP); + if (!chann) { + rc = -ENOMEM; + goto out; + } + + chann->conn = conn; + memcpy(chann->sess_key, sess_key, sizeof(chann->sess_key)); + old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, + KSMBD_DEFAULT_GFP); + if (xa_is_err(old)) { + kfree_sensitive(chann); + rc = xa_err(old); + } +out: + up_write(&sess->chann_lock); + return rc; +} + /** * smb2_get_ksmbd_tcon() - get tree connection information using a tree id. * @work: smb work @@ -244,6 +294,13 @@ void set_smb2_rsp_status(struct ksmbd_work *work, __le32 err) { struct smb2_hdr *rsp_hdr; + if (work->next_smb2_rcv_hdr_off) { + rsp_hdr = ksmbd_resp_buf_next(work); + rsp_hdr->Status = err; + smb2_set_err_rsp(work); + return; + } + rsp_hdr = smb_get_msg(work->response_buf); rsp_hdr->Status = err; @@ -350,6 +407,7 @@ int smb2_set_rsp_credits(struct ksmbd_work *work) conn->total_credits -= credit_charge; conn->outstanding_credits -= credit_charge; + work->credit_charge = 0; credits_requested = max_t(unsigned short, le16_to_cpu(req_hdr->CreditRequest), 1); @@ -403,6 +461,59 @@ static void init_chained_smb2_rsp(struct ksmbd_work *work) work->compound_fid = ((struct smb2_create_rsp *)rsp)->VolatileFileId; work->compound_pfid = ((struct smb2_create_rsp *)rsp)->PersistentFileId; work->compound_sid = le64_to_cpu(rsp->SessionId); + work->compound_status = STATUS_SUCCESS; + } else if ((req->Command == SMB2_FLUSH || + req->Command == SMB2_READ || + req->Command == SMB2_WRITE) && + rsp->Status == STATUS_SUCCESS) { + u64 volatile_id = KSMBD_NO_FID; + u64 persistent_id = KSMBD_NO_FID; + + if (req->Command == SMB2_FLUSH) { + struct smb2_flush_req *flush_req = + (struct smb2_flush_req *)req; + + volatile_id = flush_req->VolatileFileId; + persistent_id = flush_req->PersistentFileId; + } else if (req->Command == SMB2_READ) { + struct smb2_read_req *read_req = + (struct smb2_read_req *)req; + + volatile_id = read_req->VolatileFileId; + persistent_id = read_req->PersistentFileId; + } else { + struct smb2_write_req *write_req = + (struct smb2_write_req *)req; + + volatile_id = write_req->VolatileFileId; + persistent_id = write_req->PersistentFileId; + } + + if (has_file_id(volatile_id)) { + work->compound_fid = volatile_id; + work->compound_pfid = persistent_id; + work->compound_sid = le64_to_cpu(rsp->SessionId); + work->compound_status = STATUS_SUCCESS; + } + } else if (req->Command == SMB2_CREATE) { + work->compound_fid = KSMBD_NO_FID; + work->compound_pfid = KSMBD_NO_FID; + work->compound_sid = le64_to_cpu(rsp->SessionId); + work->compound_status = rsp->Status; + } else if (rsp->Status != STATUS_SUCCESS) { + work->compound_sid = le64_to_cpu(rsp->SessionId); + /* + * Only carry the failed status forward when the failing command + * was itself part of the related chain. An unrelated command + * that fails (e.g. a standalone request with a bad session id) + * must not seed the status for a following related command, + * which has to be evaluated on its own (and may legitimately + * fail with a different status such as INVALID_PARAMETER). The + * compound session id is still tracked so a following related + * command can validate it. + */ + if (req->Flags & SMB2_FLAGS_RELATED_OPERATIONS) + work->compound_status = rsp->Status; } len = get_rfc1002_len(work->response_buf) - work->next_smb2_rsp_hdr_off; @@ -428,6 +539,7 @@ static void init_chained_smb2_rsp(struct ksmbd_work *work) ksmbd_debug(SMB, "related flag should be set\n"); work->compound_fid = KSMBD_NO_FID; work->compound_pfid = KSMBD_NO_FID; + work->compound_status = STATUS_SUCCESS; } memset((char *)rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; @@ -447,6 +559,19 @@ static void init_chained_smb2_rsp(struct ksmbd_work *work) memcpy(rsp_hdr->Signature, rcv_hdr->Signature, 16); } +static bool smb2_compound_has_failed(struct ksmbd_work *work, + struct smb2_hdr *rsp) +{ + if (!work->next_smb2_rcv_hdr_off || + has_file_id(work->compound_fid) || + work->compound_status == STATUS_SUCCESS) + return false; + + rsp->Status = work->compound_status; + smb2_set_err_rsp(work); + return true; +} + /** * is_chained_smb2_message() - check for chained command * @work: smb work containing smb request buffer @@ -613,6 +738,11 @@ int smb2_check_user_session(struct ksmbd_work *work) sess_id, work->sess->id); return -EINVAL; } + if (work->sess->state != SMB2_SESSION_VALID) { + pr_err("compound request on a non-valid session (state %d)\n", + work->sess->state); + return -EINVAL; + } return 1; } @@ -803,6 +933,30 @@ static void build_encrypt_ctxt(struct smb2_encryption_neg_context *pneg_ctxt, pneg_ctxt->Ciphers[0] = cipher_type; } +static void build_compress_ctxt(struct smb2_compression_capabilities_context *pneg_ctxt, + __le16 compress_algorithm, bool compress_chained, + bool compress_pattern) +{ + /* + * Return only algorithms implemented by ksmbd. Pattern_V1 is advertised + * as a second ID when the client also enabled chained transforms. + */ + pneg_ctxt->ContextType = SMB2_COMPRESSION_CAPABILITIES; + pneg_ctxt->DataLength = cpu_to_le16(compress_pattern ? 12 : 10); + pneg_ctxt->Reserved = cpu_to_le32(0); + pneg_ctxt->CompressionAlgorithmCount = + cpu_to_le16(compress_pattern ? 2 : 1); + pneg_ctxt->Padding = cpu_to_le16(0); + pneg_ctxt->Flags = compress_chained ? + SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED : + SMB2_COMPRESSION_CAPABILITIES_FLAG_NONE; + pneg_ctxt->CompressionAlgorithms[0] = compress_algorithm; + pneg_ctxt->CompressionAlgorithms[1] = compress_pattern ? + SMB3_COMPRESS_PATTERN : 0; + pneg_ctxt->CompressionAlgorithms[2] = 0; + pneg_ctxt->CompressionAlgorithms[3] = 0; +} + static void build_sign_cap_ctxt(struct smb2_signing_capabilities *pneg_ctxt, __le16 sign_algo) { @@ -864,8 +1018,19 @@ static unsigned int assemble_neg_contexts(struct ksmbd_conn *conn, ctxt_size += sizeof(struct smb2_encryption_neg_context) + 2; } - /* compression context not yet supported */ - WARN_ON(conn->compress_algorithm != SMB3_COMPRESS_NONE); + if (conn->compress_algorithm != SMB3_COMPRESS_NONE) { + ctxt_size = round_up(ctxt_size, 8); + ksmbd_debug(SMB, + "assemble SMB2_COMPRESSION_CAPABILITIES context\n"); + build_compress_ctxt((struct smb2_compression_capabilities_context *) + (pneg_ctxt + ctxt_size), + conn->compress_algorithm, + conn->compress_chained, + conn->compress_pattern); + neg_ctxt_cnt++; + ctxt_size += sizeof(struct smb2_neg_context) + + (conn->compress_pattern ? 12 : 10); + } if (conn->posix_ext_supported) { ctxt_size = round_up(ctxt_size, 8); @@ -969,10 +1134,67 @@ bool smb3_encryption_negotiated(struct ksmbd_conn *conn) conn->cipher_type; } -static void decode_compress_ctxt(struct ksmbd_conn *conn, - struct smb2_compression_capabilities_context *pneg_ctxt) +static __le32 decode_compress_ctxt(struct ksmbd_conn *conn, + struct smb2_compression_capabilities_context *pneg_ctxt, + int ctxt_len) { + int alg_cnt, algs_size, i; + __le16 *algs; + + if (sizeof(struct smb2_neg_context) + 10 > ctxt_len) { + pr_err("Invalid SMB2_COMPRESSION_CAPABILITIES context length\n"); + return STATUS_INVALID_PARAMETER; + } + conn->compress_algorithm = SMB3_COMPRESS_NONE; + conn->compress_chained = false; + conn->compress_pattern = false; + + alg_cnt = le16_to_cpu(pneg_ctxt->CompressionAlgorithmCount); + if (!alg_cnt) + return STATUS_INVALID_PARAMETER; + + if (pneg_ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_NONE && + pneg_ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED) + return STATUS_INVALID_PARAMETER; + + algs_size = alg_cnt * sizeof(__le16); + if (sizeof(struct smb2_neg_context) + 8 + algs_size > ctxt_len) { + pr_err("Invalid compression algorithm count(%d)\n", alg_cnt); + return STATUS_INVALID_PARAMETER; + } + + /* + * CompressionAlgorithms[] is declared as a fixed 4-element array, but + * the actual element count is variable (clients such as Windows may + * advertise more). The on-wire length was validated above, so walk the + * algorithms through a pointer to avoid a fixed-array bounds check. + */ + algs = pneg_ctxt->CompressionAlgorithms; + for (i = 0; i < alg_cnt; i++) { + __le16 alg = algs[i]; + + /* + * LZ77 is the required general-purpose codec. Pattern_V1 is an + * optional chained payload type and cannot stand alone. + */ + if (alg == SMB3_COMPRESS_LZ77) { + conn->compress_algorithm = alg; + conn->compress_chained = + pneg_ctxt->Flags == + SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED; + ksmbd_debug(SMB, "Compression Algorithm ID = 0x%x\n", + le16_to_cpu(alg)); + } else if (alg == SMB3_COMPRESS_PATTERN) { + conn->compress_pattern = true; + } + } + + if (conn->compress_algorithm == SMB3_COMPRESS_NONE || + !conn->compress_chained) + conn->compress_pattern = false; + + return STATUS_SUCCESS; } static void decode_sign_cap_ctxt(struct ksmbd_conn *conn, @@ -1020,6 +1242,7 @@ static __le32 deassemble_neg_contexts(struct ksmbd_conn *conn, unsigned int offset = le32_to_cpu(req->NegotiateContextOffset); unsigned int neg_ctxt_cnt = le16_to_cpu(req->NegotiateContextCount); __le32 status = STATUS_INVALID_PARAMETER; + int compress_ctxt_cnt = 0; ksmbd_debug(SMB, "decoding %d negotiate contexts\n", neg_ctxt_cnt); if (len_of_smb <= offset) { @@ -1065,11 +1288,16 @@ static __le32 deassemble_neg_contexts(struct ksmbd_conn *conn, } else if (pctx->ContextType == SMB2_COMPRESSION_CAPABILITIES) { ksmbd_debug(SMB, "deassemble SMB2_COMPRESSION_CAPABILITIES context\n"); - if (conn->compress_algorithm) + if (compress_ctxt_cnt++) { + status = STATUS_INVALID_PARAMETER; break; + } - decode_compress_ctxt(conn, - (struct smb2_compression_capabilities_context *)pctx); + status = decode_compress_ctxt(conn, + (struct smb2_compression_capabilities_context *) + pctx, ctxt_len); + if (status != STATUS_SUCCESS) + break; } else if (pctx->ContextType == SMB2_NETNAME_NEGOTIATE_CONTEXT_ID) { ksmbd_debug(SMB, "deassemble SMB2_NETNAME_NEGOTIATE_CONTEXT_ID context\n"); @@ -1460,9 +1688,11 @@ static int ntlm_authenticate(struct ksmbd_work *work, { struct ksmbd_conn *conn = work->conn; struct ksmbd_session *sess = work->sess; - struct channel *chann = NULL, *old; struct ksmbd_user *user; + char channel_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : sess->sess_key; u64 prev_id; + bool binding = conn->binding; int sz, rc; ksmbd_debug(SMB, "authenticate phase\n"); @@ -1504,7 +1734,7 @@ static int ntlm_authenticate(struct ksmbd_work *work, if (!ksmbd_compare_user(sess->user, user)) { ksmbd_free_user(user); - return -EPERM; + return -EKEYREJECTED; } ksmbd_free_user(user); } else { @@ -1521,11 +1751,13 @@ static int ntlm_authenticate(struct ksmbd_work *work, sz = conn->mechTokenLen; else sz = le16_to_cpu(req->SecurityBufferLength); - rc = ksmbd_decode_ntlmssp_auth_blob(authblob, sz, conn, sess); + rc = ksmbd_decode_ntlmssp_auth_blob(authblob, sz, conn, sess, + auth_key); if (rc) { set_user_flag(sess->user, KSMBD_USER_FLAG_BAD_PASSWORD); ksmbd_debug(SMB, "authentication failed\n"); - return -EPERM; + rc = -EPERM; + goto out; } } @@ -1560,37 +1792,30 @@ static int ntlm_authenticate(struct ksmbd_work *work, binding_session: if (conn->dialect >= SMB30_PROT_ID) { - chann = lookup_chann_list(sess, conn); - if (!chann) { - chann = kmalloc_obj(struct channel, KSMBD_DEFAULT_GFP); - if (!chann) - return -ENOMEM; - - chann->conn = conn; - down_write(&sess->chann_lock); - old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, - KSMBD_DEFAULT_GFP); - up_write(&sess->chann_lock); - if (xa_is_err(old)) { - kfree(chann); - return xa_err(old); - } - } + rc = register_session_channel(sess, conn, auth_key); + if (rc) + goto out; } if (conn->ops->generate_signingkey) { rc = conn->ops->generate_signingkey(sess, conn); if (rc) { ksmbd_debug(SMB, "SMB3 signing key generation failed\n"); - return -EINVAL; + rc = -EINVAL; + goto out; } } if (!ksmbd_conn_lookup_dialect(conn)) { pr_err("fail to verify the dialect\n"); - return -ENOENT; + rc = -ENOENT; + goto out; } - return 0; + rc = 0; +out: + if (binding) + memzero_explicit(channel_key, sizeof(channel_key)); + return rc; } #ifdef CONFIG_SMB_SERVER_KERBEROS5 @@ -1601,8 +1826,10 @@ static int krb5_authenticate(struct ksmbd_work *work, struct ksmbd_conn *conn = work->conn; struct ksmbd_session *sess = work->sess; char *in_blob, *out_blob; - struct channel *chann = NULL, *old; + char channel_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : sess->sess_key; u64 prev_sess_id; + bool binding = conn->binding; int in_len, out_len; int retval; @@ -1615,10 +1842,12 @@ static int krb5_authenticate(struct ksmbd_work *work, (le16_to_cpu(rsp->SecurityBufferOffset) + 4); retval = ksmbd_krb5_authenticate(sess, in_blob, in_len, - out_blob, &out_len); + out_blob, &out_len, auth_key); if (retval) { ksmbd_debug(SMB, "krb5 authentication failed\n"); - return -EINVAL; + if (retval != -EKEYREJECTED) + retval = -EINVAL; + goto out; } /* Check previous session */ @@ -1655,37 +1884,30 @@ static int krb5_authenticate(struct ksmbd_work *work, binding_session: if (conn->dialect >= SMB30_PROT_ID) { - chann = lookup_chann_list(sess, conn); - if (!chann) { - chann = kmalloc_obj(struct channel, KSMBD_DEFAULT_GFP); - if (!chann) - return -ENOMEM; - - chann->conn = conn; - down_write(&sess->chann_lock); - old = xa_store(&sess->ksmbd_chann_list, (long)conn, - chann, KSMBD_DEFAULT_GFP); - up_write(&sess->chann_lock); - if (xa_is_err(old)) { - kfree(chann); - return xa_err(old); - } - } + retval = register_session_channel(sess, conn, auth_key); + if (retval) + goto out; } if (conn->ops->generate_signingkey) { retval = conn->ops->generate_signingkey(sess, conn); if (retval) { ksmbd_debug(SMB, "SMB3 signing key generation failed\n"); - return -EINVAL; + retval = -EINVAL; + goto out; } } if (!ksmbd_conn_lookup_dialect(conn)) { pr_err("fail to verify the dialect\n"); - return -ENOENT; + retval = -ENOENT; + goto out; } - return 0; + retval = 0; +out: + if (binding) + memzero_explicit(channel_key, sizeof(channel_key)); + return retval; } #else static int krb5_authenticate(struct ksmbd_work *work, @@ -1791,13 +2013,36 @@ int smb2_sess_setup(struct ksmbd_work *work) } else if ((conn->dialect < SMB30_PROT_ID || server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) && (req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { - sess = NULL; + sess = ksmbd_session_lookup_slowpath(le64_to_cpu(req->hdr.SessionId)); + if (sess) { + int sign_ret; + + work->sess = sess; + if (sess->dialect >= SMB30_PROT_ID) + sign_ret = smb3_check_sign_req(work); + else + sign_ret = smb2_check_sign_req(work); + if (sess->state != SMB2_SESSION_VALID || + !(req->hdr.Flags & SMB2_FLAGS_SIGNED) || + !sign_ret) { + ksmbd_user_session_put(sess); + work->sess = NULL; + sess = NULL; + } + } rc = -EACCES; goto out_err; } else { sess = ksmbd_session_lookup(conn, le64_to_cpu(req->hdr.SessionId)); if (!sess) { + sess = ksmbd_session_lookup_slowpath(le64_to_cpu(req->hdr.SessionId)); + if (sess && !lookup_chann_list(sess, conn)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } + if (!sess) { rc = -ENOENT; goto out_err; } @@ -1907,12 +2152,19 @@ out_err: rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; else if (rc == -EFAULT) rsp->hdr.Status = STATUS_NETWORK_SESSION_EXPIRED; - else if (rc == -ENOMEM) + else if (rc == -ENOMEM || rc == -ENOSPC) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; else if (rc == -EOPNOTSUPP) rsp->hdr.Status = STATUS_NOT_SUPPORTED; + else if (rc == -EKEYREJECTED) + rsp->hdr.Status = STATUS_ACCESS_DENIED; else if (rc) rsp->hdr.Status = STATUS_LOGON_FAILURE; + if ((rsp->hdr.Status == STATUS_USER_SESSION_DELETED || + (rsp->hdr.Status == STATUS_INVALID_PARAMETER && + (req->Flags & SMB2_SESSION_REQ_FLAG_BINDING))) && + (req->hdr.Flags & SMB2_FLAGS_SIGNED)) + rsp->hdr.Flags |= SMB2_FLAGS_SIGNED; if (conn->mechToken) { kfree(conn->mechToken); @@ -1920,6 +2172,17 @@ out_err: } if (rc < 0) { + if (sess && conn->dialect == SMB311_PROT_ID && + (req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + struct preauth_session *preauth_sess; + + preauth_sess = ksmbd_preauth_session_lookup(conn, sess->id); + if (preauth_sess) { + list_del(&preauth_sess->preauth_entry); + kfree(preauth_sess); + } + } + /* * SecurityBufferOffset should be set to zero * in session setup error response. @@ -1946,8 +2209,16 @@ out_err: sess->last_active = jiffies; sess->state = SMB2_SESSION_EXPIRED; } - ksmbd_user_session_put(sess); - work->sess = NULL; + /* + * Keep the binding session reference until the response is + * signed and sent. Error responses for a signed binding + * request are signed with the existing session signing key. + */ + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING) || + work->sess != sess) { + ksmbd_user_session_put(sess); + work->sess = NULL; + } if (try_delay) { ksmbd_conn_set_need_reconnect(conn); ssleep(5); @@ -2060,6 +2331,10 @@ out_err1: rsp->Reserved = 0; /* default manual caching */ rsp->ShareFlags = SMB2_SHAREFLAG_MANUAL_CACHING; + /* Tell the client that READ requests may request compressed responses. */ + if (conn->dialect == SMB311_PROT_ID && + conn->compress_algorithm != SMB3_COMPRESS_NONE) + rsp->ShareFlags |= cpu_to_le32(SMB2_SHAREFLAG_COMPRESS_DATA); rc = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_tree_connect_rsp)); if (rc) @@ -2706,8 +2981,10 @@ struct durable_info { unsigned short int type; bool persistent; bool reconnected; + bool app_instance_id; unsigned int timeout; char *CreateGuid; + char AppInstanceId[SMB2_CREATE_GUID_SIZE]; }; static int parse_durable_handle_context(struct ksmbd_work *work, @@ -2744,9 +3021,8 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } - if (le16_to_cpu(context->DataOffset) + - le32_to_cpu(context->DataLength) < - sizeof(struct create_durable_handle_reconnect_v2)) { + if (le32_to_cpu(context->DataLength) < + sizeof(recon_v2->dcontext)) { err = -EINVAL; goto out; } @@ -2760,6 +3036,13 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } + if (dh_info->fp->durable_volatile_id != + recon_v2->dcontext.Fid.VolatileFileId) { + err = -EBADF; + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + if (memcmp(dh_info->fp->create_guid, recon_v2->dcontext.CreateGuid, SMB2_CREATE_GUID_SIZE)) { err = -EBADF; @@ -2784,9 +3067,8 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } - if (le16_to_cpu(context->DataOffset) + - le32_to_cpu(context->DataLength) < - sizeof(create_durable_reconn_t)) { + if (le32_to_cpu(context->DataLength) < + sizeof(recon->Data)) { err = -EINVAL; goto out; } @@ -2800,6 +3082,13 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } + if (dh_info->fp->durable_volatile_id != + recon->Data.Fid.VolatileFileId) { + err = -EBADF; + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + dh_info->type = dh_idx; dh_info->reconnected = true; ksmbd_debug(SMB, "reconnect Persistent-id from reconnect = %llu\n", @@ -2816,9 +3105,8 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } - if (le16_to_cpu(context->DataOffset) + - le32_to_cpu(context->DataLength) < - sizeof(struct create_durable_req_v2)) { + if (le32_to_cpu(context->DataLength) < + sizeof(durable_v2_blob->dcontext)) { err = -EINVAL; goto out; } @@ -2881,6 +3169,31 @@ out: return err; } +static int parse_app_instance_id(struct smb2_create_req *req, + struct durable_info *dh_info) +{ + struct create_context *context; + char *data; + + context = smb2_find_context_vals(req, SMB2_CREATE_APP_INSTANCE_ID, + SMB2_CREATE_GUID_SIZE); + if (IS_ERR(context)) + return PTR_ERR(context); + if (!context) + return 0; + + if (le32_to_cpu(context->DataLength) < 20) + return -EINVAL; + + data = (char *)context + le16_to_cpu(context->DataOffset); + if (data[0] != 20 || data[1]) + return -EINVAL; + + memcpy(dh_info->AppInstanceId, data + 4, SMB2_CREATE_GUID_SIZE); + dh_info->app_instance_id = true; + return 0; +} + /** * smb2_open() - handler for smb file open request * @work: smb work containing request buffer @@ -2916,7 +3229,7 @@ int smb2_open(struct ksmbd_work *work) char *stream_name = NULL; bool file_present = false, created = false, already_permitted = false; int share_ret, need_truncate = 0; - u64 time; + u64 time, alloc_size = 0; umode_t posix_mode = 0; __le32 daccess, maximal_access = 0; int iov_len = 0; @@ -3007,11 +3320,25 @@ int smb2_open(struct ksmbd_work *work) if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE && req->CreateContextsOffset) { lc = parse_lease_state(req); + if (IS_ERR(lc)) { + rc = PTR_ERR(lc); + lc = NULL; + goto err_out2; + } + if (lc && lc->version == 2 && conn->dialect < SMB30_PROT_ID) { + kfree(lc); + lc = NULL; + if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + } rc = parse_durable_handle_context(work, req, lc, &dh_info); if (rc) { ksmbd_debug(SMB, "error parsing durable handle context\n"); goto err_out2; } + rc = parse_app_instance_id(req, &dh_info); + if (rc) + goto err_out2; if (dh_info.reconnected == true) { rc = smb2_check_durable_oplock(conn, share, dh_info.fp, @@ -3038,8 +3365,22 @@ int smb2_open(struct ksmbd_work *work) goto reconnected_fp; } - } else if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) + + if (dh_info.type == DURABLE_REQ_V2 && dh_info.app_instance_id) + ksmbd_close_fd_app_instance_id(dh_info.AppInstanceId); + } else if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) { lc = parse_lease_state(req); + if (IS_ERR(lc)) { + rc = PTR_ERR(lc); + lc = NULL; + goto err_out2; + } + if (lc && lc->version == 2 && conn->dialect < SMB30_PROT_ID) { + kfree(lc); + lc = NULL; + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + } + } if (le32_to_cpu(req->ImpersonationLevel) > le32_to_cpu(IL_DELEGATE)) { pr_err("Invalid impersonationlevel : 0x%x\n", @@ -3091,7 +3432,15 @@ int smb2_open(struct ksmbd_work *work) goto err_out2; } - if (req->FileAttributes && !(req->FileAttributes & FILE_ATTRIBUTE_MASK_LE)) { + if (req->DesiredAccess == FILE_SYNCHRONIZE_LE && + req->CreateDisposition == FILE_OPEN_IF_LE && + !req->FileAttributes) { + rc = -EACCES; + goto err_out2; + } + + if (req->FileAttributes && + (req->FileAttributes & ~cpu_to_le32(SMB2_CREATE_FILE_ATTRIBUTE_MASK))) { pr_err("Invalid file attribute : 0x%x\n", le32_to_cpu(req->FileAttributes)); rc = -EINVAL; @@ -3149,6 +3498,23 @@ int smb2_open(struct ksmbd_work *work) rc = ksmbd_vfs_kern_path(work, name, LOOKUP_NO_SYMLINKS, &path, 1); + + /* + * A durable handle opened with delete-on-close is preserved across a + * disconnect so it can be reclaimed by a durable reconnect. When a new + * delete-on-close open for the same name arrives instead, the + * disconnected handle must give way: close it so its delete-on-close + * removes the file, then re-resolve so this open can create a fresh one. + */ + if (!rc && (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) && + (req->CreateDisposition == FILE_OVERWRITE_IF_LE || + req->CreateDisposition == FILE_OPEN_IF_LE) && + ksmbd_close_disconnected_durable_delete_on_close(path.dentry)) { + path_put(&path); + rc = ksmbd_vfs_kern_path(work, name, LOOKUP_NO_SYMLINKS, + &path, 1); + } + if (!rc) { file_present = true; @@ -3183,7 +3549,14 @@ int smb2_open(struct ksmbd_work *work) rc = 0; } - if (stream_name) { + /* + * An explicit ::$DATA suffix names the unnamed data stream and is + * canonicalized to a NULL stream name (base file), but the request + * still has to be validated against the data-stream type, e.g. opening + * <dir>::$DATA with FILE_DIRECTORY_FILE must fail with + * STATUS_NOT_A_DIRECTORY. + */ + if (stream_name || s_type == DATA_STREAM) { if (req->CreateOptions & FILE_DIRECTORY_FILE_LE) { if (s_type == DATA_STREAM) { rc = -EIO; @@ -3381,14 +3754,16 @@ int smb2_open(struct ksmbd_work *work) if (posix_acl_rc) ksmbd_debug(SMB, "inherit posix acl failed : %d\n", posix_acl_rc); - if (test_share_config_flag(work->tcon->share_conf, - KSMBD_SHARE_FLAG_ACL_XATTR)) { - rc = smb_inherit_dacl(conn, &path, sess->user->uid, - sess->user->gid); - } + rc = smb2_create_sd_buffer(work, req, &path); + if (rc && rc != -ENOENT) + goto err_out; - if (rc) { - rc = smb2_create_sd_buffer(work, req, &path); + if (rc == -ENOENT) { + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_ACL_XATTR)) { + rc = smb_inherit_dacl(conn, &path, sess->user->uid, + sess->user->gid); + } if (rc) { if (posix_acl_rc) ksmbd_vfs_set_init_posix_acl(idmap, @@ -3479,6 +3854,12 @@ int smb2_open(struct ksmbd_work *work) goto err_out; } + if (!stream_name && daccess & FILE_DELETE_LE && + ksmbd_has_stream_without_delete_share(fp)) { + rc = -EPERM; + goto err_out; + } + if (file_present || created) path_put(&path); @@ -3514,7 +3895,7 @@ int smb2_open(struct ksmbd_work *work) ksmbd_debug(SMB, "lease req for(%s) req oplock state 0x%x, lease state 0x%x\n", name, req_op_level, lc->req_state); - rc = find_same_lease_key(sess, fp->f_ci, lc); + rc = find_same_lease_key(conn, fp->f_ci, lc); if (rc) goto err_out1; } else if (open_flags == O_RDONLY && @@ -3530,8 +3911,10 @@ int smb2_open(struct ksmbd_work *work) goto err_out1; } - if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) + if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) { + smb_break_all_levII_oplock_for_delete(work, fp); ksmbd_fd_set_delete_on_close(fp, file_info); + } if (need_truncate) { rc = smb2_create_truncate(&fp->filp->f_path); @@ -3548,7 +3931,6 @@ int smb2_open(struct ksmbd_work *work) rc = PTR_ERR(az_req); goto err_out1; } else if (az_req) { - loff_t alloc_size; int err; if (le16_to_cpu(az_req->ccontext.DataOffset) + @@ -3597,13 +3979,19 @@ int smb2_open(struct ksmbd_work *work) fp->create_time = ksmbd_UnixTimeToNT(stat.btime); else fp->create_time = ksmbd_UnixTimeToNT(stat.ctime); + fp->change_time = ksmbd_UnixTimeToNT(stat.ctime); + fp->allocation_size = S_ISDIR(stat.mode) ? 0 : + (alloc_size ?: stat.blocks << 9); if (req->FileAttributes || fp->f_ci->m_fattr == 0) fp->f_ci->m_fattr = cpu_to_le32(smb2_get_dos_mode(&stat, le32_to_cpu(req->FileAttributes))); if (!created) smb2_update_xattrs(tcon, &path, fp); - else + + ksmbd_vfs_update_compressed_fattr(path.dentry, &fp->f_ci->m_fattr); + + if (created) smb2_new_xattrs(tcon, &path, fp); memcpy(fp->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); @@ -3619,6 +4007,10 @@ int smb2_open(struct ksmbd_work *work) if (dh_info.type == DURABLE_REQ_V2) { memcpy(fp->create_guid, dh_info.CreateGuid, SMB2_CREATE_GUID_SIZE); + if (dh_info.app_instance_id) + memcpy(fp->app_instance_id, + dh_info.AppInstanceId, + SMB2_CREATE_GUID_SIZE); if (dh_info.timeout) fp->durable_timeout = min_t(unsigned int, dh_info.timeout, @@ -3638,11 +4030,22 @@ reconnected_fp: time = ksmbd_UnixTimeToNT(stat.atime); rsp->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); + fp->open_mtime = time; rsp->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - rsp->ChangeTime = cpu_to_le64(time); - rsp->AllocationSize = S_ISDIR(stat.mode) ? 0 : - cpu_to_le64(stat.blocks << 9); + rsp->ChangeTime = cpu_to_le64(fp->change_time); + /* + * The cached allocation size hides filesystem rounding for the + * requested allocation, but it can go stale when the file grows past + * it via writes (e.g. across a durable reconnect). Refresh it once the + * file exceeds the cached value, rounding the end of file up to the + * volume allocation unit (the filesystem block size, matching the + * SectorsPerAllocationUnit/BytesPerSector ksmbd advertises) rather than + * using the raw on-disk block count, which can include filesystem + * preallocation and metadata rounding. + */ + if (!S_ISDIR(stat.mode) && stat.size > fp->allocation_size) + fp->allocation_size = round_up(stat.size, stat.blksize); + rsp->AllocationSize = cpu_to_le64(fp->allocation_size); rsp->EndofFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); rsp->FileAttributes = fp->f_ci->m_fattr; @@ -3767,8 +4170,10 @@ err_out1: err_out2: if (!rc) { - ksmbd_update_fstate(&work->sess->file_table, fp, FP_INITED); - rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len); + rc = ksmbd_update_fstate(&work->sess->file_table, fp, + FP_INITED); + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len); } if (rc) { if (rc == -EINVAL) @@ -3802,8 +4207,19 @@ err_out2: ksmbd_debug(SMB, "Error response: %x\n", rsp->hdr.Status); } - if (dh_info.reconnected) - ksmbd_put_durable_fd(dh_info.fp); + if (dh_info.reconnected) { + /* + * If reconnect succeeded, fp was republished in the + * session file table. On a later error, ksmbd_fd_put() + * above drops the session reference; drop the durable + * lookup reference through the same session-aware path so + * final close removes the volatile id before freeing fp. + */ + if (rc && fp == dh_info.fp) + ksmbd_fd_put(work, dh_info.fp); + else + ksmbd_put_durable_fd(dh_info.fp); + } kfree(name); kfree(lc); @@ -4402,11 +4818,28 @@ int smb2_query_dir(struct ksmbd_work *work) unsigned char srch_flag; int buffer_sz; struct smb2_query_dir_private query_dir_private = {NULL, }; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; ksmbd_debug(SMB, "Received smb2 query directory request\n"); WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + if (ksmbd_override_fsids(work)) { rsp->hdr.Status = STATUS_NO_MEMORY; smb2_set_err_rsp(work); @@ -4419,7 +4852,7 @@ int smb2_query_dir(struct ksmbd_work *work) goto err_out2; } - dir_fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + dir_fp = ksmbd_lookup_fd_slow(work, id, pid); if (!dir_fp) { rc = -EBADF; goto err_out2; @@ -4452,6 +4885,8 @@ int smb2_query_dir(struct ksmbd_work *work) ksmbd_debug(SMB, "Search pattern is %s\n", srch_ptr); } + mutex_lock(&dir_fp->readdir_lock); + if (srch_flag & SMB2_REOPEN || srch_flag & SMB2_RESTART_SCANS) { ksmbd_debug(SMB, "Restart directory scan\n"); generic_file_llseek(dir_fp->filp, 0, SEEK_SET); @@ -4556,6 +4991,7 @@ no_buf_len: goto err_out; } + mutex_unlock(&dir_fp->readdir_lock); kfree(srch_ptr); ksmbd_fd_put(work, dir_fp); ksmbd_revert_fsids(work); @@ -4563,6 +4999,7 @@ no_buf_len: err_out: pr_err("error while processing smb2 query dir rc = %d\n", rc); + mutex_unlock(&dir_fp->readdir_lock); kfree(srch_ptr); err_out2: @@ -4892,8 +5329,7 @@ static int get_file_basic_info(struct smb2_query_info_rsp *rsp, basic_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); basic_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - basic_info->ChangeTime = cpu_to_le64(time); + basic_info->ChangeTime = cpu_to_le64(fp->change_time); basic_info->Attributes = fp->f_ci->m_fattr; basic_info->Pad = 0; rsp->OutputBufferLength = @@ -4918,7 +5354,7 @@ static int get_file_standard_info(struct smb2_query_info_rsp *rsp, delete_pending = ksmbd_inode_pending_delete(fp); if (ksmbd_stream_fd(fp) == false) { - sinfo->AllocationSize = cpu_to_le64(stat.blocks << 9); + sinfo->AllocationSize = cpu_to_le64(fp->allocation_size); sinfo->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); } else { sinfo->AllocationSize = cpu_to_le64(fp->stream.size); @@ -4995,13 +5431,11 @@ static int get_file_all_info(struct ksmbd_work *work, file_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); file_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - file_info->ChangeTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); file_info->Attributes = fp->f_ci->m_fattr; file_info->Pad1 = 0; if (ksmbd_stream_fd(fp) == false) { - file_info->AllocationSize = - cpu_to_le64(stat.blocks << 9); + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); file_info->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); } else { file_info->AllocationSize = cpu_to_le64(fp->stream.size); @@ -5205,11 +5639,10 @@ static int get_file_network_open_info(struct smb2_query_info_rsp *rsp, file_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); file_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - file_info->ChangeTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); file_info->Attributes = fp->f_ci->m_fattr; if (ksmbd_stream_fd(fp) == false) { - file_info->AllocationSize = cpu_to_le64(stat.blocks << 9); + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); file_info->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); } else { file_info->AllocationSize = cpu_to_le64(fp->stream.size); @@ -5262,6 +5695,7 @@ static int get_file_compression_info(struct smb2_query_info_rsp *rsp, { struct smb2_file_comp_info *file_info; struct kstat stat; + u16 fmt; int ret; ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, @@ -5269,9 +5703,13 @@ static int get_file_compression_info(struct smb2_query_info_rsp *rsp, if (ret) return ret; + ret = ksmbd_vfs_get_compression(fp, &fmt); + if (ret) + return ret; + file_info = (struct smb2_file_comp_info *)rsp->Buffer; - file_info->CompressedFileSize = cpu_to_le64(stat.blocks << 9); - file_info->CompressionFormat = COMPRESSION_FORMAT_NONE; + file_info->CompressedFileSize = cpu_to_le64(min_t(u64, stat.blocks << 9, stat.size)); + file_info->CompressionFormat = cpu_to_le16(fmt); file_info->CompressionUnitShift = 0; file_info->ChunkShift = 0; file_info->ClusterShift = 0; @@ -5315,6 +5753,12 @@ static int find_file_posix_info(struct smb2_query_info_rsp *rsp, int out_buf_len = sizeof(struct smb311_posix_qinfo) + 32; int ret; + if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { + pr_err("no right to read the attributes : 0x%x\n", + fp->daccess); + return -EACCES; + } + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); if (ret) @@ -5326,13 +5770,12 @@ static int find_file_posix_info(struct smb2_query_info_rsp *rsp, file_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); file_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - file_info->ChangeTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); file_info->DosAttributes = fp->f_ci->m_fattr; file_info->Inode = cpu_to_le64(stat.ino); if (ksmbd_stream_fd(fp) == false) { file_info->EndOfFile = cpu_to_le64(stat.size); - file_info->AllocationSize = cpu_to_le64(stat.blocks << 9); + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); } else { file_info->EndOfFile = cpu_to_le64(fp->stream.size); file_info->AllocationSize = cpu_to_le64(fp->stream.size); @@ -5548,16 +5991,33 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, case FS_ATTRIBUTE_INFORMATION: { FILE_SYSTEM_ATTRIBUTE_INFO *info; + struct file_kattr fa = {}; size_t sz; + u32 attrs; + int err; info = (FILE_SYSTEM_ATTRIBUTE_INFO *)rsp->Buffer; - info->Attributes = cpu_to_le32(FILE_SUPPORTS_OBJECT_IDS | - FILE_PERSISTENT_ACLS | - FILE_UNICODE_ON_DISK | - FILE_CASE_PRESERVED_NAMES | - FILE_CASE_SENSITIVE_SEARCH | - FILE_SUPPORTS_BLOCK_REFCOUNTING); + attrs = FILE_SUPPORTS_OBJECT_IDS | + FILE_PERSISTENT_ACLS | + FILE_UNICODE_ON_DISK | + FILE_SUPPORTS_BLOCK_REFCOUNTING; + + err = vfs_fileattr_get(path.dentry, &fa); + /* + * -EINVAL, -EOPNOTSUPP: ntfs-3g and other FUSE + * filesystems that lack FS_IOC_FSGETXATTR support. + */ + if (err && err != -ENOIOCTLCMD && err != -ENOTTY && + err != -EINVAL && err != -EOPNOTSUPP) { + path_put(&path); + return err; + } + if (!(fa.fsx_xflags & FS_XFLAG_CASEFOLD)) + attrs |= FILE_CASE_SENSITIVE_SEARCH; + if (!(fa.fsx_xflags & FS_XFLAG_CASENONPRESERVING)) + attrs |= FILE_CASE_PRESERVED_NAMES; + info->Attributes = cpu_to_le32(attrs); info->Attributes |= cpu_to_le32(server_conf.share_fake_fscaps); if (test_share_config_flag(work->tcon->share_conf, @@ -5854,6 +6314,9 @@ int smb2_query_info(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (ksmbd_override_fsids(work)) { rc = -ENOMEM; goto err_out; @@ -5958,6 +6421,9 @@ int smb2_close(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { ksmbd_debug(SMB, "IPC pipe close request\n"); @@ -6024,17 +6490,18 @@ int smb2_close(struct ksmbd_work *work) } rsp->Flags = SMB2_CLOSE_FLAG_POSTQUERY_ATTRIB; - rsp->AllocationSize = S_ISDIR(stat.mode) ? 0 : - cpu_to_le64(stat.blocks << 9); + rsp->AllocationSize = cpu_to_le64(fp->allocation_size); rsp->EndOfFile = cpu_to_le64(stat.size); rsp->Attributes = fp->f_ci->m_fattr; rsp->CreationTime = cpu_to_le64(fp->create_time); time = ksmbd_UnixTimeToNT(stat.atime); rsp->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); + if (time > fp->open_mtime && + time - fp->open_mtime < KSMBD_WRITE_TIME_RESOLUTION) + time = fp->open_mtime; rsp->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - rsp->ChangeTime = cpu_to_le64(time); + rsp->ChangeTime = cpu_to_le64(fp->change_time); ksmbd_fd_put(work, fp); } else { rsp->Flags = 0; @@ -6130,9 +6597,8 @@ static int smb2_rename(struct ksmbd_work *work, pr_err("failed to store stream name in xattr: %d\n", rc); rc = -EINVAL; - goto out; } - + kfree(xattr_stream_name); goto out; } @@ -6247,9 +6713,11 @@ static int set_file_basic_info(struct ksmbd_file *fp, attrs.ia_valid |= (ATTR_ATIME | ATTR_ATIME_SET); } - if (file_info->ChangeTime) + if (file_info->ChangeTime) { + fp->change_time = le64_to_cpu(file_info->ChangeTime); inode_set_ctime_to_ts(inode, ksmbd_NTtimeToUnix(file_info->ChangeTime)); + } if (file_info->LastWriteTime) { attrs.ia_mtime = ksmbd_NTtimeToUnix(file_info->LastWriteTime); @@ -6312,6 +6780,7 @@ static int set_file_allocation_info(struct ksmbd_work *work, */ loff_t alloc_blks; + u64 alloc_size; struct inode *inode; struct kstat stat; int rc; @@ -6327,7 +6796,19 @@ static int set_file_allocation_info(struct ksmbd_work *work, if (rc) return rc; - alloc_blks = (le64_to_cpu(file_alloc_info->AllocationSize) + 511) >> 9; + /* + * AllocationSize is fully client-controlled (the caller only + * validates the fixed 8-byte buffer length). Reject values that + * would overflow the "round up to 512-byte blocks" conversion + * below instead of silently wrapping it to a tiny block count, + * which would truncate the file to a size the client never + * asked for. + */ + alloc_size = le64_to_cpu(file_alloc_info->AllocationSize); + if (alloc_size > MAX_LFS_FILESIZE - 511) + return -EINVAL; + + alloc_blks = (alloc_size + 511) >> 9; inode = file_inode(fp->filp); if (alloc_blks > stat.blocks) { @@ -6357,6 +6838,8 @@ static int set_file_allocation_info(struct ksmbd_work *work, if (size < alloc_blks * 512) i_size_write(inode, size); } + + fp->allocation_size = le64_to_cpu(file_alloc_info->AllocationSize); return 0; } @@ -6413,7 +6896,8 @@ static int set_rename_info(struct ksmbd_work *work, struct ksmbd_file *fp, return smb2_rename(work, fp, rename_info, work->conn->local_nls); } -static int set_file_disposition_info(struct ksmbd_file *fp, +static int set_file_disposition_info(struct ksmbd_work *work, + struct ksmbd_file *fp, struct smb2_file_disposition_info *file_info) { struct inode *inode; @@ -6425,9 +6909,13 @@ static int set_file_disposition_info(struct ksmbd_file *fp, inode = file_inode(fp->filp); if (file_info->DeletePending) { + if (ksmbd_has_stream_without_delete_share(fp)) + return -ESHARE; + if (S_ISDIR(inode->i_mode) && ksmbd_vfs_empty_dir(fp) == -ENOTEMPTY) return -EBUSY; + smb_break_all_levII_oplock_for_delete(work, fp); ksmbd_set_inode_pending_delete(fp); } else { ksmbd_clear_inode_pending_delete(fp); @@ -6536,11 +7024,18 @@ static int smb2_set_info_file(struct ksmbd_work *work, struct ksmbd_file *fp, } case FILE_LINK_INFORMATION: { + struct smb2_file_link_info *file_info; + if (buf_len < sizeof(struct smb2_file_link_info)) return -EMSGSIZE; - return smb2_create_link(work, work->tcon->share_conf, - (struct smb2_file_link_info *)buffer, + file_info = (struct smb2_file_link_info *)buffer; + if (file_info->ReplaceIfExists && !(fp->daccess & FILE_DELETE_LE)) { + pr_err("no right to delete : 0x%x\n", fp->daccess); + return -EACCES; + } + + return smb2_create_link(work, work->tcon->share_conf, file_info, buf_len, fp->filp, work->conn->local_nls); } @@ -6549,7 +7044,7 @@ static int smb2_set_info_file(struct ksmbd_work *work, struct ksmbd_file *fp, if (buf_len < sizeof(struct smb2_file_disposition_info)) return -EMSGSIZE; - return set_file_disposition_info(fp, + return set_file_disposition_info(work, fp, (struct smb2_file_disposition_info *)buffer); } case FILE_FULL_EA_INFORMATION: @@ -6593,6 +7088,9 @@ static int smb2_set_info_sec(struct ksmbd_file *fp, int addition_info, fp->saccess |= FILE_SHARE_DELETE_LE; + if (!(fp->daccess & (FILE_WRITE_DAC_LE | FILE_WRITE_OWNER_LE))) + return -EACCES; + return set_info_sec(fp->conn, fp->tcon, &fp->filp->f_path, pntsd, buf_len, false, true); } @@ -6605,6 +7103,7 @@ static int smb2_set_info_sec(struct ksmbd_file *fp, int addition_info, */ int smb2_set_info(struct ksmbd_work *work) { + const struct cred *saved_cred; struct smb2_set_info_req *req; struct smb2_set_info_rsp *rsp; struct ksmbd_file *fp = NULL; @@ -6616,6 +7115,8 @@ int smb2_set_info(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); @@ -6646,6 +7147,7 @@ int smb2_set_info(struct ksmbd_work *work) goto err_out; } + saved_cred = override_creds(fp->filp->f_cred); switch (req->InfoType) { case SMB2_O_INFO_FILE: ksmbd_debug(SMB, "GOT SMB2_O_INFO_FILE\n"); @@ -6653,19 +7155,15 @@ int smb2_set_info(struct ksmbd_work *work) break; case SMB2_O_INFO_SECURITY: ksmbd_debug(SMB, "GOT SMB2_O_INFO_SECURITY\n"); - if (ksmbd_override_fsids(work)) { - rc = -ENOMEM; - goto err_out; - } rc = smb2_set_info_sec(fp, le32_to_cpu(req->AdditionalInformation), (char *)req + le16_to_cpu(req->BufferOffset), le32_to_cpu(req->BufferLength)); - ksmbd_revert_fsids(work); break; default: rc = -EOPNOTSUPP; } + revert_creds(saved_cred); if (rc < 0) goto err_out; @@ -6685,6 +7183,8 @@ err_out: rsp->hdr.Status = STATUS_INVALID_PARAMETER; else if (rc == -EMSGSIZE) rsp->hdr.Status = STATUS_INFO_LENGTH_MISMATCH; + else if (rc == -ENOSPC || rc == -EFBIG) + rsp->hdr.Status = STATUS_DISK_FULL; else if (rc == -ESHARE) rsp->hdr.Status = STATUS_SHARING_VIOLATION; else if (rc == -ENOENT) @@ -6834,7 +7334,7 @@ int smb2_read(struct ksmbd_work *work) size_t length, mincount; ssize_t nbytes = 0, remain_bytes = 0; int err = 0; - bool is_rdma_channel = false; + bool is_rdma_channel = false, async_interim = false; unsigned int max_read_size = conn->vals->max_read_size; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; void *aux_payload_buf; @@ -6850,6 +7350,8 @@ int smb2_read(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); @@ -6904,6 +7406,14 @@ int smb2_read(struct ksmbd_work *work) goto out; } + if (work->next_smb2_rcv_hdr_off && !req->hdr.NextCommand) { + err = setup_async_work(work, NULL, NULL); + if (err) + goto out; + smb2_send_interim_resp(work, STATUS_PENDING); + async_interim = true; + } + offset = le64_to_cpu(req->Offset); if (offset < 0) { err = -EINVAL; @@ -6922,7 +7432,7 @@ int smb2_read(struct ksmbd_work *work) ksmbd_debug(SMB, "filename %pD, offset %lld, len %zu\n", fp->filp, offset, length); - aux_payload_buf = kvzalloc(ALIGN(length, 8), KSMBD_DEFAULT_GFP); + aux_payload_buf = kvmalloc(ALIGN(length, 8), KSMBD_DEFAULT_GFP); if (!aux_payload_buf) { err = -ENOMEM; goto out; @@ -6935,10 +7445,21 @@ int smb2_read(struct ksmbd_work *work) goto out; } + /* + * ksmbd_vfs_read() fills only nbytes; the [nbytes, ALIGN(nbytes, 8)) + * tail of the un-zeroed buffer is transmitted as compound-response + * alignment padding, leaking uninitialized kernel memory to the + * client. Zero just that tail. + */ + if (nbytes & 7) + memset(aux_payload_buf + nbytes, 0, ALIGN(nbytes, 8) - nbytes); + if ((nbytes == 0 && length != 0) || nbytes < mincount) { kvfree(aux_payload_buf); rsp->hdr.Status = STATUS_END_OF_FILE; smb2_set_err_rsp(work); + if (async_interim) + release_async_work(work); ksmbd_fd_put(work, fp); return -ENODATA; } @@ -6973,10 +7494,23 @@ int smb2_read(struct ksmbd_work *work) kvfree(aux_payload_buf); goto out; } + if (async_interim) + release_async_work(work); + /* + * RDMA responses are transferred through channel buffers and encrypted + * responses use the encryption transform, so only normal SMB transport + * responses are candidates for compression. + */ + if (!is_rdma_channel && nbytes && + (req->Flags & SMB2_READFLAG_REQUEST_COMPRESSED) && + conn->compress_algorithm != SMB3_COMPRESS_NONE) + work->compress_response = true; ksmbd_fd_put(work, fp); return 0; out: + if (async_interim) + release_async_work(work); if (err) { if (err == -EISDIR) rsp->hdr.Status = STATUS_INVALID_DEVICE_REQUEST; @@ -7112,13 +7646,31 @@ int smb2_write(struct ksmbd_work *work) ssize_t nbytes; char *data_buf; bool writethrough = false, is_rdma_channel = false; + bool async_interim = false; int err = 0; unsigned int max_write_size = work->conn->vals->max_write_size; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; ksmbd_debug(SMB, "Received smb2 write request\n"); WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { ksmbd_debug(SMB, "IPC pipe write request\n"); return smb2_write_pipe(work); @@ -7163,7 +7715,7 @@ int smb2_write(struct ksmbd_work *work) goto out; } - fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + fp = ksmbd_lookup_fd_slow(work, id, pid); if (!fp) { err = -ENOENT; goto out; @@ -7175,6 +7727,14 @@ int smb2_write(struct ksmbd_work *work) goto out; } + if (work->next_smb2_rcv_hdr_off && !req->hdr.NextCommand) { + err = setup_async_work(work, NULL, NULL); + if (err) + goto out; + smb2_send_interim_resp(work, STATUS_PENDING); + async_interim = true; + } + if (length > max_write_size) { ksmbd_debug(SMB, "limiting write size to max size(%u)\n", max_write_size); @@ -7223,10 +7783,15 @@ int smb2_write(struct ksmbd_work *work) err = ksmbd_iov_pin_rsp(work, rsp, offsetof(struct smb2_write_rsp, Buffer)); if (err) goto out; + if (async_interim) + release_async_work(work); ksmbd_fd_put(work, fp); return 0; out: + if (async_interim) + release_async_work(work); + if (err == -EAGAIN) rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; else if (err == -ENOSPC || err == -EFBIG) @@ -7257,13 +7822,30 @@ int smb2_flush(struct ksmbd_work *work) { struct smb2_flush_req *req; struct smb2_flush_rsp *rsp; + u64 id = KSMBD_NO_FID, pid = KSMBD_NO_FID; int err; WORK_BUFFERS(work, req, rsp); ksmbd_debug(SMB, "Received smb2 flush request(fid : %llu)\n", req->VolatileFileId); - err = ksmbd_vfs_fsync(work, req->VolatileFileId, req->PersistentFileId); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + err = ksmbd_vfs_fsync(work, id, pid); if (err) goto out; @@ -7295,7 +7877,8 @@ int smb2_cancel(struct ksmbd_work *work) hdr = ksmbd_resp_buf_next(work); ksmbd_debug(SMB, "smb2 cancel called on mid %llu, async flags 0x%x\n", - hdr->MessageId, hdr->Flags); + le64_to_cpu(hdr->MessageId), + le32_to_cpu(hdr->Flags)); if (hdr->Flags & SMB2_FLAGS_ASYNC_COMMAND) { command_list = &conn->async_requests; @@ -7309,6 +7892,17 @@ int smb2_cancel(struct ksmbd_work *work) le64_to_cpu(hdr->Id.AsyncId)) continue; + /* + * Only an ACTIVE deferred work may have its cancel_fn + * fired. A CANCELLED or CLOSED work already took the + * smb2_lock() non-ACTIVE early-exit that frees the + * file_lock and skips release_async_work(), so it is + * still on conn->async_requests with a live cancel_fn + * pointing at the freed file_lock. + */ + if (iter->state != KSMBD_WORK_ACTIVE) + break; + ksmbd_debug(SMB, "smb2 with AsyncId %llu cancelled command = 0x%x\n", le64_to_cpu(hdr->Id.AsyncId), @@ -7470,11 +8064,29 @@ int smb2_lock(struct ksmbd_work *work) LIST_HEAD(lock_list); LIST_HEAD(rollback_list); int prior_lock = 0, bkt; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; WORK_BUFFERS(work, req, rsp); ksmbd_debug(SMB, "Received smb2 lock request\n"); - fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); if (!fp) { ksmbd_debug(SMB, "Invalid file id for lock : %llu\n", req->VolatileFileId); err = -ENOENT; @@ -7599,9 +8211,11 @@ int smb2_lock(struct ksmbd_work *work) nolock = 0; list_del(&cmp_lock->flist); list_del(&cmp_lock->clist); + cmp_lock->conn = NULL; spin_unlock(&conn->llist_lock); up_read(&conn_list_lock); + ksmbd_conn_put(conn); locks_free_lock(cmp_lock->fl); kfree(cmp_lock); goto out_check_cl; @@ -7713,31 +8327,30 @@ skip: list_del(&work->fp_entry); spin_unlock(&fp->f_lock); - if (work->state != KSMBD_WORK_ACTIVE) { - list_del(&smb_lock->llist); - locks_free_lock(flock); + list_del(&smb_lock->llist); + release_async_work(work); - if (work->state == KSMBD_WORK_CANCELLED) { - rsp->hdr.Status = - STATUS_CANCELLED; - kfree(smb_lock); - smb2_send_interim_resp(work, - STATUS_CANCELLED); - work->send_no_response = 1; - goto out; - } + if (work->state == KSMBD_WORK_ACTIVE) + goto retry; + + locks_free_lock(flock); - rsp->hdr.Status = - STATUS_RANGE_NOT_LOCKED; + if (work->state == KSMBD_WORK_CANCELLED) { + rsp->hdr.Status = STATUS_CANCELLED; kfree(smb_lock); - goto out2; + smb2_send_interim_resp(work, + STATUS_CANCELLED); + work->send_no_response = 1; + goto out; } - list_del(&smb_lock->llist); - release_async_work(work); - goto retry; + rsp->hdr.Status = + STATUS_RANGE_NOT_LOCKED; + kfree(smb_lock); + goto out2; } else if (!rc) { list_add(&smb_lock->llist, &rollback_list); + smb_lock->conn = ksmbd_conn_get(work->conn); spin_lock(&work->conn->llist_lock); list_add_tail(&smb_lock->clist, &work->conn->lock_list); @@ -7792,11 +8405,14 @@ out: } list_del(&smb_lock->llist); - spin_lock(&work->conn->llist_lock); + conn = smb_lock->conn; + spin_lock(&conn->llist_lock); if (!list_empty(&smb_lock->flist)) list_del(&smb_lock->flist); list_del(&smb_lock->clist); - spin_unlock(&work->conn->llist_lock); + smb_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + ksmbd_conn_put(conn); locks_free_lock(smb_lock->fl); if (rlock) @@ -7892,9 +8508,9 @@ static int fsctl_copychunk(struct ksmbd_work *work, /* * FILE_READ_DATA should only be included in - * the FSCTL_COPYCHUNK case + * the FSCTL_SRV_COPYCHUNK case */ - if (cnt_code == FSCTL_COPYCHUNK && + if (cnt_code == FSCTL_SRV_COPYCHUNK && !(dst_fp->daccess & (FILE_READ_DATA_LE | FILE_GENERIC_READ_LE))) { rsp->hdr.Status = STATUS_ACCESS_DENIED; goto out; @@ -8189,9 +8805,20 @@ static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id, int ret = 0; __le32 old_fattr; + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, "User does not have write permission\n"); + return -EACCES; + } + fp = ksmbd_lookup_fd_fast(work, id); if (!fp) return -ENOENT; + + if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_WRITE_ATTRIBUTES_LE))) { + ret = -EACCES; + goto out; + } + idmap = file_mnt_idmap(fp->filp); old_fattr = fp->f_ci->m_fattr; @@ -8203,6 +8830,7 @@ static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id, if (fp->f_ci->m_fattr != old_fattr && test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_STORE_DOS_ATTRS)) { + const struct cred *saved_cred; struct xattr_dos_attrib da; ret = ksmbd_vfs_get_dos_attrib_xattr(idmap, @@ -8211,9 +8839,11 @@ static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id, goto out; da.attr = le32_to_cpu(fp->f_ci->m_fattr); + saved_cred = override_creds(fp->filp->f_cred); ret = ksmbd_vfs_set_dos_attrib_xattr(idmap, &fp->filp->f_path, &da, true); + revert_creds(saved_cred); if (ret) fp->f_ci->m_fattr = old_fattr; } @@ -8262,6 +8892,8 @@ int smb2_ioctl(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); @@ -8300,9 +8932,74 @@ int smb2_ioctl(struct ksmbd_work *work) ret = -EOPNOTSUPP; rsp->hdr.Status = STATUS_FS_DRIVER_REQUIRED; goto out2; + case FSCTL_GET_COMPRESSION: { + struct compress_ioctl *cmpr_rsp; + struct ksmbd_file *fp; + u16 fmt; + + if (out_buf_len < sizeof(struct compress_ioctl)) { + ret = -EINVAL; + goto out; + } + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -ENOENT; + goto out; + } + + ret = ksmbd_vfs_get_compression(fp, &fmt); + ksmbd_fd_put(work, fp); + if (ret < 0) + goto out; + + cmpr_rsp = (struct compress_ioctl *)&rsp->Buffer[0]; + cmpr_rsp->CompressionState = cpu_to_le16(fmt); + nbytes = sizeof(struct compress_ioctl); + rsp->PersistentFileId = req->PersistentFileId; + rsp->VolatileFileId = req->VolatileFileId; + break; + } + case FSCTL_SET_COMPRESSION: { + struct compress_ioctl *cmpr_req; + struct ksmbd_file *fp; + + if (in_buf_len < sizeof(struct compress_ioctl)) { + ret = -EINVAL; + goto out; + } + + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, "User does not have write permission\n"); + ret = -EACCES; + goto out; + } + + cmpr_req = (struct compress_ioctl *)buffer; + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -ENOENT; + goto out; + } + + ret = ksmbd_vfs_set_compression(work, fp, le16_to_cpu(cmpr_req->CompressionState)); + ksmbd_fd_put(work, fp); + if (ret) + goto out; + break; + } case FSCTL_CREATE_OR_GET_OBJECT_ID: { struct file_object_buf_type1_ioctl_rsp *obj_buf; + struct ksmbd_file *fp; + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -EBADF; + rsp->hdr.Status = STATUS_FILE_CLOSED; + goto out2; + } + ksmbd_fd_put(work, fp); nbytes = sizeof(struct file_object_buf_type1_ioctl_rsp); obj_buf = (struct file_object_buf_type1_ioctl_rsp *) @@ -8357,7 +9054,7 @@ int smb2_ioctl(struct ksmbd_work *work) goto out; nbytes = ret; break; - case FSCTL_REQUEST_RESUME_KEY: + case FSCTL_SRV_REQUEST_RESUME_KEY: if (out_buf_len < sizeof(struct resume_key_ioctl_rsp)) { ret = -EINVAL; goto out; @@ -8371,8 +9068,8 @@ int smb2_ioctl(struct ksmbd_work *work) rsp->VolatileFileId = req->VolatileFileId; nbytes = sizeof(struct resume_key_ioctl_rsp); break; - case FSCTL_COPYCHUNK: - case FSCTL_COPYCHUNK_WRITE: + case FSCTL_SRV_COPYCHUNK: + case FSCTL_SRV_COPYCHUNK_WRITE: if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { ksmbd_debug(SMB, "User does not have write permission\n"); @@ -8447,6 +9144,12 @@ int smb2_ioctl(struct ksmbd_work *work) goto out; } + if (!(fp->daccess & FILE_WRITE_DATA_LE)) { + ksmbd_fd_put(work, fp); + ret = -EACCES; + goto out; + } + ret = ksmbd_vfs_zero_data(work, fp, off, len); ksmbd_fd_put(work, fp); if (ret < 0) @@ -8522,6 +9225,21 @@ int smb2_ioctl(struct ksmbd_work *work) goto dup_ext_out; } + if (!test_tree_conn_flag(work->tcon, + KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ret = -EACCES; + goto dup_ext_out; + } + + if (!(fp_out->daccess & FILE_WRITE_DATA_LE)) { + ret = -EACCES; + goto dup_ext_out; + } + if (!(fp_in->daccess & FILE_READ_DATA_LE)) { + ret = -EACCES; + goto dup_ext_out; + } + src_off = le64_to_cpu(dup_ext->SourceFileOffset); dst_off = le64_to_cpu(dup_ext->TargetFileOffset); length = le64_to_cpu(dup_ext->ByteCount); @@ -8607,11 +9325,10 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) struct smb2_oplock_break *rsp; struct ksmbd_file *fp; struct oplock_info *opinfo = NULL; - __le32 err = 0; - int ret = 0; + __le32 status = STATUS_SUCCESS; + int ret; u64 volatile_id, persistent_id; char req_oplevel = 0, rsp_oplevel = 0; - unsigned int oplock_change_type; WORK_BUFFERS(work, req, rsp); @@ -8637,71 +9354,58 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) return; } - if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) { - rsp->hdr.Status = STATUS_INVALID_OPLOCK_PROTOCOL; + if (opinfo->op_state != OPLOCK_ACK_WAIT) { + ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", + opinfo->op_state); + if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) + status = STATUS_INVALID_OPLOCK_PROTOCOL; + else + status = STATUS_INVALID_DEVICE_STATE; goto err_out; } - if (opinfo->op_state == OPLOCK_STATE_NONE) { - ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", opinfo->op_state); - rsp->hdr.Status = STATUS_UNSUCCESSFUL; + if (req_oplevel == SMB2_OPLOCK_LEVEL_LEASE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_PARAMETER; goto err_out; } - if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - (req_oplevel != SMB2_OPLOCK_LEVEL_II && - req_oplevel != SMB2_OPLOCK_LEVEL_NONE)) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - oplock_change_type = OPLOCK_WRITE_TO_NONE; - } else if (opinfo->level == SMB2_OPLOCK_LEVEL_II && - req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - oplock_change_type = OPLOCK_READ_TO_NONE; - } else if (req_oplevel == SMB2_OPLOCK_LEVEL_II || - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - err = STATUS_INVALID_DEVICE_STATE; - if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - req_oplevel == SMB2_OPLOCK_LEVEL_II) { - oplock_change_type = OPLOCK_WRITE_TO_READ; - } else if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - oplock_change_type = OPLOCK_WRITE_TO_NONE; - } else if (opinfo->level == SMB2_OPLOCK_LEVEL_II && - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - oplock_change_type = OPLOCK_READ_TO_NONE; - } else { - oplock_change_type = 0; - } - } else { - oplock_change_type = 0; + if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) { + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; } - switch (oplock_change_type) { - case OPLOCK_WRITE_TO_READ: - ret = opinfo_write_to_read(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_II; - break; - case OPLOCK_WRITE_TO_NONE: - ret = opinfo_write_to_none(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; - break; - case OPLOCK_READ_TO_NONE: - ret = opinfo_read_to_none(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; - break; - default: - pr_err("unknown oplock change 0x%x -> 0x%x\n", - opinfo->level, rsp_oplevel); + if (opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; } - if (ret < 0) { - rsp->hdr.Status = err; + if (opinfo->level == SMB2_OPLOCK_LEVEL_BATCH && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE && + req_oplevel != SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; goto err_out; } + if (opinfo->level == SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; + } + + if (req_oplevel == SMB2_OPLOCK_LEVEL_EXCLUSIVE) + rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; + else + rsp_oplevel = req_oplevel; + + opinfo->level = rsp_oplevel; + rsp->StructureSize = cpu_to_le16(24); rsp->OplockLevel = rsp_oplevel; rsp->Reserved = 0; @@ -8709,27 +9413,33 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) rsp->VolatileFid = volatile_id; rsp->PersistentFid = persistent_id; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_oplock_break)); - if (ret) { + if (ret) + ksmbd_debug(SMB, "failed to pin oplock break response: %d\n", + ret); + goto out; + err_out: - smb2_set_err_rsp(work); - } + rsp->hdr.Status = status; + smb2_set_err_rsp(work); +out: opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); opinfo_put(opinfo); ksmbd_fd_put(work, fp); } -static int check_lease_state(struct lease *lease, __le32 req_state) +static bool smb2_lease_state_valid(__le32 state) { - if ((lease->new_state == - (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE)) && - !(req_state & SMB2_LEASE_WRITE_CACHING_LE)) { - lease->new_state = req_state; - return 0; - } + return !(state & ~(SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE)); +} - if (lease->new_state == req_state) +static int check_lease_state(struct lease *lease, __le32 req_state) +{ + if (smb2_lease_state_valid(req_state) && + !(req_state & ~lease->new_state)) return 0; return 1; @@ -8747,9 +9457,7 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) struct smb2_lease_ack *req; struct smb2_lease_ack *rsp; struct oplock_info *opinfo; - __le32 err = 0; int ret = 0; - unsigned int lease_change_type; __le32 lease_state; struct lease *lease; @@ -8773,80 +9481,23 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) goto err_out; } - if (check_lease_state(lease, req->LeaseState)) { - rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; - ksmbd_debug(OPLOCK, - "req lease state: 0x%x, expected state: 0x%x\n", - req->LeaseState, lease->new_state); - goto err_out; - } - if (!atomic_read(&opinfo->breaking_cnt)) { rsp->hdr.Status = STATUS_UNSUCCESSFUL; goto err_out; } - /* check for bad lease state */ - if (req->LeaseState & - (~(SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE))) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_NONE; - else - lease_change_type = OPLOCK_READ_TO_NONE; - ksmbd_debug(OPLOCK, "handle bad lease state 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } else if (lease->state == SMB2_LEASE_READ_CACHING_LE && - req->LeaseState != SMB2_LEASE_NONE_LE) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - lease_change_type = OPLOCK_READ_TO_NONE; - ksmbd_debug(OPLOCK, "handle bad lease state 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } else { - /* valid lease state changes */ - err = STATUS_INVALID_DEVICE_STATE; - if (req->LeaseState == SMB2_LEASE_NONE_LE) { - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_NONE; - else - lease_change_type = OPLOCK_READ_TO_NONE; - } else if (req->LeaseState & SMB2_LEASE_READ_CACHING_LE) { - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_READ; - else - lease_change_type = OPLOCK_READ_HANDLE_TO_READ; - } else { - lease_change_type = 0; - } - } - - switch (lease_change_type) { - case OPLOCK_WRITE_TO_READ: - ret = opinfo_write_to_read(opinfo); - break; - case OPLOCK_READ_HANDLE_TO_READ: - ret = opinfo_read_handle_to_read(opinfo); - break; - case OPLOCK_WRITE_TO_NONE: - ret = opinfo_write_to_none(opinfo); - break; - case OPLOCK_READ_TO_NONE: - ret = opinfo_read_to_none(opinfo); - break; - default: - ksmbd_debug(OPLOCK, "unknown lease change 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } - - if (ret < 0) { - rsp->hdr.Status = err; + if (check_lease_state(lease, req->LeaseState)) { + rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; + ksmbd_debug(OPLOCK, + "req lease state: 0x%x, expected state: 0x%x\n", + req->LeaseState, lease->new_state); goto err_out; } - lease_state = lease->state; + lease_state = req->LeaseState; + lease->state = lease_state; + lease->new_state = SMB2_LEASE_NONE_LE; + lease_update_oplock_levels(lease); rsp->StructureSize = cpu_to_le16(36); rsp->Reserved = 0; @@ -8855,16 +9506,20 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) rsp->LeaseState = lease_state; rsp->LeaseDuration = 0; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); - if (ret) { -err_out: - smb2_set_err_rsp(work); - } + if (ret) + goto err_out; opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); atomic_dec(&opinfo->breaking_cnt); wake_up_interruptible_all(&opinfo->oplock_brk); opinfo_put(opinfo); + return; + +err_out: + smb2_set_err_rsp(work); + opinfo_put(opinfo); + return; } /** @@ -8915,6 +9570,9 @@ int smb2_notify(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (work->next_smb2_rcv_hdr_off && req->hdr.NextCommand) { rsp->hdr.Status = STATUS_INTERNAL_ERROR; smb2_set_err_rsp(work); @@ -8939,7 +9597,6 @@ bool smb2_is_sign_req(struct ksmbd_work *work, unsigned int command) if ((rcv_hdr2->Flags & SMB2_FLAGS_SIGNED) && command != SMB2_NEGOTIATE_HE && - command != SMB2_SESSION_SETUP_HE && command != SMB2_OPLOCK_BREAK_HE) return true; @@ -9051,9 +9708,13 @@ int smb3_check_sign_req(struct ksmbd_work *work) } else { chann = lookup_chann_list(work->sess, conn); if (!chann) { - return 0; + if (le16_to_cpu(hdr->Command) != SMB2_SESSION_SETUP_HE || + !(hdr->Flags & SMB2_FLAGS_SIGNED)) + return 0; + signing_key = work->sess->smb3signingkey; + } else { + signing_key = chann->smb3signingkey; } - signing_key = chann->smb3signingkey; } if (!signing_key) { @@ -9088,13 +9749,14 @@ void smb3_set_sign_rsp(struct ksmbd_work *work) struct channel *chann; char signature[SMB2_CMACAES_SIZE]; struct kvec *iov; + u16 command = conn->ops->get_cmd_val(work); int n_vec = 1; char *signing_key; hdr = ksmbd_resp_buf_curr(work); - if (conn->binding == false && - le16_to_cpu(hdr->Command) == SMB2_SESSION_SETUP_HE) { + if (command == SMB2_SESSION_SETUP_HE && + (!conn->binding || hdr->Status != STATUS_SUCCESS)) { signing_key = work->sess->smb3signingkey; } else { chann = lookup_chann_list(work->sess, work->conn); @@ -9137,28 +9799,30 @@ void smb3_preauth_hash_rsp(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); - if (le16_to_cpu(req->Command) == SMB2_NEGOTIATE_HE && - conn->preauth_info) - ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, - conn->preauth_info->Preauth_HashValue); + if (le16_to_cpu(req->Command) == SMB2_NEGOTIATE_HE) { + ksmbd_conn_lock(conn); + if (conn->preauth_info) + ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, + conn->preauth_info->Preauth_HashValue); + ksmbd_conn_unlock(conn); + } if (le16_to_cpu(rsp->Command) == SMB2_SESSION_SETUP_HE && sess) { - __u8 *hash_value; + ksmbd_conn_lock(conn); if (conn->binding) { struct preauth_session *preauth_sess; preauth_sess = ksmbd_preauth_session_lookup(conn, sess->id); - if (!preauth_sess) - return; - hash_value = preauth_sess->Preauth_HashValue; - } else { - hash_value = sess->Preauth_HashValue; - if (!hash_value) - return; + if (preauth_sess) + ksmbd_gen_preauth_integrity_hash(conn, + work->response_buf, + preauth_sess->Preauth_HashValue); + } else if (sess->Preauth_HashValue) { + ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, + sess->Preauth_HashValue); } - ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, - hash_value); + ksmbd_conn_unlock(conn); } } diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h index e7cf573e59f0..aa06c8c905f1 100644 --- a/fs/smb/server/smb2pdu.h +++ b/fs/smb/server/smb2pdu.h @@ -23,7 +23,7 @@ #define MAX_SMB2_HDR_SIZE 0x78 /* 4 len + 64 hdr + (2*24 wct) + 2 bct + 2 pad */ #define SMB21_DEFAULT_IOSIZE (1024 * 1024) -#define SMB3_DEFAULT_TRANS_SIZE (1024 * 1024) +#define SMB3_DEFAULT_TRANS_SIZE (4 * 1024 * 1024) #define SMB3_MIN_IOSIZE (64 * 1024) #define SMB3_MAX_IOSIZE (8 * 1024 * 1024) #define SMB3_MAX_MSGSIZE (4 * 4096) @@ -212,10 +212,6 @@ struct smb2_file_ea_info { __le32 EASize; } __packed; -struct smb2_file_alloc_info { - __le64 AllocationSize; -} __packed; - struct smb2_file_disposition_info { __u8 DeletePending; } __packed; @@ -230,9 +226,6 @@ struct smb2_file_mode_info { __le32 Mode; } __packed; -#define COMPRESSION_FORMAT_NONE 0x0000 -#define COMPRESSION_FORMAT_LZNT1 0x0002 - struct smb2_file_comp_info { __le64 CompressedFileSize; __le16 CompressionFormat; diff --git a/fs/smb/server/smb_common.c b/fs/smb/server/smb_common.c index 741aabdfcef5..7de73223189a 100644 --- a/fs/smb/server/smb_common.c +++ b/fs/smb/server/smb_common.c @@ -102,9 +102,6 @@ static const struct { int version; const char *string; } version_strings[] = { -#ifdef CONFIG_SMB_INSECURE_SERVER - {SMB1_PROT, SMB1_VERSION_STRING}, -#endif {SMB2_PROT, SMB20_VERSION_STRING}, {SMB21_PROT, SMB21_VERSION_STRING}, {SMB30_PROT, SMB30_VERSION_STRING}, @@ -188,11 +185,6 @@ bool ksmbd_smb_request(struct ksmbd_conn *conn) return false; proto = (__le32 *)smb_get_msg(conn->request_buf); - if (*proto == SMB2_COMPRESSION_TRANSFORM_ID) { - pr_err_ratelimited("smb2 compression not support yet"); - return false; - } - if (*proto != SMB1_PROTO_NUMBER && *proto != SMB2_PROTO_NUMBER && *proto != SMB2_TRANSFORM_PROTO_NUM) diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index 4bbc2c27e680..67b39b4d218c 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -258,6 +258,7 @@ static int sid_to_id(struct mnt_idmap *idmap, struct smb_sid *psid, uint sidtype, struct smb_fattr *fattr) { + const struct smb_sid *sid_prefix; int rc = -EINVAL; /* @@ -279,6 +280,12 @@ static int sid_to_id(struct mnt_idmap *idmap, kuid_t uid; uid_t id; + /* Only the server domain RID has a local uid representation. */ + sid_prefix = &server_conf.domain_sid; + if (psid->num_subauth != sid_prefix->num_subauth + 1 || + compare_sids(psid, sid_prefix)) + return -EINVAL; + id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); uid = KUIDT_INIT(id); uid = from_vfsuid(idmap, &init_user_ns, VFSUIDT_INIT(uid)); @@ -290,6 +297,12 @@ static int sid_to_id(struct mnt_idmap *idmap, kgid_t gid; gid_t id; + /* Local gids are represented by S-1-22-2-<gid>. */ + sid_prefix = &sid_unix_groups; + if (psid->num_subauth != sid_prefix->num_subauth + 1 || + compare_sids(psid, sid_prefix)) + return -EINVAL; + id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); gid = KGIDT_INIT(id); gid = from_vfsgid(idmap, &init_user_ns, VFSGIDT_INIT(gid)); @@ -374,6 +387,7 @@ static void parse_dacl(struct mnt_idmap *idmap, { int i, ret; u16 num_aces = 0; + u16 dacl_size; unsigned int acl_size; char *acl_base; struct smb_ace **ppace; @@ -403,7 +417,11 @@ static void parse_dacl(struct mnt_idmap *idmap, if (num_aces <= 0) return; - if (num_aces > (le16_to_cpu(pdacl->size) - sizeof(struct smb_acl)) / + dacl_size = le16_to_cpu(pdacl->size); + if (dacl_size < sizeof(struct smb_acl)) + return; + + if (num_aces > (dacl_size - sizeof(struct smb_acl)) / (offsetof(struct smb_ace, sid) + offsetof(struct smb_sid, sub_auth) + sizeof(__le16))) return; @@ -643,8 +661,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, ntace = (struct smb_ace *)((char *)pndace + *size); ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, pace->e_perm, 0777); - if (check_add_overflow(*size, ace_sz, size)) + if (check_add_overflow(*size, ace_sz, size)) { + kfree(sid); break; + } (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -655,8 +675,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, ntace = (struct smb_ace *)((char *)pndace + *size); ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x03, pace->e_perm, 0777); - if (check_add_overflow(*size, ace_sz, size)) + if (check_add_overflow(*size, ace_sz, size)) { + kfree(sid); break; + } (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -698,8 +720,10 @@ posix_default_acl: ntace = (struct smb_ace *)((char *)pndace + *size); ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, pace->e_perm, 0777); - if (check_add_overflow(*size, ace_sz, size)) + if (check_add_overflow(*size, ace_sz, size)) { + kfree(sid); break; + } (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -734,12 +758,18 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap, if (nt_ace_size > aces_size) break; + if (ntace->sid.num_subauth == 0 || + ntace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) + goto next_ace; + memcpy((char *)pndace + size, ntace, nt_ace_size); if (check_add_overflow(size, nt_ace_size, &size)) break; + num_aces++; + +next_ace: aces_size -= nt_ace_size; ntace = (struct smb_ace *)((char *)ntace + nt_ace_size); - num_aces++; } } @@ -883,9 +913,9 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, rc = sid_to_id(idmap, owner_sid_ptr, SIDOWNER, fattr); if (rc) { - pr_err("%s: Error %d mapping Owner SID to uid\n", - __func__, rc); + ksmbd_debug(SMB, "Owner SID has no Unix uid mapping\n"); owner_sid_ptr = NULL; + rc = 0; } } @@ -901,9 +931,9 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, } rc = sid_to_id(idmap, group_sid_ptr, SIDUNIX_GROUP, fattr); if (rc) { - pr_err("%s: Error %d mapping Group SID to gid\n", - __func__, rc); + ksmbd_debug(SMB, "Group SID has no Unix gid mapping\n"); group_sid_ptr = NULL; + rc = 0; } } @@ -1068,7 +1098,60 @@ static void smb_set_ace(struct smb_ace *ace, const struct smb_sid *sid, u8 type, ace->flags = flags; ace->access_req = access_req; smb_copy_sid(&ace->sid, sid); - ace->size = cpu_to_le16(1 + 1 + 2 + 4 + 1 + 1 + 6 + (sid->num_subauth * 4)); + ace->size = cpu_to_le16(1 + 1 + 2 + 4 + 1 + 1 + 6 + + (ace->sid.num_subauth * 4)); +} + +static int smb_append_inherited_ace(struct smb_ace **ace, int *nt_size, + u16 *ace_cnt, const struct smb_sid *sid, + u8 type, u8 flags, __le32 access_req) +{ + int ace_size; + + smb_set_ace(*ace, sid, type, flags, access_req); + ace_size = le16_to_cpu((*ace)->size); + /* pdacl->size is __le16 and includes struct smb_acl. */ + if (check_add_overflow(*nt_size, ace_size, nt_size) || + *nt_size > U16_MAX - (int)sizeof(struct smb_acl)) + return -EINVAL; + + (*ace_cnt)++; + *ace = (struct smb_ace *)((char *)*ace + ace_size); + return 0; +} + +static int smb_validate_ntsd_sid(struct smb_ntsd *pntsd, size_t pntsd_size, + unsigned int sid_offset, struct smb_sid **sid, + size_t *sid_size) +{ + size_t sid_end; + + *sid = NULL; + *sid_size = 0; + + if (!sid_offset) + return 0; + + if (sid_offset < sizeof(struct smb_ntsd) || + check_add_overflow(sid_offset, (size_t)CIFS_SID_BASE_SIZE, + &sid_end) || + sid_end > pntsd_size) + return -EINVAL; + + *sid = (struct smb_sid *)((char *)pntsd + sid_offset); + if ((*sid)->num_subauth > SID_MAX_SUB_AUTHORITIES) + return -EINVAL; + + if (check_add_overflow((size_t)CIFS_SID_BASE_SIZE, + sizeof(__le32) * (size_t)(*sid)->num_subauth, + &sid_end)) + return -EINVAL; + + if (sid_offset > pntsd_size || sid_end > pntsd_size - sid_offset) + return -EINVAL; + + *sid_size = sid_end; + return 0; } int smb_inherit_dacl(struct ksmbd_conn *conn, @@ -1083,28 +1166,28 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, struct dentry *parent = path->dentry->d_parent; struct mnt_idmap *idmap = mnt_idmap(path->mnt); int inherited_flags = 0, flags = 0, i, nt_size = 0, pdacl_size; - int rc = 0, pntsd_type, pntsd_size, acl_len, aces_size; + int rc = 0, pntsd_type, ppntsd_size, acl_len, aces_size; unsigned int dacloffset; size_t dacl_struct_end; u16 num_aces, ace_cnt = 0; char *aces_base; bool is_dir = S_ISDIR(d_inode(path->dentry)->i_mode); - pntsd_size = ksmbd_vfs_get_sd_xattr(conn, idmap, + ppntsd_size = ksmbd_vfs_get_sd_xattr(conn, idmap, parent, &parent_pntsd); - if (pntsd_size <= 0) + if (ppntsd_size <= 0) return -ENOENT; dacloffset = le32_to_cpu(parent_pntsd->dacloffset); if (!dacloffset || check_add_overflow(dacloffset, sizeof(struct smb_acl), &dacl_struct_end) || - dacl_struct_end > (size_t)pntsd_size) { + dacl_struct_end > (size_t)ppntsd_size) { rc = -EINVAL; goto free_parent_pntsd; } parent_pdacl = (struct smb_acl *)((char *)parent_pntsd + dacloffset); - acl_len = pntsd_size - dacloffset; + acl_len = ppntsd_size - dacloffset; num_aces = le16_to_cpu(parent_pdacl->num_aces); pntsd_type = le16_to_cpu(parent_pntsd->type); pdacl_size = le16_to_cpu(parent_pdacl->size); @@ -1157,6 +1240,12 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, CIFS_SID_BASE_SIZE) break; + if (parent_aces->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + pace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * parent_aces->sid.num_subauth) + break; + aces_size -= pace_size; flags = parent_aces->flags; @@ -1186,22 +1275,24 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, } if (is_dir && creator && flags & CONTAINER_INHERIT_ACE) { - smb_set_ace(aces, psid, parent_aces->type, inherited_flags, - parent_aces->access_req); - nt_size += le16_to_cpu(aces->size); - ace_cnt++; - aces = (struct smb_ace *)((char *)aces + le16_to_cpu(aces->size)); + rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt, + psid, parent_aces->type, + inherited_flags, + parent_aces->access_req); + if (rc) + goto free_aces_base; flags |= INHERIT_ONLY_ACE; psid = creator; } else if (is_dir && !(parent_aces->flags & NO_PROPAGATE_INHERIT_ACE)) { psid = &parent_aces->sid; } - smb_set_ace(aces, psid, parent_aces->type, flags | inherited_flags, - parent_aces->access_req); - nt_size += le16_to_cpu(aces->size); - aces = (struct smb_ace *)((char *)aces + le16_to_cpu(aces->size)); - ace_cnt++; + rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt, psid, + parent_aces->type, + flags | inherited_flags, + parent_aces->access_req); + if (rc) + goto free_aces_base; pass: parent_aces = (struct smb_ace *)((char *)parent_aces + pace_size); } @@ -1210,22 +1301,33 @@ pass: struct smb_ntsd *pntsd; struct smb_acl *pdacl; struct smb_sid *powner_sid = NULL, *pgroup_sid = NULL; - int powner_sid_size = 0, pgroup_sid_size = 0, pntsd_size; - int pntsd_alloc_size; + size_t powner_sid_size = 0, pgroup_sid_size = 0, pntsd_size; + size_t pntsd_alloc_size; - if (parent_pntsd->osidoffset) { - powner_sid = (struct smb_sid *)((char *)parent_pntsd + - le32_to_cpu(parent_pntsd->osidoffset)); - powner_sid_size = 1 + 1 + 6 + (powner_sid->num_subauth * 4); - } - if (parent_pntsd->gsidoffset) { - pgroup_sid = (struct smb_sid *)((char *)parent_pntsd + - le32_to_cpu(parent_pntsd->gsidoffset)); - pgroup_sid_size = 1 + 1 + 6 + (pgroup_sid->num_subauth * 4); - } + rc = smb_validate_ntsd_sid(parent_pntsd, ppntsd_size, + le32_to_cpu(parent_pntsd->osidoffset), + &powner_sid, &powner_sid_size); + if (rc) + goto free_aces_base; + rc = smb_validate_ntsd_sid(parent_pntsd, ppntsd_size, + le32_to_cpu(parent_pntsd->gsidoffset), + &pgroup_sid, &pgroup_sid_size); + if (rc) + goto free_aces_base; - pntsd_alloc_size = sizeof(struct smb_ntsd) + powner_sid_size + - pgroup_sid_size + sizeof(struct smb_acl) + nt_size; + if (check_add_overflow(sizeof(struct smb_ntsd), + (size_t)powner_sid_size, + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, + (size_t)pgroup_sid_size, + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, sizeof(struct smb_acl), + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, (size_t)nt_size, + &pntsd_alloc_size)) { + rc = -EINVAL; + goto free_aces_base; + } pntsd = kzalloc(pntsd_alloc_size, KSMBD_DEFAULT_GFP); if (!pntsd) { @@ -1368,8 +1470,8 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, sid) + - aces_size < CIFS_SID_BASE_SIZE) + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); if (ace_size > aces_size || @@ -1389,8 +1491,8 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, sid) + - aces_size < CIFS_SID_BASE_SIZE) + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); if (ace_size > aces_size || @@ -1399,7 +1501,9 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, break; aces_size -= ace_size; - if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + ace_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE + + sizeof(__le32) * ace->sid.num_subauth) break; if (!compare_sids(&sid, &ace->sid) || diff --git a/fs/smb/server/smbfsctl.h b/fs/smb/server/smbfsctl.h deleted file mode 100644 index ecdf8f6e0df4..000000000000 --- a/fs/smb/server/smbfsctl.h +++ /dev/null @@ -1,91 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1+ */ -/* - * fs/smb/server/smbfsctl.h: SMB, CIFS, SMB2 FSCTL definitions - * - * Copyright (c) International Business Machines Corp., 2002,2009 - * Author(s): Steve French (sfrench@us.ibm.com) - */ - -/* IOCTL information */ -/* - * List of ioctl/fsctl function codes that are or could be useful in the - * future to remote clients like cifs or SMB2 client. There is probably - * a slightly larger set of fsctls that NTFS local filesystem could handle, - * including the seven below that we do not have struct definitions for. - * Even with protocol definitions for most of these now available, we still - * need to do some experimentation to identify which are practical to do - * remotely. Some of the following, such as the encryption/compression ones - * could be invoked from tools via a specialized hook into the VFS rather - * than via the standard vfs entry points - */ - -#ifndef __KSMBD_SMBFSCTL_H -#define __KSMBD_SMBFSCTL_H - -#define FSCTL_DFS_GET_REFERRALS 0x00060194 -#define FSCTL_DFS_GET_REFERRALS_EX 0x000601B0 -#define FSCTL_REQUEST_OPLOCK_LEVEL_1 0x00090000 -#define FSCTL_REQUEST_OPLOCK_LEVEL_2 0x00090004 -#define FSCTL_REQUEST_BATCH_OPLOCK 0x00090008 -#define FSCTL_LOCK_VOLUME 0x00090018 -#define FSCTL_UNLOCK_VOLUME 0x0009001C -#define FSCTL_IS_PATHNAME_VALID 0x0009002C /* BB add struct */ -#define FSCTL_GET_COMPRESSION 0x0009003C /* BB add struct */ -#define FSCTL_SET_COMPRESSION 0x0009C040 /* BB add struct */ -#define FSCTL_QUERY_FAT_BPB 0x00090058 /* BB add struct */ -/* Verify the next FSCTL number, we had it as 0x00090090 before */ -#define FSCTL_FILESYSTEM_GET_STATS 0x00090060 /* BB add struct */ -#define FSCTL_GET_NTFS_VOLUME_DATA 0x00090064 /* BB add struct */ -#define FSCTL_GET_RETRIEVAL_POINTERS 0x00090073 /* BB add struct */ -#define FSCTL_IS_VOLUME_DIRTY 0x00090078 /* BB add struct */ -#define FSCTL_ALLOW_EXTENDED_DASD_IO 0x00090083 /* BB add struct */ -#define FSCTL_REQUEST_FILTER_OPLOCK 0x0009008C -#define FSCTL_FIND_FILES_BY_SID 0x0009008F /* BB add struct */ -#define FSCTL_SET_OBJECT_ID 0x00090098 /* BB add struct */ -#define FSCTL_GET_OBJECT_ID 0x0009009C /* BB add struct */ -#define FSCTL_DELETE_OBJECT_ID 0x000900A0 /* BB add struct */ -#define FSCTL_SET_REPARSE_POINT 0x000900A4 /* BB add struct */ -#define FSCTL_GET_REPARSE_POINT 0x000900A8 /* BB add struct */ -#define FSCTL_DELETE_REPARSE_POINT 0x000900AC /* BB add struct */ -#define FSCTL_SET_OBJECT_ID_EXTENDED 0x000900BC /* BB add struct */ -#define FSCTL_CREATE_OR_GET_OBJECT_ID 0x000900C0 /* BB add struct */ -#define FSCTL_SET_SPARSE 0x000900C4 /* BB add struct */ -#define FSCTL_SET_ZERO_DATA 0x000980C8 /* BB add struct */ -#define FSCTL_SET_ENCRYPTION 0x000900D7 /* BB add struct */ -#define FSCTL_ENCRYPTION_FSCTL_IO 0x000900DB /* BB add struct */ -#define FSCTL_WRITE_RAW_ENCRYPTED 0x000900DF /* BB add struct */ -#define FSCTL_READ_RAW_ENCRYPTED 0x000900E3 /* BB add struct */ -#define FSCTL_READ_FILE_USN_DATA 0x000900EB /* BB add struct */ -#define FSCTL_WRITE_USN_CLOSE_RECORD 0x000900EF /* BB add struct */ -#define FSCTL_SIS_COPYFILE 0x00090100 /* BB add struct */ -#define FSCTL_RECALL_FILE 0x00090117 /* BB add struct */ -#define FSCTL_QUERY_SPARING_INFO 0x00090138 /* BB add struct */ -#define FSCTL_SET_ZERO_ON_DEALLOC 0x00090194 /* BB add struct */ -#define FSCTL_SET_SHORT_NAME_BEHAVIOR 0x000901B4 /* BB add struct */ -#define FSCTL_QUERY_ALLOCATED_RANGES 0x000940CF /* BB add struct */ -#define FSCTL_SET_DEFECT_MANAGEMENT 0x00098134 /* BB add struct */ -#define FSCTL_DUPLICATE_EXTENTS_TO_FILE 0x00098344 -#define FSCTL_SIS_LINK_FILES 0x0009C104 -#define FSCTL_PIPE_PEEK 0x0011400C /* BB add struct */ -#define FSCTL_PIPE_TRANSCEIVE 0x0011C017 /* BB add struct */ -/* strange that the number for this op is not sequential with previous op */ -#define FSCTL_PIPE_WAIT 0x00110018 /* BB add struct */ -#define FSCTL_REQUEST_RESUME_KEY 0x00140078 -#define FSCTL_LMR_GET_LINK_TRACK_INF 0x001400E8 /* BB add struct */ -#define FSCTL_LMR_SET_LINK_TRACK_INF 0x001400EC /* BB add struct */ -#define FSCTL_VALIDATE_NEGOTIATE_INFO 0x00140204 -#define FSCTL_QUERY_NETWORK_INTERFACE_INFO 0x001401FC -#define FSCTL_COPYCHUNK 0x001440F2 -#define FSCTL_COPYCHUNK_WRITE 0x001480F2 - -#define IO_REPARSE_TAG_MOUNT_POINT 0xA0000003 -#define IO_REPARSE_TAG_HSM 0xC0000004 -#define IO_REPARSE_TAG_SIS 0x80000007 - -/* WSL reparse tags */ -#define IO_REPARSE_TAG_LX_SYMLINK_LE cpu_to_le32(0xA000001D) -#define IO_REPARSE_TAG_AF_UNIX_LE cpu_to_le32(0x80000023) -#define IO_REPARSE_TAG_LX_FIFO_LE cpu_to_le32(0x80000024) -#define IO_REPARSE_TAG_LX_CHR_LE cpu_to_le32(0x80000025) -#define IO_REPARSE_TAG_LX_BLK_LE cpu_to_le32(0x80000026) -#endif /* __KSMBD_SMBFSCTL_H */ diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index a8242c00096f..b6d63ff8a8a3 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -18,7 +18,6 @@ #include "smb_common.h" #include "../common/smb2status.h" #include "transport_rdma.h" -#include "../smbdirect/public.h" #define SMB_DIRECT_PORT_IWARP 5445 @@ -540,3 +539,5 @@ static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = { .rdma_write = smb_direct_rdma_write, .free_transport = smb_direct_free_transport, }; + +MODULE_IMPORT_NS("SMBDIRECT"); diff --git a/fs/smb/server/transport_rdma.h b/fs/smb/server/transport_rdma.h index bde3d88aecc7..8b78917a1795 100644 --- a/fs/smb/server/transport_rdma.h +++ b/fs/smb/server/transport_rdma.h @@ -25,6 +25,6 @@ static inline void init_smbd_max_io_size(unsigned int sz) { } static inline unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) { return 0; } #endif -#include "../smbdirect/smbdirect.h" +#include <linux/smbdirect.h> #endif /* __KSMBD_TRANSPORT_RDMA_H__ */ diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index d08973b288e5..d0a0ad15d803 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -19,8 +19,8 @@ #include <linux/vmalloc.h> #include <linux/sched/xacct.h> #include <linux/crc32c.h> -#include <linux/namei.h> #include <linux/splice.h> +#include <linux/fileattr.h> #include "glob.h" #include "oplock.h" @@ -56,7 +56,7 @@ static int ksmbd_vfs_path_lookup(struct ksmbd_share_config *share_conf, { struct qstr last; const struct path *root_share_path = &share_conf->vfs_path; - int err, type; + int err; struct dentry *d; if (pathname[0] == '\0') { @@ -67,17 +67,11 @@ static int ksmbd_vfs_path_lookup(struct ksmbd_share_config *share_conf, } CLASS(filename_kernel, filename)(pathname); - err = vfs_path_parent_lookup(filename, flags, - path, &last, &type, + err = vfs_path_parent_lookup(filename, flags, path, &last, root_share_path); if (err) return err; - if (unlikely(type != LAST_NORM)) { - path_put(path); - return -ENOENT; - } - if (for_remove) { err = mnt_want_write(path->mnt); if (err) { @@ -254,17 +248,20 @@ out: static int ksmbd_vfs_stream_read(struct ksmbd_file *fp, char *buf, loff_t *pos, size_t count) { + const struct cred *saved_cred; ssize_t v_len; char *stream_buf = NULL; ksmbd_debug(VFS, "read stream data pos : %llu, count : %zd\n", *pos, count); + saved_cred = override_creds(fp->filp->f_cred); v_len = ksmbd_vfs_getcasexattr(file_mnt_idmap(fp->filp), fp->filp->f_path.dentry, fp->stream.name, fp->stream.size, &stream_buf); + revert_creds(saved_cred); if ((int)v_len <= 0) return (int)v_len; @@ -388,6 +385,7 @@ int ksmbd_vfs_read(struct ksmbd_work *work, struct ksmbd_file *fp, size_t count, static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, size_t count) { + const struct cred *saved_cred; char *stream_buf = NULL, *wbuf; struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); size_t size; @@ -408,6 +406,7 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, count = XATTR_SIZE_MAX - *pos; } + saved_cred = override_creds(fp->filp->f_cred); v_len = ksmbd_vfs_getcasexattr(idmap, fp->filp->f_path.dentry, fp->stream.name, @@ -416,14 +415,14 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, if (v_len < 0) { pr_err("not found stream in xattr : %zd\n", v_len); err = v_len; - goto out; + goto out_revert; } if (v_len < size) { wbuf = kvzalloc(size, KSMBD_DEFAULT_GFP); if (!wbuf) { err = -ENOMEM; - goto out; + goto out_revert; } if (v_len > 0) @@ -441,6 +440,8 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, size, 0, true); +out_revert: + revert_creds(saved_cred); if (err < 0) goto out; else @@ -668,7 +669,6 @@ int ksmbd_vfs_rename(struct ksmbd_work *work, const struct path *old_path, struct renamedata rd; struct ksmbd_share_config *share_conf = work->tcon->share_conf; struct ksmbd_file *parent_fp; - int new_type; int err, lookup_flags = LOOKUP_NO_SYMLINKS; if (ksmbd_override_fsids(work)) @@ -678,8 +678,7 @@ int ksmbd_vfs_rename(struct ksmbd_work *work, const struct path *old_path, retry: err = vfs_path_parent_lookup(to, lookup_flags | LOOKUP_BENEATH, - &new_path, &new_last, &new_type, - &share_conf->vfs_path); + &new_path, &new_last, &share_conf->vfs_path); if (err) goto out1; @@ -701,10 +700,18 @@ retry: if (err) goto out_drop_write; + if (!work->tcon->posix_extensions && d_is_dir(old_child) && + ksmbd_has_open_files(old_child)) { + err = -EACCES; + goto out3; + } + parent_fp = ksmbd_lookup_fd_inode(old_child->d_parent); if (parent_fp) { - if (parent_fp->daccess & FILE_DELETE_LE) { - pr_err("parent dir is opened with delete access\n"); + if ((parent_fp->daccess & FILE_DELETE_LE) || + (!parent_fp->attrib_only && + !(parent_fp->saccess & FILE_SHARE_DELETE_LE))) { + pr_err("parent dir blocks delete sharing\n"); err = -ESHARE; ksmbd_fd_put(work, parent_fp); goto out3; @@ -919,15 +926,21 @@ void ksmbd_vfs_set_fadvise(struct file *filp, __le32 option) int ksmbd_vfs_zero_data(struct ksmbd_work *work, struct ksmbd_file *fp, loff_t off, loff_t len) { + const struct cred *saved_cred; + int err; + smb_break_all_levII_oplock(work, fp, 1); + saved_cred = override_creds(fp->filp->f_cred); if (fp->f_ci->m_fattr & FILE_ATTRIBUTE_SPARSE_FILE_LE) - return vfs_fallocate(fp->filp, - FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - off, len); - - return vfs_fallocate(fp->filp, - FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE, - off, len); + err = vfs_fallocate(fp->filp, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, len); + else + err = vfs_fallocate(fp->filp, + FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE, + off, len); + revert_creds(saved_cred); + return err; } int ksmbd_vfs_fqar_lseek(struct ksmbd_file *fp, loff_t start, loff_t length, @@ -1009,13 +1022,15 @@ int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap, int ksmbd_vfs_unlink(struct file *filp) { + const struct cred *saved_cred; int err = 0; struct dentry *dir, *dentry = filp->f_path.dentry; struct mnt_idmap *idmap = file_mnt_idmap(filp); + saved_cred = override_creds(filp->f_cred); err = mnt_want_write(filp->f_path.mnt); if (err) - return err; + goto out_revert; dir = dget_parent(dentry); dentry = start_removing_dentry(dir, dentry); @@ -1034,7 +1049,8 @@ int ksmbd_vfs_unlink(struct file *filp) out: dput(dir); mnt_drop_write(filp->f_path.mnt); - +out_revert: + revert_creds(saved_cred); return err; } @@ -1149,7 +1165,7 @@ int __ksmbd_vfs_kern_path(struct ksmbd_work *work, char *filepath, retry: err = ksmbd_vfs_path_lookup(share_conf, filepath, flags, path, for_remove); - if (!err || !caseless) + if (!err || err != -ENOENT || !caseless) return err; path_len = strlen(filepath); @@ -1251,15 +1267,30 @@ struct dentry *ksmbd_vfs_kern_path_create(struct ksmbd_work *work, unsigned int flags, struct path *path) { - char *abs_name; + struct ksmbd_share_config *share_conf = work->tcon->share_conf; + struct qstr last; struct dentry *dent; + int err; + + /* resolve the name beneath the share root so ".." cannot escape */ + CLASS(filename_kernel, filename)(name); - abs_name = convert_to_unix_name(work->tcon->share_conf, name); - if (!abs_name) - return ERR_PTR(-ENOMEM); + err = vfs_path_parent_lookup(filename, flags | LOOKUP_BENEATH, + path, &last, &share_conf->vfs_path); + if (err) + return ERR_PTR(err); + + err = mnt_want_write(path->mnt); + if (err) { + path_put(path); + return ERR_PTR(err); + } - dent = start_creating_path(AT_FDCWD, abs_name, path, flags); - kfree(abs_name); + dent = start_creating_noperm(path->dentry, &last); + if (IS_ERR(dent)) { + mnt_drop_write(path->mnt); + path_put(path); + } return dent; } @@ -1456,8 +1487,8 @@ int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn, if (rc < 0) pr_err("Failed to store XATTR ntacl :%d\n", rc); - kfree(sd_ndr.data); out: + kfree(sd_ndr.data); kfree(acl_ndr.data); kfree(smb_acl); kfree(def_smb_acl); @@ -1473,7 +1504,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, struct ndr n; struct inode *inode = d_inode(dentry); struct ndr acl_ndr = {0}; - struct xattr_ntacl acl; + struct xattr_ntacl acl = {0}; struct xattr_smb_acl *smb_acl = NULL, *def_smb_acl = NULL; __u8 cmp_hash[XATTR_SD_HASH_SIZE] = {0}; @@ -1484,7 +1515,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, n.length = rc; rc = ndr_decode_v4_ntacl(&n, &acl); if (rc) - goto free_n_data; + goto out_free; smb_acl = ksmbd_vfs_make_xattr_posix_acl(idmap, inode, ACL_TYPE_ACCESS); @@ -1510,6 +1541,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, *pntsd = acl.sd_buf; if (acl.sd_size < sizeof(struct smb_ntsd)) { pr_err("sd size is invalid\n"); + rc = -EINVAL; goto out_free; } @@ -1529,8 +1561,6 @@ out_free: kfree(acl.sd_buf); *pntsd = NULL; } - -free_n_data: kfree(n.data); return rc; } @@ -1545,14 +1575,15 @@ int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap, err = ndr_encode_dos_attr(&n, da); if (err) - return err; + goto out; err = ksmbd_vfs_setxattr(idmap, path, XATTR_NAME_DOS_ATTRIBUTE, (void *)n.data, n.offset, 0, get_write); if (err) ksmbd_debug(SMB, "failed to store dos attribute in xattr\n"); - kfree(n.data); +out: + kfree(n.data); return err; } @@ -1890,3 +1921,117 @@ int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap, posix_acl_release(acls); return rc; } + +void ksmbd_vfs_update_compressed_fattr(struct dentry *dentry, __le32 *fattr) +{ + int rc; + struct file_kattr fa = { .flags_valid = true }; + + rc = vfs_fileattr_get(dentry, &fa); + if (rc == -ENOIOCTLCMD) + *fattr &= ~FILE_ATTRIBUTE_COMPRESSED_LE; + if (rc) + return; + + if (fa.flags & FS_COMPR_FL) + *fattr |= FILE_ATTRIBUTE_COMPRESSED_LE; + else + *fattr &= ~FILE_ATTRIBUTE_COMPRESSED_LE; +} + +int ksmbd_vfs_get_compression(struct ksmbd_file *fp, u16 *fmt) +{ + struct file_kattr fa = { .flags_valid = true }; + int rc; + + rc = vfs_fileattr_get(fp->filp->f_path.dentry, &fa); + if (rc == -ENOIOCTLCMD) { + *fmt = COMPRESSION_FORMAT_NONE; + rc = 0; + goto out; + } + if (rc) + goto out; + + if (fa.flags & FS_COMPR_FL) + *fmt = COMPRESSION_FORMAT_LZNT1; + else + *fmt = COMPRESSION_FORMAT_NONE; + +out: + return rc; +} + +int ksmbd_vfs_set_compression(struct ksmbd_work *work, struct ksmbd_file *fp, u16 fmt) +{ + const struct cred *saved_cred = NULL; + struct file_kattr fa; + struct dentry *dentry = fp->filp->f_path.dentry; + struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); + u32 flags; + __le32 old_fattr; + int rc; + + if (!(fp->daccess & FILE_WRITE_DATA_LE)) { + rc = -EACCES; + goto out; + } + + saved_cred = override_creds(fp->filp->f_cred); + rc = vfs_fileattr_get(dentry, &fa); + if (rc) + goto out; + + flags = fa.flags; + if (fmt == COMPRESSION_FORMAT_NONE) { + flags &= ~FS_COMPR_FL; + } else if (fmt == COMPRESSION_FORMAT_DEFAULT || + fmt == COMPRESSION_FORMAT_LZNT1) { + flags |= FS_COMPR_FL; + } else { + rc = -EINVAL; + goto out; + } + + if (flags != fa.flags) { + fileattr_fill_flags(&fa, flags); + rc = mnt_want_write_file(fp->filp); + if (rc) + goto out; + + rc = vfs_fileattr_set(idmap, dentry, &fa); + mnt_drop_write_file(fp->filp); + if (rc) + goto out; + } + + old_fattr = fp->f_ci->m_fattr; + if (fmt == COMPRESSION_FORMAT_NONE) + fp->f_ci->m_fattr &= ~FILE_ATTRIBUTE_COMPRESSED_LE; + else + fp->f_ci->m_fattr |= FILE_ATTRIBUTE_COMPRESSED_LE; + + if (fp->f_ci->m_fattr != old_fattr && + test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_STORE_DOS_ATTRS)) { + struct xattr_dos_attrib da; + + rc = ksmbd_vfs_get_dos_attrib_xattr(idmap, dentry, &da); + if (rc <= 0) { + rc = 0; + goto out; + } + + da.attr = le32_to_cpu(fp->f_ci->m_fattr); + rc = ksmbd_vfs_set_dos_attrib_xattr(idmap, + &fp->filp->f_path, + &da, true); + if (rc) + rc = 0; + } + +out: + if (saved_cred) + revert_creds(saved_cred); + return rc; +} diff --git a/fs/smb/server/vfs.h b/fs/smb/server/vfs.h index 16ca29ee16e5..7b3d2f4fd985 100644 --- a/fs/smb/server/vfs.h +++ b/fs/smb/server/vfs.h @@ -168,4 +168,7 @@ int ksmbd_vfs_set_init_posix_acl(struct mnt_idmap *idmap, int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap, const struct path *path, struct inode *parent_inode); +void ksmbd_vfs_update_compressed_fattr(struct dentry *dentry, __le32 *fattr); +int ksmbd_vfs_get_compression(struct ksmbd_file *fp, u16 *fmt); +int ksmbd_vfs_set_compression(struct ksmbd_work *work, struct ksmbd_file *fp, u16 fmt); #endif /* __KSMBD_VFS_H__ */ diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index 3551f01a3fa0..d95c405eab11 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -10,6 +10,7 @@ #include <linux/vmalloc.h> #include <linux/kthread.h> #include <linux/freezer.h> +#include <linux/dcache.h> #include "glob.h" #include "vfs_cache.h" @@ -37,6 +38,8 @@ static struct ksmbd_file_table global_ft; static atomic_long_t fd_limit; static struct kmem_cache *filp_cache; +static int ksmbd_mark_fp_closed(struct ksmbd_file *fp); + #define OPLOCK_NONE 0 #define OPLOCK_EXCLUSIVE 1 #define OPLOCK_BATCH 2 @@ -81,7 +84,7 @@ static int proc_show_files(struct seq_file *m, void *v) read_lock(&global_ft.lock); idr_for_each_entry(global_ft.idr, fp, id) { seq_printf(m, "%#-10x %#-10llx %#-10llx %#-10x", - fp->tcon->id, + fp->tcon ? fp->tcon->id : 0, fp->persistent_id, fp->volatile_id, atomic_read(&fp->refcount)); @@ -211,13 +214,13 @@ int ksmbd_query_inode_status(struct dentry *dentry) return ret; down_read(&ci->m_lock); - if (ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS)) + if (ci->m_flags & S_DEL_PENDING) ret = KSMBD_INODE_STATUS_PENDING_DELETE; else ret = KSMBD_INODE_STATUS_OK; up_read(&ci->m_lock); - atomic_dec(&ci->m_count); + ksmbd_inode_put(ci); return ret; } @@ -227,7 +230,7 @@ bool ksmbd_inode_pending_delete(struct ksmbd_file *fp) int ret; down_read(&ci->m_lock); - ret = (ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS)); + ret = (ci->m_flags & S_DEL_PENDING); up_read(&ci->m_lock); return ret; @@ -251,6 +254,33 @@ void ksmbd_clear_inode_pending_delete(struct ksmbd_file *fp) up_write(&ci->m_lock); } +bool ksmbd_has_stream_without_delete_share(struct ksmbd_file *fp) +{ + struct ksmbd_file *prev_fp; + struct ksmbd_inode *ci = fp->f_ci; + bool ret = false; + + if (ksmbd_stream_fd(fp)) + return false; + + down_read(&ci->m_lock); + list_for_each_entry(prev_fp, &ci->m_fp_list, node) { + if (prev_fp == fp || !ksmbd_stream_fd(prev_fp)) + continue; + + if (file_inode(fp->filp) != file_inode(prev_fp->filp)) + continue; + + if (!(prev_fp->saccess & FILE_SHARE_DELETE_LE)) { + ret = true; + break; + } + } + up_read(&ci->m_lock); + + return ret; +} + void ksmbd_fd_set_delete_on_close(struct ksmbd_file *fp, int file_info) { @@ -385,22 +415,34 @@ static void __ksmbd_inode_close(struct ksmbd_file *fp) up_write(&ci->m_lock); if (remove_stream_xattr) { + const struct cred *saved_cred; + + saved_cred = override_creds(filp->f_cred); err = ksmbd_vfs_remove_xattr(file_mnt_idmap(filp), &filp->f_path, fp->stream.name, true); + revert_creds(saved_cred); if (err) pr_err("remove xattr failed : %s\n", fp->stream.name); } } + down_write(&ci->m_lock); + /* Promote S_DEL_ON_CLS to S_DEL_PENDING when close */ + if (ci->m_flags & S_DEL_ON_CLS) { + ci->m_flags &= ~S_DEL_ON_CLS; + ci->m_flags |= S_DEL_PENDING; + } + up_write(&ci->m_lock); + if (atomic_dec_and_test(&ci->m_count)) { bool do_unlink = false; down_write(&ci->m_lock); - if (ci->m_flags & (S_DEL_ON_CLS | S_DEL_PENDING)) { - ci->m_flags &= ~(S_DEL_ON_CLS | S_DEL_PENDING); + if (ci->m_flags & S_DEL_PENDING) { + ci->m_flags &= ~S_DEL_PENDING; do_unlink = true; } up_write(&ci->m_lock); @@ -418,6 +460,14 @@ static void __ksmbd_remove_durable_fd(struct ksmbd_file *fp) return; idr_remove(global_ft.idr, fp->persistent_id); + /* + * Clear persistent_id so a later __ksmbd_close_fd() that runs from a + * delayed putter (e.g. when a concurrent ksmbd_lookup_fd_inode() + * walker held the final reference) does not re-issue idr_remove() on + * an id that idr_alloc_cyclic() may have already handed out to a new + * durable handle. + */ + fp->persistent_id = KSMBD_NO_FID; } static void ksmbd_remove_durable_fd(struct ksmbd_file *fp) @@ -431,13 +481,13 @@ static void ksmbd_remove_durable_fd(struct ksmbd_file *fp) static void __ksmbd_remove_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) { - if (!has_file_id(fp->volatile_id)) - return; - down_write(&fp->f_ci->m_lock); list_del_init(&fp->node); up_write(&fp->f_ci->m_lock); + if (!has_file_id(fp->volatile_id)) + return; + write_lock(&ft->lock); idr_remove(ft->idr, fp->volatile_id); write_unlock(&ft->lock); @@ -464,10 +514,14 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) * there are not accesses to fp->lock_list. */ list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - if (!list_empty(&smb_lock->clist) && fp->conn) { - spin_lock(&fp->conn->llist_lock); - list_del(&smb_lock->clist); - spin_unlock(&fp->conn->llist_lock); + struct ksmbd_conn *conn = smb_lock->conn; + + if (conn) { + spin_lock(&conn->llist_lock); + list_del_init(&smb_lock->clist); + smb_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + ksmbd_conn_put(conn); } list_del(&smb_lock->flist); @@ -475,6 +529,17 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) kfree(smb_lock); } + /* + * Drop fp's strong reference on conn (taken in ksmbd_open_fd() / + * ksmbd_reopen_durable_fd()). Durable fps that reached the + * scavenger have already had fp->conn cleared by session_fd_check(), + * in which case there is nothing to drop here. + */ + if (fp->conn) { + ksmbd_conn_put(fp->conn); + fp->conn = NULL; + } + if (ksmbd_stream_fd(fp)) kfree(fp->stream.name); kfree(fp->owner.name); @@ -482,6 +547,78 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) kmem_cache_free(filp_cache, fp); } +/** + * ksmbd_close_disconnected_durable_delete_on_close() - drop a delete-on-close + * file kept present only by disconnected durable handles + * @dentry: dentry of the file being opened + * + * A durable handle opened with delete-on-close is preserved across a + * disconnect so it can be reclaimed by a durable reconnect. When a new + * (non-reconnect) open arrives for the same name instead, the disconnected + * handle has to give way. Close such handles so their delete-on-close is + * applied and the file is removed once the last handle is gone, letting the + * new open create a fresh file. + * + * The caller's inode reference is dropped before closing so that the final + * close can promote S_DEL_ON_CLS to S_DEL_PENDING and unlink the file. + * + * Return: true if a disconnected durable handle was closed. + */ +bool ksmbd_close_disconnected_durable_delete_on_close(struct dentry *dentry) +{ + struct ksmbd_inode *ci; + struct ksmbd_file *fp, *tmp; + LIST_HEAD(dispose); + bool closed = false; + + ci = ksmbd_inode_lookup_lock(dentry); + if (!ci) + return false; + + down_write(&ci->m_lock); + if (ci->m_flags & (S_DEL_ON_CLS | S_DEL_ON_CLS_STREAM | S_DEL_PENDING)) { + list_for_each_entry_safe(fp, tmp, &ci->m_fp_list, node) { + if (fp->conn || !fp->is_durable || + fp->f_state != FP_INITED) + continue; + + /* + * Claim the close before unlinking fp from m_fp_list. + * refcount == 1 means only the durable lifetime ref is + * left. Add a transient ref so final close can drop both. + */ + write_lock(&global_ft.lock); + if (atomic_read(&fp->refcount) == 1) { + atomic_inc(&fp->refcount); + __ksmbd_remove_durable_fd(fp); + ksmbd_mark_fp_closed(fp); + list_move_tail(&fp->node, &dispose); + } + write_unlock(&global_ft.lock); + } + } + up_write(&ci->m_lock); + + /* + * Drop our lookup reference before closing so the last __ksmbd_close_fd() + * can drop m_count to zero and unlink the delete-on-close file. The + * collected handles still hold the transient reference taken above, so + * ci stays valid until they are closed below. + */ + ksmbd_inode_put(ci); + + while (!list_empty(&dispose)) { + fp = list_first_entry(&dispose, struct ksmbd_file, node); + list_del_init(&fp->node); + if (atomic_sub_and_test(2, &fp->refcount)) { + __ksmbd_close_fd(NULL, fp); + closed = true; + } + } + + return closed; +} + static struct ksmbd_file *ksmbd_fp_get(struct ksmbd_file *fp) { if (fp->f_state != FP_INITED) @@ -510,6 +647,20 @@ static struct ksmbd_file *__ksmbd_lookup_fd(struct ksmbd_file_table *ft, static void __put_fd_final(struct ksmbd_work *work, struct ksmbd_file *fp) { + /* + * Detached durable fp -- session_fd_check() cleared fp->conn at + * preserve, so this fp is no longer tracked by any conn's + * stats.open_files_count. This happens when + * ksmbd_scavenger_dispose_dh() hands the final close off to an + * m_fp_list walker (e.g. ksmbd_lookup_fd_inode()) whose work->conn + * is unrelated to the conn that originally opened the handle; close + * via the NULL-ft path so we do not underflow that unrelated + * counter. + */ + if (!fp->conn) { + __ksmbd_close_fd(NULL, fp); + return; + } __ksmbd_close_fd(&work->sess->file_table, fp); atomic_dec(&work->conn->stats.open_files_count); } @@ -531,6 +682,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) { struct ksmbd_file *fp; struct ksmbd_file_table *ft; + bool closed = false; if (!has_file_id(id)) return 0; @@ -545,6 +697,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) fp = NULL; else { fp->f_state = FP_CLOSED; + closed = true; if (!atomic_dec_and_test(&fp->refcount)) fp = NULL; } @@ -552,7 +705,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) write_unlock(&ft->lock); if (!fp) - return -EINVAL; + return closed ? 0 : -EINVAL; __put_fd_final(work, fp); return 0; @@ -625,7 +778,8 @@ struct ksmbd_file *ksmbd_lookup_durable_fd(unsigned long long id) struct ksmbd_file *fp; fp = __ksmbd_lookup_fd(&global_ft, id); - if (fp && (fp->conn || + if (fp && (fp->durable_reconnect_disabled || + fp->conn || (fp->durable_scavenger_timeout && (fp->durable_scavenger_timeout < jiffies_to_msecs(jiffies))))) { @@ -644,6 +798,127 @@ void ksmbd_put_durable_fd(struct ksmbd_file *fp) __ksmbd_close_fd(NULL, fp); } +bool ksmbd_has_other_active_fd(struct ksmbd_file *fp) +{ + struct ksmbd_file *lfp; + struct ksmbd_inode *ci = fp->f_ci; + bool ret = false; + + down_read(&ci->m_lock); + list_for_each_entry(lfp, &ci->m_fp_list, node) { + if (lfp == fp) + continue; + + if (lfp->f_state == FP_INITED && + (READ_ONCE(lfp->conn) || READ_ONCE(lfp->tcon))) { + ret = true; + break; + } + } + up_read(&ci->m_lock); + + return ret; +} + +static struct ksmbd_file *ksmbd_lookup_fd_app_instance_id(char *app_instance_id) +{ + struct ksmbd_file *fp = NULL; + unsigned int id; + + if (!memchr_inv(app_instance_id, 0, SMB2_CREATE_GUID_SIZE)) + return NULL; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + if (!memcmp(fp->app_instance_id, app_instance_id, + SMB2_CREATE_GUID_SIZE)) { + fp = ksmbd_fp_get(fp); + break; + } + } + read_unlock(&global_ft.lock); + + return fp; +} + +int ksmbd_close_fd_app_instance_id(char *app_instance_id) +{ + struct ksmbd_file_table *ft; + struct ksmbd_file *fp; + struct oplock_info *opinfo; + int n_to_drop = 0; + + fp = ksmbd_lookup_fd_app_instance_id(app_instance_id); + if (!fp) + return 0; + + opinfo = opinfo_get(fp); + if (!opinfo) + goto out; + + down_read(&fp->f_ci->m_lock); + if (!opinfo->conn) { + up_read(&fp->f_ci->m_lock); + goto out; + } + + ft = &opinfo->sess->file_table; + write_lock(&ft->lock); + if (fp->f_state == FP_INITED && has_file_id(fp->volatile_id)) { + idr_remove(ft->idr, fp->volatile_id); + fp->volatile_id = KSMBD_NO_FID; + n_to_drop = ksmbd_mark_fp_closed(fp); + } + write_unlock(&ft->lock); + up_read(&fp->f_ci->m_lock); + opinfo_put(opinfo); + opinfo = NULL; + + if (!n_to_drop) + goto out; + + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); + + if (atomic_sub_and_test(n_to_drop, &fp->refcount)) { + if (fp->conn) + atomic_dec(&fp->conn->stats.open_files_count); + __ksmbd_close_fd(NULL, fp); + } + return 0; + +out: + if (opinfo) + opinfo_put(opinfo); + ksmbd_put_durable_fd(fp); + return 0; +} + +int ksmbd_invalidate_durable_fd(unsigned long long id) +{ + struct ksmbd_file *fp; + + fp = ksmbd_lookup_global_fd(id); + if (!fp) + return -ENOENT; + + fp->durable_reconnect_disabled = true; + + if (fp->conn) { + ksmbd_put_durable_fd(fp); + return -ENOENT; + } + + fp->durable_timeout = 1; + fp->durable_scavenger_timeout = jiffies_to_msecs(jiffies); + ksmbd_put_durable_fd(fp); + if (waitqueue_active(&dh_wq)) + wake_up(&dh_wq); + + return -ENOENT; +} + struct ksmbd_file *ksmbd_lookup_fd_cguid(char *cguid) { struct ksmbd_file *fp = NULL; @@ -678,17 +953,41 @@ struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry) down_read(&ci->m_lock); list_for_each_entry(lfp, &ci->m_fp_list, node) { if (inode == file_inode(lfp->filp)) { - atomic_dec(&ci->m_count); lfp = ksmbd_fp_get(lfp); up_read(&ci->m_lock); + ksmbd_inode_put(ci); return lfp; } } - atomic_dec(&ci->m_count); up_read(&ci->m_lock); + ksmbd_inode_put(ci); return NULL; } +bool ksmbd_has_open_files(struct dentry *dentry) +{ + struct ksmbd_file *fp; + unsigned int id; + bool ret = false; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + struct dentry *fp_dentry = fp->filp->f_path.dentry; + + if (fp->f_state != FP_INITED) + continue; + if (fp_dentry == dentry) + continue; + if (is_subdir(fp_dentry, dentry)) { + ret = true; + break; + } + } + read_unlock(&global_ft.lock); + + return ret; +} + #define OPEN_ID_TYPE_VOLATILE_ID (0) #define OPEN_ID_TYPE_PERSISTENT_ID (1) @@ -713,7 +1012,8 @@ static int __open_id(struct ksmbd_file_table *ft, struct ksmbd_file *fp, idr_preload(KSMBD_DEFAULT_GFP); write_lock(&ft->lock); - ret = idr_alloc_cyclic(ft->idr, fp, 0, INT_MAX - 1, GFP_NOWAIT); + ret = idr_alloc_cyclic(ft->idr, fp, KSMBD_START_FID, INT_MAX - 1, + GFP_NOWAIT); if (ret >= 0) { id = ret; ret = 0; @@ -749,10 +1049,18 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp) INIT_LIST_HEAD(&fp->node); INIT_LIST_HEAD(&fp->lock_list); spin_lock_init(&fp->f_lock); + mutex_init(&fp->readdir_lock); atomic_set(&fp->refcount, 1); fp->filp = filp; - fp->conn = work->conn; + /* + * fp owns a strong reference on fp->conn for as long as fp->conn is + * non-NULL, so session_fd_check() and __ksmbd_close_fd() never + * dereference a dangling pointer. Paired with ksmbd_conn_put() in + * session_fd_check() (durable preserve), in __ksmbd_close_fd() + * (final close), and on the error paths below. + */ + fp->conn = ksmbd_conn_get(work->conn); fp->tcon = work->tcon; fp->volatile_id = KSMBD_NO_FID; fp->persistent_id = KSMBD_NO_FID; @@ -774,19 +1082,64 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp) return fp; err_out: + /* fp->conn was set and refcounted before every branch here. */ + ksmbd_conn_put(fp->conn); kmem_cache_free(filp_cache, fp); return ERR_PTR(ret); } -void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, - unsigned int state) +/** + * ksmbd_update_fstate() - update an fp state under the file-table lock + * @ft: file table that publishes @fp's volatile id + * @fp: file pointer to update + * @state: new state + * + * Return: 0 on success. The FP_NEW -> FP_INITED transition is special: + * -ENOENT if teardown already unpublished @fp by advancing the state or + * clearing the volatile id. Other state updates preserve the historical + * fire-and-forget behavior. + */ +int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, + unsigned int state) { + int ret; + if (!fp) - return; + return -ENOENT; write_lock(&ft->lock); - fp->f_state = state; + if (state == FP_INITED && + (fp->f_state != FP_NEW || !has_file_id(fp->volatile_id))) { + ret = -ENOENT; + } else { + fp->f_state = state; + ret = 0; + } write_unlock(&ft->lock); + + return ret; +} + +/* + * ksmbd_mark_fp_closed() - mark fp closed under ft->lock and return how many + * refs the teardown path owns. + * + * FP_INITED has a normal idr-owned reference, so teardown owns both that + * reference and the transient lookup reference. FP_NEW is still owned by the + * in-flight opener/reopener, which will drop the original reference after + * ksmbd_update_fstate(..., FP_INITED) observes the cleared volatile id. + * FP_CLOSED on entry means an earlier ksmbd_close_fd() already consumed the + * idr-owned ref. + */ +static int ksmbd_mark_fp_closed(struct ksmbd_file *fp) +{ + if (fp->f_state == FP_INITED) { + set_close_state_blocked_works(fp); + fp->f_state = FP_CLOSED; + return 2; + } + + return 1; } static int @@ -794,7 +1147,8 @@ __close_file_table_ids(struct ksmbd_session *sess, struct ksmbd_tree_connect *tcon, bool (*skip)(struct ksmbd_tree_connect *tcon, struct ksmbd_file *fp, - struct ksmbd_user *user)) + struct ksmbd_user *user), + bool skip_preserves_fp) { struct ksmbd_file_table *ft = &sess->file_table; struct ksmbd_file *fp; @@ -802,32 +1156,121 @@ __close_file_table_ids(struct ksmbd_session *sess, int num = 0; while (1) { + int n_to_drop; + write_lock(&ft->lock); fp = idr_get_next(ft->idr, &id); if (!fp) { write_unlock(&ft->lock); break; } - - if (skip(tcon, fp, sess->user) || - !atomic_dec_and_test(&fp->refcount)) { + if (!atomic_inc_not_zero(&fp->refcount)) { id++; write_unlock(&ft->lock); continue; } - set_close_state_blocked_works(fp); - idr_remove(ft->idr, fp->volatile_id); - fp->volatile_id = KSMBD_NO_FID; - write_unlock(&ft->lock); + if (skip_preserves_fp) { + /* + * Session teardown: skip() is session_fd_check(), + * which may sleep and mutates fp->conn / fp->tcon / + * fp->volatile_id when it chooses to preserve fp + * for durable reconnect. Unpublish fp from the + * session idr here, under ft->lock, so that + * __ksmbd_lookup_fd() through this session cannot + * grant a new ksmbd_fp_get() reference to an fp + * whose fields are about to be rewritten outside + * the lock. Durable reconnect still reaches fp via + * global_ft. + */ + idr_remove(ft->idr, id); + fp->durable_volatile_id = fp->volatile_id; + fp->volatile_id = KSMBD_NO_FID; + write_unlock(&ft->lock); + + if (skip(tcon, fp, sess->user)) { + /* + * session_fd_check() has converted fp to + * durable-preserve state and cleared its + * per-conn fields. fp is already unpublished + * above; the original idr-owned ref keeps it + * alive for the durable scavenger. Drop only + * the transient ref. atomic_dec() is safe -- + * atomic_inc_not_zero() succeeded on a + * positive value and we added one more, so + * refcount cannot be zero here. + */ + atomic_dec(&fp->refcount); + id++; + continue; + } + /* + * Keep the close-state decision under the same lock + * observed by ksmbd_update_fstate(), which is how an + * in-flight FP_NEW opener learns that teardown has + * cleared its volatile id. + */ + write_lock(&ft->lock); + n_to_drop = ksmbd_mark_fp_closed(fp); + write_unlock(&ft->lock); + } else { + /* + * Tree teardown: skip() is tree_conn_fd_check(), a + * cheap pointer compare that doesn't sleep and has + * no side effects, so keep the skip decision plus + * the unpublish-and-mark-closed sequence atomic + * under ft->lock. fps belonging to other tree + * connects (skip() == true) stay fully published in + * the session idr with no lock window. + */ + if (skip(tcon, fp, sess->user)) { + atomic_dec(&fp->refcount); + write_unlock(&ft->lock); + id++; + continue; + } + idr_remove(ft->idr, id); + fp->volatile_id = KSMBD_NO_FID; + n_to_drop = ksmbd_mark_fp_closed(fp); + write_unlock(&ft->lock); + } + + /* + * fp->volatile_id is already cleared to prevent stale idr + * removal from a deferred final close. Remove fp from + * m_fp_list here because __ksmbd_remove_fd() will skip the + * list unlink when volatile_id is KSMBD_NO_FID. + */ down_write(&fp->f_ci->m_lock); list_del_init(&fp->node); up_write(&fp->f_ci->m_lock); - __ksmbd_close_fd(ft, fp); - - num++; + /* + * Drop the references this iteration owns: + * + * n_to_drop == 2: we observed FP_INITED and committed + * the FP_CLOSED transition ourselves, so we own the + * transient (+1) and the still-intact idr-owned ref. + * + * n_to_drop == 1: either a prior ksmbd_close_fd() + * already consumed the idr-owned ref, or fp was still + * FP_NEW and the in-flight opener/reopener must keep + * the original reference until ksmbd_update_fstate() + * observes the cleared volatile id. + * + * If we end up as the final putter, finalize fp and + * account the open_files_count decrement via the caller's + * atomic_sub(num, ...). Otherwise the remaining user's + * ksmbd_fd_put() reaches __put_fd_final(), which does its + * own atomic_dec(&open_files_count), so we must not count + * this fp here -- doing so would double-decrement the + * connection-wide counter. + */ + if (atomic_sub_and_test(n_to_drop, &fp->refcount)) { + __ksmbd_close_fd(NULL, fp); + num++; + } id++; } @@ -881,24 +1324,37 @@ static bool ksmbd_durable_scavenger_alive(void) return true; } -static void ksmbd_scavenger_dispose_dh(struct list_head *head) +static void ksmbd_scavenger_dispose_dh(struct ksmbd_file *fp) { - while (!list_empty(head)) { - struct ksmbd_file *fp; + /* + * Durable-preserved fp can remain linked on f_ci->m_fp_list for + * share-mode checks. Unlink it before final close; fp->node is not + * available as a scavenger-private list node because re-adding it to + * another list corrupts m_fp_list. + */ + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); - fp = list_first_entry(head, struct ksmbd_file, node); - list_del_init(&fp->node); + /* + * Drop both the durable lifetime reference and the transient reference + * taken by the scavenger under global_ft.lock. If a concurrent + * ksmbd_lookup_fd_inode() (or any other m_fp_list walker) snatched fp + * before the unlink above, that holder owns the final close via + * ksmbd_fd_put() -> __ksmbd_close_fd(). Otherwise the scavenger is + * the last putter and finalises fp here. + */ + if (atomic_sub_and_test(2, &fp->refcount)) __ksmbd_close_fd(NULL, fp); - } } static int ksmbd_durable_scavenger(void *dummy) { struct ksmbd_file *fp = NULL; + struct ksmbd_file *expired_fp; unsigned int id; unsigned int min_timeout = 1; bool found_fp_timeout; - LIST_HEAD(scavenger_list); unsigned long remaining_jiffies; __module_get(THIS_MODULE); @@ -908,33 +1364,47 @@ static int ksmbd_durable_scavenger(void *dummy) if (try_to_freeze()) continue; - found_fp_timeout = false; - - remaining_jiffies = wait_event_timeout(dh_wq, + remaining_jiffies = wait_event_interruptible_timeout(dh_wq, ksmbd_durable_scavenger_alive() == false, __msecs_to_jiffies(min_timeout)); - if (remaining_jiffies) + if ((long)remaining_jiffies > 0) min_timeout = jiffies_to_msecs(remaining_jiffies); else min_timeout = DURABLE_HANDLE_MAX_TIMEOUT; - write_lock(&global_ft.lock); - idr_for_each_entry(global_ft.idr, fp, id) { - if (!fp->durable_timeout) - continue; - - if (atomic_read(&fp->refcount) > 1 || - fp->conn) - continue; + do { + expired_fp = NULL; + found_fp_timeout = false; - found_fp_timeout = true; - if (fp->durable_scavenger_timeout <= - jiffies_to_msecs(jiffies)) { - __ksmbd_remove_durable_fd(fp); - list_add(&fp->node, &scavenger_list); - } else { + write_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { unsigned long durable_timeout; + if (!fp->durable_timeout) + continue; + + if (atomic_read(&fp->refcount) > 1 || + fp->conn) + continue; + + found_fp_timeout = true; + if (fp->durable_scavenger_timeout <= + jiffies_to_msecs(jiffies)) { + __ksmbd_remove_durable_fd(fp); + /* + * Take a transient reference so fp + * cannot be freed by an in-flight + * ksmbd_lookup_fd_inode() that found + * it through f_ci->m_fp_list while we + * drop global_ft.lock and reach the + * m_fp_list unlink in + * ksmbd_scavenger_dispose_dh(). + */ + atomic_inc(&fp->refcount); + expired_fp = fp; + break; + } + durable_timeout = fp->durable_scavenger_timeout - jiffies_to_msecs(jiffies); @@ -942,10 +1412,11 @@ static int ksmbd_durable_scavenger(void *dummy) if (min_timeout > durable_timeout) min_timeout = durable_timeout; } - } - write_unlock(&global_ft.lock); + write_unlock(&global_ft.lock); - ksmbd_scavenger_dispose_dh(&scavenger_list); + if (expired_fp) + ksmbd_scavenger_dispose_dh(expired_fp); + } while (expired_fp); if (found_fp_timeout == false) break; @@ -1010,16 +1481,21 @@ void ksmbd_stop_durable_scavenger(void) static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user) { + char *name; + if (!user) return -EINVAL; /* Duplicate the user name to ensure identity persistence */ - fp->owner.name = kstrdup(user->name, GFP_KERNEL); - if (!fp->owner.name) + name = kstrdup(user->name, GFP_KERNEL); + if (!name) return -ENOMEM; + spin_lock(&fp->f_lock); fp->owner.uid = user->uid; fp->owner.gid = user->gid; + fp->owner.name = name; + spin_unlock(&fp->f_lock); return 0; } @@ -1037,18 +1513,24 @@ static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user) { - if (!user || !fp->owner.name) + bool ret = false; + + if (!user) return false; + spin_lock(&fp->f_lock); + if (!fp->owner.name) + goto out; + /* Check if the UID and GID match first (fast path) */ if (fp->owner.uid != user->uid || fp->owner.gid != user->gid) - return false; + goto out; /* Validate the account name to ensure the same SecurityContext */ - if (strcmp(fp->owner.name, user->name)) - return false; - - return true; + ret = (strcmp(fp->owner.name, user->name) == 0); +out: + spin_unlock(&fp->f_lock); + return ret; } static bool session_fd_check(struct ksmbd_tree_connect *tcon, @@ -1062,25 +1544,43 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, if (!is_reconnectable(fp)) return false; + if (fp->f_state != FP_INITED) + return false; + + if (WARN_ON_ONCE(!fp->conn)) + return false; + if (ksmbd_vfs_copy_durable_owner(fp, user)) return false; + /* + * fp owns a strong reference on fp->conn (taken in ksmbd_open_fd() + * / ksmbd_reopen_durable_fd()), so conn stays valid for the whole + * body of this function regardless of any op->conn puts below. + */ conn = fp->conn; ci = fp->f_ci; down_write(&ci->m_lock); - list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) { + list_for_each_entry_rcu(op, &ci->m_op_list, op_entry, + lockdep_is_held(&ci->m_lock)) { if (op->conn != conn) continue; - if (op->conn && atomic_dec_and_test(&op->conn->refcnt)) - kfree(op->conn); + ksmbd_conn_put(op->conn); op->conn = NULL; + op->sess = NULL; } up_write(&ci->m_lock); list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - spin_lock(&fp->conn->llist_lock); + struct ksmbd_conn *lock_conn = smb_lock->conn; + + if (!lock_conn) + continue; + spin_lock(&lock_conn->llist_lock); list_del_init(&smb_lock->clist); - spin_unlock(&fp->conn->llist_lock); + smb_lock->conn = NULL; + spin_unlock(&lock_conn->llist_lock); + ksmbd_conn_put(lock_conn); } fp->conn = NULL; @@ -1091,6 +1591,8 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, fp->durable_scavenger_timeout = jiffies_to_msecs(jiffies) + fp->durable_timeout; + /* Drop fp's own reference on conn. */ + ksmbd_conn_put(conn); return true; } @@ -1098,7 +1600,8 @@ void ksmbd_close_tree_conn_fds(struct ksmbd_work *work) { int num = __close_file_table_ids(work->sess, work->tcon, - tree_conn_fd_check); + tree_conn_fd_check, + false); atomic_sub(num, &work->conn->stats.open_files_count); } @@ -1107,7 +1610,8 @@ void ksmbd_close_session_fds(struct ksmbd_work *work) { int num = __close_file_table_ids(work->sess, work->tcon, - session_fd_check); + session_fd_check, + true); atomic_sub(num, &work->conn->stats.open_files_count); } @@ -1166,28 +1670,47 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) struct ksmbd_lock *smb_lock; unsigned int old_f_state; + write_lock(&global_ft.lock); if (!fp->is_durable || fp->conn || fp->tcon) { + write_unlock(&global_ft.lock); pr_err("Invalid durable fd [%p:%p]\n", fp->conn, fp->tcon); return -EBADF; } if (has_file_id(fp->volatile_id)) { + write_unlock(&global_ft.lock); pr_err("Still in use durable fd: %llu\n", fp->volatile_id); return -EBADF; } + /* + * Initialize fp's connection binding before publishing fp into the + * session's file table. If __open_id() is ordered first, a + * concurrent teardown that iterates the table can observe a valid + * volatile_id with fp->conn == NULL and preserve a + * partially-initialized fp. fp owns a strong reference on the new + * conn (see ksmbd_open_fd()); undo it on __open_id() failure. + */ + fp->conn = ksmbd_conn_get(conn); + fp->tcon = work->tcon; + write_unlock(&global_ft.lock); + old_f_state = fp->f_state; fp->f_state = FP_NEW; + __open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID); if (!has_file_id(fp->volatile_id)) { + write_lock(&global_ft.lock); + fp->conn = NULL; + fp->tcon = NULL; + write_unlock(&global_ft.lock); + ksmbd_conn_put(conn); fp->f_state = old_f_state; return -EBADF; } - fp->conn = conn; - fp->tcon = work->tcon; - list_for_each_entry(smb_lock, &fp->lock_list, flist) { + smb_lock->conn = ksmbd_conn_get(conn); spin_lock(&conn->llist_lock); list_add_tail(&smb_lock->clist, &conn->lock_list); spin_unlock(&conn->llist_lock); @@ -1195,17 +1718,20 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) ci = fp->f_ci; down_write(&ci->m_lock); - list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) { + list_for_each_entry_rcu(op, &ci->m_op_list, op_entry, + lockdep_is_held(&ci->m_lock)) { if (op->conn) continue; - op->conn = fp->conn; - atomic_inc(&op->conn->refcnt); + op->conn = ksmbd_conn_get(fp->conn); + op->sess = work->sess; } up_write(&ci->m_lock); + spin_lock(&fp->f_lock); fp->owner.uid = fp->owner.gid = 0; kfree(fp->owner.name); fp->owner.name = NULL; + spin_unlock(&fp->f_lock); return 0; } @@ -1228,7 +1754,7 @@ void ksmbd_destroy_file_table(struct ksmbd_session *sess) if (!ft->idr) return; - __close_file_table_ids(sess, NULL, session_fd_check); + __close_file_table_ids(sess, NULL, session_fd_check, true); idr_destroy(ft->idr); kfree(ft->idr); ft->idr = NULL; diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index 866f32c10d4d..b9e27307a26c 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -8,6 +8,7 @@ #include <linux/file.h> #include <linux/fs.h> +#include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/spinlock.h> #include <linux/idr.h> @@ -22,7 +23,12 @@ #define FILE_GENERIC_WRITE 0x120116 #define FILE_GENERIC_EXECUTE 0X1200a0 -#define KSMBD_START_FID 0 +/* + * Start volatile/persistent file id allocation at 1. A file id of 0 yields an + * SMB2 FileId of {0, 0}, which clients (e.g. Windows, Samba) treat as a null + * handle and never close, leaking the open on the server. + */ +#define KSMBD_START_FID 1 #define KSMBD_NO_FID (INT_MAX) #define SMB2_NO_FID (0xFFFFFFFFFFFFFFFFULL) @@ -31,6 +37,7 @@ struct ksmbd_session; struct ksmbd_lock { struct file_lock *fl; + struct ksmbd_conn *conn; struct list_head clist; struct list_head flist; struct list_head llist; @@ -79,6 +86,7 @@ struct ksmbd_file { struct file *filp; u64 persistent_id; u64 volatile_id; + u64 durable_volatile_id; spinlock_t f_lock; @@ -94,7 +102,10 @@ struct ksmbd_file { __le32 coption; __le32 cdoption; __u64 create_time; + __u64 change_time; + __u64 allocation_size; __u64 itime; + __u64 open_mtime; bool is_nt_open; bool attrib_only; @@ -113,12 +124,14 @@ struct ksmbd_file { /* if ls is happening on directory, below is valid*/ struct ksmbd_readdir_data readdir_data; + struct mutex readdir_lock; int dot_dotdot[2]; unsigned int f_state; bool reserve_lease_break; bool is_durable; bool is_persistent; bool is_resilient; + bool durable_reconnect_disabled; bool is_posix_ctxt; struct durable_owner owner; @@ -157,11 +170,17 @@ struct ksmbd_file *ksmbd_lookup_fd_slow(struct ksmbd_work *work, u64 id, void ksmbd_fd_put(struct ksmbd_work *work, struct ksmbd_file *fp); struct ksmbd_inode *ksmbd_inode_lookup_lock(struct dentry *d); void ksmbd_inode_put(struct ksmbd_inode *ci); +bool ksmbd_close_disconnected_durable_delete_on_close(struct dentry *dentry); struct ksmbd_file *ksmbd_lookup_global_fd(unsigned long long id); struct ksmbd_file *ksmbd_lookup_durable_fd(unsigned long long id); void ksmbd_put_durable_fd(struct ksmbd_file *fp); +int ksmbd_invalidate_durable_fd(unsigned long long id); +bool ksmbd_has_other_active_fd(struct ksmbd_file *fp); +bool ksmbd_has_stream_without_delete_share(struct ksmbd_file *fp); +int ksmbd_close_fd_app_instance_id(char *app_instance_id); struct ksmbd_file *ksmbd_lookup_fd_cguid(char *cguid); struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry); +bool ksmbd_has_open_files(struct dentry *dentry); unsigned int ksmbd_open_durable_fd(struct ksmbd_file *fp); struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp); void ksmbd_launch_ksmbd_durable_scavenger(void); @@ -172,8 +191,8 @@ int ksmbd_close_inode_fds(struct ksmbd_work *work, struct inode *inode); int ksmbd_init_global_file_table(void); void ksmbd_free_global_file_table(void); void ksmbd_set_fd_limit(unsigned long limit); -void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, - unsigned int state); +int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, + unsigned int state); bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user); diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c index 704b271af3a8..529740005838 100644 --- a/fs/smb/smbdirect/accept.c +++ b/fs/smb/smbdirect/accept.c @@ -854,4 +854,4 @@ struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, return nsc; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_accept); +EXPORT_SYMBOL_GPL(smbdirect_socket_accept); diff --git a/fs/smb/smbdirect/connect.c b/fs/smb/smbdirect/connect.c index 8addee43a381..cd726b399afe 100644 --- a/fs/smb/smbdirect/connect.c +++ b/fs/smb/smbdirect/connect.c @@ -60,7 +60,7 @@ int smbdirect_connect(struct smbdirect_socket *sc, const struct sockaddr *dst) */ return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect); +EXPORT_SYMBOL_GPL(smbdirect_connect); static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc) { @@ -922,4 +922,4 @@ int smbdirect_connect_sync(struct smbdirect_socket *sc, return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect_sync); +EXPORT_SYMBOL_GPL(smbdirect_connect_sync); diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c index 822366718d45..8adf58097534 100644 --- a/fs/smb/smbdirect/connection.c +++ b/fs/smb/smbdirect/connection.c @@ -706,7 +706,7 @@ bool smbdirect_connection_is_connected(struct smbdirect_socket *sc) return false; return true; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_is_connected); +EXPORT_SYMBOL_GPL(smbdirect_connection_is_connected); int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc) { @@ -779,7 +779,7 @@ int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc) return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_wait_for_connected); +EXPORT_SYMBOL_GPL(smbdirect_connection_wait_for_connected); void smbdirect_connection_idle_timer_work(struct work_struct *work) { @@ -958,7 +958,7 @@ release_credit: return ret; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_batch_flush); +EXPORT_SYMBOL_GPL(smbdirect_connection_send_batch_flush); struct smbdirect_send_batch * smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, @@ -976,7 +976,7 @@ smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, return batch; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_init_send_batch_storage); +EXPORT_SYMBOL_GPL(smbdirect_init_send_batch_storage); static int smbdirect_connection_wait_for_send_bcredit(struct smbdirect_socket *sc, struct smbdirect_send_batch *batch) @@ -1263,7 +1263,7 @@ lcredit_failed: bcredit_failed: return ret; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_single_iter); +EXPORT_SYMBOL_GPL(smbdirect_connection_send_single_iter); int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc) { @@ -1288,7 +1288,7 @@ int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc) return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_wait_zero_pending); +EXPORT_SYMBOL_GPL(smbdirect_connection_send_wait_zero_pending); int smbdirect_connection_send_iter(struct smbdirect_socket *sc, struct iov_iter *iter, @@ -1373,7 +1373,7 @@ int smbdirect_connection_send_iter(struct smbdirect_socket *sc, return total_count; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_iter); +EXPORT_SYMBOL_GPL(smbdirect_connection_send_iter); static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc) { @@ -1937,7 +1937,7 @@ read_rfc1002_done: goto again; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_recvmsg); +EXPORT_SYMBOL_GPL(smbdirect_connection_recvmsg); static bool smbdirect_map_sges_single_page(struct smbdirect_map_sges *state, struct page *page, size_t off, size_t len) @@ -2168,7 +2168,7 @@ static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len, if (ret < 0) { while (state->num_sge > before) { - struct ib_sge *sge = &state->sge[state->num_sge--]; + struct ib_sge *sge = &state->sge[--state->num_sge]; ib_dma_unmap_page(state->device, sge->addr, diff --git a/fs/smb/smbdirect/debug.c b/fs/smb/smbdirect/debug.c index a66a19d4a463..3445843445bf 100644 --- a/fs/smb/smbdirect/debug.c +++ b/fs/smb/smbdirect/debug.c @@ -40,7 +40,7 @@ void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, seq_puts(m, "\n"); seq_printf(m, "Conn keep_alive_interval: %u ", - sp->keepalive_interval_msec * 1000); + sp->keepalive_interval_msec / 1000); seq_printf(m, "max_readwrite_size: %u rdma_readwrite_threshold: %u", sp->max_read_write_size, rdma_readwrite_threshold); @@ -85,4 +85,4 @@ void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, atomic_read(&sc->mr_io.ready.count), atomic_read(&sc->mr_io.used.count)); } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_legacy_debug_proc_show); +EXPORT_SYMBOL_GPL(smbdirect_connection_legacy_debug_proc_show); diff --git a/fs/smb/smbdirect/devices.c b/fs/smb/smbdirect/devices.c index 44962f221c35..7adacbdfe12e 100644 --- a/fs/smb/smbdirect/devices.c +++ b/fs/smb/smbdirect/devices.c @@ -238,7 +238,7 @@ u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev) return RDMA_NODE_UNSPECIFIED; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_netdev_rdma_capable_node_type); +EXPORT_SYMBOL_GPL(smbdirect_netdev_rdma_capable_node_type); __init int smbdirect_devices_init(void) { diff --git a/fs/smb/smbdirect/internal.h b/fs/smb/smbdirect/internal.h index 2d5acf2c21bc..e9959e6dc13a 100644 --- a/fs/smb/smbdirect/internal.h +++ b/fs/smb/smbdirect/internal.h @@ -6,11 +6,11 @@ #ifndef __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ #define __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ +#define DEFAULT_SYMBOL_NAMESPACE "SMBDIRECT" #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include "smbdirect.h" +#include <linux/smbdirect.h> #include "pdu.h" -#include "public.h" #include <linux/mutex.h> diff --git a/fs/smb/smbdirect/listen.c b/fs/smb/smbdirect/listen.c index 143a7618d95f..2f78bcaedbf8 100644 --- a/fs/smb/smbdirect/listen.c +++ b/fs/smb/smbdirect/listen.c @@ -90,7 +90,7 @@ int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog) */ return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_listen); +EXPORT_SYMBOL_GPL(smbdirect_socket_listen); static int smbdirect_new_rdma_event_handler(struct rdma_cm_id *new_id, struct rdma_cm_event *event) diff --git a/fs/smb/smbdirect/mr.c b/fs/smb/smbdirect/mr.c index 5228e699cd5d..15c6363a2f97 100644 --- a/fs/smb/smbdirect/mr.c +++ b/fs/smb/smbdirect/mr.c @@ -269,7 +269,7 @@ smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, { const struct smbdirect_socket_parameters *sp = &sc->parameters; struct smbdirect_mr_io *mr; - int ret, num_pages; + int ret, num_pages, num_mapped; struct ib_reg_wr *reg_wr; num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1); @@ -300,19 +300,22 @@ smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, num_pages, iov_iter_count(iter), sp->max_frmr_depth); smbdirect_iter_to_sgt(iter, &mr->sgt, sp->max_frmr_depth); - ret = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); - if (!ret) { + num_mapped = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + if (!num_mapped) { smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "ib_dma_map_sg num_pages=%u dir=%x ret=%d (%1pe)\n", - num_pages, mr->dir, ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + "ib_dma_map_sg num_pages=%u dir=%x num_mapped=%d\n", + num_pages, mr->dir, num_mapped); + ret = -EIO; goto dma_map_error; } - ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, mr->sgt.nents, NULL, PAGE_SIZE); - if (ret != mr->sgt.nents) { + ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, num_mapped, NULL, PAGE_SIZE); + if (ret != num_mapped) { smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "ib_map_mr_sg failed ret = %d nents = %u\n", - ret, mr->sgt.nents); + "ib_map_mr_sg failed ret = %d num_mapped = %u\n", + ret, num_mapped); + if (ret >= 0) + ret = -EIO; goto map_mr_error; } @@ -380,7 +383,7 @@ dma_map_error: mutex_unlock(&mr->mutex); return NULL; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_register_mr_io); +EXPORT_SYMBOL_GPL(smbdirect_connection_register_mr_io); void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, struct smbdirect_buffer_descriptor_v1 *v1) @@ -397,7 +400,7 @@ void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, } mutex_unlock(&mr->mutex); } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_mr_io_fill_buffer_descriptor); +EXPORT_SYMBOL_GPL(smbdirect_mr_io_fill_buffer_descriptor); /* * Deregister a MR after I/O is done @@ -490,4 +493,4 @@ put_kref: if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) mutex_unlock(&mr->mutex); } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_deregister_mr_io); +EXPORT_SYMBOL_GPL(smbdirect_connection_deregister_mr_io); diff --git a/fs/smb/smbdirect/public.h b/fs/smb/smbdirect/public.h deleted file mode 100644 index 50088155e7c3..000000000000 --- a/fs/smb/smbdirect/public.h +++ /dev/null @@ -1,148 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2025, Stefan Metzmacher - */ - -#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ -#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ - -struct smbdirect_buffer_descriptor_v1; -struct smbdirect_socket_parameters; - -struct smbdirect_socket; -struct smbdirect_send_batch; -struct smbdirect_mr_io; - -#define __SMBDIRECT_EXPORT_SYMBOL__(__sym) EXPORT_SYMBOL_FOR_MODULES(__sym, "cifs,ksmbd") - -#include <rdma/rw.h> - -u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev); - -bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs); - -int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc); - -int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc); - -int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, - const struct smbdirect_socket_parameters *sp); - -const struct smbdirect_socket_parameters * -smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc); - -int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, - enum ib_poll_context poll_ctx, - gfp_t gfp_mask); - -#define SMBDIRECT_LOG_ERR 0x0 -#define SMBDIRECT_LOG_INFO 0x1 - -#define SMBDIRECT_LOG_OUTGOING 0x1 -#define SMBDIRECT_LOG_INCOMING 0x2 -#define SMBDIRECT_LOG_READ 0x4 -#define SMBDIRECT_LOG_WRITE 0x8 -#define SMBDIRECT_LOG_RDMA_SEND 0x10 -#define SMBDIRECT_LOG_RDMA_RECV 0x20 -#define SMBDIRECT_LOG_KEEP_ALIVE 0x40 -#define SMBDIRECT_LOG_RDMA_EVENT 0x80 -#define SMBDIRECT_LOG_RDMA_MR 0x100 -#define SMBDIRECT_LOG_RDMA_RW 0x200 -#define SMBDIRECT_LOG_NEGOTIATE 0x400 -void smbdirect_socket_set_logging(struct smbdirect_socket *sc, - void *private_ptr, - bool (*needed)(struct smbdirect_socket *sc, - void *private_ptr, - unsigned int lvl, - unsigned int cls), - void (*vaprintf)(struct smbdirect_socket *sc, - const char *func, - unsigned int line, - void *private_ptr, - unsigned int lvl, - unsigned int cls, - struct va_format *vaf)); - -bool smbdirect_connection_is_connected(struct smbdirect_socket *sc); - -int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc); - -int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr); - -void smbdirect_socket_shutdown(struct smbdirect_socket *sc); - -void smbdirect_socket_release(struct smbdirect_socket *sc); - -int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch, - bool is_last); - -/* - * This is only temporary and only needed - * as long as the client still requires - * to use smbdirect_connection_send_single_iter() - */ -struct smbdirect_send_batch_storage { - union { - struct list_head __msg_list; - __aligned_u64 __space[5]; - }; -}; - -struct smbdirect_send_batch * -smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, - bool need_invalidate_rkey, - unsigned int remote_key); - -int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch, - struct iov_iter *iter, - unsigned int flags, - u32 remaining_data_length); - -int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc); - -int smbdirect_connection_send_iter(struct smbdirect_socket *sc, - struct iov_iter *iter, - unsigned int flags, - bool need_invalidate, - unsigned int remote_key); - -int smbdirect_connection_recvmsg(struct smbdirect_socket *sc, - struct msghdr *msg, - unsigned int flags); - -int smbdirect_connect(struct smbdirect_socket *sc, - const struct sockaddr *dst); - -int smbdirect_connect_sync(struct smbdirect_socket *sc, - const struct sockaddr *dst); - -int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog); - -struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, - long timeo, - struct proto_accept_arg *arg); - -int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc, - void *buf, size_t buf_len, - struct smbdirect_buffer_descriptor_v1 *desc, - size_t desc_len, - bool is_read); - -struct smbdirect_mr_io * -smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, - struct iov_iter *iter, - bool writing, - bool need_invalidate); - -void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, - struct smbdirect_buffer_descriptor_v1 *v1); - -void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr); - -void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, - unsigned int rdma_readwrite_threshold, - struct seq_file *m); - -#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ */ diff --git a/fs/smb/smbdirect/rw.c b/fs/smb/smbdirect/rw.c index c2f46b17731e..6fe38042cfb9 100644 --- a/fs/smb/smbdirect/rw.c +++ b/fs/smb/smbdirect/rw.c @@ -252,4 +252,4 @@ free_msg: kfree(msg); goto out; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_rdma_xmit); +EXPORT_SYMBOL_GPL(smbdirect_connection_rdma_xmit); diff --git a/fs/smb/smbdirect/smbdirect.h b/fs/smb/smbdirect/smbdirect.h deleted file mode 100644 index bbab5f7f7cc9..000000000000 --- a/fs/smb/smbdirect/smbdirect.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2025 Stefan Metzmacher - */ - -#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ -#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ - -#include <linux/types.h> - -/* SMB-DIRECT buffer descriptor V1 structure [MS-SMBD] 2.2.3.1 */ -struct smbdirect_buffer_descriptor_v1 { - __le64 offset; - __le32 token; - __le32 length; -} __packed; - -/* - * Connection parameters mostly from [MS-SMBD] 3.1.1.1 - * - * These are setup and negotiated at the beginning of a - * connection and remain constant unless explicitly changed. - * - * Some values are important for the upper layer. - */ -struct smbdirect_socket_parameters { - __u64 flags; -#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB ((__u64)0x1) -#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW ((__u64)0x2) - __u32 resolve_addr_timeout_msec; - __u32 resolve_route_timeout_msec; - __u32 rdma_connect_timeout_msec; - __u32 negotiate_timeout_msec; - __u16 initiator_depth; /* limited to U8_MAX */ - __u16 responder_resources; /* limited to U8_MAX */ - __u16 recv_credit_max; - __u16 send_credit_target; - __u32 max_send_size; - __u32 max_fragmented_send_size; - __u32 max_recv_size; - __u32 max_fragmented_recv_size; - __u32 max_read_write_size; - __u32 max_frmr_depth; - __u32 keepalive_interval_msec; - __u32 keepalive_timeout_msec; -} __packed; - -#define SMBDIRECT_FLAG_PORT_RANGE_MASK ( \ - SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB | \ - SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) - -#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ */ diff --git a/fs/smb/smbdirect/socket.c b/fs/smb/smbdirect/socket.c index 1b4ab01b745e..39cca7219c4d 100644 --- a/fs/smb/smbdirect/socket.c +++ b/fs/smb/smbdirect/socket.c @@ -20,7 +20,7 @@ bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs) return false; return true; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_frwr_is_supported); +EXPORT_SYMBOL_GPL(smbdirect_frwr_is_supported); static void smbdirect_socket_cleanup_work(struct work_struct *work); @@ -107,7 +107,7 @@ init_failed: alloc_failed: return ret; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_kern); +EXPORT_SYMBOL_GPL(smbdirect_socket_create_kern); int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc) { @@ -148,7 +148,7 @@ init_failed: alloc_failed: return ret; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_accepting); +EXPORT_SYMBOL_GPL(smbdirect_socket_create_accepting); int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, const struct smbdirect_socket_parameters *sp) @@ -189,14 +189,14 @@ int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_initial_parameters); +EXPORT_SYMBOL_GPL(smbdirect_socket_set_initial_parameters); const struct smbdirect_socket_parameters * smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc) { return &sc->parameters; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_get_current_parameters); +EXPORT_SYMBOL_GPL(smbdirect_socket_get_current_parameters); int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, enum ib_poll_context poll_ctx, @@ -220,7 +220,7 @@ int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_kernel_settings); +EXPORT_SYMBOL_GPL(smbdirect_socket_set_kernel_settings); void smbdirect_socket_set_logging(struct smbdirect_socket *sc, void *private_ptr, @@ -240,7 +240,7 @@ void smbdirect_socket_set_logging(struct smbdirect_socket *sc, sc->logging.needed = needed; sc->logging.vaprintf = vaprintf; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_logging); +EXPORT_SYMBOL_GPL(smbdirect_socket_set_logging); static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc) { @@ -663,13 +663,13 @@ int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr) return 0; } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_bind); +EXPORT_SYMBOL_GPL(smbdirect_socket_bind); void smbdirect_socket_shutdown(struct smbdirect_socket *sc) { smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN); } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_shutdown); +EXPORT_SYMBOL_GPL(smbdirect_socket_shutdown); static void smbdirect_socket_release_disconnect(struct kref *kref) { @@ -712,7 +712,7 @@ void smbdirect_socket_release(struct smbdirect_socket *sc) */ kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy); } -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_release); +EXPORT_SYMBOL_GPL(smbdirect_socket_release); int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, enum smbdirect_socket_status expected_status, diff --git a/fs/super.c b/fs/super.c index 378e81efe643..a8fd61136aaf 100644 --- a/fs/super.c +++ b/fs/super.c @@ -328,7 +328,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, init_rwsem(&s->s_umount); lockdep_set_class(&s->s_umount, &type->s_umount_key); /* - * sget() can have s_umount recursion. + * sget_fc() can have s_umount recursion. * * When it cannot find a suitable sb, it allocates a new * one (this one), and tries again to find a suitable old @@ -359,6 +359,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, s->s_iflags |= SB_I_NODEV; INIT_HLIST_NODE(&s->s_instances); INIT_HLIST_BL_HEAD(&s->s_roots); + spin_lock_init(&s->s_roots_lock); mutex_init(&s->s_sync_lock); INIT_LIST_HEAD(&s->s_inodes); spin_lock_init(&s->s_inode_list_lock); @@ -439,7 +440,7 @@ static void kill_super_notify(struct super_block *sb) /* * Remove it from @fs_supers so it isn't found by new - * sget{_fc}() walkers anymore. Any concurrent mounter still + * sget_fc() walkers anymore. Any concurrent mounter still * managing to grab a temporary reference is guaranteed to * already see SB_DYING and will wait until we notify them about * SB_DEAD. @@ -517,7 +518,7 @@ EXPORT_SYMBOL(deactivate_super); * @sb: superblock to acquire * * Acquire a temporary reference on a superblock and try to trade it for - * an active reference. This is used in sget{_fc}() to wait for a + * an active reference. This is used in sget_fc() to wait for a * superblock to either become SB_BORN or for it to pass through * sb->kill() and be marked as SB_DEAD. * @@ -673,11 +674,11 @@ void generic_shutdown_super(struct super_block *sb) /* * Broadcast to everyone that grabbed a temporary reference to this * superblock before we removed it from @fs_supers that the superblock - * is dying. Every walker of @fs_supers outside of sget{_fc}() will now + * is dying. Every walker of @fs_supers outside of sget_fc() will now * discard this superblock and treat it as dead. * * We leave the superblock on @fs_supers so it can be found by - * sget{_fc}() until we passed sb->kill_sb(). + * sget_fc() until we passed sb->kill_sb(). */ super_wake(sb, SB_DYING); super_unlock_excl(sb); @@ -741,12 +742,13 @@ struct super_block *sget_fc(struct fs_context *fc, int err; /* - * Never allow s_user_ns != &init_user_ns when FS_USERNS_MOUNT is - * not set, as the filesystem is likely unprepared to handle it. - * This can happen when fsconfig() is called from init_user_ns with - * an fs_fd opened in another user namespace. + * Never allow s_user_ns != &init_user_ns when FS_USERNS_MOUNT or + * FS_USERNS_DELEGATABLE is not set, as the filesystem is likely + * unprepared to handle it. This can happen when fsconfig() is called + * from init_user_ns with an fs_fd opened in another user namespace. */ - if (user_ns != &init_user_ns && !(fc->fs_type->fs_flags & FS_USERNS_MOUNT)) { + if (user_ns != &init_user_ns && + !(fc->fs_type->fs_flags & (FS_USERNS_MOUNT | FS_USERNS_DELEGATABLE))) { errorfc(fc, "VFS: Mounting from non-initial user namespace is not allowed"); return ERR_PTR(-EPERM); } @@ -808,67 +810,6 @@ share_extant_sb: } EXPORT_SYMBOL(sget_fc); -/** - * sget - find or create a superblock - * @type: filesystem type superblock should belong to - * @test: comparison callback - * @set: setup callback - * @flags: mount flags - * @data: argument to each of them - */ -struct super_block *sget(struct file_system_type *type, - int (*test)(struct super_block *,void *), - int (*set)(struct super_block *,void *), - int flags, - void *data) -{ - struct user_namespace *user_ns = current_user_ns(); - struct super_block *s = NULL; - struct super_block *old; - int err; - -retry: - spin_lock(&sb_lock); - if (test) { - hlist_for_each_entry(old, &type->fs_supers, s_instances) { - if (!test(old, data)) - continue; - if (user_ns != old->s_user_ns) { - spin_unlock(&sb_lock); - destroy_unused_super(s); - return ERR_PTR(-EBUSY); - } - if (!grab_super(old)) - goto retry; - destroy_unused_super(s); - return old; - } - } - if (!s) { - spin_unlock(&sb_lock); - s = alloc_super(type, flags, user_ns); - if (!s) - return ERR_PTR(-ENOMEM); - goto retry; - } - - err = set(s, data); - if (err) { - spin_unlock(&sb_lock); - destroy_unused_super(s); - return ERR_PTR(err); - } - s->s_type = type; - strscpy(s->s_id, type->name, sizeof(s->s_id)); - list_add_tail(&s->s_list, &super_blocks); - hlist_add_head(&s->s_instances, &type->fs_supers); - spin_unlock(&sb_lock); - get_filesystem(type); - shrinker_register(s->s_shrink); - return s; -} -EXPORT_SYMBOL(sget); - void drop_super(struct super_block *sb) { super_unlock_shared(sb); @@ -882,7 +823,6 @@ void drop_super_exclusive(struct super_block *sb) super_unlock_excl(sb); put_super(sb); } -EXPORT_SYMBOL(drop_super_exclusive); enum super_iter_flags_t { SUPER_ITER_EXCL = (1U << 0), diff --git a/fs/sync.c b/fs/sync.c index 942a60cfedfb..4a84dd837b86 100644 --- a/fs/sync.c +++ b/fs/sync.c @@ -266,8 +266,7 @@ int sync_file_range(struct file *file, loff_t offset, loff_t nbytes, i_mode = file_inode(file)->i_mode; ret = -ESPIPE; - if (!S_ISREG(i_mode) && !S_ISBLK(i_mode) && !S_ISDIR(i_mode) && - !S_ISLNK(i_mode)) + if (!S_ISREG(i_mode) && !S_ISBLK(i_mode) && !S_ISDIR(i_mode)) goto out; mapping = file->f_mapping; diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c index 5709cede1d75..cd5bb0f9fee6 100644 --- a/fs/sysfs/file.c +++ b/fs/sysfs/file.c @@ -70,9 +70,8 @@ static int sysfs_kf_seq_show(struct seq_file *sf, void *v) * The code works fine with PAGE_SIZE return but it's likely to * indicate truncated result or overflow in normal use cases. */ - if (count >= (ssize_t)PAGE_SIZE) { - printk("fill_read_buffer: %pS returned bad count\n", - ops->show); + if (count >= PAGE_SIZE) { + WARN(1, "OOB write or bad count %zd at %pS\n", count, ops->show); /* Try to struggle along */ count = PAGE_SIZE - 1; } @@ -120,6 +119,10 @@ static ssize_t sysfs_kf_read(struct kernfs_open_file *of, char *buf, len = ops->show(kobj, of->kn->priv, buf); if (len < 0) return len; + if (len >= (ssize_t)PAGE_SIZE) { + printk("fill_read_buffer: %pS returned bad count\n", ops->show); + len = PAGE_SIZE - 1; + } if (pos) { if (len <= pos) return 0; diff --git a/fs/sysfs/group.c b/fs/sysfs/group.c index 182e54e575ee..4e1e4f18a166 100644 --- a/fs/sysfs/group.c +++ b/fs/sysfs/group.c @@ -188,7 +188,7 @@ static int internal_create_group(struct kobject *kobj, int update, kernfs_get(kn); error = create_files(kn, kobj, uid, gid, grp, update); if (error) { - if (grp->name) + if (grp->name && !update) kernfs_remove(kn); } kernfs_put(kn); diff --git a/fs/sysfs/mount.c b/fs/sysfs/mount.c index b199e8ff79b1..88c10823fcaf 100644 --- a/fs/sysfs/mount.c +++ b/fs/sysfs/mount.c @@ -23,20 +23,6 @@ static struct kernfs_root *sysfs_root; struct kernfs_node *sysfs_root_kn; -static int sysfs_get_tree(struct fs_context *fc) -{ - struct kernfs_fs_context *kfc = fc->fs_private; - int ret; - - ret = kernfs_get_tree(fc); - if (ret) - return ret; - - if (kfc->new_sb_created) - fc->root->d_sb->s_iflags |= SB_I_USERNS_VISIBLE; - return 0; -} - static void sysfs_fs_context_free(struct fs_context *fc) { struct kernfs_fs_context *kfc = fc->fs_private; @@ -49,7 +35,7 @@ static void sysfs_fs_context_free(struct fs_context *fc) static const struct fs_context_operations sysfs_fs_context_ops = { .free = sysfs_fs_context_free, - .get_tree = sysfs_get_tree, + .get_tree = kernfs_get_tree, }; static int sysfs_init_fs_context(struct fs_context *fc) @@ -93,7 +79,7 @@ static struct file_system_type sysfs_fs_type = { .name = "sysfs", .init_fs_context = sysfs_init_fs_context, .kill_sb = sysfs_kill_sb, - .fs_flags = FS_USERNS_MOUNT, + .fs_flags = FS_USERNS_MOUNT | FS_USERNS_MOUNT_RESTRICTED, }; int __init sysfs_init(void) diff --git a/fs/timerfd.c b/fs/timerfd.c index 73104f36bcae..fe845af0b74e 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -55,6 +55,15 @@ static inline bool isalarm(struct timerfd_ctx *ctx) ctx->clockid == CLOCK_BOOTTIME_ALARM; } +static void __timerfd_triggered(struct timerfd_ctx *ctx) +{ + lockdep_assert_held(&ctx->wqh.lock); + + ctx->expired = 1; + ctx->ticks++; + wake_up_locked_poll(&ctx->wqh, EPOLLIN); +} + /* * This gets called when the timer event triggers. We set the "expired" * flag, but we do not re-arm the timer (in case it's necessary, @@ -62,13 +71,8 @@ static inline bool isalarm(struct timerfd_ctx *ctx) */ static void timerfd_triggered(struct timerfd_ctx *ctx) { - unsigned long flags; - - spin_lock_irqsave(&ctx->wqh.lock, flags); - ctx->expired = 1; - ctx->ticks++; - wake_up_locked_poll(&ctx->wqh, EPOLLIN); - spin_unlock_irqrestore(&ctx->wqh.lock, flags); + guard(spinlock_irqsave)(&ctx->wqh.lock); + __timerfd_triggered(ctx); } static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr) @@ -184,15 +188,54 @@ static ktime_t timerfd_get_remaining(struct timerfd_ctx *ctx) return remaining < 0 ? 0: remaining; } +static void timerfd_alarm_start(struct timerfd_ctx *ctx, ktime_t exp, bool relative) +{ + /* Start the timer. If it's expired already, handle the callback. */ + if (!alarm_start_timer(&ctx->t.alarm, exp, relative)) + __timerfd_triggered(ctx); +} + +static u64 timerfd_alarm_restart(struct timerfd_ctx *ctx) +{ + /* -1 to account for ctx->ticks++ in __timerfd_triggered() */ + u64 ticks = alarm_forward_now(&ctx->t.alarm, ctx->tintv) - 1; + + timerfd_alarm_start(ctx, alarm_get_expires(&ctx->t.alarm), false); + return ticks; +} + +static void timerfd_hrtimer_start(struct timerfd_ctx *ctx, ktime_t exp, + const enum hrtimer_mode mode) +{ + /* Start the timer. If it's expired already, handle the callback. */ + if (!hrtimer_start_range_ns_user(&ctx->t.tmr, exp, 0, mode)) + __timerfd_triggered(ctx); +} + +static u64 timerfd_hrtimer_restart(struct timerfd_ctx *ctx) +{ + /* -1 to account for ctx->ticks++ in __timerfd_triggered() */ + u64 ticks = hrtimer_forward_now(&ctx->t.tmr, ctx->tintv) - 1; + + timerfd_hrtimer_start(ctx, hrtimer_get_expires(&ctx->t.tmr), HRTIMER_MODE_ABS); + return ticks; +} + +static u64 timerfd_restart(struct timerfd_ctx *ctx) +{ + if (isalarm(ctx)) + return timerfd_alarm_restart(ctx); + return timerfd_hrtimer_restart(ctx); +} + static int timerfd_setup(struct timerfd_ctx *ctx, int flags, const struct itimerspec64 *ktmr) { + int clockid = ctx->clockid; enum hrtimer_mode htmode; ktime_t texp; - int clockid = ctx->clockid; - htmode = (flags & TFD_TIMER_ABSTIME) ? - HRTIMER_MODE_ABS: HRTIMER_MODE_REL; + htmode = (flags & TFD_TIMER_ABSTIME) ? HRTIMER_MODE_ABS: HRTIMER_MODE_REL; texp = timespec64_to_ktime(ktmr->it_value); ctx->expired = 0; @@ -206,20 +249,15 @@ static int timerfd_setup(struct timerfd_ctx *ctx, int flags, timerfd_alarmproc); } else { hrtimer_setup(&ctx->t.tmr, timerfd_tmrproc, clockid, htmode); - hrtimer_set_expires(&ctx->t.tmr, texp); } if (texp != 0) { if (flags & TFD_TIMER_ABSTIME) texp = timens_ktime_to_host(clockid, texp); - if (isalarm(ctx)) { - if (flags & TFD_TIMER_ABSTIME) - alarm_start(&ctx->t.alarm, texp); - else - alarm_start_relative(&ctx->t.alarm, texp); - } else { - hrtimer_start(&ctx->t.tmr, texp, htmode); - } + if (isalarm(ctx)) + timerfd_alarm_start(ctx, texp, !(flags & TFD_TIMER_ABSTIME)); + else + timerfd_hrtimer_start(ctx, texp, htmode); if (timerfd_canceled(ctx)) return -ECANCELED; @@ -287,27 +325,19 @@ static ssize_t timerfd_read_iter(struct kiocb *iocb, struct iov_iter *to) } if (ctx->ticks) { - ticks = ctx->ticks; + unsigned int expired = ctx->expired; - if (ctx->expired && ctx->tintv) { - /* - * If tintv != 0, this is a periodic timer that - * needs to be re-armed. We avoid doing it in the timer - * callback to avoid DoS attacks specifying a very - * short timer period. - */ - if (isalarm(ctx)) { - ticks += alarm_forward_now( - &ctx->t.alarm, ctx->tintv) - 1; - alarm_restart(&ctx->t.alarm); - } else { - ticks += hrtimer_forward_now(&ctx->t.tmr, - ctx->tintv) - 1; - hrtimer_restart(&ctx->t.tmr); - } - } + ticks = ctx->ticks; ctx->expired = 0; ctx->ticks = 0; + + /* + * If tintv != 0, this is a periodic timer that needs to be + * re-armed. We avoid doing it in the timer callback to avoid + * DoS attacks specifying a very short timer period. + */ + if (expired && ctx->tintv) + ticks += timerfd_restart(ctx); } spin_unlock_irq(&ctx->wqh.lock); if (ticks) { @@ -526,18 +556,7 @@ static int do_timerfd_gettime(int ufd, struct itimerspec64 *t) spin_lock_irq(&ctx->wqh.lock); if (ctx->expired && ctx->tintv) { ctx->expired = 0; - - if (isalarm(ctx)) { - ctx->ticks += - alarm_forward_now( - &ctx->t.alarm, ctx->tintv) - 1; - alarm_restart(&ctx->t.alarm); - } else { - ctx->ticks += - hrtimer_forward_now(&ctx->t.tmr, ctx->tintv) - - 1; - hrtimer_restart(&ctx->t.tmr); - } + ctx->ticks += timerfd_restart(ctx); } t->it_value = ktime_to_timespec64(timerfd_get_remaining(ctx)); t->it_interval = ktime_to_timespec64(ctx->tintv); diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 26b6453de30e..39c7a34531e8 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -389,8 +389,7 @@ static struct dentry *lookup_file(struct eventfs_inode *parent_ei, // Files have their parent's ei as their fsdata dentry->d_fsdata = get_ei(parent_ei); - d_add(dentry, inode); - return NULL; + return d_splice_alias(inode, dentry); }; /** @@ -420,8 +419,7 @@ static struct dentry *lookup_dir_entry(struct dentry *dentry, dentry->d_fsdata = get_ei(ei); - d_add(dentry, inode); - return NULL; + return d_splice_alias(inode, dentry); } static inline struct eventfs_inode *init_ei(struct eventfs_inode *ei, const char *name) diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index 9a77d8b64ffa..38972786817e 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c @@ -358,7 +358,7 @@ static void ubifs_evict_inode(struct inode *inode) goto out; dbg_gen("inode %llu, mode %#x", inode->i_ino, (int)inode->i_mode); - ubifs_assert(c, !icount_read(inode)); + ubifs_assert(c, !icount_read_once(inode)); truncate_inode_pages_final(&inode->i_data); diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c index 807c493ed0cd..cc6dc6e1d84d 100644 --- a/fs/udf/balloc.c +++ b/fs/udf/balloc.c @@ -82,8 +82,9 @@ static int load_block_bitmap(struct super_block *sb, int nr_groups = bitmap->s_nr_groups; if (block_group >= nr_groups) { - udf_debug("block_group (%u) > nr_groups (%d)\n", + udf_debug("block_group (%u) >= nr_groups (%d)\n", block_group, nr_groups); + return -EFSCORRUPTED; } if (bitmap->s_block_bitmap[block_group]) { @@ -662,7 +663,7 @@ void udf_free_blocks(struct super_block *sb, struct inode *inode, if (check_add_overflow(bloc->logicalBlockNum, offset, &blk) || check_add_overflow(blk, count, &blk) || - bloc->logicalBlockNum + count > map->s_partition_len) { + blk > map->s_partition_len) { udf_debug("Invalid request to free blocks: (%d, %u), off %u, " "len %u, partition len %u\n", partition, bloc->logicalBlockNum, offset, count, diff --git a/fs/udf/super.c b/fs/udf/super.c index cad2e15d633b..7b85f5a2b79f 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1242,6 +1242,11 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) if (map->s_partition_type == UDF_VIRTUAL_MAP15) { map->s_type_specific.s_virtual.s_start_offset = 0; + if (sbi->s_vat_inode->i_size < 36) { + udf_err(sb, "Too short VAT inode size %lld\n", + sbi->s_vat_inode->i_size); + return -EFSCORRUPTED; + } map->s_type_specific.s_virtual.s_num_entries = (sbi->s_vat_inode->i_size - 36) >> 2; } else if (map->s_partition_type == UDF_VIRTUAL_MAP20) { @@ -1263,6 +1268,14 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) map->s_type_specific.s_virtual.s_start_offset = le16_to_cpu(vat20->lengthHeader); + if (map->s_type_specific.s_virtual.s_start_offset + > sbi->s_vat_inode->i_size) { + udf_err(sb, "Corrupted VAT header length %u (VAT inode size %lld)\n", + map->s_type_specific.s_virtual.s_start_offset, + sbi->s_vat_inode->i_size); + brelse(bh); + return -EFSCORRUPTED; + } map->s_type_specific.s_virtual.s_num_entries = (sbi->s_vat_inode->i_size - map->s_type_specific.s_virtual. @@ -1418,7 +1431,8 @@ static int udf_load_sparable_map(struct super_block *sb, if (ident != 0 || strncmp(st->sparingIdent.ident, UDF_ID_SPARING, strlen(UDF_ID_SPARING)) || - sizeof(*st) + le16_to_cpu(st->reallocationTableLen) > + struct_size(st, mapEntry, + le16_to_cpu(st->reallocationTableLen)) > sb->s_blocksize) { brelse(bh); continue; @@ -2321,7 +2335,7 @@ static int udf_fill_super(struct super_block *sb, struct fs_context *fc) error_out: iput(sbi->s_vat_inode); - unload_nls(uopt->nls_map); + unload_nls(sbi->s_nls_map); if (lvid_open) udf_close_lvid(sb); brelse(sbi->s_lvid_bh); diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c deleted file mode 100644 index 4b53dc4a3266..000000000000 --- a/fs/userfaultfd.c +++ /dev/null @@ -1,2231 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * fs/userfaultfd.c - * - * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org> - * Copyright (C) 2008-2009 Red Hat, Inc. - * Copyright (C) 2015 Red Hat, Inc. - * - * Some part derived from fs/eventfd.c (anon inode setup) and - * mm/ksm.c (mm hashing). - */ - -#include <linux/list.h> -#include <linux/hashtable.h> -#include <linux/sched/signal.h> -#include <linux/sched/mm.h> -#include <linux/mm.h> -#include <linux/mm_inline.h> -#include <linux/mmu_notifier.h> -#include <linux/poll.h> -#include <linux/slab.h> -#include <linux/seq_file.h> -#include <linux/file.h> -#include <linux/bug.h> -#include <linux/anon_inodes.h> -#include <linux/syscalls.h> -#include <linux/userfaultfd_k.h> -#include <linux/mempolicy.h> -#include <linux/ioctl.h> -#include <linux/security.h> -#include <linux/hugetlb.h> -#include <linux/leafops.h> -#include <linux/miscdevice.h> -#include <linux/uio.h> - -static int sysctl_unprivileged_userfaultfd __read_mostly; - -#ifdef CONFIG_SYSCTL -static const struct ctl_table vm_userfaultfd_table[] = { - { - .procname = "unprivileged_userfaultfd", - .data = &sysctl_unprivileged_userfaultfd, - .maxlen = sizeof(sysctl_unprivileged_userfaultfd), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -}; -#endif - -static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init; - -struct userfaultfd_fork_ctx { - struct userfaultfd_ctx *orig; - struct userfaultfd_ctx *new; - struct list_head list; -}; - -struct userfaultfd_unmap_ctx { - struct userfaultfd_ctx *ctx; - unsigned long start; - unsigned long end; - struct list_head list; -}; - -struct userfaultfd_wait_queue { - struct uffd_msg msg; - wait_queue_entry_t wq; - struct userfaultfd_ctx *ctx; - bool waken; -}; - -struct userfaultfd_wake_range { - unsigned long start; - unsigned long len; -}; - -/* internal indication that UFFD_API ioctl was successfully executed */ -#define UFFD_FEATURE_INITIALIZED (1u << 31) - -static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) -{ - return ctx->features & UFFD_FEATURE_INITIALIZED; -} - -static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) -{ - return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC); -} - -/* - * Whether WP_UNPOPULATED is enabled on the uffd context. It is only - * meaningful when userfaultfd_wp()==true on the vma and when it's - * anonymous. - */ -bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) -{ - struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; - - if (!ctx) - return false; - - return ctx->features & UFFD_FEATURE_WP_UNPOPULATED; -} - -static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode, - int wake_flags, void *key) -{ - struct userfaultfd_wake_range *range = key; - int ret; - struct userfaultfd_wait_queue *uwq; - unsigned long start, len; - - uwq = container_of(wq, struct userfaultfd_wait_queue, wq); - ret = 0; - /* len == 0 means wake all */ - start = range->start; - len = range->len; - if (len && (start > uwq->msg.arg.pagefault.address || - start + len <= uwq->msg.arg.pagefault.address)) - goto out; - WRITE_ONCE(uwq->waken, true); - /* - * The Program-Order guarantees provided by the scheduler - * ensure uwq->waken is visible before the task is woken. - */ - ret = wake_up_state(wq->private, mode); - if (ret) { - /* - * Wake only once, autoremove behavior. - * - * After the effect of list_del_init is visible to the other - * CPUs, the waitqueue may disappear from under us, see the - * !list_empty_careful() in handle_userfault(). - * - * try_to_wake_up() has an implicit smp_mb(), and the - * wq->private is read before calling the extern function - * "wake_up_state" (which in turns calls try_to_wake_up). - */ - list_del_init(&wq->entry); - } -out: - return ret; -} - -/** - * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd - * context. - * @ctx: [in] Pointer to the userfaultfd context. - */ -static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx) -{ - refcount_inc(&ctx->refcount); -} - -/** - * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd - * context. - * @ctx: [in] Pointer to userfaultfd context. - * - * The userfaultfd context reference must have been previously acquired either - * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget(). - */ -static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) -{ - if (refcount_dec_and_test(&ctx->refcount)) { - VM_WARN_ON_ONCE(spin_is_locked(&ctx->fault_pending_wqh.lock)); - VM_WARN_ON_ONCE(waitqueue_active(&ctx->fault_pending_wqh)); - VM_WARN_ON_ONCE(spin_is_locked(&ctx->fault_wqh.lock)); - VM_WARN_ON_ONCE(waitqueue_active(&ctx->fault_wqh)); - VM_WARN_ON_ONCE(spin_is_locked(&ctx->event_wqh.lock)); - VM_WARN_ON_ONCE(waitqueue_active(&ctx->event_wqh)); - VM_WARN_ON_ONCE(spin_is_locked(&ctx->fd_wqh.lock)); - VM_WARN_ON_ONCE(waitqueue_active(&ctx->fd_wqh)); - mmdrop(ctx->mm); - kmem_cache_free(userfaultfd_ctx_cachep, ctx); - } -} - -static inline void msg_init(struct uffd_msg *msg) -{ - BUILD_BUG_ON(sizeof(struct uffd_msg) != 32); - /* - * Must use memset to zero out the paddings or kernel data is - * leaked to userland. - */ - memset(msg, 0, sizeof(struct uffd_msg)); -} - -static inline struct uffd_msg userfault_msg(unsigned long address, - unsigned long real_address, - unsigned int flags, - unsigned long reason, - unsigned int features) -{ - struct uffd_msg msg; - - msg_init(&msg); - msg.event = UFFD_EVENT_PAGEFAULT; - - msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ? - real_address : address; - - /* - * These flags indicate why the userfault occurred: - * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault. - * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault. - * - Neither of these flags being set indicates a MISSING fault. - * - * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write - * fault. Otherwise, it was a read fault. - */ - if (flags & FAULT_FLAG_WRITE) - msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE; - if (reason & VM_UFFD_WP) - msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP; - if (reason & VM_UFFD_MINOR) - msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR; - if (features & UFFD_FEATURE_THREAD_ID) - msg.arg.pagefault.feat.ptid = task_pid_vnr(current); - return msg; -} - -#ifdef CONFIG_HUGETLB_PAGE -/* - * Same functionality as userfaultfd_must_wait below with modifications for - * hugepmd ranges. - */ -static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, - struct vm_fault *vmf, - unsigned long reason) -{ - struct vm_area_struct *vma = vmf->vma; - pte_t *ptep, pte; - - assert_fault_locked(vmf); - - ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma)); - if (!ptep) - return true; - - pte = huge_ptep_get(vma->vm_mm, vmf->address, ptep); - - /* - * Lockless access: we're in a wait_event so it's ok if it - * changes under us. - */ - - /* Entry is still missing, wait for userspace to resolve the fault. */ - if (huge_pte_none(pte)) - return true; - /* UFFD PTE markers require userspace to resolve the fault. */ - if (pte_is_uffd_marker(pte)) - return true; - /* - * If VMA has UFFD WP faults enabled and WP fault, wait for userspace to - * resolve the fault. - */ - if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) - return true; - - return false; -} -#else -static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, - struct vm_fault *vmf, - unsigned long reason) -{ - /* Should never get here. */ - VM_WARN_ON_ONCE(1); - return false; -} -#endif /* CONFIG_HUGETLB_PAGE */ - -/* - * Verify the pagetables are still not ok after having registered into - * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any - * userfault that has already been resolved, if userfaultfd_read_iter and - * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different - * threads. - */ -static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, - struct vm_fault *vmf, - unsigned long reason) -{ - struct mm_struct *mm = ctx->mm; - unsigned long address = vmf->address; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd, _pmd; - pte_t *pte; - pte_t ptent; - bool ret; - - assert_fault_locked(vmf); - - pgd = pgd_offset(mm, address); - if (!pgd_present(*pgd)) - return true; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) - return true; - pud = pud_offset(p4d, address); - if (!pud_present(*pud)) - return true; - pmd = pmd_offset(pud, address); -again: - _pmd = pmdp_get_lockless(pmd); - if (pmd_none(_pmd)) - return true; - - /* - * A race could arise which would result in a softleaf entry such as - * migration entry unexpectedly being present in the PMD, so explicitly - * check for this and bail out if so. - */ - if (!pmd_present(_pmd)) - return false; - - if (pmd_trans_huge(_pmd)) - return !pmd_write(_pmd) && (reason & VM_UFFD_WP); - - pte = pte_offset_map(pmd, address); - if (!pte) - goto again; - - /* - * Lockless access: we're in a wait_event so it's ok if it - * changes under us. - */ - ptent = ptep_get(pte); - - ret = true; - /* Entry is still missing, wait for userspace to resolve the fault. */ - if (pte_none(ptent)) - goto out; - /* UFFD PTE markers require userspace to resolve the fault. */ - if (pte_is_uffd_marker(ptent)) - goto out; - /* - * If VMA has UFFD WP faults enabled and WP fault, wait for userspace to - * resolve the fault. - */ - if (!pte_write(ptent) && (reason & VM_UFFD_WP)) - goto out; - - ret = false; -out: - pte_unmap(pte); - return ret; -} - -static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags) -{ - if (flags & FAULT_FLAG_INTERRUPTIBLE) - return TASK_INTERRUPTIBLE; - - if (flags & FAULT_FLAG_KILLABLE) - return TASK_KILLABLE; - - return TASK_UNINTERRUPTIBLE; -} - -/* - * The locking rules involved in returning VM_FAULT_RETRY depending on - * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and - * FAULT_FLAG_KILLABLE are not straightforward. The "Caution" - * recommendation in __lock_page_or_retry is not an understatement. - * - * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released - * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is - * not set. - * - * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not - * set, VM_FAULT_RETRY can still be returned if and only if there are - * fatal_signal_pending()s, and the mmap_lock must be released before - * returning it. - */ -vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) -{ - struct vm_area_struct *vma = vmf->vma; - struct mm_struct *mm = vma->vm_mm; - struct userfaultfd_ctx *ctx; - struct userfaultfd_wait_queue uwq; - vm_fault_t ret = VM_FAULT_SIGBUS; - bool must_wait; - unsigned int blocking_state; - - /* - * We don't do userfault handling for the final child pid update - * and when coredumping (faults triggered by get_dump_page()). - */ - if (current->flags & (PF_EXITING|PF_DUMPCORE)) - goto out; - - assert_fault_locked(vmf); - - ctx = vma->vm_userfaultfd_ctx.ctx; - if (!ctx) - goto out; - - VM_WARN_ON_ONCE(ctx->mm != mm); - - /* Any unrecognized flag is a bug. */ - VM_WARN_ON_ONCE(reason & ~__VM_UFFD_FLAGS); - /* 0 or > 1 flags set is a bug; we expect exactly 1. */ - VM_WARN_ON_ONCE(!reason || (reason & (reason - 1))); - - if (ctx->features & UFFD_FEATURE_SIGBUS) - goto out; - if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY)) - goto out; - - /* - * Check that we can return VM_FAULT_RETRY. - * - * NOTE: it should become possible to return VM_FAULT_RETRY - * even if FAULT_FLAG_TRIED is set without leading to gup() - * -EBUSY failures, if the userfaultfd is to be extended for - * VM_UFFD_WP tracking and we intend to arm the userfault - * without first stopping userland access to the memory. For - * VM_UFFD_MISSING userfaults this is enough for now. - */ - if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) { - /* - * Validate the invariant that nowait must allow retry - * to be sure not to return SIGBUS erroneously on - * nowait invocations. - */ - VM_WARN_ON_ONCE(vmf->flags & FAULT_FLAG_RETRY_NOWAIT); -#ifdef CONFIG_DEBUG_VM - if (printk_ratelimit()) { - pr_warn("FAULT_FLAG_ALLOW_RETRY missing %x\n", - vmf->flags); - dump_stack(); - } -#endif - goto out; - } - - /* - * Handle nowait, not much to do other than tell it to retry - * and wait. - */ - ret = VM_FAULT_RETRY; - if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) - goto out; - - if (unlikely(READ_ONCE(ctx->released))) { - /* - * If a concurrent release is detected, do not return - * VM_FAULT_SIGBUS or VM_FAULT_NOPAGE, but instead always - * return VM_FAULT_RETRY with lock released proactively. - * - * If we were to return VM_FAULT_SIGBUS here, the non - * cooperative manager would be instead forced to - * always call UFFDIO_UNREGISTER before it can safely - * close the uffd, to avoid involuntary SIGBUS triggered. - * - * If we were to return VM_FAULT_NOPAGE, it would work for - * the fault path, in which the lock will be released - * later. However for GUP, faultin_page() does nothing - * special on NOPAGE, so GUP would spin retrying without - * releasing the mmap read lock, causing possible livelock. - * - * Here only VM_FAULT_RETRY would make sure the mmap lock - * be released immediately, so that the thread concurrently - * releasing the userfault would always make progress. - */ - release_fault_lock(vmf); - goto out; - } - - /* take the reference before dropping the mmap_lock */ - userfaultfd_ctx_get(ctx); - - init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function); - uwq.wq.private = current; - uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags, - reason, ctx->features); - uwq.ctx = ctx; - uwq.waken = false; - - blocking_state = userfaultfd_get_blocking_state(vmf->flags); - - /* - * Take the vma lock now, in order to safely call - * userfaultfd_huge_must_wait() later. Since acquiring the - * (sleepable) vma lock can modify the current task state, that - * must be before explicitly calling set_current_state(). - */ - if (is_vm_hugetlb_page(vma)) - hugetlb_vma_lock_read(vma); - - spin_lock_irq(&ctx->fault_pending_wqh.lock); - /* - * After the __add_wait_queue the uwq is visible to userland - * through poll/read(). - */ - __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq); - /* - * The smp_mb() after __set_current_state prevents the reads - * following the spin_unlock to happen before the list_add in - * __add_wait_queue. - */ - set_current_state(blocking_state); - spin_unlock_irq(&ctx->fault_pending_wqh.lock); - - if (is_vm_hugetlb_page(vma)) { - must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason); - hugetlb_vma_unlock_read(vma); - } else { - must_wait = userfaultfd_must_wait(ctx, vmf, reason); - } - - release_fault_lock(vmf); - - if (likely(must_wait && !READ_ONCE(ctx->released))) { - wake_up_poll(&ctx->fd_wqh, EPOLLIN); - schedule(); - } - - __set_current_state(TASK_RUNNING); - - /* - * Here we race with the list_del; list_add in - * userfaultfd_ctx_read(), however because we don't ever run - * list_del_init() to refile across the two lists, the prev - * and next pointers will never point to self. list_add also - * would never let any of the two pointers to point to - * self. So list_empty_careful won't risk to see both pointers - * pointing to self at any time during the list refile. The - * only case where list_del_init() is called is the full - * removal in the wake function and there we don't re-list_add - * and it's fine not to block on the spinlock. The uwq on this - * kernel stack can be released after the list_del_init. - */ - if (!list_empty_careful(&uwq.wq.entry)) { - spin_lock_irq(&ctx->fault_pending_wqh.lock); - /* - * No need of list_del_init(), the uwq on the stack - * will be freed shortly anyway. - */ - list_del(&uwq.wq.entry); - spin_unlock_irq(&ctx->fault_pending_wqh.lock); - } - - /* - * ctx may go away after this if the userfault pseudo fd is - * already released. - */ - userfaultfd_ctx_put(ctx); - -out: - return ret; -} - -static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, - struct userfaultfd_wait_queue *ewq) -{ - struct userfaultfd_ctx *release_new_ctx; - - if (WARN_ON_ONCE(current->flags & PF_EXITING)) - goto out; - - ewq->ctx = ctx; - init_waitqueue_entry(&ewq->wq, current); - release_new_ctx = NULL; - - spin_lock_irq(&ctx->event_wqh.lock); - /* - * After the __add_wait_queue the uwq is visible to userland - * through poll/read(). - */ - __add_wait_queue(&ctx->event_wqh, &ewq->wq); - for (;;) { - set_current_state(TASK_KILLABLE); - if (ewq->msg.event == 0) - break; - if (READ_ONCE(ctx->released) || - fatal_signal_pending(current)) { - /* - * &ewq->wq may be queued in fork_event, but - * __remove_wait_queue ignores the head - * parameter. It would be a problem if it - * didn't. - */ - __remove_wait_queue(&ctx->event_wqh, &ewq->wq); - if (ewq->msg.event == UFFD_EVENT_FORK) { - struct userfaultfd_ctx *new; - - new = (struct userfaultfd_ctx *) - (unsigned long) - ewq->msg.arg.reserved.reserved1; - release_new_ctx = new; - } - break; - } - - spin_unlock_irq(&ctx->event_wqh.lock); - - wake_up_poll(&ctx->fd_wqh, EPOLLIN); - schedule(); - - spin_lock_irq(&ctx->event_wqh.lock); - } - __set_current_state(TASK_RUNNING); - spin_unlock_irq(&ctx->event_wqh.lock); - - if (release_new_ctx) { - userfaultfd_release_new(release_new_ctx); - userfaultfd_ctx_put(release_new_ctx); - } - - /* - * ctx may go away after this if the userfault pseudo fd is - * already released. - */ -out: - atomic_dec(&ctx->mmap_changing); - VM_WARN_ON_ONCE(atomic_read(&ctx->mmap_changing) < 0); - userfaultfd_ctx_put(ctx); -} - -static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx, - struct userfaultfd_wait_queue *ewq) -{ - ewq->msg.event = 0; - wake_up_locked(&ctx->event_wqh); - __remove_wait_queue(&ctx->event_wqh, &ewq->wq); -} - -int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) -{ - struct userfaultfd_ctx *ctx = NULL, *octx; - struct userfaultfd_fork_ctx *fctx; - - octx = vma->vm_userfaultfd_ctx.ctx; - if (!octx) - return 0; - - if (!(octx->features & UFFD_FEATURE_EVENT_FORK)) { - userfaultfd_reset_ctx(vma); - return 0; - } - - list_for_each_entry(fctx, fcs, list) - if (fctx->orig == octx) { - ctx = fctx->new; - break; - } - - if (!ctx) { - fctx = kmalloc_obj(*fctx); - if (!fctx) - return -ENOMEM; - - ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); - if (!ctx) { - kfree(fctx); - return -ENOMEM; - } - - refcount_set(&ctx->refcount, 1); - ctx->flags = octx->flags; - ctx->features = octx->features; - ctx->released = false; - init_rwsem(&ctx->map_changing_lock); - atomic_set(&ctx->mmap_changing, 0); - ctx->mm = vma->vm_mm; - mmgrab(ctx->mm); - - userfaultfd_ctx_get(octx); - down_write(&octx->map_changing_lock); - atomic_inc(&octx->mmap_changing); - up_write(&octx->map_changing_lock); - fctx->orig = octx; - fctx->new = ctx; - list_add_tail(&fctx->list, fcs); - } - - vma->vm_userfaultfd_ctx.ctx = ctx; - return 0; -} - -static void dup_fctx(struct userfaultfd_fork_ctx *fctx) -{ - struct userfaultfd_ctx *ctx = fctx->orig; - struct userfaultfd_wait_queue ewq; - - msg_init(&ewq.msg); - - ewq.msg.event = UFFD_EVENT_FORK; - ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new; - - userfaultfd_event_wait_completion(ctx, &ewq); -} - -void dup_userfaultfd_complete(struct list_head *fcs) -{ - struct userfaultfd_fork_ctx *fctx, *n; - - list_for_each_entry_safe(fctx, n, fcs, list) { - dup_fctx(fctx); - list_del(&fctx->list); - kfree(fctx); - } -} - -void dup_userfaultfd_fail(struct list_head *fcs) -{ - struct userfaultfd_fork_ctx *fctx, *n; - - /* - * An error has occurred on fork, we will tear memory down, but have - * allocated memory for fctx's and raised reference counts for both the - * original and child contexts (and on the mm for each as a result). - * - * These would ordinarily be taken care of by a user handling the event, - * but we are no longer doing so, so manually clean up here. - * - * mm tear down will take care of cleaning up VMA contexts. - */ - list_for_each_entry_safe(fctx, n, fcs, list) { - struct userfaultfd_ctx *octx = fctx->orig; - struct userfaultfd_ctx *ctx = fctx->new; - - atomic_dec(&octx->mmap_changing); - VM_WARN_ON_ONCE(atomic_read(&octx->mmap_changing) < 0); - userfaultfd_ctx_put(octx); - userfaultfd_ctx_put(ctx); - - list_del(&fctx->list); - kfree(fctx); - } -} - -void mremap_userfaultfd_prep(struct vm_area_struct *vma, - struct vm_userfaultfd_ctx *vm_ctx) -{ - struct userfaultfd_ctx *ctx; - - ctx = vma->vm_userfaultfd_ctx.ctx; - - if (!ctx) - return; - - if (ctx->features & UFFD_FEATURE_EVENT_REMAP) { - vm_ctx->ctx = ctx; - userfaultfd_ctx_get(ctx); - down_write(&ctx->map_changing_lock); - atomic_inc(&ctx->mmap_changing); - up_write(&ctx->map_changing_lock); - } else { - /* Drop uffd context if remap feature not enabled */ - userfaultfd_reset_ctx(vma); - } -} - -void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx, - unsigned long from, unsigned long to, - unsigned long len) -{ - struct userfaultfd_ctx *ctx = vm_ctx->ctx; - struct userfaultfd_wait_queue ewq; - - if (!ctx) - return; - - msg_init(&ewq.msg); - - ewq.msg.event = UFFD_EVENT_REMAP; - ewq.msg.arg.remap.from = from; - ewq.msg.arg.remap.to = to; - ewq.msg.arg.remap.len = len; - - userfaultfd_event_wait_completion(ctx, &ewq); -} - -void mremap_userfaultfd_fail(struct vm_userfaultfd_ctx *vm_ctx) -{ - struct userfaultfd_ctx *ctx = vm_ctx->ctx; - - if (!ctx) - return; - - userfaultfd_ctx_put(ctx); -} - -bool userfaultfd_remove(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - struct userfaultfd_ctx *ctx; - struct userfaultfd_wait_queue ewq; - - ctx = vma->vm_userfaultfd_ctx.ctx; - if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE)) - return true; - - userfaultfd_ctx_get(ctx); - down_write(&ctx->map_changing_lock); - atomic_inc(&ctx->mmap_changing); - up_write(&ctx->map_changing_lock); - mmap_read_unlock(mm); - - msg_init(&ewq.msg); - - ewq.msg.event = UFFD_EVENT_REMOVE; - ewq.msg.arg.remove.start = start; - ewq.msg.arg.remove.end = end; - - userfaultfd_event_wait_completion(ctx, &ewq); - - return false; -} - -static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, - unsigned long start, unsigned long end) -{ - struct userfaultfd_unmap_ctx *unmap_ctx; - - list_for_each_entry(unmap_ctx, unmaps, list) - if (unmap_ctx->ctx == ctx && unmap_ctx->start == start && - unmap_ctx->end == end) - return true; - - return false; -} - -int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, - unsigned long end, struct list_head *unmaps) -{ - struct userfaultfd_unmap_ctx *unmap_ctx; - struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; - - if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) || - has_unmap_ctx(ctx, unmaps, start, end)) - return 0; - - unmap_ctx = kzalloc_obj(*unmap_ctx); - if (!unmap_ctx) - return -ENOMEM; - - userfaultfd_ctx_get(ctx); - down_write(&ctx->map_changing_lock); - atomic_inc(&ctx->mmap_changing); - up_write(&ctx->map_changing_lock); - unmap_ctx->ctx = ctx; - unmap_ctx->start = start; - unmap_ctx->end = end; - list_add_tail(&unmap_ctx->list, unmaps); - - return 0; -} - -void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf) -{ - struct userfaultfd_unmap_ctx *ctx, *n; - struct userfaultfd_wait_queue ewq; - - list_for_each_entry_safe(ctx, n, uf, list) { - msg_init(&ewq.msg); - - ewq.msg.event = UFFD_EVENT_UNMAP; - ewq.msg.arg.remove.start = ctx->start; - ewq.msg.arg.remove.end = ctx->end; - - userfaultfd_event_wait_completion(ctx->ctx, &ewq); - - list_del(&ctx->list); - kfree(ctx); - } -} - -static int userfaultfd_release(struct inode *inode, struct file *file) -{ - struct userfaultfd_ctx *ctx = file->private_data; - struct mm_struct *mm = ctx->mm; - /* len == 0 means wake all */ - struct userfaultfd_wake_range range = { .len = 0, }; - - WRITE_ONCE(ctx->released, true); - - userfaultfd_release_all(mm, ctx); - - /* - * After no new page faults can wait on this fault_*wqh, flush - * the last page faults that may have been already waiting on - * the fault_*wqh. - */ - spin_lock_irq(&ctx->fault_pending_wqh.lock); - __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range); - __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range); - spin_unlock_irq(&ctx->fault_pending_wqh.lock); - - /* Flush pending events that may still wait on event_wqh */ - wake_up_all(&ctx->event_wqh); - - wake_up_poll(&ctx->fd_wqh, EPOLLHUP); - userfaultfd_ctx_put(ctx); - return 0; -} - -/* fault_pending_wqh.lock must be hold by the caller */ -static inline struct userfaultfd_wait_queue *find_userfault_in( - wait_queue_head_t *wqh) -{ - wait_queue_entry_t *wq; - struct userfaultfd_wait_queue *uwq; - - lockdep_assert_held(&wqh->lock); - - uwq = NULL; - if (!waitqueue_active(wqh)) - goto out; - /* walk in reverse to provide FIFO behavior to read userfaults */ - wq = list_last_entry(&wqh->head, typeof(*wq), entry); - uwq = container_of(wq, struct userfaultfd_wait_queue, wq); -out: - return uwq; -} - -static inline struct userfaultfd_wait_queue *find_userfault( - struct userfaultfd_ctx *ctx) -{ - return find_userfault_in(&ctx->fault_pending_wqh); -} - -static inline struct userfaultfd_wait_queue *find_userfault_evt( - struct userfaultfd_ctx *ctx) -{ - return find_userfault_in(&ctx->event_wqh); -} - -static __poll_t userfaultfd_poll(struct file *file, poll_table *wait) -{ - struct userfaultfd_ctx *ctx = file->private_data; - __poll_t ret; - - poll_wait(file, &ctx->fd_wqh, wait); - - if (!userfaultfd_is_initialized(ctx)) - return EPOLLERR; - - /* - * poll() never guarantees that read won't block. - * userfaults can be waken before they're read(). - */ - if (unlikely(!(file->f_flags & O_NONBLOCK))) - return EPOLLERR; - /* - * lockless access to see if there are pending faults - * __pollwait last action is the add_wait_queue but - * the spin_unlock would allow the waitqueue_active to - * pass above the actual list_add inside - * add_wait_queue critical section. So use a full - * memory barrier to serialize the list_add write of - * add_wait_queue() with the waitqueue_active read - * below. - */ - ret = 0; - smp_mb(); - if (waitqueue_active(&ctx->fault_pending_wqh)) - ret = EPOLLIN; - else if (waitqueue_active(&ctx->event_wqh)) - ret = EPOLLIN; - - return ret; -} - -static const struct file_operations userfaultfd_fops; - -static int resolve_userfault_fork(struct userfaultfd_ctx *new, - struct inode *inode, - struct uffd_msg *msg) -{ - int fd; - - fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new, - O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode); - if (fd < 0) - return fd; - - msg->arg.reserved.reserved1 = 0; - msg->arg.fork.ufd = fd; - return 0; -} - -static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, - struct uffd_msg *msg, struct inode *inode) -{ - ssize_t ret; - DECLARE_WAITQUEUE(wait, current); - struct userfaultfd_wait_queue *uwq; - /* - * Handling fork event requires sleeping operations, so - * we drop the event_wqh lock, then do these ops, then - * lock it back and wake up the waiter. While the lock is - * dropped the ewq may go away so we keep track of it - * carefully. - */ - LIST_HEAD(fork_event); - struct userfaultfd_ctx *fork_nctx = NULL; - - /* always take the fd_wqh lock before the fault_pending_wqh lock */ - spin_lock_irq(&ctx->fd_wqh.lock); - __add_wait_queue(&ctx->fd_wqh, &wait); - for (;;) { - set_current_state(TASK_INTERRUPTIBLE); - spin_lock(&ctx->fault_pending_wqh.lock); - uwq = find_userfault(ctx); - if (uwq) { - /* - * Use a seqcount to repeat the lockless check - * in wake_userfault() to avoid missing - * wakeups because during the refile both - * waitqueue could become empty if this is the - * only userfault. - */ - write_seqcount_begin(&ctx->refile_seq); - - /* - * The fault_pending_wqh.lock prevents the uwq - * to disappear from under us. - * - * Refile this userfault from - * fault_pending_wqh to fault_wqh, it's not - * pending anymore after we read it. - * - * Use list_del() by hand (as - * userfaultfd_wake_function also uses - * list_del_init() by hand) to be sure nobody - * changes __remove_wait_queue() to use - * list_del_init() in turn breaking the - * !list_empty_careful() check in - * handle_userfault(). The uwq->wq.head list - * must never be empty at any time during the - * refile, or the waitqueue could disappear - * from under us. The "wait_queue_head_t" - * parameter of __remove_wait_queue() is unused - * anyway. - */ - list_del(&uwq->wq.entry); - add_wait_queue(&ctx->fault_wqh, &uwq->wq); - - write_seqcount_end(&ctx->refile_seq); - - /* careful to always initialize msg if ret == 0 */ - *msg = uwq->msg; - spin_unlock(&ctx->fault_pending_wqh.lock); - ret = 0; - break; - } - spin_unlock(&ctx->fault_pending_wqh.lock); - - spin_lock(&ctx->event_wqh.lock); - uwq = find_userfault_evt(ctx); - if (uwq) { - *msg = uwq->msg; - - if (uwq->msg.event == UFFD_EVENT_FORK) { - fork_nctx = (struct userfaultfd_ctx *) - (unsigned long) - uwq->msg.arg.reserved.reserved1; - list_move(&uwq->wq.entry, &fork_event); - /* - * fork_nctx can be freed as soon as - * we drop the lock, unless we take a - * reference on it. - */ - userfaultfd_ctx_get(fork_nctx); - spin_unlock(&ctx->event_wqh.lock); - ret = 0; - break; - } - - userfaultfd_event_complete(ctx, uwq); - spin_unlock(&ctx->event_wqh.lock); - ret = 0; - break; - } - spin_unlock(&ctx->event_wqh.lock); - - if (signal_pending(current)) { - ret = -ERESTARTSYS; - break; - } - if (no_wait) { - ret = -EAGAIN; - break; - } - spin_unlock_irq(&ctx->fd_wqh.lock); - schedule(); - spin_lock_irq(&ctx->fd_wqh.lock); - } - __remove_wait_queue(&ctx->fd_wqh, &wait); - __set_current_state(TASK_RUNNING); - spin_unlock_irq(&ctx->fd_wqh.lock); - - if (!ret && msg->event == UFFD_EVENT_FORK) { - ret = resolve_userfault_fork(fork_nctx, inode, msg); - spin_lock_irq(&ctx->event_wqh.lock); - if (!list_empty(&fork_event)) { - /* - * The fork thread didn't abort, so we can - * drop the temporary refcount. - */ - userfaultfd_ctx_put(fork_nctx); - - uwq = list_first_entry(&fork_event, - typeof(*uwq), - wq.entry); - /* - * If fork_event list wasn't empty and in turn - * the event wasn't already released by fork - * (the event is allocated on fork kernel - * stack), put the event back to its place in - * the event_wq. fork_event head will be freed - * as soon as we return so the event cannot - * stay queued there no matter the current - * "ret" value. - */ - list_del(&uwq->wq.entry); - __add_wait_queue(&ctx->event_wqh, &uwq->wq); - - /* - * Leave the event in the waitqueue and report - * error to userland if we failed to resolve - * the userfault fork. - */ - if (likely(!ret)) - userfaultfd_event_complete(ctx, uwq); - } else { - /* - * Here the fork thread aborted and the - * refcount from the fork thread on fork_nctx - * has already been released. We still hold - * the reference we took before releasing the - * lock above. If resolve_userfault_fork - * failed we've to drop it because the - * fork_nctx has to be freed in such case. If - * it succeeded we'll hold it because the new - * uffd references it. - */ - if (ret) - userfaultfd_ctx_put(fork_nctx); - } - spin_unlock_irq(&ctx->event_wqh.lock); - } - - return ret; -} - -static ssize_t userfaultfd_read_iter(struct kiocb *iocb, struct iov_iter *to) -{ - struct file *file = iocb->ki_filp; - struct userfaultfd_ctx *ctx = file->private_data; - ssize_t _ret, ret = 0; - struct uffd_msg msg; - struct inode *inode = file_inode(file); - bool no_wait; - - if (!userfaultfd_is_initialized(ctx)) - return -EINVAL; - - no_wait = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT; - for (;;) { - if (iov_iter_count(to) < sizeof(msg)) - return ret ? ret : -EINVAL; - _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode); - if (_ret < 0) - return ret ? ret : _ret; - _ret = !copy_to_iter_full(&msg, sizeof(msg), to); - if (_ret) - return ret ? ret : -EFAULT; - ret += sizeof(msg); - /* - * Allow to read more than one fault at time but only - * block if waiting for the very first one. - */ - no_wait = true; - } -} - -static void __wake_userfault(struct userfaultfd_ctx *ctx, - struct userfaultfd_wake_range *range) -{ - spin_lock_irq(&ctx->fault_pending_wqh.lock); - /* wake all in the range and autoremove */ - if (waitqueue_active(&ctx->fault_pending_wqh)) - __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, - range); - if (waitqueue_active(&ctx->fault_wqh)) - __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range); - spin_unlock_irq(&ctx->fault_pending_wqh.lock); -} - -static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, - struct userfaultfd_wake_range *range) -{ - unsigned seq; - bool need_wakeup; - - /* - * To be sure waitqueue_active() is not reordered by the CPU - * before the pagetable update, use an explicit SMP memory - * barrier here. PT lock release or mmap_read_unlock(mm) still - * have release semantics that can allow the - * waitqueue_active() to be reordered before the pte update. - */ - smp_mb(); - - /* - * Use waitqueue_active because it's very frequent to - * change the address space atomically even if there are no - * userfaults yet. So we take the spinlock only when we're - * sure we've userfaults to wake. - */ - do { - seq = read_seqcount_begin(&ctx->refile_seq); - need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) || - waitqueue_active(&ctx->fault_wqh); - cond_resched(); - } while (read_seqcount_retry(&ctx->refile_seq, seq)); - if (need_wakeup) - __wake_userfault(ctx, range); -} - -static __always_inline int validate_unaligned_range( - struct mm_struct *mm, __u64 start, __u64 len) -{ - __u64 task_size = mm->task_size; - - if (len & ~PAGE_MASK) - return -EINVAL; - if (!len) - return -EINVAL; - if (start >= task_size) - return -EINVAL; - if (len > task_size - start) - return -EINVAL; - if (start + len <= start) - return -EINVAL; - return 0; -} - -static __always_inline int validate_range(struct mm_struct *mm, - __u64 start, __u64 len) -{ - if (start & ~PAGE_MASK) - return -EINVAL; - - return validate_unaligned_range(mm, start, len); -} - -static int userfaultfd_register(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - struct mm_struct *mm = ctx->mm; - struct vm_area_struct *vma, *cur; - int ret; - struct uffdio_register uffdio_register; - struct uffdio_register __user *user_uffdio_register; - vm_flags_t vm_flags; - bool found; - bool basic_ioctls; - unsigned long start, end; - struct vma_iterator vmi; - bool wp_async = userfaultfd_wp_async_ctx(ctx); - - user_uffdio_register = (struct uffdio_register __user *) arg; - - ret = -EFAULT; - if (copy_from_user(&uffdio_register, user_uffdio_register, - sizeof(uffdio_register)-sizeof(__u64))) - goto out; - - ret = -EINVAL; - if (!uffdio_register.mode) - goto out; - if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES) - goto out; - vm_flags = 0; - if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) - vm_flags |= VM_UFFD_MISSING; - if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { - if (!pgtable_supports_uffd_wp()) - goto out; - - vm_flags |= VM_UFFD_WP; - } - if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) { -#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR - goto out; -#endif - vm_flags |= VM_UFFD_MINOR; - } - - ret = validate_range(mm, uffdio_register.range.start, - uffdio_register.range.len); - if (ret) - goto out; - - start = uffdio_register.range.start; - end = start + uffdio_register.range.len; - - ret = -ENOMEM; - if (!mmget_not_zero(mm)) - goto out; - - ret = -EINVAL; - mmap_write_lock(mm); - vma_iter_init(&vmi, mm, start); - vma = vma_find(&vmi, end); - if (!vma) - goto out_unlock; - - /* - * If the first vma contains huge pages, make sure start address - * is aligned to huge page size. - */ - if (is_vm_hugetlb_page(vma)) { - unsigned long vma_hpagesize = vma_kernel_pagesize(vma); - - if (start & (vma_hpagesize - 1)) - goto out_unlock; - } - - /* - * Search for not compatible vmas. - */ - found = false; - basic_ioctls = false; - cur = vma; - do { - cond_resched(); - - VM_WARN_ON_ONCE(!!cur->vm_userfaultfd_ctx.ctx ^ - !!(cur->vm_flags & __VM_UFFD_FLAGS)); - - /* check not compatible vmas */ - ret = -EINVAL; - if (!vma_can_userfault(cur, vm_flags, wp_async)) - goto out_unlock; - - /* - * UFFDIO_COPY will fill file holes even without - * PROT_WRITE. This check enforces that if this is a - * MAP_SHARED, the process has write permission to the backing - * file. If VM_MAYWRITE is set it also enforces that on a - * MAP_SHARED vma: there is no F_WRITE_SEAL and no further - * F_WRITE_SEAL can be taken until the vma is destroyed. - */ - ret = -EPERM; - if (unlikely(!(cur->vm_flags & VM_MAYWRITE))) - goto out_unlock; - - /* - * If this vma contains ending address, and huge pages - * check alignment. - */ - if (is_vm_hugetlb_page(cur) && end <= cur->vm_end && - end > cur->vm_start) { - unsigned long vma_hpagesize = vma_kernel_pagesize(cur); - - ret = -EINVAL; - - if (end & (vma_hpagesize - 1)) - goto out_unlock; - } - if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE)) - goto out_unlock; - - /* - * Check that this vma isn't already owned by a - * different userfaultfd. We can't allow more than one - * userfaultfd to own a single vma simultaneously or we - * wouldn't know which one to deliver the userfaults to. - */ - ret = -EBUSY; - if (cur->vm_userfaultfd_ctx.ctx && - cur->vm_userfaultfd_ctx.ctx != ctx) - goto out_unlock; - - /* - * Note vmas containing huge pages - */ - if (is_vm_hugetlb_page(cur)) - basic_ioctls = true; - - found = true; - } for_each_vma_range(vmi, cur, end); - VM_WARN_ON_ONCE(!found); - - ret = userfaultfd_register_range(ctx, vma, vm_flags, start, end, - wp_async); - -out_unlock: - mmap_write_unlock(mm); - mmput(mm); - if (!ret) { - __u64 ioctls_out; - - ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : - UFFD_API_RANGE_IOCTLS; - - /* - * Declare the WP ioctl only if the WP mode is - * specified and all checks passed with the range - */ - if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP)) - ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT); - - /* CONTINUE ioctl is only supported for MINOR ranges. */ - if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR)) - ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE); - - /* - * Now that we scanned all vmas we can already tell - * userland which ioctls methods are guaranteed to - * succeed on this range. - */ - if (put_user(ioctls_out, &user_uffdio_register->ioctls)) - ret = -EFAULT; - } -out: - return ret; -} - -static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - struct mm_struct *mm = ctx->mm; - struct vm_area_struct *vma, *prev, *cur; - int ret; - struct uffdio_range uffdio_unregister; - bool found; - unsigned long start, end, vma_end; - const void __user *buf = (void __user *)arg; - struct vma_iterator vmi; - bool wp_async = userfaultfd_wp_async_ctx(ctx); - - ret = -EFAULT; - if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) - goto out; - - ret = validate_range(mm, uffdio_unregister.start, - uffdio_unregister.len); - if (ret) - goto out; - - start = uffdio_unregister.start; - end = start + uffdio_unregister.len; - - ret = -ENOMEM; - if (!mmget_not_zero(mm)) - goto out; - - mmap_write_lock(mm); - ret = -EINVAL; - vma_iter_init(&vmi, mm, start); - vma = vma_find(&vmi, end); - if (!vma) - goto out_unlock; - - /* - * If the first vma contains huge pages, make sure start address - * is aligned to huge page size. - */ - if (is_vm_hugetlb_page(vma)) { - unsigned long vma_hpagesize = vma_kernel_pagesize(vma); - - if (start & (vma_hpagesize - 1)) - goto out_unlock; - } - - /* - * Search for not compatible vmas. - */ - found = false; - cur = vma; - do { - cond_resched(); - - VM_WARN_ON_ONCE(!!cur->vm_userfaultfd_ctx.ctx ^ - !!(cur->vm_flags & __VM_UFFD_FLAGS)); - - /* - * Prevent unregistering through a different userfaultfd than - * the one used for registration. - */ - if (cur->vm_userfaultfd_ctx.ctx && - cur->vm_userfaultfd_ctx.ctx != ctx) - goto out_unlock; - - /* - * Check not compatible vmas, not strictly required - * here as not compatible vmas cannot have an - * userfaultfd_ctx registered on them, but this - * provides for more strict behavior to notice - * unregistration errors. - */ - if (!vma_can_userfault(cur, cur->vm_flags, wp_async)) - goto out_unlock; - - found = true; - } for_each_vma_range(vmi, cur, end); - VM_WARN_ON_ONCE(!found); - - vma_iter_set(&vmi, start); - prev = vma_prev(&vmi); - if (vma->vm_start < start) - prev = vma; - - ret = 0; - for_each_vma_range(vmi, vma, end) { - cond_resched(); - - /* VMA not registered with userfaultfd. */ - if (!vma->vm_userfaultfd_ctx.ctx) - goto skip; - - VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx != ctx); - VM_WARN_ON_ONCE(!vma_can_userfault(vma, vma->vm_flags, wp_async)); - VM_WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)); - - if (vma->vm_start > start) - start = vma->vm_start; - vma_end = min(end, vma->vm_end); - - if (userfaultfd_missing(vma)) { - /* - * Wake any concurrent pending userfault while - * we unregister, so they will not hang - * permanently and it avoids userland to call - * UFFDIO_WAKE explicitly. - */ - struct userfaultfd_wake_range range; - range.start = start; - range.len = vma_end - start; - wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range); - } - - vma = userfaultfd_clear_vma(&vmi, prev, vma, - start, vma_end); - if (IS_ERR(vma)) { - ret = PTR_ERR(vma); - break; - } - - skip: - prev = vma; - start = vma->vm_end; - } - -out_unlock: - mmap_write_unlock(mm); - mmput(mm); -out: - return ret; -} - -/* - * userfaultfd_wake may be used in combination with the - * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches. - */ -static int userfaultfd_wake(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - int ret; - struct uffdio_range uffdio_wake; - struct userfaultfd_wake_range range; - const void __user *buf = (void __user *)arg; - - ret = -EFAULT; - if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake))) - goto out; - - ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len); - if (ret) - goto out; - - range.start = uffdio_wake.start; - range.len = uffdio_wake.len; - - /* - * len == 0 means wake all and we don't want to wake all here, - * so check it again to be sure. - */ - VM_WARN_ON_ONCE(!range.len); - - wake_userfault(ctx, &range); - ret = 0; - -out: - return ret; -} - -static int userfaultfd_copy(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - __s64 ret; - struct uffdio_copy uffdio_copy; - struct uffdio_copy __user *user_uffdio_copy; - struct userfaultfd_wake_range range; - uffd_flags_t flags = 0; - - user_uffdio_copy = (struct uffdio_copy __user *) arg; - - ret = -EAGAIN; - if (unlikely(atomic_read(&ctx->mmap_changing))) { - if (unlikely(put_user(ret, &user_uffdio_copy->copy))) - return -EFAULT; - goto out; - } - - ret = -EFAULT; - if (copy_from_user(&uffdio_copy, user_uffdio_copy, - /* don't copy "copy" last field */ - sizeof(uffdio_copy)-sizeof(__s64))) - goto out; - - ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, - uffdio_copy.len); - if (ret) - goto out; - ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); - if (ret) - goto out; - - ret = -EINVAL; - if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) - goto out; - if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP) - flags |= MFILL_ATOMIC_WP; - if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_copy(ctx, uffdio_copy.dst, uffdio_copy.src, - uffdio_copy.len, flags); - mmput(ctx->mm); - } else { - return -ESRCH; - } - if (unlikely(put_user(ret, &user_uffdio_copy->copy))) - return -EFAULT; - if (ret < 0) - goto out; - VM_WARN_ON_ONCE(!ret); - /* len == 0 would wake all */ - range.len = ret; - if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) { - range.start = uffdio_copy.dst; - wake_userfault(ctx, &range); - } - ret = range.len == uffdio_copy.len ? 0 : -EAGAIN; -out: - return ret; -} - -static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - __s64 ret; - struct uffdio_zeropage uffdio_zeropage; - struct uffdio_zeropage __user *user_uffdio_zeropage; - struct userfaultfd_wake_range range; - - user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg; - - ret = -EAGAIN; - if (unlikely(atomic_read(&ctx->mmap_changing))) { - if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage))) - return -EFAULT; - goto out; - } - - ret = -EFAULT; - if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage, - /* don't copy "zeropage" last field */ - sizeof(uffdio_zeropage)-sizeof(__s64))) - goto out; - - ret = validate_range(ctx->mm, uffdio_zeropage.range.start, - uffdio_zeropage.range.len); - if (ret) - goto out; - ret = -EINVAL; - if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE) - goto out; - - if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_zeropage(ctx, uffdio_zeropage.range.start, - uffdio_zeropage.range.len); - mmput(ctx->mm); - } else { - return -ESRCH; - } - if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage))) - return -EFAULT; - if (ret < 0) - goto out; - /* len == 0 would wake all */ - VM_WARN_ON_ONCE(!ret); - range.len = ret; - if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) { - range.start = uffdio_zeropage.range.start; - wake_userfault(ctx, &range); - } - ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN; -out: - return ret; -} - -static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - int ret; - struct uffdio_writeprotect uffdio_wp; - struct uffdio_writeprotect __user *user_uffdio_wp; - struct userfaultfd_wake_range range; - bool mode_wp, mode_dontwake; - - if (atomic_read(&ctx->mmap_changing)) - return -EAGAIN; - - user_uffdio_wp = (struct uffdio_writeprotect __user *) arg; - - if (copy_from_user(&uffdio_wp, user_uffdio_wp, - sizeof(struct uffdio_writeprotect))) - return -EFAULT; - - ret = validate_range(ctx->mm, uffdio_wp.range.start, - uffdio_wp.range.len); - if (ret) - return ret; - - if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE | - UFFDIO_WRITEPROTECT_MODE_WP)) - return -EINVAL; - - mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP; - mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE; - - if (mode_wp && mode_dontwake) - return -EINVAL; - - if (mmget_not_zero(ctx->mm)) { - ret = mwriteprotect_range(ctx, uffdio_wp.range.start, - uffdio_wp.range.len, mode_wp); - mmput(ctx->mm); - } else { - return -ESRCH; - } - - if (ret) - return ret; - - if (!mode_wp && !mode_dontwake) { - range.start = uffdio_wp.range.start; - range.len = uffdio_wp.range.len; - wake_userfault(ctx, &range); - } - return ret; -} - -static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) -{ - __s64 ret; - struct uffdio_continue uffdio_continue; - struct uffdio_continue __user *user_uffdio_continue; - struct userfaultfd_wake_range range; - uffd_flags_t flags = 0; - - user_uffdio_continue = (struct uffdio_continue __user *)arg; - - ret = -EAGAIN; - if (unlikely(atomic_read(&ctx->mmap_changing))) { - if (unlikely(put_user(ret, &user_uffdio_continue->mapped))) - return -EFAULT; - goto out; - } - - ret = -EFAULT; - if (copy_from_user(&uffdio_continue, user_uffdio_continue, - /* don't copy the output fields */ - sizeof(uffdio_continue) - (sizeof(__s64)))) - goto out; - - ret = validate_range(ctx->mm, uffdio_continue.range.start, - uffdio_continue.range.len); - if (ret) - goto out; - - ret = -EINVAL; - if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE | - UFFDIO_CONTINUE_MODE_WP)) - goto out; - if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP) - flags |= MFILL_ATOMIC_WP; - - if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_continue(ctx, uffdio_continue.range.start, - uffdio_continue.range.len, flags); - mmput(ctx->mm); - } else { - return -ESRCH; - } - - if (unlikely(put_user(ret, &user_uffdio_continue->mapped))) - return -EFAULT; - if (ret < 0) - goto out; - - /* len == 0 would wake all */ - VM_WARN_ON_ONCE(!ret); - range.len = ret; - if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) { - range.start = uffdio_continue.range.start; - wake_userfault(ctx, &range); - } - ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN; - -out: - return ret; -} - -static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg) -{ - __s64 ret; - struct uffdio_poison uffdio_poison; - struct uffdio_poison __user *user_uffdio_poison; - struct userfaultfd_wake_range range; - - user_uffdio_poison = (struct uffdio_poison __user *)arg; - - ret = -EAGAIN; - if (unlikely(atomic_read(&ctx->mmap_changing))) { - if (unlikely(put_user(ret, &user_uffdio_poison->updated))) - return -EFAULT; - goto out; - } - - ret = -EFAULT; - if (copy_from_user(&uffdio_poison, user_uffdio_poison, - /* don't copy the output fields */ - sizeof(uffdio_poison) - (sizeof(__s64)))) - goto out; - - ret = validate_range(ctx->mm, uffdio_poison.range.start, - uffdio_poison.range.len); - if (ret) - goto out; - - ret = -EINVAL; - if (uffdio_poison.mode & ~UFFDIO_POISON_MODE_DONTWAKE) - goto out; - - if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_poison(ctx, uffdio_poison.range.start, - uffdio_poison.range.len, 0); - mmput(ctx->mm); - } else { - return -ESRCH; - } - - if (unlikely(put_user(ret, &user_uffdio_poison->updated))) - return -EFAULT; - if (ret < 0) - goto out; - - /* len == 0 would wake all */ - VM_WARN_ON_ONCE(!ret); - range.len = ret; - if (!(uffdio_poison.mode & UFFDIO_POISON_MODE_DONTWAKE)) { - range.start = uffdio_poison.range.start; - wake_userfault(ctx, &range); - } - ret = range.len == uffdio_poison.range.len ? 0 : -EAGAIN; - -out: - return ret; -} - -bool userfaultfd_wp_async(struct vm_area_struct *vma) -{ - return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx); -} - -static inline unsigned int uffd_ctx_features(__u64 user_features) -{ - /* - * For the current set of features the bits just coincide. Set - * UFFD_FEATURE_INITIALIZED to mark the features as enabled. - */ - return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED; -} - -static int userfaultfd_move(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - __s64 ret; - struct uffdio_move uffdio_move; - struct uffdio_move __user *user_uffdio_move; - struct userfaultfd_wake_range range; - struct mm_struct *mm = ctx->mm; - - user_uffdio_move = (struct uffdio_move __user *) arg; - - ret = -EAGAIN; - if (unlikely(atomic_read(&ctx->mmap_changing))) { - if (unlikely(put_user(ret, &user_uffdio_move->move))) - return -EFAULT; - goto out; - } - - if (copy_from_user(&uffdio_move, user_uffdio_move, - /* don't copy "move" last field */ - sizeof(uffdio_move)-sizeof(__s64))) - return -EFAULT; - - /* Do not allow cross-mm moves. */ - if (mm != current->mm) - return -EINVAL; - - ret = validate_range(mm, uffdio_move.dst, uffdio_move.len); - if (ret) - return ret; - - ret = validate_range(mm, uffdio_move.src, uffdio_move.len); - if (ret) - return ret; - - if (uffdio_move.mode & ~(UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES| - UFFDIO_MOVE_MODE_DONTWAKE)) - return -EINVAL; - - if (mmget_not_zero(mm)) { - ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src, - uffdio_move.len, uffdio_move.mode); - mmput(mm); - } else { - return -ESRCH; - } - - if (unlikely(put_user(ret, &user_uffdio_move->move))) - return -EFAULT; - if (ret < 0) - goto out; - - /* len == 0 would wake all */ - VM_WARN_ON(!ret); - range.len = ret; - if (!(uffdio_move.mode & UFFDIO_MOVE_MODE_DONTWAKE)) { - range.start = uffdio_move.dst; - wake_userfault(ctx, &range); - } - ret = range.len == uffdio_move.len ? 0 : -EAGAIN; - -out: - return ret; -} - -/* - * userland asks for a certain API version and we return which bits - * and ioctl commands are implemented in this kernel for such API - * version or -EINVAL if unknown. - */ -static int userfaultfd_api(struct userfaultfd_ctx *ctx, - unsigned long arg) -{ - struct uffdio_api uffdio_api; - void __user *buf = (void __user *)arg; - unsigned int ctx_features; - int ret; - __u64 features; - - ret = -EFAULT; - if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api))) - goto out; - features = uffdio_api.features; - ret = -EINVAL; - if (uffdio_api.api != UFFD_API) - goto err_out; - ret = -EPERM; - if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE)) - goto err_out; - - /* WP_ASYNC relies on WP_UNPOPULATED, choose it unconditionally */ - if (features & UFFD_FEATURE_WP_ASYNC) - features |= UFFD_FEATURE_WP_UNPOPULATED; - - /* report all available features and ioctls to userland */ - uffdio_api.features = UFFD_API_FEATURES; -#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR - uffdio_api.features &= - ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); -#endif - if (!pgtable_supports_uffd_wp()) - uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; - - if (!uffd_supports_wp_marker()) { - uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM; - uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; - uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; - } - - ret = -EINVAL; - if (features & ~uffdio_api.features) - goto err_out; - - uffdio_api.ioctls = UFFD_API_IOCTLS; - ret = -EFAULT; - if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) - goto out; - - /* only enable the requested features for this uffd context */ - ctx_features = uffd_ctx_features(features); - ret = -EINVAL; - if (cmpxchg(&ctx->features, 0, ctx_features) != 0) - goto err_out; - - ret = 0; -out: - return ret; -err_out: - memset(&uffdio_api, 0, sizeof(uffdio_api)); - if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) - ret = -EFAULT; - goto out; -} - -static long userfaultfd_ioctl(struct file *file, unsigned cmd, - unsigned long arg) -{ - int ret = -EINVAL; - struct userfaultfd_ctx *ctx = file->private_data; - - if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) - return -EINVAL; - - switch(cmd) { - case UFFDIO_API: - ret = userfaultfd_api(ctx, arg); - break; - case UFFDIO_REGISTER: - ret = userfaultfd_register(ctx, arg); - break; - case UFFDIO_UNREGISTER: - ret = userfaultfd_unregister(ctx, arg); - break; - case UFFDIO_WAKE: - ret = userfaultfd_wake(ctx, arg); - break; - case UFFDIO_COPY: - ret = userfaultfd_copy(ctx, arg); - break; - case UFFDIO_ZEROPAGE: - ret = userfaultfd_zeropage(ctx, arg); - break; - case UFFDIO_MOVE: - ret = userfaultfd_move(ctx, arg); - break; - case UFFDIO_WRITEPROTECT: - ret = userfaultfd_writeprotect(ctx, arg); - break; - case UFFDIO_CONTINUE: - ret = userfaultfd_continue(ctx, arg); - break; - case UFFDIO_POISON: - ret = userfaultfd_poison(ctx, arg); - break; - } - return ret; -} - -#ifdef CONFIG_PROC_FS -static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) -{ - struct userfaultfd_ctx *ctx = f->private_data; - wait_queue_entry_t *wq; - unsigned long pending = 0, total = 0; - - spin_lock_irq(&ctx->fault_pending_wqh.lock); - list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) { - pending++; - total++; - } - list_for_each_entry(wq, &ctx->fault_wqh.head, entry) { - total++; - } - spin_unlock_irq(&ctx->fault_pending_wqh.lock); - - /* - * If more protocols will be added, there will be all shown - * separated by a space. Like this: - * protocols: aa:... bb:... - */ - seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", - pending, total, UFFD_API, ctx->features, - UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); -} -#endif - -static const struct file_operations userfaultfd_fops = { -#ifdef CONFIG_PROC_FS - .show_fdinfo = userfaultfd_show_fdinfo, -#endif - .release = userfaultfd_release, - .poll = userfaultfd_poll, - .read_iter = userfaultfd_read_iter, - .unlocked_ioctl = userfaultfd_ioctl, - .compat_ioctl = compat_ptr_ioctl, - .llseek = noop_llseek, -}; - -static void init_once_userfaultfd_ctx(void *mem) -{ - struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem; - - init_waitqueue_head(&ctx->fault_pending_wqh); - init_waitqueue_head(&ctx->fault_wqh); - init_waitqueue_head(&ctx->event_wqh); - init_waitqueue_head(&ctx->fd_wqh); - seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock); -} - -static int new_userfaultfd(int flags) -{ - struct userfaultfd_ctx *ctx __free(kfree) = NULL; - - VM_WARN_ON_ONCE(!current->mm); - - /* Check the UFFD_* constants for consistency. */ - BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS); - - if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY)) - return -EINVAL; - - ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); - if (!ctx) - return -ENOMEM; - - refcount_set(&ctx->refcount, 1); - ctx->flags = flags; - ctx->features = 0; - ctx->released = false; - init_rwsem(&ctx->map_changing_lock); - atomic_set(&ctx->mmap_changing, 0); - ctx->mm = current->mm; - - FD_PREPARE(fdf, flags & UFFD_SHARED_FCNTL_FLAGS, - anon_inode_create_getfile("[userfaultfd]", &userfaultfd_fops, ctx, - O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), - NULL)); - if (fdf.err) - return fdf.err; - - /* prevent the mm struct to be freed */ - mmgrab(ctx->mm); - fd_prepare_file(fdf)->f_mode |= FMODE_NOWAIT; - retain_and_null_ptr(ctx); - return fd_publish(fdf); -} - -static inline bool userfaultfd_syscall_allowed(int flags) -{ - /* Userspace-only page faults are always allowed */ - if (flags & UFFD_USER_MODE_ONLY) - return true; - - /* - * The user is requesting a userfaultfd which can handle kernel faults. - * Privileged users are always allowed to do this. - */ - if (capable(CAP_SYS_PTRACE)) - return true; - - /* Otherwise, access to kernel fault handling is sysctl controlled. */ - return sysctl_unprivileged_userfaultfd; -} - -SYSCALL_DEFINE1(userfaultfd, int, flags) -{ - if (!userfaultfd_syscall_allowed(flags)) - return -EPERM; - - return new_userfaultfd(flags); -} - -static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags) -{ - if (cmd != USERFAULTFD_IOC_NEW) - return -EINVAL; - - return new_userfaultfd(flags); -} - -static const struct file_operations userfaultfd_dev_fops = { - .unlocked_ioctl = userfaultfd_dev_ioctl, - .compat_ioctl = userfaultfd_dev_ioctl, - .owner = THIS_MODULE, - .llseek = noop_llseek, -}; - -static struct miscdevice userfaultfd_misc = { - .minor = MISC_DYNAMIC_MINOR, - .name = "userfaultfd", - .fops = &userfaultfd_dev_fops -}; - -static int __init userfaultfd_init(void) -{ - int ret; - - ret = misc_register(&userfaultfd_misc); - if (ret) - return ret; - - userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache", - sizeof(struct userfaultfd_ctx), - 0, - SLAB_HWCACHE_ALIGN|SLAB_PANIC, - init_once_userfaultfd_ctx); -#ifdef CONFIG_SYSCTL - register_sysctl_init("vm", vm_userfaultfd_table); -#endif - return 0; -} -__initcall(userfaultfd_init); diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c index 42bedc4ec7af..c5bd3271aa96 100644 --- a/fs/vboxsf/dir.c +++ b/fs/vboxsf/dir.c @@ -477,4 +477,5 @@ const struct inode_operations vboxsf_dir_iops = { .symlink = vboxsf_dir_symlink, .getattr = vboxsf_getattr, .setattr = vboxsf_setattr, + .fileattr_get = vboxsf_fileattr_get, }; diff --git a/fs/vboxsf/file.c b/fs/vboxsf/file.c index 7a7a3fbb2651..943953867e18 100644 --- a/fs/vboxsf/file.c +++ b/fs/vboxsf/file.c @@ -222,7 +222,8 @@ const struct file_operations vboxsf_reg_fops = { const struct inode_operations vboxsf_reg_iops = { .getattr = vboxsf_getattr, - .setattr = vboxsf_setattr + .setattr = vboxsf_setattr, + .fileattr_get = vboxsf_fileattr_get, }; static int vboxsf_read_folio(struct file *file, struct folio *folio) @@ -389,5 +390,6 @@ static const char *vboxsf_get_link(struct dentry *dentry, struct inode *inode, } const struct inode_operations vboxsf_lnk_iops = { - .get_link = vboxsf_get_link + .get_link = vboxsf_get_link, + .fileattr_get = vboxsf_fileattr_get, }; diff --git a/fs/vboxsf/super.c b/fs/vboxsf/super.c index a618cb093e00..a61fbab51d37 100644 --- a/fs/vboxsf/super.c +++ b/fs/vboxsf/super.c @@ -185,6 +185,13 @@ static int vboxsf_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto fail_unmap; + /* + * A failed query leaves sbi->case_insensitive false, so the + * mount defaults to reporting case-sensitive behavior. Do not + * fail the mount over an advisory attribute. + */ + vboxsf_query_case_sensitive(sbi); + sb->s_magic = VBOXSF_SUPER_MAGIC; sb->s_blocksize = 1024; sb->s_maxbytes = MAX_LFS_FILESIZE; diff --git a/fs/vboxsf/utils.c b/fs/vboxsf/utils.c index 440e8c50629d..298bfc93255c 100644 --- a/fs/vboxsf/utils.c +++ b/fs/vboxsf/utils.c @@ -11,6 +11,7 @@ #include <linux/sizes.h> #include <linux/pagemap.h> #include <linux/vfs.h> +#include <linux/fileattr.h> #include "vfsmod.h" struct inode *vboxsf_new_inode(struct super_block *sb) @@ -567,3 +568,32 @@ int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d, return err; } + +int vboxsf_query_case_sensitive(struct vboxsf_sbi *sbi) +{ + struct shfl_volinfo volinfo = {}; + u32 buf_len; + int err; + + buf_len = sizeof(volinfo); + err = vboxsf_fsinfo(sbi->root, 0, SHFL_INFO_GET | SHFL_INFO_VOLUME, + &buf_len, &volinfo); + if (err) + return err; + if (buf_len < sizeof(volinfo)) + return 0; + + sbi->case_insensitive = !volinfo.properties.case_sensitive; + return 0; +} + +int vboxsf_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb); + + if (sbi->case_insensitive) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + return 0; +} diff --git a/fs/vboxsf/vfsmod.h b/fs/vboxsf/vfsmod.h index 05973eb89d52..b61afd0ce842 100644 --- a/fs/vboxsf/vfsmod.h +++ b/fs/vboxsf/vfsmod.h @@ -47,6 +47,7 @@ struct vboxsf_sbi { u32 next_generation; u32 root; int bdi_id; + bool case_insensitive; }; /* per-inode information */ @@ -111,6 +112,11 @@ void vboxsf_dir_info_free(struct vboxsf_dir_info *p); int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d, u64 handle); +int vboxsf_query_case_sensitive(struct vboxsf_sbi *sbi); + +struct file_kattr; +int vboxsf_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + /* from vboxsf_wrappers.c */ int vboxsf_connect(void); void vboxsf_disconnect(void); diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 6e6854c19078..881d46f25e08 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -53,6 +53,9 @@ struct merkle_tree_params { u64 tree_size; /* Merkle tree size in bytes */ unsigned long tree_pages; /* Merkle tree size in pages */ + /* the hash of an all-zeroes block */ + u8 zero_digest[FS_VERITY_MAX_DIGEST_SIZE]; + /* * Starting block index for each tree level, ordered from leaf level (0) * to root level ('num_levels - 1') diff --git a/fs/verity/measure.c b/fs/verity/measure.c index 6a35623ebdf0..cfe2d5e535f9 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -68,8 +68,8 @@ EXPORT_SYMBOL_GPL(fsverity_ioctl_measure); * @alg: (out) the digest's algorithm, as a FS_VERITY_HASH_ALG_* value * @halg: (out) the digest's algorithm, as a HASH_ALGO_* value * - * Retrieves the fsverity digest of the given file. The file must have been - * opened at least once since the inode was last loaded into the inode cache; + * Retrieves the fsverity digest of the given file. The + * fsverity_ensure_verity_info() must be called on the inode beforehand; * otherwise this function will not recognize when fsverity is enabled. * * The file's fsverity digest consists of @raw_digest in combination with either @@ -118,9 +118,9 @@ __bpf_kfunc_start_defs(); * * Return: 0 on success, a negative value on error. */ -__bpf_kfunc int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr *digest_p) +__bpf_kfunc int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynptr *digest_p) { - struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p; + const struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p; const struct inode *inode = file_inode(file); u32 dynptr_sz = __bpf_dynptr_size(digest_ptr); struct fsverity_digest *arg; diff --git a/fs/verity/open.c b/fs/verity/open.c index dfa0d1afe0fe..d0c56a7faa3b 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -153,6 +153,9 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, goto out_err; } + fsverity_hash_block(params, page_address(ZERO_PAGE(0)), + params->zero_digest); + params->tree_size = offset << log_blocksize; params->tree_pages = PAGE_ALIGN(params->tree_size) >> PAGE_SHIFT; return 0; diff --git a/fs/verity/pagecache.c b/fs/verity/pagecache.c index 1819314ecaa3..99f5f53eea98 100644 --- a/fs/verity/pagecache.c +++ b/fs/verity/pagecache.c @@ -2,6 +2,7 @@ /* * Copyright 2019 Google LLC */ +#include "fsverity_private.h" #include <linux/export.h> #include <linux/fsverity.h> @@ -56,3 +57,24 @@ void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index, folio_put(folio); } EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree); + +/** + * fsverity_fill_zerohash() - fill folio with hashes of zero data block + * @folio: folio to fill + * @offset: offset in the folio to start + * @len: length of the range to fill with hashes + * @vi: fsverity info + */ +void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len, + struct fsverity_info *vi) +{ + size_t off = offset; + + WARN_ON_ONCE(!IS_ALIGNED(offset, vi->tree_params.digest_size)); + WARN_ON_ONCE(!IS_ALIGNED(len, vi->tree_params.digest_size)); + + for (; off < (offset + len); off += vi->tree_params.digest_size) + memcpy_to_folio(folio, off, vi->tree_params.zero_digest, + vi->tree_params.digest_size); +} +EXPORT_SYMBOL_GPL(fsverity_fill_zerohash); diff --git a/fs/xattr.c b/fs/xattr.c index 09ecbaaa1660..d58979115200 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -28,6 +28,11 @@ #include "internal.h" +struct sx_key { + const struct list_head *parent; + const char *name; +}; + static const char * strcmp_prefix(const char *a, const char *a_prefix) { @@ -306,7 +311,7 @@ __vfs_setxattr_locked(struct mnt_idmap *idmap, struct dentry *dentry, if (error) goto out; - error = try_break_deleg(inode, delegated_inode); + error = try_break_deleg(inode, 0, delegated_inode); if (error) goto out; @@ -510,9 +515,12 @@ vfs_listxattr(struct dentry *dentry, char *list, size_t size) if (inode->i_op->listxattr) { error = inode->i_op->listxattr(dentry, list, size); } else { - error = security_inode_listsecurity(inode, list, size); - if (size && error > size) - error = -ERANGE; + ssize_t remaining = size; + + error = security_inode_listsecurity(inode, &list, &remaining); + if (error) + return error; + error = size - remaining; } return error; } @@ -564,7 +572,7 @@ __vfs_removexattr_locked(struct mnt_idmap *idmap, if (error) goto out; - error = try_break_deleg(inode, delegated_inode); + error = try_break_deleg(inode, 0, delegated_inode); if (error) goto out; @@ -1269,23 +1277,32 @@ struct simple_xattr *simple_xattr_alloc(const void *value, size_t size) return new_xattr; } +static u32 sx_hashfn(const char *name, const struct list_head *parent, u32 seed) +{ + return jhash(name, strlen(name), jhash(&parent, sizeof(parent), seed)); +} + static u32 simple_xattr_hashfn(const void *data, u32 len, u32 seed) { - const char *name = data; - return jhash(name, strlen(name), seed); + const struct sx_key *key = data; + + return sx_hashfn(key->name, key->parent, seed); } static u32 simple_xattr_obj_hashfn(const void *obj, u32 len, u32 seed) { const struct simple_xattr *xattr = obj; - return jhash(xattr->name, strlen(xattr->name), seed); + + return sx_hashfn(xattr->name, xattr->parent, seed); } static int simple_xattr_obj_cmpfn(struct rhashtable_compare_arg *arg, const void *obj) { const struct simple_xattr *xattr = obj; - return strcmp(xattr->name, arg->key); + const struct sx_key *key = arg->key; + + return xattr->parent != key->parent || strcmp(xattr->name, key->name); } static const struct rhashtable_params simple_xattr_params = { @@ -1298,6 +1315,7 @@ static const struct rhashtable_params simple_xattr_params = { /** * simple_xattr_get - get an xattr object + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @name: the name of the xattr to retrieve * @buffer: the buffer to store the value into @@ -1311,14 +1329,19 @@ static const struct rhashtable_params simple_xattr_params = { * Return: On success the length of the xattr value is returned. On error a * negative error code is returned. */ -int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, - void *buffer, size_t size) +int simple_xattr_get(struct simple_xattr_cache *cache, struct list_head *xattrs, + const char *name, void *buffer, size_t size) { struct simple_xattr *xattr; + struct sx_key key = { .parent = xattrs, .name = name }; + struct rhashtable *ht = READ_ONCE(cache->ht); int ret = -ENODATA; + if (!ht) + return ret; + guard(rcu)(); - xattr = rhashtable_lookup(&xattrs->ht, name, simple_xattr_params); + xattr = rhashtable_lookup(ht, &key, simple_xattr_params); if (xattr) { ret = xattr->size; if (buffer) { @@ -1331,8 +1354,45 @@ int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, return ret; } +static struct rhashtable *simple_xattrs_lazy_alloc(struct simple_xattr_cache *cache, + const void *value, int flags) +{ + struct rhashtable *oldht, *ht = READ_ONCE(cache->ht); + int err; + + if (unlikely(!ht)) { + if (!value) + return (flags & XATTR_REPLACE) ? ERR_PTR(-ENODATA) : NULL; + + ht = kzalloc_obj(*ht); + if (!ht) + return ERR_PTR(-ENOMEM); + + err = rhashtable_init(ht, &simple_xattr_params); + if (err) { + kfree(ht); + return ERR_PTR(err); + } + + /* + * Provides release semantics on success, so that use of a + * non-NULL READ_ONCE(cache->ht) will be ordered relative to the + * above initialization, due to implicit address dependency. + */ + oldht = cmpxchg_release(&cache->ht, NULL, ht); + if (oldht) { + /* Race lost */ + rhashtable_destroy(ht); + kfree(ht); + ht = oldht; + } + } + return ht; +} + /** * simple_xattr_set - set an xattr object + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @name: the name of the xattr to retrieve * @value: the value to store along the xattr @@ -1362,45 +1422,58 @@ int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, * Return: On success, the removed or replaced xattr is returned, to be freed * by the caller; or NULL if none. On failure a negative error code is returned. */ -struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs, +struct simple_xattr *simple_xattr_set(struct simple_xattr_cache *cache, struct list_head *xattrs, const char *name, const void *value, size_t size, int flags) { + struct sx_key key = { .parent = xattrs, .name = name }; struct simple_xattr *old_xattr = NULL; + struct rhashtable *ht; int err; + ht = simple_xattrs_lazy_alloc(cache, value, flags); + if (IS_ERR_OR_NULL(ht)) + return ERR_CAST(ht); + CLASS(simple_xattr, new_xattr)(value, size); if (IS_ERR(new_xattr)) return new_xattr; if (new_xattr) { + new_xattr->parent = xattrs; new_xattr->name = kstrdup(name, GFP_KERNEL_ACCOUNT); if (!new_xattr->name) return ERR_PTR(-ENOMEM); } - /* Lookup is safe without RCU here since writes are serialized. */ - old_xattr = rhashtable_lookup_fast(&xattrs->ht, name, - simple_xattr_params); - + /* + * Hash table lookup/replace/remove will grab RCU read lock themselves. + * This makes sure that hash table lookup is safe against concurrent + * modification on another inode. + */ + old_xattr = rhashtable_lookup_fast(ht, &key, simple_xattr_params); if (old_xattr) { /* Fail if XATTR_CREATE is requested and the xattr exists. */ if (flags & XATTR_CREATE) return ERR_PTR(-EEXIST); if (new_xattr) { - err = rhashtable_replace_fast(&xattrs->ht, + err = rhashtable_replace_fast(ht, &old_xattr->hash_node, &new_xattr->hash_node, simple_xattr_params); if (err) return ERR_PTR(err); + + list_replace_rcu(&old_xattr->node, &new_xattr->node); } else { - err = rhashtable_remove_fast(&xattrs->ht, + err = rhashtable_remove_fast(ht, &old_xattr->hash_node, simple_xattr_params); if (err) return ERR_PTR(err); + + list_del_rcu(&old_xattr->node); } } else { /* Fail if XATTR_REPLACE is requested but no xattr is found. */ @@ -1412,11 +1485,13 @@ struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs, * new value simply insert it. */ if (new_xattr) { - err = rhashtable_insert_fast(&xattrs->ht, + err = rhashtable_insert_fast(ht, &new_xattr->hash_node, simple_xattr_params); if (err) return ERR_PTR(err); + + list_add_tail_rcu(&new_xattr->node, xattrs); } /* @@ -1453,6 +1528,7 @@ static inline int simple_xattr_limits_inc(struct simple_xattr_limits *limits, /** * simple_xattr_set_limited - set an xattr with per-inode user.* limits + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @limits: per-inode limit counters for user.* xattrs * @name: the name of the xattr to set or remove @@ -1467,7 +1543,7 @@ static inline int simple_xattr_limits_inc(struct simple_xattr_limits *limits, * Return: On success zero is returned. On failure a negative error code is * returned. */ -int simple_xattr_set_limited(struct simple_xattrs *xattrs, +int simple_xattr_set_limited(struct simple_xattr_cache *cache, struct list_head *xattrs, struct simple_xattr_limits *limits, const char *name, const void *value, size_t size, int flags) @@ -1481,7 +1557,7 @@ int simple_xattr_set_limited(struct simple_xattrs *xattrs, return ret; } - old_xattr = simple_xattr_set(xattrs, name, value, size, flags); + old_xattr = simple_xattr_set(cache, xattrs, name, value, size, flags); if (IS_ERR(old_xattr)) { if (value) simple_xattr_limits_dec(limits, size); @@ -1527,11 +1603,10 @@ static bool xattr_is_maclabel(const char *name) * Return: On success the required size or the size of the copied xattrs is * returned. On error a negative error code is returned. */ -ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, +ssize_t simple_xattr_list(struct inode *inode, struct list_head *xattrs, char *buffer, size_t size) { bool trusted = ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN); - struct rhashtable_iter iter; struct simple_xattr *xattr; ssize_t remaining_size = size; int err = 0; @@ -1540,7 +1615,7 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, if (err) return err; - err = security_inode_listsecurity(inode, buffer, remaining_size); + err = security_inode_listsecurity(inode, &buffer, &remaining_size); if (err < 0) return err; @@ -1555,17 +1630,8 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, if (!xattrs) return size - remaining_size; - rhashtable_walk_enter(&xattrs->ht, &iter); - rhashtable_walk_start(&iter); - - while ((xattr = rhashtable_walk_next(&iter)) != NULL) { - if (IS_ERR(xattr)) { - if (PTR_ERR(xattr) == -EAGAIN) - continue; - err = PTR_ERR(xattr); - break; - } - + rcu_read_lock(); + list_for_each_entry_rcu(xattr, xattrs, node) { /* skip "trusted." attributes for unprivileged callers */ if (!trusted && xattr_is_trusted(xattr->name)) continue; @@ -1578,15 +1644,14 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, if (err) break; } - - rhashtable_walk_stop(&iter); - rhashtable_walk_exit(&iter); + rcu_read_unlock(); return err ? err : size - remaining_size; } /** * simple_xattr_add - add xattr objects + * @cache: anchor for the hash table * @xattrs: the header of the xattr object * @new_xattr: the xattr object to add * @@ -1597,112 +1662,100 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, * Return: On success zero is returned. On failure a negative error code is * returned. */ -int simple_xattr_add(struct simple_xattrs *xattrs, +int simple_xattr_add(struct simple_xattr_cache *cache, struct list_head *xattrs, struct simple_xattr *new_xattr) { - return rhashtable_insert_fast(&xattrs->ht, &new_xattr->hash_node, - simple_xattr_params); + struct rhashtable *ht; + int err; + + ht = simple_xattrs_lazy_alloc(cache, new_xattr->value, 0); + if (IS_ERR(ht)) + return PTR_ERR(ht); + + new_xattr->parent = xattrs; + err = rhashtable_insert_fast(ht, &new_xattr->hash_node, simple_xattr_params); + if (err) + return err; + + list_add_tail_rcu(&new_xattr->node, xattrs); + return 0; } /** - * simple_xattrs_init - initialize new xattr header - * @xattrs: header to initialize + * simple_xattr_add_limited - add an xattr object, charging per-inode limits + * @cache: anchor for the hash table + * @xattrs: the header of the xattr object + * @limits: per-inode limit counters + * @new_xattr: the xattr object to add * - * Initialize the rhashtable used to store xattr objects. + * Like simple_xattr_add(), but also accounts @new_xattr against @limits so + * that a later removal or replacement of it through simple_xattr_set_limited() + * decrements counters that were actually incremented, rather than underflowing + * them. Use this instead of simple_xattr_add() when seeding initial xattrs + * that share a namespace with the limited set/remove path. * * Return: On success zero is returned. On failure a negative error code is * returned. */ -int simple_xattrs_init(struct simple_xattrs *xattrs) -{ - return rhashtable_init(&xattrs->ht, &simple_xattr_params); -} - -/** - * simple_xattrs_alloc - allocate and initialize a new xattr header - * - * Dynamically allocate a simple_xattrs header and initialize the - * underlying rhashtable. This is intended for consumers that want - * to lazily allocate xattr storage only when the first xattr is set, - * avoiding the per-inode rhashtable overhead when no xattrs are used. - * - * Return: On success a new simple_xattrs is returned. On failure an - * ERR_PTR is returned. - */ -struct simple_xattrs *simple_xattrs_alloc(void) +int simple_xattr_add_limited(struct simple_xattr_cache *cache, + struct list_head *xattrs, + struct simple_xattr_limits *limits, + struct simple_xattr *new_xattr) { - struct simple_xattrs *xattrs __free(kfree) = NULL; - int ret; - - xattrs = kzalloc(sizeof(*xattrs), GFP_KERNEL); - if (!xattrs) - return ERR_PTR(-ENOMEM); + int err; - ret = simple_xattrs_init(xattrs); - if (ret) - return ERR_PTR(ret); + err = simple_xattr_limits_inc(limits, new_xattr->size); + if (err) + return err; - return no_free_ptr(xattrs); + err = simple_xattr_add(cache, xattrs, new_xattr); + if (err) + simple_xattr_limits_dec(limits, new_xattr->size); + return err; } /** - * simple_xattrs_lazy_alloc - get or allocate xattrs for a set operation - * @xattrsp: pointer to the xattrs pointer (may point to NULL) - * @value: value being set (NULL means remove) - * @flags: xattr set flags - * - * For lazily-allocated xattrs on the write path. If no xattrs exist yet - * and this is a remove operation, returns the appropriate result without - * allocating. Otherwise ensures xattrs is allocated and published with - * store-release semantics. + * simple_xattrs_free - free xattrs + * @cache: anchor for the hash table + * @xattrs: xattr header whose xattrs to destroy + * @freed_space: approximate number of bytes of memory freed from @xattrs * - * Return: On success a valid pointer to the xattrs is returned. On - * failure or early-exit an ERR_PTR or NULL is returned. Callers should - * check with IS_ERR_OR_NULL() and propagate with PTR_ERR() which - * correctly returns 0 for the NULL no-op case. + * Destroy all xattrs in @xattrs. When this is called no one can hold a + * reference to any of the xattrs anymore. */ -struct simple_xattrs *simple_xattrs_lazy_alloc(struct simple_xattrs **xattrsp, - const void *value, int flags) +void simple_xattrs_free(struct simple_xattr_cache *cache, struct list_head *xattrs, + size_t *freed_space) { - struct simple_xattrs *xattrs; - - xattrs = READ_ONCE(*xattrsp); - if (xattrs) - return xattrs; - - if (!value) - return (flags & XATTR_REPLACE) ? ERR_PTR(-ENODATA) : NULL; - - xattrs = simple_xattrs_alloc(); - if (!IS_ERR(xattrs)) - smp_store_release(xattrsp, xattrs); - return xattrs; -} + if (freed_space) + *freed_space = 0; -static void simple_xattr_ht_free(void *ptr, void *arg) -{ - struct simple_xattr *xattr = ptr; - size_t *freed_space = arg; + while (!list_empty(xattrs)) { + struct simple_xattr *xattr = list_first_entry(xattrs, typeof(*xattr), node); - if (freed_space) - *freed_space += simple_xattr_space(xattr->name, xattr->size); - simple_xattr_free(xattr); + rhashtable_remove_fast(cache->ht, &xattr->hash_node, simple_xattr_params); + list_del(&xattr->node); + if (freed_space) + *freed_space += simple_xattr_space(xattr->name, xattr->size); + /* + * Free with RCU, since the xattr might still get accessed by + * the hash compare function + */ + simple_xattr_free_rcu(xattr); + } } /** - * simple_xattrs_free - free xattrs - * @xattrs: xattr header whose xattrs to destroy - * @freed_space: approximate number of bytes of memory freed from @xattrs + * simple_xattr_cache_cleanup - free the cache + * @cache: anchor for the hash table * - * Destroy all xattrs in @xattr. When this is called no one can hold a - * reference to any of the xattrs anymore. + * Destroy the cache table, which was lazily allocated on adding the first xattr. */ -void simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space) +void simple_xattr_cache_cleanup(struct simple_xattr_cache *cache) { - might_sleep(); - - if (freed_space) - *freed_space = 0; - rhashtable_free_and_destroy(&xattrs->ht, simple_xattr_ht_free, - freed_space); + if (cache->ht) { + WARN_ON(atomic_read(&cache->ht->nelems)); + rhashtable_destroy(cache->ht); + kfree(cache->ht); + cache->ht = NULL; + } } diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c index dcd2f93b6a6c..6cab66781d66 100644 --- a/fs/xfs/libxfs/xfs_ag.c +++ b/fs/xfs/libxfs/xfs_ag.c @@ -863,32 +863,30 @@ resv_err: return err2; } -void -xfs_growfs_compute_deltas( +/* + * Return the agcount for the new file system size passed in *nb and adjust *nb + * when it has to be reduced because of maximum AG count or because it would + * create a below minimum size AG. + */ +xfs_agnumber_t +xfs_growfs_compute_agcount( struct xfs_mount *mp, - xfs_rfsblock_t nb, - int64_t *deltap, - xfs_agnumber_t *nagcountp) + xfs_rfsblock_t *nb) { - xfs_rfsblock_t nb_div, nb_mod; - int64_t delta; - xfs_agnumber_t nagcount; - - nb_div = nb; - nb_mod = do_div(nb_div, mp->m_sb.sb_agblocks); - if (nb_mod && nb_mod >= XFS_MIN_AG_BLOCKS) - nb_div++; - else if (nb_mod) - nb = nb_div * mp->m_sb.sb_agblocks; - - if (nb_div > XFS_MAX_AGNUMBER + 1) { - nb_div = XFS_MAX_AGNUMBER + 1; - nb = nb_div * mp->m_sb.sb_agblocks; + uint64_t agcount; /* 64-bits wide to catch overflows */ + xfs_extlen_t remainder; + + agcount = div_u64_rem(*nb, mp->m_sb.sb_agblocks, &remainder); + if (agcount >= XFS_MAX_AGNUMBER + 1) { + agcount = XFS_MAX_AGNUMBER + 1; + remainder = 0; + } + *nb = (xfs_rfsblock_t)agcount * mp->m_sb.sb_agblocks; + if (remainder >= XFS_MIN_AG_BLOCKS) { + *nb += remainder; + agcount++; } - nagcount = nb_div; - delta = nb - mp->m_sb.sb_dblocks; - *deltap = delta; - *nagcountp = nagcount; + return agcount; } /* diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h index 16a9b43a3c27..fd22fe598931 100644 --- a/fs/xfs/libxfs/xfs_ag.h +++ b/fs/xfs/libxfs/xfs_ag.h @@ -329,12 +329,11 @@ struct aghdr_init_data { int xfs_ag_init_headers(struct xfs_mount *mp, struct aghdr_init_data *id); int xfs_ag_shrink_space(struct xfs_perag *pag, struct xfs_trans **tpp, xfs_extlen_t delta); -void -xfs_growfs_compute_deltas(struct xfs_mount *mp, xfs_rfsblock_t nb, - int64_t *deltap, xfs_agnumber_t *nagcountp); int xfs_ag_extend_space(struct xfs_perag *pag, struct xfs_trans *tp, xfs_extlen_t len); int xfs_ag_get_geometry(struct xfs_perag *pag, struct xfs_ag_geometry *ageo); +xfs_agnumber_t xfs_growfs_compute_agcount(struct xfs_mount *mp, + xfs_rfsblock_t *nb); static inline xfs_fsblock_t xfs_agbno_to_fsb( diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 93caa1dae501..b3f7b2c34ad7 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -276,7 +276,7 @@ xfs_attr_get( return -EIO; if (!args->owner) - args->owner = args->dp->i_ino; + args->owner = I_INO(args->dp); args->geo = args->dp->i_mount->m_attr_geo; args->whichfork = XFS_ATTR_FORK; xfs_attr_sethash(args); diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 2b78041e8672..86c5c09a5db4 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -1429,7 +1429,7 @@ xfs_attr3_leaf_init( struct xfs_da_args args = { .trans = tp, .dp = dp, - .owner = dp->i_ino, + .owner = I_INO(dp), .geo = dp->i_mount->m_attr_geo, }; diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 7a4c8f1aa76c..d64defeda645 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -602,7 +602,7 @@ xfs_bmap_btree_to_extents( if ((error = xfs_btree_check_block(cur, cblock, 0, cbp))) return error; - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, whichfork); error = xfs_free_extent_later(cur->bc_tp, cbno, 1, &oinfo, XFS_AG_RESV_NONE, 0); if (error) @@ -676,13 +676,12 @@ xfs_bmap_extents_to_btree( memset(&args, 0, sizeof(args)); args.tp = tp; args.mp = mp; - xfs_rmap_ino_bmbt_owner(&args.oinfo, ip->i_ino, whichfork); + xfs_rmap_inode_bmbt_owner(&args.oinfo, ip, whichfork); args.minlen = args.maxlen = args.prod = 1; args.wasdel = wasdel; *logflagsp = 0; - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(mp, ip->i_ino)); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); if (error) goto out_root_realloc; @@ -820,7 +819,7 @@ xfs_bmap_local_to_extents( args.mp = ip->i_mount; args.total = total; args.minlen = args.maxlen = args.prod = 1; - xfs_rmap_ino_owner(&args.oinfo, ip->i_ino, whichfork, 0); + xfs_rmap_inode_owner(&args.oinfo, ip, whichfork, 0); /* * Allocate a block. We know we need only one, since the @@ -828,8 +827,7 @@ xfs_bmap_local_to_extents( */ args.total = total; args.minlen = args.maxlen = args.prod = 1; - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(args.mp, ip->i_ino)); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); if (error) goto done; @@ -976,7 +974,7 @@ xfs_bmap_add_attrfork_local( dargs.total = dargs.geo->fsbcount; dargs.whichfork = XFS_DATA_FORK; dargs.trans = tp; - dargs.owner = ip->i_ino; + dargs.owner = I_INO(ip); return xfs_dir2_sf_to_block(&dargs); } @@ -1110,7 +1108,7 @@ xfs_bmap_complain_bad_rec( xfs_warn(mp, "Bmap BTree record corruption in inode 0x%llx %s fork detected at %pS!", - ip->i_ino, forkname, fa); + I_INO(ip), forkname, fa); xfs_warn(mp, "Offset 0x%llx, start block 0x%llx, block count 0x%llx state 0x%x", irec->br_startoff, irec->br_startblock, irec->br_blockcount, @@ -1143,7 +1141,7 @@ xfs_iread_bmbt_block( num_recs = xfs_btree_get_numrecs(block); if (unlikely(ir->loaded + num_recs > ifp->if_nextents)) { xfs_warn(ip->i_mount, "corrupt dinode %llu, (btree extents).", - (unsigned long long)ip->i_ino); + (unsigned long long)I_INO(ip)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, block, sizeof(*block), __this_address); xfs_bmap_mark_sick(ip, whichfork); @@ -3590,7 +3588,7 @@ xfs_bmap_btalloc_best_length( xfs_extlen_t blen = 0; int error; - ap->blkno = XFS_INO_TO_FSB(args->mp, ap->ip->i_ino); + ap->blkno = XFS_INODE_TO_FSB(ap->ip); if (!xfs_bmap_adjacent(ap)) ap->eof = false; diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c index 1c7165df483a..758a4b1ccf5b 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.c +++ b/fs/xfs/libxfs/xfs_bmap_btree.c @@ -36,10 +36,10 @@ xfs_bmbt_init_block( { if (bp) xfs_btree_init_buf(ip->i_mount, bp, &xfs_bmbt_ops, level, - numrecs, ip->i_ino); + numrecs, I_INO(ip)); else xfs_btree_init_block(ip->i_mount, buf, &xfs_bmbt_ops, level, - numrecs, ip->i_ino); + numrecs, I_INO(ip)); } /* @@ -217,7 +217,7 @@ xfs_bmbt_alloc_block( memset(&args, 0, sizeof(args)); args.tp = cur->bc_tp; args.mp = cur->bc_mp; - xfs_rmap_ino_bmbt_owner(&args.oinfo, cur->bc_ino.ip->i_ino, + xfs_rmap_inode_bmbt_owner(&args.oinfo, cur->bc_ino.ip, cur->bc_ino.whichfork); args.minlen = args.maxlen = args.prod = 1; args.wasdel = cur->bc_flags & XFS_BTREE_BMBT_WASDEL; @@ -280,7 +280,7 @@ xfs_bmbt_free_block( struct xfs_owner_info oinfo; int error; - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, cur->bc_ino.whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, cur->bc_ino.whichfork); error = xfs_free_extent_later(cur->bc_tp, fsbno, 1, &oinfo, XFS_AG_RESV_NONE, 0); if (error) diff --git a/fs/xfs/libxfs/xfs_bmap_btree.h b/fs/xfs/libxfs/xfs_bmap_btree.h index b238d559ab03..e0c870beaf67 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.h +++ b/fs/xfs/libxfs/xfs_bmap_btree.h @@ -89,7 +89,7 @@ xfs_bmbt_key_addr( { return (struct xfs_bmbt_key *) ((char *)block + xfs_bmbt_block_len(mp) + - (index - 1) * sizeof(struct xfs_bmbt_key *)); + (index - 1) * sizeof(struct xfs_bmbt_key)); } static inline xfs_bmbt_ptr_t * diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index 7012f3570c8d..60ef7f08b1d3 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -371,7 +371,7 @@ xfs_btree_check_ptr( case XFS_BTREE_TYPE_INODE: xfs_err(cur->bc_mp, "Inode %llu fork %d: Corrupt %sbt pointer at level %d index %d.", - cur->bc_ino.ip->i_ino, + I_INO(cur->bc_ino.ip), cur->bc_ino.whichfork, cur->bc_ops->name, level, index); break; @@ -1305,7 +1305,7 @@ xfs_btree_owner( case XFS_BTREE_TYPE_MEM: return cur->bc_mem.xfbtree->owner; case XFS_BTREE_TYPE_INODE: - return cur->bc_ino.ip->i_ino; + return I_INO(cur->bc_ino.ip); case XFS_BTREE_TYPE_AG: return cur->bc_group->xg_gno; default: @@ -3129,7 +3129,7 @@ xfs_btree_promote_leaf_iroot( */ broot = cur->bc_ops->broot_realloc(cur, 1); xfs_btree_init_block(cur->bc_mp, broot, cur->bc_ops, - cur->bc_nlevels - 1, 1, cur->bc_ino.ip->i_ino); + cur->bc_nlevels - 1, 1, I_INO(cur->bc_ino.ip)); pp = xfs_btree_ptr_addr(cur, 1, broot); kp = xfs_btree_key_addr(cur, 1, broot); @@ -3243,8 +3243,7 @@ xfs_btree_new_iroot( if (level > 0) aptr = *xfs_btree_ptr_addr(cur, 1, block); else - aptr.l = cpu_to_be64(XFS_INO_TO_FSB(cur->bc_mp, - cur->bc_ino.ip->i_ino)); + aptr.l = cpu_to_be64(XFS_INODE_TO_FSB(cur->bc_ino.ip)); /* Allocate the new block. If we can't do it, we're toast. Give up. */ error = xfs_btree_alloc_block(cur, &aptr, &nptr, stat); @@ -3827,7 +3826,7 @@ xfs_btree_demote_leaf_child( */ broot = cur->bc_ops->broot_realloc(cur, numrecs); xfs_btree_init_block(cur->bc_mp, broot, cur->bc_ops, 0, numrecs, - cur->bc_ino.ip->i_ino); + I_INO(cur->bc_ino.ip)); rp = xfs_btree_rec_addr(cur, 1, broot); crp = xfs_btree_rec_addr(cur, 1, cblock); @@ -5608,9 +5607,8 @@ xfs_btree_alloc_metafile_block( ASSERT(xfs_is_metadir_inode(ip)); - xfs_rmap_ino_bmbt_owner(&args.oinfo, ip->i_ino, cur->bc_ino.whichfork); - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(cur->bc_mp, ip->i_ino)); + xfs_rmap_inode_bmbt_owner(&args.oinfo, ip, cur->bc_ino.whichfork); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); if (error) return error; if (args.fsbno == NULLFSBLOCK) { @@ -5641,7 +5639,7 @@ xfs_btree_free_metafile_block( ASSERT(xfs_is_metadir_inode(ip)); - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, cur->bc_ino.whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, cur->bc_ino.whichfork); error = xfs_free_extent_later(tp, fsbno, 1, &oinfo, XFS_AG_RESV_METAFILE, 0); if (error) diff --git a/fs/xfs/libxfs/xfs_btree_staging.c b/fs/xfs/libxfs/xfs_btree_staging.c index 4300c058807b..c3c7ea54895a 100644 --- a/fs/xfs/libxfs/xfs_btree_staging.c +++ b/fs/xfs/libxfs/xfs_btree_staging.c @@ -309,7 +309,7 @@ xfs_btree_bload_prep_block( /* Initialize it and send it out. */ xfs_btree_init_block(cur->bc_mp, ifp->if_broot, cur->bc_ops, - level, nr_this_block, cur->bc_ino.ip->i_ino); + level, nr_this_block, I_INO(cur->bc_ino.ip)); *bpp = NULL; *blockp = ifp->if_broot; diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index ad801b7bd2dd..9debb95d86fa 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -2780,7 +2780,7 @@ invalid_mapping: error = -EFSCORRUPTED; if (xfs_error_level >= XFS_ERRLEVEL_LOW) { xfs_alert(mp, "%s: bno %u inode %llu", - __func__, bno, dp->i_ino); + __func__, bno, I_INO(dp)); for (i = 0; i < nirecs; i++) { xfs_alert(mp, diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index c6909716b041..89501e8bd2f8 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -878,7 +878,7 @@ xfs_defer_add_barrier( if (dfp) return; - xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type); + dfp = xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type); trace_xfs_defer_add_item(tp->t_mountp, dfp, NULL); } diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 495620cc001f..0c0402a29b6b 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -244,7 +244,7 @@ xfs_dir_init( int error; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); - error = xfs_dir_ino_validate(tp->t_mountp, pdp->i_ino); + error = xfs_dir_ino_validate(tp->t_mountp, I_INO(pdp)); if (error) return error; @@ -255,8 +255,8 @@ xfs_dir_init( args->geo = dp->i_mount->m_dir_geo; args->dp = dp; args->trans = tp; - args->owner = dp->i_ino; - error = xfs_dir2_sf_create(args, pdp->i_ino); + args->owner = I_INO(dp); + error = xfs_dir2_sf_create(args, I_INO(pdp)); kfree(args); return error; } @@ -356,7 +356,7 @@ xfs_dir_createname( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; - args->owner = dp->i_ino; + args->owner = I_INO(dp); rval = xfs_dir_createname_args(args); kfree(args); @@ -447,7 +447,7 @@ xfs_dir_lookup( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->op_flags = XFS_DA_OP_OKNOENT; - args->owner = dp->i_ino; + args->owner = I_INO(dp); if (ci_name) args->op_flags |= XFS_DA_OP_CILOOKUP; @@ -516,7 +516,7 @@ xfs_dir_removename( args->total = total; args->whichfork = XFS_DATA_FORK; args->trans = tp; - args->owner = dp->i_ino; + args->owner = I_INO(dp); rval = xfs_dir_removename_args(args); kfree(args); return rval; @@ -576,7 +576,7 @@ xfs_dir_replace( args->total = total; args->whichfork = XFS_DATA_FORK; args->trans = tp; - args->owner = dp->i_ino; + args->owner = I_INO(dp); rval = xfs_dir_replace_args(args); kfree(args); return rval; @@ -855,7 +855,7 @@ xfs_dir_create_child( xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); xfs_assert_ilocked(dp, XFS_ILOCK_EXCL); - error = xfs_dir_createname(tp, dp, name, ip->i_ino, resblks); + error = xfs_dir_createname(tp, dp, name, I_INO(ip), resblks); if (error) { ASSERT(error != -ENOSPC); return error; @@ -919,14 +919,14 @@ xfs_dir_add_child( if (VFS_I(ip)->i_nlink == 0) { struct xfs_perag *pag; - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); error = xfs_iunlink_remove(tp, pag, ip); xfs_perag_put(pag); if (error) return error; } - error = xfs_dir_createname(tp, dp, name, ip->i_ino, resblks); + error = xfs_dir_createname(tp, dp, name, I_INO(ip), resblks); if (error) return error; @@ -995,7 +995,7 @@ xfs_dir_remove_child( * get freed before the child directory is closed. If the fs * gets shrunk, this can lead to dirent inode validation errors. */ - if (dp->i_ino != tp->t_mountp->m_sb.sb_rootino) { + if (I_INO(dp) != tp->t_mountp->m_sb.sb_rootino) { error = xfs_dir_replace(tp, ip, &xfs_name_dotdot, tp->t_mountp->m_sb.sb_rootino, 0); if (error) @@ -1016,7 +1016,7 @@ xfs_dir_remove_child( if (error) return error; - error = xfs_dir_removename(tp, dp, name, ip->i_ino, resblks); + error = xfs_dir_removename(tp, dp, name, I_INO(ip), resblks); if (error) { ASSERT(error != -ENOENT); return error; @@ -1059,12 +1059,12 @@ xfs_dir_exchange_children( int error; /* Swap inode number for dirent in first parent */ - error = xfs_dir_replace(tp, dp1, name1, ip2->i_ino, spaceres); + error = xfs_dir_replace(tp, dp1, name1, I_INO(ip2), spaceres); if (error) return error; /* Swap inode number for dirent in second parent */ - error = xfs_dir_replace(tp, dp2, name2, ip1->i_ino, spaceres); + error = xfs_dir_replace(tp, dp2, name2, I_INO(ip1), spaceres); if (error) return error; @@ -1078,7 +1078,7 @@ xfs_dir_exchange_children( if (S_ISDIR(VFS_I(ip2)->i_mode)) { error = xfs_dir_replace(tp, ip2, &xfs_name_dotdot, - dp1->i_ino, spaceres); + I_INO(dp1), spaceres); if (error) return error; @@ -1102,7 +1102,7 @@ xfs_dir_exchange_children( if (S_ISDIR(VFS_I(ip1)->i_mode)) { error = xfs_dir_replace(tp, ip1, &xfs_name_dotdot, - dp2->i_ino, spaceres); + I_INO(dp2), spaceres); if (error) return error; @@ -1246,7 +1246,7 @@ xfs_dir_rename_children( ASSERT(VFS_I(du_wip->ip)->i_nlink == 0); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, du_wip->ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(du_wip->ip)); error = xfs_iunlink_remove(tp, pag, du_wip->ip); xfs_perag_put(pag); if (error) @@ -1265,7 +1265,7 @@ xfs_dir_rename_children( * to account for the ".." reference from the new entry. */ error = xfs_dir_createname(tp, target_dp, target_name, - src_ip->i_ino, spaceres); + I_INO(src_ip), spaceres); if (error) return error; @@ -1286,7 +1286,7 @@ xfs_dir_rename_children( * name at the destination directory, remove it first. */ error = xfs_dir_replace(tp, target_dp, target_name, - src_ip->i_ino, spaceres); + I_INO(src_ip), spaceres); if (error) return error; @@ -1320,7 +1320,7 @@ xfs_dir_rename_children( * directory. */ error = xfs_dir_replace(tp, src_ip, &xfs_name_dotdot, - target_dp->i_ino, spaceres); + I_INO(target_dp), spaceres); ASSERT(error != -EEXIST); if (error) return error; @@ -1358,10 +1358,10 @@ xfs_dir_rename_children( * altogether. */ if (du_wip->ip) - error = xfs_dir_replace(tp, src_dp, src_name, du_wip->ip->i_ino, + error = xfs_dir_replace(tp, src_dp, src_name, I_INO(du_wip->ip), spaceres); else - error = xfs_dir_removename(tp, src_dp, src_name, src_ip->i_ino, + error = xfs_dir_removename(tp, src_dp, src_name, I_INO(src_ip), spaceres); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c index 80ba94f51e5c..aecbab61014c 100644 --- a/fs/xfs/libxfs/xfs_dir2_data.c +++ b/fs/xfs/libxfs/xfs_dir2_data.c @@ -382,6 +382,7 @@ xfs_dir3_data_write_verify( struct xfs_mount *mp = bp->b_mount; struct xfs_buf_log_item *bip = bp->b_log_item; struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + struct xfs_dir3_data_hdr *datahdr3 = bp->b_addr; xfs_failaddr_t fa; fa = xfs_dir3_data_verify(bp); @@ -396,6 +397,11 @@ xfs_dir3_data_write_verify( if (bip) hdr3->lsn = cpu_to_be64(bip->bli_item.li_lsn); + /* + * Zero padding that may be stale from old kernels. + */ + datahdr3->pad = 0; + xfs_buf_update_cksum(bp, XFS_DIR3_DATA_CRC_OFF); } @@ -728,7 +734,6 @@ xfs_dir3_data_init( struct xfs_dir2_data_unused *dup; struct xfs_dir2_data_free *bf; int error; - int i; /* * Get the buffer set up for the block. @@ -741,13 +746,16 @@ xfs_dir3_data_init( xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_DATA_BUF); /* - * Initialize the header. + * Initialize the whole directory header region to zero + * so that all padding, bestfree entries, and any + * future header fields are clean. */ hdr = bp->b_addr; + memset(hdr, 0, geo->data_entry_offset); + if (xfs_has_crc(mp)) { struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; - memset(hdr3, 0, sizeof(*hdr3)); hdr3->magic = cpu_to_be32(XFS_DIR3_DATA_MAGIC); hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp)); hdr3->owner = cpu_to_be64(args->owner); @@ -759,10 +767,6 @@ xfs_dir3_data_init( bf = xfs_dir2_data_bestfree_p(mp, hdr); bf[0].offset = cpu_to_be16(geo->data_entry_offset); bf[0].length = cpu_to_be16(geo->blksize - geo->data_entry_offset); - for (i = 1; i < XFS_DIR2_DATA_FD_COUNT; i++) { - bf[i].length = 0; - bf[i].offset = 0; - } /* * Set up an unused entry for the block's body. diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index ed0b5287a44f..19cb536e53ef 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -1739,7 +1739,7 @@ xfs_dir2_node_add_datablk( fbno)) { xfs_alert(mp, "%s: dir ino %llu needed freesp block %lld for data block %lld, got %lld", - __func__, (unsigned long long)dp->i_ino, + __func__, (unsigned long long)I_INO(dp), (long long)xfs_dir2_db_to_fdb(args->geo, *dbno), (long long)*dbno, (long long)fbno); if (fblk) { diff --git a/fs/xfs/libxfs/xfs_dir2_sf.c b/fs/xfs/libxfs/xfs_dir2_sf.c index 1a67cdd6a707..0567cf8b9c1b 100644 --- a/fs/xfs/libxfs/xfs_dir2_sf.c +++ b/fs/xfs/libxfs/xfs_dir2_sf.c @@ -301,7 +301,7 @@ xfs_dir2_block_to_sf( * Skip . */ if (dep->namelen == 1 && dep->name[0] == '.') - ASSERT(be64_to_cpu(dep->inumber) == dp->i_ino); + ASSERT(be64_to_cpu(dep->inumber) == I_INO(dp)); /* * Skip .., but make sure the inode number is right. */ @@ -863,7 +863,7 @@ xfs_dir2_sf_lookup( * Special case for . */ if (args->namelen == 1 && args->name[0] == '.') { - args->inumber = dp->i_ino; + args->inumber = I_INO(dp); args->cmpresult = XFS_CMP_EXACT; args->filetype = XFS_DIR3_FT_DIR; return -EEXIST; diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c index ce767b40482f..bbada0d3cc08 100644 --- a/fs/xfs/libxfs/xfs_dquot_buf.c +++ b/fs/xfs/libxfs/xfs_dquot_buf.c @@ -436,17 +436,27 @@ xfs_dqinode_metadir_create( error = xfs_metadir_create(&upd, S_IFREG); if (error) - return error; + goto out_cancel; xfs_trans_log_inode(upd.tp, upd.ip, XFS_ILOG_CORE); error = xfs_metadir_commit(&upd); if (error) - return error; + goto out_irele; xfs_finish_inode_setup(upd.ip); *ipp = upd.ip; return 0; + +out_cancel: + xfs_metadir_cancel(&upd, error); +out_irele: + /* Have to finish setting up the inode to ensure it's deleted. */ + if (upd.ip) { + xfs_finish_inode_setup(upd.ip); + xfs_irele(upd.ip); + } + return error; } #ifndef __KERNEL__ diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 5d28f4eac527..dcd0bd0b13b4 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -429,7 +429,7 @@ xfs_exchmaps_attr_to_sf( .geo = tp->t_mountp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .trans = tp, - .owner = xmi->xmi_ip2->i_ino, + .owner = I_INO(xmi->xmi_ip2), }; struct xfs_buf *bp; int forkoff; @@ -438,7 +438,7 @@ xfs_exchmaps_attr_to_sf( if (!xfs_attr_is_leaf(xmi->xmi_ip2)) return 0; - error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, 0, + error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), 0, &bp); if (error) return error; @@ -461,7 +461,7 @@ xfs_exchmaps_dir_to_sf( .geo = tp->t_mountp->m_dir_geo, .whichfork = XFS_DATA_FORK, .trans = tp, - .owner = xmi->xmi_ip2->i_ino, + .owner = I_INO(xmi->xmi_ip2), }; struct xfs_dir2_sf_hdr sfh; struct xfs_buf *bp; @@ -471,7 +471,7 @@ xfs_exchmaps_dir_to_sf( if (xfs_dir2_format(&args, &error) != XFS_DIR2_FMT_BLOCK) return error; - error = xfs_dir3_block_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, &bp); + error = xfs_dir3_block_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), &bp); if (error) return error; @@ -711,7 +711,7 @@ xfs_exchmaps_estimate_overhead( return -ENOSPC; /* Can't actually reserve more than UINT_MAX blocks. */ - if (req->resblks > UINT_MAX) + if (resblks > UINT_MAX) return -ENOSPC; req->resblks = resblks; diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 779dac59b1f3..dd0ed046fbe9 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -1276,8 +1276,12 @@ static inline bool xfs_dinode_is_metadir(const struct xfs_dinode *dip) XFS_INO_AGNO_BITS(mp) + XFS_INO_AGINO_BITS(mp) #define XFS_INO_TO_AGNO(mp,i) \ ((xfs_agnumber_t)((i) >> XFS_INO_AGINO_BITS(mp))) +#define XFS_INODE_TO_AGNO(ip) \ + XFS_INO_TO_AGNO((ip)->i_mount, I_INO(ip)) #define XFS_INO_TO_AGINO(mp,i) \ ((xfs_agino_t)(i) & XFS_INO_MASK(XFS_INO_AGINO_BITS(mp))) +#define XFS_INODE_TO_AGINO(ip) \ + XFS_INO_TO_AGINO((ip)->i_mount, I_INO(ip)) #define XFS_INO_TO_AGBNO(mp,i) \ (((xfs_agblock_t)(i) >> XFS_INO_OFFSET_BITS(mp)) & \ XFS_INO_MASK(XFS_INO_AGBNO_BITS(mp))) @@ -1285,6 +1289,8 @@ static inline bool xfs_dinode_is_metadir(const struct xfs_dinode *dip) ((int)(i) & XFS_INO_MASK(XFS_INO_OFFSET_BITS(mp))) #define XFS_INO_TO_FSB(mp,i) \ XFS_AGB_TO_FSB(mp, XFS_INO_TO_AGNO(mp,i), XFS_INO_TO_AGBNO(mp,i)) +#define XFS_INODE_TO_FSB(ip) \ + XFS_INO_TO_FSB((ip)->i_mount, I_INO(ip)) #define XFS_AGINO_TO_INO(mp,a,i) \ (((xfs_ino_t)(a) << XFS_INO_AGINO_BITS(mp)) | (i)) #define XFS_AGINO_TO_AGBNO(mp,i) ((i) >> XFS_INO_OFFSET_BITS(mp)) diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index dcef06ec0a02..ffcdd1f691fd 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -1075,7 +1075,7 @@ xfs_dialloc_check_ino( if (error) return -EAGAIN; - error = xfs_imap_to_bp(pag_mount(pag), tp, &imap, &bp); + error = xfs_read_icluster(pag, tp, imap.im_agbno, &bp); if (error) return -EAGAIN; @@ -1870,7 +1870,7 @@ xfs_dialloc_pick_ag( if (S_ISDIR(mode)) return (atomic_inc_return(&mp->m_agirotor) - 1) % mp->m_maxagi; - start_agno = XFS_INO_TO_AGNO(mp, dp->i_ino); + start_agno = XFS_INODE_TO_AGNO(dp); if (start_agno >= mp->m_maxagi) start_agno = 0; @@ -1895,7 +1895,7 @@ xfs_dialloc( struct xfs_perag *pag; struct xfs_ino_geometry *igeo = M_IGEO(mp); xfs_ino_t ino = NULLFSINO; - xfs_ino_t parent = args->pip ? args->pip->i_ino : 0; + xfs_ino_t parent = args->pip ? I_INO(args->pip) : 0; xfs_agnumber_t agno; xfs_agnumber_t start_agno; umode_t mode = args->mode & S_IFMT; @@ -2511,44 +2511,38 @@ xfs_imap( * inodes in stale state on disk. Hence we have to do a btree lookup * in all cases where an untrusted inode number is passed. */ - if (flags & XFS_IGET_UNTRUSTED) { - error = xfs_imap_lookup(pag, tp, agino, agbno, - &chunk_agbno, &offset_agbno, flags); - if (error) - return error; - goto out_map; - } + if (!(flags & XFS_IGET_UNTRUSTED)) { + /* + * If the inode cluster size is the same or smaller than the + * blocksize, get to the buffer by simple arithmetics. + */ + if (M_IGEO(mp)->blocks_per_cluster == 1) { + cluster_agbno = agbno; + offset = XFS_INO_TO_OFFSET(mp, ino); + ASSERT(offset < mp->m_sb.sb_inopblock); + goto out; + } - /* - * If the inode cluster size is the same as the blocksize or - * smaller we get to the buffer by simple arithmetics. - */ - if (M_IGEO(mp)->blocks_per_cluster == 1) { - offset = XFS_INO_TO_OFFSET(mp, ino); - ASSERT(offset < mp->m_sb.sb_inopblock); - - imap->im_blkno = xfs_agbno_to_daddr(pag, agbno); - imap->im_len = XFS_FSB_TO_BB(mp, 1); - imap->im_boffset = (unsigned short)(offset << - mp->m_sb.sb_inodelog); - return 0; - } + /* + * If the inode chunks are aligned, use simple maths to find the + * location. + */ + if (M_IGEO(mp)->inoalign_mask) { + offset_agbno = agbno & M_IGEO(mp)->inoalign_mask; + chunk_agbno = agbno - offset_agbno; + goto out_map; + } - /* - * If the inode chunks are aligned then use simple maths to - * find the location. Otherwise we have to do a btree - * lookup to find the location. - */ - if (M_IGEO(mp)->inoalign_mask) { - offset_agbno = agbno & M_IGEO(mp)->inoalign_mask; - chunk_agbno = agbno - offset_agbno; - } else { - error = xfs_imap_lookup(pag, tp, agino, agbno, - &chunk_agbno, &offset_agbno, flags); - if (error) - return error; + /* + * Otherwise we have to do a btree lookup to find the location. + */ } + error = xfs_imap_lookup(pag, tp, agino, agbno, &chunk_agbno, + &offset_agbno, flags); + if (error) + return error; + out_map: ASSERT(agbno >= chunk_agbno); cluster_agbno = chunk_agbno + @@ -2556,24 +2550,14 @@ out_map: M_IGEO(mp)->blocks_per_cluster); offset = ((agbno - cluster_agbno) * mp->m_sb.sb_inopblock) + XFS_INO_TO_OFFSET(mp, ino); - - imap->im_blkno = xfs_agbno_to_daddr(pag, cluster_agbno); - imap->im_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster); +out: + imap->im_agbno = cluster_agbno; imap->im_boffset = (unsigned short)(offset << mp->m_sb.sb_inodelog); - - /* - * If the inode number maps to a block outside the bounds - * of the file system then return NULL rather than calling - * read_buf and panicing when we get an error from the - * driver. - */ - if ((imap->im_blkno + imap->im_len) > - XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks)) { - xfs_alert(mp, - "%s: (im_blkno (0x%llx) + im_len (0x%llx)) > sb_dblocks (0x%llx)", - __func__, (unsigned long long) imap->im_blkno, - (unsigned long long) imap->im_len, - XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks)); + if (imap->im_agbno + M_IGEO(mp)->blocks_per_cluster > + pag_group(pag)->xg_block_count) { + xfs_alert(mp, "inode cluster out of range: %u/%u > %u", + imap->im_agbno, M_IGEO(mp)->blocks_per_cluster, + pag_group(pag)->xg_block_count); return -EINVAL; } return 0; diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 3794e5412eba..336ef843f2fe 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -123,24 +123,24 @@ const struct xfs_buf_ops xfs_inode_buf_ra_ops = { /* - * This routine is called to map an inode to the buffer containing the on-disk - * version of the inode. It returns a pointer to the buffer containing the - * on-disk inode in the bpp parameter. + * Read the inode cluster at @bno and return it in @bpp. */ int -xfs_imap_to_bp( - struct xfs_mount *mp, +xfs_read_icluster( + struct xfs_perag *pag, struct xfs_trans *tp, - struct xfs_imap *imap, + xfs_agblock_t agbno, struct xfs_buf **bpp) { + struct xfs_mount *mp = pag_mount(pag); int error; - error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, imap->im_blkno, - imap->im_len, 0, bpp, &xfs_inode_buf_ops); + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + xfs_agbno_to_daddr(pag, agbno), + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), + 0, bpp, &xfs_inode_buf_ops); if (xfs_metadata_is_sick(error)) - xfs_agno_mark_sick(mp, xfs_daddr_to_agno(mp, imap->im_blkno), - XFS_SICK_AG_INODES); + xfs_agno_mark_sick(mp, pag_agno(pag), XFS_SICK_AG_INODES); return error; } @@ -185,7 +185,7 @@ xfs_inode_from_disk( ASSERT(ip->i_cowfp == NULL); - fa = xfs_dinode_verify(ip->i_mount, ip->i_ino, from); + fa = xfs_dinode_verify(ip->i_mount, I_INO(ip), from); if (fa) { xfs_inode_verifier_error(ip, -EFSCORRUPTED, "dinode", from, sizeof(*from), fa); @@ -358,7 +358,7 @@ xfs_inode_to_disk( to->di_flags2 = cpu_to_be64(ip->i_diflags2); /* also covers the di_used_blocks union arm: */ to->di_cowextsize = cpu_to_be32(ip->i_cowextsize); - to->di_ino = cpu_to_be64(ip->i_ino); + to->di_ino = cpu_to_be64(I_INO(ip)); to->di_lsn = cpu_to_be64(lsn); memset(to->di_pad2, 0, sizeof(to->di_pad2)); uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index 8d43d2641c73..f3624532b023 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -11,16 +11,15 @@ struct xfs_dinode; /* * Inode location information. Stored in the inode and passed to - * xfs_imap_to_bp() to get a buffer and dinode for a given inode. + * xfs_read_icluster() to get a buffer and dinode for a given inode. */ struct xfs_imap { - xfs_daddr_t im_blkno; /* starting BB of inode chunk */ - unsigned short im_len; /* length in BBs of inode chunk */ - unsigned short im_boffset; /* inode offset in block in bytes */ -}; + xfs_agblock_t im_agbno; /* starting agbno of inode cluster */ + unsigned short im_boffset; /* offset in inode cluster in bytes */ +} __packed; -int xfs_imap_to_bp(struct xfs_mount *mp, struct xfs_trans *tp, - struct xfs_imap *imap, struct xfs_buf **bpp); +int xfs_read_icluster(struct xfs_perag *pag, struct xfs_trans *tp, + xfs_agblock_t agbno, struct xfs_buf **bpp); void xfs_dinode_calc_crc(struct xfs_mount *mp, struct xfs_dinode *dip); void xfs_inode_to_disk(struct xfs_inode *ip, struct xfs_dinode *to, xfs_lsn_t lsn); diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index d14a7f2f4c03..606a36526ce2 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -87,7 +87,7 @@ xfs_iformat_local( if (unlikely(size > XFS_DFORK_SIZE(dip, ip->i_mount, whichfork))) { xfs_warn(ip->i_mount, "corrupt inode %llu (bad size %d for local fork, size = %zd).", - (unsigned long long) ip->i_ino, size, + (unsigned long long)I_INO(ip), size, XFS_DFORK_SIZE(dip, ip->i_mount, whichfork)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "xfs_iformat_local", dip, sizeof(*dip), @@ -126,7 +126,7 @@ xfs_iformat_extents( */ if (unlikely(size < 0 || size > XFS_DFORK_SIZE(dip, mp, whichfork))) { xfs_warn(ip->i_mount, "corrupt inode %llu ((a)extents = %llu).", - ip->i_ino, nex); + I_INO(ip), nex); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "xfs_iformat_extents(1)", dip, sizeof(*dip), __this_address); @@ -205,7 +205,7 @@ xfs_iformat_btree( ifp->if_nextents > ip->i_nblocks) || level == 0 || level > XFS_BM_MAXLEVELS(mp, whichfork)) { xfs_warn(mp, "corrupt inode %llu (btree).", - (unsigned long long) ip->i_ino); + (unsigned long long)I_INO(ip)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "xfs_iformat_btree", dfp, size, __this_address); diff --git a/fs/xfs/libxfs/xfs_inode_util.c b/fs/xfs/libxfs/xfs_inode_util.c index 551fa51befb6..258ac3d0d486 100644 --- a/fs/xfs/libxfs/xfs_inode_util.c +++ b/fs/xfs/libxfs/xfs_inode_util.c @@ -130,6 +130,8 @@ xfs_ip2xflags( if (xfs_inode_has_attr_fork(ip)) flags |= FS_XFLAG_HASATTR; + if (xfs_has_asciici(ip->i_mount)) + flags |= FS_XFLAG_CASEFOLD; return flags; } @@ -462,10 +464,9 @@ xfs_iunlink_insert_inode( struct xfs_buf *agibp, struct xfs_inode *ip) { - struct xfs_mount *mp = tp->t_mountp; struct xfs_agi *agi = agibp->b_addr; xfs_agino_t next_agino; - xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ip->i_ino); + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); short bucket_index = agino % XFS_AGI_UNLINKED_BUCKETS; int error; @@ -529,7 +530,7 @@ xfs_iunlink( ASSERT(VFS_I(ip)->i_mode != 0); trace_xfs_iunlink(ip); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); /* Get the agi buffer first. It ensures lock ordering on the list. */ error = xfs_read_agi(pag, tp, 0, &agibp); @@ -551,7 +552,7 @@ xfs_iunlink_remove_inode( { struct xfs_mount *mp = tp->t_mountp; struct xfs_agi *agi = agibp->b_addr; - xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ip->i_ino); + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); xfs_agino_t head_agino; short bucket_index = agino % XFS_AGI_UNLINKED_BUCKETS; int error; @@ -653,7 +654,7 @@ xfs_droplink( if (inode->i_nlink == 0) { xfs_info_ratelimited(tp->t_mountp, "Inode 0x%llx link count dropped below zero. Pinning link count.", - ip->i_ino); + I_INO(ip)); set_nlink(inode, XFS_NLINK_PINNED); } if (inode->i_nlink != XFS_NLINK_PINNED) @@ -682,7 +683,7 @@ xfs_bumplink( if (inode->i_nlink == XFS_NLINK_PINNED - 1) xfs_info_ratelimited(tp->t_mountp, "Inode 0x%llx link count exceeded maximum. Pinning link count.", - ip->i_ino); + I_INO(ip)); if (inode->i_nlink != XFS_NLINK_PINNED) inc_nlink(inode); @@ -706,7 +707,7 @@ xfs_inode_uninit( * makes the AGI lock -> unlinked list modification order the same as * used in O_TMPFILE creation. */ - error = xfs_difree(tp, pag, ip->i_ino, xic); + error = xfs_difree(tp, pag, I_INO(ip), xic); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 3f5a24dda907..a4e1b3eb425c 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -52,6 +52,19 @@ typedef uint32_t xlog_tid_t; #define CYCLE_LSN(lsn) ((uint)((lsn)>>32)) #define BLOCK_LSN(lsn) ((uint)(lsn)) +/* + * By comparing each component, we don't have to worry about extra endian issues + * in treating two 32 bit numbers as one 64 bit number + */ +static inline xfs_lsn_t XFS_LSN_CMP(xfs_lsn_t lsn1, xfs_lsn_t lsn2) +{ + if (CYCLE_LSN(lsn1) != CYCLE_LSN(lsn2)) + return CYCLE_LSN(lsn1) < CYCLE_LSN(lsn2) ? -999 : 999; + if (BLOCK_LSN(lsn1) != BLOCK_LSN(lsn2)) + return BLOCK_LSN(lsn1) < BLOCK_LSN(lsn2) ? -999 : 999; + return 0; +} + /* this is used in a spot where we might otherwise double-endian-flip */ #define CYCLE_LSN_DISK(lsn) (((__be32 *)&(lsn))[0]) diff --git a/fs/xfs/libxfs/xfs_metadir.c b/fs/xfs/libxfs/xfs_metadir.c index 3e5c61188927..74c4596ee4cf 100644 --- a/fs/xfs/libxfs/xfs_metadir.c +++ b/fs/xfs/libxfs/xfs_metadir.c @@ -95,7 +95,7 @@ xfs_metadir_lookup( .hashval = xfs_dir2_hashname(mp, xname), .whichfork = XFS_DATA_FORK, .op_flags = XFS_DA_OP_OKNOENT, - .owner = dp->i_ino, + .owner = I_INO(dp), }; int error; diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 3509cc4b2175..8d111c9b6527 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -23,7 +23,6 @@ #include "xfs_attr_sf.h" #include "xfs_bmap.h" #include "xfs_defer.h" -#include "xfs_log.h" #include "xfs_xattr.h" #include "xfs_parent.h" #include "xfs_trans_space.h" @@ -202,7 +201,7 @@ xfs_parent_addname( xfs_inode_to_parent_rec(&ppargs->rec, dp); xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, - child->i_ino, parent_name); + I_INO(child), parent_name); return xfs_attr_setname(&ppargs->args, 0); } @@ -224,7 +223,7 @@ xfs_parent_removename( xfs_inode_to_parent_rec(&ppargs->rec, dp); xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, - child->i_ino, parent_name); + I_INO(child), parent_name); return xfs_attr_removename(&ppargs->args); } @@ -248,7 +247,7 @@ xfs_parent_replacename( xfs_inode_to_parent_rec(&ppargs->rec, old_dp); xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, - child->i_ino, old_name); + I_INO(child), old_name); xfs_inode_to_parent_rec(&ppargs->new_rec, new_dp); @@ -312,7 +311,7 @@ xfs_parent_lookup( struct xfs_da_args *scratch) { memset(scratch, 0, sizeof(struct xfs_da_args)); - xfs_parent_da_args_init(scratch, tp, pptr, ip, ip->i_ino, parent_name); + xfs_parent_da_args_init(scratch, tp, pptr, ip, I_INO(ip), parent_name); return xfs_attr_get_ilocked(scratch); } diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index b8036527cdc7..8eb4de9c5f1a 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -34,7 +34,7 @@ xfs_inode_to_parent_rec( struct xfs_parent_rec *rec, const struct xfs_inode *dp) { - xfs_parent_rec_init(rec, dp->i_ino, VFS_IC(dp)->i_generation); + xfs_parent_rec_init(rec, I_INO(dp), VFS_IC(dp)->i_generation); } extern struct kmem_cache *xfs_parent_args_cache; diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 40c7f0ff6cf3..0ec6ccd8b4dc 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -1414,8 +1414,7 @@ xfs_refcount_finish_one( if (rcur == NULL) { struct xfs_perag *pag = to_perag(ri->ri_group); - error = xfs_alloc_read_agf(pag, tp, - XFS_ALLOC_FLAG_FREEING, &agbp); + error = xfs_alloc_read_agf(pag, tp, 0, &agbp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_rmap.c b/fs/xfs/libxfs/xfs_rmap.c index e78133c908ca..34d218de21a9 100644 --- a/fs/xfs/libxfs/xfs_rmap.c +++ b/fs/xfs/libxfs/xfs_rmap.c @@ -2780,7 +2780,7 @@ xfs_rmap_map_extent( if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) type = XFS_RMAP_MAP_SHARED; - __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV); + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); } /* Unmap an extent out of a file. */ @@ -2800,7 +2800,7 @@ xfs_rmap_unmap_extent( if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) type = XFS_RMAP_UNMAP_SHARED; - __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV); + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); } /* @@ -2826,7 +2826,7 @@ xfs_rmap_convert_extent( if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) type = XFS_RMAP_CONVERT_SHARED; - __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV); + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); } /* Schedule the creation of an rmap for non-file data. */ diff --git a/fs/xfs/libxfs/xfs_rmap.h b/fs/xfs/libxfs/xfs_rmap.h index 5f39f6e53cd1..4afa33575ce2 100644 --- a/fs/xfs/libxfs/xfs_rmap.h +++ b/fs/xfs/libxfs/xfs_rmap.h @@ -21,6 +21,8 @@ xfs_rmap_ino_bmbt_owner( if (whichfork == XFS_ATTR_FORK) oi->oi_flags |= XFS_OWNER_INFO_ATTR_FORK; } +#define xfs_rmap_inode_bmbt_owner(oi, ip, whichfork) \ + xfs_rmap_ino_bmbt_owner(oi, I_INO(ip), whichfork) static inline void xfs_rmap_ino_owner( @@ -35,6 +37,8 @@ xfs_rmap_ino_owner( if (whichfork == XFS_ATTR_FORK) oi->oi_flags |= XFS_OWNER_INFO_ATTR_FORK; } +#define xfs_rmap_inode_owner(oi, ip, whichfork, offset) \ + xfs_rmap_ino_owner(oi, I_INO(ip), whichfork, offset) static inline bool xfs_rmap_should_skip_owner_update( diff --git a/fs/xfs/libxfs/xfs_rtbitmap.c b/fs/xfs/libxfs/xfs_rtbitmap.c index bc4c0a99f4dd..e62786e90f7b 100644 --- a/fs/xfs/libxfs/xfs_rtbitmap.c +++ b/fs/xfs/libxfs/xfs_rtbitmap.c @@ -214,7 +214,7 @@ xfs_rtbuf_get( if (xfs_has_rtgroups(mp)) { struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; - if (hdr->rt_owner != cpu_to_be64(ip->i_ino)) { + if (hdr->rt_owner != cpu_to_be64(I_INO(ip))) { xfs_buf_mark_corrupt(bp); xfs_trans_brelse(args->tp, bp); xfs_rtginode_mark_sick(args->rtg, type); @@ -1409,7 +1409,7 @@ xfs_rtfile_initialize_block( hdr->rt_magic = cpu_to_be32(XFS_RTBITMAP_MAGIC); else hdr->rt_magic = cpu_to_be32(XFS_RTSUMMARY_MAGIC); - hdr->rt_owner = cpu_to_be64(ip->i_ino); + hdr->rt_owner = cpu_to_be64(I_INO(ip)); hdr->rt_blkno = cpu_to_be64(XFS_FSB_TO_DADDR(mp, fsbno)); hdr->rt_lsn = 0; uuid_copy(&hdr->rt_uuid, &mp->m_sb.sb_meta_uuid); diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.c b/fs/xfs/libxfs/xfs_rtrefcount_btree.c index c1518267eb17..f27b80a199ba 100644 --- a/fs/xfs/libxfs/xfs_rtrefcount_btree.c +++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.c @@ -602,7 +602,7 @@ xfs_rtrefcountbt_from_disk( rblocklen = xfs_rtrefcount_broot_space(mp, dblock); xfs_btree_init_block(mp, rblock, &xfs_rtrefcountbt_ops, 0, 0, - ip->i_ino); + I_INO(ip)); rblock->bb_level = dblock->bb_level; rblock->bb_numrecs = dblock->bb_numrecs; @@ -751,7 +751,7 @@ xfs_rtrefcountbt_create( xfs_rtrefcount_broot_space_calc(mp, 0, 0)); if (broot) xfs_btree_init_block(mp, broot, &xfs_rtrefcountbt_ops, 0, 0, - ip->i_ino); + I_INO(ip)); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE | XFS_ILOG_DBROOT); return 0; } diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.c b/fs/xfs/libxfs/xfs_rtrmap_btree.c index 00557b7ef298..c264bc5651c0 100644 --- a/fs/xfs/libxfs/xfs_rtrmap_btree.c +++ b/fs/xfs/libxfs/xfs_rtrmap_btree.c @@ -839,7 +839,7 @@ xfs_rtrmapbt_from_disk( unsigned int numrecs; unsigned int maxrecs; - xfs_btree_init_block(mp, rblock, &xfs_rtrmapbt_ops, 0, 0, ip->i_ino); + xfs_btree_init_block(mp, rblock, &xfs_rtrmapbt_ops, 0, 0, I_INO(ip)); rblock->bb_level = dblock->bb_level; rblock->bb_numrecs = dblock->bb_numrecs; @@ -981,7 +981,7 @@ xfs_rtrmapbt_create( broot = xfs_broot_realloc(ifp, xfs_rtrmap_broot_space_calc(mp, 0, 0)); if (broot) xfs_btree_init_block(mp, broot, &xfs_rtrmapbt_ops, 0, 0, - ip->i_ino); + I_INO(ip)); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE | XFS_ILOG_DBROOT); return 0; diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c index f9a5966d8048..b0dc3888bf1b 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.c +++ b/fs/xfs/libxfs/xfs_symlink_remote.c @@ -196,7 +196,7 @@ xfs_symlink_local_to_remote( bp->b_ops = &xfs_symlink_buf_ops; buf = bp->b_addr; - buf += xfs_symlink_hdr_set(mp, ip->i_ino, 0, ifp->if_bytes, bp); + buf += xfs_symlink_hdr_set(mp, I_INO(ip), 0, ifp->if_bytes, bp); memcpy(buf, ifp->if_data, ifp->if_bytes); xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsymlink_hdr) + ifp->if_bytes - 1); @@ -277,13 +277,13 @@ xfs_symlink_remote_read( cur_chunk = bp->b_addr; if (xfs_has_crc(mp)) { - if (!xfs_symlink_hdr_ok(ip->i_ino, offset, + if (!xfs_symlink_hdr_ok(I_INO(ip), offset, byte_cnt, bp)) { xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); error = -EFSCORRUPTED; xfs_alert(mp, "symlink header does not match required off/len/owner (0x%x/0x%x,0x%llx)", - offset, byte_cnt, ip->i_ino); + offset, byte_cnt, I_INO(ip)); xfs_buf_relse(bp); goto out; diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index ae9ed5f280d0..2554494847ff 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -652,7 +652,7 @@ xrep_agfl_fill( while (agbno < start + len && af->fl_off < af->flcount) af->agfl_bno[af->fl_off++] = cpu_to_be32(agbno++); - error = xagb_bitmap_set(&af->used_extents, start, agbno - 1); + error = xagb_bitmap_set(&af->used_extents, start, agbno - start); if (error) return error; @@ -1116,9 +1116,7 @@ xrep_iunlink_igrab( struct xfs_perag *pag, struct xfs_inode *ip) { - struct xfs_mount *mp = pag_mount(pag); - - if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag_agno(pag)) + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) return false; if (!xfs_inode_on_unlinked_list(ip)) @@ -1136,18 +1134,14 @@ xrep_iunlink_visit( struct xrep_agi *ragi, unsigned int batch_idx) { - struct xfs_mount *mp = ragi->sc->mp; struct xfs_inode *ip = ragi->lookup_batch[batch_idx]; - xfs_agino_t agino; - unsigned int bucket; + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); + unsigned int bucket = agino % XFS_AGI_UNLINKED_BUCKETS; int error; - ASSERT(XFS_INO_TO_AGNO(mp, ip->i_ino) == pag_agno(ragi->sc->sa.pag)); + ASSERT(XFS_INODE_TO_AGNO(ip) == pag_agno(ragi->sc->sa.pag)); ASSERT(xfs_inode_on_unlinked_list(ip)); - agino = XFS_INO_TO_AGINO(mp, ip->i_ino); - bucket = agino % XFS_AGI_UNLINKED_BUCKETS; - trace_xrep_iunlink_visit(ragi->sc->sa.pag, bucket, ragi->iunlink_heads[bucket], ip); @@ -1213,10 +1207,10 @@ xrep_iunlink_mark_incore( * us to see this inode, so another lookup from the * same index will not find it again. */ - if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag_agno(pag)) + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) continue; - first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); - if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) + first_index = XFS_INO_TO_AGINO(mp, I_INO(ip) + 1); + if (first_index < XFS_INODE_TO_AGINO(ip)) done = true; } diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 390ac2e11ee0..d5226939c810 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -182,7 +182,7 @@ xchk_xattr_actor( .namelen = namelen, .trans = sc->tp, .valuelen = valuelen, - .owner = ip->i_ino, + .owner = I_INO(ip), }; struct xchk_xattr_buf *ab; int error = 0; @@ -199,7 +199,7 @@ xchk_xattr_actor( if (attr_flags & XFS_ATTR_INCOMPLETE) { /* Incomplete attr key, just mark the inode for preening. */ - xchk_ino_set_preen(sc, ip->i_ino); + xchk_ino_set_preen(sc, I_INO(ip)); return 0; } diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 7d00a1ce0bac..be627ab655ad 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -329,7 +329,7 @@ xrep_xattr_salvage_remote_attr( .dp = rx->sc->ip, .index = ent_idx, .geo = rx->sc->mp->m_attr_geo, - .owner = rx->sc->ip->i_ino, + .owner = I_INO(rx->sc->ip), .attr_filter = ent->flags & XFS_ATTR_NSP_ONDISK_MASK, .namelen = rentry->namelen, .name = rentry->name, @@ -590,7 +590,7 @@ xrep_xattr_recover_block( * as much as we can from the block. */ if (info->magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC) && xrep_buf_verify_struct(bp, &xfs_attr3_leaf_buf_ops) && - xfs_attr3_leaf_header_check(bp, rx->sc->ip->i_ino) == NULL) + xfs_attr3_leaf_header_check(bp, I_INO(rx->sc->ip)) == NULL) error = xrep_xattr_recover_leaf(rx, bp); /* @@ -617,7 +617,7 @@ xrep_xattr_insert_rec( .attr_filter = key->flags, .namelen = key->namelen, .valuelen = key->valuelen, - .owner = rx->sc->ip->i_ino, + .owner = I_INO(rx->sc->ip), .geo = rx->sc->mp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .op_flags = XFS_DA_OP_OKNOENT, @@ -982,7 +982,7 @@ xrep_xattr_fork_remove( xfs_emerg(sc->mp, "inode 0x%llx attr fork still has %llu attr extents, format %d?!", - ip->i_ino, ifp->if_nextents, ifp->if_format); + I_INO(ip), ifp->if_nextents, ifp->if_format); for_each_xfs_iext(ifp, &icur, &irec) { xfs_err(sc->mp, "[%u]: startoff %llu startblock %llu blockcount %llu state %u", @@ -1116,7 +1116,7 @@ xrep_xattr_replay_pptr_update( trace_xrep_xattr_replay_parentadd(sc->tempip, xname, &pptr->pptr_rec); - error = xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, + error = xfs_parent_set(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rx->pptr_args); ASSERT(error != -EEXIST); return error; @@ -1125,7 +1125,7 @@ xrep_xattr_replay_pptr_update( trace_xrep_xattr_replay_parentremove(sc->tempip, xname, &pptr->pptr_rec); - error = xfs_parent_unset(sc->tempip, sc->ip->i_ino, xname, + error = xfs_parent_unset(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rx->pptr_args); ASSERT(error != -ENOATTR); return error; @@ -1256,7 +1256,7 @@ xrep_xattr_live_dirent_update( * repairing, so stash the update for replay against the temporary * file. */ - if (p->ip->i_ino != sc->ip->i_ino) + if (I_INO(p->ip) != I_INO(sc->ip)) return NOTIFY_DONE; mutex_lock(&rx->lock); @@ -1295,7 +1295,7 @@ xrep_xattr_swap_prep( .whichfork = XFS_ATTR_FORK, .trans = sc->tp, .total = 1, - .owner = sc->ip->i_ino, + .owner = I_INO(sc->ip), }; error = xfs_attr_shortform_to_leaf(&args); diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index d40534bf9ef9..70028da1aacc 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -205,7 +205,7 @@ xchk_bmap_xref_rmap( { struct xfs_rmap_irec rmap; unsigned long long rmap_end; - uint64_t owner = info->sc->ip->i_ino; + uint64_t owner = I_INO(info->sc->ip); if (xchk_skip_xref(info->sc->sm)) return; @@ -352,7 +352,7 @@ xchk_bmap_rt_iextent_xref( case XFS_DATA_FORK: xchk_bmap_xref_rmap(info, irec, rgbno); if (!xfs_is_reflink_inode(info->sc->ip)) { - xfs_rmap_ino_owner(&oinfo, info->sc->ip->i_ino, + xfs_rmap_inode_owner(&oinfo, info->sc->ip, info->whichfork, irec->br_startoff); xchk_xref_is_only_rt_owned_by(info->sc, rgbno, irec->br_blockcount, &oinfo); @@ -407,7 +407,7 @@ xchk_bmap_iextent_xref( case XFS_DATA_FORK: xchk_bmap_xref_rmap(info, irec, agbno); if (!xfs_is_reflink_inode(info->sc->ip)) { - xfs_rmap_ino_owner(&oinfo, info->sc->ip->i_ino, + xfs_rmap_inode_owner(&oinfo, info->sc->ip, info->whichfork, irec->br_startoff); xchk_xref_is_only_owned_by(info->sc, agbno, irec->br_blockcount, &oinfo); @@ -419,7 +419,7 @@ xchk_bmap_iextent_xref( break; case XFS_ATTR_FORK: xchk_bmap_xref_rmap(info, irec, agbno); - xfs_rmap_ino_owner(&oinfo, info->sc->ip->i_ino, + xfs_rmap_inode_owner(&oinfo, info->sc->ip, info->whichfork, irec->br_startoff); xchk_xref_is_only_owned_by(info->sc, agbno, irec->br_blockcount, &oinfo); @@ -543,7 +543,7 @@ xchk_bmapbt_rec( for (i = 0; i < bs->cur->bc_nlevels - 1; i++) { block = xfs_btree_get_block(bs->cur, i, &bp); owner = be64_to_cpu(block->bb_u.l.bb_owner); - if (owner != ip->i_ino) + if (owner != I_INO(ip)) xchk_fblock_set_corrupt(bs->sc, info->whichfork, 0); } @@ -600,7 +600,7 @@ xchk_bmap_btree( /* Check the btree structure. */ cur = xfs_bmbt_init_cursor(mp, sc->tp, ip, whichfork); - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, whichfork); error = xchk_btree(sc, cur, xchk_bmapbt_rec, &oinfo, info); xfs_btree_del_cursor(cur, error); out: @@ -628,7 +628,7 @@ xchk_bmap_check_rmap( bool have_map; /* Is this even the right fork? */ - if (rec->rm_owner != sc->ip->i_ino) + if (rec->rm_owner != I_INO(sc->ip)) return 0; if ((sbcri->whichfork == XFS_ATTR_FORK) ^ !!(rec->rm_flags & XFS_RMAP_ATTR_FORK)) @@ -1003,7 +1003,7 @@ xchk_bmap_iext_iter( */ if (nr > 1 && info->whichfork != XFS_COW_FORK && howmany_64(irec->br_blockcount, XFS_MAX_BMBT_EXTLEN) < nr) - xchk_ino_set_preen(info->sc, info->sc->ip->i_ino); + xchk_ino_set_preen(info->sc, I_INO(info->sc->ip)); return true; } @@ -1040,7 +1040,7 @@ xchk_bmap( case XFS_COW_FORK: /* No CoW forks filesystem doesn't support out of place writes */ if (!xfs_has_reflink(mp) && !xfs_has_zoned(mp)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } break; @@ -1052,7 +1052,7 @@ xchk_bmap( * attr here. */ if (!xfs_has_attr(mp)) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); break; default: ASSERT(whichfork == XFS_DATA_FORK); @@ -1137,7 +1137,7 @@ xchk_bmap_data( int error; if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTD_ZAPPED)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1165,7 +1165,7 @@ xchk_bmap_attr( * returning immediately. */ if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTA_ZAPPED)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } diff --git a/fs/xfs/scrub/bmap_repair.c b/fs/xfs/scrub/bmap_repair.c index 822a4af43833..eabffba47776 100644 --- a/fs/xfs/scrub/bmap_repair.c +++ b/fs/xfs/scrub/bmap_repair.c @@ -252,7 +252,7 @@ xrep_bmap_walk_rmap( if (xchk_should_terminate(rb->sc, &error)) return error; - if (rec->rm_owner != rb->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rb->sc->ip)) return 0; error = xrep_bmap_check_fork_rmap(rb, cur, rec); @@ -416,7 +416,7 @@ xrep_bmap_walk_rtrmap( return error; /* Skip extents which are not owned by this inode and fork. */ - if (rec->rm_owner != rb->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rb->sc->ip)) return 0; error = xrep_bmap_check_rtfork_rmap(rb->sc, cur, rec); @@ -760,7 +760,7 @@ xrep_bmap_build_new_fork( * Prepare to construct the new fork by initializing the new btree * structure and creating a fake ifork in the ifakeroot structure. */ - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, rb->whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, rb->whichfork); error = xrep_newbt_init_inode(&rb->new_bmapbt, sc, rb->whichfork, &oinfo); if (error) @@ -833,7 +833,7 @@ xrep_bmap_remove_old_tree( struct xfs_owner_info oinfo; /* Free the old bmbt blocks if they're not in use. */ - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, rb->whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, rb->whichfork); return xrep_reap_fsblocks(sc, &rb->old_bmbt_blocks, &oinfo); } diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 20e63069088b..e5233e31abb7 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -251,6 +251,17 @@ xchk_ino_set_preen( trace_xchk_ino_preen(sc, ino, __return_address); } +/* Record a block indexed by a file fork that could be optimized. */ +void +xchk_fblock_set_preen( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_PREEN; + trace_xchk_fblock_preen(sc, whichfork, offset, __return_address); +} + /* Record something being wrong with the filesystem primary superblock. */ void xchk_set_corrupt( @@ -309,12 +320,12 @@ xchk_ino_set_corrupt( /* Record a corruption while cross-referencing with an inode. */ void -xchk_ino_xref_set_corrupt( +xchk_ip_xref_set_corrupt( struct xfs_scrub *sc, - xfs_ino_t ino) + struct xfs_inode *ip) { sc->sm->sm_flags |= XFS_SCRUB_OFLAG_XCORRUPT; - trace_xchk_ino_error(sc, ino, __return_address); + trace_xchk_ino_error(sc, I_INO(ip), __return_address); } /* Record corruption in a block indexed by a file fork. */ @@ -1099,7 +1110,7 @@ xchk_install_live_inode( struct xfs_inode *ip) { if (!igrab(VFS_I(ip))) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); return -EFSCORRUPTED; } @@ -1130,7 +1141,7 @@ xchk_iget_for_scrubbing( ASSERT(sc->tp == NULL); /* We want to scan the inode we already had opened. */ - if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == ip_in->i_ino) + if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == I_INO(ip_in)) return xchk_install_live_inode(sc, ip_in); /* @@ -1428,13 +1439,13 @@ xchk_metadata_inode_forks( /* Metadata inodes don't live on the rt device. */ if (sc->ip->i_diflags & XFS_DIFLAG_REALTIME) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } /* They should never participate in reflink. */ if (xfs_is_reflink_inode(sc->ip)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1451,7 +1462,7 @@ xchk_metadata_inode_forks( &error)) return error; if (shared) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } /* @@ -1460,7 +1471,7 @@ xchk_metadata_inode_forks( */ if (xfs_inode_hasattr(sc->ip)) { if (!xfs_has_metadir(sc->mp)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1562,7 +1573,7 @@ xchk_inode_is_allocated( * This isn't the inode we want. */ spin_lock(&ip->i_flags_lock); - if (ip->i_ino != ino) + if (I_INO(ip) != ino) goto out_skip; trace_xchk_inode_is_allocated(ip); @@ -1670,7 +1681,7 @@ bool xchk_inode_is_sb_rooted(const struct xfs_inode *ip) { return xchk_inode_is_dirtree_root(ip) || - xfs_is_sb_inum(ip->i_mount, ip->i_ino); + xfs_is_sb_inum(ip->i_mount, I_INO(ip)); } /* What is the root directory inumber for this inode? */ @@ -1680,8 +1691,8 @@ xchk_inode_rootdir_inum(const struct xfs_inode *ip) struct xfs_mount *mp = ip->i_mount; if (xfs_is_metadir_inode(ip)) - return mp->m_metadirip->i_ino; - return mp->m_rootip->i_ino; + return I_INO(mp->m_metadirip); + return I_INO(mp->m_rootip); } static int diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index f2ecc68538f0..9d627fd50687 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -25,11 +25,15 @@ bool xchk_fblock_xref_process_error(struct xfs_scrub *sc, void xchk_block_set_preen(struct xfs_scrub *sc, struct xfs_buf *bp); void xchk_ino_set_preen(struct xfs_scrub *sc, xfs_ino_t ino); +void xchk_fblock_set_preen(struct xfs_scrub *sc, + int whichfork, xfs_fileoff_t offset); void xchk_set_corrupt(struct xfs_scrub *sc); void xchk_block_set_corrupt(struct xfs_scrub *sc, struct xfs_buf *bp); void xchk_ino_set_corrupt(struct xfs_scrub *sc, xfs_ino_t ino); +#define xchk_ip_set_corrupt(_sc, _ip) \ + xchk_ino_set_corrupt((_sc), I_INO(_ip)) void xchk_fblock_set_corrupt(struct xfs_scrub *sc, int whichfork, xfs_fileoff_t offset); #ifdef CONFIG_XFS_QUOTA @@ -39,8 +43,8 @@ void xchk_qcheck_set_corrupt(struct xfs_scrub *sc, unsigned int dqtype, void xchk_block_xref_set_corrupt(struct xfs_scrub *sc, struct xfs_buf *bp); -void xchk_ino_xref_set_corrupt(struct xfs_scrub *sc, - xfs_ino_t ino); +void xchk_ip_xref_set_corrupt(struct xfs_scrub *sc, + struct xfs_inode *ip); void xchk_fblock_xref_set_corrupt(struct xfs_scrub *sc, int whichfork, xfs_fileoff_t offset); diff --git a/fs/xfs/scrub/cow_repair.c b/fs/xfs/scrub/cow_repair.c index bffc4666ce60..8dd9c0266e21 100644 --- a/fs/xfs/scrub/cow_repair.c +++ b/fs/xfs/scrub/cow_repair.c @@ -29,6 +29,7 @@ #include "xfs_rtalloc.h" #include "xfs_rtbitmap.h" #include "xfs_rtgroup.h" +#include "xfs_bmap_util.h" #include "scrub/xfs_scrub.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -80,12 +81,6 @@ struct xrep_cow { unsigned int next_bno; }; -/* CoW staging extent. */ -struct xrep_cow_extent { - xfs_fsblock_t fsbno; - xfs_extlen_t len; -}; - /* * Mark the part of the file range that corresponds to the given physical * space. Caller must ensure that the physical range is within xc->irec. @@ -231,6 +226,29 @@ xrep_cow_mark_missing_staging_rmap( } /* + * Trim the start and end of the current mapping by up to 1/4 of the length + * and mark that as "bad" to test the cow fork repair mechanism. + */ +static inline int +xrep_cow_debug_replacement( + struct xrep_cow *xc) +{ + xfs_fsblock_t fsbno = xc->irec.br_startblock; + xfs_extlen_t len = xc->irec.br_blockcount; + uint32_t trim; + + /* get_random_u32_below requires a nonzero argument */ + trim = len > 4 ? get_random_u32_below(len / 4) : 0; + len -= trim; + + trim = len > 4 ? get_random_u32_below(len / 4) : 0; + fsbno += trim; + len -= trim; + + return xrep_cow_mark_file_range(xc, fsbno, len); +} + +/* * Find any part of the CoW fork mapping that isn't a single-owner CoW staging * extent and mark the corresponding part of the file range in the bitmap. */ @@ -299,19 +317,17 @@ xrep_cow_find_bad( * If userspace is forcing us to rebuild the CoW fork or someone turned * on the debugging knob, replace everything in the CoW fork. */ - if ((sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) || - XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR)) { + if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR)) + error = xrep_cow_debug_replacement(xc); + else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock, xc->irec.br_blockcount); - if (error) - return error; - } out_sa: xchk_ag_free(sc, &sc->sa); out_pag: xfs_perag_put(pag); - return 0; + return error; } /* @@ -384,13 +400,11 @@ xrep_cow_find_bad_rt( * turned on the debugging knob, replace everything in the * CoW fork and then scan for staging extents in the refcountbt. */ - if ((sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) || - XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR)) { + if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR)) + error = xrep_cow_debug_replacement(xc); + else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock, xc->irec.br_blockcount); - if (error) - goto out_rtg; - } out_sr: xchk_rtgroup_btcur_free(&sc->sr); @@ -407,27 +421,25 @@ out_rtg: STATIC int xrep_cow_alloc( struct xfs_scrub *sc, - xfs_extlen_t maxlen, - struct xrep_cow_extent *repl) + struct xfs_bmbt_irec *del) { struct xfs_alloc_arg args = { .tp = sc->tp, .mp = sc->mp, .oinfo = XFS_RMAP_OINFO_SKIP_UPDATE, .minlen = 1, - .maxlen = maxlen, + .maxlen = del->br_blockcount, .prod = 1, .resv = XFS_AG_RESV_NONE, .datatype = XFS_ALLOC_USERDATA, }; int error; - error = xfs_trans_reserve_more(sc->tp, maxlen, 0); + error = xfs_trans_reserve_more(sc->tp, del->br_blockcount, 0); if (error) return error; - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(sc->mp, sc->ip->i_ino)); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(sc->ip)); if (error) return error; if (args.fsbno == NULLFSBLOCK) @@ -435,8 +447,8 @@ xrep_cow_alloc( xfs_refcount_alloc_cow_extent(sc->tp, false, args.fsbno, args.len); - repl->fsbno = args.fsbno; - repl->len = args.len; + del->br_startblock = args.fsbno; + del->br_blockcount = args.len; return 0; } @@ -447,10 +459,12 @@ xrep_cow_alloc( STATIC int xrep_cow_alloc_rt( struct xfs_scrub *sc, - xfs_extlen_t maxlen, - struct xrep_cow_extent *repl) + struct xfs_bmbt_irec *del) { - xfs_rtxlen_t maxrtx = xfs_rtb_to_rtx(sc->mp, maxlen); + xfs_fsblock_t fsbno; + xfs_rtxlen_t maxrtx = + min(U32_MAX, xfs_blen_to_rtbxlen(sc->mp, del->br_blockcount)); + xfs_extlen_t len; int error; error = xfs_trans_reserve_more(sc->tp, 0, maxrtx); @@ -458,11 +472,14 @@ xrep_cow_alloc_rt( return error; error = xfs_rtallocate_rtgs(sc->tp, NULLRTBLOCK, 1, maxrtx, 1, false, - false, &repl->fsbno, &repl->len); + false, &fsbno, &len); if (error) return error; - xfs_refcount_alloc_cow_extent(sc->tp, true, repl->fsbno, repl->len); + xfs_refcount_alloc_cow_extent(sc->tp, true, fsbno, len); + + del->br_startblock = fsbno; + del->br_blockcount = len; return 0; } @@ -476,19 +493,19 @@ static inline int xrep_cow_find_mapping( struct xrep_cow *xc, struct xfs_iext_cursor *icur, - xfs_fileoff_t startoff, - struct xfs_bmbt_irec *got) + xfs_fileoff_t badoff, + xfs_extlen_t badlen, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep) { struct xfs_inode *ip = xc->sc->ip; struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); - if (!xfs_iext_lookup_extent(ip, ifp, startoff, icur, got)) + if (!xfs_iext_lookup_extent(ip, ifp, badoff, icur, got)) goto bad; + memcpy(rep, got, sizeof(*rep)); - if (got->br_startoff > startoff) - goto bad; - - if (got->br_blockcount == 0) + if (got->br_startoff > badoff) goto bad; if (isnullstartblock(got->br_startblock)) @@ -497,56 +514,28 @@ xrep_cow_find_mapping( if (xfs_bmap_is_written_extent(got)) goto bad; - return 0; -bad: - ASSERT(0); - return -EFSCORRUPTED; -} - -#define REPLACE_LEFT_SIDE (1U << 0) -#define REPLACE_RIGHT_SIDE (1U << 1) - -/* - * Given a CoW fork mapping @got and a replacement mapping @repl, remap the - * beginning of @got with the space described by @rep. - */ -static inline void -xrep_cow_replace_mapping( - struct xfs_inode *ip, - struct xfs_iext_cursor *icur, - const struct xfs_bmbt_irec *got, - const struct xrep_cow_extent *repl) -{ - struct xfs_bmbt_irec new = *got; /* struct copy */ + if (got->br_startoff < badoff) { + const int64_t delta = badoff - got->br_startoff; - ASSERT(repl->len > 0); - ASSERT(!isnullstartblock(got->br_startblock)); + rep->br_blockcount -= delta; + rep->br_startoff += delta; + rep->br_startblock += delta; + } - trace_xrep_cow_replace_mapping(ip, got, repl->fsbno, repl->len); + if (got->br_startoff + got->br_blockcount > badoff + badlen) { + const int64_t delta = (got->br_startoff + got->br_blockcount) - + (badoff + badlen); - if (got->br_blockcount == repl->len) { - /* - * The new extent is a complete replacement for the existing - * extent. Update the COW fork record. - */ - new.br_startblock = repl->fsbno; - xfs_iext_update_extent(ip, BMAP_COWFORK, icur, &new); - return; + rep->br_blockcount -= delta; } - /* - * The new extent can replace the beginning of the COW fork record. - * Move the left side of @got upwards, then insert the new record. - */ - new.br_startoff += repl->len; - new.br_startblock += repl->len; - new.br_blockcount -= repl->len; - xfs_iext_update_extent(ip, BMAP_COWFORK, icur, &new); - - new.br_startoff = got->br_startoff; - new.br_startblock = repl->fsbno; - new.br_blockcount = repl->len; - xfs_iext_insert(ip, icur, &new, BMAP_COWFORK); + if (got->br_blockcount == 0) + goto bad; + + return 0; +bad: + ASSERT(0); + return -EFSCORRUPTED; } /* @@ -560,33 +549,30 @@ xrep_cow_replace_range( xfs_extlen_t *blockcount) { struct xfs_iext_cursor icur; - struct xrep_cow_extent repl; - struct xfs_bmbt_irec got; + struct xfs_bmbt_irec got, rep; struct xfs_scrub *sc = xc->sc; - xfs_fileoff_t nextoff; - xfs_extlen_t alloc_len; + xfs_fsblock_t old_fsbno; int error; /* - * Put the existing CoW fork mapping in @got. If @got ends before - * @rep, truncate @rep so we only replace one extent mapping at a time. + * Put the existing CoW fork mapping in @got, and put in @rep the + * contents of @got trimmed to @startoff/@blockcount. We only want + * to replace the bad region, and only one mapping at a time. */ - error = xrep_cow_find_mapping(xc, &icur, startoff, &got); + error = xrep_cow_find_mapping(xc, &icur, startoff, *blockcount, &got, + &rep); if (error) return error; - nextoff = min(startoff + *blockcount, - got.br_startoff + got.br_blockcount); + old_fsbno = rep.br_startblock; /* * Allocate a replacement extent. If we don't fill all the blocks, * shorten the quantity that will be deleted in this step. */ - alloc_len = min_t(xfs_fileoff_t, XFS_MAX_BMBT_EXTLEN, - nextoff - startoff); if (XFS_IS_REALTIME_INODE(sc->ip)) - error = xrep_cow_alloc_rt(sc, alloc_len, &repl); + error = xrep_cow_alloc_rt(sc, &rep); else - error = xrep_cow_alloc(sc, alloc_len, &repl); + error = xrep_cow_alloc(sc, &rep); if (error) return error; @@ -594,7 +580,7 @@ xrep_cow_replace_range( * Replace the old mapping with the new one, and commit the metadata * changes made so far. */ - xrep_cow_replace_mapping(sc->ip, &icur, &got, &repl); + xfs_bmap_replace_cow_mapping(sc->ip, &icur, &got, &rep); xfs_inode_set_cowblocks_tag(sc->ip); error = xfs_defer_finish(&sc->tp); @@ -603,15 +589,15 @@ xrep_cow_replace_range( /* Note the old CoW staging extents; we'll reap them all later. */ if (XFS_IS_REALTIME_INODE(sc->ip)) - error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks, - got.br_startblock, repl.len); + error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks, old_fsbno, + rep.br_blockcount); else - error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks, - got.br_startblock, repl.len); + error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks, old_fsbno, + rep.br_blockcount); if (error) return error; - *blockcount = repl.len; + *blockcount = rep.br_blockcount; return 0; } diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 1a71d36898b1..d4e5afbde856 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -393,7 +393,7 @@ xchk_da_btree_block( /* Check the owner. */ if (xfs_has_crc(ip->i_mount)) { owner = be64_to_cpu(hdr3->owner); - if (owner != ip->i_ino) + if (owner != I_INO(ip)) xchk_da_set_corrupt(ds, level); } @@ -454,7 +454,12 @@ xchk_da_btree_block( } } - /* XXX: Check hdr3.pad32 once we know how to fix it. */ + if (xfs_has_crc(ip->i_mount)) { + struct xfs_da3_node_hdr *nodehdr3 = blk->bp->b_addr; + + if (nodehdr3->__pad32) + xchk_da_set_preen(ds, level); + } break; default: xchk_da_set_corrupt(ds, level); @@ -519,7 +524,7 @@ xchk_da_btree( ds->dargs.whichfork = whichfork; ds->dargs.trans = sc->tp; ds->dargs.op_flags = XFS_DA_OP_OKNOENT; - ds->dargs.owner = sc->ip->i_ino; + ds->dargs.owner = I_INO(sc->ip); ds->state = xfs_da_state_alloc(&ds->dargs); ds->sc = sc; ds->private = private; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index e09724cd3725..2a037aae904d 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -187,11 +187,11 @@ xchk_dir_check_pptr_fast( if (!lockmode) { struct xchk_dirent save_de = { .namelen = name->len, - .ino = ip->i_ino, + .ino = I_INO(ip), }; /* Couldn't lock the inode, so save the dirent for later. */ - trace_xchk_dir_defer(sc->ip, name, ip->i_ino); + trace_xchk_dir_defer(sc->ip, name, I_INO(ip)); error = xfblob_storename(sd->dir_names, &save_de.name_cookie, name); @@ -254,14 +254,14 @@ xchk_dir_actor( if (xfs_dir2_samename(name, &xfs_name_dot)) { /* If this is "." then check that the inum matches the dir. */ - if (ino != dp->i_ino) + if (ino != I_INO(dp)) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); } else if (xfs_dir2_samename(name, &xfs_name_dotdot)) { /* * If this is ".." in the root inode, check that the inum * matches this dir. */ - if (xchk_inode_is_dirtree_root(dp) && ino != dp->i_ino) + if (xchk_inode_is_dirtree_root(dp) && ino != I_INO(dp)) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); } @@ -482,17 +482,22 @@ xchk_directory_data_bestfree( /* dir block format */ if (lblk != XFS_B_TO_FSBT(mp, XFS_DIR2_DATA_OFFSET)) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); - error = xfs_dir3_block_read(sc->tp, sc->ip, sc->ip->i_ino, &bp); + error = xfs_dir3_block_read(sc->tp, sc->ip, I_INO(sc->ip), &bp); } else { /* dir data format */ - error = xfs_dir3_data_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, + error = xfs_dir3_data_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, 0, &bp); } if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) goto out; xchk_buffer_recheck(sc, bp); - /* XXX: Check xfs_dir3_data_hdr.pad is zero once we start setting it. */ + if (xfs_has_crc(sc->mp)) { + struct xfs_dir3_data_hdr *hdr3 = bp->b_addr; + + if (hdr3->pad) + xchk_fblock_set_preen(sc, XFS_DATA_FORK, lblk); + } if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) goto out_buf; @@ -638,7 +643,7 @@ xchk_directory_leaf1_bestfree( int error; /* Read the free space block. */ - error = xfs_dir3_leaf_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, &bp); + error = xfs_dir3_leaf_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) return error; xchk_buffer_recheck(sc, bp); @@ -744,7 +749,7 @@ xchk_directory_free_bestfree( int error; /* Read the free space block */ - error = xfs_dir2_free_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, &bp); + error = xfs_dir2_free_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) return error; xchk_buffer_recheck(sc, bp); @@ -792,7 +797,7 @@ xchk_directory_blocks( .whichfork = XFS_DATA_FORK, .geo = sc->mp->m_dir_geo, .trans = sc->tp, - .owner = sc->ip->i_ino, + .owner = I_INO(sc->ip), }; struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); struct xfs_mount *mp = sc->mp; @@ -991,7 +996,7 @@ xchk_dir_slow_dirent( */ lockmode = xchk_dir_lock_child(sc, ip); if (lockmode) { - trace_xchk_dir_slowpath(sc->ip, xname, ip->i_ino); + trace_xchk_dir_slowpath(sc->ip, xname, I_INO(ip)); goto check_pptr; } @@ -1002,7 +1007,7 @@ xchk_dir_slow_dirent( xchk_iunlock(sc, sc->ilock_flags); sd->need_revalidate = true; - trace_xchk_dir_ultraslowpath(sc->ip, xname, ip->i_ino); + trace_xchk_dir_ultraslowpath(sc->ip, xname, I_INO(ip)); error = xchk_dir_trylock_for_pptrs(sc, ip, &lockmode); if (error) @@ -1075,7 +1080,7 @@ xchk_directory( /* Plausible size? */ if (sc->ip->i_disk_size < xfs_dir2_sf_hdr_size(0)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 23b80c54aa60..1c088cfba10e 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -304,7 +304,7 @@ xrep_dir_want_salvage( struct xfs_mount *mp = rd->sc->mp; /* No pointers to ourselves or to garbage. */ - if (ino == rd->sc->ip->i_ino) + if (ino == I_INO(rd->sc->ip)) return false; if (!xfs_verify_dir_ino(mp, ino)) return false; @@ -644,14 +644,14 @@ xrep_dir_recover_dirblock( case cpu_to_be32(XFS_DIR3_BLOCK_MAGIC): if (!xrep_buf_verify_struct(bp, &xfs_dir3_block_buf_ops)) goto out; - if (xfs_dir3_block_header_check(bp, rd->sc->ip->i_ino) != NULL) + if (xfs_dir3_block_header_check(bp, I_INO(rd->sc->ip)) != NULL) goto out; break; case cpu_to_be32(XFS_DIR2_DATA_MAGIC): case cpu_to_be32(XFS_DIR3_DATA_MAGIC): if (!xrep_buf_verify_struct(bp, &xfs_dir3_data_buf_ops)) goto out; - if (xfs_dir3_data_header_check(bp, rd->sc->ip->i_ino) != NULL) + if (xfs_dir3_data_header_check(bp, I_INO(rd->sc->ip)) != NULL) goto out; break; default: @@ -676,7 +676,7 @@ xrep_dir_init_args( memset(&rd->args, 0, sizeof(struct xfs_da_args)); rd->args.geo = rd->sc->mp->m_dir_geo; rd->args.whichfork = XFS_DATA_FORK; - rd->args.owner = rd->sc->ip->i_ino; + rd->args.owner = I_INO(rd->sc->ip); rd->args.trans = rd->sc->tp; rd->args.dp = dp; if (!name) @@ -1125,12 +1125,12 @@ xrep_dir_scan_pptr( if (error) return error; - if (parent_ino != sc->ip->i_ino || + if (parent_ino != I_INO(sc->ip) || parent_gen != VFS_I(sc->ip)->i_generation) return 0; mutex_lock(&rd->pscan.lock); - error = xrep_dir_stash_createname(rd, &xname, ip->i_ino); + error = xrep_dir_stash_createname(rd, &xname, I_INO(ip)); mutex_unlock(&rd->pscan.lock); return error; } @@ -1151,7 +1151,7 @@ xrep_dir_scan_dirent( struct xrep_dir *rd = priv; /* Dirent doesn't point to this directory. */ - if (ino != rd->sc->ip->i_ino) + if (ino != I_INO(rd->sc->ip)) return 0; /* Ignore garbage inum. */ @@ -1168,9 +1168,9 @@ xrep_dir_scan_dirent( return 0; trace_xrep_dir_stash_createname(sc->tempip, &xfs_name_dotdot, - dp->i_ino); + I_INO(dp)); - xrep_findparent_scan_found(&rd->pscan, dp->i_ino); + xrep_findparent_scan_found(&rd->pscan, I_INO(dp)); return 0; } @@ -1281,7 +1281,7 @@ xrep_dir_scan_dirtree( /* Roots of directory trees are their own parents. */ if (xchk_inode_is_dirtree_root(sc->ip)) - xrep_findparent_scan_found(&rd->pscan, sc->ip->i_ino); + xrep_findparent_scan_found(&rd->pscan, I_INO(sc->ip)); /* * Filesystem scans are time consuming. Drop the directory ILOCK and @@ -1369,15 +1369,15 @@ xrep_dir_live_update( * rebuilding. Stash the update for replay against the temporary * directory. */ - if (p->dp->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&rd->pscan.iscan, p->ip->i_ino)) { + if (I_INO(p->dp) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rd->pscan.iscan, I_INO(p->ip))) { mutex_lock(&rd->pscan.lock); if (p->delta > 0) error = xrep_dir_stash_createname(rd, p->name, - p->ip->i_ino); + I_INO(p->ip)); else error = xrep_dir_stash_removename(rd, p->name, - p->ip->i_ino); + I_INO(p->ip)); mutex_unlock(&rd->pscan.lock); if (error) goto out_abort; @@ -1388,14 +1388,14 @@ xrep_dir_live_update( * the directory that we're rebuilding, so remember the new dotdot * target. */ - if (p->ip->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&rd->pscan.iscan, p->dp->i_ino)) { + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rd->pscan.iscan, I_INO(p->dp))) { if (p->delta > 0) { trace_xrep_dir_stash_createname(sc->tempip, &xfs_name_dotdot, - p->dp->i_ino); + I_INO(p->dp)); - xrep_findparent_scan_found(&rd->pscan, p->dp->i_ino); + xrep_findparent_scan_found(&rd->pscan, I_INO(p->dp)); } else { trace_xrep_dir_stash_removename(sc->tempip, &xfs_name_dotdot, @@ -1468,7 +1468,7 @@ xrep_dir_swap_prep( .whichfork = XFS_DATA_FORK, .trans = sc->tp, .total = 1, - .owner = sc->ip->i_ino, + .owner = I_INO(sc->ip), }; error = xfs_dir2_sf_to_block(&args); @@ -1576,7 +1576,7 @@ xrep_dir_set_nlink( * count. If the directory has no parent, it will be moved to the * orphanage. */ - pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, dp->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(dp)); if (!pag) { ASSERT(0); return -EFSCORRUPTED; @@ -1764,7 +1764,7 @@ xrep_dir_rebuild_tree( * directory to an empty shortform directory because inactivation does * nothing for directories. */ - error = xrep_dir_reset_fork(rd, sc->mp->m_rootip->i_ino); + error = xrep_dir_reset_fork(rd, I_INO(sc->mp->m_rootip)); if (error) return error; diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 143114718d75..b2cf6e5439d9 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -175,7 +175,7 @@ xchk_dirpath_append( if (error) return error; - error = xino_bitmap_set(&path->seen_inodes, ip->i_ino); + error = xino_bitmap_set(&path->seen_inodes, I_INO(ip)); if (error) return error; @@ -383,6 +383,14 @@ xchk_dirpath_step_up( goto out_scanlock; } + /* The handle encoded in the parent pointer must match. */ + if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) { + trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + /* We've reached the root directory; the path is ok. */ if (parent_ino == dl->root_ino) { xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_OK); @@ -394,7 +402,7 @@ xchk_dirpath_step_up( * The inode being scanned is its own distant ancestor! Get rid of * this path. */ - if (parent_ino == sc->ip->i_ino) { + if (parent_ino == I_INO(sc->ip)) { xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); error = 0; goto out_scanlock; @@ -411,14 +419,6 @@ xchk_dirpath_step_up( goto out_scanlock; } - /* The handle encoded in the parent pointer must match. */ - if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) { - trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr, - path->nr_steps, &dl->xname, &dl->pptr_rec); - error = -EFSCORRUPTED; - goto out_scanlock; - } - /* Parent pointer must point up to a directory. */ if (!S_ISDIR(VFS_I(dp)->i_mode)) { trace_xchk_dirpath_nondir_parent(dl->sc, dp, path->path_nr, @@ -533,7 +533,7 @@ xchk_dirpath_walk_upwards( * The inode being scanned is its own direct ancestor! * Get rid of this path. */ - if (be64_to_cpu(dl->pptr_rec.p_ino) == sc->ip->i_ino) { + if (be64_to_cpu(dl->pptr_rec.p_ino) == I_INO(sc->ip)) { xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); return 0; } @@ -610,7 +610,7 @@ xchk_dirpath_step_is_stale( * If the parent and child being updated are not the ones mentioned in * this path step, the scan data is still ok. */ - if (p->ip->i_ino != child_ino || p->dp->i_ino != *cursor) + if (I_INO(p->ip) != child_ino || I_INO(p->dp) != *cursor) return 0; /* @@ -632,13 +632,13 @@ xchk_dirpath_step_is_stale( * If the update comes from the repair code itself, walk the state * machine forward. */ - if (p->ip->i_ino == dl->scan_ino && + if (I_INO(p->ip) == dl->scan_ino && path->outcome == XREP_DIRPATH_ADOPTING) { xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_ADOPTED); return 0; } - if (p->ip->i_ino == dl->scan_ino && + if (I_INO(p->ip) == dl->scan_ino && path->outcome == XREP_DIRPATH_DELETING) { xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_DELETED); return 0; @@ -669,7 +669,7 @@ xchk_dirpath_is_stale( * The child being updated has not been seen by this path at all; this * path cannot be stale. */ - if (!xino_bitmap_test(&path->seen_inodes, p->ip->i_ino)) + if (!xino_bitmap_test(&path->seen_inodes, I_INO(p->ip))) return 0; ret = xchk_dirpath_step_is_stale(dl, path, 0, idx, p, &cursor); @@ -928,7 +928,7 @@ xchk_dirtree( * released during teardown. */ dl->root_ino = xchk_inode_rootdir_inum(sc->ip); - dl->scan_ino = sc->ip->i_ino; + dl->scan_ino = I_INO(sc->ip); trace_xchk_dirtree_start(sc->ip, sc->sm, 0); @@ -954,7 +954,7 @@ xchk_dirtree( * parent pointers are corrupt; this scan cannot be completed * without full information. */ - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); error = 0; goto out_scanlock; } @@ -979,12 +979,12 @@ xchk_dirtree( xchk_dirtree_evaluate(dl, &oc); if (xchk_dirtree_parentless(dl)) { if (oc.good || oc.bad || oc.suspect) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } else { if (oc.bad || oc.good + oc.suspect != 1) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); if (oc.suspect) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); } out_scanlock: diff --git a/fs/xfs/scrub/dirtree_repair.c b/fs/xfs/scrub/dirtree_repair.c index bcfbbf5efc6c..1c0d7ea4a5be 100644 --- a/fs/xfs/scrub/dirtree_repair.c +++ b/fs/xfs/scrub/dirtree_repair.c @@ -459,7 +459,7 @@ xrep_dirtree_unlink( if (error) goto out_trans_cancel; - error = xfs_dir_removename(sc->tp, dp, &dl->xname, sc->ip->i_ino, + error = xfs_dir_removename(sc->tp, dp, &dl->xname, I_INO(sc->ip), resblks); if (error) { ASSERT(error != -ENOENT); @@ -583,7 +583,7 @@ xrep_dirtree_create_adoption_path( */ xfs_inode_to_parent_rec(&dl->pptr_rec, sc->orphanage); - error = xino_bitmap_set(&path->seen_inodes, sc->orphanage->i_ino); + error = xino_bitmap_set(&path->seen_inodes, I_INO(sc->orphanage)); if (error) goto out_path; diff --git a/fs/xfs/scrub/dqiterate.c b/fs/xfs/scrub/dqiterate.c index 10950e4bd4c3..079dc4e691a0 100644 --- a/fs/xfs/scrub/dqiterate.c +++ b/fs/xfs/scrub/dqiterate.c @@ -205,7 +205,7 @@ xchk_dquot_iter( if (error) return error; - cursor->id = dq->q_id + 1; + cursor->id = (uint64_t)dq->q_id + 1; *dqpp = dq; return 1; } diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index 2076f028d271..04b6b96b0a30 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -65,7 +65,7 @@ struct xrep_findparent_info { /* * Scrub context. We're looking for a @dp containing a directory - * entry pointing to sc->ip->i_ino. + * entry pointing to I_INO(sc->ip). */ struct xfs_scrub *sc; @@ -106,7 +106,7 @@ xrep_findparent_dirent( if (xchk_should_terminate(fpi->sc, &error)) return error; - if (ino != fpi->sc->ip->i_ino) + if (ino != I_INO(fpi->sc->ip)) return 0; /* Ignore garbage directory entry names. */ @@ -123,18 +123,18 @@ xrep_findparent_dirent( /* Uhoh, more than one parent for a dir? */ if (fpi->found_parent != NULLFSINO && - !(fpi->parent_tentative && fpi->found_parent == fpi->dp->i_ino)) { + !(fpi->parent_tentative && fpi->found_parent == I_INO(fpi->dp))) { trace_xrep_findparent_dirent(fpi->sc->ip, 0); return -EFSCORRUPTED; } /* We found a potential parent; remember this. */ - trace_xrep_findparent_dirent(fpi->sc->ip, fpi->dp->i_ino); - fpi->found_parent = fpi->dp->i_ino; + trace_xrep_findparent_dirent(fpi->sc->ip, I_INO(fpi->dp)); + fpi->found_parent = I_INO(fpi->dp); fpi->parent_tentative = false; if (fpi->parent_scan) - xrep_findparent_scan_found(fpi->parent_scan, fpi->dp->i_ino); + xrep_findparent_scan_found(fpi->parent_scan, I_INO(fpi->dp)); return 0; } @@ -227,13 +227,12 @@ xrep_findparent_live_update( * already scanned @p->dp, update the dotdot target inumber to the * parent inode. */ - if (p->ip->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&pscan->iscan, p->dp->i_ino)) { - if (p->delta > 0) { - xrep_findparent_scan_found(pscan, p->dp->i_ino); - } else { + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&pscan->iscan, I_INO(p->dp))) { + if (p->delta > 0) + xrep_findparent_scan_found(pscan, I_INO(p->dp)); + else xrep_findparent_scan_found(pscan, NULLFSINO); - } } return NOTIFY_DONE; @@ -388,7 +387,7 @@ xrep_findparent_confirm( if (*parent_ino == NULLFSINO) return 0; if (!xfs_verify_dir_ino(sc->mp, *parent_ino) || - *parent_ino == sc->ip->i_ino) { + *parent_ino == I_INO(sc->ip)) { *parent_ino = NULLFSINO; return 0; } @@ -422,10 +421,10 @@ xfs_ino_t xrep_findparent_self_reference( struct xfs_scrub *sc) { - if (sc->ip->i_ino == sc->mp->m_sb.sb_rootino) + if (I_INO(sc->ip) == sc->mp->m_sb.sb_rootino) return sc->mp->m_sb.sb_rootino; - if (sc->ip->i_ino == sc->mp->m_sb.sb_metadirino) + if (I_INO(sc->ip) == sc->mp->m_sb.sb_metadirino) return sc->mp->m_sb.sb_metadirino; if (VFS_I(sc->ip)->i_nlink == 0) @@ -457,8 +456,8 @@ xrep_findparent_from_dcache( dput(parent); if (S_ISDIR(pip->i_mode)) { - trace_xrep_findparent_from_dcache(sc->ip, XFS_I(pip)->i_ino); - ret = XFS_I(pip)->i_ino; + ret = pip->i_ino; + trace_xrep_findparent_from_dcache(sc->ip, ret); } xchk_irele(sc, XFS_I(pip)); diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c index 911dc0f9a79d..19c0b1b2a787 100644 --- a/fs/xfs/scrub/ialloc.c +++ b/fs/xfs/scrub/ialloc.c @@ -363,6 +363,7 @@ xchk_iallocbt_check_cluster( struct xfs_inobt_rec_incore *irec, unsigned int cluster_base) { + struct xfs_perag *pag = to_perag(bs->cur->bc_group); struct xfs_imap imap; struct xfs_mount *mp = bs->cur->bc_mp; struct xfs_buf *cluster_bp; @@ -394,8 +395,7 @@ xchk_iallocbt_check_cluster( * ir_startino can be large enough to make im_boffset nonzero. */ ir_holemask = (irec->ir_holemask & cluster_mask); - imap.im_blkno = xfs_agbno_to_daddr(to_perag(bs->cur->bc_group), agbno); - imap.im_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster); + imap.im_agbno = agbno; imap.im_boffset = XFS_INO_TO_OFFSET(mp, irec->ir_startino) << mp->m_sb.sb_inodelog; @@ -405,8 +405,8 @@ xchk_iallocbt_check_cluster( return 0; } - trace_xchk_iallocbt_check_cluster(to_perag(bs->cur->bc_group), - irec->ir_startino, imap.im_blkno, imap.im_len, + trace_xchk_iallocbt_check_cluster(pag, irec->ir_startino, imap.im_agbno, + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), cluster_base, nr_inodes, cluster_mask, ir_holemask, XFS_INO_TO_OFFSET(mp, irec->ir_startino + cluster_base)); @@ -429,7 +429,8 @@ xchk_iallocbt_check_cluster( &XFS_RMAP_OINFO_INODES); /* Grab the inode cluster buffer. */ - error = xfs_imap_to_bp(mp, bs->cur->bc_tp, &imap, &cluster_bp); + error = xfs_read_icluster(pag, bs->cur->bc_tp, imap.im_agbno, + &cluster_bp); if (!xchk_btree_xref_process_error(bs->sc, bs->cur, 0, &error)) return error; diff --git a/fs/xfs/scrub/ialloc_repair.c b/fs/xfs/scrub/ialloc_repair.c index 608c7022ac15..46b1c8ac5543 100644 --- a/fs/xfs/scrub/ialloc_repair.c +++ b/fs/xfs/scrub/ialloc_repair.c @@ -287,7 +287,6 @@ xrep_ibt_process_cluster( struct xrep_ibt *ri, xfs_agblock_t cluster_bno) { - struct xfs_imap imap; struct xfs_buf *cluster_bp; struct xfs_scrub *sc = ri->sc; struct xfs_mount *mp = sc->mp; @@ -305,10 +304,8 @@ xrep_ibt_process_cluster( * inobt because imap_to_bp directly maps the buffer without touching * either inode btree. */ - imap.im_blkno = xfs_agbno_to_daddr(sc->sa.pag, cluster_bno); - imap.im_len = XFS_FSB_TO_BB(mp, igeo->blocks_per_cluster); - imap.im_boffset = 0; - error = xfs_imap_to_bp(mp, sc->tp, &imap, &cluster_bp); + error = xfs_read_icluster(sc->sa.pag, sc->tp, cluster_bno, + &cluster_bp); if (error) return error; diff --git a/fs/xfs/scrub/inode.c b/fs/xfs/scrub/inode.c index 948d04dcba2a..65b13e311916 100644 --- a/fs/xfs/scrub/inode.c +++ b/fs/xfs/scrub/inode.c @@ -102,7 +102,7 @@ xchk_setup_inode( xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); /* We want to scan the opened inode, so lock it and exit. */ - if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == ip_in->i_ino) { + if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == I_INO(ip_in)) { error = xchk_install_live_inode(sc, ip_in); if (error) return error; @@ -710,17 +710,17 @@ xchk_inode_xref_bmap( if (!xchk_should_check_xref(sc, &error, NULL)) return; if (nextents < xfs_dfork_data_extents(dip)) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); error = xchk_inode_count_blocks(sc, XFS_ATTR_FORK, &nextents, &acount); if (!xchk_should_check_xref(sc, &error, NULL)) return; if (nextents != xfs_dfork_attr_extents(dip)) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); /* Check nblocks against the inode. */ if (count + acount != be64_to_cpu(dip->di_nblocks)) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); } /* Cross-reference with the other btrees. */ @@ -794,10 +794,10 @@ xchk_inode_check_unlinked( { if (VFS_I(sc->ip)->i_nlink == 0) { if (!xfs_inode_on_unlinked_list(sc->ip)) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ino_set_corrupt(sc, I_INO(sc->ip)); } else { if (xfs_inode_on_unlinked_list(sc->ip)) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ino_set_corrupt(sc, I_INO(sc->ip)); } } @@ -821,7 +821,7 @@ xchk_inode( /* Scrub the inode core. */ xfs_inode_to_disk(sc->ip, &di, 0); - xchk_dinode(sc, &di, sc->ip->i_ino); + xchk_dinode(sc, &di, I_INO(sc->ip)); if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) goto out; @@ -831,11 +831,11 @@ xchk_inode( * we scrubbed the dinode. */ if (S_ISREG(VFS_I(sc->ip)->i_mode)) - xchk_inode_check_reflink_iflag(sc, sc->ip->i_ino); + xchk_inode_check_reflink_iflag(sc, I_INO(sc->ip)); xchk_inode_check_unlinked(sc); - xchk_inode_xref(sc, sc->ip->i_ino, &di); + xchk_inode_xref(sc, I_INO(sc->ip), &di); out: return error; } diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 9738b9ce3f2d..3ec41c198351 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -921,7 +921,7 @@ xrep_dinode_bad_bmbt_fork( if (nrecs == 0 || xfs_bmdr_space_calc(nrecs) > dfork_size) return true; - if (level == 0 || level >= XFS_BM_MAXLEVELS(sc->mp, whichfork)) + if (level == 0 || level > XFS_BM_MAXLEVELS(sc->mp, whichfork)) return true; dmxr = xfs_bmdr_maxrecs(dfork_size, 0); @@ -1547,6 +1547,7 @@ xrep_dinode_core( struct xrep_inode *ri) { struct xfs_scrub *sc = ri->sc; + struct xfs_mount *mp = sc->mp; struct xfs_buf *bp; struct xfs_dinode *dip; xfs_ino_t ino = sc->sm->sm_ino; @@ -1559,8 +1560,11 @@ xrep_dinode_core( return error; /* Read the inode cluster buffer. */ - error = xfs_trans_read_buf(sc->mp, sc->tp, sc->mp->m_ddev_targp, - ri->imap.im_blkno, ri->imap.im_len, 0, &bp, NULL); + error = xfs_trans_read_buf(mp, sc->tp, mp->m_ddev_targp, + XFS_AGB_TO_DADDR(mp, XFS_INO_TO_AGNO(mp, ino), + ri->imap.im_agbno), + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), + 0, &bp, NULL); if (error) return error; @@ -1583,10 +1587,10 @@ xrep_dinode_core( write: /* Write out the inode. */ trace_xrep_dinode_fixed(sc, dip); - xfs_dinode_calc_crc(sc->mp, dip); + xfs_dinode_calc_crc(mp, dip); xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_DINO_BUF); xfs_trans_log_buf(sc->tp, bp, ri->imap.im_boffset, - ri->imap.im_boffset + sc->mp->m_sb.sb_inodesize - 1); + ri->imap.im_boffset + mp->m_sb.sb_inodesize - 1); /* * In theory, we've fixed the ondisk inode record enough that we should @@ -1753,7 +1757,7 @@ xrep_clamp_timestamp( struct xfs_inode *ip, struct timespec64 *ts) { - ts->tv_nsec = clamp_t(long, ts->tv_nsec, 0, NSEC_PER_SEC); + ts->tv_nsec = clamp_t(long, ts->tv_nsec, 0, NSEC_PER_SEC - 1); *ts = timestamp_truncate(*ts, VFS_I(ip)); } @@ -1796,7 +1800,7 @@ xrep_inode_flags( sc->ip->i_diflags &= ~XFS_DIFLAG_ANY; /* NEWRTBM only applies to realtime bitmaps */ - if (sc->ip->i_ino == sc->mp->m_sb.sb_rbmino) + if (I_INO(sc->ip) == sc->mp->m_sb.sb_rbmino) sc->ip->i_diflags |= XFS_DIFLAG_NEWRTBM; else sc->ip->i_diflags &= ~XFS_DIFLAG_NEWRTBM; @@ -2011,8 +2015,7 @@ xrep_inode_unlinked( struct xfs_perag *pag; int error; - pag = xfs_perag_get(sc->mp, - XFS_INO_TO_AGNO(sc->mp, sc->ip->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(sc->ip)); error = xfs_iunlink_remove(sc->tp, pag, sc->ip); xfs_perag_put(pag); if (error) diff --git a/fs/xfs/scrub/iscan.c b/fs/xfs/scrub/iscan.c index 2a974eed00cc..03f72ba9e7a3 100644 --- a/fs/xfs/scrub/iscan.c +++ b/fs/xfs/scrub/iscan.c @@ -728,7 +728,7 @@ xchk_iscan_mark_visited( struct xfs_inode *ip) { mutex_lock(&iscan->lock); - iscan->__visited_ino = ip->i_ino; + iscan->__visited_ino = I_INO(ip); trace_xchk_iscan_visit(iscan); mutex_unlock(&iscan->lock); } diff --git a/fs/xfs/scrub/listxattr.c b/fs/xfs/scrub/listxattr.c index 0863db64b1b2..9b4cf393788b 100644 --- a/fs/xfs/scrub/listxattr.c +++ b/fs/xfs/scrub/listxattr.c @@ -115,7 +115,7 @@ xchk_xattr_walk_leaf( struct xfs_buf *leaf_bp; int error; - error = xfs_attr3_leaf_read(sc->tp, ip, ip->i_ino, 0, &leaf_bp); + error = xfs_attr3_leaf_read(sc->tp, ip, I_INO(ip), 0, &leaf_bp); if (error) return error; @@ -166,7 +166,7 @@ xchk_xattr_find_leftmost_leaf( magic != XFS_DA3_NODE_MAGIC) goto out_buf; - fa = xfs_da3_node_header_check(bp, ip->i_ino); + fa = xfs_da3_node_header_check(bp, I_INO(ip)); if (fa) goto out_buf; @@ -195,7 +195,7 @@ xchk_xattr_find_leftmost_leaf( } error = -EFSCORRUPTED; - fa = xfs_attr3_leaf_header_check(bp, ip->i_ino); + fa = xfs_attr3_leaf_header_check(bp, I_INO(ip)); if (fa) goto out_buf; @@ -266,8 +266,8 @@ xchk_xattr_walk_node( goto out_bitmap; } - error = xfs_attr3_leaf_read(sc->tp, ip, ip->i_ino, - leafhdr.forw, &leaf_bp); + error = xfs_attr3_leaf_read(sc->tp, ip, I_INO(ip), leafhdr.forw, + &leaf_bp); if (error) goto out_bitmap; diff --git a/fs/xfs/scrub/metapath.c b/fs/xfs/scrub/metapath.c index 050b86eb12d3..ff1ff762b300 100644 --- a/fs/xfs/scrub/metapath.c +++ b/fs/xfs/scrub/metapath.c @@ -314,7 +314,7 @@ xchk_metapath( /* Parent required to do anything else. */ if (mpath->dp == NULL) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -329,15 +329,15 @@ xchk_metapath( trace_xchk_metapath_lookup(sc, mpath->path, mpath->dp, ino); if (error == -ENOENT) { /* No directory entry at all */ - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); error = 0; goto out_ilock; } if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) goto out_ilock; - if (ino != sc->ip->i_ino) { + if (ino != I_INO(sc->ip)) { /* Pointing to wrong inode */ - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } out_ilock: @@ -364,7 +364,7 @@ xrep_metapath_link( else mpath->du.ppargs = NULL; - trace_xrep_metapath_link(sc, mpath->path, mpath->dp, sc->ip->i_ino); + trace_xrep_metapath_link(sc, mpath->path, mpath->dp, I_INO(sc->ip)); return xfs_dir_add_child(sc->tp, mpath->link_resblks, &mpath->du); } @@ -462,7 +462,7 @@ xrep_metapath_try_link( if (error) goto out_cancel; - if (ino == sc->ip->i_ino) { + if (ino == I_INO(sc->ip)) { /* The dirent already points to @sc->ip; we're done. */ error = 0; goto out_cancel; @@ -530,7 +530,7 @@ xrep_metapath_try_unlink( xfs_ino_t ino; int error; - ASSERT(*alleged_child != sc->ip->i_ino); + ASSERT(*alleged_child != I_INO(sc->ip)); trace_xrep_metapath_try_unlink(sc, mpath->path, mpath->dp, *alleged_child); @@ -575,7 +575,7 @@ xrep_metapath_try_unlink( if (error) goto out_cancel; - if (ino == sc->ip->i_ino) { + if (ino == I_INO(sc->ip)) { /* The dirent already points to @sc->ip; we're done. */ error = -EEXIST; goto out_cancel; diff --git a/fs/xfs/scrub/newbt.c b/fs/xfs/scrub/newbt.c index 2e4981c1baf2..c82f4631fd9c 100644 --- a/fs/xfs/scrub/newbt.c +++ b/fs/xfs/scrub/newbt.c @@ -123,8 +123,7 @@ xrep_newbt_init_inode( if (!ifp) return -ENOMEM; - xrep_newbt_init_ag(xnr, sc, oinfo, - XFS_INO_TO_FSB(sc->mp, sc->ip->i_ino), + xrep_newbt_init_ag(xnr, sc, oinfo, XFS_INODE_TO_FSB(sc->ip), XFS_AG_RESV_NONE); xnr->ifake.if_fork = ifp; xnr->ifake.if_fork_size = xfs_inode_fork_size(sc->ip, whichfork); @@ -146,7 +145,7 @@ xrep_newbt_init_metadir_inode( ASSERT(xfs_is_metadir_inode(sc->ip)); - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, XFS_DATA_FORK); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, XFS_DATA_FORK); ifp = kmem_cache_zalloc(xfs_ifork_cache, XCHK_GFP_FLAGS); if (!ifp) @@ -160,8 +159,7 @@ xrep_newbt_init_metadir_inode( * as if they were regular file blocks. This exposes us to a higher * risk of the repair being cancelled due to ENOSPC. */ - xrep_newbt_init_ag(xnr, sc, &oinfo, - XFS_INO_TO_FSB(sc->mp, sc->ip->i_ino), + xrep_newbt_init_ag(xnr, sc, &oinfo, XFS_INODE_TO_FSB(sc->ip), XFS_AG_RESV_NONE); xnr->ifake.if_fork = ifp; xnr->ifake.if_fork_size = xfs_inode_fork_size(sc->ip, XFS_DATA_FORK); diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index 1e0effdb65a6..355ab6de23ea 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -175,7 +175,7 @@ xchk_nlinks_live_update( if (xrep_is_tempfile(p->dp)) return NOTIFY_DONE; - trace_xchk_nlinks_live_update(xnc->sc->mp, p->dp, action, p->ip->i_ino, + trace_xchk_nlinks_live_update(xnc->sc->mp, p->dp, action, I_INO(p->ip), p->delta, p->name->name, p->name->len); /* @@ -183,12 +183,12 @@ xchk_nlinks_live_update( * to @ip. If @ip is a subdirectory, update the number of child links * going out of @dp. */ - if (xchk_iscan_want_live_update(&xnc->collect_iscan, p->dp->i_ino)) { + if (xchk_iscan_want_live_update(&xnc->collect_iscan, I_INO(p->dp))) { mutex_lock(&xnc->lock); - error = xchk_nlinks_update_incore(xnc, p->ip->i_ino, p->delta, + error = xchk_nlinks_update_incore(xnc, I_INO(p->ip), p->delta, 0, 0); if (!error && S_ISDIR(VFS_IC(p->ip)->i_mode)) - error = xchk_nlinks_update_incore(xnc, p->dp->i_ino, 0, + error = xchk_nlinks_update_incore(xnc, I_INO(p->dp), 0, 0, p->delta); mutex_unlock(&xnc->lock); if (error) @@ -200,9 +200,9 @@ xchk_nlinks_live_update( * number of backrefs pointing to @dp. */ if (S_ISDIR(VFS_IC(p->ip)->i_mode) && - xchk_iscan_want_live_update(&xnc->collect_iscan, p->ip->i_ino)) { + xchk_iscan_want_live_update(&xnc->collect_iscan, I_INO(p->ip))) { mutex_lock(&xnc->lock); - error = xchk_nlinks_update_incore(xnc, p->dp->i_ino, 0, + error = xchk_nlinks_update_incore(xnc, I_INO(p->dp), 0, p->delta, 0); mutex_unlock(&xnc->lock); if (error) @@ -243,7 +243,7 @@ xchk_nlinks_collect_dirent( dotdot = true; /* Don't accept a '.' entry that points somewhere else. */ - if (dot && ino != dp->i_ino) { + if (dot && ino != I_INO(dp)) { error = -ECANCELED; goto out_abort; } @@ -304,7 +304,7 @@ xchk_nlinks_collect_dirent( * number of child links of dp. */ if (!dot && !dotdot && name->type == XFS_DIR3_FT_DIR) { - error = xchk_nlinks_update_incore(xnc, dp->i_ino, 0, 0, 1); + error = xchk_nlinks_update_incore(xnc, I_INO(dp), 0, 0, 1); if (error) goto out_unlock; } @@ -692,7 +692,7 @@ xchk_nlinks_compare_inode( goto out_scanlock; } - error = xchk_nlinks_comparison_read(xnc, ip->i_ino, &obs); + error = xchk_nlinks_comparison_read(xnc, I_INO(ip), &obs); if (error) goto out_scanlock; @@ -718,15 +718,15 @@ xchk_nlinks_compare_inode( * count, but it will let them decrease it. */ if (total_links > XFS_NLINK_PINNED) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } else if (total_links > XFS_MAXLINK) { - xchk_ino_set_warning(sc, ip->i_ino); + xchk_ino_set_warning(sc, I_INO(ip)); } /* Link counts should match. */ if (total_links != actual_nlink) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } @@ -740,14 +740,14 @@ xchk_nlinks_compare_inode( * number of subdirectory entries in the directory. */ if (obs.children != obs.backrefs) - xchk_ino_xref_set_corrupt(sc, ip->i_ino); + xchk_ip_xref_set_corrupt(sc, ip); } else { /* * Non-directories and unlinked directories should not have * back references. */ if (obs.backrefs != 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } @@ -756,7 +756,7 @@ xchk_nlinks_compare_inode( * children. */ if (obs.children != 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } @@ -769,7 +769,7 @@ xchk_nlinks_compare_inode( * the root directory. */ if (obs.parents != 1) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } else if (actual_nlink > 0) { @@ -778,7 +778,7 @@ xchk_nlinks_compare_inode( * least one parent. */ if (obs.parents == 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 9049215c6eae..fbc2ff809fc0 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -75,7 +75,7 @@ xrep_nlinks_iunlink_remove( struct xfs_perag *pag; int error; - pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, sc->ip->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(sc->ip)); error = xfs_iunlink_remove(sc->tp, pag, sc->ip); xfs_perag_put(pag); return error; @@ -152,7 +152,7 @@ xrep_nlinks_repair_inode( goto out_scanlock; } - error = xfarray_load_sparse(xnc->nlinks, ip->i_ino, &obs); + error = xfarray_load_sparse(xnc->nlinks, I_INO(ip), &obs); if (error) goto out_scanlock; @@ -206,7 +206,7 @@ xrep_nlinks_repair_inode( * updated our scan info. */ mutex_lock(&xnc->lock); - error = xfarray_load_sparse(xnc->nlinks, ip->i_ino, &obs); + error = xfarray_load_sparse(xnc->nlinks, I_INO(ip), &obs); mutex_unlock(&xnc->lock); if (error) goto out_trans; diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 33c6db6b4498..3aca66869b80 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -402,14 +402,14 @@ xrep_adoption_compute_name( int error = 0; adopt->xname = xname; - xname->len = snprintf(namebuf, MAXNAMELEN, "%llu", sc->ip->i_ino); + xname->len = snprintf(namebuf, MAXNAMELEN, "%llu", I_INO(sc->ip)); xname->type = xfs_mode_to_ftype(VFS_I(sc->ip)->i_mode); /* Make sure the filename is unique in the lost+found. */ error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); while (error == 0 && incr < 10000) { xname->len = snprintf(namebuf, MAXNAMELEN, "%llu.%u", - sc->ip->i_ino, ++incr); + I_INO(sc->ip), ++incr); error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); } if (error == 0) { @@ -532,7 +532,7 @@ xrep_adoption_move( int error; trace_xrep_adoption_reparent(sc->orphanage, adopt->xname, - sc->ip->i_ino); + I_INO(sc->ip)); error = xrep_adoption_check_dcache(adopt); if (error) @@ -553,7 +553,7 @@ xrep_adoption_move( /* Create the new name in the orphanage. */ error = xfs_dir_createname(sc->tp, sc->orphanage, adopt->xname, - sc->ip->i_ino, adopt->orphanage_blkres); + I_INO(sc->ip), adopt->orphanage_blkres); if (error) return error; @@ -576,7 +576,7 @@ xrep_adoption_move( /* Replace the dotdot entry if the child is a subdirectory. */ if (isdir) { error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, - sc->orphanage->i_ino, adopt->child_blkres); + I_INO(sc->orphanage), adopt->child_blkres); if (error) return error; } diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 931604fa1294..a8c4807e1d94 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -72,7 +72,7 @@ xchk_parent_actor( if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) return error; - if (sc->ip->i_ino == ino) + if (I_INO(sc->ip) == ino) spc->nlink++; if (xchk_should_terminate(spc->sc, &error)) @@ -119,6 +119,7 @@ xchk_parent_validate( .nlink = 0, }; struct xfs_mount *mp = sc->mp; + xfs_ino_t ino = I_INO(sc->ip); struct xfs_inode *dp = NULL; xfs_nlink_t expected_nlink; unsigned int lock_mode; @@ -126,22 +127,20 @@ xchk_parent_validate( /* Is this the root dir? Then '..' must point to itself. */ if (sc->ip == mp->m_rootip) { - if (sc->ip->i_ino != mp->m_sb.sb_rootino || - sc->ip->i_ino != parent_ino) + if (ino != mp->m_sb.sb_rootino || ino != parent_ino) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } /* Is this the metadata root dir? Then '..' must point to itself. */ if (sc->ip == mp->m_metadirip) { - if (sc->ip->i_ino != mp->m_sb.sb_metadirino || - sc->ip->i_ino != parent_ino) + if (ino != mp->m_sb.sb_metadirino || ino != parent_ino) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } /* '..' must not point to ourselves. */ - if (sc->ip->i_ino == parent_ino) { + if (ino == parent_ino) { xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } @@ -315,7 +314,7 @@ xchk_parent_pptr_and_dotdot( /* Is this the root dir? Then '..' must point to itself. */ if (xchk_inode_is_dirtree_root(sc->ip)) { - if (sc->ip->i_ino != pp->parent_ino) + if (I_INO(sc->ip) != pp->parent_ino) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } @@ -398,7 +397,7 @@ xchk_parent_dirent( return error; /* Does the inode number match? */ - if (child_ino != sc->ip->i_ino) { + if (child_ino != I_INO(sc->ip)) { xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); return 0; } @@ -490,7 +489,7 @@ xchk_parent_scan_attr( } /* No self-referential parent pointers. */ - if (parent_ino == sc->ip->i_ino) { + if (parent_ino == I_INO(sc->ip)) { xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); return -ECANCELED; } @@ -512,7 +511,7 @@ xchk_parent_scan_attr( }; /* Couldn't lock the inode, so save the pptr for later. */ - trace_xchk_parent_defer(sc->ip, &xname, dp->i_ino); + trace_xchk_parent_defer(sc->ip, &xname, I_INO(dp)); error = xfblob_storename(pp->pptr_names, &save_pp.name_cookie, &xname); @@ -599,7 +598,7 @@ xchk_parent_slow_pptr( */ lockmode = xchk_parent_lock_dir(sc, dp); if (lockmode) { - trace_xchk_parent_slowpath(sc->ip, xname, dp->i_ino); + trace_xchk_parent_slowpath(sc->ip, xname, I_INO(dp)); goto check_dirent; } @@ -610,7 +609,7 @@ xchk_parent_slow_pptr( xchk_iunlock(sc, sc->ilock_flags); pp->need_revalidate = true; - trace_xchk_parent_ultraslowpath(sc->ip, xname, dp->i_ino); + trace_xchk_parent_ultraslowpath(sc->ip, xname, I_INO(dp)); error = xchk_dir_trylock_for_pptrs(sc, dp, &lockmode); if (error) @@ -729,10 +728,10 @@ xchk_parent_count_pptrs( pp->pptrs_found++; if (VFS_I(sc->ip)->i_nlink == 0 && pp->pptrs_found > 0) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); else if (VFS_I(sc->ip)->i_nlink > 0 && pp->pptrs_found == 0) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } else { /* * Starting with metadir, we allow checking of parent pointers @@ -743,7 +742,7 @@ xchk_parent_count_pptrs( pp->pptrs_found++; if (VFS_I(sc->ip)->i_nlink != pp->pptrs_found) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } return 0; @@ -856,7 +855,7 @@ xchk_parent( return -ENOENT; /* We're not a special inode, are we? */ - if (!xfs_verify_dir_ino(mp, sc->ip->i_ino)) { + if (!xfs_verify_dir_ino(mp, I_INO(sc->ip))) { xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 2ce0cefad362..f643724910f9 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -302,14 +302,14 @@ xrep_parent_replay_update( trace_xrep_parent_replay_parentadd(sc->tempip, xname, &pptr->pptr_rec); - return xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, + return xfs_parent_set(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rp->pptr_args); case XREP_PPTR_REMOVE: /* Remove parent pointer. */ trace_xrep_parent_replay_parentremove(sc->tempip, xname, &pptr->pptr_rec); - return xfs_parent_unset(sc->tempip, sc->ip->i_ino, xname, + return xfs_parent_unset(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rp->pptr_args); } @@ -434,7 +434,7 @@ xrep_parent_scan_dirent( int error; /* Dirent doesn't point to this directory. */ - if (ino != rp->sc->ip->i_ino) + if (ino != I_INO(rp->sc->ip)) return 0; /* No weird looking names. */ @@ -645,8 +645,8 @@ xrep_parent_live_update( * repairing, so stash the update for replay against the temporary * file. */ - if (p->ip->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&rp->pscan.iscan, p->dp->i_ino)) { + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rp->pscan.iscan, I_INO(p->dp))) { mutex_lock(&rp->pscan.lock); if (p->delta > 0) error = xrep_parent_stash_parentadd(rp, p->name, p->dp); @@ -906,7 +906,7 @@ xrep_parent_fetch_xattr_remote( .namelen = namelen, .trans = sc->tp, .valuelen = valuelen, - .owner = ip->i_ino, + .owner = I_INO(ip), }; int error; @@ -984,7 +984,7 @@ xrep_parent_insert_xattr( .attr_filter = key->flags, .namelen = key->namelen, .valuelen = key->valuelen, - .owner = rp->sc->ip->i_ino, + .owner = I_INO(rp->sc->ip), .geo = rp->sc->mp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .op_flags = XFS_DA_OP_OKNOENT, @@ -1329,7 +1329,7 @@ xrep_parent_rebuild_pptrs( * For this purpose, root directories are their own parents. */ if (xchk_inode_is_dirtree_root(sc->ip)) { - xrep_findparent_scan_found(&rp->pscan, sc->ip->i_ino); + xrep_findparent_scan_found(&rp->pscan, I_INO(sc->ip)); } else { error = xrep_parent_lookup_pptrs(sc, &parent_ino); if (error) @@ -1451,7 +1451,7 @@ xrep_parent_set_nondir_nlink( * The file is on the unlinked list but we found parents. * Remove the file from the unlinked list. */ - pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, ip->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(ip)); if (!pag) { ASSERT(0); return -EFSCORRUPTED; diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c index 9ea014e2ecfd..c199d128538e 100644 --- a/fs/xfs/scrub/quotacheck.c +++ b/fs/xfs/scrub/quotacheck.c @@ -399,7 +399,7 @@ xqcheck_collect_inode( int error = 0; if (xfs_is_metadir_inode(ip) || - xfs_is_quota_inode(&tp->t_mountp->m_sb, ip->i_ino)) { + xfs_is_quota_inode(&tp->t_mountp->m_sb, I_INO(ip))) { /* * Quota files are never counted towards quota, so we do not * need to take the lock. Files do not switch between the diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index c66ec9093a38..d68bf20b4a14 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -50,7 +50,7 @@ xchk_dir_walk_sf( dapos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, geo->data_entry_offset); - error = dirent_fn(sc, dp, dapos, &name, dp->i_ino, priv); + error = dirent_fn(sc, dp, dapos, &name, I_INO(dp), priv); if (error) return error; @@ -100,7 +100,7 @@ xchk_dir_walk_block( unsigned int off, next_off, end; int error; - error = xfs_dir3_block_read(sc->tp, dp, dp->i_ino, &bp); + error = xfs_dir3_block_read(sc->tp, dp, I_INO(dp), &bp); if (error) return error; @@ -176,7 +176,7 @@ xchk_read_leaf_dir_buf( if (new_off > *curoff) *curoff = new_off; - return xfs_dir3_data_read(tp, dp, dp->i_ino, map.br_startoff, 0, bpp); + return xfs_dir3_data_read(tp, dp, I_INO(dp), map.br_startoff, 0, bpp); } /* Call a function for every entry in a leaf directory. */ @@ -274,7 +274,7 @@ xchk_dir_walk( .dp = dp, .geo = dp->i_mount->m_dir_geo, .trans = sc->tp, - .owner = dp->i_ino, + .owner = I_INO(dp), }; int error; @@ -320,7 +320,7 @@ xchk_dir_lookup( .hashval = xfs_dir2_hashname(dp->i_mount, name), .whichfork = XFS_DATA_FORK, .op_flags = XFS_DA_OP_OKNOENT, - .owner = dp->i_ino, + .owner = I_INO(dp), }; int error; @@ -332,7 +332,7 @@ xchk_dir_lookup( * set to sc->ip, so we must switch the owner here for the lookup. */ if (dp == sc->tempip) - args.owner = sc->ip->i_ino; + args.owner = I_INO(sc->ip); ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index fff23932828b..fcd14c1703ea 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -1205,7 +1205,7 @@ xrep_reap_metadir_fsblocks( ASSERT(xfs_is_metadir_inode(sc->ip)); xreap_configure_agextent_limits(&rs); - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, XFS_DATA_FORK); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, XFS_DATA_FORK); error = xfsb_bitmap_walk(bitmap, xreap_fsmeta_extent, &rs); if (error) return error; @@ -1249,8 +1249,7 @@ xreap_bmapi_select( cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, sc->sa.pag); - xfs_rmap_ino_owner(&oinfo, rs->ip->i_ino, rs->whichfork, - imap->br_startoff); + xfs_rmap_inode_owner(&oinfo, rs->ip, rs->whichfork, imap->br_startoff); error = xfs_rmap_has_other_keys(cur, agbno, 1, &oinfo, crosslinked); if (error) goto out_cur; diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index ac8c592579bd..11697a8b2a1d 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -816,7 +816,7 @@ xrep_ino_dqattach( case -ENOENT: xfs_err_ratelimited(sc->mp, "inode %llu repair encountered quota error %d, quotacheck forced.", - (unsigned long long)sc->ip->i_ino, error); + (unsigned long long)I_INO(sc->ip), error); if (XFS_IS_UQUOTA_ON(sc->mp) && !sc->ip->i_udquot) xrep_force_quotacheck(sc, XFS_DQTYPE_USER); if (XFS_IS_GQUOTA_ON(sc->mp) && !sc->ip->i_gdquot) diff --git a/fs/xfs/scrub/rgsuper.c b/fs/xfs/scrub/rgsuper.c index 482f899a518a..2bd2c0351b35 100644 --- a/fs/xfs/scrub/rgsuper.c +++ b/fs/xfs/scrub/rgsuper.c @@ -23,6 +23,8 @@ int xchk_setup_rgsuperblock( struct xfs_scrub *sc) { + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); return xchk_trans_alloc(sc, 0); } @@ -43,6 +45,7 @@ xchk_rgsuperblock( struct xfs_scrub *sc) { xfs_rgnumber_t rgno = sc->sm->sm_agno; + unsigned int flags; int error; /* @@ -63,7 +66,12 @@ xchk_rgsuperblock( if (!xchk_xref_process_error(sc, 0, 0, &error)) return error; - error = xchk_rtgroup_lock(sc, &sc->sr, XFS_RTGLOCK_BITMAP_SHARED); + if (xfs_has_rtrmapbt(sc->mp)) + flags = XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP; + else + flags = XFS_RTGLOCK_BITMAP_SHARED; + + error = xchk_rtgroup_lock(sc, &sc->sr, flags); if (error) return error; @@ -80,9 +88,13 @@ int xrep_rgsuperblock( struct xfs_scrub *sc) { + struct xfs_buf *sb_bp; + ASSERT(rtg_rgno(sc->sr.rtg) == 0); + sb_bp = xfs_trans_getsb(sc->tp); xfs_log_sb(sc->tp); + xfs_log_rtsb(sc->tp, sb_bp); return 0; } #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index 6766df9922e8..590f9f41856e 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -570,7 +570,7 @@ xrep_rmap_scan_ifork( int whichfork) { struct xrep_rmap_ifork rf = { - .accum = { .rm_owner = ip->i_ino, }, + .accum = { .rm_owner = I_INO(ip), }, .rr = rr, .whichfork = whichfork, }; diff --git a/fs/xfs/scrub/rtbitmap.c b/fs/xfs/scrub/rtbitmap.c index b3a4972d423e..52c24d3d4be6 100644 --- a/fs/xfs/scrub/rtbitmap.c +++ b/fs/xfs/scrub/rtbitmap.c @@ -200,13 +200,13 @@ xchk_rtbitmap( /* Is sb_rextents correct? */ if (mp->m_sb.sb_rextents != rtb->rextents) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* Is sb_rextslog correct? */ if (mp->m_sb.sb_rextslog != rtb->rextslog) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -215,17 +215,17 @@ xchk_rtbitmap( * case can we exceed 4bn bitmap blocks since the super field is a u32. */ if (rtb->rbmblocks > U32_MAX) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } if (mp->m_sb.sb_rbmblocks != rtb->rbmblocks) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* The bitmap file length must be aligned to an fsblock. */ if (rbmip->i_disk_size & mp->m_blockmask) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -235,7 +235,7 @@ xchk_rtbitmap( * file can be larger than sb_rbmblocks. */ if (rbmip->i_disk_size < XFS_FSB_TO_B(mp, rtb->rbmblocks)) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -258,7 +258,7 @@ xchk_rtbitmap( * the last free extent we saw and the last possible extent in the rt * group. */ - last_rgbno = rtg->rtg_extents * mp->m_sb.sb_rextsize - 1; + last_rgbno = rtg->rtg_extents * mp->m_sb.sb_rextsize; if (rtb->next_free_rgbno < last_rgbno) xchk_xref_has_rt_owner(sc, rtb->next_free_rgbno, last_rgbno - rtb->next_free_rgbno); @@ -284,7 +284,7 @@ xchk_xref_is_used_rt_space( if (xfs_has_zoned(sc->mp)) { if (!xfs_zone_rgbno_is_valid(rtg, xfs_rtb_to_rgbno(sc->mp, rtbno) + len - 1)) - xchk_ino_xref_set_corrupt(sc, rtg_rmap(rtg)->i_ino); + xchk_ip_xref_set_corrupt(sc, rtg_rmap(rtg)); return; } @@ -295,5 +295,5 @@ xchk_xref_is_used_rt_space( if (!xchk_should_check_xref(sc, &error, NULL)) return; if (is_free) - xchk_ino_xref_set_corrupt(sc, rtg_bitmap(rtg)->i_ino); + xchk_ip_xref_set_corrupt(sc, rtg_bitmap(rtg)); } diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c index f4ca86a2ea1b..dc64902d6c25 100644 --- a/fs/xfs/scrub/rtbitmap_repair.c +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -382,7 +382,7 @@ xrep_rtbitmap_prep_buf( struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; hdr->rt_magic = cpu_to_be32(XFS_RTBITMAP_MAGIC); - hdr->rt_owner = cpu_to_be64(sc->ip->i_ino); + hdr->rt_owner = cpu_to_be64(I_INO(sc->ip)); hdr->rt_blkno = cpu_to_be64(xfs_buf_daddr(bp)); hdr->rt_lsn = 0; uuid_copy(&hdr->rt_uuid, &sc->mp->m_sb.sb_meta_uuid); diff --git a/fs/xfs/scrub/rtrefcount.c b/fs/xfs/scrub/rtrefcount.c index 4951cc271788..4e7c540c8d23 100644 --- a/fs/xfs/scrub/rtrefcount.c +++ b/fs/xfs/scrub/rtrefcount.c @@ -555,7 +555,7 @@ xchk_rtrefcountbt( if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) return error; - xfs_rmap_ino_bmbt_owner(&btree_oinfo, rtg_refcount(sc->sr.rtg)->i_ino, + xfs_rmap_inode_bmbt_owner(&btree_oinfo, rtg_refcount(sc->sr.rtg), XFS_DATA_FORK); error = xchk_btree(sc, sc->sr.refc_cur, xchk_rtrefcountbt_rec, &btree_oinfo, &rrc); @@ -607,7 +607,7 @@ xchk_xref_is_rt_cow_staging( /* CoW lookup returned a shared extent record? */ if (rc.rc_domain != XFS_REFC_DOMAIN_COW) - xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); /* Must be at least as long as what was passed in */ if (rc.rc_blockcount < len) diff --git a/fs/xfs/scrub/rtrefcount_repair.c b/fs/xfs/scrub/rtrefcount_repair.c index f165fb397647..2b939960c7dd 100644 --- a/fs/xfs/scrub/rtrefcount_repair.c +++ b/fs/xfs/scrub/rtrefcount_repair.c @@ -360,7 +360,7 @@ xrep_rtrefc_walk_rmap( return error; /* Skip extents which are not owned by this inode and fork. */ - if (rec->rm_owner != rr->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rr->sc->ip)) return 0; error = xrep_check_ino_btree_mapping(rr->sc, rec); diff --git a/fs/xfs/scrub/rtrmap.c b/fs/xfs/scrub/rtrmap.c index 8b1a8389d32f..564d19a97a2f 100644 --- a/fs/xfs/scrub/rtrmap.c +++ b/fs/xfs/scrub/rtrmap.c @@ -87,6 +87,9 @@ xchk_rtrmapbt_is_shareable( return false; if (irec->rm_flags & XFS_RMAP_UNWRITTEN) return false; + if (irec->rm_owner == XFS_RMAP_OWN_COW || + irec->rm_owner == XFS_RMAP_OWN_FS) + return false; return true; } @@ -146,6 +149,9 @@ xchk_rtrmap_mergeable( return false; if (r1->rm_flags != r2->rm_flags) return false; + if (r1->rm_owner == XFS_RMAP_OWN_COW || + r1->rm_owner == XFS_RMAP_OWN_FS) + return true; return r1->rm_offset + r1->rm_blockcount == r2->rm_offset; } @@ -209,7 +215,7 @@ xchk_rtrmapbt_xref( xfs_rgbno_to_rtb(sc->sr.rtg, irec->rm_startblock), irec->rm_blockcount); if (irec->rm_owner == XFS_RMAP_OWN_COW) - xchk_xref_is_cow_staging(sc, irec->rm_startblock, + xchk_xref_is_rt_cow_staging(sc, irec->rm_startblock, irec->rm_blockcount); else xchk_rtrmapbt_xref_rtrefc(sc, irec); @@ -244,7 +250,6 @@ int xchk_rtrmapbt( struct xfs_scrub *sc) { - struct xfs_inode *ip = rtg_rmap(sc->sr.rtg); struct xfs_owner_info oinfo; struct xchk_rtrmap cr = { }; int error; @@ -253,7 +258,7 @@ xchk_rtrmapbt( if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) return error; - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, XFS_DATA_FORK); + xfs_rmap_inode_bmbt_owner(&oinfo, rtg_rmap(sc->sr.rtg), XFS_DATA_FORK); return xchk_btree(sc, sc->sr.rmap_cur, xchk_rtrmapbt_rec, &oinfo, &cr); } diff --git a/fs/xfs/scrub/rtrmap_repair.c b/fs/xfs/scrub/rtrmap_repair.c index f2d16ca8d251..a2b72e61edf5 100644 --- a/fs/xfs/scrub/rtrmap_repair.c +++ b/fs/xfs/scrub/rtrmap_repair.c @@ -324,7 +324,7 @@ xrep_rtrmap_scan_dfork( struct xfs_inode *ip) { struct xrep_rtrmap_ifork rf = { - .accum = { .rm_owner = ip->i_ino, }, + .accum = { .rm_owner = I_INO(ip), }, .rr = rr, }; struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); @@ -392,7 +392,7 @@ xrep_rtrmap_walk_rmap( return error; /* Skip extents which are not owned by this inode and fork. */ - if (rec->rm_owner != rr->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rr->sc->ip)) return 0; error = xrep_check_ino_btree_mapping(rr->sc, rec); diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c index efce39d7a1ef..78f72a046887 100644 --- a/fs/xfs/scrub/rtsummary.c +++ b/fs/xfs/scrub/rtsummary.c @@ -189,7 +189,7 @@ xchk_rtsum_record_free( rtlen = xfs_rtxlen_to_extlen(mp, rec->ar_extcount); if (!xfs_verify_rtbext(mp, rtbno, rtlen)) { - xchk_ino_xref_set_corrupt(sc, rtg_bitmap(rtg)->i_ino); + xchk_ip_xref_set_corrupt(sc, rtg_bitmap(rtg)); return -EFSCORRUPTED; } @@ -314,25 +314,25 @@ xchk_rtsummary( /* Is sb_rextents correct? */ if (mp->m_sb.sb_rextents != rts->rextents) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* Is m_rsumlevels correct? */ if (mp->m_rsumlevels != rts->rsumlevels) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } /* Is m_rsumsize correct? */ if (mp->m_rsumblocks != rts->rsumblocks) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } /* The summary file length must be aligned to an fsblock. */ if (rsumip->i_disk_size & mp->m_blockmask) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } @@ -342,7 +342,7 @@ xchk_rtsummary( * the file can be larger than rsumsize. */ if (rsumip->i_disk_size < XFS_FSB_TO_B(mp, rts->rsumblocks)) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } @@ -358,7 +358,7 @@ xchk_rtsummary( * EFSCORRUPTED means the rtbitmap is corrupt, which is an xref * error since we're checking the summary file. */ - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } if (error) diff --git a/fs/xfs/scrub/rtsummary_repair.c b/fs/xfs/scrub/rtsummary_repair.c index afffbd6e0ad1..f065c3e51ce2 100644 --- a/fs/xfs/scrub/rtsummary_repair.c +++ b/fs/xfs/scrub/rtsummary_repair.c @@ -91,7 +91,7 @@ xrep_rtsummary_prep_buf( struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; hdr->rt_magic = cpu_to_be32(XFS_RTSUMMARY_MAGIC); - hdr->rt_owner = cpu_to_be64(sc->ip->i_ino); + hdr->rt_owner = cpu_to_be64(I_INO(sc->ip)); hdr->rt_blkno = cpu_to_be64(xfs_buf_daddr(bp)); hdr->rt_lsn = 0; uuid_copy(&hdr->rt_uuid, &sc->mp->m_sb.sb_meta_uuid); diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 27511a125b39..8742445c86f4 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -955,7 +955,7 @@ xfs_ioc_scrubv_metadata( * because each scrubber gets to decide its own strategy and return * values for getting an inode. */ - if (head.svh_ino && head.svh_ino != ip_in->i_ino) + if (head.svh_ino && head.svh_ino != I_INO(ip_in)) handle_ip = xchk_scrubv_open_by_handle(mp, &head); /* Run all the scrubbers. */ diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index a3f1abc91390..6d7d3523b71f 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -11,7 +11,7 @@ struct xfs_scrub; struct xchk_relax { unsigned long next_resched; unsigned int resched_nr; - bool interruptible; + bool killable; }; /* Yield to the scheduler at most 10x per second. */ @@ -21,7 +21,7 @@ struct xchk_relax { (struct xchk_relax){ \ .next_resched = XCHK_RELAX_NEXT, \ .resched_nr = 0, \ - .interruptible = true, \ + .killable = true, \ } /* @@ -45,7 +45,7 @@ static inline int xchk_maybe_relax(struct xchk_relax *widget) widget->next_resched = XCHK_RELAX_NEXT; } - if (widget->interruptible && fatal_signal_pending(current)) + if (widget->killable && fatal_signal_pending(current)) return -EINTR; return 0; diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c index 25416dfb5189..91c86ea0e0f1 100644 --- a/fs/xfs/scrub/symlink_repair.c +++ b/fs/xfs/scrub/symlink_repair.c @@ -144,7 +144,7 @@ xrep_symlink_salvage_remote( fa = bp->b_ops->verify_struct(bp); dsl = bp->b_addr; magic_ok = dsl->sl_magic == cpu_to_be32(XFS_SYMLINK_MAGIC); - hdr_ok = xfs_symlink_hdr_ok(ip->i_ino, offset, byte_cnt, bp); + hdr_ok = xfs_symlink_hdr_ok(I_INO(ip), offset, byte_cnt, bp); if (!hdr_ok || (fa != NULL && !magic_ok)) break; @@ -259,7 +259,7 @@ xrep_symlink_local_to_remote( if (!xfs_has_crc(sc->mp)) return; - dsl->sl_owner = cpu_to_be64(sc->ip->i_ino); + dsl->sl_owner = cpu_to_be64(I_INO(sc->ip)); xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsymlink_hdr) + ifp->if_bytes - 1); } @@ -375,7 +375,7 @@ xrep_symlink_reset_fork( /* Reset the temp symlink target to dummy content. */ xfs_idestroy_fork(ifp); - return xfs_symlink_write_target(sc->tp, sc->tempip, sc->tempip->i_ino, + return xfs_symlink_write_target(sc->tp, sc->tempip, I_INO(sc->tempip), "?", 1, 0, 0); } @@ -434,7 +434,7 @@ xrep_symlink_rebuild( sc->tempip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; /* Write the salvaged target to the temporary link. */ - error = xfs_symlink_write_target(sc->tp, sc->tempip, sc->ip->i_ino, + error = xfs_symlink_write_target(sc->tp, sc->tempip, I_INO(sc->ip), target_buf, target_len, fs_blocks, resblks); if (error) return error; diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 8d754df72aa5..e0c630f888cf 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -121,7 +121,7 @@ xrep_tempfile_create( * remote target block, so the owner is irrelevant. */ error = xfs_symlink_write_target(tp, sc->tempip, - sc->tempip->i_ino, ".", 1, 0, 0); + I_INO(sc->tempip), ".", 1, 0, 0); if (error) goto out_trans_cancel; } diff --git a/fs/xfs/scrub/trace.c b/fs/xfs/scrub/trace.c index 70d353287993..e5efc680651f 100644 --- a/fs/xfs/scrub/trace.c +++ b/fs/xfs/scrub/trace.c @@ -48,7 +48,7 @@ xchk_btree_cur_fsbno( if (level == cur->bc_nlevels - 1 && cur->bc_ops->type == XFS_BTREE_TYPE_INODE) - return XFS_INO_TO_FSB(cur->bc_mp, cur->bc_ino.ip->i_ino); + return XFS_INODE_TO_FSB(cur->bc_ino.ip); return NULLFSBLOCK; } diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 286c5f5e0544..d5d39d82749e 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -162,7 +162,7 @@ DECLARE_EVENT_CLASS(xchk_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->type = sm->sm_type; __entry->agno = sm->sm_agno; __entry->inum = sm->sm_ino; @@ -236,7 +236,7 @@ DECLARE_EVENT_CLASS(xchk_vector_head_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->agno = vhead->svh_agno; __entry->inum = vhead->svh_ino; __entry->gen = vhead->svh_gen; @@ -340,7 +340,7 @@ TRACE_EVENT(xchk_file_op_error, ), TP_fast_assign( __entry->dev = sc->ip->i_mount->m_super->s_dev; - __entry->ino = sc->ip->i_ino; + __entry->ino = I_INO(sc->ip); __entry->whichfork = whichfork; __entry->type = sc->sm->sm_type; __entry->offset = offset; @@ -438,7 +438,7 @@ DECLARE_EVENT_CLASS(xchk_fblock_error_class, ), TP_fast_assign( __entry->dev = sc->ip->i_mount->m_super->s_dev; - __entry->ino = sc->ip->i_ino; + __entry->ino = I_INO(sc->ip); __entry->whichfork = whichfork; __entry->type = sc->sm->sm_type; __entry->offset = offset; @@ -481,7 +481,7 @@ DECLARE_EVENT_CLASS(xchk_dqiter_class, TP_fast_assign( __entry->dev = cursor->sc->mp->m_super->s_dev; __entry->dqtype = cursor->dqtype; - __entry->ino = cursor->quota_ip->i_ino; + __entry->ino = I_INO(cursor->quota_ip); __entry->cur_id = cursor->id; __entry->startoff = cursor->bmap.br_startoff; __entry->startblock = cursor->bmap.br_startblock; @@ -613,7 +613,7 @@ TRACE_EVENT(xchk_ifork_btree_op_error, TP_fast_assign( xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); __entry->whichfork = cur->bc_ino.whichfork; __entry->type = sc->sm->sm_type; __assign_str(name); @@ -693,7 +693,7 @@ TRACE_EVENT(xchk_ifork_btree_error, TP_fast_assign( xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = sc->ip->i_ino; + __entry->ino = I_INO(sc->ip); __entry->whichfork = cur->bc_ino.whichfork; __entry->type = sc->sm->sm_type; __assign_str(name); @@ -785,17 +785,17 @@ TRACE_EVENT(xchk_xref_error, TRACE_EVENT(xchk_iallocbt_check_cluster, TP_PROTO(const struct xfs_perag *pag, xfs_agino_t startino, - xfs_daddr_t map_daddr, unsigned short map_len, + xfs_agblock_t map_agblock, unsigned short map_len, unsigned int chunk_ino, unsigned int nr_inodes, uint16_t cluster_mask, uint16_t holemask, unsigned int cluster_ino), - TP_ARGS(pag, startino, map_daddr, map_len, chunk_ino, nr_inodes, + TP_ARGS(pag, startino, map_agblock, map_len, chunk_ino, nr_inodes, cluster_mask, holemask, cluster_ino), TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_agnumber_t, agno) __field(xfs_agino_t, startino) - __field(xfs_daddr_t, map_daddr) + __field(xfs_agblock_t, map_agblock) __field(unsigned short, map_len) __field(unsigned int, chunk_ino) __field(unsigned int, nr_inodes) @@ -807,7 +807,7 @@ TRACE_EVENT(xchk_iallocbt_check_cluster, __entry->dev = pag_mount(pag)->m_super->s_dev; __entry->agno = pag_agno(pag); __entry->startino = startino; - __entry->map_daddr = map_daddr; + __entry->map_agblock = map_agblock; __entry->map_len = map_len; __entry->chunk_ino = chunk_ino; __entry->nr_inodes = nr_inodes; @@ -815,11 +815,11 @@ TRACE_EVENT(xchk_iallocbt_check_cluster, __entry->holemask = holemask; __entry->cluster_ino = cluster_ino; ), - TP_printk("dev %d:%d agno 0x%x startino 0x%x daddr 0x%llx bbcount 0x%x chunkino 0x%x nr_inodes %u cluster_mask 0x%x holemask 0x%x cluster_ino 0x%x", + TP_printk("dev %d:%d agno 0x%x startino 0x%x agblock 0x%x bbcount 0x%x chunkino 0x%x nr_inodes %u cluster_mask 0x%x holemask 0x%x cluster_ino 0x%x", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, __entry->startino, - __entry->map_daddr, + __entry->map_agblock, __entry->map_len, __entry->chunk_ino, __entry->nr_inodes, @@ -839,7 +839,7 @@ TRACE_EVENT(xchk_inode_is_allocated, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->iflags = ip->i_flags; __entry->mode = VFS_I(ip)->i_mode; ), @@ -1437,7 +1437,7 @@ TRACE_EVENT(xchk_nlinks_collect_dirent, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->dir = dp->i_ino; + __entry->dir = I_INO(dp); __entry->ino = ino; __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -1464,7 +1464,7 @@ TRACE_EVENT(xchk_nlinks_collect_pptr, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->dir = dp->i_ino; + __entry->dir = I_INO(dp); __entry->ino = be64_to_cpu(pptr->p_ino); __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -1509,7 +1509,7 @@ TRACE_EVENT(xchk_nlinks_live_update, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->dir = dp ? dp->i_ino : NULLFSINO; + __entry->dir = dp ? I_INO(dp) : NULLFSINO; __entry->action = action; __entry->ino = ino; __entry->delta = delta; @@ -1602,7 +1602,7 @@ DECLARE_EVENT_CLASS(xchk_nlinks_diff_class, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->ftype = xfs_mode_to_ftype(VFS_I(ip)->i_mode); __entry->nlink = VFS_I(ip)->i_nlink; __entry->parents = live->parents; @@ -1638,7 +1638,7 @@ DECLARE_EVENT_CLASS(xchk_pptr_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->namelen = name->len; memcpy(__get_str(name), name, name->len); __entry->far_ino = far_ino; @@ -1680,7 +1680,7 @@ DECLARE_EVENT_CLASS(xchk_dirtree_class, TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; __entry->path_nr = path_nr; - __entry->child_ino = ip->i_ino; + __entry->child_ino = I_INO(ip); __entry->child_gen = VFS_I(ip)->i_generation; __entry->parent_ino = be64_to_cpu(pptr->p_ino); __entry->parent_gen = be32_to_cpu(pptr->p_gen); @@ -1727,7 +1727,7 @@ DECLARE_EVENT_CLASS(xchk_dirpath_class, __entry->dev = sc->mp->m_super->s_dev; __entry->path_nr = path_nr; __entry->step_nr = step_nr; - __entry->child_ino = ip->i_ino; + __entry->child_ino = I_INO(ip); __entry->child_gen = VFS_I(ip)->i_generation; __entry->parent_ino = be64_to_cpu(pptr->p_ino); __entry->parent_gen = be32_to_cpu(pptr->p_gen); @@ -1830,7 +1830,7 @@ DECLARE_EVENT_CLASS(xchk_dirtree_evaluate_class, ), TP_fast_assign( __entry->dev = dl->sc->mp->m_super->s_dev; - __entry->ino = dl->sc->ip->i_ino; + __entry->ino = I_INO(dl->sc->ip); __entry->rootino = dl->root_ino; __entry->nr_paths = dl->nr_paths; __entry->bad = oc->bad; @@ -1873,8 +1873,8 @@ TRACE_EVENT(xchk_dirpath_changed, __entry->dev = sc->mp->m_super->s_dev; __entry->path_nr = path_nr; __entry->step_nr = step_nr; - __entry->child_ino = ip->i_ino; - __entry->parent_ino = dp->i_ino; + __entry->child_ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); __entry->namelen = xname->len; memcpy(__get_str(name), xname->name, xname->len); ), @@ -1904,9 +1904,9 @@ TRACE_EVENT(xchk_dirtree_live_update, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->parent_ino = dp->i_ino; + __entry->parent_ino = I_INO(dp); __entry->action = action; - __entry->child_ino = ip->i_ino; + __entry->child_ino = I_INO(ip); __entry->delta = delta; __entry->namelen = xname->len; memcpy(__get_str(name), xname->name, xname->len); @@ -1933,8 +1933,8 @@ DECLARE_EVENT_CLASS(xchk_metapath_class, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->scrub_ino = sc->ip ? sc->ip->i_ino : NULLFSINO; - __entry->parent_ino = dp ? dp->i_ino : NULLFSINO; + __entry->scrub_ino = sc->ip ? I_INO(sc->ip) : NULLFSINO; + __entry->parent_ino = dp ? I_INO(dp) : NULLFSINO; __entry->ino = ino; __assign_str(name); ), @@ -2208,7 +2208,7 @@ TRACE_EVENT(xrep_bmap_found, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->whichfork = whichfork; __entry->lblk = irec->br_startoff; __entry->len = irec->br_blockcount; @@ -2609,7 +2609,7 @@ TRACE_EVENT(xrep_dinode_findmode_dirent, TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; __entry->ino = sc->sm->sm_ino; - __entry->parent_ino = dp->i_ino; + __entry->parent_ino = I_INO(dp); __entry->ftype = ftype; ), TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx ftype '%s'", @@ -2633,7 +2633,7 @@ TRACE_EVENT(xrep_dinode_findmode_dirent_inval, TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; __entry->ino = sc->sm->sm_ino; - __entry->parent_ino = dp->i_ino; + __entry->parent_ino = I_INO(dp); __entry->ftype = ftype; __entry->found_ftype = found_ftype; ), @@ -2658,7 +2658,7 @@ TRACE_EVENT(xrep_cow_mark_file_range, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->startoff = startoff; __entry->startblock = startblock; __entry->blockcount = blockcount; @@ -2671,41 +2671,6 @@ TRACE_EVENT(xrep_cow_mark_file_range, __entry->blockcount) ); -TRACE_EVENT(xrep_cow_replace_mapping, - TP_PROTO(struct xfs_inode *ip, const struct xfs_bmbt_irec *irec, - xfs_fsblock_t new_startblock, xfs_extlen_t new_blockcount), - TP_ARGS(ip, irec, new_startblock, new_blockcount), - TP_STRUCT__entry( - __field(dev_t, dev) - __field(xfs_ino_t, ino) - __field(xfs_fsblock_t, startblock) - __field(xfs_fileoff_t, startoff) - __field(xfs_filblks_t, blockcount) - __field(xfs_exntst_t, state) - __field(xfs_fsblock_t, new_startblock) - __field(xfs_extlen_t, new_blockcount) - ), - TP_fast_assign( - __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; - __entry->startoff = irec->br_startoff; - __entry->startblock = irec->br_startblock; - __entry->blockcount = irec->br_blockcount; - __entry->state = irec->br_state; - __entry->new_startblock = new_startblock; - __entry->new_blockcount = new_blockcount; - ), - TP_printk("dev %d:%d ino 0x%llx startoff 0x%llx startblock 0x%llx fsbcount 0x%llx state 0x%x new_startblock 0x%llx new_fsbcount 0x%x", - MAJOR(__entry->dev), MINOR(__entry->dev), - __entry->ino, - __entry->startoff, - __entry->startblock, - __entry->blockcount, - __entry->state, - __entry->new_startblock, - __entry->new_blockcount) -); - TRACE_EVENT(xrep_cow_free_staging, TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t agbno, xfs_extlen_t blockcount), @@ -2817,13 +2782,13 @@ TRACE_EVENT(xrep_tempfile_create, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = sc->file ? XFS_I(file_inode(sc->file))->i_ino : 0; + __entry->ino = sc->file ? file_inode(sc->file)->i_ino : 0; __entry->type = sc->sm->sm_type; __entry->agno = sc->sm->sm_agno; __entry->inum = sc->sm->sm_ino; __entry->gen = sc->sm->sm_gen; __entry->flags = sc->sm->sm_flags; - __entry->temp_inum = sc->tempip->i_ino; + __entry->temp_inum = I_INO(sc->tempip); ), TP_printk("dev %d:%d ino 0x%llx type %s inum 0x%llx gen 0x%x flags 0x%x temp_inum 0x%llx", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -2850,7 +2815,7 @@ DECLARE_EVENT_CLASS(xrep_tempfile_class, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = sc->tempip->i_ino; + __entry->ino = I_INO(sc->tempip); __entry->whichfork = whichfork; __entry->lblk = irec->br_startoff; __entry->len = irec->br_blockcount; @@ -2890,7 +2855,7 @@ TRACE_EVENT(xreap_ifork_extent, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->whichfork = whichfork; __entry->fileoff = irec->br_startoff; __entry->len = irec->br_blockcount; @@ -2946,7 +2911,7 @@ TRACE_EVENT(xrep_xattr_recover_leafblock, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->dabno = dabno; __entry->magic = magic; ), @@ -2971,7 +2936,7 @@ DECLARE_EVENT_CLASS(xrep_xattr_salvage_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->flags = flags; __entry->namelen = namelen; memcpy(__get_str(name), name, namelen); @@ -3011,7 +2976,7 @@ DECLARE_EVENT_CLASS(xrep_pptr_salvage_class, const struct xfs_parent_rec *rec = value; __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->parent_ino = be64_to_cpu(rec->p_ino); __entry->parent_gen = be32_to_cpu(rec->p_gen); __entry->namelen = namelen; @@ -3043,8 +3008,8 @@ DECLARE_EVENT_CLASS(xrep_xattr_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; - __entry->src_ino = arg_ip->i_ino; + __entry->ino = I_INO(ip); + __entry->src_ino = I_INO(arg_ip); ), TP_printk("dev %d:%d ino 0x%llx src 0x%llx", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -3073,8 +3038,8 @@ DECLARE_EVENT_CLASS(xrep_xattr_pptr_scan_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; - __entry->parent_ino = dp->i_ino; + __entry->ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); __entry->parent_gen = VFS_IC(dp)->i_generation; __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -3108,7 +3073,7 @@ TRACE_EVENT(xrep_dir_recover_dirblock, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->dabno = dabno; __entry->magic = magic; __entry->magic_guess = magic_guess; @@ -3131,7 +3096,7 @@ DECLARE_EVENT_CLASS(xrep_dir_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->parent_ino = parent_ino; ), TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", @@ -3161,7 +3126,7 @@ DECLARE_EVENT_CLASS(xrep_dirent_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); __entry->ino = ino; @@ -3198,8 +3163,8 @@ DECLARE_EVENT_CLASS(xrep_adoption_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; - __entry->child_ino = ip->i_ino; + __entry->dir_ino = I_INO(dp); + __entry->child_ino = I_INO(ip); __entry->moved = moved; ), TP_printk("dev %d:%d dir 0x%llx child 0x%llx moved? %d", @@ -3224,7 +3189,7 @@ DECLARE_EVENT_CLASS(xrep_parent_salvage_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->ino = ino; ), TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", @@ -3254,7 +3219,7 @@ DECLARE_EVENT_CLASS(xrep_pptr_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->parent_ino = be64_to_cpu(pptr->p_ino); __entry->parent_gen = be32_to_cpu(pptr->p_gen); __entry->namelen = name->len; @@ -3292,8 +3257,8 @@ DECLARE_EVENT_CLASS(xrep_pptr_scan_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; - __entry->parent_ino = dp->i_ino; + __entry->ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); __entry->parent_gen = VFS_IC(dp)->i_generation; __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -3392,7 +3357,7 @@ TRACE_EVENT(xrep_symlink_salvage_target, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->targetlen = targetlen; memcpy(__get_str(target), target, targetlen); __get_str(target)[targetlen] = 0; @@ -3413,7 +3378,7 @@ DECLARE_EVENT_CLASS(xrep_symlink_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); ), TP_printk("dev %d:%d ip 0x%llx", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -3443,7 +3408,7 @@ TRACE_EVENT(xrep_iunlink_visit, TP_fast_assign( __entry->dev = pag_mount(pag)->m_super->s_dev; __entry->agno = pag_agno(pag); - __entry->agino = XFS_INO_TO_AGINO(pag_mount(pag), ip->i_ino); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->bucket = bucket; __entry->bucket_agino = bucket_agino; __entry->prev_agino = ip->i_prev_unlinked; @@ -3473,8 +3438,8 @@ TRACE_EVENT(xrep_iunlink_reload_next, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->old_prev_agino = ip->i_prev_unlinked; __entry->prev_agino = prev_agino; __entry->next_agino = ip->i_next_unlinked; @@ -3503,8 +3468,8 @@ TRACE_EVENT(xrep_iunlink_reload_ondisk, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->nlink = VFS_I(ip)->i_nlink; __entry->next_agino = ip->i_next_unlinked; ), @@ -3590,8 +3555,8 @@ TRACE_EVENT(xrep_iunlink_relink_next, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->next_agino = ip->i_next_unlinked; __entry->new_next_agino = next_agino; ), @@ -3616,8 +3581,8 @@ TRACE_EVENT(xrep_iunlink_relink_prev, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->prev_agino = ip->i_prev_unlinked; __entry->new_prev_agino = prev_agino; ), diff --git a/fs/xfs/scrub/xfarray.c b/fs/xfs/scrub/xfarray.c index c7c4a71b6fa7..2ce24bfe4c0f 100644 --- a/fs/xfs/scrub/xfarray.c +++ b/fs/xfs/scrub/xfarray.c @@ -487,8 +487,7 @@ xfarray_sortinfo_alloc( xfarray_sortinfo_lo(si)[0] = 0; xfarray_sortinfo_hi(si)[0] = array->nr - 1; si->relax = INIT_XCHK_RELAX; - if (flags & XFARRAY_SORT_KILLABLE) - si->relax.interruptible = false; + si->relax.killable = !!(flags & XFARRAY_SORT_KILLABLE); trace_xfarray_sort(si, nr_bytes); *infop = si; diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index f279055fcea0..51293b6f331f 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -139,6 +139,17 @@ xfs_end_ioend_write( */ error = blk_status_to_errno(ioend->io_bio.bi_status); if (unlikely(error)) { + /* + * Zoned writes update the in-core open zone accounting before + * I/O submission. A failed write leaves that state + * inconsistent, so shut down the filesystem instead of letting + * later writers wait forever for open zone space to become + * available. + */ + if (is_zoned) { + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + goto done; + } if (ioend->io_flags & IOMAP_IOEND_SHARED) { ASSERT(!is_zoned); xfs_reflink_cancel_cow_range(ip, offset, size, true); @@ -268,7 +279,7 @@ xfs_discard_folio( xfs_alert_ratelimited(mp, "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.", - folio, ip->i_ino, pos); + folio, I_INO(ip), pos); /* * The end of the punch range is always the offset of the first @@ -753,8 +764,7 @@ xfs_bio_submit_read( /* defer read completions to the ioend workqueue */ iomap_init_ioend(iter->inode, bio, ctx->read_ctx_file_offset, 0); - bio->bi_end_io = xfs_end_bio; - submit_bio(bio); + iomap_bio_submit_read_endio(iter, ctx, xfs_end_bio); } static const struct iomap_read_ops xfs_iomap_read_ops = { diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index deab14f31b38..a53ee2b0c54a 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -378,7 +378,7 @@ xfs_attr_log_item( * structure with fields from this xfs_attr_intent */ attrp = &attrip->attri_format; - attrp->alfi_ino = args->dp->i_ino; + attrp->alfi_ino = I_INO(args->dp); ASSERT(!(attr->xattri_op_flags & ~XFS_ATTRI_OP_FLAGS_TYPE_MASK)); attrp->alfi_op_flags = attr->xattri_op_flags; attrp->alfi_value_len = nv->value.iov_len; @@ -695,7 +695,7 @@ xfs_attri_recover_work( args->attr_filter = attrp->alfi_attr_filter & XFS_ATTRI_FILTER_MASK; args->op_flags = XFS_DA_OP_RECOVERY | XFS_DA_OP_OKNOENT | XFS_DA_OP_LOGGED; - args->owner = args->dp->i_ino; + args->owner = I_INO(args->dp); xfs_attr_sethash(args); switch (xfs_attr_intent_op(attr)) { diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 114566b1ae5c..f751edb930b6 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -248,7 +248,7 @@ xfs_attr_node_list_lookup( goto out_corruptbuf; } - fa = xfs_da3_node_header_check(bp, dp->i_ino); + fa = xfs_da3_node_header_check(bp, I_INO(dp)); if (fa) goto out_corruptbuf; @@ -287,7 +287,7 @@ xfs_attr_node_list_lookup( } } - fa = xfs_attr3_leaf_header_check(bp, dp->i_ino); + fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); goto out_releasebuf; @@ -348,7 +348,7 @@ xfs_attr_node_list( case XFS_DA_NODE_MAGIC: case XFS_DA3_NODE_MAGIC: trace_xfs_attr_list_wrong_blk(context); - fa = xfs_da3_node_header_check(bp, dp->i_ino); + fa = xfs_da3_node_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); @@ -359,7 +359,7 @@ xfs_attr_node_list( case XFS_ATTR_LEAF_MAGIC: case XFS_ATTR3_LEAF_MAGIC: leaf = bp->b_addr; - fa = xfs_attr3_leaf_header_check(bp, dp->i_ino); + fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); xfs_trans_brelse(context->tp, bp); @@ -417,7 +417,7 @@ need_lookup: break; cursor->blkno = leafhdr.forw; xfs_trans_brelse(context->tp, bp); - error = xfs_attr3_leaf_read(context->tp, dp, dp->i_ino, + error = xfs_attr3_leaf_read(context->tp, dp, I_INO(dp), cursor->blkno, &bp); if (error) return error; @@ -543,7 +543,7 @@ xfs_attr_leaf_list( context->cursor.blkno = 0; error = xfs_attr3_leaf_read(context->tp, context->dp, - context->dp->i_ino, 0, &bp); + I_INO(context->dp), 0, &bp); if (error) return error; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index b237a25d6045..89f6e79a955f 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -245,7 +245,7 @@ xfs_bmap_update_diff_items( struct xfs_bmap_intent *ba = bi_entry(a); struct xfs_bmap_intent *bb = bi_entry(b); - return cmp_int(ba->bi_owner->i_ino, bb->bi_owner->i_ino); + return cmp_int(I_INO(ba->bi_owner), I_INO(bb->bi_owner)); } /* Log bmap updates in the intent item. */ @@ -266,7 +266,7 @@ xfs_bmap_update_log_item( next_extent = atomic_inc_return(&buip->bui_next_extent) - 1; ASSERT(next_extent < buip->bui_format.bui_nextents); map = &buip->bui_format.bui_extents[next_extent]; - map->me_owner = bi->bi_owner->i_ino; + map->me_owner = I_INO(bi->bi_owner); map->me_startblock = bi->bi_bmap.br_startblock; map->me_startoff = bi->bi_bmap.br_startoff; map->me_len = bi->bi_bmap.br_blockcount; diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 0ab00615f1ad..c88b9ade7389 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1479,7 +1479,7 @@ xfs_swap_change_owner( struct xfs_trans *tp = *tpp; do { - error = xfs_bmbt_change_owner(tp, ip, XFS_DATA_FORK, ip->i_ino, + error = xfs_bmbt_change_owner(tp, ip, XFS_DATA_FORK, I_INO(ip), NULL); /* success or fatal error */ if (error != -EAGAIN) @@ -1631,7 +1631,7 @@ xfs_swap_extents( if (error) { xfs_notice(mp, "%s: inode 0x%llx format is incompatible for exchanging.", - __func__, ip->i_ino); + __func__, I_INO(ip)); goto out_trans_cancel; } @@ -1744,3 +1744,92 @@ out_trans_cancel: xfs_trans_cancel(tp); goto out_unlock_ilock; } + +/* + * Given a CoW fork mapping @got and a replacement mapping @rep, map the space + * described by @rep into the cow fork, pushing aside @got as necessary. @icur + * must point to iext tree leaf containing @got. + */ +void +xfs_bmap_replace_cow_mapping( + struct xfs_inode *ip, + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + xfs_fileoff_t rep_endoff = + rep->br_startoff + rep->br_blockcount; + xfs_fileoff_t got_endoff = + got->br_startoff + got->br_blockcount; + uint32_t state = BMAP_COWFORK; + + ASSERT(rep->br_blockcount > 0); + ASSERT(!isnullstartblock(got->br_startblock)); + ASSERT(got->br_startoff <= rep->br_startoff); + ASSERT(got_endoff >= rep_endoff); + + trace_xfs_bmap_replace_cow_mapping(ip, got, rep); + + if (got->br_startoff == rep->br_startoff) + state |= BMAP_LEFT_FILLING; + if (got_endoff == rep_endoff) + state |= BMAP_RIGHT_FILLING; + + switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) { + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: + /* + * Replacement matches the whole mapping, update the record. + */ + xfs_iext_update_extent(ip, state, icur, rep); + break; + case BMAP_LEFT_FILLING: + /* + * Replace the first part of the mapping: Update the cursor + * position with the new mapping, then add a record with the + * tail of the old mapping. + */ + got->br_startoff = rep_endoff; + got->br_blockcount -= rep->br_blockcount; + got->br_startblock += rep->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, rep); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, got, state); + break; + case BMAP_RIGHT_FILLING: + /* + * Replacing the last part of the mapping. Shorten the current + * mapping then add a record with the new mapping. + */ + got->br_blockcount -= rep->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, rep, state); + break; + case 0: + /* + * Replacing the middle of the extent. Shorten the current + * mapping, add a new record with the new mapping, and add a + * second new record with the tail of the old mapping. + */ + got->br_blockcount = rep->br_startoff - got->br_startoff; + + struct xfs_bmbt_irec new = { + .br_startoff = rep_endoff, + .br_blockcount = got_endoff - rep_endoff, + .br_state = got->br_state, + .br_startblock = got->br_startblock + + rep->br_blockcount + + got->br_blockcount, + }; + + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, rep, state); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, &new, state); + break; + } +} diff --git a/fs/xfs/xfs_bmap_util.h b/fs/xfs/xfs_bmap_util.h index c477b3361630..eaaf094154b9 100644 --- a/fs/xfs/xfs_bmap_util.h +++ b/fs/xfs/xfs_bmap_util.h @@ -81,4 +81,8 @@ int xfs_bmap_count_blocks(struct xfs_trans *tp, struct xfs_inode *ip, int xfs_flush_unmap_range(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t len); +void xfs_bmap_replace_cow_mapping(struct xfs_inode *ip, + struct xfs_iext_cursor *icur, struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep); + #endif /* __XFS_BMAP_UTIL_H__ */ diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 580d40a5ee57..e1465e950acc 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -121,6 +121,22 @@ xfs_buf_free( } static int +xfs_buf_alloc_folio( + struct xfs_buf *bp, + size_t size, + gfp_t gfp_mask) +{ + struct folio *folio; + + folio = folio_alloc(gfp_mask, get_order(size)); + if (!folio) + return -ENOMEM; + bp->b_addr = folio_address(folio); + trace_xfs_buf_backing_folio(bp, _RET_IP_); + return 0; +} + +static int xfs_buf_alloc_kmem( struct xfs_buf *bp, size_t size, @@ -129,7 +145,7 @@ xfs_buf_alloc_kmem( ASSERT(is_power_of_2(size)); ASSERT(size < PAGE_SIZE); - bp->b_addr = kmalloc(size, gfp_mask | __GFP_NOFAIL); + bp->b_addr = kmalloc(size, gfp_mask); if (!bp->b_addr) return -ENOMEM; @@ -148,6 +164,26 @@ xfs_buf_alloc_kmem( return 0; } +static int +xfs_buf_alloc_vmalloc( + struct xfs_buf *bp, + size_t size, + gfp_t gfp_mask) +{ + for (;;) { + bp->b_addr = __vmalloc(size, gfp_mask); + if (bp->b_addr) + break; + if (gfp_mask & __GFP_NORETRY) + return -ENOMEM; + XFS_STATS_INC(bp->b_mount, xb_page_retries); + memalloc_retry_wait(gfp_mask); + } + + trace_xfs_buf_backing_vmalloc(bp, _RET_IP_); + return 0; +} + /* * Allocate backing memory for a buffer. * @@ -175,7 +211,6 @@ xfs_buf_alloc_backing_mem( { size_t size = BBTOB(bp->b_length); gfp_t gfp_mask = GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOWARN; - struct folio *folio; if (xfs_buftarg_is_mem(bp->b_target)) return xmbuf_map_backing_mem(bp); @@ -188,22 +223,6 @@ xfs_buf_alloc_backing_mem( gfp_mask |= __GFP_NORETRY; /* - * For buffers smaller than PAGE_SIZE use a kmalloc allocation if that - * is properly aligned. The slab allocator now guarantees an aligned - * allocation for all power of two sizes, which matches most of the - * smaller than PAGE_SIZE buffers used by XFS. - */ - if (size < PAGE_SIZE && is_power_of_2(size)) - return xfs_buf_alloc_kmem(bp, size, gfp_mask); - - /* - * Don't bother with the retry loop for single PAGE allocations: vmalloc - * won't do any better. - */ - if (size <= PAGE_SIZE) - gfp_mask |= __GFP_NOFAIL; - - /* * Optimistically attempt a single high order folio allocation for * larger than PAGE_SIZE buffers. * @@ -215,35 +234,31 @@ xfs_buf_alloc_backing_mem( * path for them instead of wasting memory here. */ if (size > PAGE_SIZE) { - if (!is_power_of_2(size)) - goto fallback; - gfp_mask &= ~__GFP_DIRECT_RECLAIM; - gfp_mask |= __GFP_NORETRY; - } - folio = folio_alloc(gfp_mask, get_order(size)); - if (!folio) { - if (size <= PAGE_SIZE) - return -ENOMEM; - trace_xfs_buf_backing_fallback(bp, _RET_IP_); - goto fallback; - } - bp->b_addr = folio_address(folio); - trace_xfs_buf_backing_folio(bp, _RET_IP_); - return 0; - -fallback: - for (;;) { - bp->b_addr = __vmalloc(size, gfp_mask); - if (bp->b_addr) - break; - if (flags & XBF_READ_AHEAD) - return -ENOMEM; - XFS_STATS_INC(bp->b_mount, xb_page_retries); - memalloc_retry_wait(gfp_mask); + if (is_power_of_2(size)) { + gfp_t folio_gfp = gfp_mask; + + folio_gfp &= ~__GFP_DIRECT_RECLAIM; + folio_gfp |= __GFP_NORETRY; + if (xfs_buf_alloc_folio(bp, size, folio_gfp) == 0) + return 0; + trace_xfs_buf_backing_fallback(bp, _RET_IP_); + } + return xfs_buf_alloc_vmalloc(bp, size, gfp_mask); } - trace_xfs_buf_backing_vmalloc(bp, _RET_IP_); - return 0; + /* + * The slab allocator now guarantees aligned allocations for all power + * of two sizes. This covers most smaller XFS buffers, so just use + * kmalloc in this case. + * + * Don't bother with the vmalloc fallback for allocations of page size + * or less: vmalloc won't do any better. + */ + if (!(gfp_mask & __GFP_NORETRY)) + gfp_mask |= __GFP_NOFAIL; + if (size < PAGE_SIZE && is_power_of_2(size)) + return xfs_buf_alloc_kmem(bp, size, gfp_mask); + return xfs_buf_alloc_folio(bp, size, gfp_mask); } static int @@ -472,6 +487,7 @@ xfs_buf_find_insert( /* The new buffer keeps the perag reference until it is freed. */ new_bp->b_pag = pag; +retry: rcu_read_lock(); bp = rhashtable_lookup_get_insert_fast(&btp->bt_hash, &new_bp->b_rhash_head, xfs_buf_hash_params); @@ -480,8 +496,16 @@ xfs_buf_find_insert( error = PTR_ERR(bp); goto out_free_buf; } - if (bp && lockref_get_not_dead(&bp->b_lockref)) { - /* found an existing buffer */ + if (bp) { + /* + * If there is an existing buffer with a dead lockref, retry + * until the new buffer is added, or a usable buffer is found. + */ + if (!lockref_get_not_dead(&bp->b_lockref)) { + rcu_read_unlock(); + cpu_relax(); + goto retry; + } rcu_read_unlock(); error = xfs_buf_find_lock(bp, flags); if (error) @@ -614,7 +638,7 @@ _xfs_buf_read( * type. If repair can't establish that, the buffer will be left in memory * with NULL buffer ops. */ -int +static int xfs_buf_reverify( struct xfs_buf *bp, const struct xfs_buf_ops *ops) @@ -820,15 +844,20 @@ xfs_buf_destroy( ASSERT(__lockref_is_dead(&bp->b_lockref)); ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); + if (bp->b_pag) + xfs_perag_put(bp->b_pag); + xfs_buf_free(bp); +} + +static inline void +xfs_buf_kill( + struct xfs_buf *bp) +{ + lockref_mark_dead(&bp->b_lockref); if (!xfs_buf_is_uncached(bp)) { rhashtable_remove_fast(&bp->b_target->bt_hash, &bp->b_rhash_head, xfs_buf_hash_params); - - if (bp->b_pag) - xfs_perag_put(bp->b_pag); } - - xfs_buf_free(bp); } /* @@ -851,7 +880,7 @@ xfs_buf_rele( return; kill: - lockref_mark_dead(&bp->b_lockref); + xfs_buf_kill(bp); list_lru_del_obj(&bp->b_target->bt_lru, &bp->b_lru); spin_unlock(&bp->b_lockref.lock); @@ -1069,11 +1098,17 @@ out_stale: return false; } -/* returns false if the caller needs to resubmit the I/O, else true */ -static bool -__xfs_buf_ioend( +/* + * Complete a buffer read or write. + * + * Releases the buffer if the I/O was asynchronous. + */ +static void +xfs_buf_ioend( struct xfs_buf *bp) { + bool async = bp->b_flags & XBF_ASYNC; + trace_xfs_buf_iodone(bp, _RET_IP_); if (bp->b_flags & XBF_READ) { @@ -1087,14 +1122,16 @@ __xfs_buf_ioend( if (bp->b_flags & XBF_READ_AHEAD) percpu_counter_dec(&bp->b_target->bt_readahead_count); } else { - if (!bp->b_error) { + if (unlikely(bp->b_error)) { + if (xfs_buf_ioend_handle_error(bp)) { + ASSERT(async); + return; + } + } else { bp->b_flags &= ~XBF_WRITE_FAIL; bp->b_flags |= XBF_DONE; } - if (unlikely(bp->b_error) && xfs_buf_ioend_handle_error(bp)) - return false; - /* clear the retry state */ bp->b_last_error = 0; bp->b_retries = 0; @@ -1114,30 +1151,15 @@ __xfs_buf_ioend( bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD | _XBF_LOGRECOVERY); - return true; -} - -static void -xfs_buf_ioend( - struct xfs_buf *bp) -{ - if (!__xfs_buf_ioend(bp)) - return; - if (bp->b_flags & XBF_ASYNC) + if (async) xfs_buf_relse(bp); - else - complete(&bp->b_iowait); } static void xfs_buf_ioend_work( struct work_struct *work) { - struct xfs_buf *bp = - container_of(work, struct xfs_buf, b_ioend_work); - - if (__xfs_buf_ioend(bp)) - xfs_buf_relse(bp); + xfs_buf_ioend(container_of(work, struct xfs_buf, b_ioend_work)); } void @@ -1163,17 +1185,18 @@ xfs_buf_ioerror_alert( } /* - * To simulate an I/O failure, the buffer must be locked and held with at least - * two references. + * Fail a locked and referenced buffer outside the I/O path. * - * The buf item reference is dropped via ioend processing. The second reference - * is owned by the caller and is dropped on I/O completion if the buffer is - * XBF_ASYNC. + * The caller transfers a reference which will be released after processing the + * error. */ void -xfs_buf_ioend_fail( +xfs_buf_fail( struct xfs_buf *bp) { + ASSERT(xfs_buf_islocked(bp)); + + bp->b_flags |= XBF_ASYNC; bp->b_flags &= ~XBF_DONE; xfs_buf_stale(bp); xfs_buf_ioerror(bp, -EIO); @@ -1286,12 +1309,11 @@ xfs_buf_iowait( { ASSERT(!(bp->b_flags & XBF_ASYNC)); - do { - trace_xfs_buf_iowait(bp, _RET_IP_); - wait_for_completion(&bp->b_iowait); - trace_xfs_buf_iowait_done(bp, _RET_IP_); - } while (!__xfs_buf_ioend(bp)); + trace_xfs_buf_iowait(bp, _RET_IP_); + wait_for_completion(&bp->b_iowait); + trace_xfs_buf_iowait_done(bp, _RET_IP_); + xfs_buf_ioend(bp); return bp->b_error; } @@ -1355,8 +1377,8 @@ xfs_buf_submit( * on shutdown. */ if (bp->b_mount->m_log && xlog_is_shutdown(bp->b_mount->m_log)) { - xfs_buf_ioend_fail(bp); - return; + xfs_buf_ioerror(bp, -EIO); + goto ioerror; } if (bp->b_flags & XBF_WRITE) @@ -1369,18 +1391,26 @@ xfs_buf_submit( bp->b_error = 0; if ((bp->b_flags & XBF_WRITE) && !xfs_buf_verify_write(bp)) { + /* ->verify_write should have set b_error already */ xfs_force_shutdown(bp->b_mount, SHUTDOWN_CORRUPT_INCORE); - xfs_buf_ioend(bp); - return; + goto ioerror; } /* In-memory targets are directly mapped, no I/O required. */ - if (xfs_buftarg_is_mem(bp->b_target)) { - xfs_buf_ioend(bp); - return; - } + if (xfs_buftarg_is_mem(bp->b_target)) + goto end_io; xfs_buf_submit_bio(bp); + return; + +ioerror: + bp->b_flags &= ~XBF_DONE; + xfs_buf_stale(bp); +end_io: + if (bp->b_flags & XBF_ASYNC) + xfs_buf_ioend(bp); + else + complete(&bp->b_iowait); } /* @@ -1433,7 +1463,7 @@ xfs_buftarg_drain_rele( return LRU_SKIP; } - lockref_mark_dead(&bp->b_lockref); + xfs_buf_kill(bp); list_lru_isolate_move(lru, item, dispose); spin_unlock(&bp->b_lockref.lock); return LRU_REMOVED; @@ -1545,7 +1575,7 @@ xfs_buftarg_isolate( return LRU_ROTATE; } - lockref_mark_dead(&bp->b_lockref); + xfs_buf_kill(bp); list_lru_isolate_move(lru, item, dispose); spin_unlock(&bp->b_lockref.lock); return LRU_REMOVED; diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index bf39d89f0f6d..79cc9c3f0254 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -290,7 +290,7 @@ extern void __xfs_buf_ioerror(struct xfs_buf *bp, int error, xfs_failaddr_t failaddr); #define xfs_buf_ioerror(bp, err) __xfs_buf_ioerror((bp), (err), __this_address) extern void xfs_buf_ioerror_alert(struct xfs_buf *bp, xfs_failaddr_t fa); -void xfs_buf_ioend_fail(struct xfs_buf *); +void xfs_buf_fail(struct xfs_buf *bp); void __xfs_buf_mark_corrupt(struct xfs_buf *bp, xfs_failaddr_t fa); #define xfs_buf_mark_corrupt(bp) __xfs_buf_mark_corrupt((bp), __this_address) @@ -365,7 +365,6 @@ int xfs_configure_buftarg(struct xfs_buftarg *btp, unsigned int sectorsize, #define xfs_readonly_buftarg(buftarg) bdev_read_only((buftarg)->bt_bdev) -int xfs_buf_reverify(struct xfs_buf *bp, const struct xfs_buf_ops *ops); bool xfs_verify_magic(struct xfs_buf *bp, __be32 dmagic); bool xfs_verify_magic16(struct xfs_buf *bp, __be16 dmagic); diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 8487635579e5..1f055cd6732e 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -549,8 +549,7 @@ xfs_buf_item_unpin( * wait for the lock and then run the IO failure completion. */ xfs_buf_lock(bp); - bp->b_flags |= XBF_ASYNC; - xfs_buf_ioend_fail(bp); + xfs_buf_fail(bp); return; } diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index 60a80d4173f7..577f7a581043 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -84,7 +84,7 @@ xfs_dir2_sf_getdents( */ if (ctx->pos <= dot_offset) { ctx->pos = dot_offset & 0x7fffffff; - if (!dir_emit(ctx, ".", 1, dp->i_ino, DT_DIR)) + if (!dir_emit(ctx, ".", 1, I_INO(dp), DT_DIR)) return 0; } @@ -532,7 +532,7 @@ xfs_readdir( args.dp = dp; args.geo = dp->i_mount->m_dir_geo; args.trans = tp; - args.owner = dp->i_ino; + args.owner = I_INO(dp); if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) return xfs_dir2_sf_getdents(&args, ctx); diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 69e9bc588c8b..c311f61d9554 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1216,6 +1216,14 @@ xfs_qm_dqflush_check( type != XFS_DQTYPE_PROJ) return __this_address; + /* bigtime flag should never be set on root dquots */ + if (dqp->q_type & XFS_DQTYPE_BIGTIME) { + if (!xfs_has_bigtime(dqp->q_mount)) + return __this_address; + if (dqp->q_id == 0) + return __this_address; + } + if (dqp->q_id == 0) return NULL; @@ -1231,14 +1239,6 @@ xfs_qm_dqflush_check( !dqp->q_rtb.timer) return __this_address; - /* bigtime flag should never be set on root dquots */ - if (dqp->q_type & XFS_DQTYPE_BIGTIME) { - if (!xfs_has_bigtime(dqp->q_mount)) - return __this_address; - if (dqp->q_id == 0) - return __this_address; - } - return NULL; } diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index d652240a1dca..9ece70fd94ca 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -369,7 +369,7 @@ xfs_inode_verifier_error( xfs_alert(mp, "Metadata %s detected at %pS, inode 0x%llx %s", error == -EFSBADCRC ? "CRC error" : "corruption", - fa, ip->i_ino, name); + fa, I_INO(ip), name); xfs_alert(mp, "Unmount and run xfs_repair"); diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c index 13a42467370f..c3745d33e54e 100644 --- a/fs/xfs/xfs_exchmaps_item.c +++ b/fs/xfs/xfs_exchmaps_item.c @@ -227,9 +227,9 @@ xfs_exchmaps_create_intent( xmi_lip = xfs_xmi_init(tp->t_mountp); xlf = &xmi_lip->xmi_format; - xlf->xmi_inode1 = xmi->xmi_ip1->i_ino; + xlf->xmi_inode1 = I_INO(xmi->xmi_ip1); xlf->xmi_igen1 = VFS_I(xmi->xmi_ip1)->i_generation; - xlf->xmi_inode2 = xmi->xmi_ip2->i_ino; + xlf->xmi_inode2 = I_INO(xmi->xmi_ip2); xlf->xmi_igen2 = VFS_I(xmi->xmi_ip2)->i_generation; xlf->xmi_startoff1 = xmi->xmi_startoff1; xlf->xmi_startoff2 = xmi->xmi_startoff2; diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 5c083f29ea65..94965a6c2187 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -91,7 +91,7 @@ xfs_exchrange_check_freshness( trace_xfs_exchrange_freshness(fxr, ip2); /* Check that file2 hasn't otherwise been modified. */ - if (fxr->file2_ino != ip2->i_ino || + if (fxr->file2_ino != inode2->i_ino || fxr->file2_gen != inode2->i_generation || !timespec64_equal(&fxr->file2_ctime, &ctime) || !timespec64_equal(&fxr->file2_mtime, &mtime)) @@ -863,7 +863,7 @@ xfs_ioc_start_commit( kern_f->file2_ctime_nsec = kstat.ctime.tv_nsec; kern_f->file2_mtime = kstat.mtime.tv_sec; kern_f->file2_mtime_nsec = kstat.mtime.tv_nsec; - kern_f->file2_ino = ip2->i_ino; + kern_f->file2_ino = inode2->i_ino; kern_f->file2_gen = inode2->i_generation; kern_f->magic = XCR_FRESH_MAGIC; xfs_iunlock(ip2, lockflags); diff --git a/fs/xfs/xfs_export.c b/fs/xfs/xfs_export.c index e3e3c3c89840..9afda52ed1a2 100644 --- a/fs/xfs/xfs_export.c +++ b/fs/xfs/xfs_export.c @@ -82,19 +82,19 @@ xfs_fs_encode_fh( switch (fileid_type) { case FILEID_INO32_GEN_PARENT: - fid->i32.parent_ino = XFS_I(parent)->i_ino; + fid->i32.parent_ino = parent->i_ino; fid->i32.parent_gen = parent->i_generation; fallthrough; case FILEID_INO32_GEN: - fid->i32.ino = XFS_I(inode)->i_ino; + fid->i32.ino = inode->i_ino; fid->i32.gen = inode->i_generation; break; case FILEID_INO32_GEN_PARENT | XFS_FILEID_TYPE_64FLAG: - fid64->parent_ino = XFS_I(parent)->i_ino; + fid64->parent_ino = parent->i_ino; fid64->parent_gen = parent->i_generation; fallthrough; case FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG: - fid64->ino = XFS_I(inode)->i_ino; + fid64->ino = inode->i_ino; fid64->gen = inode->i_generation; break; } @@ -244,8 +244,6 @@ const struct export_operations xfs_export_operations = { .get_parent = xfs_fs_get_parent, .commit_metadata = xfs_fs_nfs_commit_metadata, #ifdef CONFIG_EXPORTFS_BLOCK_OPS - .get_uuid = xfs_fs_get_uuid, - .map_blocks = xfs_fs_map_blocks, - .commit_blocks = xfs_fs_commit_blocks, + .block_ops = &xfs_export_block_ops, #endif }; diff --git a/fs/xfs/xfs_filestream.c b/fs/xfs/xfs_filestream.c index 32e641d76e92..bb46852075f2 100644 --- a/fs/xfs/xfs_filestream.c +++ b/fs/xfs/xfs_filestream.c @@ -225,7 +225,7 @@ xfs_filestream_lookup_association( atomic_inc(&pag_group(pag)->xg_active_ref); xfs_mru_cache_done(mp->m_filestream); - trace_xfs_filestream_lookup(pag, ap->ip->i_ino); + trace_xfs_filestream_lookup(pag, I_INO(ap->ip)); ap->blkno = xfs_agbno_to_fsb(pag, 0); xfs_bmap_adjacent(ap); @@ -345,7 +345,7 @@ xfs_filestream_select_ag( args->total = ap->total; pip = xfs_filestream_get_parent(ap->ip); if (pip) { - ino = pip->i_ino; + ino = I_INO(pip); error = xfs_filestream_lookup_association(ap, args, ino, longest); xfs_irele(pip); @@ -370,7 +370,7 @@ void xfs_filestream_deassociate( struct xfs_inode *ip) { - xfs_mru_cache_delete(ip->i_mount->m_filestream, ip->i_ino); + xfs_mru_cache_delete(ip->i_mount->m_filestream, I_INO(ip)); } int diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 8d64d904d73c..67624a804a7f 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -124,7 +124,9 @@ xfs_growfs_data_private( mp->m_sb.sb_rextsize); if (error) return error; - xfs_growfs_compute_deltas(mp, nb, &delta, &nagcount); + + nagcount = xfs_growfs_compute_agcount(mp, &nb); + delta = nb - mp->m_sb.sb_dblocks; /* * Reject filesystems with a single AG because they are not diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c index 2b8617ae7ec2..0689cade8f74 100644 --- a/fs/xfs/xfs_handle.c +++ b/fs/xfs/xfs_handle.c @@ -124,7 +124,7 @@ xfs_find_handle( if (cmd == XFS_IOC_PATH_TO_FSHANDLE) hsize = xfs_fshandle_init(ip->i_mount, &handle); else - hsize = xfs_filehandle_init(ip->i_mount, ip->i_ino, + hsize = xfs_filehandle_init(ip->i_mount, inode->i_ino, inode->i_generation, &handle); error = -EFAULT; @@ -808,7 +808,7 @@ xfs_getparents( sizeof(struct xfs_attrlist_cursor)); /* Is this the root directory? */ - if (ip->i_ino == mp->m_sb.sb_rootino) + if (I_INO(ip) == mp->m_sb.sb_rootino) gp->gp_oflags |= XFS_GETPARENTS_OFLAG_ROOT; if (gpx->context.seen_enough == 0) { diff --git a/fs/xfs/xfs_healthmon.c b/fs/xfs/xfs_healthmon.c index 26c325d34bd1..4521ffdab9f1 100644 --- a/fs/xfs/xfs_healthmon.c +++ b/fs/xfs/xfs_healthmon.c @@ -533,7 +533,7 @@ xfs_healthmon_report_inode( struct xfs_healthmon_event event = { .type = type, .domain = XFS_HEALTHMON_INODE, - .ino = ip->i_ino, + .ino = I_INO(ip), .gen = VFS_I(ip)->i_generation, }; struct xfs_healthmon *hm = xfs_healthmon_get(ip->i_mount); @@ -646,7 +646,7 @@ xfs_healthmon_report_file_ioerror( struct xfs_healthmon_event event = { .type = file_ioerr_type(p->type), .domain = XFS_HEALTHMON_FILERANGE, - .fino = ip->i_ino, + .fino = I_INO(ip), .fgen = VFS_I(ip)->i_generation, .fpos = p->pos, .flen = p->len, @@ -1182,7 +1182,7 @@ xfs_ioc_health_monitor( */ if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (ip->i_ino != mp->m_sb.sb_rootino) + if (I_INO(ip) != mp->m_sb.sb_rootino) return -EPERM; if (current_user_ns() != &init_user_ns) return -EPERM; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 2040a9292ee6..9d8dd30bd927 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -101,6 +101,7 @@ xfs_inode_alloc( return NULL; } + VFS_I(ip)->i_ino = ino; /* VFS doesn't initialise i_mode! */ VFS_I(ip)->i_mode = 0; mapping_set_folio_min_order(VFS_I(ip)->i_mapping, @@ -108,10 +109,8 @@ xfs_inode_alloc( XFS_STATS_INC(mp, xs_inodes_active); ASSERT(atomic_read(&ip->i_pincount) == 0); - ASSERT(ip->i_ino == 0); /* initialise the xfs inode */ - ip->i_ino = ino; ip->i_mount = mp; memset(&ip->i_imap, 0, sizeof(struct xfs_imap)); ip->i_cowfp = NULL; @@ -193,7 +192,7 @@ xfs_inode_free( */ spin_lock(&ip->i_flags_lock); ip->i_flags = XFS_IRECLAIM; - ip->i_ino = 0; + VFS_I(ip)->i_ino = 0; spin_unlock(&ip->i_flags_lock); __xfs_inode_free(ip); @@ -329,6 +328,7 @@ xfs_reinit_inode( struct inode *inode) { int error; + u64 ino = inode->i_ino; uint32_t nlink = inode->i_nlink; uint32_t generation = inode->i_generation; uint64_t version = inode_peek_iversion(inode); @@ -340,6 +340,7 @@ xfs_reinit_inode( error = inode_init_always(mp->m_super, inode); + inode->i_ino = ino; set_nlink(inode, nlink); inode->i_generation = generation; inode_set_iversion_queried(inode, version); @@ -397,7 +398,7 @@ xfs_iget_recycle( */ ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS; ip->i_flags |= XFS_INEW; - xfs_perag_clear_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_clear_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_RECLAIM_TAG); inode_state_assign_raw(inode, I_NEW); spin_unlock(&ip->i_flags_lock); @@ -426,9 +427,8 @@ xfs_iget_check_free_state( if (VFS_I(ip)->i_mode != 0) { xfs_warn(ip->i_mount, "Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)", - ip->i_ino, VFS_I(ip)->i_mode); - xfs_agno_mark_sick(ip->i_mount, - XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), + I_INO(ip), VFS_I(ip)->i_mode); + xfs_agno_mark_sick(ip->i_mount, XFS_INODE_TO_AGNO(ip), XFS_SICK_AG_INOBT); return -EFSCORRUPTED; } @@ -436,9 +436,8 @@ xfs_iget_check_free_state( if (ip->i_nblocks != 0) { xfs_warn(ip->i_mount, "Corruption detected! Free inode 0x%llx has blocks allocated!", - ip->i_ino); - xfs_agno_mark_sick(ip->i_mount, - XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), + I_INO(ip)); + xfs_agno_mark_sick(ip->i_mount, XFS_INODE_TO_AGNO(ip), XFS_SICK_AG_INOBT); return -EFSCORRUPTED; } @@ -516,7 +515,7 @@ xfs_iget_cache_hit( * will not match, so check for that, too. */ spin_lock(&ip->i_flags_lock); - if (ip->i_ino != ino) + if (I_INO(ip) != ino) goto out_skip; /* @@ -646,7 +645,7 @@ xfs_iget_cache_miss( */ xfs_iflags_set(ip, XFS_INEW); - error = xfs_imap(pag, tp, ip->i_ino, &ip->i_imap, flags); + error = xfs_imap(pag, tp, I_INO(ip), &ip->i_imap, flags); if (error) goto out_destroy; @@ -664,7 +663,7 @@ xfs_iget_cache_miss( } else { struct xfs_buf *bp; - error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &bp); + error = xfs_read_icluster(pag, tp, ip->i_imap.im_agbno, &bp); if (error) goto out_destroy; @@ -819,8 +818,11 @@ again: * now. If it's a new inode being created, xfs_init_new_inode will * handle it. */ - if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0) - xfs_setup_existing_inode(ip); + if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0) { + xfs_setup_inode(ip); + xfs_setup_iops(ip); + xfs_finish_inode_setup(ip); + } return 0; out_error_or_again: @@ -962,7 +964,7 @@ xfs_reclaim_inode( struct xfs_inode *ip, struct xfs_perag *pag) { - xfs_ino_t ino = ip->i_ino; /* for radix_tree_delete */ + xfs_ino_t ino = I_INO(ip); /* for radix_tree_delete */ if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) goto out; @@ -1010,7 +1012,7 @@ reclaim: */ spin_lock(&ip->i_flags_lock); ip->i_flags = XFS_IRECLAIM; - ip->i_ino = 0; + VFS_I(ip)->i_ino = 0; ip->i_sick = 0; ip->i_checked = 0; spin_unlock(&ip->i_flags_lock); @@ -1287,10 +1289,10 @@ xfs_blockgc_set_iflag( ip->i_flags |= iflag; spin_unlock(&ip->i_flags_lock); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); spin_lock(&pag->pag_ici_lock); - xfs_perag_set_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_set_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_BLOCKGC_TAG); spin_unlock(&pag->pag_ici_lock); @@ -1324,10 +1326,10 @@ xfs_blockgc_clear_iflag( if (!clear_tag) return; - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); spin_lock(&pag->pag_ici_lock); - xfs_perag_clear_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_clear_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_BLOCKGC_TAG); spin_unlock(&pag->pag_ici_lock); @@ -1510,7 +1512,7 @@ xfs_blockgc_igrab( /* Check for stale RCU freed inode */ spin_lock(&ip->i_flags_lock); - if (!ip->i_ino) + if (!I_INO(ip)) goto out_unlock_noent; if (ip->i_flags & XFS_BLOCKGC_NOGRAB_IFLAGS) @@ -1802,10 +1804,10 @@ restart: * us to see this inode, so another lookup from the * same index will not find it again. */ - if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag_agno(pag)) + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) continue; - first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); - if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) + first_index = XFS_INO_TO_AGINO(mp, I_INO(ip) + 1); + if (first_index < XFS_INODE_TO_AGINO(ip)) done = true; } @@ -1892,7 +1894,7 @@ xfs_check_delalloc( if (isnullstartblock(got.br_startblock)) { xfs_warn(ip->i_mount, "ino %llx %s fork has delalloc extent at [0x%llx:0x%llx]", - ip->i_ino, + I_INO(ip), whichfork == XFS_DATA_FORK ? "data" : "cow", got.br_startoff, got.br_blockcount); } @@ -1916,14 +1918,14 @@ xfs_inodegc_set_reclaimable( ASSERT(0); } - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); spin_lock(&pag->pag_ici_lock); spin_lock(&ip->i_flags_lock); trace_xfs_inode_set_reclaimable(ip); ip->i_flags &= ~(XFS_NEED_INACTIVE | XFS_INACTIVATING); ip->i_flags |= XFS_IRECLAIMABLE; - xfs_perag_set_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_set_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_RECLAIM_TAG); spin_unlock(&ip->i_flags_lock); diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index beaa26ec62da..15279d22a894 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -495,9 +495,9 @@ xfs_lock_two_inodes( ASSERT(!(ip1_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL))); ASSERT(!(ip0_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))); ASSERT(!(ip1_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))); - ASSERT(ip0->i_ino != ip1->i_ino); + ASSERT(I_INO(ip0) != I_INO(ip1)); - if (ip0->i_ino > ip1->i_ino) { + if (I_INO(ip0) > I_INO(ip1)) { swap(ip0, ip1); swap(ip0_mode, ip1_mode); } @@ -699,12 +699,6 @@ xfs_create( */ error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks, &tp); - if (error == -ENOSPC) { - /* flush outstanding delalloc blocks and retry */ - xfs_flush_inodes(mp); - error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, - resblks, &tp); - } if (error) goto out_parent; @@ -1046,7 +1040,7 @@ xfs_itruncate_extents_flags( int error = 0; xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); - if (icount_read(VFS_I(ip))) + if (icount_read_once(VFS_I(ip))) xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL); if (whichfork == XFS_DATA_FORK) ASSERT(new_size <= XFS_ISIZE(ip)); @@ -1363,7 +1357,7 @@ xfs_inactive_health( if (sick & XFS_SICK_INO_FORGET) return; - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); if (!pag) { /* There had better still be a perag structure! */ ASSERT(0); @@ -1386,7 +1380,7 @@ int xfs_inactive( xfs_inode_t *ip) { - struct xfs_mount *mp; + struct xfs_mount *mp = ip->i_mount; int error = 0; int truncate = 0; @@ -1399,7 +1393,20 @@ xfs_inactive( goto out; } - mp = ip->i_mount; + /* + * If the filesystem has been shut down - for example a mount that + * failed after background inactivation was enabled - do not + * inactivate the inode. Inactivation modifies persistent metadata, + * its transactions cannot complete on a shut down mount, and the + * subsystems it relies on (e.g. quota, mp->m_quotainfo) may not be + * set up. The attached dquots are dropped at the out: label and the + * inode then goes straight to reclaim, the same way + * xfs_inode_needs_inactive() already declines to inactivate on a shut + * down mount at queue time. + */ + if (xfs_is_shutdown(mp)) + goto out; + ASSERT(!xfs_iflags_test(ip, XFS_IRECOVERY)); xfs_inactive_health(ip); @@ -1519,7 +1526,7 @@ xfs_iunlink_lookup( * Inode in RCU freeing limbo should not happen. Warn about this and * let the caller handle the failure. */ - if (WARN_ON_ONCE(!ip->i_ino)) { + if (WARN_ON_ONCE(!I_INO(ip))) { rcu_read_unlock(); return NULL; } @@ -1620,7 +1627,7 @@ retry: * valid, the wrong inode or stale. */ spin_lock(&ip->i_flags_lock); - if (ip->i_ino != inum || __xfs_iflags_test(ip, XFS_ISTALE)) + if (I_INO(ip) != inum || __xfs_iflags_test(ip, XFS_ISTALE)) goto out_iflags_unlock; /* @@ -1802,7 +1809,7 @@ xfs_ifree( ASSERT(ip->i_disk_size == 0 || !S_ISREG(VFS_I(ip)->i_mode)); ASSERT(ip->i_nblocks == 0); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); error = xfs_inode_uninit(tp, pag, ip, &xic); if (error) @@ -2061,7 +2068,7 @@ xfs_sort_inodes( */ for (i = 0; i < num_inodes; i++) { for (j = 1; j < num_inodes; j++) { - if (i_tab[j]->i_ino < i_tab[j-1]->i_ino) + if (I_INO(i_tab[j]) < I_INO(i_tab[j-1])) swap(i_tab[j], i_tab[j - 1]); } } @@ -2311,8 +2318,7 @@ retry: struct xfs_perag *pag; struct xfs_buf *bp; - pag = xfs_perag_get(mp, - XFS_INO_TO_AGNO(mp, inodes[i]->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(inodes[i])); error = xfs_read_agi(pag, tp, 0, &bp); xfs_perag_put(pag); if (error) @@ -2393,7 +2399,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_1)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad inode %llu magic number 0x%x, ptr "PTR_FMT, - __func__, ip->i_ino, be16_to_cpu(dip->di_magic), dip); + __func__, I_INO(ip), be16_to_cpu(dip->di_magic), dip); goto flush_out; } if (ip->i_df.if_format == XFS_DINODE_FMT_META_BTREE) { @@ -2402,7 +2408,7 @@ xfs_iflush( xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad %s meta btree inode %Lu, ptr "PTR_FMT, __func__, xfs_metafile_type_str(ip->i_metatype), - ip->i_ino, ip); + I_INO(ip), ip); goto flush_out; } } else if (S_ISREG(VFS_I(ip)->i_mode)) { @@ -2411,7 +2417,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_3)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad regular inode %llu, ptr "PTR_FMT, - __func__, ip->i_ino, ip); + __func__, I_INO(ip), ip); goto flush_out; } } else if (S_ISDIR(VFS_I(ip)->i_mode)) { @@ -2421,7 +2427,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_4)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad directory inode %llu, ptr "PTR_FMT, - __func__, ip->i_ino, ip); + __func__, I_INO(ip), ip); goto flush_out; } } @@ -2430,7 +2436,7 @@ xfs_iflush( xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: detected corrupt incore inode %llu, " "total extents = %llu nblocks = %lld, ptr "PTR_FMT, - __func__, ip->i_ino, + __func__, I_INO(ip), ip->i_df.if_nextents + xfs_ifork_nextents(&ip->i_af), ip->i_nblocks, ip); goto flush_out; @@ -2439,7 +2445,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_6)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: bad inode %llu, forkoff 0x%x, ptr "PTR_FMT, - __func__, ip->i_ino, ip->i_forkoff, ip); + __func__, I_INO(ip), ip->i_forkoff, ip); goto flush_out; } @@ -2447,7 +2453,7 @@ xfs_iflush( ip->i_af.if_format == XFS_DINODE_FMT_META_BTREE) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: meta btree in inode %Lu attr fork, ptr "PTR_FMT, - __func__, ip->i_ino, ip); + __func__, I_INO(ip), ip); goto flush_out; } @@ -2640,8 +2646,7 @@ xfs_iflush_cluster( * inode cluster buffers. */ xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); - bp->b_flags |= XBF_ASYNC; - xfs_buf_ioend_fail(bp); + xfs_buf_fail(bp); return error; } @@ -2747,7 +2752,7 @@ xfs_mmaplock_two_inodes_and_break_dax_layout( { int error; - if (ip1->i_ino > ip2->i_ino) + if (I_INO(ip1) > I_INO(ip2)) swap(ip1, ip2); again: @@ -2859,8 +2864,8 @@ xfs_inode_reload_unlinked_bucket( struct xfs_buf *agibp; struct xfs_agi *agi; struct xfs_perag *pag; - xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, ip->i_ino); - xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ip->i_ino); + xfs_agnumber_t agno = XFS_INODE_TO_AGNO(ip); + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); xfs_agino_t prev_agino, next_agino; unsigned int bucket; bool foundit = false; diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index bd6d33557194..34c1038ebfcd 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -30,7 +30,6 @@ typedef struct xfs_inode { struct xfs_dquot *i_pdquot; /* project dquot */ /* Inode location stuff */ - xfs_ino_t i_ino; /* inode number (agno/agino)*/ struct xfs_imap i_imap; /* location for xfs_imap() */ /* Extent information. */ @@ -184,6 +183,11 @@ static inline const struct inode *VFS_IC(const struct xfs_inode *ip) return &ip->i_vnode; } +static inline xfs_ino_t I_INO(const struct xfs_inode *ip) +{ + return VFS_IC(ip)->i_ino; +} + /* * For regular files we only update the on-disk filesize when actually * writing data back to disk. Until then only the copy in the VFS inode @@ -299,9 +303,9 @@ static inline bool xfs_is_internal_inode(const struct xfs_inode *ip) * Before metadata directories, the only metadata inodes were the * three quota files, the realtime bitmap, and the realtime summary. */ - return ip->i_ino == mp->m_sb.sb_rbmino || - ip->i_ino == mp->m_sb.sb_rsumino || - xfs_is_quota_inode(&mp->m_sb, ip->i_ino); + return I_INO(ip) == mp->m_sb.sb_rbmino || + I_INO(ip) == mp->m_sb.sb_rsumino || + xfs_is_quota_inode(&mp->m_sb, I_INO(ip)); } static inline bool xfs_is_zoned_inode(const struct xfs_inode *ip) @@ -630,13 +634,6 @@ static inline void xfs_finish_inode_setup(struct xfs_inode *ip) unlock_new_inode(VFS_I(ip)); } -static inline void xfs_setup_existing_inode(struct xfs_inode *ip) -{ - xfs_setup_inode(ip); - xfs_setup_iops(ip); - xfs_finish_inode_setup(ip); -} - void xfs_irele(struct xfs_inode *ip); extern struct kmem_cache *xfs_inode_cache; diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 4ae81eed0442..99d6ecccdaa7 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -18,6 +18,7 @@ #include "xfs_buf_item.h" #include "xfs_log.h" #include "xfs_log_priv.h" +#include "xfs_ag.h" #include "xfs_error.h" #include "xfs_rtbitmap.h" @@ -34,7 +35,7 @@ static uint64_t xfs_inode_item_sort( struct xfs_log_item *lip) { - return INODE_ITEM(lip)->ili_inode->i_ino; + return I_INO(INODE_ITEM(lip)->ili_inode); } #ifdef DEBUG_EXPENSIVE @@ -54,7 +55,7 @@ xfs_inode_item_precommit_check( xfs_inode_to_disk(ip, dip, 0); xfs_dinode_calc_crc(mp, dip); - fa = xfs_dinode_verify(mp, ip->i_ino, dip); + fa = xfs_dinode_verify(mp, I_INO(ip), dip); if (fa) { xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, dip, sizeof(*dip), fa); @@ -104,6 +105,7 @@ xfs_inode_item_precommit( { struct xfs_inode_log_item *iip = INODE_ITEM(lip); struct xfs_inode *ip = iip->ili_inode; + struct xfs_mount *mp = ip->i_mount; struct inode *inode = VFS_I(ip); unsigned int flags = iip->ili_dirty_flags; @@ -124,8 +126,7 @@ xfs_inode_item_precommit( * to upgrade this inode to bigtime format, do so now. */ if ((flags & (XFS_ILOG_CORE | XFS_ILOG_TIMESTAMP)) && - xfs_has_bigtime(ip->i_mount) && - !xfs_inode_has_bigtime(ip)) { + xfs_has_bigtime(mp) && !xfs_inode_has_bigtime(ip)) { ip->i_diflags2 |= XFS_DIFLAG2_BIGTIME; flags |= XFS_ILOG_CORE; } @@ -138,14 +139,14 @@ xfs_inode_item_precommit( */ if (ip->i_diflags & XFS_DIFLAG_RTINHERIT) { if ((ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) && - xfs_extlen_to_rtxmod(ip->i_mount, ip->i_extsize) > 0) { + xfs_extlen_to_rtxmod(mp, ip->i_extsize) > 0) { ip->i_diflags &= ~(XFS_DIFLAG_EXTSIZE | XFS_DIFLAG_EXTSZINHERIT); ip->i_extsize = 0; flags |= XFS_ILOG_CORE; } if ((ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) && - xfs_extlen_to_rtxmod(ip->i_mount, ip->i_cowextsize) > 0) { + xfs_extlen_to_rtxmod(mp, ip->i_cowextsize) > 0) { ip->i_diflags2 &= ~XFS_DIFLAG2_COWEXTSIZE; ip->i_cowextsize = 0; flags |= XFS_ILOG_CORE; @@ -154,6 +155,7 @@ xfs_inode_item_precommit( spin_lock(&iip->ili_lock); if (!iip->ili_item.li_buf) { + struct xfs_perag *pag; struct xfs_buf *bp; int error; @@ -167,7 +169,9 @@ xfs_inode_item_precommit( * here. */ spin_unlock(&iip->ili_lock); - error = xfs_imap_to_bp(ip->i_mount, tp, &ip->i_imap, &bp); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + error = xfs_read_icluster(pag, tp, ip->i_imap.im_agbno, &bp); + xfs_perag_put(pag); if (error) return error; @@ -588,7 +592,7 @@ xfs_inode_to_log_dinode( to->di_flags2 = ip->i_diflags2; /* also covers the di_used_blocks union arm: */ to->di_cowextsize = ip->i_cowextsize; - to->di_ino = ip->i_ino; + to->di_ino = I_INO(ip); to->di_lsn = lsn; memset(to->di_pad2, 0, sizeof(to->di_pad2)); uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); @@ -647,13 +651,15 @@ xfs_inode_item_format( { struct xfs_inode_log_item *iip = INODE_ITEM(lip); struct xfs_inode *ip = iip->ili_inode; + struct xfs_mount *mp = ip->i_mount; struct xfs_inode_log_format *ilf; ilf = xlog_format_start(lfb, XLOG_REG_TYPE_IFORMAT); ilf->ilf_type = XFS_LI_INODE; - ilf->ilf_ino = ip->i_ino; - ilf->ilf_blkno = ip->i_imap.im_blkno; - ilf->ilf_len = ip->i_imap.im_len; + ilf->ilf_ino = I_INO(ip); + ilf->ilf_blkno = XFS_AGB_TO_DADDR(mp, XFS_INODE_TO_AGNO(ip), + ip->i_imap.im_agbno); + ilf->ilf_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster); ilf->ilf_boffset = ip->i_imap.im_boffset; ilf->ilf_fields = XFS_ILOG_CORE; ilf->ilf_size = 2; /* format + core */ diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index b42106c4a9ef..169a8fe3bf0a 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -99,7 +99,7 @@ xfs_recover_inode_owner_change( if (in_f->ilf_fields & XFS_ILOG_DOWNER) { ASSERT(in_f->ilf_fields & XFS_ILOG_DBROOT); error = xfs_bmbt_change_owner(NULL, ip, XFS_DATA_FORK, - ip->i_ino, buffer_list); + I_INO(ip), buffer_list); if (error) goto out_free_ip; } @@ -107,7 +107,7 @@ xfs_recover_inode_owner_change( if (in_f->ilf_fields & XFS_ILOG_AOWNER) { ASSERT(in_f->ilf_fields & XFS_ILOG_ABROOT); error = xfs_bmbt_change_owner(NULL, ip, XFS_ATTR_FORK, - ip->i_ino, buffer_list); + I_INO(ip), buffer_list); if (error) goto out_free_ip; } diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 46e234863644..1b53701bebea 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -409,6 +409,26 @@ xfs_ioc_ag_geometry( return 0; } +static void +xfs_rtgroup_report_write_pointer( + struct xfs_rtgroup *rtg, + struct xfs_rtgroup_geometry *rgeo) +{ + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + if (rtg->rtg_open_zone) { + rgeo->rg_writepointer = rtg->rtg_open_zone->oz_allocated; + } else { + xfs_rgblock_t highest_rgbno = xfs_rtrmap_highest_rgbno(rtg); + + if (highest_rgbno == NULLRGBLOCK) + rgeo->rg_writepointer = 0; + else + rgeo->rg_writepointer = highest_rgbno + 1; + } + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + rgeo->rg_flags |= XFS_RTGROUP_GEOM_WRITEPOINTER; +} + STATIC int xfs_ioc_rtgroup_geometry( struct xfs_mount *mp, @@ -416,7 +436,6 @@ xfs_ioc_rtgroup_geometry( { struct xfs_rtgroup *rtg; struct xfs_rtgroup_geometry rgeo; - xfs_rgblock_t highest_rgbno; int error; if (copy_from_user(&rgeo, arg, sizeof(rgeo))) @@ -433,28 +452,16 @@ xfs_ioc_rtgroup_geometry( return -EINVAL; error = xfs_rtgroup_get_geometry(rtg, &rgeo); - xfs_rtgroup_put(rtg); if (error) - return error; - - if (xfs_has_zoned(mp)) { - xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); - if (rtg->rtg_open_zone) { - rgeo.rg_writepointer = rtg->rtg_open_zone->oz_allocated; - } else { - highest_rgbno = xfs_rtrmap_highest_rgbno(rtg); - if (highest_rgbno == NULLRGBLOCK) - rgeo.rg_writepointer = 0; - else - rgeo.rg_writepointer = highest_rgbno + 1; - } - xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); - rgeo.rg_flags |= XFS_RTGROUP_GEOM_WRITEPOINTER; - } + goto out_put_rtg; + if (xfs_has_zoned(mp)) + xfs_rtgroup_report_write_pointer(rtg, &rgeo); if (copy_to_user(arg, &rgeo, sizeof(rgeo))) - return -EFAULT; - return 0; + error = -EFAULT; +out_put_rtg: + xfs_rtgroup_put(rtg); + return error; } /* @@ -517,7 +524,7 @@ xfs_ioc_fsgetxattra( xfs_inode_t *ip, void __user *arg) { - struct file_kattr fa; + struct file_kattr fa = {}; xfs_ilock(ip, XFS_ILOCK_SHARED); xfs_fill_fsxattr(ip, XFS_ATTR_FORK, &fa); @@ -755,9 +762,23 @@ xfs_fileattr_set( trace_xfs_ioctl_setattr(ip); if (!fa->fsx_valid) { - if (fa->flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | - FS_NOATIME_FL | FS_NODUMP_FL | - FS_SYNC_FL | FS_DAX_FL | FS_PROJINHERIT_FL)) + unsigned int allowed = FS_IMMUTABLE_FL | FS_APPEND_FL | + FS_NOATIME_FL | FS_NODUMP_FL | + FS_SYNC_FL | FS_DAX_FL | + FS_PROJINHERIT_FL; + + /* + * FS_CASEFOLD_FL reflects the ASCIICI superblock feature, + * a read-only property. Accept it as a no-op so chattr's + * RMW round-trip succeeds; reject any attempt to enable + * it on a non-ASCIICI filesystem. xfs_flags2diflags() + * has no clause for CASEFOLD, so the bit is dropped from + * the on-disk diflags regardless. + */ + if (xfs_has_asciici(mp)) + allowed |= FS_CASEFOLD_FL; + + if (fa->flags & ~allowed) return -EOPNOTSUPP; } @@ -984,7 +1005,7 @@ xfs_ioc_swapext( if (ip->i_mount != tip->i_mount) return -EINVAL; - if (ip->i_ino == tip->i_ino) + if (I_INO(ip) == I_INO(tip)) return -EINVAL; if (xfs_is_shutdown(ip->i_mount)) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index f20a02f49ed9..225c3de88d03 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -45,7 +45,7 @@ xfs_alert_fsblock_zero( "Access to block zero in inode %llu " "start_block: %llx start_off: %llx " "blkcnt: %llx extent-state: %x", - (unsigned long long)ip->i_ino, + (unsigned long long)I_INO(ip), (unsigned long long)imap->br_startblock, (unsigned long long)imap->br_startoff, (unsigned long long)imap->br_blockcount, @@ -113,6 +113,7 @@ xfs_bmbt_to_iomap( return xfs_alert_fsblock_zero(ip, imap); } + iomap->flags = iomap_flags; if (imap->br_startblock == HOLESTARTBLOCK) { iomap->addr = IOMAP_NULL_ADDR; iomap->type = IOMAP_HOLE; @@ -143,7 +144,6 @@ xfs_bmbt_to_iomap( } iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff); iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount); - iomap->flags = iomap_flags; if (mapping_flags & IOMAP_DAX) { iomap->dax_dev = target->bt_daxdev; } else { @@ -1736,7 +1736,7 @@ restart: if (count_fsb > ac->reserved_blocks) { xfs_warn_ratelimited(mp, "Short write on ino 0x%llx comm %.20s due to three-way race with write fault and direct I/O", - ip->i_ino, current->comm); + I_INO(ip), current->comm); count_fsb = ac->reserved_blocks; if (!count_fsb) { error = -EIO; diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 325c2200c501..6339f4956ecb 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -703,7 +703,7 @@ xfs_vn_getattr( stat->nlink = inode->i_nlink; stat->uid = vfsuid_into_kuid(vfsuid); stat->gid = vfsgid_into_kgid(vfsgid); - stat->ino = ip->i_ino; + stat->ino = inode->i_ino; stat->atime = inode_get_atime(inode); fill_mg_cmtime(stat, request_mask, inode); @@ -1423,7 +1423,6 @@ xfs_setup_inode( gfp_t gfp_mask; bool is_meta = xfs_is_internal_inode(ip); - inode->i_ino = ip->i_ino; inode_state_set_raw(inode, I_NEW); inode_sb_list_add(inode); diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index 97f106d2b4cd..159295c63e8f 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -97,7 +97,7 @@ xfs_bulkstat_one_int( } ASSERT(ip != NULL); - ASSERT(ip->i_imap.im_blkno != 0); + ASSERT(ip->i_imap.im_agbno != 0); inode = VFS_I(ip); vfsuid = i_uid_into_vfsuid(idmap, inode); vfsgid = i_gid_into_vfsgid(idmap, inode); diff --git a/fs/xfs/xfs_iunlink_item.c b/fs/xfs/xfs_iunlink_item.c index a03a48eeb9a8..d76d3f0434f1 100644 --- a/fs/xfs/xfs_iunlink_item.c +++ b/fs/xfs/xfs_iunlink_item.c @@ -40,7 +40,7 @@ static uint64_t xfs_iunlink_item_sort( struct xfs_log_item *lip) { - return IUL_ITEM(lip)->ip->i_ino; + return I_INO(IUL_ITEM(lip)->ip); } /* @@ -59,7 +59,7 @@ xfs_iunlink_log_dinode( int offset; int error; - error = xfs_imap_to_bp(tp->t_mountp, tp, &ip->i_imap, &ibp); + error = xfs_read_icluster(iup->pag, tp, ip->i_imap.im_agbno, &ibp); if (error) return error; /* diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index 0f23812b0b31..ca66429bf6c9 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -81,23 +81,6 @@ xlog_format_copy( } /* - * By comparing each component, we don't have to worry about extra - * endian issues in treating two 32 bit numbers as one 64 bit number - */ -static inline xfs_lsn_t _lsn_cmp(xfs_lsn_t lsn1, xfs_lsn_t lsn2) -{ - if (CYCLE_LSN(lsn1) != CYCLE_LSN(lsn2)) - return (CYCLE_LSN(lsn1)<CYCLE_LSN(lsn2))? -999 : 999; - - if (BLOCK_LSN(lsn1) != BLOCK_LSN(lsn2)) - return (BLOCK_LSN(lsn1)<BLOCK_LSN(lsn2))? -999 : 999; - - return 0; -} - -#define XFS_LSN_CMP(x,y) _lsn_cmp(x,y) - -/* * Flags to xfs_log_force() * * XFS_LOG_SYNC: Synchronous force in-core log to disk diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c index edc368938f30..639f875a8fb2 100644 --- a/fs/xfs/xfs_log_cil.c +++ b/fs/xfs/xfs_log_cil.c @@ -1710,7 +1710,7 @@ xlog_cil_push_background( static void xlog_cil_push_now( struct xlog *log, - xfs_lsn_t push_seq, + xfs_csn_t push_seq, bool async) { struct xfs_cil *cil = log->l_cilp; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 09e6678ca487..fdb011e6ef60 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1028,7 +1028,7 @@ xlog_verify_head( { struct xlog_rec_header *tmp_rhead; char *tmp_buffer; - xfs_daddr_t first_bad; + xfs_daddr_t first_bad = XFS_BUF_DADDR_NULL; xfs_daddr_t tmp_rhead_blk; int found; int error; @@ -1057,7 +1057,8 @@ xlog_verify_head( */ error = xlog_do_recovery_pass(log, *head_blk, tmp_rhead_blk, XLOG_RECOVER_CRCPASS, &first_bad); - if ((error == -EFSBADCRC || error == -EFSCORRUPTED) && first_bad) { + if ((error == -EFSBADCRC || error == -EFSCORRUPTED) && + first_bad != XFS_BUF_DADDR_NULL) { /* * We've hit a potential torn write. Reset the error and warn * about it. @@ -1906,18 +1907,20 @@ xlog_recover_reorder_trans( list_for_each_entry_safe(item, n, &sort_list, ri_list) { enum xlog_recover_reorder fate = XLOG_REORDER_ITEM_LIST; + /* a committed item with no regions has a NULL ri_buf[0] */ + if (!item->ri_cnt || !item->ri_buf) { + xfs_warn(log->l_mp, + "%s: committed log item has no regions", + __func__); + error = -EFSCORRUPTED; + break; + } + item->ri_ops = xlog_find_item_ops(item); if (!item->ri_ops) { xfs_warn(log->l_mp, "%s: unrecognized type of log operation (%d)", __func__, ITEM_TYPE(item)); - ASSERT(0); - /* - * return the remaining items back to the transaction - * item list so they can be freed in caller. - */ - if (!list_empty(&sort_list)) - list_splice_init(&sort_list, &trans->r_itemq); error = -EFSCORRUPTED; break; } @@ -1945,7 +1948,15 @@ xlog_recover_reorder_trans( } } - ASSERT(list_empty(&sort_list)); + /* + * Return the remaining items back to the transaction item list so they + * can be freed in caller. This should only happen when we encounter + * an error. + */ + if (!list_empty(&sort_list)) { + ASSERT(error); + list_splice_init(&sort_list, &trans->r_itemq); + } if (!list_empty(&buffer_list)) list_splice(&buffer_list, &trans->r_itemq); if (!list_empty(&item_list)) @@ -3575,4 +3586,3 @@ xlog_recover_cancel( if (xlog_recovery_needed(log)) xlog_recover_cancel_intents(log); } - diff --git a/fs/xfs/xfs_message.c b/fs/xfs/xfs_message.c index fd297082aeb8..0243e509a468 100644 --- a/fs/xfs/xfs_message.c +++ b/fs/xfs/xfs_message.c @@ -149,10 +149,6 @@ xfs_warn_experimental( .opstate = XFS_OPSTATE_WARNED_LARP, .name = "logged extended attributes", }, - [XFS_EXPERIMENTAL_ZONED] = { - .opstate = XFS_OPSTATE_WARNED_ZONED, - .name = "zoned RT device", - }, }; ASSERT(feat >= 0 && feat < XFS_EXPERIMENTAL_MAX); BUILD_BUG_ON(ARRAY_SIZE(features) != XFS_EXPERIMENTAL_MAX); diff --git a/fs/xfs/xfs_message.h b/fs/xfs/xfs_message.h index 49b0ef40d299..811b885f41c3 100644 --- a/fs/xfs/xfs_message.h +++ b/fs/xfs/xfs_message.h @@ -93,7 +93,6 @@ void xfs_buf_alert_ratelimited(struct xfs_buf *bp, const char *rlmsg, enum xfs_experimental_feat { XFS_EXPERIMENTAL_SHRINK, XFS_EXPERIMENTAL_LARP, - XFS_EXPERIMENTAL_ZONED, XFS_EXPERIMENTAL_MAX, }; diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index b24195f570cd..be90c7b03994 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1080,7 +1080,7 @@ xfs_mountfs( if (XFS_IS_CORRUPT(mp, !S_ISDIR(VFS_I(rip)->i_mode))) { xfs_warn(mp, "corrupted root inode %llu: not a directory", - (unsigned long long)rip->i_ino); + (unsigned long long)I_INO(rip)); xfs_iunlock(rip, XFS_ILOCK_EXCL); error = -EFSCORRUPTED; goto out_rele_rip; @@ -1149,9 +1149,12 @@ xfs_mountfs( * blocks. */ error = xfs_fs_reserve_ag_blocks(mp); - if (error && error == -ENOSPC) + if (error) { + if (error != -ENOSPC) + goto out_rtunmount; xfs_warn(mp, - "ENOSPC reserving per-AG metadata pool, log recovery may fail."); +"ENOSPC reserving per-AG metadata pool, log recovery may fail."); + } error = xfs_log_mount_finish(mp); xfs_fs_unreserve_ag_blocks(mp); if (error) { @@ -1243,11 +1246,19 @@ xfs_mountfs( xfs_irele(mp->m_metadirip); /* - * Inactivate all inodes that might still be in memory after a log - * intent recovery failure so that reclaim can free them. Metadata - * inodes and the root directory shouldn't need inactivation, but the - * mount failed for some reason, so pull down all the state and flee. + * The mount has failed. Mark the filesystem shut down so that any + * inodes still queued for background inactivation are dropped + * straight to reclaim instead of being inactivated: a failed mount + * must not write to the (possibly corrupt, only partially set up) + * persistent metadata, and parts of the mount it would need - e.g. + * the quota subsystem (mp->m_quotainfo) - may never have been + * initialised. + * + * Flush the queue so that those inodes are pulled down and reclaim + * can free them; with the fs shut down xfs_inodegc_inactivate() + * turns each one reclaimable without touching the on-disk structures. */ + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); xfs_inodegc_flush(mp); /* diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index d964bae096ef..66a02d1b9ad7 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -580,8 +580,6 @@ __XFS_HAS_FEAT(nouuid, NOUUID) #define XFS_OPSTATE_WARNED_METADIR 17 /* Filesystem should use qflags to determine quotaon status */ #define XFS_OPSTATE_RESUMING_QUOTAON 18 -/* Kernel has logged a warning about zoned RT device being used on this fs. */ -#define XFS_OPSTATE_WARNED_ZONED 19 /* (Zoned) GC is in progress */ #define XFS_OPSTATE_ZONEGC_RUNNING 20 diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c index 64c8afb935c2..b994ff15d5e4 100644 --- a/fs/xfs/xfs_notify_failure.c +++ b/fs/xfs/xfs_notify_failure.c @@ -350,7 +350,7 @@ xfs_dax_notify_dev_failure( /* * Shutdown fs from a force umount in pre-remove case which won't fail, * so errors can be ignored. Otherwise, shutdown the filesystem with - * CORRUPT flag if error occured or notify.want_shutdown was set during + * CORRUPT flag if error occurred or notify.want_shutdown was set during * RMAP querying. */ if (mf_flags & MF_MEM_PRE_REMOVE) diff --git a/fs/xfs/xfs_pnfs.c b/fs/xfs/xfs_pnfs.c index 221e55887a2a..f8535ecde21a 100644 --- a/fs/xfs/xfs_pnfs.c +++ b/fs/xfs/xfs_pnfs.c @@ -13,6 +13,7 @@ #include "xfs_bmap.h" #include "xfs_iomap.h" #include "xfs_pnfs.h" +#include <linux/exportfs_block.h> /* * Ensure that we do not have any outstanding pNFS layouts that can be used by @@ -45,11 +46,22 @@ xfs_break_leased_layouts( return error; } +static expfs_block_layouts_t +xfs_fs_layouts_supported( + struct super_block *sb) +{ + expfs_block_layouts_t supported = EXPFS_BLOCK_IN_BAND_ID; + + if (exportfs_bdev_supports_out_of_band_id(sb->s_bdev)) + supported |= EXPFS_BLOCK_OUT_OF_BAND_ID; + return supported; +} + /* * Get a unique ID including its location so that the client can identify * the exported device. */ -int +static int xfs_fs_get_uuid( struct super_block *sb, u8 *buf, @@ -104,7 +116,7 @@ xfs_fs_map_update_inode( /* * Get a layout for the pNFS client. */ -int +static int xfs_fs_map_blocks( struct inode *inode, loff_t offset, @@ -118,7 +130,6 @@ xfs_fs_map_blocks( struct xfs_bmbt_irec imap; xfs_fileoff_t offset_fsb, end_fsb; loff_t limit; - int bmapi_flags = XFS_BMAPI_ENTIRE; int nimaps = 1; uint lock_flags; int error = 0; @@ -172,14 +183,18 @@ xfs_fs_map_blocks( offset_fsb = XFS_B_TO_FSBT(mp, offset); lock_flags = xfs_ilock_data_map_shared(ip); + /* request mappings for the specified range only */ error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, - &imap, &nimaps, bmapi_flags); + &imap, &nimaps, 0); + if (error) { + xfs_iunlock(ip, lock_flags); + goto out_unlock; + } seq = xfs_iomap_inode_sequence(ip, 0); ASSERT(!nimaps || imap.br_startblock != DELAYSTARTBLOCK); - if (!error && write && - (!nimaps || imap.br_startblock == HOLESTARTBLOCK)) { + if (write && (!nimaps || imap.br_startblock == HOLESTARTBLOCK)) { if (offset + length > XFS_ISIZE(ip)) end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb); else if (nimaps && imap.br_startblock == HOLESTARTBLOCK) @@ -252,28 +267,27 @@ xfs_pnfs_validate_isize( * to manually flush the cache here similar to what the fsync code path does * for datasyncs on files that have no dirty metadata. */ -int +static int xfs_fs_commit_blocks( struct inode *inode, struct iomap *maps, int nr_maps, - struct iattr *iattr) + loff_t new_size) { struct xfs_inode *ip = XFS_I(inode); struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; + struct timespec64 now; bool update_isize = false; int error, i; loff_t size; - ASSERT(iattr->ia_valid & (ATTR_ATIME|ATTR_CTIME|ATTR_MTIME)); - xfs_ilock(ip, XFS_IOLOCK_EXCL); size = i_size_read(inode); - if ((iattr->ia_valid & ATTR_SIZE) && iattr->ia_size > size) { + if (new_size > size) { update_isize = true; - size = iattr->ia_size; + size = new_size; } for (i = 0; i < nr_maps; i++) { @@ -318,11 +332,13 @@ xfs_fs_commit_blocks( xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); - ASSERT(!(iattr->ia_valid & (ATTR_UID | ATTR_GID))); - setattr_copy(&nop_mnt_idmap, inode, iattr); + now = inode_set_ctime_current(inode); + inode_set_atime_to_ts(inode, now); + inode_set_mtime_to_ts(inode, now); + if (update_isize) { - i_size_write(inode, iattr->ia_size); - ip->i_disk_size = iattr->ia_size; + i_size_write(inode, new_size); + ip->i_disk_size = new_size; } xfs_trans_set_sync(tp); @@ -332,3 +348,10 @@ out_drop_iolock: xfs_iunlock(ip, XFS_IOLOCK_EXCL); return error; } + +const struct exportfs_block_ops xfs_export_block_ops = { + .layouts_supported = xfs_fs_layouts_supported, + .get_uuid = xfs_fs_get_uuid, + .map_blocks = xfs_fs_map_blocks, + .commit_blocks = xfs_fs_commit_blocks, +}; diff --git a/fs/xfs/xfs_pnfs.h b/fs/xfs/xfs_pnfs.h index 940c6c2ad88c..bf43b2009e4c 100644 --- a/fs/xfs/xfs_pnfs.h +++ b/fs/xfs/xfs_pnfs.h @@ -2,13 +2,9 @@ #ifndef _XFS_PNFS_H #define _XFS_PNFS_H 1 -#ifdef CONFIG_EXPORTFS_BLOCK_OPS -int xfs_fs_get_uuid(struct super_block *sb, u8 *buf, u32 *len, u64 *offset); -int xfs_fs_map_blocks(struct inode *inode, loff_t offset, u64 length, - struct iomap *iomap, bool write, u32 *device_generation); -int xfs_fs_commit_blocks(struct inode *inode, struct iomap *maps, int nr_maps, - struct iattr *iattr); +#include <linux/exportfs_block.h> +#ifdef CONFIG_EXPORTFS_BLOCK_OPS int xfs_break_leased_layouts(struct inode *inode, uint *iolock, bool *did_unlock); #else @@ -18,4 +14,7 @@ xfs_break_leased_layouts(struct inode *inode, uint *iolock, bool *did_unlock) return 0; } #endif /* CONFIG_EXPORTFS_BLOCK_OPS */ + +extern const struct exportfs_block_ops xfs_export_block_ops; + #endif /* _XFS_PNFS_H */ diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 7bd15d9e7fba..896b24f87ac9 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -166,10 +166,9 @@ xfs_qm_dqpurge( * does it on success. */ error = xfs_qm_dqflush(dqp, bp); - if (!error) { + if (!error) error = xfs_bwrite(bp); - xfs_buf_relse(bp); - } + xfs_buf_relse(bp); xfs_dqflock(dqp); } xfs_dquot_detach_buf(dqp); @@ -297,7 +296,7 @@ xfs_qm_need_dqattach( return false; if (!XFS_NOT_DQATTACHED(mp, ip)) return false; - if (xfs_is_quota_inode(&mp->m_sb, ip->i_ino)) + if (xfs_is_quota_inode(&mp->m_sb, I_INO(ip))) return false; if (xfs_is_metadir_inode(ip)) return false; @@ -390,7 +389,7 @@ xfs_qm_dqdetach( trace_xfs_dquot_dqdetach(ip); - ASSERT(!xfs_is_quota_inode(&ip->i_mount->m_sb, ip->i_ino)); + ASSERT(!xfs_is_quota_inode(&ip->i_mount->m_sb, I_INO(ip))); if (ip->i_udquot) { xfs_qm_dqrele(ip->i_udquot); ip->i_udquot = NULL; @@ -986,11 +985,11 @@ xfs_qm_qino_alloc( mp->m_sb.sb_qflags = mp->m_qflags & XFS_ALL_QUOTA_ACCT; } if (flags & XFS_QMOPT_UQUOTA) - mp->m_sb.sb_uquotino = (*ipp)->i_ino; + mp->m_sb.sb_uquotino = I_INO(*ipp); else if (flags & XFS_QMOPT_GQUOTA) - mp->m_sb.sb_gquotino = (*ipp)->i_ino; + mp->m_sb.sb_gquotino = I_INO(*ipp); else - mp->m_sb.sb_pquotino = (*ipp)->i_ino; + mp->m_sb.sb_pquotino = I_INO(*ipp); spin_unlock(&mp->m_sb_lock); xfs_log_sb(tp); diff --git a/fs/xfs/xfs_quotaops.c b/fs/xfs/xfs_quotaops.c index 8804508cc2b8..81f739e3aa0c 100644 --- a/fs/xfs/xfs_quotaops.c +++ b/fs/xfs/xfs_quotaops.c @@ -34,7 +34,7 @@ xfs_qm_fill_state( defq = xfs_get_defquota(mp->m_quotainfo, type); - tstate->ino = ip->i_ino; + tstate->ino = I_INO(ip); tstate->flags |= QCI_SYSFILE; tstate->blocks = ip->i_nblocks; tstate->nextents = ip->i_df.if_nextents; diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index db23a0f231d6..480136136635 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -440,6 +440,7 @@ xfs_reflink_fill_cow_hole( struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; xfs_filblks_t resaligned; + unsigned int seq_before = READ_ONCE(ip->i_df.if_seq); unsigned int dblocks = 0, rblocks = 0; int nimaps; int error; @@ -465,6 +466,22 @@ xfs_reflink_fill_cow_hole( *lockmode = XFS_ILOCK_EXCL; + /* + * The data fork mapping may have changed while we dropped the ILOCK + * (a racing O_DIRECT writer under IOLOCK_SHARED can complete a full + * CoW cycle including xfs_reflink_end_cow(), which remaps this offset + * and drops the refcount of the old shared block). Re-read it so the + * shared-status recheck below and the caller's in-place iomap both + * operate on the current mapping rather than a stale physical block. + */ + if (seq_before != READ_ONCE(ip->i_df.if_seq)) { + nimaps = 1; + error = xfs_bmapi_read(ip, imap->br_startoff, + imap->br_blockcount, imap, &nimaps, 0); + if (error) + goto out_trans_cancel; + } + error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); if (error || !*shared) goto out_trans_cancel; @@ -511,6 +528,8 @@ xfs_reflink_fill_delalloc( bool found; do { + unsigned int seq_before = READ_ONCE(ip->i_df.if_seq); + xfs_iunlock(ip, *lockmode); *lockmode = 0; @@ -521,6 +540,23 @@ xfs_reflink_fill_delalloc( *lockmode = XFS_ILOCK_EXCL; + /* + * The data fork mapping may have changed while we dropped the + * ILOCK (a racing O_DIRECT writer under IOLOCK_SHARED can + * complete a full CoW cycle including xfs_reflink_end_cow(), + * which remaps this offset and drops the refcount of the old + * shared block). Re-read it so the shared-status recheck + * below and the caller's in-place iomap both operate on the + * current mapping rather than a stale physical block. + */ + if (seq_before != READ_ONCE(ip->i_df.if_seq)) { + nimaps = 1; + error = xfs_bmapi_read(ip, imap->br_startoff, + imap->br_blockcount, imap, &nimaps, 0); + if (error) + goto out_trans_cancel; + } + error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); if (error || !*shared) @@ -949,16 +985,16 @@ xfs_reflink_end_cow( * repeatedly cycles the ILOCK to allocate one transaction per remapped * extent. * - * If we're being called by writeback then the pages will still - * have PageWriteback set, which prevents races with reflink remapping - * and truncate. Reflink remapping prevents races with writeback by - * taking the iolock and mmaplock before flushing the pages and - * remapping, which means there won't be any further writeback or page - * cache dirtying until the reflink completes. + * If we're being called by writeback then the folios will still + * have the writeback flag set, which prevents races with reflink + * remapping and truncate. Reflink remapping prevents races with + * writeback by taking the iolock and mmaplock before flushing + * the folios and remapping, which means there won't be any further + * writeback or page cache dirtying until the reflink completes. * * We should never have two threads issuing writeback for the same file * region. There are also have post-eof checks in the writeback - * preparation code so that we don't bother writing out pages that are + * preparation code so that we don't bother writing out folios that are * about to be truncated. * * If we're being called as part of directio write completion, the dio @@ -1188,7 +1224,7 @@ xfs_reflink_set_inode_flag( goto out_error; /* Lock both files against IO */ - if (src->i_ino == dest->i_ino) + if (I_INO(src) == I_INO(dest)) xfs_ilock(src, XFS_ILOCK_EXCL); else xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); @@ -1202,7 +1238,7 @@ xfs_reflink_set_inode_flag( } else xfs_iunlock(src, XFS_ILOCK_EXCL); - if (src->i_ino == dest->i_ino) + if (I_INO(src) == I_INO(dest)) goto commit_flags; if (!xfs_is_reflink_inode(dest)) { diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index 153f3c378f9f..7a3f97686989 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -890,8 +890,7 @@ xfs_growfs_rt_sb_fields( static int xfs_growfs_rt_zoned( - struct xfs_rtgroup *rtg, - xfs_rfsblock_t nrblocks) + struct xfs_rtgroup *rtg) { struct xfs_mount *mp = rtg_mount(rtg); struct xfs_mount *nmp; @@ -903,7 +902,8 @@ xfs_growfs_rt_zoned( * Calculate new sb and mount fields for this round. Also ensure the * rtg_extents value is uptodate as the rtbitmap code relies on it. */ - nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks, + nmp = xfs_growfs_rt_alloc_fake_mount(mp, + xfs_rtgs_to_rfsbs(mp, rtg_rgno(rtg) + 1), mp->m_sb.sb_rextsize); if (!nmp) return -ENOMEM; @@ -933,6 +933,14 @@ xfs_growfs_rt_zoned( mp->m_features |= XFS_FEAT_REALTIME; xfs_rtrmapbt_compute_maxlevels(mp); xfs_rtrefcountbt_compute_maxlevels(mp); + + /* + * Finally add the newly added zone to the freelist and add the space + * to the available counter. The order is important here: only add + * the available space after the zones, as available space guarantees + * that zones to back it are available. + */ + xfs_zone_mark_free(rtg); xfs_zoned_add_available(mp, freed_rtx); out_free: kfree(nmp); @@ -1218,7 +1226,7 @@ xfs_growfs_rtg( } if (xfs_has_zoned(mp)) { - error = xfs_growfs_rt_zoned(rtg, nrblocks); + error = xfs_growfs_rt_zoned(rtg); goto out_rele; } diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index f8de44443e81..8531d526fc44 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -534,8 +534,11 @@ xfs_open_devices( out_free_rtdev_targ: if (mp->m_rtdev_targp) xfs_free_buftarg(mp->m_rtdev_targp); + mp->m_rtdev_targp = NULL; + rtdev_file = NULL; /* released by xfs_free_buftarg() */ out_free_ddev_targ: xfs_free_buftarg(mp->m_ddev_targp); + mp->m_ddev_targp = NULL; out_close_rtdev: if (rtdev_file) bdev_fput(rtdev_file); @@ -1901,7 +1904,6 @@ xfs_fs_fill_super( error = -EINVAL; goto out_filestream_unmount; } - xfs_warn_experimental(mp, XFS_EXPERIMENTAL_ZONED); } if (xfs_has_reflink(mp)) { diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index c4da624fb296..5585ac7f4d16 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -52,7 +52,7 @@ xfs_readlink( if (pathlen < 0 || pathlen > XFS_SYMLINK_MAXLEN) { xfs_alert(mp, "%s: inode (%llu) bad symlink length (%lld)", - __func__, (unsigned long long) ip->i_ino, + __func__, (unsigned long long)I_INO(ip), (long long) pathlen); ASSERT(0); goto out_corrupt; @@ -186,7 +186,7 @@ xfs_symlink( xfs_qm_vop_create_dqattach(tp, du.ip, udqp, gdqp, pdqp); resblks -= XFS_IALLOC_SPACE_RES(mp); - error = xfs_symlink_write_target(tp, du.ip, du.ip->i_ino, target_path, + error = xfs_symlink_write_target(tp, du.ip, I_INO(du.ip), target_path, pathlen, fs_blocks, resblks); if (error) goto out_trans_cancel; @@ -339,7 +339,7 @@ xfs_inactive_symlink( if (pathlen <= 0 || pathlen > XFS_SYMLINK_MAXLEN) { xfs_alert(mp, "%s: inode (0x%llx) bad symlink length (%d)", - __func__, (unsigned long long)ip->i_ino, pathlen); + __func__, (unsigned long long)I_INO(ip), pathlen); xfs_iunlock(ip, XFS_ILOCK_EXCL); ASSERT(0); xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); diff --git a/fs/xfs/xfs_sysfs.c b/fs/xfs/xfs_sysfs.c index 676777064c2d..b62712187324 100644 --- a/fs/xfs/xfs_sysfs.c +++ b/fs/xfs/xfs_sysfs.c @@ -781,6 +781,23 @@ static const struct kobj_type xfs_zoned_ktype = { }; int +xfs_zoned_sysfs_init(struct xfs_mount *mp) +{ + if (!IS_ENABLED(CONFIG_XFS_RT) || !xfs_has_zoned(mp)) + return 0; + + return xfs_sysfs_init(&mp->m_zoned_kobj, &xfs_zoned_ktype, + &mp->m_kobj, "zoned"); +} + +void +xfs_zoned_sysfs_del(struct xfs_mount *mp) +{ + if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp)) + xfs_sysfs_del(&mp->m_zoned_kobj); +} + +int xfs_mount_sysfs_init( struct xfs_mount *mp) { @@ -820,14 +837,6 @@ xfs_mount_sysfs_init( if (error) goto out_remove_error_dir; - if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp)) { - /* .../xfs/<dev>/zoned/ */ - error = xfs_sysfs_init(&mp->m_zoned_kobj, &xfs_zoned_ktype, - &mp->m_kobj, "zoned"); - if (error) - goto out_remove_error_dir; - } - return 0; out_remove_error_dir: @@ -846,9 +855,6 @@ xfs_mount_sysfs_del( struct xfs_error_cfg *cfg; int i, j; - if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp)) - xfs_sysfs_del(&mp->m_zoned_kobj); - for (i = 0; i < XFS_ERR_CLASS_MAX; i++) { for (j = 0; j < XFS_ERR_ERRNO_MAX; j++) { cfg = &mp->m_error_cfg[i][j]; diff --git a/fs/xfs/xfs_sysfs.h b/fs/xfs/xfs_sysfs.h index 1622fe80ad3e..25e5f8fae2f3 100644 --- a/fs/xfs/xfs_sysfs.h +++ b/fs/xfs/xfs_sysfs.h @@ -53,6 +53,8 @@ xfs_sysfs_del( } int xfs_mount_sysfs_init(struct xfs_mount *mp); +int xfs_zoned_sysfs_init(struct xfs_mount *mp); +void xfs_zoned_sysfs_del(struct xfs_mount *mp); void xfs_mount_sysfs_del(struct xfs_mount *mp); #endif /* __XFS_SYSFS_H__ */ diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 1c098cfc5c00..aeb89ac53bf1 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -132,7 +132,7 @@ DECLARE_EVENT_CLASS(xfs_attr_list_class, ), TP_fast_assign( __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; - __entry->ino = ctx->dp->i_ino; + __entry->ino = I_INO(ctx->dp); __entry->hashval = ctx->cursor.hashval; __entry->blkno = ctx->cursor.blkno; __entry->offset = ctx->cursor.offset; @@ -634,7 +634,7 @@ TRACE_EVENT(xfs_attr_list_node_descend, ), TP_fast_assign( __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; - __entry->ino = ctx->dp->i_ino; + __entry->ino = I_INO(ctx->dp); __entry->hashval = ctx->cursor.hashval; __entry->blkno = ctx->cursor.blkno; __entry->offset = ctx->cursor.offset; @@ -688,7 +688,7 @@ DECLARE_EVENT_CLASS(xfs_bmap_class, ifp = xfs_iext_state_to_fork(ip, state); xfs_iext_get_extent(ifp, cur, &r); __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->leaf = cur->leaf; __entry->pos = cur->pos; __entry->startoff = r.br_startoff; @@ -1021,7 +1021,7 @@ DECLARE_EVENT_CLASS(xfs_lock_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->lock_flags = lock_flags; __entry->caller_ip = caller_ip; ), @@ -1052,7 +1052,7 @@ DECLARE_EVENT_CLASS(xfs_inode_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->iflags = ip->i_flags; ), TP_printk("dev %d:%d ino 0x%llx iflags 0x%lx", @@ -1128,7 +1128,7 @@ DECLARE_EVENT_CLASS(xfs_fault_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->order = order; ), TP_printk("dev %d:%d ino 0x%llx order %u", @@ -1157,8 +1157,8 @@ DECLARE_EVENT_CLASS(xfs_iref_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; - __entry->count = icount_read(VFS_I(ip)); + __entry->ino = I_INO(ip); + __entry->count = icount_read_once(VFS_I(ip)); __entry->pincount = atomic_read(&ip->i_pincount); __entry->iflags = ip->i_flags; __entry->caller_ip = caller_ip; @@ -1185,7 +1185,7 @@ TRACE_EVENT(xfs_iomap_prealloc_size, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->blocks = blocks; __entry->shift = shift; __entry->writeio_blocks = writeio_blocks; @@ -1272,7 +1272,7 @@ DECLARE_EVENT_CLASS(xfs_namespace_class, ), TP_fast_assign( __entry->dev = VFS_I(dp)->i_sb->s_dev; - __entry->dp_ino = dp->i_ino; + __entry->dp_ino = I_INO(dp); __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); ), @@ -1308,8 +1308,8 @@ TRACE_EVENT(xfs_rename, ), TP_fast_assign( __entry->dev = VFS_I(src_dp)->i_sb->s_dev; - __entry->src_dp_ino = src_dp->i_ino; - __entry->target_dp_ino = target_dp->i_ino; + __entry->src_dp_ino = I_INO(src_dp); + __entry->target_dp_ino = I_INO(target_dp); __entry->src_namelen = src_name->len; __entry->target_namelen = target_name->len; memcpy(__get_str(src_name), src_name->name, src_name->len); @@ -1760,7 +1760,7 @@ DECLARE_EVENT_CLASS(xfs_file_class, ), TP_fast_assign( __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; - __entry->ino = XFS_I(file_inode(iocb->ki_filp))->i_ino; + __entry->ino = file_inode(iocb->ki_filp)->i_ino; __entry->size = XFS_I(file_inode(iocb->ki_filp))->i_disk_size; __entry->offset = iocb->ki_pos; __entry->count = iov_iter_count(iter); @@ -1796,7 +1796,7 @@ TRACE_EVENT(xfs_iomap_atomic_write_cow, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->offset = offset; __entry->count = count; ), @@ -1824,7 +1824,7 @@ DECLARE_EVENT_CLASS(xfs_imap_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->size = ip->i_disk_size; __entry->offset = offset; __entry->count = count; @@ -1869,7 +1869,7 @@ DECLARE_EVENT_CLASS(xfs_simple_io_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->isize = VFS_I(ip)->i_size; __entry->disize = ip->i_disk_size; __entry->offset = offset; @@ -1908,7 +1908,7 @@ DECLARE_EVENT_CLASS(xfs_itrunc_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->size = ip->i_disk_size; __entry->new_size = new_size; ), @@ -1941,7 +1941,7 @@ TRACE_EVENT(xfs_bunmap, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->size = ip->i_disk_size; __entry->fileoff = fileoff; __entry->len = len; @@ -2344,7 +2344,7 @@ DECLARE_EVENT_CLASS(xfs_da_class, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); if (args->namelen) memcpy(__get_str(name), args->name, args->namelen); __entry->namelen = args->namelen; @@ -2411,7 +2411,7 @@ DECLARE_EVENT_CLASS(xfs_attr_class, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); if (args->namelen) memcpy(__get_str(name), args->name, args->namelen); __entry->namelen = args->namelen; @@ -2511,7 +2511,7 @@ DECLARE_EVENT_CLASS(xfs_dir2_space_class, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); __entry->op_flags = args->op_flags; __entry->idx = idx; ), @@ -2544,7 +2544,7 @@ TRACE_EVENT(xfs_dir2_leafn_moveents, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); __entry->op_flags = args->op_flags; __entry->src_idx = src_idx; __entry->dst_idx = dst_idx; @@ -2586,7 +2586,7 @@ DECLARE_EVENT_CLASS(xfs_swap_extent_class, TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; __entry->which = which; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->format = ip->i_df.if_format; __entry->nex = ip->i_df.if_nextents; __entry->broot_size = ip->i_df.if_broot_bytes; @@ -2944,7 +2944,7 @@ TRACE_EVENT(xfs_btree_alloc_block, switch (cur->bc_ops->type) { case XFS_BTREE_TYPE_INODE: __entry->agno = 0; - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); break; case XFS_BTREE_TYPE_AG: __entry->agno = cur->bc_group->xg_gno; @@ -2996,7 +2996,7 @@ TRACE_EVENT(xfs_btree_free_block, __entry->agno = xfs_daddr_to_agno(cur->bc_mp, xfs_buf_daddr(bp)); if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); else __entry->ino = 0; __assign_str(name); @@ -3251,7 +3251,7 @@ DECLARE_EVENT_CLASS(xfs_btree_error_class, switch (cur->bc_ops->type) { case XFS_BTREE_TYPE_INODE: __entry->agno = 0; - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); break; case XFS_BTREE_TYPE_AG: __entry->agno = cur->bc_group->xg_gno; @@ -3470,7 +3470,7 @@ DECLARE_EVENT_CLASS(xfs_bmap_deferred_class, bi->bi_bmap.br_startblock, bi->bi_group->xg_type); } - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->whichfork = bi->bi_whichfork; __entry->l_loff = bi->bi_bmap.br_startoff; __entry->l_len = bi->bi_bmap.br_blockcount; @@ -3974,7 +3974,7 @@ DECLARE_EVENT_CLASS(xfs_inode_error_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->error = error; __entry->caller_ip = caller_ip; ), @@ -4012,12 +4012,12 @@ DECLARE_EVENT_CLASS(xfs_double_io_class, ), TP_fast_assign( __entry->dev = VFS_I(src)->i_sb->s_dev; - __entry->src_ino = src->i_ino; + __entry->src_ino = I_INO(src); __entry->src_isize = VFS_I(src)->i_size; __entry->src_disize = src->i_disk_size; __entry->src_offset = soffset; __entry->len = len; - __entry->dest_ino = dest->i_ino; + __entry->dest_ino = I_INO(dest); __entry->dest_isize = VFS_I(dest)->i_size; __entry->dest_disize = dest->i_disk_size; __entry->dest_offset = doffset; @@ -4057,7 +4057,7 @@ DECLARE_EVENT_CLASS(xfs_inode_irec_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->lblk = irec->br_startoff; __entry->len = irec->br_blockcount; __entry->pblk = irec->br_startblock; @@ -4093,7 +4093,7 @@ DECLARE_EVENT_CLASS(xfs_wb_invalid_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->addr = iomap->addr; __entry->pos = iomap->offset; __entry->len = iomap->length; @@ -4136,7 +4136,7 @@ DECLARE_EVENT_CLASS(xfs_iomap_invalid_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->addr = iomap->addr; __entry->pos = iomap->offset; __entry->len = iomap->length; @@ -4183,10 +4183,10 @@ TRACE_EVENT(xfs_reflink_remap_blocks, ), TP_fast_assign( __entry->dev = VFS_I(src)->i_sb->s_dev; - __entry->src_ino = src->i_ino; + __entry->src_ino = I_INO(src); __entry->src_lblk = soffset; __entry->len = len; - __entry->dest_ino = dest->i_ino; + __entry->dest_ino = I_INO(dest); __entry->dest_lblk = doffset; ), TP_printk("dev %d:%d fsbcount 0x%llx " @@ -4504,8 +4504,7 @@ TRACE_EVENT(xfs_iunlink_update_dinode, TP_fast_assign( __entry->dev = pag_mount(iup->pag)->m_super->s_dev; __entry->agno = pag_agno(iup->pag); - __entry->agino = - XFS_INO_TO_AGINO(iup->ip->i_mount, iup->ip->i_ino); + __entry->agino = XFS_INODE_TO_AGINO(iup->ip); __entry->old_ptr = old_ptr; __entry->new_ptr = iup->next_agino; ), @@ -4529,8 +4528,8 @@ TRACE_EVENT(xfs_iunlink_reload_next, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->prev_agino = ip->i_prev_unlinked; __entry->next_agino = ip->i_next_unlinked; ), @@ -4552,8 +4551,8 @@ TRACE_EVENT(xfs_inode_reload_unlinked_bucket, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); ), TP_printk("dev %d:%d agno 0x%x agino 0x%x bucket %u", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -4572,8 +4571,8 @@ DECLARE_EVENT_CLASS(xfs_ag_inode_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); ), TP_printk("dev %d:%d agno 0x%x agino 0x%x", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -4650,7 +4649,7 @@ DECLARE_EVENT_CLASS(xfs_inode_corrupt_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->flags = flags; ), TP_printk("dev %d:%d ino 0x%llx flags 0x%x", @@ -4769,10 +4768,8 @@ TRACE_EVENT(xfs_btree_commit_ifakeroot, TP_fast_assign( __entry->dev = cur->bc_mp->m_super->s_dev; __assign_str(name); - __entry->agno = XFS_INO_TO_AGNO(cur->bc_mp, - cur->bc_ino.ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(cur->bc_mp, - cur->bc_ino.ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(cur->bc_ino.ip); + __entry->agino = XFS_INODE_TO_AGINO(cur->bc_ino.ip); __entry->levels = cur->bc_ino.ifake->if_levels; __entry->blocks = cur->bc_ino.ifake->if_blocks; __entry->whichfork = cur->bc_ino.whichfork; @@ -5031,7 +5028,7 @@ DECLARE_EVENT_CLASS(xfs_das_state_class, ), TP_fast_assign( __entry->das = das; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); ), TP_printk("state change %s ino 0x%llx", __print_symbolic(__entry->das, XFS_DAS_STRINGS), @@ -5296,7 +5293,7 @@ DECLARE_EVENT_CLASS(xfs_exchrange_inode_class, TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; __entry->whichfile = whichfile; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->format = ip->i_df.if_format; __entry->nex = ip->i_df.if_nextents; __entry->fork_off = xfs_inode_fork_boff(ip); @@ -5349,10 +5346,10 @@ DECLARE_EVENT_CLASS(xfs_exchrange_class, ), TP_fast_assign( __entry->dev = VFS_I(ip1)->i_sb->s_dev; - __entry->ip1_ino = ip1->i_ino; + __entry->ip1_ino = I_INO(ip1); __entry->ip1_isize = VFS_I(ip1)->i_size; __entry->ip1_disize = ip1->i_disk_size; - __entry->ip2_ino = ip2->i_ino; + __entry->ip2_ino = I_INO(ip2); __entry->ip2_isize = VFS_I(ip2)->i_size; __entry->ip2_disize = ip2->i_disk_size; @@ -5408,7 +5405,7 @@ TRACE_EVENT(xfs_exchrange_freshness, struct inode *inode2 = VFS_I(ip2); __entry->dev = inode2->i_sb->s_dev; - __entry->ip2_ino = ip2->i_ino; + __entry->ip2_ino = I_INO(ip2); ts64 = inode_get_ctime(inode2); __entry->ip2_ctime = ts64.tv_sec; @@ -5480,8 +5477,8 @@ DECLARE_EVENT_CLASS(xfs_exchmaps_estimate_class, ), TP_fast_assign( __entry->dev = req->ip1->i_mount->m_super->s_dev; - __entry->ino1 = req->ip1->i_ino; - __entry->ino2 = req->ip2->i_ino; + __entry->ino1 = I_INO(req->ip1); + __entry->ino2 = I_INO(req->ip2); __entry->startoff1 = req->startoff1; __entry->startoff2 = req->startoff2; __entry->blockcount = req->blockcount; @@ -5532,8 +5529,8 @@ DECLARE_EVENT_CLASS(xfs_exchmaps_intent_class, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->ino1 = xmi->xmi_ip1->i_ino; - __entry->ino2 = xmi->xmi_ip2->i_ino; + __entry->ino1 = I_INO(xmi->xmi_ip1); + __entry->ino2 = I_INO(xmi->xmi_ip2); __entry->flags = xmi->xmi_flags; __entry->startoff1 = xmi->xmi_startoff1; __entry->startoff2 = xmi->xmi_startoff2; @@ -5628,8 +5625,8 @@ TRACE_EVENT(xfs_exchmaps_delta_nextents, int whichfork = xfs_exchmaps_reqfork(req); __entry->dev = req->ip1->i_mount->m_super->s_dev; - __entry->ino1 = req->ip1->i_ino; - __entry->ino2 = req->ip2->i_ino; + __entry->ino1 = I_INO(req->ip1); + __entry->ino2 = I_INO(req->ip2); __entry->nexts1 = xfs_ifork_ptr(req->ip1, whichfork)->if_nextents; __entry->nexts2 = xfs_ifork_ptr(req->ip2, whichfork)->if_nextents; __entry->d_nexts1 = d_nexts1; @@ -5659,7 +5656,7 @@ DECLARE_EVENT_CLASS(xfs_getparents_rec_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->firstu = context->firstu; __entry->reclen = pptr->gpr_reclen; __entry->bufsize = ppi->gp_bufsize; @@ -5703,7 +5700,7 @@ DECLARE_EVENT_CLASS(xfs_getparents_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->iflags = ppi->gp_iflags; __entry->oflags = ppi->gp_oflags; __entry->bufsize = ppi->gp_bufsize; @@ -5742,8 +5739,8 @@ DECLARE_EVENT_CLASS(xfs_metadir_update_class, ), TP_fast_assign( __entry->dev = upd->dp->i_mount->m_super->s_dev; - __entry->dp_ino = upd->dp->i_ino; - __entry->ino = upd->ip ? upd->ip->i_ino : NULLFSINO; + __entry->dp_ino = I_INO(upd->dp); + __entry->ino = upd->ip ? I_INO(upd->ip) : NULLFSINO; __assign_str(fname); ), TP_printk("dev %d:%d dp 0x%llx fname '%s' ino 0x%llx", @@ -5777,8 +5774,8 @@ DECLARE_EVENT_CLASS(xfs_metadir_update_error_class, ), TP_fast_assign( __entry->dev = upd->dp->i_mount->m_super->s_dev; - __entry->dp_ino = upd->dp->i_ino; - __entry->ino = upd->ip ? upd->ip->i_ino : NULLFSINO; + __entry->dp_ino = I_INO(upd->dp); + __entry->ino = upd->ip ? I_INO(upd->ip) : NULLFSINO; __entry->error = error; __assign_str(fname); ), @@ -5810,7 +5807,7 @@ DECLARE_EVENT_CLASS(xfs_metadir_class, ), TP_fast_assign( __entry->dev = VFS_I(dp)->i_sb->s_dev; - __entry->dp_ino = dp->i_ino; + __entry->dp_ino = I_INO(dp); __entry->ino = ino, __entry->ftype = name->type; __entry->namelen = name->len; @@ -6328,7 +6325,7 @@ TRACE_EVENT(xfs_healthmon_report_file_ioerror, TP_fast_assign( __entry->dev = hm->dev; __entry->type = p->type; - __entry->ino = XFS_I(p->inode)->i_ino; + __entry->ino = p->inode->i_ino; __entry->gen = p->inode->i_generation; __entry->pos = p->pos; __entry->len = p->len; @@ -6442,6 +6439,47 @@ TRACE_EVENT(xfs_verify_media_error, __entry->error) ); +TRACE_EVENT(xfs_bmap_replace_cow_mapping, + TP_PROTO(struct xfs_inode *ip, const struct xfs_bmbt_irec *got, + const struct xfs_bmbt_irec *rep), + TP_ARGS(ip, got, rep), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsblock_t, startblock) + __field(xfs_fileoff_t, startoff) + __field(xfs_filblks_t, blockcount) + __field(xfs_exntst_t, state) + __field(xfs_fileoff_t, new_startoff) + __field(xfs_fsblock_t, new_startblock) + __field(xfs_extlen_t, new_blockcount) + __field(xfs_exntst_t, new_state) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->startoff = got->br_startoff; + __entry->startblock = got->br_startblock; + __entry->blockcount = got->br_blockcount; + __entry->state = got->br_state; + __entry->new_startoff = rep->br_startoff; + __entry->new_startblock = rep->br_startblock; + __entry->new_blockcount = rep->br_blockcount; + __entry->new_state = rep->br_state; + ), + TP_printk("dev %d:%d ino 0x%llx startoff 0x%llx startblock 0x%llx fsbcount 0x%llx state 0x%x new_startoff 0x%llx new_startblock 0x%llx new_fsbcount 0x%x new_state 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->startoff, + __entry->startblock, + __entry->blockcount, + __entry->state, + __entry->new_startoff, + __entry->new_startblock, + __entry->new_blockcount, + __entry->new_state) +); + #endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index bcc470f56e46..7bfbd9f6f0df 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1164,7 +1164,7 @@ xfs_trans_reserve_more_inode( if (error) return error; - if (!XFS_IS_QUOTA_ON(mp) || xfs_is_quota_inode(&mp->m_sb, ip->i_ino)) + if (!XFS_IS_QUOTA_ON(mp) || xfs_is_quota_inode(&mp->m_sb, I_INO(ip))) return 0; if (tp->t_flags & XFS_TRANS_RESERVE) @@ -1199,10 +1199,21 @@ xfs_trans_alloc_icreate( { struct xfs_trans *tp; bool retried = false; + bool flushed = false; int error; retry: error = xfs_trans_alloc(mp, resv, dblocks, 0, 0, &tp); + if (error == -ENOSPC && !flushed) { + /* + * Flush all delalloc blocks to reclaim space from speculative + * preallocation. This is similar to the quota retry below + * but targets FS-wide ENOSPC. + */ + xfs_flush_inodes(mp); + flushed = true; + goto retry; + } if (error) return error; diff --git a/fs/xfs/xfs_trans_buf.c b/fs/xfs/xfs_trans_buf.c index 95db73a37e57..7e17b93fe9ad 100644 --- a/fs/xfs/xfs_trans_buf.c +++ b/fs/xfs/xfs_trans_buf.c @@ -234,7 +234,6 @@ xfs_trans_read_buf_map( const struct xfs_buf_ops *ops) { struct xfs_buf *bp = NULL; - struct xfs_buf_log_item *bip; int error; *bpp = NULL; @@ -251,9 +250,10 @@ xfs_trans_read_buf_map( if (bp) { ASSERT(xfs_buf_islocked(bp)); ASSERT(bp->b_transp == tp); - ASSERT(bp->b_log_item != NULL); ASSERT(!bp->b_error); ASSERT(bp->b_flags & XBF_DONE); + ASSERT(atomic_read(&bp->b_log_item->bli_refcount) > 0); + ASSERT(bp->b_ops); /* * We never locked this buf ourselves, so we shouldn't @@ -264,39 +264,8 @@ xfs_trans_read_buf_map( return -EIO; } - /* - * Check if the caller is trying to read a buffer that is - * already attached to the transaction yet has no buffer ops - * assigned. Ops are usually attached when the buffer is - * attached to the transaction, or by the read caller if - * special circumstances. That didn't happen, which is not - * how this is supposed to go. - * - * If the buffer passes verification we'll let this go, but if - * not we have to shut down. Let the transaction cleanup code - * release this buffer when it kills the tranaction. - */ - ASSERT(bp->b_ops != NULL); - error = xfs_buf_reverify(bp, ops); - if (error) { - xfs_buf_ioerror_alert(bp, __return_address); - - if (tp->t_flags & XFS_TRANS_DIRTY) - xfs_force_shutdown(tp->t_mountp, - SHUTDOWN_META_IO_ERROR); - - /* bad CRC means corrupted metadata */ - if (error == -EFSBADCRC) - error = -EFSCORRUPTED; - return error; - } - - bip = bp->b_log_item; - bip->bli_recur++; - - ASSERT(atomic_read(&bip->bli_refcount) > 0); - trace_xfs_trans_read_buf_recur(bip); - ASSERT(bp->b_ops != NULL || ops == NULL); + bp->b_log_item->bli_recur++; + trace_xfs_trans_read_buf_recur(bp->b_log_item); *bpp = bp; return 0; } diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index eaf9de6e07fd..1606c614f205 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -164,7 +164,7 @@ xfs_trans_mod_ino_dquot( if (xfs_hooks_switched_on(&xfs_dqtrx_hooks_switch)) { struct xfs_mod_ino_dqtrx_params p = { .tx_id = (uintptr_t)tp, - .ino = ip->i_ino, + .ino = I_INO(ip), .q_type = xfs_dquot_type(dqp), .q_id = dqp->q_id, .delta = delta @@ -247,7 +247,7 @@ xfs_trans_mod_dquot_byino( xfs_mount_t *mp = tp->t_mountp; if (!XFS_IS_QUOTA_ON(mp) || - xfs_is_quota_inode(&mp->m_sb, ip->i_ino) || + xfs_is_quota_inode(&mp->m_sb, I_INO(ip)) || xfs_is_metadir_inode(ip)) return; @@ -990,7 +990,7 @@ xfs_trans_reserve_quota_nblks( if (xfs_is_metadir_inode(ip)) return 0; - ASSERT(!xfs_is_quota_inode(&mp->m_sb, ip->i_ino)); + ASSERT(!xfs_is_quota_inode(&mp->m_sb, I_INO(ip))); xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); if (force) diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 544213067d59..1efe6c8139b2 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -104,7 +104,7 @@ xfs_attr_change( args->op_flags |= XFS_DA_OP_LOGGED; } - args->owner = args->dp->i_ino; + args->owner = I_INO(args->dp); args->geo = mp->m_attr_geo; args->whichfork = XFS_ATTR_FORK; xfs_attr_sethash(args); diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c index a851b98143c0..7d13fa7ab30a 100644 --- a/fs/xfs/xfs_zone_alloc.c +++ b/fs/xfs/xfs_zone_alloc.c @@ -21,6 +21,7 @@ #include "xfs_rtbitmap.h" #include "xfs_rtrmap_btree.h" #include "xfs_zone_alloc.h" +#include "xfs_sysfs.h" #include "xfs_zone_priv.h" #include "xfs_zones.h" #include "xfs_trace.h" @@ -944,6 +945,14 @@ xfs_zone_rgbno_is_valid( rtg_rgno(rtg), XFS_RTG_FREE); } +void +xfs_zone_mark_free( + struct xfs_rtgroup *rtg) +{ + xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); + atomic_inc(&rtg_mount(rtg)->m_zone_info->zi_nr_free_zones); +} + static void xfs_free_open_zones( struct xfs_zone_info *zi) @@ -1082,8 +1091,7 @@ xfs_init_zone( if (write_pointer == 0) { /* zone is empty */ - atomic_inc(&zi->zi_nr_free_zones); - xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); + xfs_zone_mark_free(rtg); iz->available += rtg_blocks(rtg); } else if (write_pointer < rtg_blocks(rtg)) { /* zone is open */ @@ -1170,7 +1178,7 @@ xfs_calc_open_zones( if (bdev_open_zones && bdev_open_zones < mp->m_max_open_zones) { mp->m_max_open_zones = bdev_open_zones; - xfs_info(mp, "limiting open zones to %u due to hardware limit.\n", + xfs_info(mp, "limiting open zones to %u due to hardware limit.", bdev_open_zones); } @@ -1217,7 +1225,7 @@ xfs_alloc_zone_info( return zi; out_free_bitmaps: - while (--i > 0) + while (--i >= 0) kvfree(zi->zi_used_bucket_bitmap[i]); kfree(zi); return NULL; @@ -1413,11 +1421,17 @@ xfs_mount_zones( if (error) goto out_free_zone_info; + error = xfs_zoned_sysfs_init(mp); + if (error) + goto out_zone_gc_unmount; + xfs_info(mp, "%u zones of %u blocks (%u max open zones)", mp->m_sb.sb_rgcount, iz.zone_capacity, mp->m_max_open_zones); trace_xfs_zones_mount(mp); return 0; +out_zone_gc_unmount: + xfs_zone_gc_unmount(mp); out_free_zone_info: xfs_free_zone_info(mp->m_zone_info); return error; @@ -1427,6 +1441,7 @@ void xfs_unmount_zones( struct xfs_mount *mp) { + xfs_zoned_sysfs_del(mp); xfs_zone_gc_unmount(mp); xfs_free_zone_info(mp->m_zone_info); } diff --git a/fs/xfs/xfs_zone_alloc.h b/fs/xfs/xfs_zone_alloc.h index 8b2ef98c81ef..abf8358bb266 100644 --- a/fs/xfs/xfs_zone_alloc.h +++ b/fs/xfs/xfs_zone_alloc.h @@ -42,6 +42,7 @@ void xfs_zoned_wake_all(struct xfs_mount *mp); bool xfs_zone_rgbno_is_valid(struct xfs_rtgroup *rtg, xfs_rgnumber_t rgbno); void xfs_mark_rtg_boundary(struct iomap_ioend *ioend); +void xfs_zone_mark_free(struct xfs_rtgroup *rtg); uint64_t xfs_zoned_default_resblks(struct xfs_mount *mp, enum xfs_free_counter ctr); void xfs_zoned_show_stats(struct seq_file *m, struct xfs_mount *mp); diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c index fedcc47048af..f76a09130852 100644 --- a/fs/xfs/xfs_zone_gc.c +++ b/fs/xfs/xfs_zone_gc.c @@ -400,7 +400,7 @@ retry: /* * If the inode was already deleted, skip over it. */ - if (error == -ENOENT) { + if (error == -ENOENT || error == -EINVAL) { iter->rec_idx++; goto retry; } @@ -924,9 +924,7 @@ xfs_zone_gc_finish_reset( goto out; } - xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); - atomic_inc(&zi->zi_nr_free_zones); - + xfs_zone_mark_free(rtg); xfs_zoned_add_available(mp, rtg_blocks(rtg)); wake_up_all(&zi->zi_zone_wait); @@ -1221,7 +1219,7 @@ out_put_oz: if (data->oz) xfs_open_zone_put(data->oz); out_free_gc_data: - kfree(data); + xfs_zone_gc_data_free(data); return error; } diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 9b646cb5335d..ff43d6d1ea30 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -610,10 +610,14 @@ static long zonefs_fname_to_fno(const struct qstr *fname) return c - '0'; for (i = 0, rname = name + len - 1; i < len; i++, rname--) { + long digit; + c = *rname; if (!isdigit(c)) return -ENOENT; - fno += (c - '0') * shift; + digit = (c - '0') * shift; + if (check_add_overflow(fno, digit, &fno)) + return -ENOENT; shift *= 10; } |