diff options
250 files changed, 9646 insertions, 3796 deletions
diff --git a/Documentation/ABI/testing/procfs-nfsd-reply_cache_stats b/Documentation/ABI/testing/procfs-nfsd-reply_cache_stats new file mode 100644 index 000000000000..57ed5f8e6597 --- /dev/null +++ b/Documentation/ABI/testing/procfs-nfsd-reply_cache_stats @@ -0,0 +1,38 @@ +What: /proc/fs/nfsd/reply_cache_stats +Date: March 2013 +KernelVersion: 3.10 +Contact: linux-nfs@vger.kernel.org +Description: + Provides statistics for the NFS server duplicate reply + cache (DRC). The file contains one labeled field per + line. Each line has the form "field name:" followed by + whitespace and a decimal value. + + Fields: + + ======================= ====== ========================== + max entries u32 Upper bound on cache size + num entries u32 Current entry count + hash buckets u32 Number of hash buckets + mem usage s64 Bytes consumed by the DRC + cache hits s64 Requests answered from cache + cache misses s64 Requests not found in cache + not cached s64 Idempotent requests that + bypass the cache + payload misses s64 XID matched but request + checksum did not + longest chain len u32 Longest hash chain observed + cachesize at longest u32 Cache size when longest + chain was recorded + ======================= ====== ========================== + + Counter fields (cache hits, cache misses, not cached, + payload misses, mem usage) are maintained with per-cpu + counters and may briefly show stale values under + concurrent load. There is no way to reset these + counters; consumers should compute rates by sampling + over time. + + New fields may be appended in future kernels. Parsers + should match on field name, not line position. +Users: nfs-utils (https://git.linux-nfs.org/?p=steved/nfs-utils.git) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 9b923a14ee9c..4a12805a50ba 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7470,19 +7470,13 @@ Kernel parameters sunrpc.pool_mode= [NFS] - Control how the NFS server code allocates CPUs to - service thread pools. Depending on how many NICs - you have and where their interrupts are bound, this - option will affect which CPUs will do NFS serving. - Note: this parameter cannot be changed while the - NFS server is running. - - auto the server chooses an appropriate mode - automatically using heuristics - global a single global pool contains all CPUs - percpu one pool for each CPU - pernode one pool for each NUMA node (equivalent - to global on non-NUMA machines) + Deprecated. The NFS server now always uses one + service thread pool per NUMA node (equivalent to a + single global pool on non-NUMA machines). All of + the previously accepted values (auto, global, + percpu, pernode) are still accepted for backward + compatibility but are ignored: the mode is always + pernode, and reads always return "pernode". sunrpc.tcp_slot_table_entries= sunrpc.udp_slot_table_entries= diff --git a/Documentation/filesystems/erofs.rst b/Documentation/filesystems/erofs.rst index 4230884fb359..d301d9ac946a 100644 --- a/Documentation/filesystems/erofs.rst +++ b/Documentation/filesystems/erofs.rst @@ -137,8 +137,25 @@ fsoffset=%llu Specify block-aligned filesystem offset for the primary d inode_share Enable inode page sharing for this filesystem. Inodes with identical content within the same domain ID can share the page cache. +source=%s (For file-backed mounts) Specify the backing image as a path + or as an already-opened file descriptor. =================== ========================================================= +File-backed mounts +================== + +When CONFIG_EROFS_FS_BACKED_BY_FILE is enabled, EROFS file-backed images +can be mounted directly without a loopback block device. The backing file +can be given either as a path, or as an already-opened file descriptor. + +When a file descriptor is used, the kernel resolves its path and records it +so that /proc/mounts and similar interfaces can still report the mount +source. + +Only regular files are accepted as backing files; to mount an image that +resides on a block device, use the traditional block device mount path +instead. + Sysfs Entries ============= diff --git a/Documentation/filesystems/ext4/group_descr.rst b/Documentation/filesystems/ext4/group_descr.rst index 392ec44f8fb0..9a0c2d92c2ef 100644 --- a/Documentation/filesystems/ext4/group_descr.rst +++ b/Documentation/filesystems/ext4/group_descr.rst @@ -20,11 +20,10 @@ group of the flex group. If the meta_bg feature flag is set, then several block groups are grouped together into a meta group. Note that in the meta_bg case, -however, the first and last two block groups within the larger meta -group contain only group descriptors for the groups inside the meta -group. - -flex_bg and meta_bg do not appear to be mutually exclusive features. +however, the superblock and a single block group descriptor block is +placed at the beginning of the first, second, and last block groups in a +meta-block group. The flex_bg and meta_bg features are not mutually +exclusive. In ext2, ext3, and ext4 (when the 64bit feature is not enabled), the block group descriptor was only 32 bytes long and therefore ends at diff --git a/Documentation/netlink/specs/nfsd.yaml b/Documentation/netlink/specs/nfsd.yaml index 8f36fadd68f7..642268819c6f 100644 --- a/Documentation/netlink/specs/nfsd.yaml +++ b/Documentation/netlink/specs/nfsd.yaml @@ -330,6 +330,91 @@ attribute-sets: of which client holds the state. Intended for use after all clients have been unexported from a given path, enabling the underlying filesystem to be unmounted. + - + name: server-proc-entry + attributes: + - + name: op + type: u32 + - + name: count + type: u64 + - + name: pad + type: pad + - + name: server-stats + attributes: + - + name: rc-hits + type: u64 + - + name: rc-misses + type: u64 + - + name: rc-nocache + type: u64 + - + name: pad + type: pad + - + name: fh-stale + type: u64 + - + name: io-read + type: u64 + - + name: io-write + type: u64 + - + name: netcnt + type: u32 + - + name: netudpcnt + type: u32 + - + name: nettcpcnt + type: u32 + - + name: nettcpconn + type: u32 + - + name: rpccnt + type: u32 + - + name: rpcbadfmt + type: u32 + - + name: rpcbadauth + type: u32 + - + name: rpcbadclnt + type: u32 + - + name: proc2-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc3-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc4-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc4ops-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc4cb-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true operations: list: @@ -516,6 +601,32 @@ operations: request: attributes: - path + - + name: server-stats-get + doc: dump NFS server statistics + attribute-set: server-stats + dump: + reply: + attributes: + - rc-hits + - rc-misses + - rc-nocache + - fh-stale + - io-read + - io-write + - netcnt + - netudpcnt + - nettcpcnt + - nettcpconn + - rpccnt + - rpcbadfmt + - rpcbadauth + - rpcbadclnt + - proc2-ops + - proc3-ops + - proc4-ops + - proc4ops-ops + - proc4cb-ops mcast-groups: list: diff --git a/Documentation/sunrpc/xdr/nfs4_1.x b/Documentation/sunrpc/xdr/nfs4_1.x index 5b45547b2ebc..e66f396ae659 100644 --- a/Documentation/sunrpc/xdr/nfs4_1.x +++ b/Documentation/sunrpc/xdr/nfs4_1.x @@ -45,19 +45,162 @@ pragma header nfs4; /* * Basic typedefs for RFC 1832 data type definitions */ -typedef hyper int64_t; -typedef unsigned int uint32_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef hyper int64_t; +typedef unsigned hyper uint64_t; + +const NFS4_VERIFIER_SIZE = 8; +const NFS4_FHSIZE = 128; + +enum nfsstat4 { + NFS4_OK = 0, /* everything is okay */ + NFS4ERR_PERM = 1, /* caller not privileged */ + NFS4ERR_NOENT = 2, /* no such file/directory */ + NFS4ERR_IO = 5, /* hard I/O error */ + NFS4ERR_NXIO = 6, /* no such device */ + NFS4ERR_ACCESS = 13, /* access denied */ + NFS4ERR_EXIST = 17, /* file already exists */ + NFS4ERR_XDEV = 18, /* different filesystems */ + + /* + * Please do not allocate value 19; it was used in NFSv3 + * and we do not want a value in NFSv3 to have a different + * meaning in NFSv4.x. + */ + + NFS4ERR_NOTDIR = 20, /* should be a directory */ + NFS4ERR_ISDIR = 21, /* should not be directory */ + NFS4ERR_INVAL = 22, /* invalid argument */ + NFS4ERR_FBIG = 27, /* file exceeds server max */ + NFS4ERR_NOSPC = 28, /* no space on filesystem */ + NFS4ERR_ROFS = 30, /* read-only filesystem */ + NFS4ERR_MLINK = 31, /* too many hard links */ + NFS4ERR_NAMETOOLONG = 63, /* name exceeds server max */ + NFS4ERR_NOTEMPTY = 66, /* directory not empty */ + NFS4ERR_DQUOT = 69, /* hard quota limit reached*/ + NFS4ERR_STALE = 70, /* file no longer exists */ + NFS4ERR_BADHANDLE = 10001,/* Illegal filehandle */ + NFS4ERR_BAD_COOKIE = 10003,/* READDIR cookie is stale */ + NFS4ERR_NOTSUPP = 10004,/* operation not supported */ + NFS4ERR_TOOSMALL = 10005,/* response limit exceeded */ + NFS4ERR_SERVERFAULT = 10006,/* undefined server error */ + NFS4ERR_BADTYPE = 10007,/* type invalid for CREATE */ + NFS4ERR_DELAY = 10008,/* file "busy" - retry */ + NFS4ERR_SAME = 10009,/* nverify says attrs same */ + NFS4ERR_DENIED = 10010,/* lock unavailable */ + NFS4ERR_EXPIRED = 10011,/* lock lease expired */ + NFS4ERR_LOCKED = 10012,/* I/O failed due to lock */ + NFS4ERR_GRACE = 10013,/* in grace period */ + NFS4ERR_FHEXPIRED = 10014,/* filehandle expired */ + NFS4ERR_SHARE_DENIED = 10015,/* share reserve denied */ + NFS4ERR_WRONGSEC = 10016,/* wrong security flavor */ + NFS4ERR_CLID_INUSE = 10017,/* clientid in use */ + + /* NFS4ERR_RESOURCE is not a valid error in NFSv4.1 */ + NFS4ERR_RESOURCE = 10018,/* resource exhaustion */ + + NFS4ERR_MOVED = 10019,/* filesystem relocated */ + NFS4ERR_NOFILEHANDLE = 10020,/* current FH is not set */ + NFS4ERR_MINOR_VERS_MISMATCH= 10021,/* minor vers not supp */ + NFS4ERR_STALE_CLIENTID = 10022,/* server has rebooted */ + NFS4ERR_STALE_STATEID = 10023,/* server has rebooted */ + NFS4ERR_OLD_STATEID = 10024,/* state is out of sync */ + NFS4ERR_BAD_STATEID = 10025,/* incorrect stateid */ + NFS4ERR_BAD_SEQID = 10026,/* request is out of seq. */ + NFS4ERR_NOT_SAME = 10027,/* verify - attrs not same */ + NFS4ERR_LOCK_RANGE = 10028,/* overlapping lock range */ + NFS4ERR_SYMLINK = 10029,/* should be file/directory*/ + NFS4ERR_RESTOREFH = 10030,/* no saved filehandle */ + NFS4ERR_LEASE_MOVED = 10031,/* some filesystem moved */ + NFS4ERR_ATTRNOTSUPP = 10032,/* recommended attr not sup*/ + NFS4ERR_NO_GRACE = 10033,/* reclaim outside of grace*/ + NFS4ERR_RECLAIM_BAD = 10034,/* reclaim error at server */ + NFS4ERR_RECLAIM_CONFLICT= 10035,/* conflict on reclaim */ + NFS4ERR_BADXDR = 10036,/* XDR decode failed */ + NFS4ERR_LOCKS_HELD = 10037,/* file locks held at CLOSE*/ + NFS4ERR_OPENMODE = 10038,/* conflict in OPEN and I/O*/ + NFS4ERR_BADOWNER = 10039,/* owner translation bad */ + NFS4ERR_BADCHAR = 10040,/* utf-8 char not supported*/ + NFS4ERR_BADNAME = 10041,/* name not supported */ + NFS4ERR_BAD_RANGE = 10042,/* lock range not supported*/ + NFS4ERR_LOCK_NOTSUPP = 10043,/* no atomic up/downgrade */ + NFS4ERR_OP_ILLEGAL = 10044,/* undefined operation */ + NFS4ERR_DEADLOCK = 10045,/* file locking deadlock */ + NFS4ERR_FILE_OPEN = 10046,/* open file blocks op. */ + NFS4ERR_ADMIN_REVOKED = 10047,/* lockowner state revoked */ + NFS4ERR_CB_PATH_DOWN = 10048,/* callback path down */ + + /* NFSv4.1 errors start here. */ + + NFS4ERR_BADIOMODE = 10049, + NFS4ERR_BADLAYOUT = 10050, + NFS4ERR_BAD_SESSION_DIGEST = 10051, + NFS4ERR_BADSESSION = 10052, + NFS4ERR_BADSLOT = 10053, + NFS4ERR_COMPLETE_ALREADY = 10054, + NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, + NFS4ERR_DELEG_ALREADY_WANTED = 10056, + NFS4ERR_BACK_CHAN_BUSY = 10057,/*backchan reqs outstanding*/ + NFS4ERR_LAYOUTTRYLATER = 10058, + NFS4ERR_LAYOUTUNAVAILABLE = 10059, + NFS4ERR_NOMATCHING_LAYOUT = 10060, + NFS4ERR_RECALLCONFLICT = 10061, + NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, + NFS4ERR_SEQ_MISORDERED = 10063,/* unexpected seq.ID in req*/ + NFS4ERR_SEQUENCE_POS = 10064,/* [CB_]SEQ. op not 1st op */ + NFS4ERR_REQ_TOO_BIG = 10065,/* request too big */ + NFS4ERR_REP_TOO_BIG = 10066,/* reply too big */ + NFS4ERR_REP_TOO_BIG_TO_CACHE =10067,/* rep. not all cached*/ + NFS4ERR_RETRY_UNCACHED_REP =10068,/* retry & rep. uncached*/ + NFS4ERR_UNSAFE_COMPOUND =10069,/* retry/recovery too hard */ + NFS4ERR_TOO_MANY_OPS = 10070,/*too many ops in [CB_]COMP*/ + NFS4ERR_OP_NOT_IN_SESSION =10071,/* op needs [CB_]SEQ. op */ + NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp. */ + /* Error 10073 is unused. */ + NFS4ERR_CLIENTID_BUSY = 10074,/* clientid has state */ + NFS4ERR_PNFS_IO_HOLE = 10075,/* IO to _SPARSE file hole */ + NFS4ERR_SEQ_FALSE_RETRY= 10076,/* Retry != original req. */ + NFS4ERR_BAD_HIGH_SLOT = 10077,/* req has bad highest_slot*/ + NFS4ERR_DEADSESSION = 10078,/*new req sent to dead sess*/ + NFS4ERR_ENCR_ALG_UNSUPP= 10079,/* encr alg. not supp. */ + NFS4ERR_PNFS_NO_LAYOUT = 10080,/* I/O without a layout */ + NFS4ERR_NOT_ONLY_OP = 10081,/* addl ops not allowed */ + NFS4ERR_WRONG_CRED = 10082,/* op done by wrong cred */ + NFS4ERR_WRONG_TYPE = 10083,/* op on wrong type object */ + NFS4ERR_DIRDELEG_UNAVAIL=10084,/* delegation not avail. */ + NFS4ERR_REJECT_DELEG = 10085,/* cb rejected delegation */ + NFS4ERR_RETURNCONFLICT = 10086,/* layout get before return*/ + NFS4ERR_DELEG_REVOKED = 10087, /* deleg./layout revoked */ + NFS4ERR_PARTNER_NOTSUPP = 10088, + NFS4ERR_PARTNER_NO_AUTH = 10089, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096 +}; /* * Basic data types */ +typedef opaque attrlist4<>; typedef uint32_t bitmap4<>; +typedef opaque verifier4[NFS4_VERIFIER_SIZE]; +typedef uint64_t nfs_cookie4; +typedef opaque nfs_fh4<NFS4_FHSIZE>; typedef opaque utf8string<>; typedef utf8string utf8str_cis; typedef utf8string utf8str_cs; typedef utf8string utf8str_mixed; +typedef utf8str_cs component4; +typedef utf8str_cs linktext4; +typedef component4 pathname4<>; + /* * Timeval */ @@ -66,6 +209,21 @@ struct nfstime4 { uint32_t nseconds; }; +/* + * File attribute container + */ +struct fattr4 { + bitmap4 attrmask; + attrlist4 attr_vals; +}; + +/* + * Stateid + */ +struct stateid4 { + uint32_t seqid; + opaque other[12]; +}; /* * The following content was extracted from draft-ietf-nfsv4-delstid @@ -245,3 +403,99 @@ const FATTR4_ACL_TRUEFORM = 89; const FATTR4_ACL_TRUEFORM_SCOPE = 90; const FATTR4_POSIX_DEFAULT_ACL = 91; const FATTR4_POSIX_ACCESS_ACL = 92; + +/* + * Directory notification types. + */ +enum notify_type4 { + NOTIFY4_CHANGE_CHILD_ATTRS = 0, + NOTIFY4_CHANGE_DIR_ATTRS = 1, + NOTIFY4_REMOVE_ENTRY = 2, + NOTIFY4_ADD_ENTRY = 3, + NOTIFY4_RENAME_ENTRY = 4, + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5, + /* Proposed in RFC8881bis */ + NOTIFY4_GFLAG_EXTEND = 6, + NOTIFY4_AUFLAG_VALID = 7, + NOTIFY4_AUFLAG_USER = 8, + NOTIFY4_AUFLAG_GROUP = 9, + NOTIFY4_AUFLAG_OTHER = 10, + NOTIFY4_CHANGE_AUTH = 11, + NOTIFY4_CFLAG_ORDER = 12, + NOTIFY4_AUFLAG_GANOW = 13, + NOTIFY4_AUFLAG_GALATER = 14, + NOTIFY4_CHANGE_GA = 15, + NOTIFY4_CHANGE_AMASK = 16 +}; + +/* Changed entry information. */ +struct notify_entry4 { + component4 ne_file; + fattr4 ne_attrs; +}; + +/* Previous entry information */ +struct prev_entry4 { + notify_entry4 pe_prev_entry; + /* what READDIR returned for this entry */ + nfs_cookie4 pe_prev_entry_cookie; +}; + +struct notify_remove4 { + notify_entry4 nrm_old_entry; + nfs_cookie4 nrm_old_entry_cookie; +}; +pragma public notify_remove4; + +struct notify_add4 { + /* + * Information on object + * possibly renamed over. + */ + notify_remove4 nad_old_entry<1>; + notify_entry4 nad_new_entry; + /* what READDIR would have returned for this entry */ + nfs_cookie4 nad_new_entry_cookie<1>; + prev_entry4 nad_prev_entry<1>; + bool nad_last_entry; +}; +pragma public notify_add4; + +struct notify_attr4 { + notify_entry4 na_changed_entry; +}; +pragma public notify_attr4; + +struct notify_rename4 { + notify_remove4 nrn_old_entry; + notify_add4 nrn_new_entry; +}; +pragma public notify_rename4; + +struct notify_verifier4 { + verifier4 nv_old_cookieverf; + verifier4 nv_new_cookieverf; +}; + +/* + * Objects of type notify_<>4 and + * notify_device_<>4 are encoded in this. + */ +typedef opaque notifylist4<>; + +struct notify4 { + /* composed from notify_type4 or notify_deviceid_type4 */ + bitmap4 notify_mask; + notifylist4 notify_vals; +}; + +struct CB_NOTIFY4args { + stateid4 cna_stateid; + nfs_fh4 cna_fh; + notify4 cna_changes<>; +}; +pragma public CB_NOTIFY4args; + +struct CB_NOTIFY4res { + nfsstat4 cnr_status; +}; diff --git a/MAINTAINERS b/MAINTAINERS index 64dc2a2869ab..2f7cd9accca8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9915,7 +9915,6 @@ F: net/core/failover.c FANOTIFY M: Jan Kara <jack@suse.cz> R: Amir Goldstein <amir73il@gmail.com> -R: Matthew Bobrowski <repnop@google.com> L: linux-fsdevel@vger.kernel.org S: Maintained F: fs/notify/fanotify/ diff --git a/arch/loongarch/configs/loongson32_defconfig b/arch/loongarch/configs/loongson32_defconfig index 6bf2867dbdc6..e76ea9b62974 100644 --- a/arch/loongarch/configs/loongson32_defconfig +++ b/arch/loongarch/configs/loongson32_defconfig @@ -973,7 +973,6 @@ CONFIG_FS_VERITY=y CONFIG_FANOTIFY=y CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y CONFIG_QUOTA=y -# CONFIG_PRINT_QUOTA_WARNING is not set CONFIG_QFMT_V1=m CONFIG_QFMT_V2=m CONFIG_AUTOFS_FS=y diff --git a/arch/loongarch/configs/loongson64_defconfig b/arch/loongarch/configs/loongson64_defconfig index def104c9d405..976340d4d514 100644 --- a/arch/loongarch/configs/loongson64_defconfig +++ b/arch/loongarch/configs/loongson64_defconfig @@ -1004,7 +1004,6 @@ CONFIG_FS_VERITY=y CONFIG_FANOTIFY=y CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y CONFIG_QUOTA=y -# CONFIG_PRINT_QUOTA_WARNING is not set CONFIG_QFMT_V1=m CONFIG_QFMT_V2=m CONFIG_AUTOFS_FS=y diff --git a/arch/mips/configs/bigsur_defconfig b/arch/mips/configs/bigsur_defconfig index 611dc0dd392d..d2b18d5033be 100644 --- a/arch/mips/configs/bigsur_defconfig +++ b/arch/mips/configs/bigsur_defconfig @@ -140,7 +140,6 @@ CONFIG_EXT4_FS_SECURITY=y CONFIG_EXT4_FS=y CONFIG_QUOTA=y CONFIG_QUOTA_NETLINK_INTERFACE=y -# CONFIG_PRINT_QUOTA_WARNING is not set CONFIG_QFMT_V2=m CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m diff --git a/arch/mips/configs/ip22_defconfig b/arch/mips/configs/ip22_defconfig index 009e66299b69..09a77e5b249c 100644 --- a/arch/mips/configs/ip22_defconfig +++ b/arch/mips/configs/ip22_defconfig @@ -238,7 +238,6 @@ CONFIG_XFS_FS=m CONFIG_XFS_QUOTA=y CONFIG_QUOTA=y CONFIG_QUOTA_NETLINK_INTERFACE=y -# CONFIG_PRINT_QUOTA_WARNING is not set CONFIG_QFMT_V2=m CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m diff --git a/arch/mips/configs/loongson2k_defconfig b/arch/mips/configs/loongson2k_defconfig index ca534a6b66de..8fd863f4f7b0 100644 --- a/arch/mips/configs/loongson2k_defconfig +++ b/arch/mips/configs/loongson2k_defconfig @@ -303,7 +303,6 @@ CONFIG_XFS_FS=y CONFIG_XFS_QUOTA=y CONFIG_XFS_POSIX_ACL=y CONFIG_QUOTA=y -# CONFIG_PRINT_QUOTA_WARNING is not set CONFIG_AUTOFS_FS=y CONFIG_FUSE_FS=m CONFIG_ISO9660_FS=m diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h index 0de9df759c99..4133c790e71a 100644 --- a/arch/x86/include/asm/elf.h +++ b/arch/x86/include/asm/elf.h @@ -322,7 +322,7 @@ extern unsigned long get_sigframe_size(void); #else /* CONFIG_X86_32 */ -/* 1GB for 64bit, 8MB for 32bit */ +/* 16GB for 64bit, 8MB for 32bit */ #define __STACK_RND_MASK(is32bit) ((is32bit) ? 0x7ff : 0x3fffff) #define STACK_RND_MASK __STACK_RND_MASK(mmap_is_ia32()) diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h index 81e4af49e39c..607a53a91f58 100644 --- a/drivers/dax/dax-private.h +++ b/drivers/dax/dax-private.h @@ -69,7 +69,6 @@ struct dev_dax_range { * data while the device is activated in the driver. * @region: parent region * @dax_dev: core dax functionality - * @virt_addr: kva from memremap; used by fsdev_dax * @cached_size: size of daxdev cached by fsdev_dax * @align: alignment of this instance * @target_node: effective numa node if dev_dax memory range is onlined @@ -85,7 +84,6 @@ struct dev_dax_range { struct dev_dax { struct dax_region *region; struct dax_device *dax_dev; - void *virt_addr; u64 cached_size; unsigned int align; int target_node; diff --git a/drivers/dax/fsdev.c b/drivers/dax/fsdev.c index 188b2526bee4..598604bf5ac5 100644 --- a/drivers/dax/fsdev.c +++ b/drivers/dax/fsdev.c @@ -45,15 +45,13 @@ static void fsdev_write_dax(void *addr, struct page *page, } static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, - long nr_pages, enum dax_access_mode mode, void **kaddr, - unsigned long *pfn) + long nr_pages, enum dax_access_mode mode, void **kaddr, + unsigned long *pfn) { struct dev_dax *dev_dax = dax_get_private(dax_dev); size_t size = nr_pages << PAGE_SHIFT; size_t offset = pgoff << PAGE_SHIFT; - void *virt_addr = dev_dax->virt_addr + offset; phys_addr_t phys; - unsigned long local_pfn; phys = dax_pgoff_to_phys(dev_dax, pgoff, size); if (phys == -1) { @@ -63,11 +61,10 @@ static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, } if (kaddr) - *kaddr = virt_addr; + *kaddr = __va(phys); - local_pfn = PHYS_PFN(phys); if (pfn) - *pfn = local_pfn; + *pfn = PHYS_PFN(phys); /* * Use cached_size which was computed at probe time. The size cannot @@ -83,7 +80,8 @@ static int fsdev_dax_zero_page_range(struct dax_device *dax_dev, long rc; WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__); - rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, &kaddr, NULL); + rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, + &kaddr, NULL); if (rc < 0) return rc; fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE); @@ -91,15 +89,15 @@ static int fsdev_dax_zero_page_range(struct dax_device *dax_dev, } static long fsdev_dax_direct_access(struct dax_device *dax_dev, - pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, - void **kaddr, unsigned long *pfn) + pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, + void **kaddr, unsigned long *pfn) { return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode, kaddr, pfn); } -static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) +static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, + pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) { return _copy_from_iter_flushcache(addr, bytes, i); } @@ -127,6 +125,23 @@ static void fsdev_clear_ops(void *data) dax_set_ops(dev_dax->dax_dev, NULL); } +static void fsdev_clear_pgmap_ops(void *data) +{ + struct dev_pagemap *pgmap = data; + + /* + * fsdev installs pgmap->ops and ->owner at probe. For a static device + * the pgmap is shared and long-lived (owned by the dax bus), so + * leaving fsdev's ops behind on unbind would let a later + * memory_failure -- after rebind to another driver, or after this + * module is unloaded -- dispatch through a stale or freed + * ->memory_failure handler. Clear them so the pgmap carries no fsdev + * state once we are unbound. + */ + pgmap->ops = NULL; + pgmap->owner = NULL; +} + /* * Page map operations for FS-DAX mode * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c @@ -135,11 +150,26 @@ static void fsdev_clear_ops(void *data) * The core mm code in free_zone_device_folio() handles the wake_up_var() * directly for this memory type. */ +static u64 fsdev_pfn_to_offset(struct dev_dax *dev_dax, unsigned long pfn) +{ + phys_addr_t phys = PFN_PHYS(pfn); + u64 offset = 0; + + for (int i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + + if (phys >= range->start && phys <= range->end) + return offset + (phys - range->start); + offset += range_len(range); + } + return -1ULL; +} + static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap, unsigned long pfn, unsigned long nr_pages, int mf_flags) { struct dev_dax *dev_dax = pgmap->owner; - u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start; + u64 offset = fsdev_pfn_to_offset(dev_dax, pfn); u64 len = nr_pages << PAGE_SHIFT; return dax_holder_notify_failure(dev_dax->dax_dev, offset, @@ -204,46 +234,62 @@ static const struct file_operations fsdev_fops = { .release = fsdev_release, }; -static int fsdev_dax_probe(struct dev_dax *dev_dax) +/* + * Acquire the dev_pagemap for probe: the static (pre-populated) one if + * present, or a devm-allocated one for the dynamic case. Note that + * dev_dax->pgmap is not set here; fsdev_dax_probe() sets it only once + * probe succeeds, so a failed probe never leaves a dangling pointer + * to a devres-freed pgmap. + */ +static struct dev_pagemap *fsdev_acquire_pgmap(struct dev_dax *dev_dax) { - struct dax_device *dax_dev = dev_dax->dax_dev; struct device *dev = &dev_dax->dev; struct dev_pagemap *pgmap; - struct inode *inode; - u64 data_offset = 0; - struct cdev *cdev; - void *addr; - int rc, i; + size_t pgmap_size; if (static_dev_dax(dev_dax)) { if (dev_dax->nr_range > 1) { - dev_warn(dev, "static pgmap / multi-range device conflict\n"); - return -EINVAL; + dev_warn(dev, + "static pgmap / multi-range device conflict\n"); + return ERR_PTR(-EINVAL); } pgmap = dev_dax->pgmap; - } else { - size_t pgmap_size; + pgmap->vmemmap_shift = 0; + return pgmap; + } - if (dev_dax->pgmap) { - dev_warn(dev, "dynamic-dax with pre-populated page map\n"); - return -EINVAL; - } + if (dev_dax->pgmap) { + dev_warn(dev, "dynamic-dax with pre-populated page map\n"); + return ERR_PTR(-EINVAL); + } - pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); - pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); - if (!pgmap) - return -ENOMEM; + pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); + pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); + if (!pgmap) + return ERR_PTR(-ENOMEM); - pgmap->nr_range = dev_dax->nr_range; - dev_dax->pgmap = pgmap; + pgmap->nr_range = dev_dax->nr_range; + for (int i = 0; i < dev_dax->nr_range; i++) + pgmap->ranges[i] = dev_dax->ranges[i].range; - for (i = 0; i < dev_dax->nr_range; i++) { - struct range *range = &dev_dax->ranges[i].range; + return pgmap; +} - pgmap->ranges[i] = *range; - } - } +static int fsdev_dax_probe(struct dev_dax *dev_dax) +{ + struct dax_device *dax_dev = dev_dax->dax_dev; + struct device *dev = &dev_dax->dev; + struct dev_pagemap *pgmap; + struct inode *inode; + u64 data_offset = 0; + struct cdev *cdev; + void *addr; + int rc, i; + + pgmap = fsdev_acquire_pgmap(dev_dax); + if (IS_ERR(pgmap)) + return PTR_ERR(pgmap); for (i = 0; i < dev_dax->nr_range; i++) { struct range *range = &dev_dax->ranges[i].range; @@ -275,6 +321,11 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) if (IS_ERR(addr)) return PTR_ERR(addr); + /* Drop fsdev's pgmap->ops/owner on unbind so no stale ops survive. */ + rc = devm_add_action_or_reset(dev, fsdev_clear_pgmap_ops, pgmap); + if (rc) + return rc; + /* * Clear any stale compound folio state left over from a previous * driver (e.g., device_dax with vmemmap_shift). Also register this @@ -290,15 +341,18 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) /* Detect whether the data is at a non-zero offset into the memory */ if (pgmap->range.start != dev_dax->ranges[0].range.start) { u64 phys = dev_dax->ranges[0].range.start; - u64 pgmap_phys = dev_dax->pgmap[0].range.start; + u64 pgmap_phys = pgmap[0].range.start; - if (!WARN_ON(pgmap_phys > phys)) - data_offset = phys - pgmap_phys; + if (pgmap_phys > phys) { + dev_err(dev, "pgmap start %#llx exceeds data start %#llx\n", + pgmap_phys, phys); + return -EINVAL; + } + data_offset = phys - pgmap_phys; pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n", __func__, phys, pgmap_phys, data_offset); } - dev_dax->virt_addr = addr + data_offset; inode = dax_inode(dax_dev); cdev = inode->i_cdev; @@ -323,7 +377,13 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) return rc; run_dax(dax_dev); - return devm_add_action_or_reset(dev, fsdev_kill, dev_dax); + rc = devm_add_action_or_reset(dev, fsdev_kill, dev_dax); + if (rc) + return rc; + + /* Probe can no longer fail; expose the pgmap via dev_dax */ + dev_dax->pgmap = pgmap; + return 0; } static struct dax_device_driver fsdev_dax_driver = { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 25cf99dd9360..45f84b0eb909 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -116,11 +116,47 @@ EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); #if IS_ENABLED(CONFIG_FS_DAX) +/** + * fs_put_dax() - release holder ownership of a dax_device + * @dax_dev: dax device to release (may be NULL) + * @holder: the holder pointer previously passed to fs_dax_get() or + * fs_dax_get_by_bdev(); must match exactly, as it is used + * in a cmpxchg to atomically release ownership + * + * Must only be called by the current holder. Clears holder_ops before + * holder_data to avoid a race where a concurrent fs_dax_get() could have + * its newly installed holder_ops overwritten. + */ void fs_put_dax(struct dax_device *dax_dev, void *holder) { - if (dax_dev && holder && - cmpxchg(&dax_dev->holder_data, holder, NULL) == holder) - dax_dev->holder_ops = NULL; + if (dax_dev && holder) { + void *prev; + + /* + * Clear holder_ops before releasing holder_data. A concurrent + * dax_holder_notify_failure() that sees NULL ops returns + * -EOPNOTSUPP cleanly. A concurrent fs_dax_get() that acquires + * holder_data after the cmpxchg below is guaranteed to observe + * holder_ops=NULL first (cmpxchg provides release ordering), so + * its subsequent store of new ops will not be overwritten. + */ + WRITE_ONCE(dax_dev->holder_ops, NULL); + prev = cmpxchg(&dax_dev->holder_data, holder, NULL); + + /* + * prev == holder: normal release. + * prev == NULL: already released by kill_dax() when the + * device was removed under a live holder; + * not a bug. + * prev != holder (non-NULL): fs_put_dax() called by something + * that is not the current holder; an API + * contract violation. A lock would be needed + * to guard against this, but we WARN_ON() + * instead since violating the contract is + * a bug. + */ + WARN_ON(prev && prev != holder); + } put_dax(dax_dev); } EXPORT_SYMBOL_GPL(fs_put_dax); @@ -303,6 +339,7 @@ EXPORT_SYMBOL_GPL(dax_recovery_write); int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len, int mf_flags) { + const struct dax_holder_operations *ops; int rc, id; id = dax_read_lock(); @@ -311,12 +348,19 @@ int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, goto out; } - if (!dax_dev->holder_ops) { + /* + * Read holder_ops once: a concurrent fs_put_dax() can clear it without + * synchronizing against readers. Without the single fetch the compiler + * could reload between the NULL check and the call and dereference a + * NULL ops. + */ + ops = READ_ONCE(dax_dev->holder_ops); + if (!ops) { rc = -EOPNOTSUPP; goto out; } - rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags); + rc = ops->notify_failure(dax_dev, off, len, mf_flags); out: dax_read_unlock(id); return rc; diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 7e1112960d7f..380e352dc3cc 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -883,6 +883,14 @@ static int discover_arenas(struct btt *btt) arena->external_lba_start = cur_nlba; parse_arena_meta(arena, super, cur_off); + if (arena->nfree < btt->nd_region->num_lanes) { + dev_err(to_dev(arena), + "nfree %u smaller than lane count %d\n", + arena->nfree, btt->nd_region->num_lanes); + ret = -ENODEV; + goto out; + } + ret = log_set_indices(arena); if (ret) { dev_err(to_dev(arena), diff --git a/drivers/nvdimm/btt.h b/drivers/nvdimm/btt.h index 0c76c0333f6e..1ed245c18250 100644 --- a/drivers/nvdimm/btt.h +++ b/drivers/nvdimm/btt.h @@ -210,7 +210,7 @@ struct badblocks; * @lbasize: LBA size as requested and presented to upper layers. * This is sector_size + size of any metadata. * @sector_size: The Linux sector size - 512 or 4096 - * @lanes: Per-lane spinlocks + * @nd_region: &struct nd_region pointer * @init_lock: Mutex used for the BTT initialization * @init_state: Flag describing the initialization state for the BTT * @num_arenas: Number of arenas in the BTT instance diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c index 4218e3ac4a2a..dea2eee86d13 100644 --- a/drivers/nvdimm/label.c +++ b/drivers/nvdimm/label.c @@ -145,10 +145,21 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd) /* label sizes larger than 128 arrived with v1.2 */ version = __le16_to_cpu(nsindex[i]->major) * 100 + __le16_to_cpu(nsindex[i]->minor); - if (version >= 102) + if (version >= 102) { + /* + * labelsize feeds the shift below; only 0 (128-byte) + * and 1 (256-byte) are valid -- a larger value would + * overflow or exceed the width of int. + */ + if (nsindex[i]->labelsize > 1) { + dev_dbg(dev, "nsindex%d labelsize: %d invalid\n", + i, nsindex[i]->labelsize); + continue; + } labelsize = 1 << (7 + nsindex[i]->labelsize); - else + } else { labelsize = 128; + } if (labelsize != sizeof_namespace_label(ndd)) { dev_dbg(dev, "nsindex%d labelsize %d invalid\n", @@ -202,7 +213,7 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd) } nslot = __le32_to_cpu(nsindex[i]->nslot); - if (nslot * sizeof_namespace_label(ndd) + if ((u64)nslot * sizeof_namespace_label(ndd) + 2 * sizeof_namespace_index(ndd) > ndd->nsarea.config_size) { dev_dbg(dev, "nsindex%d nslot: %u invalid, config_size: %#x\n", diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 6b283fd03ff8..a77bbccb274d 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -1032,33 +1032,6 @@ out: return ret; } -#ifdef CONFIG_PRINT_QUOTA_WARNING -/** - * tty_write_message - write a message to a certain tty, not just the console. - * @tty: the destination tty_struct - * @msg: the message to write - * - * This is used for messages that need to be redirected to a specific tty. We - * don't put it into the syslog queue right now maybe in the future if really - * needed. - * - * We must still hold the BTM and test the CLOSING flag for the moment. - * - * This function is DEPRECATED, do not use in new code. - */ -void tty_write_message(struct tty_struct *tty, char *msg) -{ - if (tty) { - mutex_lock(&tty->atomic_write_lock); - tty_lock(tty); - if (tty->ops->write && tty->count > 0) - tty->ops->write(tty, msg, strlen(msg)); - tty_unlock(tty); - tty_write_unlock(tty); - } -} -#endif - static ssize_t file_tty_write(struct file *file, struct kiocb *iocb, struct iov_iter *from) { struct tty_struct *tty = file_tty(file); diff --git a/fs/btrfs/acl.c b/fs/btrfs/acl.c index e55b686fe1ab..662cdd1cbdef 100644 --- a/fs/btrfs/acl.c +++ b/fs/btrfs/acl.c @@ -15,6 +15,7 @@ #include "xattr.h" #include "acl.h" #include "misc.h" +#include "btrfs_inode.h" struct posix_acl *btrfs_get_acl(struct inode *inode, int type, bool rcu) { @@ -107,6 +108,9 @@ int btrfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, struct inode *inode = d_inode(dentry); umode_t old_mode = inode->i_mode; + if (btrfs_root_readonly(BTRFS_I(inode)->root)) + return -EROFS; + if (type == ACL_TYPE_ACCESS && acl) { ret = posix_acl_update_mode(idmap, inode, &inode->i_mode, &acl); diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 8def7abb728f..830460a40e86 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -2047,6 +2047,11 @@ static int btrfs_reclaim_block_group(struct btrfs_block_group *bg, int *reclaime trace_btrfs_reclaim_block_group(bg); ret = btrfs_relocate_chunk(fs_info, bg->start, false); + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + btrfs_dec_block_group_ro(bg); + btrfs_debug(fs_info, "deferring reclaim of chunk %llu", bg->start); + return ret; + } if (ret) { btrfs_dec_block_group_ro(bg); btrfs_err(fs_info, "error relocating chunk %llu", @@ -2113,7 +2118,8 @@ void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limi spin_unlock(&fs_info->unused_bgs_lock); ret = btrfs_reclaim_block_group(bg, &reclaimed); - if (ret && !READ_ONCE(space_info->periodic_reclaim)) + if ((btrfs_is_zoned(fs_info) && ret == -EAGAIN) || + (ret && !READ_ONCE(space_info->periodic_reclaim))) btrfs_link_bg_list(bg, &retry_list); btrfs_put_block_group(bg); @@ -2624,10 +2630,9 @@ static int fill_dummy_bgs(struct btrfs_fs_info *fs_info) /* Fill dummy cache as FULL */ bg->length = map->chunk_len; - bg->flags = map->type; + bg->flags = map->on_disk_type; bg->cached = BTRFS_CACHE_FINISHED; bg->used = map->chunk_len; - bg->flags = map->type; bg->space_info = btrfs_find_space_info(fs_info, bg->flags); ret = btrfs_add_block_group_cache(bg); /* @@ -3916,7 +3921,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, old_val += num_bytes; cache->used = old_val; cache->reserved -= num_bytes; - cache->reclaim_mark = 0; + cache->reclaim_mark = false; space_info->bytes_reserved -= num_bytes; space_info->bytes_used += num_bytes; space_info->disk_used += num_bytes * factor; diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 790c2d467af5..69d56864d4ba 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -263,6 +263,9 @@ struct btrfs_block_group { enum btrfs_block_group_size_class size_class:8; + /* If set, this blockgroup is not used for allocation between two reclaim sweeps. */ + bool reclaim_mark; + /* * Number of extents in this block group used for swap files. * All accesses protected by the spinlock 'lock'. @@ -281,7 +284,6 @@ struct btrfs_block_group { struct list_head active_bg_list; struct work_struct zone_finish_work; struct extent_buffer *last_eb; - u64 reclaim_mark; }; static inline u64 btrfs_block_group_end(const struct btrfs_block_group *block_group) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index ffb6b52863a7..c62b5148d5ac 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -651,9 +651,9 @@ struct heuristic_ws { u8 *sample; u32 sample_size; /* Buckets store counters for each byte value */ - struct bucket_item *bucket; + struct bucket_item bucket[BUCKET_SIZE]; /* Sorting buffer */ - struct bucket_item *bucket_b; + struct bucket_item bucket_b[BUCKET_SIZE]; struct list_head list; }; @@ -664,8 +664,6 @@ static void free_heuristic_ws(struct list_head *ws) workspace = list_entry(ws, struct heuristic_ws, list); kvfree(workspace->sample); - kfree(workspace->bucket); - kfree(workspace->bucket_b); kfree(workspace); } @@ -681,14 +679,6 @@ static struct list_head *alloc_heuristic_ws(struct btrfs_fs_info *fs_info) if (!ws->sample) goto fail; - ws->bucket = kzalloc_objs(*ws->bucket, BUCKET_SIZE); - if (!ws->bucket) - goto fail; - - ws->bucket_b = kzalloc_objs(*ws->bucket_b, BUCKET_SIZE); - if (!ws->bucket_b) - goto fail; - INIT_LIST_HEAD(&ws->list); return &ws->list; fail: diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 49fb6b816aa9..8fe330d81b8f 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1460,6 +1460,7 @@ static noinline void unlock_up(struct btrfs_path *path, int level, */ static int read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, + struct btrfs_eb_prealloc *pa, struct extent_buffer **eb_ret, int slot, const struct btrfs_key *key) { @@ -1546,7 +1547,8 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, if (p->reada != READA_NONE) reada_for_search(fs_info, p, parent_level, slot, key->objectid); - tmp = btrfs_find_create_tree_block(fs_info, blocknr, check.owner_root, check.level); + tmp = btrfs_find_create_tree_block(fs_info, pa, blocknr, + check.owner_root, check.level); if (IS_ERR(tmp)) { ret = PTR_ERR(tmp); tmp = NULL; @@ -2004,6 +2006,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, u8 lowest_level = 0; int min_write_lock_level; int prev_cmp; + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; if (!root) return -EINVAL; @@ -2058,6 +2061,11 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, } again: + if (pa.needs_prealloc) { + ret = btrfs_init_eb_prealloc(fs_info, &pa, false); + if (ret) + goto done; + } prev_cmp = -1; b = btrfs_search_slot_get_root(root, p, write_lock_level); if (IS_ERR(b)) { @@ -2187,7 +2195,7 @@ cow_done: goto done; } - ret2 = read_block_for_search(root, p, &b, slot, key); + ret2 = read_block_for_search(root, p, &pa, &b, slot, key); if (ret2 == -EAGAIN && !p->nowait) { trace_btrfs_search_slot_restart(root, level, "read_block"); goto again; @@ -2234,6 +2242,8 @@ done: ret = ret2; } + btrfs_free_eb_prealloc(&pa); + return ret; } ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); @@ -2259,6 +2269,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, int level; int lowest_unlock = 1; u8 lowest_level = 0; + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; lowest_level = p->lowest_level; WARN_ON(p->nodes[0] != NULL); @@ -2270,6 +2281,11 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, } again: + if (pa.needs_prealloc) { + ret = btrfs_init_eb_prealloc(fs_info, &pa, false); + if (ret) + goto done; + } b = btrfs_get_old_root(root, time_seq); if (unlikely(!b)) { ret = -EIO; @@ -2316,7 +2332,7 @@ again: goto done; } - ret2 = read_block_for_search(root, p, &b, slot, key); + ret2 = read_block_for_search(root, p, &pa, &b, slot, key); if (ret2 == -EAGAIN && !p->nowait) goto again; if (ret2) { @@ -2339,6 +2355,8 @@ done: if (ret < 0) btrfs_release_path(p); + btrfs_free_eb_prealloc(&pa); + return ret; } @@ -4780,6 +4798,7 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, struct extent_buffer *next; struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_key key; + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; bool need_commit_sem = false; u32 nritems; int ret; @@ -4798,6 +4817,11 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); again: + if (pa.needs_prealloc) { + ret = btrfs_init_eb_prealloc(fs_info, &pa, false); + if (ret) + goto done; + } level = 1; next = NULL; btrfs_release_path(path); @@ -4880,7 +4904,7 @@ again: } next = c; - ret = read_block_for_search(root, path, &next, slot, &key); + ret = read_block_for_search(root, path, &pa, &next, slot, &key); if (ret == -EAGAIN && !path->nowait) goto again; @@ -4923,7 +4947,7 @@ again: if (!level) break; - ret = read_block_for_search(root, path, &next, 0, &key); + ret = read_block_for_search(root, path, &pa, &next, 0, &key); if (ret == -EAGAIN && !path->nowait) goto again; @@ -4956,6 +4980,8 @@ done: ret = ret2; } + btrfs_free_eb_prealloc(&pa); + return ret; } diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 6de7ad191e04..22ba2b4505b3 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -131,7 +131,6 @@ enum { BTRFS_ROOT_ORPHAN_ITEM_INSERTED, BTRFS_ROOT_DEFRAG_RUNNING, BTRFS_ROOT_FORCE_COW, - BTRFS_ROOT_MULTI_LOG_TASKS, BTRFS_ROOT_DIRTY, BTRFS_ROOT_DELETING, @@ -196,9 +195,7 @@ struct btrfs_root { struct list_head log_ctxs[2]; /* Used only for log trees of subvolumes, not for the log root tree */ atomic_t log_writers; - atomic_t log_commit[2]; - /* Used only for log trees of subvolumes, not for the log root tree */ - atomic_t log_batch; + bool log_commit[2]; /* * Protected by the 'log_mutex' lock but can be read without holding * that lock to avoid unnecessary lock contention, in which case it @@ -216,7 +213,6 @@ struct btrfs_root { * to access this field. */ int last_log_commit; - pid_t log_start_pid; u64 last_trans; diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index f0c6758b7055..6ec5dd760d42 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -1093,7 +1093,7 @@ next: struct defrag_target_range *tmp; list_for_each_entry_safe(entry, tmp, target_list, list) { - list_del_init(&entry->list); + list_del(&entry->list); kfree(entry); } } @@ -1130,20 +1130,15 @@ static_assert(PAGE_ALIGNED(CLUSTER_SIZE)); * * - Extent bits are locked */ -static int defrag_one_locked_target(struct btrfs_inode *inode, - struct defrag_target_range *target, - struct folio **folios, int nr_pages, - struct extent_state **cached_state) +static void defrag_one_locked_target(struct btrfs_inode *inode, + struct defrag_target_range *target, + struct folio **folios, int nr_pages, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_changeset *data_reserved = NULL; const u64 start = target->start; const u64 len = target->len; - int ret = 0; - ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); - if (ret < 0) - return ret; btrfs_clear_extent_bit(&inode->io_tree, start, start + len - 1, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, cached_state); @@ -1164,10 +1159,6 @@ static int defrag_one_locked_target(struct btrfs_inode *inode, continue; btrfs_folio_clamp_set_dirty(fs_info, folio, start, len); } - btrfs_delalloc_release_extents(inode, len); - extent_changeset_free(data_reserved); - - return ret; } static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, @@ -1178,11 +1169,13 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, struct defrag_target_range *entry; struct defrag_target_range *tmp; LIST_HEAD(target_list); - struct folio **folios; + struct folio AUTO_KFREE(*folios); const u32 sectorsize = inode->root->fs_info->sectorsize; u64 cur = start; const unsigned int nr_pages = ((start + len - 1) >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; + struct extent_changeset *data_reserved = NULL; + u64 last_defrag_end = start; int ret = 0; ASSERT(nr_pages <= CLUSTER_SIZE / PAGE_SIZE); @@ -1192,6 +1185,20 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, if (!folios) return -ENOMEM; + /* + * Reserve delalloc space before locking the range and before locking + * and dirtying any folios - otherwise we could deadlock, for example + * after defrag of one range we dirty folios and keep them locked when + * we move to the next range, so reserving delalloc space right before + * each range could trigger flushing of delalloc and deadlock on the + * extent lock or trigger a transaction commit with flushoncommit, which + * can either deadlock on the lock of a folio made dirty in the previous + * range or the extent lock. + */ + ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); + if (ret < 0) + return ret; + /* Prepare all pages */ for (int i = 0; cur < start + len && i < nr_pages; i++) { folios[i] = defrag_prepare_one_folio(inode, cur >> PAGE_SHIFT); @@ -1225,15 +1232,12 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, if (ret < 0) goto unlock_extent; - list_for_each_entry(entry, &target_list, list) { - ret = defrag_one_locked_target(inode, entry, folios, nr_pages, - &cached_state); - if (ret < 0) - break; - } - list_for_each_entry_safe(entry, tmp, &target_list, list) { - list_del_init(&entry->list); + defrag_one_locked_target(inode, entry, folios, nr_pages, &cached_state); + if (entry->start > last_defrag_end) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + entry->start - last_defrag_end, true); + last_defrag_end = entry->start + entry->len; kfree(entry); } unlock_extent: @@ -1245,7 +1249,12 @@ free_folios: folio_unlock(folios[i]); folio_put(folios[i]); } - kfree(folios); + btrfs_delalloc_release_extents(inode, len); + if (last_defrag_end < start + len) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + start + len - last_defrag_end, true); + extent_changeset_free(data_reserved); + return ret; } @@ -1310,10 +1319,8 @@ static int defrag_one_cluster(struct btrfs_inode *inode, inode->root->fs_info->sectorsize_bits; } out: - list_for_each_entry_safe(entry, tmp, &target_list, list) { - list_del_init(&entry->list); + list_for_each_entry_safe(entry, tmp, &target_list, list) kfree(entry); - } if (ret >= 0) *last_scanned_ret = max(*last_scanned_ret, start + len); return ret; diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 09795439b9fb..db2ffab0941a 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1523,10 +1523,10 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, ret = __btrfs_add_delayed_item(delayed_node, delayed_item); if (unlikely(ret)) { btrfs_err(trans->fs_info, -"error adding delayed dir index item, name: %.*s, index: %llu, root: %llu, dir: %llu, dir->index_cnt: %llu, delayed_node->index_cnt: %llu, error: %d", +"error adding delayed dir index item, name: %.*s, index: %llu, root: %llu, dir: %llu, dir->index_cnt: %llu, delayed_node->index_cnt: %llu, error: %pe", name_len, name, index, btrfs_root_id(delayed_node->root), delayed_node->inode_id, dir->index_cnt, - delayed_node->index_cnt, ret); + delayed_node->index_cnt, ERR_PTR(ret)); btrfs_release_delayed_item(delayed_item); btrfs_release_dir_index_item_space(trans); mutex_unlock(&delayed_node->mutex); @@ -1645,8 +1645,8 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, */ if (ret < 0) { btrfs_err(trans->fs_info, -"metadata reservation failed for delayed dir item deletion, index: %llu, root: %llu, inode: %llu, error: %d", - index, btrfs_root_id(node->root), node->inode_id, ret); +"metadata reservation failed for delayed dir item deletion, index: %llu, root: %llu, inode: %llu, error: %pe", + index, btrfs_root_id(node->root), node->inode_id, ERR_PTR(ret)); btrfs_release_delayed_item(item); goto end; } @@ -1655,8 +1655,8 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, ret = __btrfs_add_delayed_item(node, item); if (unlikely(ret)) { btrfs_err(trans->fs_info, -"failed to add delayed dir index item, root: %llu, inode: %llu, index: %llu, error: %d", - btrfs_root_id(node->root), node->inode_id, index, ret); +"failed to add delayed dir index item, root: %llu, inode: %llu, index: %llu, error: %pe", + btrfs_root_id(node->root), node->inode_id, index, ERR_PTR(ret)); btrfs_delayed_item_release_metadata(dir->root, item); btrfs_release_delayed_item(item); } diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index d5439b06cdc9..3075d7992713 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -14,7 +14,6 @@ #include "ordered-data.h" struct btrfs_dio_data { - ssize_t submitted; loff_t old_isize; struct extent_changeset *data_reserved; struct btrfs_ordered_extent *ordered; @@ -151,7 +150,7 @@ static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, if (type != BTRFS_ORDERED_NOCOW) { em = btrfs_create_io_em(inode, start, file_extent, type); if (IS_ERR(em)) - goto out; + return em; } ordered = btrfs_alloc_ordered_extent(inode, start, file_extent, @@ -168,7 +167,6 @@ static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, ASSERT(!dio_data->ordered); dio_data->ordered = ordered; } - out: return em; } @@ -281,17 +279,24 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, em2 = btrfs_create_dio_extent(BTRFS_I(inode), dio_data, start, &file_extent, type); btrfs_dec_nocow_writers(bg); - if (type == BTRFS_ORDERED_PREALLOC) { - btrfs_free_extent_map(em); - *map = em2; - em = em2; - } - if (IS_ERR(em2)) { ret = PTR_ERR(em2); + btrfs_free_extent_map(em); + *map = NULL; goto out; } + /* + * True NOCOW writes don't need to create a new extent map, + * while PREALLOC writes must replace the existing one. + */ + if (em2) { + ASSERT(type == BTRFS_ORDERED_PREALLOC); + btrfs_free_extent_map(em); + *map = em2; + em = em2; + } + dio_data->nocow_done = true; } else { /* Our caller expects us to free the input extent map. */ @@ -619,78 +624,81 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, { struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); struct btrfs_dio_data *dio_data = iter->private; - size_t submitted = dio_data->submitted; const bool write = !!(flags & IOMAP_WRITE); int ret = 0; - if (!write && (iomap->type == IOMAP_HOLE)) { - /* If reading from a hole, unlock and return */ - btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, - pos + length - 1, NULL); + if (!write) { + /* + * Hole read, nothing is submitted, thus we have to unlock + * the whole range. + */ + if (iomap->type == IOMAP_HOLE) { + btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, + pos + length - 1, NULL); + return 0; + } + /* + * Short read, needs to unlock the remaining range, and + * return -ENOTBLK so we can later fault in the pages and retry. + */ + if (written < length) { + btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos + written, + pos + length - 1, NULL); + return -ENOTBLK; + } + /* The full range is submitted, endio will do the unlock. */ return 0; } - if (submitted < length) { - pos += submitted; - length -= submitted; - if (write) { - /* - * Got a short write and have updated the isize, need to - * revert the isize change. - * - * Normally we need to update isize with extent lock hold, - * but we're safe due to the following factors: - * - * - Only a single writer can be enlarging isize - * Enlarging isize will take the exclusive inode lock. - * - * - Buffered readers need to wait for the OE we're holding - * Buffered readers will lock extent and wait for OE - * of the folio range, and since page cache is invalidated - * the OE wait can not be skipped. - * - * So here we are safe to revert the isize before - * finishing the OE, and no reader of the remaining range - * can see the enlarged size. - * - * TODO: Extend the DIO_LOCKED lifespan for direct writes, - * and only enlarge isize after a successful write. - */ - if (dio_data->updated_isize) { - u64 new_isize; - - if (submitted == 0) - new_isize = dio_data->old_isize; - else - new_isize = max(dio_data->old_isize, pos); - i_size_write(inode, new_isize); - dio_data->updated_isize = false; - } - /* - * We have a short write, if there is any range - * that is submitted properly, that part will have - * its own OE split from the original one. - * - * So for the OE at dio_data->ordered, it's the part - * that is not submitted, and should be marked - * as fully truncated. - */ - btrfs_mark_ordered_extent_truncated(dio_data->ordered, 0); - btrfs_finish_ordered_extent(dio_data->ordered, - pos, length, true); - } else { - btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, - pos + length - 1, NULL); + if (written < length) { + /* + * Got a short write and have updated the i_size, need to revert + * the i_size change. + * + * Normally we need to update i_size with extent lock held, but + * we're safe due to the following factors: + * + * - Only a single writer can be enlarging i_size + * Enlarging i_size will take the exclusive inode lock. + * + * - Buffered readers need to wait for the OE we're holding + * Buffered readers will lock extent and wait for OE + * of the folio range, and since page cache is invalidated + * the OE wait cannot be skipped. + * + * So here we are safe to revert the isize before finishing the + * OE, and no reader of the remaining range can see the enlarged + * size. + * + * TODO: Extend the DIO_LOCKED lifespan for direct writes, + * and only enlarge isize after a successful write. + */ + if (dio_data->updated_isize) { + u64 new_isize; + + if (written == 0) + new_isize = dio_data->old_isize; + else + new_isize = max(dio_data->old_isize, pos + written); + i_size_write(inode, new_isize); + dio_data->updated_isize = false; } + /* + * We have a short write, if there is any range that is submitted + * properly, that part will have its own OE split from the + * original one. + * + * So for the OE at dio_data->ordered, it's the part that is not + * submitted, and should be marked as fully truncated. + */ + btrfs_mark_ordered_extent_truncated(dio_data->ordered, 0); + btrfs_finish_ordered_extent(dio_data->ordered, + pos + written, length - written, true); ret = -ENOTBLK; } - if (write) { - btrfs_put_ordered_extent(dio_data->ordered); - dio_data->ordered = NULL; - } - - if (write) - extent_changeset_free(dio_data->data_reserved); + btrfs_put_ordered_extent(dio_data->ordered); + dio_data->ordered = NULL; + extent_changeset_free(dio_data->data_reserved); return ret; } @@ -772,8 +780,6 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, dip->file_offset = file_offset; dip->bytes = bio->bi_iter.bi_size; - dio_data->submitted += bio->bi_iter.bi_size; - /* * Check if we are doing a partial write. If we are, we need to split * the ordered extent to match the submitted bio. Hang on to the @@ -819,13 +825,41 @@ static ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED, &data, done_before); } +static bool need_stable_write(struct btrfs_inode *inode) +{ + const u64 data_profile = btrfs_data_alloc_profile(inode->root->fs_info) & + BTRFS_BLOCK_GROUP_PROFILE_MASK; + + /* Data checksum requires stable buffer. */ + if (!(inode->flags & BTRFS_INODE_NODATASUM)) + return true; + /* + * Any profile with mirror/parity will require stable buffer. + * Otherwise the mirror may differ from each other. + * + * Thus only SINGLE and RAID0 doesn't require stable buffer. + */ + if (data_profile != 0 && data_profile != BTRFS_BLOCK_GROUP_RAID0) + return true; + return false; +} + static struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter, size_t done_before) { struct btrfs_dio_data data = { 0 }; + unsigned int dio_flags = IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED; + + if (need_stable_write(BTRFS_I(file_inode(iocb->ki_filp)))) { + /* For now no support for BOUNCE and NOWAIT direct write. */ + if (iocb->ki_flags & IOCB_NOWAIT) + return ERR_PTR(-EAGAIN); + + dio_flags |= IOMAP_DIO_BOUNCE; + } return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, - IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED, &data, done_before); + dio_flags, &data, done_before); } static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, @@ -854,8 +888,6 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) ssize_t ret; unsigned int ilock_flags = 0; struct iomap_dio *dio; - const u64 data_profile = btrfs_data_alloc_profile(fs_info) & - BTRFS_BLOCK_GROUP_PROFILE_MASK; if (iocb->ki_flags & IOCB_NOWAIT) ilock_flags |= BTRFS_ILOCK_TRY; @@ -869,16 +901,6 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode) && IS_NOSEC(inode)) ilock_flags |= BTRFS_ILOCK_SHARED; - /* - * If our data profile has duplication (either extra mirrors or RAID56), - * we can not trust the direct IO buffer, the content may change during - * writeback and cause different contents written to different mirrors. - * - * Thus only RAID0 and SINGLE can go true zero-copy direct IO. - */ - if (data_profile != BTRFS_BLOCK_GROUP_RAID0 && data_profile != 0) - goto buffered; - relock: ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags); if (ret < 0) @@ -919,22 +941,6 @@ relock: btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); goto buffered; } - /* - * We can't control the folios being passed in, applications can write - * to them while a direct IO write is in progress. This means the - * content might change after we calculated the data checksum. - * Therefore we can end up storing a checksum that doesn't match the - * persisted data. - * - * To be extra safe and avoid false data checksum mismatch, if the - * inode requires data checksum, just fallback to buffered IO. - * For buffered IO we have full control of page cache and can ensure - * no one is modifying the content during writeback. - */ - if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { - btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); - goto buffered; - } /* * The iov_iter can be mapped to the same file range we are writing to. diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2f1666d9544e..819727460bcf 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -271,14 +271,15 @@ int btree_csum_one_bio(struct btrfs_bio *bbio) return -EIO; /* - * If an extent_buffer is marked as EXTENT_BUFFER_ZONED_ZEROOUT, don't - * checksum it but zero-out its content. This is done to preserve - * ordering of I/O without unnecessarily writing out data. + * An extent_buffer marked EXTENT_BUFFER_ZONED_ZEROOUT is written out as + * zeros to preserve ordering of I/O without persisting the now + * unnecessary block. The bio is fed from the shared zero page (see + * write_one_eb()), so there is nothing to checksum here. Crucially, the + * buffer's own content is left intact: it may still be referenced, e.g. + * btrfs_free_tree_block() reads its header to add a delayed reference. */ - if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { - memzero_extent_buffer(eb, 0, eb->len); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) return 0; - } if (WARN_ON_ONCE(found_start != eb->start)) return -EIO; @@ -590,12 +591,13 @@ static const struct address_space_operations btree_aops = { struct extent_buffer *btrfs_find_create_tree_block( struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, u64 bytenr, u64 owner_root, int level) { if (btrfs_is_testing(fs_info)) return alloc_test_extent_buffer(fs_info, bytenr); - return alloc_extent_buffer(fs_info, bytenr, owner_root, level); + return alloc_extent_buffer(fs_info, pa, bytenr, owner_root, level); } /* @@ -608,12 +610,13 @@ struct extent_buffer *btrfs_find_create_tree_block( struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, struct btrfs_tree_parent_check *check) { + struct btrfs_eb_prealloc pa = { 0 }; struct extent_buffer *buf = NULL; int ret; ASSERT(check); - buf = btrfs_find_create_tree_block(fs_info, bytenr, check->owner_root, + buf = btrfs_find_create_tree_block(fs_info, &pa, bytenr, check->owner_root, check->level); if (IS_ERR(buf)) return buf; @@ -666,10 +669,7 @@ static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info, init_waitqueue_head(&root->log_commit_wait[1]); INIT_LIST_HEAD(&root->log_ctxs[0]); INIT_LIST_HEAD(&root->log_ctxs[1]); - atomic_set(&root->log_commit[0], 0); - atomic_set(&root->log_commit[1], 0); atomic_set(&root->log_writers, 0); - atomic_set(&root->log_batch, 0); refcount_set(&root->refs, 1); atomic_set(&root->snapshot_force_cow, 0); atomic_set(&root->nr_swapfiles, 0); @@ -2052,7 +2052,7 @@ static int btrfs_replay_log(struct btrfs_fs_info *fs_info, if (IS_ERR(log_tree_root->node)) { ret = PTR_ERR(log_tree_root->node); log_tree_root->node = NULL; - btrfs_err(fs_info, "failed to read log tree with error: %d", ret); + btrfs_err(fs_info, "failed to read log tree with error: %pe", ERR_PTR(ret)); btrfs_put_root(log_tree_root); return ret; } @@ -2062,7 +2062,7 @@ static int btrfs_replay_log(struct btrfs_fs_info *fs_info, btrfs_put_root(log_tree_root); if (unlikely(ret)) { ASSERT(BTRFS_FS_ERROR(fs_info) != 0); - btrfs_err(fs_info, "failed to recover log trees with error: %d", ret); + btrfs_err(fs_info, "failed to recover log trees with error: %pe", ERR_PTR(ret)); return ret; } @@ -2303,8 +2303,8 @@ static int btrfs_read_roots(struct btrfs_fs_info *fs_info) return 0; out: - btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d", - location.objectid, ret); + btrfs_warn(fs_info, "failed to read root (objectid=%llu): %pe", + location.objectid, ERR_PTR(ret)); return ret; } @@ -2395,8 +2395,8 @@ short_read: int btrfs_validate_super(const struct btrfs_fs_info *fs_info, const struct btrfs_super_block *sb, int mirror_num) { - u64 nodesize = btrfs_super_nodesize(sb); - u64 sectorsize = btrfs_super_sectorsize(sb); + const u32 nodesize = btrfs_super_nodesize(sb); + const u32 sectorsize = btrfs_super_sectorsize(sb); int ret = 0; const bool ignore_flags = btrfs_test_opt(fs_info, IGNORESUPERFLAGS); @@ -2438,24 +2438,24 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info, */ if (unlikely(!is_power_of_2(sectorsize) || sectorsize < BTRFS_MIN_BLOCKSIZE || sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE)) { - btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize); + btrfs_err(fs_info, "invalid sectorsize %u", sectorsize); ret = -EINVAL; } if (unlikely(!btrfs_supported_blocksize(sectorsize))) { btrfs_err(fs_info, - "sectorsize %llu not yet supported for page size %lu", + "sectorsize %u not yet supported for page size %lu", sectorsize, PAGE_SIZE); ret = -EINVAL; } if (unlikely(!is_power_of_2(nodesize) || nodesize < sectorsize || nodesize > BTRFS_MAX_METADATA_BLOCKSIZE)) { - btrfs_err(fs_info, "invalid nodesize %llu", nodesize); + btrfs_err(fs_info, "invalid nodesize %u", nodesize); ret = -EINVAL; } if (unlikely(nodesize != le32_to_cpu(sb->__unused_leafsize))) { - btrfs_err(fs_info, "invalid leafsize %u, should be %llu", + btrfs_err(fs_info, "invalid leafsize %u, should be %u", le32_to_cpu(sb->__unused_leafsize), nodesize); ret = -EINVAL; } @@ -2905,7 +2905,6 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) fs_info->nodesize = 4096; fs_info->sectorsize = 4096; fs_info->sectorsize_bits = ilog2(4096); - fs_info->stripesize = 4096; /* Default compress algorithm when user does -o compress */ fs_info->compress_type = BTRFS_COMPRESS_ZLIB; @@ -2979,8 +2978,8 @@ static int btrfs_uuid_rescan_kthread(void *data) ret = btrfs_uuid_tree_iterate(fs_info); if (ret < 0) { if (ret != -EINTR) - btrfs_warn(fs_info, "iterating uuid_tree failed %d", - ret); + btrfs_warn(fs_info, "iterating uuid_tree failed %pe", + ERR_PTR(ret)); up(&fs_info->uuid_tree_rescan_sem); return ret; } @@ -3083,7 +3082,7 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) ret = btrfs_rebuild_free_space_tree(fs_info); if (ret) { btrfs_warn(fs_info, - "failed to rebuild free space tree: %d", ret); + "failed to rebuild free space tree: %pe", ERR_PTR(ret)); return ret; } } @@ -3094,7 +3093,7 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) ret = btrfs_delete_free_space_tree(fs_info); if (ret) { btrfs_warn(fs_info, - "failed to disable free space tree: %d", ret); + "failed to disable free space tree: %pe", ERR_PTR(ret)); return ret; } } @@ -3105,7 +3104,8 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) */ ret = btrfs_delete_orphan_free_space_entries(fs_info); if (ret < 0) { - btrfs_err(fs_info, "failed to delete orphan free space tree entries: %d", ret); + btrfs_err(fs_info, "failed to delete orphan free space tree entries: %pe", + ERR_PTR(ret)); return ret; } /* @@ -3139,7 +3139,7 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) ret = btrfs_recover_relocation(fs_info); mutex_unlock(&fs_info->cleaner_mutex); if (ret < 0) { - btrfs_warn(fs_info, "failed to recover relocation: %d", ret); + btrfs_warn(fs_info, "failed to recover relocation: %pe", ERR_PTR(ret)); return ret; } @@ -3149,7 +3149,7 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) ret = btrfs_create_free_space_tree(fs_info); if (ret) { btrfs_warn(fs_info, - "failed to create free space tree: %d", ret); + "failed to create free space tree: %pe", ERR_PTR(ret)); return ret; } } @@ -3177,7 +3177,7 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) ret = btrfs_create_uuid_tree(fs_info); if (ret) { btrfs_warn(fs_info, - "failed to create the UUID tree %d", ret); + "failed to create the UUID tree %pe", ERR_PTR(ret)); return ret; } } @@ -3314,6 +3314,8 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) */ rcu_read_lock(); xa_for_each(&fs_info->buffer_tree, index, eb) { + unsigned int refs; + /* Increase the ref so that the eb won't disappear. */ if (!refcount_inc_not_zero(&eb->refs)) continue; @@ -3324,16 +3326,26 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE); /* + * We hold the spinlock to make sure above + * EXTENT_BUFFER_READING flag is cleared with the held + * ref dropped. + * Or we can hit a race window and lead to false alerts. + */ + spin_lock(&eb->refs_lock); + refs = refcount_read(&eb->refs); + spin_unlock(&eb->refs_lock); + + /* * The refs threshold is 2, one held by us at the beginning * of the loop, one for the ownership in the buffer tree. */ - if (unlikely(refcount_read(&eb->refs) > 2 || extent_buffer_under_io(eb))) { + if (unlikely(refs > 2 || extent_buffer_under_io(eb))) { WARN_ON_ONCE(IS_ENABLED(CONFIG_BTRFS_DEBUG)); btrfs_warn(fs_info, "unable to release extent buffer %llu owner %llu gen %llu refs %u flags 0x%lx", eb->start, btrfs_header_owner(eb), btrfs_header_generation(eb), - refcount_read(&eb->refs), eb->bflags); + refs, eb->bflags); } free_extent_buffer(eb); rcu_read_lock(); @@ -3355,7 +3367,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device { u32 sectorsize; u32 nodesize; - u32 stripesize; u64 generation; u16 csum_type; struct btrfs_super_block *disk_super; @@ -3464,7 +3475,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device /* Set up fs_info before parsing mount options */ nodesize = btrfs_super_nodesize(disk_super); sectorsize = btrfs_super_sectorsize(disk_super); - stripesize = sectorsize; fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids)); fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids)); @@ -3483,7 +3493,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device else fs_info->block_max_order = calc_block_max_order(fs_info->sectorsize_bits); fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size; - fs_info->stripesize = stripesize; fs_info->fs_devices->fs_info = fs_info; if (fs_info->sectorsize > PAGE_SIZE) @@ -3549,7 +3558,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device ret = btrfs_read_sys_array(fs_info); mutex_unlock(&fs_info->chunk_mutex); if (ret) { - btrfs_err(fs_info, "failed to read the system array: %d", ret); + btrfs_err(fs_info, "failed to read the system array: %pe", ERR_PTR(ret)); goto fail_sb_buffer; } @@ -3568,7 +3577,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device ret = btrfs_read_chunk_tree(fs_info); if (ret) { - btrfs_err(fs_info, "failed to read chunk tree: %d", ret); + btrfs_err(fs_info, "failed to read chunk tree: %pe", ERR_PTR(ret)); goto fail_tree_roots; } @@ -3598,7 +3607,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device ret = btrfs_get_dev_zone_info_all_devices(fs_info); if (ret) { btrfs_err(fs_info, - "zoned: failed to read device zone info: %d", ret); + "zoned: failed to read device zone info: %pe", ERR_PTR(ret)); goto fail_block_groups; } @@ -3621,72 +3630,73 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device ret = btrfs_verify_dev_extents(fs_info); if (ret) { btrfs_err(fs_info, - "failed to verify dev extents against chunks: %d", - ret); + "failed to verify dev extents against chunks: %pe", + ERR_PTR(ret)); goto fail_block_groups; } ret = btrfs_recover_balance(fs_info); if (ret) { - btrfs_err(fs_info, "failed to recover balance: %d", ret); + btrfs_err(fs_info, "failed to recover balance: %pe", ERR_PTR(ret)); goto fail_block_groups; } ret = btrfs_init_dev_stats(fs_info); if (ret) { - btrfs_err(fs_info, "failed to init dev_stats: %d", ret); + btrfs_err(fs_info, "failed to init dev_stats: %pe", ERR_PTR(ret)); goto fail_block_groups; } ret = btrfs_init_dev_replace(fs_info); if (ret) { - btrfs_err(fs_info, "failed to init dev_replace: %d", ret); + btrfs_err(fs_info, "failed to init dev_replace: %pe", ERR_PTR(ret)); goto fail_block_groups; } ret = btrfs_check_zoned_mode(fs_info); if (ret) { - btrfs_err(fs_info, "failed to initialize zoned mode: %d", - ret); + btrfs_err(fs_info, "failed to initialize zoned mode: %pe", + ERR_PTR(ret)); goto fail_block_groups; } ret = btrfs_sysfs_add_fsid(fs_devices); if (ret) { - btrfs_err(fs_info, "failed to init sysfs fsid interface: %d", - ret); + btrfs_err(fs_info, "failed to init sysfs fsid interface: %pe", + ERR_PTR(ret)); goto fail_block_groups; } ret = btrfs_sysfs_add_mounted(fs_info); if (ret) { - btrfs_err(fs_info, "failed to init sysfs interface: %d", ret); + btrfs_err(fs_info, "failed to init sysfs interface: %pe", ERR_PTR(ret)); goto fail_fsdev_sysfs; } ret = btrfs_init_space_info(fs_info); if (ret) { - btrfs_err(fs_info, "failed to initialize space info: %d", ret); + btrfs_err(fs_info, "failed to initialize space info: %pe", ERR_PTR(ret)); goto fail_sysfs; } ret = btrfs_read_block_groups(fs_info); if (ret) { - btrfs_err(fs_info, "failed to read block groups: %d", ret); + btrfs_err(fs_info, "failed to read block groups: %pe", ERR_PTR(ret)); goto fail_sysfs; } if (btrfs_fs_incompat(fs_info, REMAP_TREE)) { ret = btrfs_populate_fully_remapped_bgs_list(fs_info); if (ret) { - btrfs_err(fs_info, "failed to populate fully_remapped_bgs list: %d", ret); + btrfs_err(fs_info, "failed to populate fully_remapped_bgs list: %pe", + ERR_PTR(ret)); goto fail_sysfs; } } ret = btrfs_init_writeback_bio_size(fs_info); if (ret) { - btrfs_err(fs_info, "failed to get optimum writeback size: %d", - ret); + btrfs_err(fs_info, "failed to get optimum writeback size: %pe", + ERR_PTR(ret)); goto fail_sysfs; } @@ -3742,7 +3752,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true); if (IS_ERR(fs_info->fs_root)) { ret = PTR_ERR(fs_info->fs_root); - btrfs_err(fs_info, "failed to read fs tree: %d", ret); + btrfs_err(fs_info, "failed to read fs tree: %pe", ERR_PTR(ret)); fs_info->fs_root = NULL; goto fail_qgroup; } @@ -3763,7 +3773,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device btrfs_info(fs_info, "checking UUID tree"); ret = btrfs_check_uuid_tree(fs_info); if (ret) { - btrfs_err(fs_info, "failed to check the UUID tree: %d", ret); + btrfs_err(fs_info, "failed to check the UUID tree: %pe", ERR_PTR(ret)); close_ctree(fs_info); return ret; } @@ -3879,8 +3889,8 @@ static int write_dev_supers(struct btrfs_device *device, continue; } else if (ret < 0) { btrfs_err(device->fs_info, - "couldn't get super block location for mirror %d error %d", - i, ret); + "couldn't get super block location for mirror %d error %pe", + i, ERR_PTR(ret)); atomic_inc(&device->sb_write_errors); continue; } @@ -3898,8 +3908,8 @@ static int write_dev_supers(struct btrfs_device *device, GFP_NOFS); if (IS_ERR(folio)) { btrfs_err(device->fs_info, - "couldn't get super block page for bytenr %llu error %ld", - bytenr, PTR_ERR(folio)); + "couldn't get super block page for bytenr %llu error %pe", + bytenr, folio); atomic_inc(&device->sb_write_errors); continue; } @@ -4382,6 +4392,21 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) flush_workqueue(fs_info->fixup_workers); /* + * After we entered close_ctree() autodefrag could be running and before + * we parked the cleaner kthread, it dirtied folios of some inode. + * We don't want to leave any delalloc here, it may be flushed any time + * after this point and result in ordered extents that create delayed + * iputs after flushed the ordered extent queues further below, run + * delayed iputs and set BTRFS_FS_STATE_NO_DELAYED_IPUT. If we are + * mounted with flushoncommit, then btrfs_commit_super() called below + * will flush delalloc and wait for ordered extents but we end up + * getting delayed iputs than are never run. So flush delalloc and wait + * for ordered extents. + */ + btrfs_start_delalloc_roots(fs_info, LONG_MAX, false); + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); + + /* * Handle the error fs first, as it will flush and wait for all ordered * extents. This will generate delayed iputs, thus we want to handle * it first. @@ -4511,7 +4536,7 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) if (!btrfs_is_shutdown(fs_info)) { ret = btrfs_commit_super(fs_info); if (ret) - btrfs_err(fs_info, "commit super block returned %d", ret); + btrfs_err(fs_info, "commit super block returned %pe", ERR_PTR(ret)); } } @@ -4551,6 +4576,13 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) btrfs_free_fs_roots(fs_info); /* + * Drop metadata left stranded ahead of a zone write pointer while the + * endio workqueues are still up, so the final iput() of the btree inode + * below does not hang submitting a write that can no longer complete. + */ + btrfs_zoned_release_dirty_metadata(fs_info); + + /* * We must make sure there is not any read request to * submit after we stop all workers. */ @@ -4998,6 +5030,7 @@ static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info) btrfs_assert_delayed_root_empty(fs_info); btrfs_destroy_all_delalloc_inodes(fs_info); btrfs_drop_all_logs(fs_info); + btrfs_zoned_release_dirty_metadata(fs_info); btrfs_free_all_qgroup_pertrans(fs_info); mutex_unlock(&fs_info->transaction_kthread_mutex); diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 9185f8f02eeb..290508894f7c 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -15,6 +15,7 @@ struct block_device; struct super_block; struct extent_buffer; +struct btrfs_eb_prealloc; struct btrfs_device; struct btrfs_fs_devices; struct btrfs_fs_info; @@ -48,6 +49,7 @@ struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, struct btrfs_tree_parent_check *check); struct extent_buffer *btrfs_find_create_tree_block( struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, u64 bytenr, u64 owner_root, int level); int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info); diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c index c18ea5ef2974..d6df11f6088c 100644 --- a/fs/btrfs/extent-io-tree.c +++ b/fs/btrfs/extent-io-tree.c @@ -334,6 +334,21 @@ static inline struct extent_state *tree_search(struct extent_io_tree *tree, u64 return tree_search_for_insert(tree, offset, NULL, NULL); } +static void validate_extent_state(const struct extent_io_tree *tree, + const struct extent_state *state) +{ + u32 blocksize; + + if (tree->owner != IO_TREE_INODE_IO) + return; + + blocksize = btrfs_extent_io_tree_to_fs_info(tree)->sectorsize; + ASSERT(IS_ALIGNED(state->start, blocksize) && + IS_ALIGNED(state->end + 1, blocksize), + "unaligned extent state, blocksize=%u start=%llu end=%llu state=0x%x", + blocksize, state->start, state->end, state->state); +} + #define extent_io_tree_panic(tree, state, opname, err) \ btrfs_panic(btrfs_extent_io_tree_to_fs_info((tree)), (err), \ "extent io tree error on %s state start %llu end %llu", \ @@ -429,6 +444,8 @@ static struct extent_state *insert_state(struct extent_io_tree *tree, const u64 end = state->end + 1; const bool try_merge = !(bits & (EXTENT_LOCK_BITS | EXTENT_BOUNDARY)); + validate_extent_state(tree, state); + set_state_bits(tree, state, bits, changeset); node = &tree->state.rb_node; @@ -481,6 +498,8 @@ static void insert_state_fast(struct extent_io_tree *tree, struct rb_node *parent, unsigned bits, struct extent_changeset *changeset) { + validate_extent_state(tree, state); + set_state_bits(tree, state, bits, changeset); rb_link_node(&state->rb_node, parent, node); rb_insert_color(&state->rb_node, &tree->state); @@ -533,6 +552,8 @@ static int split_state(struct extent_io_tree *tree, struct extent_state *orig, } } + validate_extent_state(tree, orig); + validate_extent_state(tree, prealloc); rb_link_node(&prealloc->rb_node, parent, node); rb_insert_color(&prealloc->rb_node, &tree->state); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 624d76e0ca01..d6a4390ee34a 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -4757,7 +4757,7 @@ have_block_group: /* Checks */ ffe_ctl->search_start = round_up(ffe_ctl->found_offset, - fs_info->stripesize); + fs_info->sectorsize); /* move on to the next group */ if (ffe_ctl->search_start + ffe_ctl->num_bytes > @@ -5260,10 +5260,11 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, enum btrfs_lock_nesting nest) { struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_eb_prealloc pa = { 0 }; struct extent_buffer *buf; u64 lockdep_owner = owner; - buf = btrfs_find_create_tree_block(fs_info, bytenr, owner, level); + buf = btrfs_find_create_tree_block(fs_info, &pa, bytenr, owner, level); if (IS_ERR(buf)) return buf; @@ -5880,8 +5881,8 @@ static int maybe_drop_reference(struct btrfs_trans_handle *trans, struct btrfs_r ret = btrfs_qgroup_trace_subtree(trans, next, generation, level - 1); if (ret) { btrfs_err_rl(root->fs_info, -"error %d accounting shared subtree, quota is out of sync, rescan required", - ret); +"error %pe accounting shared subtree, quota is out of sync, rescan required", + ERR_PTR(ret)); } } @@ -5917,6 +5918,7 @@ static noinline int do_walk_down(struct btrfs_trans_handle *trans, struct walk_control *wc) { struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_eb_prealloc pa = { 0 }; u64 bytenr; u64 generation; u64 owner_root = 0; @@ -5939,7 +5941,7 @@ static noinline int do_walk_down(struct btrfs_trans_handle *trans, bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]); - next = btrfs_find_create_tree_block(fs_info, bytenr, btrfs_root_id(root), + next = btrfs_find_create_tree_block(fs_info, &pa, bytenr, btrfs_root_id(root), level - 1); if (IS_ERR(next)) return PTR_ERR(next); @@ -6096,8 +6098,8 @@ static noinline int walk_up_proc(struct btrfs_trans_handle *trans, ret = btrfs_qgroup_trace_leaf_items(trans, eb); if (ret) { btrfs_err_rl(fs_info, - "error %d accounting leaf items, quota is out of sync, rescan required", - ret); + "error %pe accounting leaf items, quota is out of sync, rescan required", + ERR_PTR(ret)); } } } @@ -6498,8 +6500,8 @@ out: ret = btrfs_qgroup_cleanup_dropped_subvolume(fs_info, rootid); if (ret < 0) btrfs_warn_rl(fs_info, - "failed to cleanup qgroup 0/%llu: %d", - rootid, ret); + "failed to cleanup qgroup 0/%llu: %pe", + rootid, ERR_PTR(ret)); ret = 0; } /* @@ -6914,8 +6916,8 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) if (bg_failed) btrfs_warn(fs_info, - "failed to trim %llu block group(s), first error %d", - bg_failed, bg_ret); + "failed to trim %llu block group(s), first error %pe", + bg_failed, ERR_PTR(bg_ret)); if (ret == -ERESTARTSYS || ret == -EINTR) return ret; @@ -6925,8 +6927,8 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) if (dev_failed) btrfs_warn(fs_info, - "failed to trim %llu device(s), first error %d", - dev_failed, dev_ret); + "failed to trim %llu device(s), first error %pe", + dev_failed, ERR_PTR(dev_ret)); range->len = trimmed; if (ret == -ERESTARTSYS || ret == -EINTR) return ret; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index f032f0858f40..d7600e5fa3d9 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -6,6 +6,7 @@ #include <linux/mm.h> #include <linux/pagemap.h> #include <linux/page-flags.h> +#include <linux/rmap.h> #include <linux/sched/mm.h> #include <linux/spinlock.h> #include <linux/blkdev.h> @@ -299,6 +300,25 @@ static noinline void unlock_delalloc_folio(const struct inode *inode, PAGE_UNLOCK); } +#ifdef CONFIG_BTRFS_DEBUG +/* + * Writeback must write-protect a folio when locking it for IO, before + * anything consumes its data (zeroing, inline copy, compression, + * checksumming). If this fails, then an mmap writer would be able to + * modify the data concurrently while we need it to be stable. + */ +void btrfs_check_folio_write_protected(struct folio *folio) +{ + if (folio_mkclean(folio)) { + const struct btrfs_inode *inode = BTRFS_I(folio->mapping->host); + + DEBUG_WARN("writable mmap PTEs, root %llu ino %llu pos %llu order %u", + btrfs_root_id(inode->root), btrfs_ino(inode), folio_pos(folio), + folio_order(folio)); + } +} +#endif + static noinline int lock_delalloc_folios(struct inode *inode, struct folio *locked_folio, u64 start, u64 end) @@ -332,6 +352,8 @@ static noinline int lock_delalloc_folios(struct inode *inode, folio_unlock(folio); goto out; } + /* Locked for writeback; revoke writable mmap PTEs before using the data. */ + folio_mkclean(folio); range_start = max_t(u64, folio_pos(folio), start); range_len = min_t(u64, folio_next_pos(folio), end + 1) - range_start; btrfs_folio_set_lock(fs_info, folio, range_start, range_len); @@ -1370,6 +1392,22 @@ again: } } +static void assert_folio_range(const struct btrfs_inode *inode, + u64 start, u64 end) +{ + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* + * For btrfs page cache, a folio always contains at least one block, + * so the range should always be block size aligned. + */ + ASSERT(IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize), + "blocksize=%u root=%lld ino=%llu start=%llu end=%llu mapping min order=%u", + blocksize, btrfs_root_id(inode->root), btrfs_ino(inode), + start, end, + mapping_min_folio_order(inode->vfs_inode.i_mapping)); +} + int btrfs_read_folio(struct file *file, struct folio *folio) { struct inode *vfs_inode = folio->mapping->host; @@ -1385,6 +1423,7 @@ int btrfs_read_folio(struct file *file, struct folio *folio) struct fsverity_info *vi = NULL; int ret; + assert_folio_range(inode, start, end); lock_extents_for_read(inode, start, end, &cached_state); if (folio_pos(folio) < i_size_read(vfs_inode)) vi = fsverity_get_info(vfs_inode); @@ -1676,13 +1715,13 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, last_finished_delalloc_end = found_start + found_len; if (unlikely(ret < 0)) btrfs_err_rl(fs_info, -"failed to run delalloc range, root=%lld ino=%llu folio=%llu submit_bitmap=%*pbl start=%llu len=%u: %d", +"failed to run delalloc range, root=%lld ino=%llu folio=%llu submit_bitmap=%*pbl start=%llu len=%u: %pe", btrfs_root_id(inode->root), btrfs_ino(inode), folio_pos(folio), blocks_per_folio, bio_ctrl->submit_bitmap, - found_start, found_len, ret); + found_start, found_len, ERR_PTR(ret)); } else { /* * We've hit an error during previous delalloc range, @@ -1892,6 +1931,14 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, ASSERT(start >= folio_start, "start=%llu folio_start=%llu", start, folio_start); ASSERT(end <= folio_end, "start=%llu len=%u folio_start=%llu folio_size=%zu", start, len, folio_start, folio_size(folio)); + assert_folio_range(inode, folio_start, folio_end - 1); + + /* + * We are about to checksum and write out the data, so it must not be + * mmap writeable, or we could corrupt the data and end up with invalid + * checksums. + */ + btrfs_check_folio_write_protected(folio); /* Truncate the submit bitmap to the current range. */ if (start > folio_start) @@ -2052,10 +2099,10 @@ static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl return 0; if (unlikely(ret < 0)) btrfs_err_rl(fs_info, -"failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %d", +"failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %pe", btrfs_root_id(inode->root), btrfs_ino(inode), folio_pos(folio), blocks_per_folio, - bio_ctrl->submit_bitmap, ret); + bio_ctrl->submit_bitmap, ERR_PTR(ret)); bio_ctrl->wbc->nr_to_write--; @@ -2350,14 +2397,17 @@ static struct extent_buffer *find_extent_buffer_nolock( static void end_bbio_meta_write(struct btrfs_bio *bbio) { struct extent_buffer *eb = bbio->private; - struct folio_iter fi; if (bbio->bio.bi_status != BLK_STS_OK) set_btree_ioerr(eb); - bio_for_each_folio_all(fi, &bbio->bio) { - btrfs_meta_folio_clear_writeback(fi.folio, eb); - } + /* + * Clear writeback on the buffer's own folios. The bio may carry the + * shared zero page instead (EXTENT_BUFFER_ZONED_ZEROOUT), so iterate + * the extent buffer folios rather than the bio folios. + */ + for (int i = 0; i < num_extent_folios(eb); i++) + btrfs_meta_folio_clear_writeback(eb->folios[i], eb); buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK); clear_and_wake_up_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); @@ -2398,7 +2448,8 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, struct btrfs_fs_info *fs_info = eb->fs_info; struct btrfs_bio *bbio; - prepare_eb_write(eb); + if (!test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) + prepare_eb_write(eb); bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc), @@ -2418,8 +2469,21 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, btrfs_meta_folio_set_writeback(folio, eb); if (!folio_test_dirty(folio)) wbc->nr_to_write -= folio_nr_pages(folio); - bio_add_folio_nofail(&bbio->bio, folio, range_len, - offset_in_folio(folio, range_start)); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { + u32 off = 0; + + while (off < range_len) { + u32 add = min_t(u32, PAGE_SIZE, range_len - off); + + bio_add_folio_nofail(&bbio->bio, + page_folio(ZERO_PAGE(0)), + add, 0); + off += add; + } + } else { + bio_add_folio_nofail(&bbio->bio, folio, range_len, + offset_in_folio(folio, range_start)); + } wbc_account_cgroup_owner(wbc, folio, range_len); folio_unlock(folio); } @@ -2467,6 +2531,76 @@ void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, } } +static int write_meta_extent_buffer(struct btrfs_eb_write_context *ctx, + struct writeback_control *wbc) +{ + struct extent_buffer *eb = ctx->eb; + int ret; + + ret = btrfs_check_meta_write_pointer(eb->fs_info, ctx); + if (ret) + return ret; + + if (!lock_extent_buffer_for_io(eb, wbc)) + return 0; + + /* Implies write in zoned mode. */ + if (ctx->zoned_bg) { + /* Mark the last eb in the block group. */ + btrfs_schedule_zone_finish_bg(ctx->zoned_bg, eb); + ctx->zoned_bg->meta_write_pointer += eb->len; + } + write_one_eb(eb, wbc); + return 0; +} + +/* + * On a zoned filesystem, write out the currently dirty metadata extent buffers + * of @bg. Used to flush the active metadata/system block group before the + * ascending-address walk in btree_writepages(), so that walk can pivot the + * active block group away (finishing it) instead of aborting the commit; see + * the caller for details. + */ +static void flush_active_meta_bg(struct address_space *mapping, + struct writeback_control *wbc, + struct btrfs_eb_write_context *ctx, + struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); + unsigned long index = bg->start >> fs_info->nodesize_bits; + unsigned long end = (btrfs_block_group_end(bg) - 1) >> fs_info->nodesize_bits; + struct eb_batch batch; + unsigned int nr_ebs; + + ASSERT(btrfs_is_zoned(fs_info)); + lockdep_assert_held(&fs_info->zoned_meta_io_lock); + + eb_batch_init(&batch); + while (index <= end && + (nr_ebs = buffer_tree_get_ebs_tag(fs_info, &index, end, + PAGECACHE_TAG_DIRTY, &batch))) { + struct extent_buffer *eb; + + while ((eb = eb_batch_next(&batch)) != NULL) { + ctx->eb = eb; + + /* + * If the eb is behind the write pointer (-EBUSY, e.g. + * already being written by someone else) skip it and + * carry on. Only a hole at the write pointer (-EAGAIN) + * stops the flush. The main walk in btree_writepages() + * then deals with it. + */ + if (write_meta_extent_buffer(ctx, wbc) == -EAGAIN) { + eb_batch_release(&batch); + return; + } + } + eb_batch_release(&batch); + cond_resched(); + } +} + int btree_writepages(struct address_space *mapping, struct writeback_control *wbc) { struct btrfs_eb_write_context ctx = { .wbc = wbc }; @@ -2502,6 +2636,22 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb else tag = PAGECACHE_TAG_DIRTY; btrfs_zoned_meta_io_lock(fs_info); + + /* + * On a zoned filesystem, flush the currently active metadata/system + * block group(s) first, under this same lock, so the ascending-address + * walk below can pivot the active block group instead of aborting the + * transaction commit with -EAGAIN. + */ + if (btrfs_is_zoned(fs_info) && wbc->sync_mode == WB_SYNC_ALL && + !wbc->for_sync) { + if (fs_info->active_meta_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_meta_bg); + if (fs_info->active_system_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_system_bg); + } retry: if (wbc->sync_mode == WB_SYNC_ALL) buffer_tree_tag_for_writeback(fs_info, index, end); @@ -2512,28 +2662,13 @@ retry: while ((eb = eb_batch_next(&batch)) != NULL) { ctx.eb = eb; - ret = btrfs_check_meta_write_pointer(eb->fs_info, &ctx); - if (ret) { - if (ret == -EBUSY) - ret = 0; - - if (ret) { - done = true; - break; - } - continue; - } - - if (!lock_extent_buffer_for_io(eb, wbc)) - continue; - - /* Implies write in zoned mode. */ - if (ctx.zoned_bg) { - /* Mark the last eb in the block group. */ - btrfs_schedule_zone_finish_bg(ctx.zoned_bg, eb); - ctx.zoned_bg->meta_write_pointer += eb->len; + ret = write_meta_extent_buffer(&ctx, wbc); + if (ret == -EBUSY) { + ret = 0; + } else if (ret) { + done = true; + break; } - write_one_eb(eb, wbc); } nr_to_write_done = (wbc->nr_to_write <= 0); eb_batch_release(&batch); @@ -2703,6 +2838,8 @@ retry: continue; } + /* Locked for writeback; revoke writable mmap PTEs before using the data. */ + folio_mkclean(folio); ret = extent_writepage(folio, bio_ctrl); if (ret < 0) { done = true; @@ -2857,13 +2994,25 @@ void btrfs_readahead(struct readahead_control *rac) struct extent_map *em_cached = NULL; struct fsverity_info *vi = NULL; + assert_folio_range(inode, start, end); lock_extents_for_read(inode, start, end, &cached_state); + /* We don't use cached state for a bulk unlock, just free it. */ + btrfs_free_extent_state(cached_state); if (start < i_size_read(vfs_inode)) vi = fsverity_get_info(vfs_inode); - while ((folio = readahead_folio(rac)) != NULL) - btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi); + while ((folio = readahead_folio(rac)) != NULL) { + /* + * Read start and end before btrfs_do_readpage(). It unlocks the + * folio, so our reference might not be valid after. + */ + const u64 folio_start = folio_pos(folio); + const u64 folio_end = folio_start + folio_size(folio) - 1; - btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state); + btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi); + /* Only unlock the range we locked, even if readahead expands. */ + if (folio_start >= start && folio_end <= end) + btrfs_unlock_extent(&inode->io_tree, folio_start, folio_end, NULL); + } if (em_cached) btrfs_free_extent_map(em_cached); @@ -3099,46 +3248,70 @@ static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) } /* + * Claim a slot to track an extent buffer in, evicting the coldest tracked buffer + * when the array is full. + * + * Slots fill in order until the array is full. After that a CLOCK (second + * chance) scan advances the hand, clearing one reference bit per step, until + * it lands on an unreferenced slot whose buffer is evicted. Clearing a bit per + * step bounds the scan to BTRFS_INHIBITED_EBS_SLOTS iterations. + */ +static int btrfs_inhibit_claim_slot(struct btrfs_trans_handle *trans) +{ + int slot; + + if (trans->nr_inhibited_ebs < BTRFS_INHIBITED_EBS_SLOTS) + return trans->nr_inhibited_ebs++; + + while (trans->inhibited_ebs_referenced & (1U << trans->inhibited_ebs_hand)) { + trans->inhibited_ebs_referenced &= ~(1U << trans->inhibited_ebs_hand); + trans->inhibited_ebs_hand = + (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + } + slot = trans->inhibited_ebs_hand; + trans->inhibited_ebs_hand = (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + + atomic_dec(&trans->inhibited_ebs[slot]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[slot]); + + return slot; +} + +/* * Inhibit writeback on buffer during transaction. * * @trans: transaction handle that will own the inhibitor * @eb: extent buffer to inhibit writeback on * - * Attempt to track this extent buffer in the transaction's inhibited set. If - * memory allocation fails, the buffer is simply not tracked. It may be written - * back and need re-COW, which is the original behavior. This is acceptable - * since inhibiting writeback is an optimization. + * Attempt to track this extent buffer in the transaction's inhibited set. When + * the set is full the coldest tracked buffer is evicted instead. An untracked + * buffer may be written back and need re-COW, which is the original behavior. + * This is acceptable since inhibiting writeback is an optimization. */ void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_buffer *eb) { - unsigned long index = eb->start >> trans->fs_info->nodesize_bits; - void *old; + int slot; lockdep_assert_held(&eb->lock); - /* Check if already inhibited by this handle. */ - old = xa_load(&trans->writeback_inhibited_ebs, index); - if (old == eb) - return; - - /* Take reference for the xarray entry. */ - refcount_inc(&eb->refs); - old = xa_store(&trans->writeback_inhibited_ebs, index, eb, GFP_NOFS); - if (xa_is_err(old)) { - /* Allocation failed, just skip inhibiting this buffer. */ - free_extent_buffer(eb); - return; + /* Already tracked: set its reference bit (second chance) and return. */ + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + if (trans->inhibited_ebs[i] == eb) { + trans->inhibited_ebs_referenced |= 1U << i; + return; + } } - /* Handle replacement of different eb at same index. */ - if (old && old != eb) { - struct extent_buffer *old_eb = old; - - atomic_dec(&old_eb->writeback_inhibitors); - free_extent_buffer(old_eb); - } + slot = btrfs_inhibit_claim_slot(trans); + /* + * Pin the eb while the array holds a raw pointer to it; the counter is + * what lock_extent_buffer_for_io() checks. + */ + refcount_inc(&eb->refs); atomic_inc(&eb->writeback_inhibitors); + trans->inhibited_ebs[slot] = eb; + trans->inhibited_ebs_referenced |= 1U << slot; } /* @@ -3146,22 +3319,18 @@ void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_ */ void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans) { - struct extent_buffer *eb; - unsigned long index; - - xa_for_each(&trans->writeback_inhibited_ebs, index, eb) { - atomic_dec(&eb->writeback_inhibitors); - free_extent_buffer(eb); + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + atomic_dec(&trans->inhibited_ebs[i]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[i]); } - xa_destroy(&trans->writeback_inhibited_ebs); + trans->nr_inhibited_ebs = 0; + trans->inhibited_ebs_referenced = 0; + trans->inhibited_ebs_hand = 0; } -static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info, - u64 start) +static void init_extent_buffer(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, u64 start) { - struct extent_buffer *eb = NULL; - - eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL); eb->start = start; eb->len = fs_info->nodesize; eb->fs_info = fs_info; @@ -3174,7 +3343,15 @@ static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info refcount_set(&eb->refs, 1); ASSERT(eb->len <= BTRFS_MAX_METADATA_BLOCKSIZE); +} + +static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start) +{ + struct extent_buffer *eb; + eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS | __GFP_NOFAIL); + init_extent_buffer(fs_info, eb, start); return eb; } @@ -3471,7 +3648,7 @@ static bool check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start) * The caller needs to free the existing folios and retry using the same order. */ static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i, - struct btrfs_folio_state *prealloc, + struct btrfs_eb_prealloc *pa, struct extent_buffer **found_eb_ret) { @@ -3493,6 +3670,7 @@ retry: if (!ret) goto finish; + /* ret == -EEXIST: a folio already lives at this index. */ existing_folio = filemap_lock_folio(mapping, index + i); /* The page cache only exists for a very short time, just retry. */ if (IS_ERR(existing_folio)) @@ -3501,7 +3679,27 @@ retry: /* For now, we should only have single-page folios for btree inode. */ ASSERT(folio_nr_pages(existing_folio) == 1); + /* + * TODO: Special handling for a corner case where the order of + * folios mismatch between the new eb and filemap. + * + * This happens when: + * + * - the new eb is using higher order folio + * + * - the filemap is still using 0-order folios for the range + * This can happen at the previous eb allocation, and we don't + * have higher order folio for the call. + * + * - the existing eb has already been freed + * + * In this case, we have to free the existing folios first, and + * re-allocate using the same order. + * Thankfully this is not going to happen yet, as we're still + * using 0-order folios. + */ if (folio_size(existing_folio) != eb->folio_size) { + DEBUG_WARN("folio order mismatch between new eb and filemap"); folio_unlock(existing_folio); folio_put(existing_folio); return -EAGAIN; @@ -3532,8 +3730,10 @@ finish: eb->folio_size = folio_size(eb->folios[i]); eb->folio_shift = folio_shift(eb->folios[i]); /* Should not fail, as we have preallocated the memory. */ - ret = attach_extent_buffer_folio(eb, eb->folios[i], prealloc); + ret = attach_extent_buffer_folio(eb, eb->folios[i], pa->bfs); ASSERT(!ret); + /* The subpage state, if any, is now attached to the folio or freed. */ + pa->bfs = NULL; /* * To inform we have an extra eb under allocation, so that * detach_extent_buffer_page() won't release the folio private when the @@ -3548,13 +3748,104 @@ finish: return 0; } +/* + * Allocate the extent_buffer, its folios, and btrfs_folio_state, if needed. + * + * @pa: The holder struct to do the allocation in. + * @nowait: Whether to do a speculative GFP_NOWAIT allocation while holding locks. + * + * Return 0 on success and a negative errno otherwise. On failure, pa->eb/bfs + * will be NULL. If @nowait=true, then on ENOMEM, mark @pa->needs_prealloc and + * return -EAGAIN to signal the caller to unlock and retry. + */ +int btrfs_init_eb_prealloc(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, bool nowait) +{ + gfp_t gfp = nowait ? GFP_NOWAIT : GFP_NOFS | __GFP_NOFAIL; + int ret; + + ASSERT(!pa->eb, "unexpected non-null eb: %p", pa->eb); + ASSERT(!pa->bfs, "unexpected non-null bfs: %p", pa->bfs); + pa->needs_prealloc = false; + + pa->eb = kmem_cache_zalloc(extent_buffer_cache, gfp); + if (!pa->eb) { + ret = -ENOMEM; + goto out; + } + /* alloc_eb_folio_array() needs len; init_extent_buffer() sets it again later. */ + pa->eb->len = fs_info->nodesize; + + /* + * Preallocate folio private for subpage case, so that we won't + * allocate memory with i_private_lock nor page lock hold. + * + * The memory will be freed by attach_extent_buffer_page() or freed + * manually if we exit earlier. + */ + if (btrfs_meta_is_subpage(fs_info)) { + pa->bfs = btrfs_alloc_folio_state(fs_info, PAGE_SIZE, + BTRFS_SUBPAGE_METADATA, gfp); + if (IS_ERR(pa->bfs)) { + ret = PTR_ERR(pa->bfs); + pa->bfs = NULL; + goto free_eb; + } + } + + /* + * Allocate pages without attaching them. Caller is ultimately responsible + * for attaching the folios to the mapping with attach_eb_folio_to_filemap(). + */ + ret = alloc_eb_folio_array(pa->eb, gfp | __GFP_MOVABLE); + if (ret < 0) + goto free_bfs; + + return 0; + +free_bfs: + btrfs_free_folio_state(pa->bfs); + pa->bfs = NULL; +free_eb: + kmem_cache_free(extent_buffer_cache, pa->eb); + pa->eb = NULL; +out: + if (nowait && ret == -ENOMEM) { + pa->needs_prealloc = true; + ret = -EAGAIN; + } + return ret; +} + +/* + * Used to cleanup a btrfs_eb_prealloc which had its contents allocated but + * folios not yet attached and eb/bfs consumed, and refs still 0. + * + * Safe to call on a fully used btrfs_eb_prealloc as the internal structs will + * be null once they are owned by the context using them. + */ +void btrfs_free_eb_prealloc(struct btrfs_eb_prealloc *pa) +{ + if (!pa->eb) + return; + + for (int i = 0; i < num_extent_pages(pa->eb); i++) { + if (pa->eb->folios[i]) + folio_put(pa->eb->folios[i]); + } + btrfs_free_folio_state(pa->bfs); + kmem_cache_free(extent_buffer_cache, pa->eb); + pa->eb = NULL; + pa->bfs = NULL; +} + struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, u64 start, u64 owner_root, int level) { int attached = 0; struct extent_buffer *eb; struct extent_buffer *existing_eb = NULL; - struct btrfs_folio_state *prealloc = NULL; u64 lockdep_owner = owner_root; bool page_contig = true; bool uptodate = true; @@ -3578,9 +3869,14 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, if (eb) return eb; - eb = __alloc_extent_buffer(fs_info, start); - if (!eb) - return ERR_PTR(-ENOMEM); + if (!pa->eb) { + ret = btrfs_init_eb_prealloc(fs_info, pa, pa->supports_nowait); + if (ret) + return ERR_PTR(ret); + } + eb = pa->eb; + pa->eb = NULL; + init_extent_buffer(fs_info, eb, start); /* * The reloc trees are just snapshots, so we need them to appear to be @@ -3591,66 +3887,18 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level); - /* - * Preallocate folio private for subpage case, so that we won't - * allocate memory with i_private_lock nor page lock hold. - * - * The memory will be freed by attach_extent_buffer_page() or freed - * manually if we exit earlier. - */ - if (btrfs_meta_is_subpage(fs_info)) { - prealloc = btrfs_alloc_folio_state(fs_info, PAGE_SIZE, BTRFS_SUBPAGE_METADATA); - if (IS_ERR(prealloc)) { - ret = PTR_ERR(prealloc); - goto out; - } - } - -reallocate: - /* - * Allocate all pages first. These will be attached to btree_inode->i_mapping - * below (added to LRU, served by btree_migrate_folio), so request - * __GFP_MOVABLE so the page allocator places them in MOVABLE pageblocks. - */ - ret = alloc_eb_folio_array(eb, GFP_NOFS | __GFP_NOFAIL | __GFP_MOVABLE); - if (ret < 0) { - btrfs_free_folio_state(prealloc); - goto out; - } - /* Attach all pages to the filemap. */ for (int i = 0; i < num_extent_folios(eb); i++) { struct folio *folio; - ret = attach_eb_folio_to_filemap(eb, i, prealloc, &existing_eb); + ret = attach_eb_folio_to_filemap(eb, i, pa, &existing_eb); if (ret > 0) { ASSERT(existing_eb); goto out; } - - /* - * TODO: Special handling for a corner case where the order of - * folios mismatch between the new eb and filemap. - * - * This happens when: - * - * - the new eb is using higher order folio - * - * - the filemap is still using 0-order folios for the range - * This can happen at the previous eb allocation, and we don't - * have higher order folio for the call. - * - * - the existing eb has already been freed - * - * In this case, we have to free the existing folios first, and - * re-allocate using the same order. - * Thankfully this is not going to happen yet, as we're still - * using 0-order folios. - */ - if (unlikely(ret == -EAGAIN)) { - DEBUG_WARN("folio order mismatch between new eb and filemap"); - goto reallocate; - } + /* -EAGAIN: folio order mismatch, unreachable with 0-order folios. */ + if (ret < 0) + goto out; attached++; /* @@ -3727,6 +3975,10 @@ again: out: WARN_ON(!refcount_dec_and_test(&eb->refs)); + /* Attach hands off pa->bfs; free it if we bailed first. */ + btrfs_free_folio_state(pa->bfs); + pa->bfs = NULL; + /* * Any attached folios need to be detached before we unlock them. This * is because when we're inserting our new folios into the mapping, and @@ -3811,12 +4063,31 @@ static int release_extent_buffer(struct extent_buffer *eb) return 0; } -void free_extent_buffer(struct extent_buffer *eb) +static void clear_extent_buffer_reading(struct extent_buffer *eb) +{ + clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags); +} + +static void free_extent_buffer_clear_reading(struct extent_buffer *eb, + bool clear_reading) { int refs; + if (!eb) return; + /* + * We want to clear EXTENT_BUFFER_READING flag and decrease refs + * in the same critical section. + * This will make sure invalidate_and_check_btree_folios() won't + * see an eb with EXTENT_BUFFER_READING cleared but refs not yet + * decreased. + */ + if (clear_reading) { + spin_lock(&eb->refs_lock); + clear_extent_buffer_reading(eb); + } + refs = refcount_read(&eb->refs); while (1) { if (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)) { @@ -3827,11 +4098,16 @@ void free_extent_buffer(struct extent_buffer *eb) } /* Optimization to avoid locking eb->refs_lock. */ - if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1)) + if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1)) { + if (clear_reading) + spin_unlock(&eb->refs_lock); return; + } } - spin_lock(&eb->refs_lock); + if (!clear_reading) + spin_lock(&eb->refs_lock); + if (refcount_read(&eb->refs) == 2 && test_bit(EXTENT_BUFFER_STALE, &eb->bflags) && !extent_buffer_under_io(eb) && @@ -3845,6 +4121,11 @@ void free_extent_buffer(struct extent_buffer *eb) release_extent_buffer(eb); } +void free_extent_buffer(struct extent_buffer *eb) +{ + return free_extent_buffer_clear_reading(eb, false); +} + void free_extent_buffer_stale(struct extent_buffer *eb) { if (!eb) @@ -3859,6 +4140,32 @@ void free_extent_buffer_stale(struct extent_buffer *eb) release_extent_buffer(eb); } +static void clear_extent_buffer_dirty(struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + + if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) + return; + + buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY); + percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len, + fs_info->dirty_metadata_batch); + + for (int i = 0; i < num_extent_folios(eb); i++) { + struct folio *folio = eb->folios[i]; + bool last; + + if (!folio_test_dirty(folio)) + continue; + folio_lock(folio); + last = btrfs_meta_folio_clear_and_test_dirty(folio, eb); + if (last) + btrfs_clear_folio_dirty_tag(folio); + folio_unlock(folio); + } + WARN_ON(refcount_read(&eb->refs) == 0); +} + void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, struct extent_buffer *eb) { @@ -3883,26 +4190,42 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, return; } - if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) - return; + clear_extent_buffer_dirty(eb); +} - buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY); - percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len, - fs_info->dirty_metadata_batch); +/* + * On a zoned filesystem a freed tree block is kept dirty and flagged as + * EXTENT_BUFFER_ZONED_ZEROOUT so a later writeback zeroes it out and advances + * the zone write pointer. Such buffers still dirty when the filesystem is torn + * down can no longer be written back and are stale; if left dirty they hang the + * final iput() of the btree inode. Drop their dirty state, and the deferred + * zero-out along with it. + */ +void btrfs_zoned_release_dirty_metadata(struct btrfs_fs_info *fs_info) +{ + struct eb_batch batch; + unsigned long index = 0; - for (int i = 0; i < num_extent_folios(eb); i++) { - struct folio *folio = eb->folios[i]; - bool last; + if (!btrfs_is_zoned(fs_info)) + return; - if (!folio_test_dirty(folio)) - continue; - folio_lock(folio); - last = btrfs_meta_folio_clear_and_test_dirty(folio, eb); - if (last) - btrfs_clear_folio_dirty_tag(folio); - folio_unlock(folio); + btrfs_zoned_meta_io_lock(fs_info); + eb_batch_init(&batch); + while (buffer_tree_get_ebs_tag(fs_info, &index, ULONG_MAX, + PAGECACHE_TAG_DIRTY, &batch)) { + struct extent_buffer *eb; + + while ((eb = eb_batch_next(&batch)) != NULL) { + btrfs_tree_lock(eb); + if (test_and_clear_bit(EXTENT_BUFFER_ZONED_ZEROOUT, + &eb->bflags)) + clear_extent_buffer_dirty(eb); + btrfs_tree_unlock(eb); + } + eb_batch_release(&batch); + cond_resched(); } - WARN_ON(refcount_read(&eb->refs) == 0); + btrfs_zoned_meta_io_unlock(fs_info); } void set_extent_buffer_dirty(struct extent_buffer *eb) @@ -3970,11 +4293,6 @@ void set_extent_buffer_uptodate(struct extent_buffer *eb) btrfs_meta_folio_set_uptodate(eb->folios[i], eb); } -static void clear_extent_buffer_reading(struct extent_buffer *eb) -{ - clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags); -} - static void end_bbio_meta_read(struct btrfs_bio *bbio) { struct extent_buffer *eb = bbio->private; @@ -3998,8 +4316,7 @@ static void end_bbio_meta_read(struct btrfs_bio *bbio) else clear_extent_buffer_uptodate(eb); - clear_extent_buffer_reading(eb); - free_extent_buffer(eb); + free_extent_buffer_clear_reading(eb, true); bio_put(&bbio->bio); } @@ -4758,6 +5075,8 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, .level = level, .transid = gen }; + /* Readahead is best effort so prefer to fail rather than block in reclaim. */ + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; struct extent_buffer *eb; int ret; @@ -4766,7 +5085,7 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, check.has_first_key = true; } - eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level); + eb = btrfs_find_create_tree_block(fs_info, &pa, bytenr, owner_root, level); if (IS_ERR(eb)) return; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 9896e15ddc40..d8dd2ae9ff9a 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -119,6 +119,25 @@ struct extent_buffer { #endif }; +/* + * Wrapper struct for managing preallocating an extent_buffer, its folios and a + * btrfs_folio_state if needed. + * + * Only used to mediate allocation, do not refer to the eb directly if not + * returned from a successful eb allocating API. + * + * The eb folios and bfs should generally not be fully attached, except briefly + * before they are NULLed in the struct after successful attachment. + */ +struct btrfs_eb_prealloc { + struct extent_buffer *eb; + struct btrfs_folio_state *bfs; + /* eb alloc may use GFP_NOWAIT; caller can drop locks and retry. */ + bool supports_nowait; + /* GFP_NOWAIT eb alloc failed; preallocate again and retry. */ + bool needs_prealloc; +}; + struct btrfs_eb_write_context { struct writeback_control *wbc; struct extent_buffer *eb; @@ -255,6 +274,11 @@ bool try_release_extent_mapping(struct folio *folio, gfp_t mask); int try_release_extent_buffer(struct folio *folio); int btrfs_read_folio(struct file *file, struct folio *folio); +#ifdef CONFIG_BTRFS_DEBUG +void btrfs_check_folio_write_protected(struct folio *folio); +#else +static inline void btrfs_check_folio_write_protected(struct folio *folio) { } +#endif void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio, u64 start, u64 end, struct writeback_control *wbc, bool pages_dirty); @@ -266,7 +290,11 @@ int set_folio_extent_mapped(struct folio *folio); void clear_folio_extent_mapped(struct folio *folio); struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, u64 start, u64 owner_root, int level); +int btrfs_init_eb_prealloc(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, bool nowait); +void btrfs_free_eb_prealloc(struct btrfs_eb_prealloc *pa); struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, u64 start); struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src); @@ -388,6 +416,7 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, u32 bits_to_clear, unsigned long page_ops); void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, struct extent_buffer *buf); +void btrfs_zoned_release_dirty_metadata(struct btrfs_fs_info *fs_info); static inline void btrfs_clear_folio_dirty_tag(struct folio *folio) { diff --git a/fs/btrfs/fiemap.c b/fs/btrfs/fiemap.c index 6263e837093e..7a2a97180099 100644 --- a/fs/btrfs/fiemap.c +++ b/fs/btrfs/fiemap.c @@ -641,7 +641,7 @@ static int extent_fiemap(struct btrfs_inode *inode, u64 prev_extent_end; u64 range_start; u64 range_end; - const u64 sectorsize = inode->root->fs_info->sectorsize; + const u32 sectorsize = inode->root->fs_info->sectorsize; bool stopped = false; int ret; @@ -660,7 +660,7 @@ restart: range_end = round_up(start + len, sectorsize); prev_extent_end = range_start; - btrfs_lock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_lock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end); if (ret < 0) @@ -840,7 +840,7 @@ check_eof_delalloc: } out_unlock: - btrfs_unlock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_unlock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); if (ret == BTRFS_FIEMAP_FLUSH_CACHE) { btrfs_release_path(path); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index a2a2df2df786..20e15dc30bfb 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -875,62 +875,56 @@ again: /* * Locks the extent and properly waits for data=ordered extents to finish - * before allowing the folios to be modified if need. + * before allowing the folios to be modified. * * Return: - * 1 - the extent is locked - * 0 - the extent is not locked, and everything is OK + * 0 - the extent is locked * -EAGAIN - need to prepare the folios again */ static noinline int -lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, - loff_t pos, size_t write_bytes, - u64 *lockstart, u64 *lockend, bool nowait, - struct extent_state **cached_state) +lock_and_cleanup_extent(struct btrfs_inode *inode, struct folio *folio, + loff_t pos, size_t write_bytes, + u64 *lockstart, u64 *lockend, bool nowait, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_ordered_extent *ordered; u64 start_pos; u64 last_pos; - int ret = 0; start_pos = round_down(pos, fs_info->sectorsize); last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1; - if (start_pos < inode->vfs_inode.i_size) { - struct btrfs_ordered_extent *ordered; - - if (nowait) { - if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, - last_pos, cached_state)) { - folio_unlock(folio); - folio_put(folio); - return -EAGAIN; - } - } else { - btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); - } - - ordered = btrfs_lookup_ordered_range(inode, start_pos, - last_pos - start_pos + 1); - if (ordered && - ordered->file_offset + ordered->num_bytes > start_pos && - ordered->file_offset <= last_pos) { - btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); + if (nowait) { + if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, + last_pos, cached_state)) { folio_unlock(folio); folio_put(folio); - btrfs_start_ordered_extent(ordered); - btrfs_put_ordered_extent(ordered); return -EAGAIN; } - if (ordered) - btrfs_put_ordered_extent(ordered); + } else { + btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + } - *lockstart = start_pos; - *lockend = last_pos; - ret = 1; + ordered = btrfs_lookup_ordered_range(inode, start_pos, + last_pos - start_pos + 1); + if (ordered && + ordered->file_offset + ordered->num_bytes > start_pos && + ordered->file_offset <= last_pos) { + btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + folio_unlock(folio); + folio_put(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + return -EAGAIN; } + if (ordered) + btrfs_put_ordered_extent(ordered); + + *lockstart = start_pos; + *lockend = last_pos; /* * We should be called after prepare_one_folio() which should have locked @@ -938,7 +932,7 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, */ WARN_ON(!folio_test_locked(folio)); - return ret; + return 0; } /* @@ -1195,7 +1189,6 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, const u64 reserved_start = round_down(start, fs_info->sectorsize); u64 reserved_len; struct folio *folio = NULL; - int extents_locked; u64 lockstart; u64 lockend; bool only_release_metadata = false; @@ -1253,18 +1246,16 @@ again: reserved_len = last_block - reserved_start; } - extents_locked = lock_and_cleanup_extent_if_need(inode, folio, start, - write_bytes, &lockstart, - &lockend, nowait, - &cached_state); - if (extents_locked < 0) { - if (!nowait && extents_locked == -EAGAIN) + ret = lock_and_cleanup_extent(inode, folio, start, write_bytes, + &lockstart, &lockend, nowait, &cached_state); + if (ret < 0) { + if (!nowait) goto again; btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); - return extents_locked; + return ret; } copied = copy_folio_from_iter_atomic(folio, offset_in_folio(folio, start), @@ -1288,11 +1279,8 @@ again: /* No copied bytes, unlock, release reserved space and exit. */ if (copied == 0) { - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, - &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, + &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); @@ -1311,17 +1299,7 @@ again: ret = btrfs_dirty_folio(inode, folio, start, copied, &cached_state, only_release_metadata); - /* - * If we have not locked the extent range, because the range's start - * offset is >= i_size, we might still have a non-NULL cached extent - * state, acquired while marking the extent range as delalloc through - * btrfs_dirty_page(). Therefore free any possible cached extent state - * to avoid a memory leak. - */ - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); if (ret) { @@ -1595,8 +1573,6 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) else btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP); - atomic_inc(&root->log_batch); - /* * Before we acquired the inode's lock and the mmap lock, someone may * have dirtied more pages in the target range. We need to make sure @@ -1679,8 +1655,6 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) if (ret) goto out_release_extents; - atomic_inc(&root->log_batch); - if (skip_inode_logging(&ctx)) { /* * We've had everything committed since the last time we were @@ -2683,8 +2657,8 @@ static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len) lockstart = round_up(offset, fs_info->sectorsize); lockend = round_down(offset + len, fs_info->sectorsize) - 1; - same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset)) - == (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)); + same_block = (offset >> fs_info->sectorsize_bits) == + ((offset + len - 1) >> fs_info->sectorsize_bits); /* * Only do this if we are in the same block and we aren't doing the * entire block. @@ -2888,7 +2862,7 @@ enum { static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode, u64 offset) { - const u64 sectorsize = inode->root->fs_info->sectorsize; + const u32 sectorsize = inode->root->fs_info->sectorsize; struct extent_map *em; int ret; @@ -2918,7 +2892,7 @@ static int btrfs_zero_range(struct inode *inode, struct extent_changeset *data_reserved = NULL; int ret; u64 alloc_hint = 0; - const u64 sectorsize = fs_info->sectorsize; + const u32 sectorsize = fs_info->sectorsize; const u64 orig_start = offset; const u64 orig_end = offset + len - 1; u64 alloc_start = round_down(offset, sectorsize); @@ -2967,8 +2941,8 @@ static int btrfs_zero_range(struct inode *inode, } btrfs_free_extent_map(em); - if (BTRFS_BYTES_TO_BLKS(fs_info, offset) == - BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)) { + if ((offset >> fs_info->sectorsize_bits) == + ((offset + len - 1) >> fs_info->sectorsize_bits)) { em = btrfs_get_extent(BTRFS_I(inode), NULL, alloc_start, sectorsize); if (IS_ERR(em)) { ret = PTR_ERR(em); diff --git a/fs/btrfs/fs.c b/fs/btrfs/fs.c index dcf12979af33..de160d29dde8 100644 --- a/fs/btrfs/fs.c +++ b/fs/btrfs/fs.c @@ -127,20 +127,9 @@ void btrfs_csum_final(struct btrfs_csum_ctx *ctx, u8 *out) } /* - * We support the following block sizes for all systems: - * - * - 4K - * This is the most common block size. For PAGE SIZE > 4K cases the subpage - * mode is used. - * - * - PAGE_SIZE - * The straightforward block size to support. - * - * And extra support for the following block sizes based on the kernel config: - * - * - MIN_BLOCKSIZE - * This is either 4K (regular builds) or 2K (debug builds) - * This allows testing subpage routines on x86_64. + * For regular builds, any block size <= page size is supported. + * For experimental builds, any block size between BTRFS_MIN_BLOCKSIZE + * and BTRFS_MAX_BLOCKSIZE (inclusive) is supported. */ bool __attribute_const__ btrfs_supported_blocksize(u32 blocksize) { @@ -148,7 +137,7 @@ bool __attribute_const__ btrfs_supported_blocksize(u32 blocksize) ASSERT(is_power_of_2(blocksize) && blocksize >= BTRFS_MIN_BLOCKSIZE && blocksize <= BTRFS_MAX_BLOCKSIZE); - if (blocksize == PAGE_SIZE || blocksize == SZ_4K || blocksize == BTRFS_MIN_BLOCKSIZE) + if (blocksize <= PAGE_SIZE) return true; #ifdef CONFIG_BTRFS_EXPERIMENTAL /* diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index f7f343fbe732..10e15a319b93 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -289,7 +289,8 @@ enum { BTRFS_MOUNT_IGNOREBADROOTS | \ BTRFS_MOUNT_IGNOREDATACSUMS | \ BTRFS_MOUNT_IGNOREMETACSUMS | \ - BTRFS_MOUNT_IGNORESUPERFLAGS) + BTRFS_MOUNT_IGNORESUPERFLAGS | \ + BTRFS_MOUNT_USEBACKUPROOT) /* * Compat flags that we support. If any incompat flags are set other than the @@ -890,7 +891,6 @@ struct btrfs_fs_info { u32 sectorsize_bits; u32 block_min_order; u32 block_max_order; - u32 stripesize; u32 writeback_bio_size; u32 csum_size; u32 csums_per_leaf; @@ -1060,8 +1060,6 @@ static inline u64 btrfs_calc_metadata_size(const struct btrfs_fs_info *fs_info, #define BTRFS_MAX_EXTENT_ITEM_SIZE(r) ((BTRFS_LEAF_DATA_SIZE(r->fs_info) >> 4) - \ sizeof(struct btrfs_item)) -#define BTRFS_BYTES_TO_BLKS(fs_info, bytes) ((bytes) >> (fs_info)->sectorsize_bits) - static inline bool btrfs_is_zoned(const struct btrfs_fs_info *fs_info) { return IS_ENABLED(CONFIG_BLK_DEV_ZONED) && fs_info->zone_size > 0; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 7e82dad952ec..3c10a0ef0002 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -250,8 +250,8 @@ static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off ret = extent_from_logical(fs_info, logical, &path, &found_key, &flags); if (ret < 0) { - btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %d", - logical, ret); + btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %pe", + logical, ERR_PTR(ret)); return; } eb = path.nodes[0]; @@ -775,19 +775,28 @@ static inline void inode_should_defrag(struct btrfs_inode *inode, static int extent_range_clear_dirty_for_io(struct btrfs_inode *inode, u64 start, u64 end) { + pgoff_t index = start >> PAGE_SHIFT; const pgoff_t end_index = end >> PAGE_SHIFT; struct folio *folio; int ret = 0; - for (pgoff_t index = start >> PAGE_SHIFT; index <= end_index; index++) { + while (index <= end_index) { folio = filemap_get_folio(inode->vfs_inode.i_mapping, index); if (IS_ERR(folio)) { if (!ret) ret = PTR_ERR(folio); + index++; continue; } + /* + * We are about to compress the folio, so it must not be mmap + * writeable or we could corrupt the data as we attempt to + * compress it. + */ + btrfs_check_folio_write_protected(folio); btrfs_folio_clamp_clear_dirty(inode->root->fs_info, folio, start, end + 1 - start); + index = folio_next_index(folio); folio_put(folio); } return ret; @@ -860,7 +869,7 @@ static void compress_file_range(struct btrfs_work *work) struct btrfs_inode *inode = async_chunk->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; struct compressed_bio *cb = NULL; - u64 blocksize = fs_info->sectorsize; + const u32 blocksize = fs_info->sectorsize; u64 start = async_chunk->start; u64 end = async_chunk->end; u64 actual_end; @@ -877,11 +886,6 @@ static void compress_file_range(struct btrfs_work *work) inode_should_defrag(inode, start, end, end - start + 1, SZ_16K); - /* - * We need to call clear_page_dirty_for_io on each page in the range. - * Otherwise applications with the file mmap'd can wander in and change - * the page contents while we are compressing them. - */ ret = extent_range_clear_dirty_for_io(inode, start, end); /* @@ -1015,9 +1019,10 @@ static void submit_uncompressed_range(struct btrfs_inode *inode, btrfs_folio_end_lock(inode->root->fs_info, locked_folio, start, async_extent->ram_size); btrfs_err_rl(inode->root->fs_info, - "%s failed, root=%llu inode=%llu start=%llu len=%llu: %d", + "%s failed, root=%llu inode=%llu start=%llu len=%llu: %pe", __func__, btrfs_root_id(inode->root), - btrfs_ino(inode), start, async_extent->ram_size, ret); + btrfs_ino(inode), start, async_extent->ram_size, + ERR_PTR(ret)); } } @@ -1504,10 +1509,10 @@ out_unlock: end - start - cur_alloc_size + 1, NULL); } btrfs_err(fs_info, -"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%u: %d", +"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%u: %pe", __func__, btrfs_root_id(inode->root), btrfs_ino(inode), orig_start, end + 1 - orig_start, - start, cur_alloc_size, ret); + start, cur_alloc_size, ERR_PTR(ret)); return ret; } @@ -1958,9 +1963,9 @@ error: PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); btrfs_err(inode->root->fs_info, - "%s failed, root=%lld inode=%llu start=%llu len=%llu: %d", + "%s failed, root=%lld inode=%llu start=%llu len=%llu: %pe", __func__, btrfs_root_id(inode->root), btrfs_ino(inode), - file_pos, len, ret); + file_pos, len, ERR_PTR(ret)); return ret; } @@ -2281,10 +2286,10 @@ error: } btrfs_free_path(path); btrfs_err(fs_info, -"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu oe_cleanup=%llu oe_cleanup_len=%llu untouched_start=%llu untouched_len=%llu: %d", +"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu oe_cleanup=%llu oe_cleanup_len=%llu untouched_start=%llu untouched_len=%llu: %pe", __func__, btrfs_root_id(inode->root), btrfs_ino(inode), start, end + 1 - start, cur_offset, oe_cleanup_start, oe_cleanup_len, - untouched_start, untouched_len, ret); + untouched_start, untouched_len, ERR_PTR(ret)); return ret; } @@ -2317,6 +2322,13 @@ static int run_delalloc_inline(struct btrfs_inode *inode, struct folio *locked_f int ret; ASSERT(folio_pos(locked_folio) == 0); + /* + * If an mmap writer could modify the folio while we copy it into an + * inline extent we might see only part of their modification then + * wrongly mark it clean again after copying, losing that write. So the + * folio must be write protected here. + */ + btrfs_check_folio_write_protected(locked_folio); if (btrfs_inode_can_compress(inode) && inode_need_compress(inode, 0, blocksize, true)) { @@ -3039,7 +3051,7 @@ static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, u64 qgroup_reserved) { struct btrfs_root *root = inode->root; - const u64 sectorsize = root->fs_info->sectorsize; + const u32 sectorsize = root->fs_info->sectorsize; BTRFS_PATH_AUTO_FREE(path); struct extent_buffer *leaf; struct btrfs_key ins; @@ -3896,7 +3908,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) out: if (ret) - btrfs_err(fs_info, "could not do orphan cleanup %d", ret); + btrfs_err(fs_info, "could not do orphan cleanup %pe", ERR_PTR(ret)); return ret; } @@ -4199,8 +4211,8 @@ cache_acl: ret = btrfs_load_inode_props(inode, path); if (ret) btrfs_err(fs_info, - "error loading props for ino %llu (root %llu): %d", - btrfs_ino(inode), btrfs_root_id(root), ret); + "error loading props for ino %llu (root %llu): %pe", + btrfs_ino(inode), btrfs_root_id(root), ERR_PTR(ret)); } /* @@ -6814,8 +6826,8 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, } if (ret) { btrfs_err(fs_info, - "error inheriting props for ino %llu (root %llu): %d", - btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ret); + "error inheriting props for ino %llu (root %llu): %pe", + btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ERR_PTR(ret)); } /* @@ -10194,6 +10206,8 @@ static void btrfs_free_swapfile_pins(struct inode *inode) struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; struct btrfs_swapfile_pin *sp; struct rb_node *node, *next; + u64 bg_bytes_released = 0; + u32 bg_nr_released = 0; spin_lock(&fs_info->swapfile_pins_lock); node = rb_first(&fs_info->swapfile_pins); @@ -10203,15 +10217,24 @@ static void btrfs_free_swapfile_pins(struct inode *inode) if (sp->inode == inode) { rb_erase(&sp->node, &fs_info->swapfile_pins); if (sp->is_block_group) { - btrfs_dec_block_group_swap_extents(sp->ptr, + struct btrfs_block_group *bg = sp->ptr; + + bg_bytes_released += bg->length; + bg_nr_released++; + btrfs_dec_block_group_swap_extents(bg, sp->bg_extent_count); - btrfs_put_block_group(sp->ptr); + btrfs_put_block_group(bg); } kfree(sp); } node = next; } spin_unlock(&fs_info->swapfile_pins_lock); + btrfs_info(fs_info, +"swapfile deactivated on root %llu ino %llu, released %llu bytes from %u block group(s)", + btrfs_root_id(BTRFS_I(inode)->root), + btrfs_ino(BTRFS_I(inode)), bg_bytes_released, + bg_nr_released); } struct btrfs_swap_info { @@ -10289,8 +10312,10 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, struct btrfs_backref_share_check_ctx *backref_ctx = NULL; struct btrfs_path *path = NULL; int ret = 0; + u32 pinned_bg_nr = 0; u64 isize; u64 prev_extent_end = 0; + u64 pinned_bg_size = 0; /* * Acquire the inode's mmap lock to prevent races with memory mapped @@ -10540,6 +10565,9 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, ret = 0; else goto out; + } else { + pinned_bg_size += bg->length; + pinned_bg_nr++; } if (bsi.block_len && @@ -10587,6 +10615,14 @@ out_unlock_mmap: if (ret) return ret; + btrfs_info(fs_info, +"swapfile activated on root %llu ino %llu, pinned down %llu bytes from %u block group(s)", + btrfs_root_id(BTRFS_I(inode)->root), + btrfs_ino(BTRFS_I(inode)), + pinned_bg_size, pinned_bg_nr); + btrfs_warn(fs_info, +"block groups with swapfile extents will not be scrubbed or balanced"); + if (device) sis->bdev = device->bdev; *span = bsi.highest_ppage - bsi.lowest_ppage + 1; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index c4e995661bcf..72bc9d4f7708 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -356,14 +356,21 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap, inode_flags |= BTRFS_INODE_NODATACOW; } } else { - /* - * Revert back under same assumptions as above - */ - if (S_ISREG(inode->vfs_inode.i_mode)) { - if (inode->vfs_inode.i_size == 0) - inode_flags &= ~(BTRFS_INODE_NODATACOW | - BTRFS_INODE_NODATASUM); - } else { + /* We can only change NODATACOW for zero-sized regular file. */ + if (S_ISREG(inode->vfs_inode.i_mode) && (inode->vfs_inode.i_size == 0)) { + inode_flags &= ~BTRFS_INODE_NODATACOW; + /* + * There is currently no way to change NODATASUM flag + * through fileattr API. If we unconditionally keep the + * current NODATASUM flag, chattr +C then chattr -C will + * keep the NODATASUM flag, and no way to remove that + * flag. + * + * So respect the current mount option for NODATASUM flag. + */ + if (!btrfs_test_opt(fs_info, NODATASUM)) + inode_flags &= ~BTRFS_INODE_NODATASUM; + } else if (!S_ISREG(inode->vfs_inode.i_mode)) { inode_flags &= ~BTRFS_INODE_NODATACOW; } } @@ -393,9 +400,9 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap, /* * 1 for inode item - * 2 for properties + * 1 for property */ - trans = btrfs_start_transaction(root, 3); + trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) return PTR_ERR(trans); @@ -1136,13 +1143,13 @@ out_drop: } static noinline int __btrfs_ioctl_snap_create(struct file *file, - struct mnt_idmap *idmap, const char *name, unsigned long fd, bool subvol, bool readonly, struct btrfs_qgroup_inherit *inherit) { int ret; struct qstr qname = QSTR(name); + struct mnt_idmap *idmap = file_mnt_idmap(file); if (!S_ISDIR(file_inode(file)->i_mode)) return -ENOTDIR; @@ -1220,8 +1227,7 @@ static noinline int btrfs_ioctl_snap_create(struct file *file, if (ret < 0) return ret; - return __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), - vol_args->name, vol_args->fd, subvol, + return __btrfs_ioctl_snap_create(file, vol_args->name, vol_args->fd, subvol, false, NULL); } @@ -1264,8 +1270,7 @@ static noinline int btrfs_ioctl_snap_create_v2(struct file *file, return ret; } - return __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), - vol_args->name, vol_args->fd, subvol, + return __btrfs_ioctl_snap_create(file, vol_args->name, vol_args->fd, subvol, readonly, inherit); } @@ -1657,13 +1662,11 @@ static noinline int btrfs_ioctl_tree_search_v2(struct btrfs_root *root, } /* - * Search INODE_REFs to identify path name of 'dirid' directory - * in a 'tree_id' tree. and sets path name to 'name'. + * Search for an INODE_REF in a 'root' tree which identifies the path name of + * 'dirid'. When found, it sets 'name' with the path name. */ -static noinline int btrfs_search_path_in_tree(struct btrfs_fs_info *info, - u64 tree_id, u64 dirid, char *name) +static noinline int btrfs_search_path_in_tree(struct btrfs_root *root, u64 dirid, char *name) { - struct btrfs_root *root; struct btrfs_key key; char *ptr; int ret = -1; @@ -1685,13 +1688,6 @@ static noinline int btrfs_search_path_in_tree(struct btrfs_fs_info *info, ptr = &name[BTRFS_INO_LOOKUP_PATH_MAX - 1]; - root = btrfs_get_fs_root(info, tree_id, true); - if (IS_ERR(root)) { - ret = PTR_ERR(root); - root = NULL; - goto out; - } - key.objectid = dirid; key.type = BTRFS_INODE_REF_KEY; key.offset = (u64)-1; @@ -1699,11 +1695,9 @@ static noinline int btrfs_search_path_in_tree(struct btrfs_fs_info *info, while (1) { ret = btrfs_search_backwards(root, &key, path); if (ret < 0) - goto out; - else if (ret > 0) { - ret = -ENOENT; - goto out; - } + return ret; + else if (ret > 0) + return -ENOENT; l = path->nodes[0]; slot = path->slots[0]; @@ -1712,10 +1706,8 @@ static noinline int btrfs_search_path_in_tree(struct btrfs_fs_info *info, len = btrfs_inode_ref_name_len(l, iref); ptr -= len + 1; total_len += len + 1; - if (ptr < name) { - ret = -ENAMETOOLONG; - goto out; - } + if (ptr < name) + return -ENAMETOOLONG; *(ptr + len) = '/'; read_extent_buffer(l, ptr, (unsigned long)(iref + 1), len); @@ -1730,10 +1722,8 @@ static noinline int btrfs_search_path_in_tree(struct btrfs_fs_info *info, } memmove(name, ptr, total_len); name[total_len] = '\0'; - ret = 0; -out: - btrfs_put_root(root); - return ret; + + return 0; } static int btrfs_search_path_in_tree_user(struct mnt_idmap *idmap, @@ -1877,6 +1867,7 @@ out_put: static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, void __user *argp) { + bool new_root = false; struct btrfs_ioctl_ino_lookup_args AUTO_KFREE(args); int ret = 0; @@ -1890,6 +1881,8 @@ static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, */ if (args->treeid == 0) args->treeid = btrfs_root_id(root); + else + new_root = true; if (args->objectid == BTRFS_FIRST_FREE_OBJECTID) { args->name[0] = 0; @@ -1901,9 +1894,14 @@ static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, goto out; } - ret = btrfs_search_path_in_tree(root->fs_info, - args->treeid, args->objectid, - args->name); + if (new_root) { + root = btrfs_get_fs_root(root->fs_info, args->treeid, true); + if (IS_ERR(root)) + return PTR_ERR(root); + } + ret = btrfs_search_path_in_tree(root, args->objectid, args->name); + if (new_root) + btrfs_put_root(root); out: if (ret == 0 && copy_to_user(argp, args, sizeof(*args))) @@ -2841,8 +2839,8 @@ static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) else ret = -ENOENT; btrfs_err(fs_info, - "could not find default diritem for dir %llu: %d", - dir_id, ret); + "could not find default diritem for dir %llu: %pe", + dir_id, ERR_PTR(ret)); goto out_free; } @@ -3613,7 +3611,7 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_root *quota_root; struct btrfs_ioctl_qgroup_assign_args AUTO_KFREE(sa); struct btrfs_qgroup_list AUTO_KFREE(prealloc); struct btrfs_trans_handle *trans; @@ -3644,10 +3642,20 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) } } + mutex_lock(&fs_info->qgroup_ioctl_lock); + quota_root = btrfs_grab_root(fs_info->quota_root); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + if (!quota_root) { + ret = -ENOTCONN; + goto drop_write; + } + /* 2 BTRFS_QGROUP_RELATION_KEY items. */ - trans = btrfs_start_transaction(root, 2); + trans = btrfs_start_transaction(quota_root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); + btrfs_put_root(quota_root); goto drop_write; } @@ -3671,6 +3679,7 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) "qgroup status update failed after %s relation, marked as inconsistent", sa->assign ? "adding" : "deleting"); err = btrfs_end_transaction(trans); + btrfs_put_root(quota_root); if (err && !ret) ret = err; @@ -3682,7 +3691,8 @@ drop_write: static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); - struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *quota_root; struct btrfs_ioctl_qgroup_create_args AUTO_KFREE(sa); struct btrfs_trans_handle *trans; int ret; @@ -3691,7 +3701,7 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (!btrfs_qgroup_enabled(root->fs_info)) + if (!btrfs_qgroup_enabled(fs_info)) return -ENOTCONN; ret = mnt_want_write_file(file); @@ -3714,13 +3724,23 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) goto drop_write; } + mutex_lock(&fs_info->qgroup_ioctl_lock); + quota_root = btrfs_grab_root(fs_info->quota_root); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + if (!quota_root) { + ret = -ENOTCONN; + goto drop_write; + } + /* * 1 BTRFS_QGROUP_INFO_KEY item. * 1 BTRFS_QGROUP_LIMIT_KEY item. */ - trans = btrfs_start_transaction(root, 2); + trans = btrfs_start_transaction(quota_root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); + btrfs_put_root(quota_root); goto drop_write; } @@ -3731,6 +3751,7 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) } err = btrfs_end_transaction(trans); + btrfs_put_root(quota_root); if (err && !ret) ret = err; @@ -3743,6 +3764,8 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_root *quota_root; + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_ioctl_qgroup_limit_args AUTO_KFREE(sa); struct btrfs_trans_handle *trans; int ret; @@ -3752,7 +3775,7 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (!btrfs_qgroup_enabled(root->fs_info)) + if (!btrfs_qgroup_enabled(fs_info)) return -ENOTCONN; ret = mnt_want_write_file(file); @@ -3765,10 +3788,20 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) goto drop_write; } + mutex_lock(&fs_info->qgroup_ioctl_lock); + quota_root = btrfs_grab_root(fs_info->quota_root); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + if (!quota_root) { + ret = -ENOTCONN; + goto drop_write; + } + /* 1 BTRFS_QGROUP_LIMIT_KEY item. */ - trans = btrfs_start_transaction(root, 1); + trans = btrfs_start_transaction(quota_root, 1); if (IS_ERR(trans)) { ret = PTR_ERR(trans); + btrfs_put_root(quota_root); goto drop_write; } @@ -3781,6 +3814,7 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) ret = btrfs_limit_qgroup(trans, qgroupid, &sa->lim); err = btrfs_end_transaction(trans); + btrfs_put_root(quota_root); if (err && !ret) ret = err; diff --git a/fs/btrfs/messages.c b/fs/btrfs/messages.c index 7c60c14e60fa..198d1747c80a 100644 --- a/fs/btrfs/messages.c +++ b/fs/btrfs/messages.c @@ -279,7 +279,6 @@ void __btrfs_panic(const struct btrfs_fs_info *fs_info, const char *function, unsigned int line, int error, const char *fmt, ...) { char *s_id = "<unknown>"; - const char *errstr; struct va_format vaf = { .fmt = fmt }; va_list args; @@ -289,13 +288,12 @@ void __btrfs_panic(const struct btrfs_fs_info *fs_info, const char *function, va_start(args, fmt); vaf.va = &args; - errstr = btrfs_decode_error(error); if (fs_info && (btrfs_test_opt(fs_info, PANIC_ON_FATAL_ERROR))) - panic(KERN_CRIT "BTRFS panic (device %s) in %s:%d: %pV (errno=%d %s)\n", - s_id, function, line, &vaf, error, errstr); + panic(KERN_CRIT "BTRFS panic (device %s) in %s:%d: %pV (errno=%d %pe)\n", + s_id, function, line, &vaf, error, ERR_PTR(error)); - btrfs_crit(fs_info, "panic in %s:%d: %pV (errno=%d %s)", - function, line, &vaf, error, errstr); + btrfs_crit(fs_info, "panic in %s:%d: %pV (errno=%d %pe)", + function, line, &vaf, error, ERR_PTR(error)); va_end(args); /* Caller calls BUG() */ } diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 502fb4a55cb2..f68b696b4bf7 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -3915,8 +3915,8 @@ out: ret = PTR_ERR(trans); trans = NULL; btrfs_err(fs_info, - "fail to start transaction for status update: %d", - ret); + "fail to start transaction for status update: %pe", + ERR_PTR(ret)); } } else { trans = NULL; @@ -3931,7 +3931,7 @@ out: if (ret2 < 0) { ret = ret2; - btrfs_err(fs_info, "fail to update qgroup status: %d", ret); + btrfs_err(fs_info, "fail to update qgroup status: %pe", ERR_PTR(ret)); } } fs_info->qgroup_rescan_running = false; @@ -3952,7 +3952,7 @@ out: btrfs_info(fs_info, "qgroup scan completed%s", ret > 0 ? " (inconsistency flag cleared)" : ""); } else { - btrfs_err(fs_info, "qgroup scan failed with %d", ret); + btrfs_err(fs_info, "qgroup scan failed with %pe", ERR_PTR(ret)); } } @@ -4339,12 +4339,13 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, struct ulist_node *unode; struct ulist_iterator uiter; struct extent_changeset changeset; + const u32 sectorsize = root->fs_info->sectorsize; + const u64 aligned_start = round_down(start, sectorsize); + const u64 aligned_len = round_up(start + len, sectorsize) - aligned_start; u64 freed = 0; int ret; extent_changeset_init_bytes_only(&changeset); - len = round_up(start + len, root->fs_info->sectorsize); - start = round_down(start, root->fs_info->sectorsize); ULIST_ITER_INIT(&uiter); while ((unode = ulist_next(&reserved->range_changed, &uiter))) { @@ -4356,12 +4357,15 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, extent_changeset_release(&changeset); - /* Only free range in range [start, start + len) */ - if (range_start >= start + len || - range_start + range_len <= start) + /* + * Only free the range within + * [aligned_start, aligned_start + aligned_len). + */ + if (range_start >= aligned_start + aligned_len || + range_start + range_len <= aligned_start) continue; - free_start = max(range_start, start); - free_len = min(start + len, range_start + range_len) - + free_start = max(range_start, aligned_start); + free_len = min(aligned_start + aligned_len, range_start + range_len) - free_start; /* * TODO: To also modify reserved->ranges_reserved to reflect diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index ffb654d36391..1ee52a9dcee3 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -2997,13 +2997,11 @@ void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio) * This is due to the fact rbio has its own page management for its cache. */ void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, - struct folio **data_folios, u64 data_logical) + void *vaddr, u64 data_logical) { struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; const u64 offset_in_full_stripe = data_logical - rbio->bioc->full_stripe_logical; - unsigned int findex = 0; - unsigned int foffset = 0; int ret; /* @@ -3026,18 +3024,10 @@ void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, cur_off < offset_in_full_stripe + BTRFS_STRIPE_LEN; cur_off += PAGE_SIZE) { const unsigned int pindex = cur_off >> PAGE_SHIFT; - void *kaddr; - kaddr = kmap_local_page(rbio->stripe_pages[pindex]); - memcpy_from_folio(kaddr, data_folios[findex], foffset, PAGE_SIZE); - kunmap_local(kaddr); - - foffset += PAGE_SIZE; - ASSERT(foffset <= folio_size(data_folios[findex])); - if (foffset == folio_size(data_folios[findex])) { - findex++; - foffset = 0; - } + ASSERT(cur_off - offset_in_full_stripe + PAGE_SIZE <= BTRFS_STRIPE_LEN); + memcpy_to_page(rbio->stripe_pages[pindex], 0, + vaddr + cur_off - offset_in_full_stripe, PAGE_SIZE); } bitmap_set(rbio->stripe_uptodate_bitmap, offset_in_full_stripe >> fs_info->sectorsize_bits, diff --git a/fs/btrfs/raid56.h b/fs/btrfs/raid56.h index 1f463ecf7e41..8542648199f1 100644 --- a/fs/btrfs/raid56.h +++ b/fs/btrfs/raid56.h @@ -283,7 +283,7 @@ struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio, void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio); void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, - struct folio **data_folios, u64 data_logical); + void *vaddr, u64 data_logical); int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info); void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info); diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 9a49d2ecb949..d2a4101912bd 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -20,30 +20,31 @@ #define BTRFS_MAX_DEDUPE_LEN SZ_16M static int clone_finish_inode_update(struct btrfs_trans_handle *trans, - struct inode *inode, + struct btrfs_inode *inode, u64 endoff, const u64 destoff, const u64 olen, bool no_time_update) { + struct inode *vfs_inode = &inode->vfs_inode; int ret; - inode_inc_iversion(inode); - if (!no_time_update) { - inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - } + inode_inc_iversion(vfs_inode); + if (!no_time_update) + inode_set_mtime_to_ts(vfs_inode, inode_set_ctime_current(vfs_inode)); + /* * We round up to the block size at eof when determining which * extents to clone above, but shouldn't round up the file size. */ if (endoff > destoff + olen) endoff = destoff + olen; - if (endoff > inode->i_size) { - i_size_write(inode, endoff); - btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); + if (endoff > vfs_inode->i_size) { + i_size_write(vfs_inode, endoff); + btrfs_inode_safe_disk_i_size_write(inode, 0); } - ret = btrfs_update_inode(trans, BTRFS_I(inode)); + ret = btrfs_update_inode(trans, inode); if (unlikely(ret)) { btrfs_abort_transaction(trans, ret); btrfs_end_transaction(trans); @@ -392,11 +393,11 @@ copy_to_page: * @destoff: Offset within @inode to start clone * @no_time_update: Whether to update mtime/ctime on the target inode */ -static int btrfs_clone(struct inode *src, struct inode *inode, +static int btrfs_clone(struct btrfs_inode *src, struct btrfs_inode *inode, const u64 off, const u64 olen, const u64 olen_aligned, const u64 destoff, bool no_time_update) { - struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_fs_info *fs_info = inode->root->fs_info; BTRFS_PATH_AUTO_FREE(path); struct extent_buffer *leaf; struct btrfs_trans_handle *trans; @@ -420,7 +421,7 @@ static int btrfs_clone(struct inode *src, struct inode *inode, path->reada = READA_FORWARD; /* Clone data */ - key.objectid = btrfs_ino(BTRFS_I(src)); + key.objectid = btrfs_ino(src); key.type = BTRFS_EXTENT_DATA_KEY; key.offset = off; @@ -436,8 +437,7 @@ static int btrfs_clone(struct inode *src, struct inode *inode, u64 drop_start; /* Note the key will change type as we walk through the tree */ - ret = btrfs_search_slot(NULL, BTRFS_I(src)->root, &key, path, - 0, 0); + ret = btrfs_search_slot(NULL, src->root, &key, path, 0, 0); if (ret < 0) goto out; /* @@ -455,7 +455,7 @@ static int btrfs_clone(struct inode *src, struct inode *inode, nritems = btrfs_header_nritems(path->nodes[0]); process_slot: if (path->slots[0] >= nritems) { - ret = btrfs_next_leaf(BTRFS_I(src)->root, path); + ret = btrfs_next_leaf(src->root, path); if (ret < 0) goto out; if (ret > 0) @@ -466,8 +466,7 @@ process_slot: slot = path->slots[0]; btrfs_item_key_to_cpu(leaf, &key, slot); - if (key.type > BTRFS_EXTENT_DATA_KEY || - key.objectid != btrfs_ino(BTRFS_I(src))) + if (key.type > BTRFS_EXTENT_DATA_KEY || key.objectid != btrfs_ino(src)) break; ASSERT(key.type == BTRFS_EXTENT_DATA_KEY, "key.type=%u", key.type); @@ -514,7 +513,7 @@ process_slot: btrfs_release_path(path); memcpy(&new_key, &key, sizeof(new_key)); - new_key.objectid = btrfs_ino(BTRFS_I(inode)); + new_key.objectid = btrfs_ino(inode); if (off <= key.offset) new_key.offset = key.offset + destoff - off; else @@ -558,7 +557,7 @@ process_slot: clone_info.extent_buf = buf; clone_info.is_new_extent = false; clone_info.update_times = !no_time_update; - ret = btrfs_replace_file_extents(BTRFS_I(inode), path, + ret = btrfs_replace_file_extents(inode, path, drop_start, new_key.offset + datal - 1, &clone_info, &trans); if (ret) @@ -582,7 +581,7 @@ process_slot: goto out; } - ret = clone_copy_inline_extent(BTRFS_I(inode), path, &new_key, + ret = clone_copy_inline_extent(inode, path, &new_key, drop_start, datal, size, comp, buf, &trans); if (ret) @@ -605,9 +604,9 @@ process_slot: * the checksums problem on fsync. */ if (extent_gen == trans->transid && disko > 0) - BTRFS_I(src)->last_reflink_trans = trans->transid; + src->last_reflink_trans = trans->transid; - BTRFS_I(inode)->last_reflink_trans = trans->transid; + inode->last_reflink_trans = trans->transid; last_dest_end = ALIGN(new_key.offset + datal, fs_info->sectorsize); @@ -653,10 +652,10 @@ process_slot: * set by previous calls to btrfs_replace_file_extents() that * replaced file extent items. */ - if (last_dest_end >= i_size_read(inode)) - btrfs_set_inode_full_sync(BTRFS_I(inode)); + if (last_dest_end >= i_size_read(&inode->vfs_inode)) + btrfs_set_inode_full_sync(inode); - ret = btrfs_replace_file_extents(BTRFS_I(inode), path, + ret = btrfs_replace_file_extents(inode, path, last_dest_end, destoff + len - 1, NULL, &trans); if (ret) goto out; @@ -666,7 +665,7 @@ process_slot: } out: - clear_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &BTRFS_I(inode)->runtime_flags); + clear_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags); return ret; } @@ -688,10 +687,10 @@ static void btrfs_double_mmap_unlock(struct btrfs_inode *inode1, struct btrfs_in static int btrfs_extent_same_range(struct btrfs_inode *src, u64 loff, u64 len, struct btrfs_inode *dst, u64 dst_loff) { - const u64 end = dst_loff + len - 1; struct extent_state *cached_state = NULL; struct btrfs_fs_info *fs_info = src->root->fs_info; - const u64 bs = fs_info->sectorsize; + const u32 bs = fs_info->sectorsize; + const u64 end = round_up(dst_loff + len, bs) - 1; int ret; /* @@ -701,8 +700,7 @@ static int btrfs_extent_same_range(struct btrfs_inode *src, u64 loff, u64 len, * mode. */ btrfs_lock_extent(&dst->io_tree, dst_loff, end, &cached_state); - ret = btrfs_clone(&src->vfs_inode, &dst->vfs_inode, loff, len, - ALIGN(len, bs), dst_loff, true); + ret = btrfs_clone(src, dst, loff, len, ALIGN(len, bs), dst_loff, true); btrfs_unlock_extent(&dst->io_tree, dst_loff, end, &cached_state); btrfs_btree_balance_dirty(fs_info); @@ -710,12 +708,12 @@ static int btrfs_extent_same_range(struct btrfs_inode *src, u64 loff, u64 len, return ret; } -static int btrfs_extent_same(struct inode *src, u64 loff, u64 olen, - struct inode *dst, u64 dst_loff) +static int btrfs_extent_same(struct btrfs_inode *src, u64 loff, u64 olen, + struct btrfs_inode *dst, u64 dst_loff) { int ret = 0; u64 i, tail_len, chunk_count; - struct btrfs_root *root_dst = BTRFS_I(dst)->root; + struct btrfs_root *root_dst = dst->root; spin_lock(&root_dst->root_item_lock); if (root_dst->send_in_progress) { @@ -733,8 +731,8 @@ static int btrfs_extent_same(struct inode *src, u64 loff, u64 olen, chunk_count = div_u64(olen, BTRFS_MAX_DEDUPE_LEN); for (i = 0; i < chunk_count; i++) { - ret = btrfs_extent_same_range(BTRFS_I(src), loff, BTRFS_MAX_DEDUPE_LEN, - BTRFS_I(dst), dst_loff); + ret = btrfs_extent_same_range(src, loff, BTRFS_MAX_DEDUPE_LEN, + dst, dst_loff); if (ret) goto out; @@ -743,8 +741,7 @@ static int btrfs_extent_same(struct inode *src, u64 loff, u64 olen, } if (tail_len > 0) - ret = btrfs_extent_same_range(BTRFS_I(src), loff, tail_len, - BTRFS_I(dst), dst_loff); + ret = btrfs_extent_same_range(src, loff, tail_len, dst, dst_loff); out: spin_lock(&root_dst->root_item_lock); root_dst->dedupe_in_progress--; @@ -757,12 +754,14 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, u64 off, u64 olen, u64 destoff) { struct extent_state *cached_state = NULL; - struct inode *inode = file_inode(file); - struct inode *src = file_inode(file_src); - struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_inode *inode = BTRFS_I(file_inode(file)); + struct btrfs_inode *src = BTRFS_I(file_inode(file_src)); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u64 src_isize = src->vfs_inode.i_size; + const u64 inode_isize = inode->vfs_inode.i_size; int ret; u64 len = olen; - u64 bs = fs_info->sectorsize; + const u32 bs = fs_info->sectorsize; u64 end; /* @@ -771,13 +770,13 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, * if the file size is not blocksize aligned. So we don't need to check * for that case here. */ - if (off + len == src->i_size) - len = ALIGN(src->i_size, bs) - off; + if (off + len == src_isize) + len = ALIGN(src_isize, bs) - off; - if (destoff > inode->i_size) { - const u64 wb_start = ALIGN_DOWN(inode->i_size, bs); + if (destoff > inode_isize) { + const u64 wb_start = ALIGN_DOWN(inode_isize, bs); - ret = btrfs_cont_expand(BTRFS_I(inode), inode->i_size, destoff); + ret = btrfs_cont_expand(inode, inode_isize, destoff); if (ret) return ret; /* @@ -789,8 +788,7 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, * we found the previous extent covering eof and before we * attempted to increment its reference count). */ - ret = btrfs_wait_ordered_range(BTRFS_I(inode), wb_start, - destoff - wb_start); + ret = btrfs_wait_ordered_range(inode, wb_start, destoff - wb_start); if (ret) return ret; } @@ -801,10 +799,10 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, * because we have already locked the inode's i_mmap_lock in exclusive * mode. */ - end = destoff + len - 1; - btrfs_lock_extent(&BTRFS_I(inode)->io_tree, destoff, end, &cached_state); + end = round_up(destoff + len, bs) - 1; + btrfs_lock_extent(&inode->io_tree, destoff, end, &cached_state); ret = btrfs_clone(src, inode, off, olen, len, destoff, false); - btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, destoff, end, &cached_state); + btrfs_unlock_extent(&inode->io_tree, destoff, end, &cached_state); if (ret < 0) return ret; @@ -818,7 +816,7 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, * could come from some range other than the copied inline extent's * destination range and we have no way to know that. */ - ret = btrfs_wait_ordered_range(BTRFS_I(inode), destoff, len); + ret = btrfs_wait_ordered_range(inode, destoff, len); if (ret < 0) return ret; @@ -826,7 +824,7 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, * Invalidate page cache so that future reads will see the cloned data * immediately and not the previous data. */ - ret = filemap_invalidate_inode(inode, false, destoff, end); + ret = filemap_invalidate_inode(&inode->vfs_inode, false, destoff, end); if (ret < 0) return ret; @@ -841,7 +839,7 @@ static int btrfs_remap_file_range_prep(struct file *file_in, loff_t pos_in, { struct btrfs_inode *inode_in = BTRFS_I(file_inode(file_in)); struct btrfs_inode *inode_out = BTRFS_I(file_inode(file_out)); - u64 bs = inode_out->root->fs_info->sectorsize; + const u32 bs = inode_out->root->fs_info->sectorsize; u64 wb_len; int ret; @@ -934,7 +932,7 @@ loff_t btrfs_remap_file_range(struct file *src_file, loff_t off, bool same_inode = dst_inode == src_inode; int ret; - if (btrfs_is_shutdown(inode_to_fs_info(file_inode(src_file)))) + if (btrfs_is_shutdown(src_inode->root->fs_info)) return -EIO; if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY)) @@ -953,8 +951,7 @@ loff_t btrfs_remap_file_range(struct file *src_file, loff_t off, goto out_unlock; if (remap_flags & REMAP_FILE_DEDUP) - ret = btrfs_extent_same(&src_inode->vfs_inode, off, len, - &dst_inode->vfs_inode, destoff); + ret = btrfs_extent_same(src_inode, off, len, dst_inode, destoff); else ret = btrfs_clone_files(dst_file, src_file, off, len, destoff); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index fc5c14b5adad..da54db75e7a9 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -339,14 +339,15 @@ static struct btrfs_backref_node *walk_down_backref( static bool reloc_root_is_dead(const struct btrfs_root *root) { + if (test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state)) + return true; /* - * Pair with set_bit/clear_bit in clean_dirty_subvols and - * btrfs_update_reloc_root. We need to see the updated bit before - * trying to access reloc_root + * Pairs with set_bit/clear_bit in clear_reloc_root() and + * btrfs_update_reloc_root(). We need to see the updated bit before + * trying to access root->reloc_root in our callers. */ smp_rmb(); - if (test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state)) - return true; + return false; } @@ -1537,6 +1538,33 @@ static void clear_reloc_root(struct btrfs_root *root) clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); } +/* Drop the reloc trees of a relocation that is being deferred and retried. */ +static void abort_reloc_roots(struct reloc_control *rc, struct list_head *list) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_root *reloc_root, *tmp; + + list_for_each_entry_safe(reloc_root, tmp, list, root_list) { + struct btrfs_root *root; + + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false); + if (!IS_ERR(root)) { + if (root->reloc_root == reloc_root) { + clear_reloc_root(root); + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } + + btrfs_set_root_refs(&reloc_root->root_item, 0); + memset(&reloc_root->root_item.drop_progress, 0, sizeof(struct btrfs_disk_key)); + btrfs_set_root_drop_level(&reloc_root->root_item, 0); + + list_del_init(&reloc_root->root_list); + list_add_tail(&reloc_root->reloc_dirty_list, &rc->dirty_subvol_roots); + } +} + static int clean_dirty_subvols(struct reloc_control *rc) { struct btrfs_root *root; @@ -1876,8 +1904,7 @@ again: return err; } -static noinline_for_stack -void merge_reloc_roots(struct reloc_control *rc) +static noinline_for_stack int merge_reloc_roots(struct reloc_control *rc) { struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_root *root; @@ -1975,7 +2002,15 @@ again: goto again; } out: - if (ret) { + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + abort_reloc_roots(rc, &reloc_roots); + + /* New reloc root may be added. */ + mutex_lock(&fs_info->reloc_mutex); + list_splice_init(&rc->reloc_roots, &reloc_roots); + mutex_unlock(&fs_info->reloc_mutex); + abort_reloc_roots(rc, &reloc_roots); + } else if (ret) { btrfs_handle_fs_error(fs_info, ret, NULL); free_reloc_roots(&reloc_roots); @@ -2001,6 +2036,7 @@ out: * * The remaining nodes will be cleaned up by put_reloc_control(). */ + return ret; } static void free_block_list(struct rb_root *blocks) @@ -3730,7 +3766,9 @@ restart: */ err = prepare_to_merge(rc, err); - merge_reloc_roots(rc); + ret = merge_reloc_roots(rc); + if (ret && !err) + err = ret; rc->merge_reloc_tree = false; unset_reloc_control(rc); @@ -4114,10 +4152,10 @@ static int copy_remapped_data(struct btrfs_fs_info *fs_info, u64 old_addr, u64 new_addr, u64 length) { int ret; - u64 copy_len = min_t(u64, length, SZ_1M); + const u64 copy_len = min_t(u64, length, SZ_1M); struct page **pages; struct reloc_io_private priv; - unsigned int nr_pages = DIV_ROUND_UP(length, PAGE_SIZE); + const unsigned int nr_pages = DIV_ROUND_UP(copy_len, PAGE_SIZE); pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); if (!pages) @@ -5555,6 +5593,24 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) return ret; } +static void release_recovered_fs_roots(struct list_head *roots, bool drop_reloc_refs) +{ + struct btrfs_root *root; + struct btrfs_root *next; + + list_for_each_entry_safe(root, next, roots, reloc_dirty_list) { + list_del_init(&root->reloc_dirty_list); + if (drop_reloc_refs) { + struct btrfs_root *reloc_root = root->reloc_root; + + ASSERT(reloc_root); + root->reloc_root = NULL; + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } +} + /* * recover relocation interrupted by system crash. * @@ -5564,6 +5620,7 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) { LIST_HEAD(reloc_roots); + LIST_HEAD(recovered_roots); struct btrfs_key key; struct btrfs_root *fs_root; struct btrfs_root *reloc_root; @@ -5680,7 +5737,7 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) ret = PTR_ERR(fs_root); list_add_tail(&reloc_root->root_list, &reloc_roots); btrfs_end_transaction(trans); - goto out_unset; + goto out_drop_reloc_refs; } ret = __add_reloc_root(reloc_root, rc); @@ -5689,17 +5746,21 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) list_add_tail(&reloc_root->root_list, &reloc_roots); btrfs_put_root(fs_root); btrfs_end_transaction(trans); - goto out_unset; + goto out_drop_reloc_refs; } + ASSERT(list_empty(&fs_root->reloc_dirty_list)); fs_root->reloc_root = btrfs_grab_root(reloc_root); - btrfs_put_root(fs_root); + list_add_tail(&fs_root->reloc_dirty_list, &recovered_roots); } ret = btrfs_commit_transaction(trans); if (ret) - goto out_unset; + goto out_drop_reloc_refs; + release_recovered_fs_roots(&recovered_roots, false); - merge_reloc_roots(rc); + ret = merge_reloc_roots(rc); + if (ret) + goto out_unset; unset_reloc_control(rc); @@ -5713,6 +5774,8 @@ out_clean: ret2 = clean_dirty_subvols(rc); if (ret2 < 0 && !ret) ret = ret2; +out_drop_reloc_refs: + release_recovered_fs_roots(&recovered_roots, true); out_unset: unset_reloc_control(rc); reloc_chunk_end(fs_info); diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c index 90659b287d90..2e4c3efbd02f 100644 --- a/fs/btrfs/root-tree.c +++ b/fs/btrfs/root-tree.c @@ -265,15 +265,15 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) if (IS_ERR(trans)) { ret = PTR_ERR(trans); btrfs_err(fs_info, - "failed to join transaction to delete orphan item: %d", - ret); + "failed to join transaction to delete orphan item: %pe", + ERR_PTR(ret)); return ret; } ret = btrfs_del_orphan_item(trans, tree_root, root_objectid); btrfs_end_transaction(trans); if (ret) { btrfs_err(fs_info, - "failed to delete root orphan item: %d", ret); + "failed to delete root orphan item: %pe", ERR_PTR(ret)); return ret; } continue; diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index d2f7ac5b6e96..f209e75f0ff5 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -57,12 +57,6 @@ struct scrub_ctx; #define SCRUB_TOTAL_STRIPES (SCRUB_GROUPS_PER_SCTX * SCRUB_STRIPES_PER_GROUP) -/* - * The following value times PAGE_SIZE needs to be large enough to match the - * largest node/leaf/sector size that shall be supported. - */ -#define SCRUB_MAX_SECTORS_PER_BLOCK (BTRFS_MAX_METADATA_BLOCKSIZE / SZ_4K) - /* Represent one sector and its needed info to verify the content. */ struct scrub_sector_verification { union { @@ -129,19 +123,17 @@ enum { scrub_bitmap_nr_last, }; -#define SCRUB_STRIPE_MAX_FOLIOS (BTRFS_STRIPE_LEN / PAGE_SIZE) - /* * Represent one contiguous range with a length of BTRFS_STRIPE_LEN. */ struct scrub_stripe { struct scrub_ctx *sctx; struct btrfs_block_group *bg; - - struct folio *folios[SCRUB_STRIPE_MAX_FOLIOS]; struct scrub_sector_verification *sectors; - struct btrfs_device *dev; + + void *buffer; + u64 logical; u64 physical; @@ -227,6 +219,9 @@ struct scrub_ctx { refcount_t refs; }; +static_assert(BTRFS_STRIPE_LEN >= PAGE_SIZE); +static_assert(IS_ALIGNED(BTRFS_STRIPE_LEN, PAGE_SIZE)); + #define scrub_calc_start_bit(stripe, name, block_nr) \ ({ \ unsigned int __start_bit; \ @@ -338,13 +333,10 @@ static void release_scrub_stripe(struct scrub_stripe *stripe) if (!stripe) return; - for (int i = 0; i < SCRUB_STRIPE_MAX_FOLIOS; i++) { - if (stripe->folios[i]) - folio_put(stripe->folios[i]); - stripe->folios[i] = NULL; - } + kvfree(stripe->buffer); kfree(stripe->sectors); kfree(stripe->csums); + stripe->buffer = NULL; stripe->sectors = NULL; stripe->csums = NULL; stripe->sctx = NULL; @@ -354,9 +346,6 @@ static void release_scrub_stripe(struct scrub_stripe *stripe) static int init_scrub_stripe(struct btrfs_fs_info *fs_info, struct scrub_stripe *stripe) { - const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; - int ret; - memset(stripe, 0, sizeof(*stripe)); stripe->nr_sectors = BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits; @@ -367,11 +356,8 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, atomic_set(&stripe->pending_io, 0); spin_lock_init(&stripe->write_error_lock); - ASSERT(BTRFS_STRIPE_LEN >> min_folio_shift <= SCRUB_STRIPE_MAX_FOLIOS); - ret = btrfs_alloc_folio_array(BTRFS_STRIPE_LEN >> min_folio_shift, - fs_info->block_min_order, stripe->folios, - GFP_NOFS); - if (ret < 0) + stripe->buffer = kvmalloc(BTRFS_STRIPE_LEN, GFP_NOFS); + if (!stripe->buffer) goto error; stripe->sectors = kzalloc_objs(struct scrub_sector_verification, @@ -682,32 +668,18 @@ static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) return ret; } -static void *scrub_stripe_get_kaddr(struct scrub_stripe *stripe, int sector_nr) -{ - struct btrfs_fs_info *fs_info = stripe->bg->fs_info; - const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; - u32 offset = (sector_nr << fs_info->sectorsize_bits); - const struct folio *folio = stripe->folios[offset >> min_folio_shift]; - - /* stripe->folios[] is allocated by us and no highmem is allowed. */ - ASSERT(folio); - ASSERT(!folio_test_highmem(folio)); - return folio_address(folio) + offset_in_folio(folio, offset); -} - -static phys_addr_t scrub_stripe_get_paddr(struct scrub_stripe *stripe, int sector_nr) +/* + * Unlike the existing csum which is based on paddr, this version is fully on + * vaddr, so no extra per-page iteration needed. + */ +static void scrub_calc_vaddr_csum(struct btrfs_fs_info *fs_info, + void *vaddr, unsigned int len, u8 *dest) { - struct btrfs_fs_info *fs_info = stripe->bg->fs_info; - const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; - u32 offset = (sector_nr << fs_info->sectorsize_bits); - const struct folio *folio = stripe->folios[offset >> min_folio_shift]; + struct btrfs_csum_ctx csum; - /* stripe->folios[] is allocated by us and no highmem is allowed. */ - ASSERT(folio); - ASSERT(!folio_test_highmem(folio)); - /* And the range must be contained inside the folio. */ - ASSERT(offset_in_folio(folio, offset) + fs_info->sectorsize <= folio_size(folio)); - return page_to_phys(folio_page(folio, 0)) + offset_in_folio(folio, offset); + btrfs_csum_init(&csum, fs_info->csum_type); + btrfs_csum_update(&csum, vaddr, len); + btrfs_csum_final(&csum, dest); } static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr) @@ -715,19 +687,10 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr struct btrfs_fs_info *fs_info = stripe->bg->fs_info; const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits); - void *first_kaddr = scrub_stripe_get_kaddr(stripe, sector_nr); - struct btrfs_header *header = first_kaddr; - struct btrfs_csum_ctx csum; - u8 on_disk_csum[BTRFS_CSUM_SIZE]; + void *first_vaddr = stripe->buffer + (sector_nr << fs_info->sectorsize_bits); + struct btrfs_header *header = first_vaddr; u8 calculated_csum[BTRFS_CSUM_SIZE]; - /* - * Here we don't have a good way to attach the pages (and subpages) - * to a dummy extent buffer, thus we have to directly grab the members - * from pages. - */ - memcpy(on_disk_csum, header->csum, fs_info->csum_size); - if (logical != btrfs_stack_header_bytenr(header)) { scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); @@ -759,23 +722,15 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr } /* Now check tree block csum. */ - btrfs_csum_init(&csum, fs_info->csum_type); - btrfs_csum_update(&csum, first_kaddr + BTRFS_CSUM_SIZE, - fs_info->sectorsize - BTRFS_CSUM_SIZE); - - for (int i = sector_nr + 1; i < sector_nr + sectors_per_tree; i++) { - btrfs_csum_update(&csum, scrub_stripe_get_kaddr(stripe, i), - fs_info->sectorsize); - } - - btrfs_csum_final(&csum, calculated_csum); - if (memcmp(calculated_csum, on_disk_csum, fs_info->csum_size) != 0) { + scrub_calc_vaddr_csum(fs_info, first_vaddr + BTRFS_CSUM_SIZE, + fs_info->nodesize - BTRFS_CSUM_SIZE, calculated_csum); + if (memcmp(calculated_csum, header->csum, fs_info->csum_size) != 0) { scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad csum, has " BTRFS_CSUM_FMT " want " BTRFS_CSUM_FMT, logical, stripe->mirror_num, - BTRFS_CSUM_FMT_VALUE(fs_info->csum_size, on_disk_csum), + BTRFS_CSUM_FMT_VALUE(fs_info->csum_size, header->csum), BTRFS_CSUM_FMT_VALUE(fs_info->csum_size, calculated_csum)); return; } @@ -801,9 +756,7 @@ static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr) struct btrfs_fs_info *fs_info = stripe->bg->fs_info; struct scrub_sector_verification *sector = &stripe->sectors[sector_nr]; const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; - phys_addr_t paddr = scrub_stripe_get_paddr(stripe, sector_nr); u8 csum_buf[BTRFS_CSUM_SIZE]; - int ret; ASSERT(sector_nr >= 0 && sector_nr < stripe->nr_sectors); @@ -846,8 +799,10 @@ static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr) return; } - ret = btrfs_check_block_csum(fs_info, paddr, csum_buf, sector->csum); - if (ret < 0) { + scrub_calc_vaddr_csum(fs_info, + stripe->buffer + (sector_nr << fs_info->sectorsize_bits), + fs_info->sectorsize, csum_buf); + if (memcmp(csum_buf, sector->csum, fs_info->csum_size)) { scrub_bitmap_set_bit_csum_error(stripe, sector_nr); scrub_bitmap_set_bit_error(stripe, sector_nr); } else { @@ -870,43 +825,65 @@ static void scrub_verify_one_stripe(struct scrub_stripe *stripe, unsigned long b } } -static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *first_bvec) +static unsigned int calc_sector_number(const struct btrfs_bio *bbio) { - int i; + const struct scrub_stripe *stripe = bbio->private; + const struct btrfs_fs_info *fs_info = stripe->bg->fs_info; - for (i = 0; i < stripe->nr_sectors; i++) { - if (scrub_stripe_get_kaddr(stripe, i) == bvec_virt(first_bvec)) - break; - } - ASSERT(i < stripe->nr_sectors); - return i; + /* Scrub bbios all have their @file_offset set to the logical bytenr. */ + ASSERT(bbio->file_offset >= stripe->logical && + bbio->file_offset < stripe->logical + (stripe->nr_sectors << + fs_info->sectorsize_bits), + "scrub bio logical=%llu stripe logical=%llu stripe len=%u", + bbio->file_offset, stripe->logical, + stripe->nr_sectors << fs_info->sectorsize_bits); + return (bbio->file_offset - stripe->logical) >> fs_info->sectorsize_bits; } /* - * Repair read is different to the regular read: + * Common handling of read endio. * - * - Only reads the failed sectors - * - May have extra blocksize limits + * The bbio will be released, so no more access to @bbio after this function. */ -static void scrub_repair_read_endio(struct btrfs_bio *bbio) +static void scrub_read_endio_common(struct btrfs_bio *bbio) { struct scrub_stripe *stripe = bbio->private; struct btrfs_fs_info *fs_info = stripe->bg->fs_info; - int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); + unsigned int sector_nr = calc_sector_number(bbio); const u32 bio_size = bio_get_size(&bbio->bio); + const u32 sectors = bio_size >> fs_info->sectorsize_bits; - ASSERT(sector_nr < stripe->nr_sectors); + + /* + * For vmallocated space, readers need to call invalidate_kernel_vmap_range() + * to manage the coherency between kernel mapping and devie space mapping. + */ + if (is_vmalloc_addr(stripe->buffer)) + invalidate_kernel_vmap_range( + stripe->buffer + (sector_nr << fs_info->sectorsize_bits), + bio_size); if (bbio->bio.bi_status) { - scrub_bitmap_set_io_error(stripe, sector_nr, - bio_size >> fs_info->sectorsize_bits); - scrub_bitmap_set_error(stripe, sector_nr, - bio_size >> fs_info->sectorsize_bits); + scrub_bitmap_set_io_error(stripe, sector_nr, sectors); + scrub_bitmap_set_error(stripe, sector_nr, sectors); } else { - scrub_bitmap_clear_io_error(stripe, sector_nr, - bio_size >> fs_info->sectorsize_bits); + scrub_bitmap_clear_io_error(stripe, sector_nr, sectors); } bio_put(&bbio->bio); +} + +/* + * Repair read is different to the regular read: + * + * - Only reads the failed sectors + * - May have extra blocksize limits + */ +static void scrub_repair_read_endio(struct btrfs_bio *bbio) +{ + struct scrub_stripe *stripe = bbio->private; + + scrub_read_endio_common(bbio); + if (atomic_dec_and_test(&stripe->pending_io)) wake_up(&stripe->io_wait); } @@ -921,30 +898,35 @@ static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *st int sector_nr) { struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; - void *kaddr = scrub_stripe_get_kaddr(stripe, sector_nr); + const u32 offset = sector_nr << fs_info->sectorsize_bits; int ret; - ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), fs_info->sectorsize, - offset_in_page(kaddr)); - /* - * Caller should ensure the bbio has enough size. - * And we cannot use __bio_add_page(), which doesn't do any merge. - * - * Meanwhile for scrub_submit_initial_read() we fully rely on the merge - * to create the minimal amount of bio vectors, for fs block size < page - * size cases. - */ + ASSERT(offset + fs_info->sectorsize <= BTRFS_STRIPE_LEN); + + if (is_vmalloc_addr(stripe->buffer)) { + ret = bio_add_vmalloc(&bbio->bio, stripe->buffer + offset, fs_info->sectorsize); + ASSERT(ret == true); + return; + } + ret = bio_add_page(&bbio->bio, virt_to_page(stripe->buffer + offset), + fs_info->sectorsize, offset_in_page(stripe->buffer + offset)); ASSERT(ret == fs_info->sectorsize); } static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info, - unsigned int nr_vecs, blk_opf_t opf, + blk_opf_t opf, u64 logical, btrfs_bio_end_io_t end_io, void *private) { struct btrfs_bio *bbio; - bbio = btrfs_bio_alloc(nr_vecs, opf, BTRFS_I(fs_info->btree_inode), + /* + * Stripe->buffer is allocated by kvmalloc(), which can be pages at + * different physical addresses, we have to ensure the bbio is large + * enough to contain the full stripe. + */ + bbio = btrfs_bio_alloc(BTRFS_STRIPE_LEN >> PAGE_SHIFT, opf, + BTRFS_I(fs_info->btree_inode), logical, end_io, private); bbio->is_scrub = true; bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT; @@ -976,7 +958,7 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, } if (!bbio) - bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, + bbio = alloc_scrub_bbio(fs_info, REQ_OP_READ, stripe->logical + (i << fs_info->sectorsize_bits), scrub_repair_read_endio, stripe); @@ -1245,20 +1227,9 @@ out: static void scrub_read_endio(struct btrfs_bio *bbio) { struct scrub_stripe *stripe = bbio->private; - int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); - int num_sectors; - const u32 bio_size = bio_get_size(&bbio->bio); - ASSERT(sector_nr < stripe->nr_sectors); - num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits; + scrub_read_endio_common(bbio); - if (bbio->bio.bi_status) { - scrub_bitmap_set_io_error(stripe, sector_nr, num_sectors); - scrub_bitmap_set_error(stripe, sector_nr, num_sectors); - } else { - scrub_bitmap_clear_io_error(stripe, sector_nr, num_sectors); - } - bio_put(&bbio->bio); if (atomic_dec_and_test(&stripe->pending_io)) { wake_up(&stripe->io_wait); INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker); @@ -1270,7 +1241,7 @@ static void scrub_write_endio(struct btrfs_bio *bbio) { struct scrub_stripe *stripe = bbio->private; struct btrfs_fs_info *fs_info = stripe->bg->fs_info; - int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); + unsigned int sector_nr = calc_sector_number(bbio); const u32 bio_size = bio_get_size(&bbio->bio); if (bbio->bio.bi_status) { @@ -1349,7 +1320,7 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *str bbio = NULL; } if (!bbio) - bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_WRITE, + bbio = alloc_scrub_bbio(fs_info, REQ_OP_WRITE, stripe->logical + (sector_nr << fs_info->sectorsize_bits), scrub_write_endio, stripe); scrub_bio_add_sector(bbio, stripe, sector_nr); @@ -1844,7 +1815,7 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) continue; } - bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, + bbio = alloc_scrub_bbio(fs_info, REQ_OP_READ, logical, scrub_read_endio, stripe); } @@ -1869,7 +1840,6 @@ static void scrub_submit_initial_read(struct scrub_ctx *sctx, { struct btrfs_fs_info *fs_info = sctx->fs_info; struct btrfs_bio *bbio; - const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits; int mirror = stripe->mirror_num; @@ -1882,7 +1852,7 @@ static void scrub_submit_initial_read(struct scrub_ctx *sctx, return; } - bbio = alloc_scrub_bbio(fs_info, BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ, + bbio = alloc_scrub_bbio(fs_info, REQ_OP_READ, stripe->logical, scrub_read_endio, stripe); /* Read the whole range inside the chunk boundary. */ for (unsigned int cur = 0; cur < nr_sectors; cur++) @@ -2138,7 +2108,7 @@ static int scrub_raid56_cached_parity(struct scrub_ctx *sctx, for (int i = 0; i < data_stripes; i++) { struct scrub_stripe *stripe = &sctx->raid56_data_stripes[i]; - raid56_parity_cache_data_folios(rbio, stripe->folios, + raid56_parity_cache_data_folios(rbio, stripe->buffer, full_stripe_start + (i << BTRFS_STRIPE_LEN_SHIFT)); } raid56_parity_submit_scrub_rbio(rbio); @@ -3091,14 +3061,6 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, /* At mount time we have ensured nodesize is in the range of [4K, 64K]. */ ASSERT(fs_info->nodesize <= BTRFS_STRIPE_LEN); - /* - * SCRUB_MAX_SECTORS_PER_BLOCK is calculated using the largest possible - * value (max nodesize / min sectorsize), thus nodesize should always - * be fine. - */ - ASSERT(fs_info->nodesize <= - SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits); - /* Allocate outside of device_list_mutex */ sctx = scrub_setup_ctx(fs_info, is_dev_replace); if (IS_ERR(sctx)) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 3ae480c7474b..dca3570168c7 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -130,10 +130,10 @@ static_assert(offsetof(struct backref_cache_entry, entry) == 0); #define SEND_MAX_DIR_CREATED_CACHE_SIZE 64 /* - * Max number of entries in the cache that stores directories that were already - * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses - * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but - * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64). + * Maximum number of entries in the cache that stores utimes values for directories. + * The cache uses raw struct btrfs_lru_cache_entry entries, so it uses at most + * 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but the + * kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64). */ #define SEND_MAX_DIR_UTIMES_CACHE_SIZE 64 @@ -625,9 +625,8 @@ static void fs_path_unreverse(struct fs_path *p) static inline bool is_current_inode_path(const struct send_ctx *sctx, const struct fs_path *path) { - const struct fs_path *cur = &sctx->cur_inode_path; - - return (strncmp(path->start, cur->start, fs_path_len(cur)) == 0); + /* Paths are always nul terminated. */ + return (strcmp(path->start, sctx->cur_inode_path.start) == 0); } static struct btrfs_path *alloc_path_for_send(void) @@ -6033,7 +6032,7 @@ static int send_write_or_clone(struct send_ctx *sctx, int ret = 0; u64 offset = key->offset; u64 end; - u64 bs = sctx->send_root->fs_info->sectorsize; + const u32 bs = sctx->send_root->fs_info->sectorsize; struct btrfs_file_extent_item *ei; u64 disk_byte; u64 data_offset; @@ -8251,7 +8250,7 @@ out: } if (sort_clone_roots) { - for (i = 0; i < sctx->clone_roots_cnt; i++) { + for (i = 0; sctx && i < sctx->clone_roots_cnt; i++) { btrfs_root_dec_send_in_progress( sctx->clone_roots[i].root); btrfs_put_root(sctx->clone_roots[i].root); diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index e6641597b321..39a28e1bec8a 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -2156,7 +2156,7 @@ again: will_reclaim = true; reclaim = true; } - bg->reclaim_mark++; + bg->reclaim_mark = true; spin_unlock(&bg->lock); if (reclaim) btrfs_mark_bg_to_reclaim(bg); diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index 27dd677ca687..ebf18efe1ea3 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -59,7 +59,7 @@ int btrfs_attach_folio_state(const struct btrfs_fs_info *fs_info, if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) return 0; - bfs = btrfs_alloc_folio_state(fs_info, folio_size(folio), type); + bfs = btrfs_alloc_folio_state(fs_info, folio_size(folio), type, GFP_NOFS); if (IS_ERR(bfs)) return PTR_ERR(bfs); @@ -86,7 +86,8 @@ void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio } struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs_info, - size_t fsize, enum btrfs_folio_type type) + size_t fsize, enum btrfs_folio_type type, + gfp_t gfp) { struct btrfs_folio_state *ret; unsigned int real_size; @@ -96,7 +97,7 @@ struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs real_size = struct_size(ret, bitmaps, BITS_TO_LONGS(btrfs_bitmap_nr_max * (fsize >> fs_info->sectorsize_bits))); - ret = kzalloc(real_size, GFP_NOFS); + ret = kzalloc(real_size, gfp); if (!ret) return ERR_PTR(-ENOMEM); diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index 9aceba93c818..9b106a73d682 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -110,7 +110,8 @@ void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio /* Allocate additional data where page represents more than one sector */ struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs_info, - size_t fsize, enum btrfs_folio_type type); + size_t fsize, enum btrfs_folio_type type, + gfp_t gfp); static inline void btrfs_free_folio_state(struct btrfs_folio_state *bfs) { kfree(bfs); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index f4e34898d581..464129b1b0d4 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -129,7 +129,6 @@ enum { /* Rescue options */ Opt_rescue, - Opt_usebackuproot, /* Debugging options */ Opt_enospc_debug, @@ -249,8 +248,6 @@ static const struct fs_parameter_spec btrfs_fs_parameters[] = { /* Rescue options. */ fsparam_enum("rescue", Opt_rescue, btrfs_parameter_rescue), - /* Deprecated, with alias rescue=usebackuproot */ - __fsparam(NULL, "usebackuproot", Opt_usebackuproot, fs_param_deprecated, NULL), /* For compatibility only, alias for "rescue=nologreplay". */ fsparam_flag("norecovery", Opt_norecovery), @@ -514,19 +511,20 @@ static int btrfs_parse_param(struct fs_context *fc, struct fs_parameter *param) btrfs_clear_opt(ctx->mount_opt, NODISCARD); break; case Opt_space_cache: - if (result.negated) { - btrfs_set_opt(ctx->mount_opt, NOSPACECACHE); - btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE); - btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE); - } else { - btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE); - btrfs_set_opt(ctx->mount_opt, SPACE_CACHE); - } + if (!result.negated) + btrfs_warn(NULL, + "v1 space cache is deprecated, falling back to no space cache"); + btrfs_set_opt(ctx->mount_opt, NOSPACECACHE); + btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE); + btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE); break; case Opt_space_cache_version: switch (result.uint_32) { case Opt_space_cache_v1: - btrfs_set_opt(ctx->mount_opt, SPACE_CACHE); + btrfs_warn(NULL, + "v1 space cache is deprecated, falling back to no space cache"); + btrfs_set_opt(ctx->mount_opt, NOSPACECACHE); + btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE); btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE); break; case Opt_space_cache_v2: @@ -560,14 +558,6 @@ static int btrfs_parse_param(struct fs_context *fc, struct fs_parameter *param) else btrfs_set_opt(ctx->mount_opt, AUTO_DEFRAG); break; - case Opt_usebackuproot: - btrfs_warn(NULL, - "'usebackuproot' is deprecated, use 'rescue=usebackuproot' instead"); - btrfs_set_opt(ctx->mount_opt, USEBACKUPROOT); - - /* If we're loading the backup roots we can't trust the space cache. */ - btrfs_set_opt(ctx->mount_opt, CLEAR_CACHE); - break; case Opt_skip_balance: btrfs_set_opt(ctx->mount_opt, SKIP_BALANCE); break; @@ -620,6 +610,7 @@ static int btrfs_parse_param(struct fs_context *fc, struct fs_parameter *param) btrfs_set_opt(ctx->mount_opt, IGNORESUPERFLAGS); btrfs_set_opt(ctx->mount_opt, IGNOREBADROOTS); btrfs_set_opt(ctx->mount_opt, NOLOGREPLAY); + btrfs_set_opt(ctx->mount_opt, USEBACKUPROOT); break; default: btrfs_info(NULL, "unrecognized rescue option '%s'", @@ -668,7 +659,6 @@ static int btrfs_parse_param(struct fs_context *fc, struct fs_parameter *param) */ static void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info) { - btrfs_clear_opt(fs_info->mount_opt, USEBACKUPROOT); btrfs_clear_opt(fs_info->mount_opt, CLEAR_CACHE); btrfs_clear_opt(fs_info->mount_opt, NOSPACECACHE); } @@ -692,7 +682,8 @@ bool btrfs_check_options(const struct btrfs_fs_info *info, bool ret = true; if (!(flags & SB_RDONLY) && - (check_ro_option(info, *mount_opt, BTRFS_MOUNT_NOLOGREPLAY, "nologreplay") || + (check_ro_option(info, *mount_opt, BTRFS_MOUNT_USEBACKUPROOT, "usebackuproot") || + check_ro_option(info, *mount_opt, BTRFS_MOUNT_NOLOGREPLAY, "nologreplay") || check_ro_option(info, *mount_opt, BTRFS_MOUNT_IGNOREBADROOTS, "ignorebadroots") || check_ro_option(info, *mount_opt, BTRFS_MOUNT_IGNOREDATACSUMS, "ignoredatacsums") || check_ro_option(info, *mount_opt, BTRFS_MOUNT_IGNOREMETACSUMS, "ignoremetacsums") || @@ -982,7 +973,7 @@ static int btrfs_fill_super(struct super_block *sb, ret = open_ctree(sb, fs_devices); if (ret) { - btrfs_err(fs_info, "open_ctree failed: %d", ret); + btrfs_err(fs_info, "open_ctree failed: %pe", ERR_PTR(ret)); return ret; } diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index 0d14570c8bc2..39cb01ee441a 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -1336,7 +1336,7 @@ char *btrfs_get_mod_read_policy(void) return read_policy; } -/* Set perms to 0, disable /sys/module/btrfs/parameter/read_policy interface. */ +/* Set perms to 0, disable /sys/module/btrfs/parameters/read_policy interface. */ module_param(read_policy, charp, 0); MODULE_PARM_DESC(read_policy, "Global read policy: pid (default), round-robin[:<min_contig_read>], devid[:<devid>]"); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 8f9419728100..bafc62cf5ebc 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -698,8 +698,6 @@ again: goto alloc_fail; } - xa_init(&h->writeback_inhibited_ebs); - /* * If we are JOIN_NOLOCK we're already committing a transaction and * waiting on this guy, so we don't need to do the sb_start_intwrite @@ -1519,12 +1517,8 @@ static noinline int commit_fs_roots(struct btrfs_trans_handle *trans) ASSERT(atomic_read(&root->log_writers) == 0, "atomic_read(&root->log_writers)=%d", atomic_read(&root->log_writers)); - ASSERT(atomic_read(&root->log_commit[0]) == 0, - "atomic_read(&root->log_commit[0])=%d", - atomic_read(&root->log_commit[0])); - ASSERT(atomic_read(&root->log_commit[1]) == 0, - "atomic_read(&root->log_commit[1])=%d", - atomic_read(&root->log_commit[1])); + ASSERT(!root->log_commit[0]); + ASSERT(!root->log_commit[1]); radix_tree_tag_clear(&fs_info->fs_roots_radix, (unsigned long)btrfs_root_id(root), @@ -1642,7 +1636,7 @@ static int qgroup_account_snapshot(struct btrfs_trans_handle *trans, ret = btrfs_write_and_wait_transaction(trans); if (unlikely(ret)) { btrfs_err(fs_info, -"error while writing out transaction during qgroup snapshot accounting: %d", ret); +"error while writing out transaction during qgroup snapshot accounting: %pe", ERR_PTR(ret)); return ret; } @@ -2588,7 +2582,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) ret = btrfs_write_and_wait_transaction(trans); if (unlikely(ret)) { - btrfs_err(fs_info, "error while writing out transaction: %d", ret); + btrfs_err(fs_info, "error while writing out transaction: %pe", ERR_PTR(ret)); mutex_unlock(&fs_info->tree_log_mutex); goto scrub_continue; } @@ -2749,8 +2743,8 @@ void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, WRITE_ONCE(trans->transaction->aborted, error); trace_btrfs_transaction_abort(trans); if (first_hit) { - btrfs_err(fs_info, "Transaction %llu aborted (error %d)", - trans->transid, error); + btrfs_err(fs_info, "Transaction %llu aborted (%pe)", + trans->transid, ERR_PTR(error)); if (error == -ENOSPC) btrfs_dump_space_info_for_trans_abort(fs_info); } diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 5e4b1106fd90..3a57f227b5ed 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -7,12 +7,12 @@ #define BTRFS_TRANSACTION_H #include <linux/atomic.h> +#include <linux/build_bug.h> #include <linux/refcount.h> #include <linux/list.h> #include <linux/time64.h> #include <linux/mutex.h> #include <linux/wait.h> -#include <linux/xarray.h> #include "btrfs_inode.h" #include "delayed-ref.h" @@ -23,6 +23,7 @@ struct btrfs_fs_info; struct btrfs_root_item; struct btrfs_root; struct btrfs_path; +struct extent_buffer; /* * Signal that a direct IO write is in progress, to avoid deadlock for sync @@ -136,6 +137,18 @@ enum { #define TRANS_EXTWRITERS (__TRANS_START | __TRANS_ATTACH) +/* + * Number of extent buffers a transaction handle tracks for writeback + * inhibition. The CLOCK reference bits pack into a u32 so this must not exceed + * 32, and keeping it a power of two lets the compiler reduce the CLOCK hand + * modulo to a mask. + */ +#define BTRFS_INHIBITED_EBS_SLOTS 8 + +static_assert(BTRFS_INHIBITED_EBS_SLOTS <= 32); +static_assert(BTRFS_INHIBITED_EBS_SLOTS != 0 && + (BTRFS_INHIBITED_EBS_SLOTS & (BTRFS_INHIBITED_EBS_SLOTS - 1)) == 0); + struct btrfs_trans_handle { u64 transid; u64 bytes_reserved; @@ -163,8 +176,14 @@ struct btrfs_trans_handle { struct btrfs_fs_info *fs_info; struct list_head new_bgs; struct btrfs_block_rsv delayed_rsv; - /* Extent buffers with writeback inhibited by this handle. */ - struct xarray writeback_inhibited_ebs; + + /* Extent buffers this handle has inhibited writeback on. */ + struct extent_buffer *inhibited_ebs[BTRFS_INHIBITED_EBS_SLOTS]; + /* CLOCK reference bit per slot. */ + u32 inhibited_ebs_referenced; + u32 nr_inhibited_ebs; + /* CLOCK hand. */ + u32 inhibited_ebs_hand; }; /* diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 875e4ddc68ea..7ba7b6098aa5 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -221,7 +221,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, static int link_to_fixup_dir(struct walk_control *wc, u64 objectid); static noinline int replay_dir_deletes(struct walk_control *wc, u64 dirid, bool del_all); -static void wait_log_commit(struct btrfs_root *root, int transid); +static bool wait_log_commit(struct btrfs_root *root, int transid); /* * tree logging is a special write ahead log used to make sure that @@ -305,24 +305,13 @@ static int start_log_trans(struct btrfs_trans_handle *trans, again: if (root->log_root) { - int index = (root->log_transid + 1) % 2; - if (btrfs_need_log_full_commit(trans)) { ret = BTRFS_LOG_FORCE_COMMIT; goto out; } - if (zoned && atomic_read(&root->log_commit[index])) { - wait_log_commit(root, root->log_transid - 1); + if (zoned && wait_log_commit(root, root->log_transid - 1)) goto again; - } - - if (!root->log_start_pid) { - clear_bit(BTRFS_ROOT_MULTI_LOG_TASKS, &root->state); - root->log_start_pid = current->pid; - } else if (root->log_start_pid != current->pid) { - set_bit(BTRFS_ROOT_MULTI_LOG_TASKS, &root->state); - } } else { /* * This means fs_info->log_root_tree was already created @@ -340,8 +329,6 @@ again: goto out; set_bit(BTRFS_ROOT_HAS_LOG_TREE, &root->state); - clear_bit(BTRFS_ROOT_MULTI_LOG_TASKS, &root->state); - root->log_start_pid = current->pid; } atomic_inc(&root->log_writers); @@ -372,13 +359,9 @@ static int join_running_log_trans(struct btrfs_root *root) mutex_lock(&root->log_mutex); again: if (root->log_root) { - int index = (root->log_transid + 1) % 2; - ret = 0; - if (zoned && atomic_read(&root->log_commit[index])) { - wait_log_commit(root, root->log_transid - 1); + if (zoned && wait_log_commit(root, root->log_transid - 1)) goto again; - } atomic_inc(&root->log_writers); } mutex_unlock(&root->log_mutex); @@ -2986,6 +2969,7 @@ static noinline int walk_down_log_tree(struct btrfs_path *path, int *level, { struct btrfs_trans_handle *trans = wc->trans; struct btrfs_fs_info *fs_info = wc->log->fs_info; + struct btrfs_eb_prealloc pa = { 0 }; u64 bytenr; u64 ptr_gen; struct extent_buffer *next; @@ -3010,7 +2994,7 @@ static noinline int walk_down_log_tree(struct btrfs_path *path, int *level, check.has_first_key = true; btrfs_node_key_to_cpu(cur, &check.first_key, path->slots[*level]); - next = btrfs_find_create_tree_block(fs_info, bytenr, + next = btrfs_find_create_tree_block(fs_info, &pa, bytenr, btrfs_header_owner(cur), *level - 1); if (IS_ERR(next)) { @@ -3181,10 +3165,14 @@ static int update_log_root(struct btrfs_trans_handle *trans, return ret; } -static void wait_log_commit(struct btrfs_root *root, int transid) +/* Returns true if we had to wait, false otherwise. */ +static bool wait_log_commit(struct btrfs_root *root, int transid) { DEFINE_WAIT(wait); - int index = transid % 2; + const int index = (transid >= 0 ? transid % 2 : -transid % 2); + + if (!root->log_commit[index]) + return false; /* * we only allow two pending log transactions at a time, @@ -3195,15 +3183,17 @@ static void wait_log_commit(struct btrfs_root *root, int transid) prepare_to_wait(&root->log_commit_wait[index], &wait, TASK_UNINTERRUPTIBLE); - if (!(root->log_transid_committed < transid && - atomic_read(&root->log_commit[index]))) - break; - mutex_unlock(&root->log_mutex); schedule(); mutex_lock(&root->log_mutex); + + if (!(root->log_transid_committed < transid && + root->log_commit[index])) + break; } finish_wait(&root->log_commit_wait[index], &wait); + + return true; } static void wait_for_writer(struct btrfs_root *root) @@ -3307,15 +3297,15 @@ static inline void btrfs_remove_all_log_ctxs(struct btrfs_root *root, int btrfs_sync_log(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_log_ctx *ctx) { - int index1; - int index2; int mark; int ret; struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *log = root->log_root; struct btrfs_root *log_root_tree = fs_info->log_root_tree; struct btrfs_root_item new_root_item; - int log_transid = 0; + int log_transid = ctx->log_transid; + int index1 = log_transid % 2; + int index2; struct btrfs_log_ctx root_log_ctx; struct blk_plug plug; u64 log_root_start; @@ -3323,41 +3313,25 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, mutex_lock(&root->log_mutex); trace_btrfs_sync_log_enter(trans, root, ctx); - log_transid = ctx->log_transid; if (root->log_transid_committed >= log_transid) { trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); mutex_unlock(&root->log_mutex); return ctx->log_ret; } - index1 = log_transid % 2; - if (atomic_read(&root->log_commit[index1])) { - wait_log_commit(root, log_transid); + if (wait_log_commit(root, log_transid)) { trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); mutex_unlock(&root->log_mutex); return ctx->log_ret; } ASSERT(log_transid == root->log_transid, "log_transid=%d root->log_transid=%d", log_transid, root->log_transid); - atomic_set(&root->log_commit[index1], 1); + root->log_commit[index1] = true; /* wait for previous tree log sync to complete */ - if (atomic_read(&root->log_commit[(index1 + 1) % 2])) - wait_log_commit(root, log_transid - 1); + wait_log_commit(root, log_transid - 1); - while (1) { - int batch = atomic_read(&root->log_batch); - /* when we're on an ssd, just kick the log commit out */ - if (!btrfs_test_opt(fs_info, SSD) && - test_bit(BTRFS_ROOT_MULTI_LOG_TASKS, &root->state)) { - mutex_unlock(&root->log_mutex); - schedule_timeout_uninterruptible(1); - mutex_lock(&root->log_mutex); - } - wait_for_writer(root); - if (batch == atomic_read(&root->log_batch)) - break; - } + wait_for_writer(root); /* bail out if we need to do a full commit */ if (btrfs_need_log_full_commit(trans)) { @@ -3414,7 +3388,6 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, btrfs_set_root_log_transid(root, root->log_transid + 1); log->log_transid = root->log_transid; - root->log_start_pid = 0; /* * IO has been started, blocks of the log tree have WRITTEN flag set * in their headers. new modifications of the log will be written to @@ -3473,7 +3446,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, goto out; } - if (atomic_read(&log_root_tree->log_commit[index2])) { + if (log_root_tree->log_commit[index2]) { blk_finish_plug(&plug); ret = btrfs_wait_tree_log_extents(log, mark); wait_log_commit(log_root_tree, @@ -3487,12 +3460,9 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, ASSERT(root_log_ctx.log_transid == log_root_tree->log_transid, "root_log_ctx.log_transid=%d log_root_tree->log_transid=%d", root_log_ctx.log_transid, log_root_tree->log_transid); - atomic_set(&log_root_tree->log_commit[index2], 1); + log_root_tree->log_commit[index2] = true; - if (atomic_read(&log_root_tree->log_commit[(index2 + 1) % 2])) { - wait_log_commit(log_root_tree, - root_log_ctx.log_transid - 1); - } + wait_log_commit(log_root_tree, root_log_ctx.log_transid - 1); /* * now that we've moved on to the tree of log tree roots, @@ -3590,7 +3560,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, /* * We know there can only be one task here, since we have not yet set - * root->log_commit[index1] to 0 and any task attempting to sync the + * root->log_commit[index1] to false and any task attempting to sync the * log must wait for the previous log transaction to commit if it's * still in progress or wait for the current log transaction commit if * someone else already started it. We use <= and not < because the @@ -3606,7 +3576,7 @@ out_wake_log_root: btrfs_remove_all_log_ctxs(log_root_tree, index2, ret); log_root_tree->log_transid_committed++; - atomic_set(&log_root_tree->log_commit[index2], 0); + log_root_tree->log_commit[index2] = false; mutex_unlock(&log_root_tree->log_mutex); /* @@ -3619,7 +3589,7 @@ out: mutex_lock(&root->log_mutex); btrfs_remove_all_log_ctxs(root, index1, ret); root->log_transid_committed++; - atomic_set(&root->log_commit[index1], 0); + root->log_commit[index1] = false; mutex_unlock(&root->log_mutex); /* @@ -5622,6 +5592,15 @@ static int btrfs_log_holes(struct btrfs_trans_handle *trans, if (!btrfs_fs_incompat(fs_info, NO_HOLES) || i_size == 0) return 0; + /* + * If there are no prealloc extents (which can be located past i_size), + * and disk space used is greater than or equals to i_size, then there + * are no holes. + */ + if (!(inode->flags & BTRFS_INODE_PREALLOC) && + i_size <= inode_get_bytes(&inode->vfs_inode)) + return 0; + key.objectid = ino; key.type = BTRFS_EXTENT_DATA_KEY; key.offset = 0; diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 983365a73541..4e0ab5842274 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -638,7 +638,7 @@ rollback: rollback_ret = rollback_verity(inode); if (rollback_ret) btrfs_err(inode->root->fs_info, - "failed to rollback verity items: %d", rollback_ret); + "failed to rollback verity items: %pe", ERR_PTR(rollback_ret)); return ret; } @@ -720,14 +720,18 @@ again: goto out; folio_lock(folio); - /* If it's not uptodate after we have the lock, we got a read error. */ - if (!folio_test_uptodate(folio)) { + /* Folio was truncated from mapping. */ + if (!folio->mapping) { folio_unlock(folio); folio_put(folio); - return ERR_PTR(-EIO); + goto again; } - folio_unlock(folio); - goto out; + /* Another reader may have filled the folio while we waited. */ + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + goto out; + } + goto read_folio; } folio = filemap_alloc_folio(mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS), @@ -744,6 +748,7 @@ again: return ERR_PTR(ret); } +read_folio: /* * Merkle item keys are indexed from byte 0 in the merkle tree. * They have the form: @@ -753,6 +758,7 @@ again: ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_MERKLE_ITEM_KEY, off, folio_address(folio), PAGE_SIZE, folio); if (ret < 0) { + folio_unlock(folio); folio_put(folio); return ERR_PTR(ret); } diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index e68ce323bb06..9b66eb584ece 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -749,41 +749,6 @@ const u8 *btrfs_sb_fsid_ptr(const struct btrfs_super_block *sb) return has_metadata_uuid ? sb->metadata_uuid : sb->fsid; } -static bool is_same_device(struct btrfs_device *device, const char *new_path) -{ - struct path old = { .mnt = NULL, .dentry = NULL }; - struct path new = { .mnt = NULL, .dentry = NULL }; - char AUTO_KFREE(old_path); - bool is_same = false; - int ret; - - if (!device->name) - goto out; - - old_path = kzalloc(PATH_MAX, GFP_NOFS); - if (!old_path) - goto out; - - rcu_read_lock(); - ret = strscpy(old_path, rcu_dereference(device->name), PATH_MAX); - rcu_read_unlock(); - if (ret < 0) - goto out; - - ret = kern_path(old_path, LOOKUP_FOLLOW, &old); - if (ret) - goto out; - ret = kern_path(new_path, LOOKUP_FOLLOW, &new); - if (ret) - goto out; - if (path_equal(&old, &new)) - is_same = true; -out: - path_put(&old); - path_put(&new); - return is_same; -} - /* * Add new device to list of registered devices * @@ -904,7 +869,7 @@ static noinline struct btrfs_device *device_list_add(const char *path, MAJOR(path_devt), MINOR(path_devt), current->comm, task_pid_nr(current)); - } else if (!device->name || !is_same_device(device, path)) { + } else if (!device->name || device->devt != path_devt) { const char *old_name; /* @@ -4670,7 +4635,7 @@ again: if (ret == -ENOSPC) { enospc_errors++; } else if (ret == -ETXTBSY) { - btrfs_info(fs_info, + btrfs_warn(fs_info, "skipping relocation of block group %llu due to active swapfile", found_key.offset); ret = 0; @@ -6131,6 +6096,19 @@ struct btrfs_chunk_map *btrfs_alloc_chunk_map(int num_stripes, gfp_t gfp) return map; } +static void set_real_chunk_type(struct btrfs_chunk_map *map) +{ + map->type = map->on_disk_type; + if (likely((map->on_disk_type & BTRFS_BLOCK_GROUP_RAID56_MASK) == 0 || + nr_data_stripes(map) > 1)) + return; + if (map->on_disk_type & BTRFS_BLOCK_GROUP_RAID5) + map->type |= BTRFS_BLOCK_GROUP_RAID1; + else + map->type |= BTRFS_BLOCK_GROUP_RAID1C3; + map->type &= ~BTRFS_BLOCK_GROUP_RAID56_MASK; +} + static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans, struct alloc_chunk_ctl *ctl, struct btrfs_device_info *devices_info) @@ -6149,11 +6127,10 @@ static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans, map->start = start; map->chunk_len = ctl->chunk_size; map->stripe_size = ctl->stripe_size; - map->type = type; - map->io_align = BTRFS_STRIPE_LEN; - map->io_width = BTRFS_STRIPE_LEN; + map->on_disk_type = type; map->sub_stripes = ctl->sub_stripes; map->num_stripes = ctl->num_stripes; + set_real_chunk_type(map); for (int i = 0; i < ctl->ndevs; i++) { for (int j = 0; j < ctl->dev_stripes; j++) { @@ -6332,7 +6309,7 @@ int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans, btrfs_set_stack_chunk_length(chunk, bg->length); btrfs_set_stack_chunk_owner(chunk, BTRFS_EXTENT_TREE_OBJECTID); btrfs_set_stack_chunk_stripe_len(chunk, BTRFS_STRIPE_LEN); - btrfs_set_stack_chunk_type(chunk, map->type); + btrfs_set_stack_chunk_type(chunk, map->on_disk_type); btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes); btrfs_set_stack_chunk_io_align(chunk, BTRFS_STRIPE_LEN); btrfs_set_stack_chunk_io_width(chunk, BTRFS_STRIPE_LEN); @@ -7714,9 +7691,7 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, map->start = logical; map->chunk_len = length; map->num_stripes = num_stripes; - map->io_width = btrfs_chunk_io_width(leaf, chunk); - map->io_align = btrfs_chunk_io_align(leaf, chunk); - map->type = type; + map->on_disk_type = type; /* * We can't use the sub_stripes value, as for profiles other than * RAID10, they may have 0 as sub_stripes for filesystems created by @@ -7727,6 +7702,7 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, */ map->sub_stripes = btrfs_raid_array[index].sub_stripes; map->verified_stripes = 0; + set_real_chunk_type(map); if (num_stripes > 0) map->stripe_size = btrfs_calc_stripe_length(map); diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index df2c671ab6fa..0415d74cad9b 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -632,9 +632,15 @@ struct btrfs_chunk_map { u64 start; u64 chunk_len; u64 stripe_size; + /* + * The real type that is utilized during logical address mapping. + * + * For most profiles it matches @on_disk_type, but for single-data-RAID56, + * the real type will be set to RAID1/RAID1C3, to avoid unsupported + * operations from raid56 lib. + */ u64 type; - int io_align; - int io_width; + u64 on_disk_type; int num_stripes; int sub_stripes; struct btrfs_io_stripe stripes[]; diff --git a/fs/buffer.c b/fs/buffer.c index 65a2893b2cbb..dd50d17b8907 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2081,6 +2081,7 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) { size_t block_start, block_end; bool partial = false; + bool uptodate = folio_test_uptodate(folio); unsigned blocksize; struct buffer_head *bh, *head; @@ -2103,6 +2104,8 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) clear_buffer_new(bh); block_start = block_end; + if (uptodate && block_start >= to) + break; bh = bh->b_this_page; } while (bh != head); @@ -392,12 +392,12 @@ int dax_folio_reset_order(struct folio *folio) int order = folio_order(folio); /* - * DAX maintains the invariant that folio->share != 0 only when - * folio->mapping == NULL (enforced by dax_folio_make_shared()). - * Equivalently: folio->mapping != NULL implies folio->share == 0. - * Callers ensure share has been decremented to zero before - * calling here, so unconditionally clearing both fields is - * correct. + * Clear the mapping and the index/share union word. folio->share + * and folio->index occupy the same union in struct folio. For + * non-shared folios (mapping != NULL), the union holds folio->index + * (file page offset); for shared folios (mapping == NULL), it holds + * folio->share (reference count). Either way, we are releasing the + * folio and both fields should be zeroed. */ folio->mapping = NULL; folio->share = 0; diff --git a/fs/erofs/data.c b/fs/erofs/data.c index d2f01245ee79..20962f8b60a8 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -30,20 +30,6 @@ void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap) { pgoff_t index = (buf->off + offset) >> PAGE_SHIFT; struct folio *folio = NULL; - loff_t fpos; - int err; - - /* - * Metadata access for file-backed mounts reuses page cache of backing - * fs inodes (only folio data will be needed) to prevent double caching. - * However, the data access range must be verified here in advance. - */ - if (buf->file) { - fpos = (loff_t)index << PAGE_SHIFT; - err = rw_verify_area(READ, buf->file, &fpos, PAGE_SIZE); - if (err < 0) - return ERR_PTR(err); - } if (buf->page) { folio = page_folio(buf->page); @@ -52,7 +38,8 @@ void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap) } if (!folio || !folio_contains(folio, index)) { erofs_put_metabuf(buf); - folio = read_mapping_folio(buf->mapping, index, buf->file); + folio = read_cache_folio(buf->mapping, index, + buf->mc ? erofs_read_meta_folio : NULL, NULL); if (IS_ERR(folio)) return folio; } @@ -69,19 +56,20 @@ int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, { struct erofs_sb_info *sbi = EROFS_SB(sb); - buf->file = NULL; + buf->mc = false; if (in_metabox) { if (unlikely(!sbi->metabox_inode)) return -EFSCORRUPTED; buf->mapping = sbi->metabox_inode->i_mapping; return 0; } - buf->off = sbi->dif0.fsoff; if (erofs_is_fileio_mode(sbi)) { - buf->file = sbi->dif0.file; /* some fs like FUSE needs it */ - buf->mapping = buf->file->f_mapping; - } else + buf->mapping = sbi->managed_cache->i_mapping; + buf->mc = true; + } else { + buf->off = sbi->dif0.fsoff; buf->mapping = sb->s_bdev->bd_mapping; + } return 0; } diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 3c54e95964c9..27caf4bebddc 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -299,6 +299,7 @@ static const char *z_erofs_transform_plain(struct z_erofs_decompress_req *rq, return ERR_PTR(-EOPNOTSUPP); if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { cur = bs - (rq->pageofs_out & (bs - 1)); + DBG_BUGON(rq->pageofs_in & (bs - 1)); pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; cur = min(cur, rq->outputsize); if (cur && rq->out[0]) { diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c index 98cdaa1cd1a7..ebb81a7ffd4c 100644 --- a/fs/erofs/fileio.c +++ b/fs/erofs/fileio.c @@ -194,3 +194,24 @@ const struct address_space_operations erofs_fileio_aops = { .read_folio = erofs_fileio_read_folio, .readahead = erofs_fileio_readahead, }; + +int erofs_read_meta_folio(struct file *file, struct folio *folio) +{ + struct erofs_fileio io = { + .dev = { .m_pa = folio_pos(folio), }, + }; + struct inode *inode = folio_inode(folio); + int err; + + err = erofs_map_dev(inode->i_sb, &io.dev); + if (err) + return err; + + io.rq = erofs_fileio_rq_alloc(&io.dev); + io.rq->bio.bi_iter.bi_sector = + (io.dev.m_dif->fsoff + io.dev.m_pa) >> 9; + erofs_onlinefolio_init(folio); + bio_add_folio_nofail(&io.rq->bio, folio, folio_size(folio), 0); + erofs_fileio_rq_submit(io.rq); + return 0; +} diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 57bd21859c65..9b7370f0f3df 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -98,11 +98,9 @@ struct erofs_sb_info { unsigned int sync_decompress; /* strategy for sync decompression */ unsigned int shrinker_run_no; - /* pseudo inode to manage cached pages */ - struct inode *managed_cache; - struct erofs_sb_lz4_info lz4; #endif /* CONFIG_EROFS_FS_ZIP */ + struct inode *managed_cache; /* pseudo inode to cache physical data */ struct inode *packed_inode; struct inode *metabox_inode; struct erofs_dev_context *devs; @@ -176,10 +174,10 @@ enum { struct erofs_buf { struct address_space *mapping; - struct file *file; u64 off; struct page *page; void *base; + bool mc; }; #define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL }) @@ -269,7 +267,7 @@ struct erofs_inode { #ifdef CONFIG_EROFS_FS_ZIP struct { unsigned short z_advise; - unsigned char z_algorithmtype[2]; + unsigned char z_algofmt[2]; unsigned char z_lclusterbits; union { u64 z_tailextent_headlcn; @@ -399,6 +397,12 @@ extern const struct file_operations erofs_ishare_fops; extern const struct iomap_ops z_erofs_iomap_report_ops; +int erofs_setup_managed_cache(struct super_block *sb); +#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE +int erofs_read_meta_folio(struct file *file, struct folio *folio); +#else +#define erofs_read_meta_folio NULL +#endif void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf, erofs_off_t *offset, int *lengthp); void erofs_unmap_metabuf(struct erofs_buf *buf); diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 770808685934..0967aace9f7f 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -386,6 +386,7 @@ static void erofs_default_options(struct erofs_sb_info *sbi) enum { Opt_user_xattr, Opt_acl, Opt_cache_strategy, Opt_dax, Opt_dax_enum, Opt_device, Opt_domain_id, Opt_directio, Opt_fsoffset, Opt_inode_share, + Opt_source, }; static const struct constant_table erofs_param_cache_strategy[] = { @@ -402,17 +403,18 @@ static const struct constant_table erofs_dax_param_enums[] = { }; static const struct fs_parameter_spec erofs_fs_parameters[] = { - fsparam_flag_no("user_xattr", Opt_user_xattr), - fsparam_flag_no("acl", Opt_acl), - fsparam_enum("cache_strategy", Opt_cache_strategy, + fsparam_flag_no("user_xattr", Opt_user_xattr), + fsparam_flag_no("acl", Opt_acl), + fsparam_enum("cache_strategy", Opt_cache_strategy, erofs_param_cache_strategy), - fsparam_flag("dax", Opt_dax), - fsparam_enum("dax", Opt_dax_enum, erofs_dax_param_enums), - fsparam_string("device", Opt_device), - fsparam_string("domain_id", Opt_domain_id), - fsparam_flag_no("directio", Opt_directio), - fsparam_u64("fsoffset", Opt_fsoffset), - fsparam_flag("inode_share", Opt_inode_share), + fsparam_flag("dax", Opt_dax), + fsparam_enum("dax", Opt_dax_enum, erofs_dax_param_enums), + fsparam_string("device", Opt_device), + fsparam_string("domain_id", Opt_domain_id), + fsparam_flag_no("directio", Opt_directio), + fsparam_u64("fsoffset", Opt_fsoffset), + fsparam_flag("inode_share", Opt_inode_share), + fsparam_file_or_string("source", Opt_source), {} }; @@ -437,6 +439,40 @@ static bool erofs_fc_set_dax_mode(struct fs_context *fc, unsigned int mode) return false; } +static int erofs_fc_parse_source(struct fs_context *fc, + struct fs_parameter *param) +{ + struct erofs_sb_info *sbi = fc->s_fs_info; + + if (fc->source || sbi->dif0.file) + return invalf(fc, "Multiple sources"); + + switch (param->type) { + case fs_value_is_string: + fc->source = param->string; + param->string = NULL; + return 0; + case fs_value_is_file: { + char *buf __free(kfree) = kmalloc(PATH_MAX, GFP_KERNEL); + char *p; + + if (!buf) + return -ENOMEM; + p = file_path(param->file, buf, PATH_MAX); + if (IS_ERR(p)) + return PTR_ERR(p); + fc->source = kstrdup(p, GFP_KERNEL); + if (!fc->source) + return -ENOMEM; + sbi->dif0.file = no_free_ptr(param->file); + return 0; + } + default: + WARN_ON_ONCE(true); + return -EINVAL; + } +} + static int erofs_fc_parse_param(struct fs_context *fc, struct fs_parameter *param) { @@ -524,6 +560,8 @@ static int erofs_fc_parse_param(struct fs_context *fc, else set_opt(&sbi->opt, INODE_SHARE); break; + case Opt_source: + return erofs_fc_parse_source(fc, param); } return 0; } @@ -595,6 +633,21 @@ static const struct export_operations erofs_export_ops = { .get_parent = erofs_get_parent, }; +int erofs_setup_managed_cache(struct super_block *sb) +{ + if (!EROFS_SB(sb)->managed_cache) { + struct inode *inode = new_inode(sb); + + if (!inode) + return -ENOMEM; + set_nlink(inode, 1); + inode->i_size = OFFSET_MAX; + mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); + EROFS_SB(sb)->managed_cache = inode; + } + return 0; +} + static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) { struct inode *inode; @@ -607,16 +660,16 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_op = &erofs_sops; if (!sbi->domain_id && test_opt(&sbi->opt, INODE_SHARE)) { - errorfc(fc, "domain_id is needed when inode_ishare is on"); + errorfc(fc, "domain_id is needed when inode_share is on"); return -EINVAL; } if (test_opt(&sbi->opt, DAX_ALWAYS) && test_opt(&sbi->opt, INODE_SHARE)) { - errorfc(fc, "FSDAX is not allowed when inode_ishare is on"); + errorfc(fc, "FSDAX is not allowed when inode_share is on"); return -EINVAL; } sbi->blkszbits = PAGE_SHIFT; - if (!sb->s_bdev) { + if (erofs_is_fileio_mode(sbi)) { /* * (File-backed mounts) EROFS claims it's safe to nest other * fs contexts (including its own) due to self-controlled RO @@ -631,14 +684,11 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) * It MUST change if another fs plans to support them, which * may also require adjusting FILESYSTEM_MAX_STACK_DEPTH. */ - if (erofs_is_fileio_mode(sbi)) { - inode = file_inode(sbi->dif0.file); - if ((inode->i_sb->s_op == &erofs_sops && - !inode->i_sb->s_bdev) || - inode->i_sb->s_stack_depth) { - erofs_err(sb, "file-backed mounts cannot be applied to stacked fses"); - return -ENOTBLK; - } + inode = file_inode(sbi->dif0.file); + if ((inode->i_sb->s_op == &erofs_sops && + !inode->i_sb->s_bdev) || inode->i_sb->s_stack_depth) { + erofs_err(sb, "file-backed mounts cannot be applied to stacked fses"); + return -ENOTBLK; } sb->s_blocksize = PAGE_SIZE; sb->s_blocksize_bits = PAGE_SHIFT; @@ -646,6 +696,9 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) err = super_setup_bdi(sb); if (err) return err; + err = erofs_setup_managed_cache(sb); + if (err) + return err; snprintf(sb->s_id, sizeof(sb->s_id), "%u:%u", MAJOR(sb->s_dev), MINOR(sb->s_dev)); @@ -743,13 +796,26 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) static int erofs_fc_get_tree(struct fs_context *fc) { + struct erofs_sb_info *sbi = fc->s_fs_info; int ret; + if (sbi->dif0.file) { + if (!IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE)) { + errorfc(fc, "source fd option not supported"); + return -EINVAL; + } + if (!S_ISREG(file_inode(sbi->dif0.file)->i_mode) || + !sbi->dif0.file->f_mapping->a_ops->read_folio) { + errorfc(fc, "source is unsupported"); + return -EINVAL; + } + return get_tree_nodev(fc, erofs_fc_fill_super); + } + ret = get_tree_bdev_flags(fc, erofs_fc_fill_super, IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) ? GET_TREE_BDEV_QUIET_LOOKUP : 0); if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && ret == -ENOTBLK) { - struct erofs_sb_info *sbi = fc->s_fs_info; struct file *file; if (!fc->source) @@ -873,10 +939,8 @@ static void erofs_drop_internal_inodes(struct erofs_sb_info *sbi) sbi->packed_inode = NULL; iput(sbi->metabox_inode); sbi->metabox_inode = NULL; -#ifdef CONFIG_EROFS_FS_ZIP iput(sbi->managed_cache); sbi->managed_cache = NULL; -#endif } static void erofs_kill_sb(struct super_block *sb) diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index d022d1dff5a1..0de7d561f189 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -128,7 +128,17 @@ struct z_erofs_pcluster_slab { #define _PCLP(n) { .maxpages = n } static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { - _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), + _PCLP(1), + _PCLP(4), +#if Z_EROFS_PCLUSTER_MAX_PAGES > 16 + _PCLP(16), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 64 + _PCLP(64), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 128 + _PCLP(128), +#endif _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1) }; @@ -666,21 +676,16 @@ static const struct address_space_operations z_erofs_cache_aops = { int z_erofs_init_super(struct super_block *sb) { - struct inode *inode; int err; err = z_erofs_init_pcpu_workers(sb); if (err) return err; - inode = new_inode(sb); - if (!inode) - return -ENOMEM; - set_nlink(inode, 1); - inode->i_size = OFFSET_MAX; - inode->i_mapping->a_ops = &z_erofs_cache_aops; - mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); - EROFS_SB(sb)->managed_cache = inode; + err = erofs_setup_managed_cache(sb); + if (err) + return err; + EROFS_SB(sb)->managed_cache->i_mapping->a_ops = &z_erofs_cache_aops; xa_init(&EROFS_SB(sb)->managed_pslots); return 0; } diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 5f33af3fdf97..f8981fc74246 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -483,14 +483,14 @@ static int z_erofs_map_blocks_fo(struct inode *inode, } if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { - if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) + if ((vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) && + !(map->m_flags & EROFS_MAP_META)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { - map->m_algorithmformat = vi->z_algorithmtype[1]; } else { - map->m_algorithmformat = vi->z_algorithmtype[0]; + map->m_algorithmformat = + vi->z_algofmt[m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2]; } if ((flags & EROFS_GET_BLOCKS_FIEMAP) || @@ -604,9 +604,14 @@ static int z_erofs_map_blocks_ext(struct inode *inode, if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; - if (fmt) - map->m_algorithmformat = fmt - 1; - else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) + if (fmt) { + map->m_algorithmformat = --fmt; + if (fmt >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", + fmt, map->m_la, vi->nid); + return -EOPNOTSUPP; + } + } else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else @@ -624,7 +629,7 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) struct super_block *const sb = inode->i_sb; struct z_erofs_map_header *h; erofs_off_t pos; - int err = 0; + int err = 0, nr; if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { /* @@ -667,12 +672,19 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) goto done; } - vi->z_algorithmtype[0] = h->h_algorithmtype & 15; - vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); else if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) vi->z_idata_size = le16_to_cpu(h->h_idata_size); + for (nr = 0; nr < 2; ++nr) { + vi->z_algofmt[nr] = (h->h_algorithmtype >> (4 * nr)) & 15; + if (vi->z_algofmt[nr] >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", + nr + 1, vi->z_algofmt[nr], vi->nid); + err = -EOPNOTSUPP; + goto out_unlock; + } + } if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | @@ -720,12 +732,8 @@ static int z_erofs_map_sanity_check(struct inode *inode, if (!(map->m_flags & EROFS_MAP_MAPPED)) return 0; - if (unlikely(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX)) { - erofs_err(inode->i_sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", - map->m_algorithmformat, map->m_la, EROFS_I(inode)->nid); - return -EOPNOTSUPP; - } + DBG_BUGON(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX); if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", diff --git a/fs/ext2/xattr.c b/fs/ext2/xattr.c index be63f89402a3..9b68c490ab26 100644 --- a/fs/ext2/xattr.c +++ b/fs/ext2/xattr.c @@ -780,16 +780,12 @@ ext2_xattr_set2(struct inode *inode, struct buffer_head *old_bh, mark_inode_dirty(inode); if (IS_SYNC(inode)) { error = sync_inode_metadata(inode, 1); - /* In case sync failed due to ENOSPC the inode was actually - * written (only some dirty data were not) so we just proceed - * as if nothing happened and cleanup the unused block */ - if (error && error != -ENOSPC) { - if (new_bh && new_bh != old_bh) { - dquot_free_block_nodirty(inode, 1); - mark_inode_dirty(inode); - } + /* + * Inode writeout failed. Backing everything out is complex so + * let's just leave it for e2fsck to cleanup the mess. + */ + if (error) goto cleanup; - } } error = 0; diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c index 8040c731b3e4..52f4c5169f91 100644 --- a/fs/ext4/balloc.c +++ b/fs/ext4/balloc.c @@ -331,9 +331,13 @@ struct ext4_group_info *ext4_get_group_info(struct super_block *sb, if (unlikely(group >= EXT4_SB(sb)->s_groups_count)) return NULL; + if (unlikely(!EXT4_SB(sb)->s_group_info)) + return NULL; indexv = group >> (EXT4_DESC_PER_BLOCK_BITS(sb)); indexh = group & ((EXT4_DESC_PER_BLOCK(sb)) - 1); grp_info = sbi_array_rcu_deref(EXT4_SB(sb), s_group_info, indexv); + if (unlikely(!grp_info)) + return NULL; return grp_info[indexh]; } diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c index 9265cfe62c83..1a0fccb084ef 100644 --- a/fs/ext4/crypto.c +++ b/fs/ext4/crypto.c @@ -144,7 +144,13 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, if (inode->i_ino == EXT4_ROOT_INO) return -EPERM; - if (WARN_ON_ONCE(IS_DAX(inode) && i_size_read(inode))) + /* + * For new encrypted inodes, S_DAX is never set in the first place. + * + * For existing inodes, this is called only on empty directories. ext4 + * never sets S_DAX on directories. + */ + if (WARN_ON_ONCE(IS_DAX(inode))) return -EINVAL; if (ext4_test_inode_flag(inode, EXT4_INODE_DAX)) @@ -164,27 +170,24 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, if (handle) { /* + * __ext4_new_inode() should have already set the encrypt flag + * on the inode and avoided enabling inline data. + */ + if (WARN_ON_ONCE(!IS_ENCRYPTED(inode))) + return -EINVAL; + if (WARN_ON_ONCE(ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))) + return -EINVAL; + /* * Since the inode is new it is ok to pass the * XATTR_CREATE flag. This is necessary to match the * remaining journal credits check in the set_handle * function with the credits allocated for the new * inode. */ - res = ext4_xattr_set_handle(handle, inode, - EXT4_XATTR_INDEX_ENCRYPTION, - EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, - ctx, len, XATTR_CREATE); - if (!res) { - ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - ext4_clear_inode_state(inode, - EXT4_STATE_MAY_INLINE_DATA); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ - ext4_set_inode_flags(inode, false); - } - return res; + return ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, XATTR_CREATE); } res = dquot_initialize(inode); @@ -205,10 +208,7 @@ retry: ctx, len, 0); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ + /* Update inode->i_flags to set S_ENCRYPTED. */ ext4_set_inode_flags(inode, false); res = ext4_mark_inode_dirty(handle, inode); if (res) diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c index 17edd678fa87..8d7b81e6948e 100644 --- a/fs/ext4/dir.c +++ b/fs/ext4/dir.c @@ -138,6 +138,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) struct buffer_head *bh = NULL; struct fscrypt_str fstr = FSTR_INIT(NULL, 0); struct dir_private_info *info = file->private_data; + bool has_csum = ext4_has_feature_metadata_csum(sb); err = fscrypt_prepare_readdir(inode); if (err) @@ -149,7 +150,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) return err; /* Can we just clear INDEX flag to ignore htree information? */ - if (!ext4_has_feature_metadata_csum(sb)) { + if (!has_csum) { /* * We don't set the inode dirty flag since it's not * critical that it gets flushed back to the disk. @@ -235,7 +236,10 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) * dirent right now. Scan from the start of the block * to make sure. */ if (!inode_eq_iversion(inode, info->cookie)) { - for (i = 0; i < sb->s_blocksize && i < offset; ) { + for (i = 0; + i <= sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode) && + i < offset;) { de = (struct ext4_dir_entry_2 *) (bh->b_data + i); /* It's too expensive to do a full @@ -257,6 +261,17 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) info->cookie = inode_query_iversion(inode); } + if (unlikely(offset < sb->s_blocksize && + offset > sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode))) { + EXT4_ERROR_FILE(file, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, size=%lu", + "directory entry too close to block end", + offset, sb->s_blocksize); + ctx->pos = round_up(ctx->pos, sb->s_blocksize); + goto next_block; + } + while (ctx->pos < inode->i_size && offset < sb->s_blocksize) { de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); @@ -312,6 +327,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) ctx->pos += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); } +next_block: if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode)) goto done; brelse(bh); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 2fdff50526e9..724a27e8be61 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -334,7 +334,7 @@ struct ext4_io_submit { #define EXT4_MAX_BLOCK_SIZE 65536 #define EXT4_MIN_BLOCK_LOG_SIZE 10 #define EXT4_MAX_BLOCK_LOG_SIZE 16 -#define EXT4_MAX_CLUSTER_LOG_SIZE 30 +#define EXT4_MAX_CLUSTER_LOG_SIZE 28 #ifdef __KERNEL__ # define EXT4_BLOCK_SIZE(s) ((s)->s_blocksize) #else @@ -1070,8 +1070,14 @@ struct ext4_inode_info { * between readers of EAs and writers of regular file data, so * instead we synchronize on xattr_sem when reading or changing * EAs. + * + * EA inodes (EXT4_EA_INODE_FL) do not use xattr_sem; they reuse + * the space for deferred iput linkage. */ - struct rw_semaphore xattr_sem; + union { + struct rw_semaphore xattr_sem; + struct llist_node i_ea_iput_node; + }; /* * Inodes with EXT4_STATE_ORPHAN_FILE use i_orphan_idx. Otherwise @@ -1770,6 +1776,11 @@ struct ext4_sb_info { struct ext4_es_stats s_es_stats; struct mb_cache *s_ea_block_cache; struct mb_cache *s_ea_inode_cache; + + /* Deferred iput for EA inodes to avoid lock ordering issues */ + struct llist_head s_ea_inode_to_free; + struct delayed_work s_ea_inode_work; + spinlock_t s_es_lock ____cacheline_aligned_in_smp; /* Journal triggers for checksum computation */ @@ -3148,14 +3159,15 @@ int do_journal_get_write_access(handle_t *handle, struct inode *inode, struct buffer_head *bh); void ext4_set_inode_mapping_order(struct inode *inode); #define FALL_BACK_TO_NONDELALLOC 1 -#define CONVERT_INLINE_DATA 2 +#define EXT4_WRITE_DATA_INLINE 2 typedef enum { EXT4_IGET_NORMAL = 0, EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */ EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */ EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */ - EXT4_IGET_EA_INODE = 0x0008 /* Inode should contain an EA value */ + EXT4_IGET_EA_INODE = 0x0008, /* Inode should contain an EA value */ + EXT4_IGET_NOWAIT = 0x0010 /* Non-blocking lookup (skip if freeing) */ } ext4_iget_flags; extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, @@ -3196,10 +3208,13 @@ extern int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode); extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks); extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks); extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks, - int pextents); + int pextents, int alloc_extents); extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end); + +#define EXT4_PARTIAL_ZERO_START 0x1 +#define EXT4_PARTIAL_ZERO_END 0x2 extern int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, - loff_t length, bool *did_zero); + loff_t length, unsigned int *partial_zeroed); extern vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf); extern qsize_t *ext4_get_reserved_space(struct inode *inode); extern int ext4_get_projid(struct inode *inode, kprojid_t *projid); @@ -3651,7 +3666,13 @@ struct ext4_group_info { #define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \ (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state))) #define EXT4_MB_GRP_TEST_AND_SET_READ(grp) \ - (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state))) + (ext4_mb_grp_test_and_set_read((grp))) + +static inline int ext4_mb_grp_test_and_set_read(struct ext4_group_info *grp) +{ + return (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state) || + test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state)); +} #define EXT4_MAX_CONTENTION 8 #define EXT4_CONTENTION_THRESHOLD 2 @@ -3760,7 +3781,7 @@ extern int ext4_generic_write_inline_data(struct address_space *mapping, struct inode *inode, loff_t pos, unsigned len, struct folio **foliop, - void **fsdata, bool da); + bool da); extern int ext4_try_add_inline_entry(handle_t *handle, struct ext4_filename *fname, struct inode *dir, struct inode *inode); @@ -3892,7 +3913,8 @@ extern void ext4_ext_release(struct super_block *); extern long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len); extern int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, - loff_t offset, ssize_t len); + loff_t offset, ssize_t len, + ext4_lblk_t *converted); extern int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, loff_t offset, ssize_t len); extern int ext4_convert_unwritten_io_end_vec(handle_t *handle, diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c index 02b066299164..53ddedb52a6f 100644 --- a/fs/ext4/ext4_jbd2.c +++ b/fs/ext4/ext4_jbd2.c @@ -33,14 +33,22 @@ int ext4_inode_journal_mode(struct inode *inode) static handle_t *ext4_get_nojournal(void) { handle_t *handle = current->journal_info; - unsigned long ref_cnt = (unsigned long)handle; - BUG_ON(ref_cnt >= EXT4_NOJOURNAL_MAX_REF_COUNT); - - ref_cnt++; - handle = (handle_t *)ref_cnt; - - current->journal_info = handle; + BUG_ON(handle && !handle->h_invalid); + + if (!handle) { + handle = jbd2_alloc_handle(GFP_NOFS); + if (!handle) + return ERR_PTR(-ENOMEM); + handle->h_invalid = 1; + /* + * This is done by start_this_handle() if journalling + * is enabled. + */ + handle->saved_alloc_context = memalloc_nofs_save(); + current->journal_info = handle; + } + handle->h_ref++; return handle; } @@ -48,14 +56,14 @@ static handle_t *ext4_get_nojournal(void) /* Decrement the non-pointer handle value */ static void ext4_put_nojournal(handle_t *handle) { - unsigned long ref_cnt = (unsigned long)handle; + BUG_ON(handle->h_ref == 0); - BUG_ON(ref_cnt == 0); - - ref_cnt--; - handle = (handle_t *)ref_cnt; - - current->journal_info = handle; + handle->h_ref--; + if (handle->h_ref == 0) { + memalloc_nofs_restore(handle->saved_alloc_context); + jbd2_free_handle(handle); + current->journal_info = NULL; + } } /* diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h index 63d17c5201b5..2fbf48b3dfe2 100644 --- a/fs/ext4/ext4_jbd2.h +++ b/fs/ext4/ext4_jbd2.h @@ -182,15 +182,11 @@ handle_t *__ext4_journal_start_sb(struct inode *inode, struct super_block *sb, int rsv_blocks, int revoke_creds); int __ext4_journal_stop(const char *where, unsigned int line, handle_t *handle); -#define EXT4_NOJOURNAL_MAX_REF_COUNT ((unsigned long) 4096) - /* Note: Do not use this for NULL handles. This is only to determine if * a properly allocated handle is using a journal or not. */ static inline int ext4_handle_valid(handle_t *handle) { - if ((unsigned long)handle < EXT4_NOJOURNAL_MAX_REF_COUNT) - return 0; - return 1; + return (handle && !handle->h_invalid); } static inline void ext4_handle_sync(handle_t *handle) diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 15972410d460..76038b6c3655 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -2427,9 +2427,17 @@ int ext4_ext_index_trans_blocks(struct inode *inode, int extents) */ if (extents <= 1) index = (EXT4_MAX_EXTENT_DEPTH * 2) + extents; - else - index = (EXT4_MAX_EXTENT_DEPTH * 3) + - DIV_ROUND_UP(extents, ext4_ext_space_block(inode, 0)); + else { + int ext_max = ext4_ext_space_block(inode, 0); + + index = EXT4_MAX_EXTENT_DEPTH * 3; + /* + * Modified extents need not start at the beginning of the + * leaf. Already two extents may need two leaf block + * modifications... + */ + index += DIV_ROUND_UP(extents + ext_max - 1, ext_max); + } return index; } @@ -4571,6 +4579,22 @@ retry_remove_space: return err; } +/* + * Pre-allocate blocks for the range [@offset, @offset + @len). Allocated + * blocks are marked as unwritten by default. If EXT4_GET_BLOCKS_ZERO is + * set, the allocated blocks are zeroed on disk and their extents are + * converted to written state. + * + * When @new_size is nonzero, the caller intends to extend the file, and + * the file size should be updated to the end of the allocated blocks. + * + * Allocation may partially succeed due to some non-fatal issues. In that + * case, i_disksize (and i_size) is advanced up to the successfully + * processed portion of the range. + * + * Return 0 on success, or a negative error code on failure or partial + * failure. + */ static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len, loff_t new_size, int flags) { @@ -4585,6 +4609,7 @@ static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len, loff_t epos = 0, old_size = i_size_read(inode); unsigned int blkbits = inode->i_blkbits; bool alloc_zero = false; + bool orphan = false; BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)); map.m_lblk = offset >> blkbits; @@ -4659,19 +4684,49 @@ retry: if (alloc_zero && (map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) { + ext4_lblk_t converted; + + WARN_ON_ONCE(map.m_lblk + map.m_len > + EXT4_B_TO_LBLK(inode, new_size ?: old_size)); + ret = ext4_issue_zeroout(inode, map.m_lblk, map.m_pblk, map.m_len); - if (likely(!ret)) - ret = ext4_convert_unwritten_extents(NULL, + if (unlikely(ret)) + break; + + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, + credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + break; + } + + ret = ext4_convert_unwritten_extents(handle, inode, (loff_t)map.m_lblk << blkbits, - (loff_t)map.m_len << blkbits); + (loff_t)map.m_len << blkbits, + &converted); if (ret) - break; + map.m_len = converted; + + /* + * If blocks beyond i_disksize are converted, add + * the inode to the orphan list and advance the epos. + */ + if (new_size && converted) { + ret2 = ext4_orphan_add(handle, inode); + ret = ret ? ret : ret2; + orphan = true; + } + + ret3 = ext4_journal_stop(handle); + ret = ret ? ret : ret3; } map.m_lblk += map.m_len; map.m_len = len_lblk = len_lblk - map.m_len; epos = EXT4_LBLK_TO_B(inode, map.m_lblk); + if (ret) + break; } if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) @@ -4687,11 +4742,23 @@ retry: if (epos > new_size) epos = new_size; - handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); - if (IS_ERR(handle)) - return ret ? ret : PTR_ERR(handle); + handle = ext4_journal_start(inode, EXT4_HT_MISC, 2); + if (IS_ERR(handle)) { + /* + * The conversion has successfully completed. Not much to + * do with the error here so just cleanup the orphan list + * and hope for the best. + */ + if (orphan && inode->i_nlink) + ext4_orphan_del(NULL, inode); + ret2 = PTR_ERR(handle); + goto out; + } ext4_update_inode_size(inode, epos); + if (orphan && inode->i_nlink) + ext4_orphan_del(handle, inode); + ret2 = ext4_mark_inode_dirty(handle, inode); ext4_update_inode_fsync_trans(handle, inode, 1); ret3 = ext4_journal_stop(handle); @@ -4699,6 +4766,9 @@ retry: if (epos > old_size) pagecache_isize_extended(inode, old_size, epos); +out: + if (ret2) + ext4_std_error(inode->i_sb, ret2); return ret ? ret : ret2; } @@ -4715,7 +4785,7 @@ static long ext4_zero_range(struct file *file, loff_t offset, loff_t align_start, align_end, new_size = 0; loff_t end = offset + len; unsigned int blocksize = i_blocksize(inode); - bool partial_zeroed = false; + unsigned int partial_zeroed = 0; int ret, flags; trace_ext4_zero_range(inode, offset, len, mode); @@ -4734,10 +4804,16 @@ static long ext4_zero_range(struct file *file, loff_t offset, } flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; - /* Preallocate the range including the unaligned edges */ + /* + * Preallocate the range including the unaligned edges, and zero + * out partial blocks if they already contain data. + */ if (!IS_ALIGNED(offset | end, blocksize)) { ret = ext4_alloc_file_blocks(file, offset, len, new_size, flags); + if (!ret) + ret = ext4_zero_partial_blocks(inode, offset, len, + &partial_zeroed); if (ret) return ret; } @@ -4754,6 +4830,21 @@ static long ext4_zero_range(struct file *file, loff_t offset, /* Zero range excluding the unaligned edges */ align_start = round_up(offset, blocksize); align_end = round_down(end, blocksize); + + /* + * In WRITE_ZEROES mode, edges that were not partial-zeroed (clean + * unwritten or hole) must be allocated and zeroed as whole blocks. + * Expand the aligned range outward to cover them. + */ + if (mode & FALLOC_FL_WRITE_ZEROES) { + if (!IS_ALIGNED(offset, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_START)) + align_start = round_down(offset, blocksize); + if (!IS_ALIGNED(end, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_END)) + align_end = round_up(end, blocksize); + } + if (align_end > align_start) { if (mode & FALLOC_FL_WRITE_ZEROES) flags = EXT4_GET_BLOCKS_CREATE_ZERO | EXT4_EX_NOCACHE; @@ -4770,11 +4861,15 @@ static long ext4_zero_range(struct file *file, loff_t offset, if (IS_ALIGNED(offset | end, blocksize)) return ret; - /* Zero out partial block at the edges of the range */ - ret = ext4_zero_partial_blocks(inode, offset, len, &partial_zeroed); - if (ret) - return ret; - if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) { + /* + * In FALLOC_FL_WRITE_ZEROES mode, edges that have been partially + * zeroed must be written back to ensure the entire zeroed range + * is converted to the written state. In SYNC mode, writeback is + * also required to persist the zeroed data to disk. + */ + if (partial_zeroed && + ((mode & FALLOC_FL_WRITE_ZEROES) || + (file->f_flags & O_SYNC) || IS_SYNC(inode))) { ret = filemap_write_and_wait_range(inode->i_mapping, offset, end - 1); if (ret) @@ -4976,7 +5071,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * it can tell if the extent in the cache is a split extent. * But for now let's assume pextents as 2 always. */ - credits = ext4_meta_trans_blocks(inode, max_blocks, 2); + credits = ext4_meta_trans_blocks(inode, max_blocks, 2, 0); } if (credits) { @@ -5026,21 +5121,26 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * all unwritten extents within this range will be converted to * written extents. * - * This function is called from the direct IO end io call back - * function, to convert the fallocated extents after IO is completed. - * Returns 0 on success. + * This function is called from the direct/buffered I/O end io call back + * function and FALLOC_FL_WRITE_ZEROES, to convert the fallocated + * unwritten extents after data I/O is completed. + * + * Returns 0 on full success, or a negative error code on partial + * success or failure. The number of blocks converted is returned via + * @converted. */ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, - loff_t offset, ssize_t len) + loff_t offset, ssize_t len, + ext4_lblk_t *converted) { - unsigned int max_blocks; + ext4_lblk_t max_blocks, conv_blocks = 0; int ret = 0, ret2 = 0, ret3 = 0; struct ext4_map_blocks map; unsigned int blkbits = inode->i_blkbits; unsigned int credits = 0; map.m_lblk = offset >> blkbits; - max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); + map.m_len = max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); if (!handle) { /* @@ -5048,9 +5148,8 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, */ credits = ext4_chunk_trans_blocks(inode, max_blocks); } - while (ret >= 0 && ret < max_blocks) { - map.m_lblk += ret; - map.m_len = (max_blocks -= ret); + + while (max_blocks) { if (credits) { handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, credits); @@ -5067,23 +5166,34 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, ret = ext4_map_blocks(handle, inode, &map, EXT4_GET_BLOCKS_IO_CONVERT_EXT | EXT4_EX_NOCACHE); - if (ret <= 0) + if (ret <= 0) { ext4_warning(inode->i_sb, - "inode #%llu: block %u: len %u: " - "ext4_ext_map_blocks returned %d", - inode->i_ino, map.m_lblk, - map.m_len, ret); + "inode #%llu: block %u: len %u: ext4_map_blocks returned %d", + inode->i_ino, map.m_lblk, map.m_len, ret); + if (unlikely(ret == 0)) + ret = -EINVAL; + } else { + conv_blocks += map.m_len; + } + ret2 = ext4_mark_inode_dirty(handle, inode); if (credits) { ret3 = ext4_journal_stop(handle); if (unlikely(ret3)) ret2 = ret3; } - - if (ret <= 0 || ret2) + ret = ret < 0 ? ret : ret2; + if (ret) break; + + map.m_lblk += map.m_len; + map.m_len = (max_blocks -= map.m_len); } - return ret > 0 ? ret2 : ret; + /* Converted some or all blocks successfully? */ + if (converted) + *converted = conv_blocks; + + return ret; } int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end) @@ -5106,7 +5216,7 @@ int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end) list_for_each_entry(io_end_vec, &io_end->list_vec, list) { ret = ext4_convert_unwritten_extents(handle, io_end->inode, io_end_vec->offset, - io_end_vec->size); + io_end_vec->size, NULL); if (ret) break; } diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 8e2259799614..062103e42cd8 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -200,25 +200,6 @@ static inline void ext4_fc_set_snap_err(int *snap_err, int err) *snap_err = err; } -static void ext4_end_buffer_io_sync(struct bio *bio) -{ - struct buffer_head *bh; - bool uptodate = bio_endio_bh(bio, &bh); - - BUFFER_TRACE(bh, ""); - if (uptodate) { - ext4_debug("%s: Block %lld up-to-date", - __func__, bh->b_blocknr); - set_buffer_uptodate(bh); - } else { - ext4_debug("%s: Block %lld not up-to-date", - __func__, bh->b_blocknr); - clear_buffer_uptodate(bh); - } - - unlock_buffer(bh); -} - static void ext4_fc_free_inode_snap(struct inode *inode); static inline void ext4_fc_reset_inode(struct inode *inode) @@ -691,7 +672,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) lock_buffer(bh); set_buffer_dirty(bh); set_buffer_uptodate(bh); - bh_submit(bh, REQ_OP_WRITE | write_flags, ext4_end_buffer_io_sync); + bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); EXT4_SB(sb)->s_fc_bh = NULL; } @@ -2196,8 +2177,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) if (ret == 0) { /* Range is not mapped */ path = ext4_find_extent(inode, cur, path, 0); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } memset(&newex, 0, sizeof(newex)); newex.ee_block = cpu_to_le32(cur); ext4_ext_store_pblock( @@ -2209,8 +2193,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) path = ext4_ext_insert_extent(NULL, inode, path, &newex, 0); up_write((&EXT4_I(inode)->i_data_sem)); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } goto next; } @@ -2257,10 +2244,11 @@ next: } ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); + ret = 0; out: ext4_free_ext_path(path); iput(inode); - return 0; + return ret; } /* Replay DEL_RANGE tag */ @@ -2320,9 +2308,10 @@ ext4_fc_replay_del_range(struct super_block *sb, u8 *val) ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); ext4_mark_inode_dirty(NULL, inode); + ret = 0; out: iput(inode); - return 0; + return ret; } static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index f20d92255546..374b4bc25bd5 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -215,31 +215,60 @@ ext4_extending_io(struct inode *inode, loff_t offset, size_t len) return false; } -/* Is IO overwriting allocated or initialized blocks? */ -static bool ext4_overwrite_io(struct inode *inode, - loff_t pos, loff_t len, bool *unwritten) +/* + * Does an unaligned DIO write require partial block zeroing? + * + * Partial block zeroing is performed only for the head and tail blocks + * when they are partially covered by the write and the underlying extent + * is a hole or unwritten. Middle blocks (fully covered by the write) + * are written as whole blocks without zeroing. + * + * When zeroing is required, two concurrent unaligned DIO writes to the + * same partial block can race and corrupt each other's data, so the + * caller must take the exclusive i_rwsem and drain in-flight DIO. When + * zeroing is not required, shared lock is safe -- block allocation and + * unwritten conversion for middle blocks are protected by i_data_sem + * and inode_dio_begin(). + */ +static bool ext4_dio_needs_zeroing(struct inode *inode, loff_t pos, loff_t len) { struct ext4_map_blocks map; unsigned int blkbits = inode->i_blkbits; - int err, blklen; + unsigned long blockmask = inode->i_sb->s_blocksize - 1; + bool head_partial, tail_partial; + ext4_lblk_t head_lblk, tail_lblk; + int err; if (pos + len > i_size_read(inode)) - return false; + return true; - map.m_lblk = pos >> blkbits; - map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits); - blklen = map.m_len; + head_partial = (pos & blockmask) != 0; + tail_partial = ((pos + len) & blockmask) != 0; + head_lblk = pos >> blkbits; + tail_lblk = (pos + len - 1) >> blkbits; + + /* Check the head partial block. */ + if (head_partial) { + map.m_lblk = head_lblk; + map.m_len = tail_lblk - head_lblk + 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + /* If this mapping already covers the tail block, we're done. */ + if (!tail_partial || map.m_lblk + err > tail_lblk) + return false; + } - err = ext4_map_blocks(NULL, inode, &map, 0); - if (err != blklen) - return false; - /* - * 'err==len' means that all of the blocks have been preallocated, - * regardless of whether they have been initialized or not. We need to - * check m_flags to distinguish the unwritten extents. - */ - *unwritten = !(map.m_flags & EXT4_MAP_MAPPED); - return true; + /* Check the tail partial block. */ + if (tail_partial) { + map.m_lblk = tail_lblk; + map.m_len = 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + } + + return false; } static ssize_t ext4_generic_write_checks(struct kiocb *iocb, @@ -280,7 +309,7 @@ static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from) if (count <= 0) return count; - ret = file_modified(iocb->ki_filp); + ret = kiocb_modified(iocb); if (ret) return ret; @@ -311,6 +340,13 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; inode_lock(inode); + + /* + * Prevent concurrent direct I/O and buffered I/O to the same file + * range. Wait for in-flight DIO to finish before dirtying pages. + */ + inode_dio_wait(inode); + ret = ext4_write_checks(iocb, from); if (ret <= 0) goto out; @@ -402,7 +438,8 @@ static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size, error = ext4_convert_unwritten_extents_atomic(NULL, inode, pos, size); else if (!error && size && flags & IOMAP_DIO_UNWRITTEN) - error = ext4_convert_unwritten_extents(NULL, inode, pos, size); + error = ext4_convert_unwritten_extents(NULL, inode, pos, size, + NULL); if (error) return error; /* @@ -430,16 +467,28 @@ static const struct iomap_dio_ops ext4_dio_write_ops = { * condition requires an exclusive inode lock. If yes, then we restart the * whole operation by releasing the shared lock and acquiring exclusive lock. * - * - For unaligned_io we never take shared lock as it may cause data corruption - * when two unaligned IO tries to modify the same block e.g. while zeroing. + * The decision is layered, evaluated in this order: * - * - For extending writes case we don't take the shared lock, since it requires - * updating inode i_disksize and/or orphan handling with exclusive lock. + * 1. If kiocb_modified() needs to update security info (!IS_NOSEC), upgrade + * to the exclusive lock -- the security update itself requires it, + * regardless of whether the write extends the file or is aligned. * - * - shared locking will only be true mostly with overwrites, including - * initialized blocks and unwritten blocks. + * 2. If the write extends i_size or i_disksize, upgrade to the exclusive + * lock to safely update i_disksize and the orphan list, regardless of + * alignment. * - * - Otherwise we will switch to exclusive i_rwsem lock. + * 3. Otherwise, for aligned non-extending writes, shared lock is always + * sufficient regardless of extent state (written, unwritten, or hole). + * truncate/punch_hole cannot run while we hold the shared i_rwsem + * (they need it exclusively); after we release it, inode_dio_begin() + * keeps their inode_dio_wait() blocked until in-flight bios complete. + * i_data_sem serializes concurrent extent tree modifications. + * + * 4. Otherwise, the write is unaligned and non-extending. Shared lock is + * safe unless the DIO layer needs to perform partial block zeroing -- + * i.e. the head or tail partial block sits on a hole or unwritten + * extent. In that case upgrade to the exclusive lock and drain + * in-flight DIO to avoid races with concurrent partial block zeroing. */ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, bool *ilock_shared, bool *extend, @@ -450,7 +499,7 @@ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, loff_t offset; size_t count; ssize_t ret; - bool overwrite, unaligned_io, unwritten; + bool needs_zeroing = false; restart: ret = ext4_generic_write_checks(iocb, from); @@ -460,24 +509,22 @@ restart: offset = iocb->ki_pos; count = ret; - unaligned_io = ext4_unaligned_io(inode, from, offset); *extend = ext4_extending_io(inode, offset, count); - overwrite = ext4_overwrite_io(inode, offset, count, &unwritten); /* - * Determine whether we need to upgrade to an exclusive lock. This is - * required to change security info in file_modified(), for extending - * I/O, any form of non-overwrite I/O, and unaligned I/O to unwritten - * extents (as partial block zeroing may be required). + * For unaligned writes, check whether partial block zeroing will be + * needed. If so, exclusive lock is required to serialize against + * concurrent DIO that could race with the zeroing. * - * Note that unaligned writes are allowed under shared lock so long as - * they are pure overwrites. Otherwise, concurrent unaligned writes risk - * data corruption due to partial block zeroing in the dio layer, and so - * the I/O must occur exclusively. + * For aligned writes we skip this check entirely since allocation + * under shared lock is safe. */ + if (ext4_unaligned_io(inode, from, offset)) + needs_zeroing = ext4_dio_needs_zeroing(inode, offset, count); + + /* Determine whether we need to upgrade to an exclusive lock. */ if (*ilock_shared && - ((!IS_NOSEC(inode) || *extend || !overwrite || - (unaligned_io && unwritten)))) { + (!IS_NOSEC(inode) || *extend || needs_zeroing)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; @@ -491,21 +538,28 @@ restart: /* * Now that locking is settled, determine dio flags and exclusivity * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce - * behavior already. The inode lock is already held exclusive if the - * write is non-overwrite or extending, so drain all outstanding dio and - * set the force wait dio flag. + * behavior already. When holding the exclusive lock for a write that + * needs partial block zeroing or is extending the file, we must wait + * for the I/O to complete synchronously: + * + * - needs_zeroing: drain in-flight DIO whose end_io could race with + * our partial block zeroing, and force synchronous completion so we + * don't leave in-flight zeroing bios for the next writer to drain. + * + * - extend: the caller must update i_disksize after I/O completion, + * which requires the data to be on disk first. */ - if (!*ilock_shared && (unaligned_io || *extend)) { + if (!*ilock_shared && (needs_zeroing || *extend)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; } - if (unaligned_io && (!overwrite || unwritten)) + if (needs_zeroing) inode_dio_wait(inode); *dio_flags = IOMAP_DIO_FORCE_WAIT; } - ret = file_modified(file); + ret = kiocb_modified(iocb); if (ret < 0) goto out; @@ -674,6 +728,11 @@ ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) count = iov_iter_count(from); if (offset + count > EXT4_I(inode)->i_disksize) { + if (iocb->ki_flags & IOCB_NOWAIT) { + ret = -EAGAIN; + goto out; + } + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); if (IS_ERR(handle)) { ret = PTR_ERR(handle); diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index a40cb27f8116..a5831fc536db 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -997,6 +997,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, err = fscrypt_prepare_new_inode(dir, inode, &encrypt); if (err) goto out; + if (encrypt) + i_flags |= EXT4_ENCRYPT_FL; } err = dquot_initialize(inode); @@ -1306,6 +1308,8 @@ got: ei->i_extra_isize = sbi->s_want_extra_isize; ei->i_inline_off = 0; if (ext4_has_feature_inline_data(sb) && + /* Encrypted inodes cannot have inline data */ + !(ei->i_flags & EXT4_ENCRYPT_FL) && (!(ei->i_flags & (EXT4_DAX_FL|EXT4_EA_INODE_FL)) || S_ISDIR(mode))) ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); ret = inode; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 8045e4ff270c..ceee69a66482 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -22,8 +22,7 @@ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, - struct inode *inode, - void **fsdata); + struct inode *inode); static int ext4_get_inline_size(struct inode *inode) { @@ -697,7 +696,7 @@ int ext4_generic_write_inline_data(struct address_space *mapping, struct inode *inode, loff_t pos, unsigned len, struct folio **foliop, - void **fsdata, bool da) + bool da) { int ret; handle_t *handle; @@ -728,7 +727,7 @@ retry_journal: return ext4_convert_inline_data_to_extent(mapping, inode); } - ret = ext4_da_convert_inline_data_to_extent(mapping, inode, fsdata); + ret = ext4_da_convert_inline_data_to_extent(mapping, inode); if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) goto retry_journal; @@ -788,7 +787,7 @@ int ext4_try_to_write_inline_data(struct address_space *mapping, if (pos + len > ext4_get_max_inline_size(inode)) return ext4_convert_inline_data_to_extent(mapping, inode); return ext4_generic_write_inline_data(mapping, inode, pos, len, - foliop, NULL, false); + foliop, false); } int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, @@ -812,7 +811,19 @@ int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, goto out; } ext4_write_lock_xattr(inode, &no_expand); - BUG_ON(!ext4_has_inline_data(inode)); + /* + * We could have raced with ext4_page_mkwrite() converting + * the inode and clearing the inline data flag, so we just + * release resources and retry the whole write. + */ + if (unlikely(!ext4_has_inline_data(inode))) { + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + return 0; + } /* * ei->i_inline_off may have changed since @@ -883,8 +894,7 @@ out: * need to start the journal since the file's metadata isn't changed now. */ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, - struct inode *inode, - void **fsdata) + struct inode *inode) { int ret = 0, inline_size; struct folio *folio; @@ -922,7 +932,6 @@ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, folio_mark_dirty(folio); folio_mark_uptodate(folio); ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); - *fsdata = (void *)CONVERT_INLINE_DATA; out: up_read(&EXT4_I(inode)->xattr_sem); @@ -1454,6 +1463,8 @@ int ext4_read_inline_dir(struct file *file, /* for other entry, the real offset in * the buf has to be tuned accordingly. */ + if (i + ext4_dir_rec_len(1, NULL) > extra_size) + break; de = (struct ext4_dir_entry_2 *) (dir_buf + i - extra_offset); /* It's too expensive to do a full @@ -1488,10 +1499,17 @@ int ext4_read_inline_dir(struct file *file, continue; } + /* + * de lives at dir_buf + ctx->pos - extra_offset, within the + * kmalloc(inline_size) buffer. Make sure its header fits before + * ext4_check_dir_entry() dereferences de->rec_len. + */ + if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size) + goto out; de = (struct ext4_dir_entry_2 *) (dir_buf + ctx->pos - extra_offset); if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, - extra_size, ctx->pos)) + inline_size, ctx->pos)) goto out; if (le32_to_cpu(de->inode)) { if (!dir_emit(ctx, de->name, de->name_len, diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 3fc235eb65f2..bd4b778df9eb 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -176,7 +176,6 @@ void ext4_evict_inode(struct inode *inode) * (xattr block freeing), bitmap, group descriptor (inode freeing) */ int extra_credits = 6; - struct ext4_xattr_inode_array *ea_inode_array = NULL; bool freeze_protected = false; trace_ext4_evict_inode(inode); @@ -266,6 +265,7 @@ void ext4_evict_inode(struct inode *inode) if (ext4_inode_is_fast_symlink(inode)) memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data)); inode->i_size = 0; + ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); err = ext4_mark_inode_dirty(handle, inode); if (err) { ext4_warning(inode->i_sb, @@ -283,8 +283,7 @@ void ext4_evict_inode(struct inode *inode) } /* Remove xattr references. */ - err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array, - extra_credits); + err = ext4_xattr_delete_inode(handle, inode, extra_credits); if (err) { ext4_warning(inode->i_sb, "xattr delete (err %d)", err); stop_handle: @@ -292,7 +291,6 @@ stop_handle: ext4_orphan_del(NULL, inode); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); goto no_delete; } @@ -322,7 +320,6 @@ stop_handle: ext4_journal_stop(handle); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); return; no_delete: /* @@ -1184,6 +1181,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, int nr_wait = 0; int i; bool should_journal_data = ext4_should_journal_data(inode); + bool folio_uptodate = folio_test_uptodate(folio); BUG_ON(!folio_test_locked(folio)); BUG_ON(to > folio_size(folio)); @@ -1195,13 +1193,13 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, head = create_empty_buffers(folio, blocksize, 0); block = EXT4_PG_TO_LBLK(inode, folio->index); - for (bh = head, block_start = 0; bh != head || !block_start; + for (bh = head, block_start = 0; + block_start < to || (!folio_uptodate && bh != head); block++, block_start = block_end, bh = bh->b_this_page) { block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { - if (folio_test_uptodate(folio)) { + if (folio_uptodate) set_buffer_uptodate(bh); - } continue; } if (WARN_ON_ONCE(buffer_new(bh))) @@ -1222,7 +1220,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, if (should_journal_data) do_journal_get_write_access(handle, inode, bh); - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { /* * Unlike __block_write_begin() we leave * dirtying of new uptodate buffers to @@ -1239,7 +1237,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, continue; } } - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { set_buffer_uptodate(bh); continue; } @@ -1293,6 +1291,8 @@ static int ext4_write_begin(const struct kiocb *iocb, if (unlikely(ret)) return ret; + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + trace_ext4_write_begin(inode, pos, len); /* * Reserve one block more for addition to orphan list in case @@ -1307,8 +1307,10 @@ static int ext4_write_begin(const struct kiocb *iocb, foliop); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } /* @@ -1441,8 +1443,7 @@ static int ext4_write_end(const struct kiocb *iocb, trace_ext4_write_end(inode, pos, len, copied); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1551,8 +1552,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -2672,13 +2672,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * page is already under writeback and we are not doing * a data integrity writeback, skip the page */ - if (!folio_test_dirty(folio) || - (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || unlikely(folio->mapping != mapping)) { folio_unlock(folio); continue; } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } folio_wait_writeback(folio); BUG_ON(folio_test_writeback(folio)); @@ -2829,10 +2841,10 @@ static int ext4_do_writepages(struct mpage_da_data *mpd) if (ext4_should_dioread_nolock(inode)) { int bpf = ext4_journal_blocks_per_folio(inode); /* - * We may need to convert up to one extent per block in - * the folio and we may dirty the inode. + * We may need to convert up to one extent per block in the + * folio. */ - rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf); + rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); } if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) @@ -3135,11 +3147,13 @@ static int ext4_da_write_begin(const struct kiocb *iocb, if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { ret = ext4_generic_write_inline_data(mapping, inode, pos, len, - foliop, fsdata, true); + foliop, true); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } retry: @@ -3268,17 +3282,15 @@ static int ext4_da_write_end(const struct kiocb *iocb, struct folio *folio, void *fsdata) { struct inode *inode = mapping->host; - int write_mode = (int)(unsigned long)fsdata; + unsigned long write_mode = (unsigned long)fsdata; - if (write_mode == FALL_BACK_TO_NONDELALLOC) + if (write_mode & FALL_BACK_TO_NONDELALLOC) return ext4_write_end(iocb, mapping, pos, len, copied, folio, fsdata); trace_ext4_da_write_end(inode, pos, len, copied); - if (write_mode != CONVERT_INLINE_DATA && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && - ext4_has_inline_data(inode)) + if (write_mode & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -3652,6 +3664,9 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, int ret, dio_credits, m_flags = 0, retries = 0; bool force_commit = false; + if (flags & IOMAP_NOWAIT) + return -EAGAIN; + /* * Trim the mapping request to the maximum value that we can map at * once for direct I/O. @@ -3673,8 +3688,8 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, return ret; if (map->m_len < orig_mlen) { map->m_len = orig_mlen; - dio_credits = ext4_meta_trans_blocks(inode, orig_mlen, - map->m_len); + dio_credits = ext4_meta_trans_blocks(inode, map->m_len, + map->m_len, 0); } else { dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); @@ -4010,6 +4025,10 @@ void ext4_set_aops(struct inode *inode) * because it might have data in pagecache (eg, if called from ext4_zero_range, * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a * racing writeback can come later and flush the stale pagecache to disk. + * + * Return the loaded bh if it actually needs zeroing - in written, dirty + * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or + * a clean unwritten block). */ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) { @@ -4021,7 +4040,7 @@ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) int err = 0; folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + FGP_WRITEBEGIN | FGP_ACCESSED, mapping_gfp_constraint(mapping, ~__GFP_FS)); if (IS_ERR(folio)) return ERR_CAST(folio); @@ -4048,8 +4067,15 @@ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) } if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "unmapped"); - ext4_get_block(inode, iblock, bh, 0); - /* unmapped? It's a hole - nothing to do */ + err = ext4_get_block(inode, iblock, bh, 0); + if (err < 0) + goto unlock; + /* + * It's a hole or a clean unwritten block - nothing to do. + * Note that a lookup-only get_block (without + * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean + * unwritten extents. + */ if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "still unmapped"); goto unlock; @@ -4191,6 +4217,14 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) offset = from & (blocksize - 1); if (!offset || from >= end) return 0; + /* + * Inline data has no tail block to zero out. Note that a race with + * ext4_page_mkwrite() converting inline data to an extent without + * holding i_rwsem is safe, as that path zeroes the full block before + * copying in the inline data. + */ + if (ext4_has_inline_data(inode)) + return 0; /* If we are processing an encrypted inode during orphan list handling */ if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) return 0; @@ -4225,13 +4259,26 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) return 0; } +/* + * Zero out the unaligned head and tail of the [lstart, lstart+length) + * range. + * + * On return, @partial_zeroed records which edges actually got + * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if + * the head/tail block got actually partially zeroed (in written, dirty + * unwritten or delalloc state). Cleared if the head/tail block is a + * hole or a clean unwritten block, in which case there is nothing that + * needs zeroing. When the head and tail land in the same block, both + * bits are set together on a successful zeroing. + */ int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, - bool *did_zero) + unsigned int *partial_zeroed) { struct super_block *sb = inode->i_sb; unsigned partial_start, partial_end; ext4_fsblk_t start, end; loff_t byte_end = (lstart + length - 1); + bool did_zero = false; int err = 0; partial_start = lstart & (sb->s_blocksize - 1); @@ -4243,21 +4290,32 @@ int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, /* Handle partial zero within the single block */ if (start == end && (partial_start || (partial_end != sb->s_blocksize - 1))) { - err = ext4_block_zero_range(inode, lstart, length, did_zero, + err = ext4_block_zero_range(inode, lstart, length, &did_zero, NULL); + if (did_zero) + *partial_zeroed |= (EXT4_PARTIAL_ZERO_START | + EXT4_PARTIAL_ZERO_END); return err; } /* Handle partial zero out on the start of the range */ if (partial_start) { err = ext4_block_zero_range(inode, lstart, sb->s_blocksize, - did_zero, NULL); + &did_zero, NULL); if (err) return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_START; } /* Handle partial zero out on the end of the range */ - if (partial_end != sb->s_blocksize - 1) + if (partial_end != sb->s_blocksize - 1) { + did_zero = false; err = ext4_block_zero_range(inode, byte_end - partial_end, - partial_end + 1, did_zero, NULL); + partial_end + 1, &did_zero, NULL); + if (err) + return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_END; + } return err; } @@ -4406,7 +4464,7 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) loff_t end = offset + length; handle_t *handle; unsigned int credits; - bool partial_zeroed = false; + unsigned int partial_zeroed = 0; int ret; trace_ext4_punch_hole(inode, offset, length, 0); @@ -5242,6 +5300,20 @@ void ext4_set_inode_mapping_order(struct inode *inode) mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); } +static int ext4_iget_match(struct inode *inode, u64 ino, void *data) +{ + if (inode->i_ino != ino) + return 0; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { + spin_unlock(&inode->i_lock); + return -1; + } + __iget(inode); + spin_unlock(&inode->i_lock); + return 1; +} + struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ext4_iget_flags flags, const char *function, unsigned int line) @@ -5270,9 +5342,24 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, return ERR_PTR(-EFSCORRUPTED); } - inode = iget_locked(sb, ino); - if (!inode) - return ERR_PTR(-ENOMEM); + if (flags & EXT4_IGET_NOWAIT) { + inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); + if (!inode) + return ERR_PTR(-ENOENT); + + if (inode_state_read_once(inode) & I_NEW) + wait_on_new_inode(inode); + + if (unlikely(inode_unhashed(inode))) { + iput(inode); + return ERR_PTR(-ENOENT); + } + } else { + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + } + if (!(inode_state_read_once(inode) & I_NEW)) { ret = check_igot_inode(inode, flags, function, line); if (ret) { @@ -6288,17 +6375,17 @@ static int ext4_index_trans_blocks(struct inode *inode, int lblocks, } /* - * Account for index blocks, block groups bitmaps and block group - * descriptor blocks if modify datablocks and index blocks - * worse case, the indexs blocks spread over different block groups - * - * If datablocks are discontiguous, they are possible to spread over - * different block groups too. If they are contiguous, with flexbg, - * they could still across block group boundary. - * - * Also account for superblock, inode, quota and xattr blocks + * Calculate number of credits needed in a transaction to: + * * Allocate data blocks from @alloc_extents different groups - note that + * with flexbg a single physical extent can span multiple groups but + * single mballoc request only returns extent within one group. + * * Allocate metatadata (extent tree blocks, indirect blocks) to store + * pointers to @pextents data extents having @lblocks in total. + * * Modify extent tree / indirect block tree, inode, superblock, quota + * tracking, xattr blocks */ -int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) +int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, + int alloc_extents) { ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); int gdpblocks; @@ -6315,7 +6402,7 @@ int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) * Now let's see how many group bitmaps and group descriptors need * to account */ - groups = idxblocks + pextents; + groups = idxblocks + alloc_extents; gdpblocks = groups; if (groups > ngroups) groups = ngroups; @@ -6341,7 +6428,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) { int ret; - ret = ext4_meta_trans_blocks(inode, nrblocks, 1); + ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); /* Account for data blocks for journalled mode */ if (ext4_should_journal_data(inode)) ret += nrblocks; @@ -6359,7 +6446,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) */ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) { - return ext4_meta_trans_blocks(inode, nrblocks, 1); + return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); } /* @@ -6506,6 +6593,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode, return -EOVERFLOW; /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + + /* * In nojournal mode, we can immediately attempt to expand * the inode. When journaled, we first need to obtain extra * buffer credits since we may write into the EA block diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index ed1bd00e11cd..06171a11db12 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2861,8 +2861,6 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, blk_start_plug(&plug); while (nr-- > 0) { - struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, - NULL); struct ext4_group_info *grp = ext4_get_group_info(sb, group); /* @@ -2872,14 +2870,17 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, * prefetch once, so we avoid getblk() call, which can * be expensive. */ - if (gdp && grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && - EXT4_MB_GRP_NEED_INIT(grp) && - ext4_free_group_clusters(sb, gdp) > 0 ) { - bh = ext4_read_block_bitmap_nowait(sb, group, true); - if (!IS_ERR_OR_NULL(bh)) { - if (!buffer_uptodate(bh) && cnt) - (*cnt)++; - brelse(bh); + if (grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && + EXT4_MB_GRP_NEED_INIT(grp)) { + struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, NULL); + + if (gdp && ext4_free_group_clusters(sb, gdp) > 0) { + bh = ext4_read_block_bitmap_nowait(sb, group, true); + if (!IS_ERR_OR_NULL(bh)) { + if (!buffer_uptodate(bh) && cnt) + (*cnt)++; + brelse(bh); + } } } if (++group >= ngroups) diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 477d43d7e294..5d60ef10fe11 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -464,6 +464,7 @@ int ext4_ext_migrate(struct inode *inode) if (IS_ERR(tmp_inode)) { retval = PTR_ERR(tmp_inode); ext4_journal_stop(handle); + tmp_inode = NULL; goto out_unlock; } /* @@ -591,9 +592,9 @@ out_stop: ext4_journal_stop(handle); out_tmp_inode: unlock_new_inode(tmp_inode); - iput(tmp_inode); out_unlock: ext4_writepages_up_write(inode->i_sb, alloc_ctx); + iput(tmp_inode); return retval; } diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 640a03ee02c7..a6386c1d237f 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -1467,6 +1467,8 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, /* this code is executed quadratically often */ /* do minimal checking `by hand' */ if (de->name + de->name_len <= dlimit && + (!ext4_hash_in_dirent(dir) || + (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) && ext4_match(dir, fname, de)) { /* found a match - just to be sure, do * a full check */ diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index 64ea47624233..b4675aa7ea96 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -4,6 +4,7 @@ #include <linux/fs.h> #include <linux/quotaops.h> #include <linux/buffer_head.h> +#include <linux/string_choices.h> #include "ext4.h" #include "ext4_jbd2.h" @@ -388,7 +389,7 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); - if (!es->s_last_orphan && !oi->of_blocks) { + if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) { ext4_debug("no orphan inodes to clean up\n"); return; } @@ -486,14 +487,12 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) } } -#define PLURAL(x) (x), ((x) == 1) ? "" : "s" - if (nr_orphans) ext4_msg(sb, KERN_INFO, "%d orphan inode%s deleted", - PLURAL(nr_orphans)); + nr_orphans, str_plural(nr_orphans)); if (nr_truncates) ext4_msg(sb, KERN_INFO, "%d truncate%s cleaned up", - PLURAL(nr_truncates)); + nr_truncates, str_plural(nr_truncates)); #ifdef CONFIG_QUOTA /* Turn off quotas if they were enabled for orphan cleanup */ if (quota_update) { @@ -572,6 +571,7 @@ int ext4_init_orphan_info(struct super_block *sb) int i, j; int ret; int free; + int loaded = 0; __le32 *bdata; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); struct ext4_orphan_block_tail *ot; @@ -613,6 +613,7 @@ int ext4_init_orphan_info(struct super_block *sb) ret = -EIO; goto out_free; } + loaded++; ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh); if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) { ext4_error(sb, "orphan file block %d: bad magic", i); @@ -635,8 +636,10 @@ int ext4_init_orphan_info(struct super_block *sb) iput(inode); return 0; out_free: - for (i--; i >= 0; i--) - brelse(oi->of_binfo[i].ob_bh); + while (loaded > 0) { + loaded--; + brelse(oi->of_binfo[loaded].ob_bh); + } kvfree(oi->of_binfo); out_put: iput(inode); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 4b6112e5d6c5..bca0dc87d0b7 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1303,6 +1303,8 @@ static void ext4_put_super(struct super_block *sb) &sb->s_uuid); ext4_unregister_li_request(sb); + /* Drain deferred EA inode iputs while quota is still active. */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_quotas_off(sb, EXT4_MAXQUOTAS); destroy_workqueue(sbi->rsv_conversion_wq); @@ -1423,6 +1425,13 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) memset(&ei->i_dquot, 0, sizeof(ei->i_dquot)); #endif ei->jinode = NULL; + /* + * Reinitialize xattr_sem every allocation because EA inodes + * share this space with i_ea_iput_node (via union) which may + * have overwritten the semaphore when the slab object was + * previously used as an EA inode. + */ + init_rwsem(&ei->xattr_sem); INIT_LIST_HEAD(&ei->i_rsv_conversion_list); spin_lock_init(&ei->i_completed_io_lock); ei->i_sync_tid = 0; @@ -1489,7 +1498,6 @@ static void init_once(void *foo) struct ext4_inode_info *ei = foo; INIT_LIST_HEAD(&ei->i_orphan); - init_rwsem(&ei->xattr_sem); init_rwsem(&ei->i_data_sem); inode_init_once(&ei->vfs_inode); ext4_fc_init_inode(&ei->vfs_inode); @@ -4483,8 +4491,9 @@ static int ext4_handle_clustersize(struct super_block *sb) sbi->s_cluster_bits = 0; } sbi->s_clusters_per_group = le32_to_cpu(es->s_clusters_per_group); - if (sbi->s_clusters_per_group > sb->s_blocksize * 8) { - ext4_msg(sb, KERN_ERR, "#clusters per group too big: %lu", + if (sbi->s_clusters_per_group > sb->s_blocksize * 8 || + sbi->s_clusters_per_group & 7) { + ext4_msg(sb, KERN_ERR, "invalid #clusters per group: %lu", sbi->s_clusters_per_group); return -EINVAL; } @@ -5316,8 +5325,10 @@ static int ext4_block_group_meta_init(struct super_block *sb, int silent) return -EINVAL; } if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || - sbi->s_inodes_per_group > sb->s_blocksize * 8) { - ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n", + sbi->s_inodes_per_group > sb->s_blocksize * 8 || + sbi->s_inodes_per_group & 7 || + sbi->s_inodes_per_group % sbi->s_inodes_per_block) { + ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu", sbi->s_inodes_per_group); return -EINVAL; } @@ -5377,7 +5388,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_set_def_opts(sb, es); sbi->s_resuid = make_kuid(&init_user_ns, ext4_get_resuid(es)); - sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resuid(es)); + sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resgid(es)); sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; @@ -5505,6 +5516,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_has_feature_orphan_present(sb) || ext4_has_feature_journal_needs_recovery(sb)); + ext4_init_ea_inode_work(sbi); + if (ext4_has_feature_mmp(sb) && !sb_rdonly(sb)) { err = ext4_multi_mount_protect(sb, le64_to_cpu(es->s_mmp_block)); if (err) @@ -5755,6 +5768,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) return 0; failed_mount9: + /* Drain deferred EA inode iputs before quota shutdown */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_quotas_off(sb, EXT4_MAXQUOTAS); failed_mount8: __maybe_unused ext4_release_orphan_info(sb); @@ -5775,6 +5790,8 @@ failed_mount4: if (EXT4_SB(sb)->rsv_conversion_wq) destroy_workqueue(EXT4_SB(sb)->rsv_conversion_wq); failed_mount_wq: + /* Drain deferred EA inode iputs before freeing structures */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_xattr_destroy_cache(sbi->s_ea_inode_cache); sbi->s_ea_inode_cache = NULL; @@ -5785,6 +5802,8 @@ failed_mount_wq: ext4_journal_destroy(sbi, sbi->s_journal); } failed_mount3a: + /* Drain deferred EA inode iputs from journal replay */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_es_unregister_shrinker(sbi); failed_mount3: /* flush s_sb_upd_work before sbi destroy */ @@ -6455,6 +6474,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait) trace_ext4_sync_fs(sb, wait); flush_workqueue(sbi->rsv_conversion_wq); + flush_delayed_work(&sbi->s_ea_inode_work); /* * Writeback quota in non-journalled quota case - journalled quota has * no dirty dquots diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 982a1f831e22..5c310747b965 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -114,10 +114,6 @@ const struct xattr_handler * const ext4_xattr_handlers[] = { #define EA_INODE_CACHE(inode) (((struct ext4_sb_info *) \ inode->i_sb->s_fs_info)->s_ea_inode_cache) -static int -ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, - struct inode *inode); - #ifdef CONFIG_LOCKDEP void ext4_xattr_inode_set_class(struct inode *ea_inode) { @@ -464,6 +460,21 @@ static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino, inode_unlock(inode); } + /* + * Since this function resolves references from active xattr entries, + * the EA inode must be in active state (i_nlink=1, ref_count>0). + * i_nlink > 1, i_nlink == 0 (dangling reference), or ref_count == 0 + * (inconsistent with an active entry) all indicate on-disk corruption. + */ + if (inode->i_nlink != 1 || !ext4_xattr_inode_get_ref(inode)) { + ext4_error(parent->i_sb, + "EA inode %lu has unexpected i_nlink=%u ref_count=%llu", + ea_ino, inode->i_nlink, + ext4_xattr_inode_get_ref(inode)); + ext4_put_ea_inode(inode); + return -EFSCORRUPTED; + } + *ea_inode = inode; return 0; } @@ -567,7 +578,7 @@ ext4_xattr_inode_get(struct inode *inode, struct ext4_xattr_entry *entry, ea_inode->i_ino, true /* reusable */); } out: - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return err; } @@ -1104,10 +1115,10 @@ static int ext4_xattr_inode_inc_ref_all(handle_t *handle, struct inode *parent, err = ext4_xattr_inode_inc_ref(handle, ea_inode); if (err) { ext4_warning_inode(ea_inode, "inc ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); goto cleanup; } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return 0; @@ -1133,7 +1144,7 @@ cleanup: if (err) ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return saved_err; } @@ -1160,7 +1171,6 @@ static void ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, struct buffer_head *bh, struct ext4_xattr_entry *first, bool block_csum, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits, bool skip_quota) { struct inode *ea_inode; @@ -1197,14 +1207,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err) continue; - err = ext4_expand_inode_array(ea_inode_array, ea_inode); - if (err) { - ext4_warning_inode(ea_inode, - "Expand inode array err=%d", err); - iput(ea_inode); - continue; - } - err = ext4_journal_ensure_credits_fn(handle, credits, credits, ext4_free_metadata_revoke_credits(parent->i_sb, 1), ext4_xattr_restart_fn(handle, parent, bh, block_csum, @@ -1212,6 +1214,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err < 0) { ext4_warning_inode(ea_inode, "Ensure credits err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } if (err > 0) { @@ -1221,6 +1224,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, ext4_warning_inode(ea_inode, "Re-get write access err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } } @@ -1229,6 +1233,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err) { ext4_warning_inode(ea_inode, "ea_inode dec ref err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } @@ -1245,6 +1250,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, entry->e_value_inum = 0; entry->e_value_size = 0; + ext4_put_ea_inode(ea_inode); dirty = true; } @@ -1271,7 +1277,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, static void ext4_xattr_release_block(handle_t *handle, struct inode *inode, struct buffer_head *bh, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits) { struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); @@ -1313,7 +1318,6 @@ retry_ref: ext4_xattr_inode_dec_ref_all(handle, inode, bh, BFIRST(bh), true /* block_csum */, - ea_inode_array, extra_credits, true /* skip_quota */); ext4_free_blocks(handle, inode, bh, 0, 1, @@ -1505,7 +1509,7 @@ static struct inode *ext4_xattr_inode_create(handle_t *handle, if (ext4_xattr_inode_dec_ref(handle, ea_inode)) ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return ERR_PTR(err); } @@ -1550,7 +1554,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, while (ce) { ea_inode = ext4_iget(inode->i_sb, ce->e_value, - EXT4_IGET_EA_INODE); + EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT); if (IS_ERR(ea_inode)) goto next_entry; ext4_xattr_inode_set_class(ea_inode); @@ -1564,7 +1568,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, kvfree(ea_data); return ea_inode; } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); next_entry: ce = mb_cache_entry_find_next(ea_inode_cache, ce); } @@ -1615,7 +1619,7 @@ static struct inode *ext4_xattr_inode_lookup_create(handle_t *handle, ea_inode->i_ino, true /* reusable */); return ea_inode; out_err: - iput(ea_inode); + ext4_put_ea_inode(ea_inode); ext4_xattr_inode_free_quota(inode, NULL, value_len); return ERR_PTR(err); } @@ -1848,7 +1852,7 @@ update_hash: ret = 0; out: - iput(old_ea_inode); + ext4_put_ea_inode(old_ea_inode); return ret; } @@ -2010,7 +2014,7 @@ clone_block: old_ea_inode_quota = le32_to_cpu( s->here->e_value_size); } - iput(tmp_inode); + ext4_put_ea_inode(tmp_inode); s->here->e_value_inum = 0; s->here->e_value_size = 0; @@ -2075,12 +2079,13 @@ inserted: * stable so we can check the additional * reference fits. */ - ref = le32_to_cpu(BHDR(new_bh)->h_refcount) + 1; - if (ref > EXT4_XATTR_REFCOUNT_MAX) { + ref = le32_to_cpu(BHDR(new_bh)->h_refcount); + if (ref >= EXT4_XATTR_REFCOUNT_MAX) { /* * Undo everything and check mbcache * again. */ + clear_bit(MBE_REUSABLE_B, &ce->e_flags); unlock_buffer(new_bh); dquot_free_block(inode, EXT4_C2B(EXT4_SB(sb), @@ -2091,6 +2096,7 @@ inserted: new_bh = NULL; goto inserted; } + ref++; BHDR(new_bh)->h_refcount = cpu_to_le32(ref); if (ref == EXT4_XATTR_REFCOUNT_MAX) clear_bit(MBE_REUSABLE_B, &ce->e_flags); @@ -2150,7 +2156,7 @@ getblk_failed: ext4_warning_inode(ea_inode, "dec ref error=%d", error); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); ea_inode = NULL; } @@ -2182,12 +2188,8 @@ getblk_failed: /* Drop the previous xattr block. */ if (bs->bh && bs->bh != new_bh) { - struct ext4_xattr_inode_array *ea_inode_array = NULL; - ext4_xattr_release_block(handle, inode, bs->bh, - &ea_inode_array, 0 /* extra_credits */); - ext4_xattr_inode_array_free(ea_inode_array); } error = 0; @@ -2203,7 +2205,7 @@ cleanup: ext4_xattr_inode_free_quota(inode, ea_inode, i_size_read(ea_inode)); } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } if (ce) mb_cache_entry_put(ea_block_cache, ce); @@ -2285,7 +2287,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, ext4_xattr_inode_free_quota(inode, ea_inode, i_size_read(ea_inode)); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return error; } @@ -2297,7 +2299,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, header->h_magic = cpu_to_le32(0); ext4_clear_inode_state(inode, EXT4_STATE_XATTR); } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return 0; } @@ -2839,6 +2841,7 @@ retry: s_min_extra_isize) { tried_min_extra_isize++; new_extra_isize = s_min_extra_isize; + error = 0; goto retry; } goto cleanup; @@ -2863,46 +2866,6 @@ cleanup: return error; } -#define EIA_INCR 16 /* must be 2^n */ -#define EIA_MASK (EIA_INCR - 1) - -/* Add the large xattr @inode into @ea_inode_array for deferred iput(). - * If @ea_inode_array is new or full it will be grown and the old - * contents copied over. - */ -static int -ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, - struct inode *inode) -{ - if (*ea_inode_array == NULL) { - /* - * Start with 15 inodes, so it fits into a power-of-two size. - */ - (*ea_inode_array) = kmalloc_flex(**ea_inode_array, inodes, - EIA_MASK, GFP_NOFS); - if (*ea_inode_array == NULL) - return -ENOMEM; - (*ea_inode_array)->count = 0; - } else if (((*ea_inode_array)->count & EIA_MASK) == EIA_MASK) { - /* expand the array once all 15 + n * 16 slots are full */ - struct ext4_xattr_inode_array *new_array = NULL; - - new_array = kmalloc_flex(**ea_inode_array, inodes, - (*ea_inode_array)->count + EIA_INCR, - GFP_NOFS); - if (new_array == NULL) - return -ENOMEM; - memcpy(new_array, *ea_inode_array, - struct_size(*ea_inode_array, inodes, - (*ea_inode_array)->count)); - kfree(*ea_inode_array); - *ea_inode_array = new_array; - } - (*ea_inode_array)->count++; - (*ea_inode_array)->inodes[(*ea_inode_array)->count - 1] = inode; - return 0; -} - /* * ext4_xattr_delete_inode() * @@ -2913,7 +2876,6 @@ ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, * references on xattr block and xattr inodes. */ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits) { struct buffer_head *bh = NULL; @@ -2952,7 +2914,6 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, ext4_xattr_inode_dec_ref_all(handle, inode, iloc.bh, IFIRST(header), false /* block_csum */, - ea_inode_array, extra_credits, false /* skip_quota */); } @@ -2986,12 +2947,12 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, continue; ext4_xattr_inode_free_quota(inode, ea_inode, le32_to_cpu(entry->e_value_size)); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } } - ext4_xattr_release_block(handle, inode, bh, ea_inode_array, + ext4_xattr_release_block(handle, inode, bh, extra_credits); /* * Update i_file_acl value in the same transaction that releases @@ -3013,16 +2974,63 @@ cleanup: return error; } -void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *ea_inode_array) +/* + * Worker function for deferred EA inode iput. Processes all inodes queued + * on s_ea_inode_to_free in a context free of xattr_sem/jbd2 handle locks. + */ +static void ext4_ea_inode_work(struct work_struct *work) { - int idx; + struct ext4_sb_info *sbi = container_of(to_delayed_work(work), + struct ext4_sb_info, + s_ea_inode_work); + struct llist_node *node = llist_del_all(&sbi->s_ea_inode_to_free); - if (ea_inode_array == NULL) + while (node) { + struct ext4_inode_info *ei = container_of(node, + struct ext4_inode_info, i_ea_iput_node); + node = node->next; + iput(&ei->vfs_inode); + } +} + +/* + * Release a VFS reference on an EA inode. Must be used instead of iput() + * in any context where xattr_sem or a jbd2 handle is held. + * + * If this is not the last reference, drops it immediately via + * iput_if_not_last() with no further action needed. + * + * If this is the last reference, the inode is linked onto a per-sb + * llist via i_ea_iput_node (embedded in ext4_inode_info, sharing space + * with the unused xattr_sem) and a delayed worker performs the final + * iput() in a clean context. + * + * Note: while an inode is on s_ea_inode_to_free, the unconsumed i_count + * reference (still 1) keeps it in the inode cache, so any concurrent + * iget() bumps i_count to >= 2 and iput_if_not_last() will succeed. + * Nobody will add the inode a second time until ext4_ea_inode_work() + * drops that reference via iput(). + */ +void ext4_put_ea_inode(struct inode *inode) +{ + if (!inode) return; + WARN_ON_ONCE(!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)); + if (iput_if_not_last(inode)) + return; + llist_add(&EXT4_I(inode)->i_ea_iput_node, + &EXT4_SB(inode->i_sb)->s_ea_inode_to_free); + /* + * Use a short delay to allow multiple EA inodes to accumulate, + * reducing workqueue wakeups when several are released together. + */ + schedule_delayed_work(&EXT4_SB(inode->i_sb)->s_ea_inode_work, 1); +} - for (idx = 0; idx < ea_inode_array->count; ++idx) - iput(ea_inode_array->inodes[idx]); - kfree(ea_inode_array); +void ext4_init_ea_inode_work(struct ext4_sb_info *sbi) +{ + init_llist_head(&sbi->s_ea_inode_to_free); + INIT_DELAYED_WORK(&sbi->s_ea_inode_work, ext4_ea_inode_work); } /* diff --git a/fs/ext4/xattr.h b/fs/ext4/xattr.h index 1fedf44d4fb6..821dc6a50e51 100644 --- a/fs/ext4/xattr.h +++ b/fs/ext4/xattr.h @@ -131,11 +131,6 @@ struct ext4_xattr_ibody_find { struct ext4_iloc iloc; }; -struct ext4_xattr_inode_array { - unsigned int count; - struct inode *inodes[] __counted_by(count); -}; - extern const struct xattr_handler ext4_xattr_user_handler; extern const struct xattr_handler ext4_xattr_trusted_handler; extern const struct xattr_handler ext4_xattr_security_handler; @@ -187,9 +182,9 @@ extern int __ext4_xattr_set_credits(struct super_block *sb, struct inode *inode, bool is_create); extern int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, - struct ext4_xattr_inode_array **array, int extra_credits); -extern void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *array); +extern void ext4_init_ea_inode_work(struct ext4_sb_info *sbi); +extern void ext4_put_ea_inode(struct inode *inode); extern int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize, struct ext4_inode *raw_inode, handle_t *handle); diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index 397568b9c7e7..f9869d62b850 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -65,12 +65,14 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, /* Empty block? */ if (block_size == 0) { for ( i = 0 ; i < pcount ; i++ ) { + unsigned int off = i ? 0 : poffset; + if (!pages[i]) continue; - memzero_page(pages[i], 0, PAGE_SIZE); + memzero_page(pages[i], off, PAGE_SIZE - off); SetPageUptodate(pages[i]); } - return ((loff_t)pcount) << PAGE_SHIFT; + return (((loff_t)pcount) << PAGE_SHIFT) - poffset; } /* Because zlib is not thread-safe, do all the I/O at the top. */ @@ -291,6 +293,7 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount, memzero_page(*pages, poffset, PAGE_SIZE - poffset); SetPageUptodate(*pages); } + brelse(bh); return 0; } diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c index cc587cd25162..c7ca7603e97a 100644 --- a/fs/isofs/dir.c +++ b/fs/isofs/dir.c @@ -84,7 +84,7 @@ int get_acorn_filename(struct iso_directory_record *de, */ static int do_isofs_readdir(struct inode *inode, struct file *file, struct dir_context *ctx, - char *tmpname, struct iso_directory_record *tmpde) + char *tmpname) { unsigned long bufsize = ISOFS_BUFFER_SIZE(inode); unsigned char bufbits = ISOFS_BUFFER_BITS(inode); @@ -133,26 +133,7 @@ static int do_isofs_readdir(struct inode *inode, struct file *file, offset_saved = offset; offset += de_len; - /* Make sure we have a full directory entry */ - if (offset >= bufsize) { - int slop = bufsize - offset + de_len; - memcpy(tmpde, de, slop); - offset &= bufsize - 1; - block++; - brelse(bh); - bh = NULL; - if (offset) { - bh = isofs_bread(inode, block); - if (!bh) - return 0; - memcpy((void *) tmpde + slop, bh->b_data, offset); - } - de = tmpde; - } - /* Basic sanity check, whether name doesn't exceed dir entry */ - if (de_len < sizeof(struct iso_directory_record) || - de_len < de->name_len[0] + - sizeof(struct iso_directory_record)) { + if (!isofs_dir_record_valid(de, offset_saved, bufsize)) { printk(KERN_NOTICE "iso9660: Corrupted directory entry" " in block %lu of inode %llu\n", block, inode->i_ino); @@ -254,16 +235,13 @@ static int isofs_readdir(struct file *file, struct dir_context *ctx) { int result; char *tmpname; - struct iso_directory_record *tmpde; struct inode *inode = file_inode(file); - tmpname = kmalloc(PAGE_SIZE, GFP_KERNEL); + tmpname = kmalloc(1024, GFP_KERNEL); if (tmpname == NULL) return -ENOMEM; - tmpde = (struct iso_directory_record *) (tmpname+1024); - - result = do_isofs_readdir(inode, file, ctx, tmpname, tmpde); + result = do_isofs_readdir(inode, file, ctx, tmpname); kfree(tmpname); return result; @@ -300,4 +278,3 @@ const struct inode_operations isofs_dir_inode_operations = .fileattr_get = isofs_fileattr_get, }; - diff --git a/fs/isofs/export.c b/fs/isofs/export.c index 78f80c1a5c54..4f7fa1d508a1 100644 --- a/fs/isofs/export.c +++ b/fs/isofs/export.c @@ -83,13 +83,21 @@ static struct dentry *isofs_export_get_parent(struct dentry *child) /* This is the "." entry. */ de = (struct iso_directory_record*)bh->b_data; + if (!isofs_dir_record_valid(de, 0, child_inode->i_sb->s_blocksize) || + isonum_711(de->name_len) != 1 || de->name[0] != 0) { + printk(KERN_ERR "isofs: Unable to find the \".\" directory for NFS.\n"); + rv = ERR_PTR(-EACCES); + goto out; + } /* The ".." entry is always the second entry. */ parent_offset = (unsigned long)isonum_711(de->length); de = (struct iso_directory_record*)(bh->b_data + parent_offset); /* Verify it is in fact the ".." entry. */ - if ((isonum_711(de->name_len) != 1) || (de->name[0] != 1)) { + if (!isofs_dir_record_valid(de, parent_offset, + child_inode->i_sb->s_blocksize) || + isonum_711(de->name_len) != 1 || de->name[0] != 1) { printk(KERN_ERR "isofs: Unable to find the \"..\" " "directory for NFS.\n"); rv = ERR_PTR(-EACCES); diff --git a/fs/isofs/isofs.h b/fs/isofs/isofs.h index 0ec8b24a42ed..dacb9cdae4fd 100644 --- a/fs/isofs/isofs.h +++ b/fs/isofs/isofs.h @@ -115,6 +115,9 @@ struct inode; /* To make gcc happy */ extern int parse_rock_ridge_inode(struct iso_directory_record *, struct inode *, int relocated); extern int get_rock_ridge_filename(struct iso_directory_record *, char *, struct inode *); extern int isofs_name_translate(struct iso_directory_record *, char *, struct inode *); +bool isofs_dir_record_valid(struct iso_directory_record *de, + unsigned long offset, + unsigned long bufsize); int get_joliet_filename(struct iso_directory_record *, unsigned char *, struct inode *); int get_acorn_filename(struct iso_directory_record *, char *, struct inode *); diff --git a/fs/isofs/namei.c b/fs/isofs/namei.c index 3ace3d6a55e7..010682f5901a 100644 --- a/fs/isofs/namei.c +++ b/fs/isofs/namei.c @@ -10,6 +10,26 @@ #include <linux/gfp.h> #include "isofs.h" +bool isofs_dir_record_valid(struct iso_directory_record *de, + unsigned long offset, + unsigned long bufsize) +{ + unsigned int len; + unsigned int name_len; + unsigned long min_len = offsetof(struct iso_directory_record, name); + + if (offset > bufsize || bufsize - offset < min_len) + return false; + + len = isonum_711(de->length); + name_len = isonum_711(de->name_len); + if (len < min_len || name_len > len - min_len) + return false; + if (len > bufsize - offset) + return false; + return true; +} + static int isofs_cmp(struct dentry *dentry, const char *compare, int dlen) { @@ -28,7 +48,7 @@ isofs_cmp(struct dentry *dentry, const char *compare, int dlen) static unsigned long isofs_find_entry(struct inode *dir, struct dentry *dentry, unsigned long *block_rv, unsigned long *offset_rv, - char *tmpname, struct iso_directory_record *tmpde) + char *tmpname) { unsigned long bufsize = ISOFS_BUFFER_SIZE(dir); unsigned char bufbits = ISOFS_BUFFER_BITS(dir); @@ -71,33 +91,15 @@ isofs_find_entry(struct inode *dir, struct dentry *dentry, offset += de_len; f_pos += de_len; - /* Make sure we have a full directory entry */ - if (offset >= bufsize) { - int slop = bufsize - offset + de_len; - memcpy(tmpde, de, slop); - offset &= bufsize - 1; - block++; - brelse(bh); - bh = NULL; - if (offset) { - bh = isofs_bread(dir, block); - if (!bh) - return 0; - memcpy((void *) tmpde + slop, bh->b_data, offset); - } - de = tmpde; - } - - dlen = de->name_len[0]; - dpnt = de->name; - /* Basic sanity check, whether name doesn't exceed dir entry */ - if (de_len < dlen + sizeof(struct iso_directory_record)) { + if (!isofs_dir_record_valid(de, offset_saved, bufsize)) { printk(KERN_NOTICE "iso9660: Corrupted directory entry" " in block %lu of inode %llu\n", block, dir->i_ino); brelse(bh); return 0; } + dlen = de->name_len[0]; + dpnt = de->name; if (sbi->s_rock && ((i = get_rock_ridge_filename(de, tmpname, dir)))) { @@ -149,17 +151,14 @@ struct dentry *isofs_lookup(struct inode *dir, struct dentry *dentry, unsigned i unsigned long block; unsigned long offset; struct inode *inode; - struct page *page; + char *tmpname; - page = alloc_page(GFP_USER); - if (!page) + tmpname = kmalloc(1024, GFP_USER); + if (!tmpname) return ERR_PTR(-ENOMEM); - found = isofs_find_entry(dir, dentry, - &block, &offset, - page_address(page), - 1024 + page_address(page)); - __free_page(page); + found = isofs_find_entry(dir, dentry, &block, &offset, tmpname); + kfree(tmpname); inode = found ? isofs_iget(dir->i_sb, block, offset) : NULL; diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 1508e2f54462..513273712010 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -358,15 +358,16 @@ int jbd2_cleanup_journal_tail(journal_t *journal) /* * journal_shrink_one_cp_list * - * Find all the written-back checkpoint buffers in the given list - * and try to release them. If the whole transaction is released, set - * the 'released' parameter. Return the number of released checkpointed - * buffers. + * Find written-back checkpoint buffers in the given list and try to release + * them. If 'nr_to_scan' is set, scan at most that many buffers. If the whole + * transaction is released, set the 'released' parameter. Return the number of + * released checkpointed buffers. * * Called with j_list_lock held. */ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, enum jbd2_shrink_type type, + unsigned long *nr_to_scan, bool *released) { struct journal_head *last_jh; @@ -375,13 +376,15 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, int ret; *released = false; - if (!jh) + if (!jh || (nr_to_scan && !*nr_to_scan)) return 0; last_jh = jh->b_cpprev; do { jh = next_jh; next_jh = jh->b_cpnext; + if (nr_to_scan) + (*nr_to_scan)--; if (type == JBD2_SHRINK_DESTROY) { ret = __jbd2_journal_remove_checkpoint(jh); @@ -389,7 +392,7 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, ret = jbd2_journal_try_remove_checkpoint(jh); if (ret < 0) { if (type == JBD2_SHRINK_BUSY_SKIP) - continue; + goto next; break; } } @@ -400,9 +403,10 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, break; } +next: if (need_resched()) break; - } while (jh != last_jh); + } while (jh != last_jh && (!nr_to_scan || *nr_to_scan)); return nr_freed; } @@ -424,7 +428,6 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, tid_t first_tid = 0, last_tid = 0, next_tid = 0; tid_t tid = 0; unsigned long nr_freed = 0; - unsigned long freed; bool first_set = false; again: @@ -457,10 +460,9 @@ again: next_transaction = transaction->t_cpnext; tid = transaction->t_tid; - freed = journal_shrink_one_cp_list(transaction->t_checkpoint_list, - JBD2_SHRINK_BUSY_SKIP, &released); - nr_freed += freed; - (*nr_to_scan) -= min(*nr_to_scan, freed); + nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_list, + JBD2_SHRINK_BUSY_SKIP, + nr_to_scan, &released); if (*nr_to_scan == 0) break; if (need_resched() || spin_needbreak(&journal->j_list_lock)) @@ -516,7 +518,7 @@ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, transaction = next_transaction; next_transaction = transaction->t_cpnext; journal_shrink_one_cp_list(transaction->t_checkpoint_list, - type, &released); + type, NULL, &released); /* * This function only frees up some memory if possible so we * dont have an obligation to finish processing. Bail out if diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 09efa337649e..00f5a98f3d4f 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -94,6 +94,7 @@ EXPORT_SYMBOL(jbd2_journal_init_jbd_inode); EXPORT_SYMBOL(jbd2_journal_release_jbd_inode); EXPORT_SYMBOL(jbd2_journal_begin_ordered_truncate); EXPORT_SYMBOL(jbd2_inode_cache); +EXPORT_SYMBOL(jbd2_handle_cache); #ifdef CONFIG_JBD2_DEBUG void __jbd2_debug(int level, const char *file, const char *func, diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h index e418a50c4180..14cc952fe81a 100644 --- a/fs/lockd/lockd.h +++ b/fs/lockd/lockd.h @@ -314,7 +314,7 @@ void nsm_release(struct nsm_handle *nsm); * This is used in garbage collection and resource reclaim * A return value != 0 means destroy the lock/block/share */ -typedef int (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref); +typedef int (*nlm_host_match_fn_t)(void *owner, struct nlm_host *ref); /* * Server-side lock handling diff --git a/fs/lockd/nlm3xdr_gen.c b/fs/lockd/nlm3xdr_gen.c index 9ed5a41b5daf..4b6e3ee7a719 100644 --- a/fs/lockd/nlm3xdr_gen.c +++ b/fs/lockd/nlm3xdr_gen.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x -// XDR specification modification time: Thu Apr 23 10:56:34 2026 +// XDR specification modification time: Mon Jun 29 20:42:29 2026 #include <linux/sunrpc/svc.h> @@ -272,20 +272,6 @@ xdrgen_decode_nlm_notifyargs(struct xdr_stream *xdr, struct nlm_notifyargs *ptr) } /** - * nlm_svc_decode_void - Decode a void argument - * @rqstp: RPC transaction context - * @xdr: source XDR data stream - * - * Return values: - * %true: procedure arguments decoded successfully - * %false: decode failed - */ -bool nlm_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - return xdrgen_decode_void(xdr); -} - -/** * nlm_svc_decode_nlm_testargs - Decode a nlm_testargs argument * @rqstp: RPC transaction context * @xdr: source XDR data stream @@ -652,20 +638,6 @@ xdrgen_encode_nlm_notifyargs(struct xdr_stream *xdr, const struct nlm_notifyargs } /** - * nlm_svc_encode_void - Encode a void result - * @rqstp: RPC transaction context - * @xdr: target XDR data stream - * - * Return values: - * %true: procedure results encoded successfully - * %false: encode failed - */ -bool nlm_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - return xdrgen_encode_void(xdr); -} - -/** * nlm_svc_encode_nlm_testres - Encode a nlm_testres result * @rqstp: RPC transaction context * @xdr: target XDR data stream diff --git a/fs/lockd/nlm3xdr_gen.h b/fs/lockd/nlm3xdr_gen.h index c99038e99805..bdbfc26ba0e4 100644 --- a/fs/lockd/nlm3xdr_gen.h +++ b/fs/lockd/nlm3xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ -/* XDR specification modification time: Thu Apr 23 10:56:34 2026 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM3_DECL_H #define _LINUX_XDRGEN_NLM3_DECL_H @@ -13,7 +13,6 @@ #include <linux/sunrpc/xdrgen/_builtins.h> #include <linux/sunrpc/xdrgen/nlm3.h> -bool nlm_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm_svc_decode_nlm_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm_svc_decode_nlm_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm_svc_decode_nlm_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); @@ -24,7 +23,6 @@ bool nlm_svc_decode_nlm_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xd bool nlm_svc_decode_nlm_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm_svc_decode_nlm_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm_svc_encode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm_svc_encode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm_svc_encode_nlm_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); diff --git a/fs/lockd/nlm4xdr_gen.c b/fs/lockd/nlm4xdr_gen.c index 1c8c221db456..f98d74ee3d41 100644 --- a/fs/lockd/nlm4xdr_gen.c +++ b/fs/lockd/nlm4xdr_gen.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x -// XDR specification modification time: Thu Dec 25 13:10:19 2025 +// XDR specification modification time: Mon Jun 29 20:42:29 2026 #include <linux/sunrpc/svc.h> @@ -20,6 +20,16 @@ xdrgen_decode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode *ptr) if (xdr_stream_decode_u32(xdr, &val) < 0) return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsm_DN: + case fsm_DR: + case fsm_DW: + case fsm_DRW: + break; + default: + return false; + } *ptr = val; return true; } @@ -31,6 +41,16 @@ xdrgen_decode_fsh4_access(struct xdr_stream *xdr, fsh4_access *ptr) if (xdr_stream_decode_u32(xdr, &val) < 0) return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsa_NONE: + case fsa_R: + case fsa_W: + case fsa_RW: + break; + default: + return false; + } *ptr = val; return true; } @@ -62,7 +82,30 @@ xdrgen_decode_int32(struct xdr_stream *xdr, int32 *ptr) static bool __maybe_unused xdrgen_decode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats *ptr) { - return xdr_stream_decode_be32(xdr, ptr) == 0; + __be32 raw; + u32 val; + + if (xdr_stream_decode_be32(xdr, &raw) < 0) + return false; + val = be32_to_cpu(raw); + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NLM4_GRANTED: + case NLM4_DENIED: + case NLM4_DENIED_NOLOCKS: + case NLM4_BLOCKED: + case NLM4_DENIED_GRACE_PERIOD: + case NLM4_DEADLCK: + case NLM4_ROFS: + case NLM4_STALE_FH: + case NLM4_FBIG: + case NLM4_FAILED: + break; + default: + return false; + } + *ptr = raw; + return true; } static bool __maybe_unused @@ -258,20 +301,6 @@ xdrgen_decode_nlm4_notifyargs(struct xdr_stream *xdr, struct nlm4_notifyargs *pt } /** - * nlm4_svc_decode_void - Decode a void argument - * @rqstp: RPC transaction context - * @xdr: source XDR data stream - * - * Return values: - * %true: procedure arguments decoded successfully - * %false: decode failed - */ -bool nlm4_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - return xdrgen_decode_void(xdr); -} - -/** * nlm4_svc_decode_nlm4_testargs - Decode a nlm4_testargs argument * @rqstp: RPC transaction context * @xdr: source XDR data stream @@ -662,20 +691,6 @@ xdrgen_encode_nlm4_notifyargs(struct xdr_stream *xdr, const struct nlm4_notifyar } /** - * nlm4_svc_encode_void - Encode a void result - * @rqstp: RPC transaction context - * @xdr: target XDR data stream - * - * Return values: - * %true: procedure results encoded successfully - * %false: encode failed - */ -bool nlm4_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) -{ - return xdrgen_encode_void(xdr); -} - -/** * nlm4_svc_encode_nlm4_testres - Encode a nlm4_testres result * @rqstp: RPC transaction context * @xdr: target XDR data stream diff --git a/fs/lockd/nlm4xdr_gen.h b/fs/lockd/nlm4xdr_gen.h index b6008b296a3e..1a72c26ad18a 100644 --- a/fs/lockd/nlm4xdr_gen.h +++ b/fs/lockd/nlm4xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ -/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM4_DECL_H #define _LINUX_XDRGEN_NLM4_DECL_H @@ -13,7 +13,6 @@ #include <linux/sunrpc/xdrgen/_builtins.h> #include <linux/sunrpc/xdrgen/nlm4.h> -bool nlm4_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm4_svc_decode_nlm4_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm4_svc_decode_nlm4_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm4_svc_decode_nlm4_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); @@ -24,7 +23,6 @@ bool nlm4_svc_decode_nlm4_notifyargs(struct svc_rqst *rqstp, struct xdr_stream * bool nlm4_svc_decode_nlm4_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm4_svc_decode_nlm4_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm4_svc_encode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm4_svc_encode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); bool nlm4_svc_encode_nlm4_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); diff --git a/fs/lockd/share.h b/fs/lockd/share.h index 1ec3ccdb2aef..a12b6c454f58 100644 --- a/fs/lockd/share.h +++ b/fs/lockd/share.h @@ -8,9 +8,14 @@ #ifndef _LOCKD_SHARE_H #define _LOCKD_SHARE_H +#include <linux/bits.h> + /* Synthetic svid for lockowner lookup during share operations */ #define LOCKD_SHARE_SVID (~(u32)0) +/* One bit per (access, deny) pair; index = (access << 2) | deny */ +#define LOCKD_FSH_BIT(a, d) BIT(((a) << 2) | (d)) + /* * DOS share for a specific file */ @@ -21,12 +26,13 @@ struct lockd_share { struct xdr_netobj s_owner; /* owner handle */ u32 s_access; /* access mode */ u32 s_mode; /* deny mode */ + u16 s_access_deny_bmap; /* held (access, deny) pairs */ }; __be32 nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file, struct xdr_netobj *oh, u32 access, u32 mode); __be32 nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, - struct xdr_netobj *oh); + struct xdr_netobj *oh, u32 access, u32 mode); void nlmsvc_traverse_shares(struct nlm_host *, struct nlm_file *, nlm_host_match_fn_t); diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index 490551369ef2..ee90e743064a 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -47,7 +47,7 @@ static struct svc_program nlmsvc_program; -const struct nlmsvc_binding *nlmsvc_ops; +const struct nlmsvc_binding __rcu *nlmsvc_ops; EXPORT_SYMBOL_GPL(nlmsvc_ops); static DEFINE_MUTEX(nlmsvc_mutex); @@ -142,7 +142,7 @@ lockd(void *vrqstp) nlmsvc_retry_blocked(rqstp); svc_recv(rqstp, 0); } - if (nlmsvc_ops) + if (rcu_access_pointer(nlmsvc_ops)) nlmsvc_invalidate_all(); nlm_shutdown_hosts(); cancel_delayed_work_sync(&ln->grace_period_end); diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index 78e675470c4b..03c5554ca579 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -128,7 +128,7 @@ nlm4svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) { struct nlm_host *host; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return NULL; host = nlmsvc_lookup_host(rqstp, caller.data, caller.len); if (!host) @@ -872,7 +872,8 @@ static __be32 nlm4svc_proc_granted_msg(struct svc_rqst *rqstp) struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; struct nlm_host *host; - host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + host = nlmsvc_lookup_host(rqstp, argp->xdrgen.alock.caller_name.data, + argp->xdrgen.alock.caller_name.len); if (!host) return rpc_system_err; @@ -894,7 +895,7 @@ static __be32 nlm4svc_proc_granted_res(struct svc_rqst *rqstp) { struct nlm4_res_wrapper *argp = rqstp->rq_argp; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return rpc_success; if (nlm4_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie)) @@ -1078,7 +1079,9 @@ static __be32 nlm4svc_proc_unshare(struct svc_rqst *rqstp) if (resp->xdrgen.stat) goto out; - resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh); + resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh, + argp->xdrgen.share.access, + argp->xdrgen.share.mode); nlmsvc_release_lockowner(lock); @@ -1164,8 +1167,8 @@ out: static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_NULL] = { .pc_func = nlm4svc_proc_null, - .pc_decode = nlm4_svc_decode_void, - .pc_encode = nlm4_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = XDR_void, .pc_argzero = 0, .pc_ressize = 0, @@ -1225,7 +1228,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_TEST_MSG] = { .pc_func = nlm4svc_proc_test_msg, .pc_decode = nlm4_svc_decode_nlm4_testargs, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_testargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1235,7 +1238,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_LOCK_MSG] = { .pc_func = nlm4svc_proc_lock_msg, .pc_decode = nlm4_svc_decode_nlm4_lockargs, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_lockargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1245,7 +1248,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_CANCEL_MSG] = { .pc_func = nlm4svc_proc_cancel_msg, .pc_decode = nlm4_svc_decode_nlm4_cancargs, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_cancargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1255,7 +1258,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_UNLOCK_MSG] = { .pc_func = nlm4svc_proc_unlock_msg, .pc_decode = nlm4_svc_decode_nlm4_unlockargs, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_unlockargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1265,7 +1268,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_GRANTED_MSG] = { .pc_func = nlm4svc_proc_granted_msg, .pc_decode = nlm4_svc_decode_nlm4_testargs, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_testargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1275,7 +1278,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_TEST_RES] = { .pc_func = nlm4svc_proc_null, .pc_decode = nlm4_svc_decode_nlm4_testres, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_testres), .pc_argzero = 0, .pc_ressize = 0, @@ -1285,7 +1288,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_LOCK_RES] = { .pc_func = nlm4svc_proc_null, .pc_decode = nlm4_svc_decode_nlm4_res, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_res), .pc_argzero = 0, .pc_ressize = 0, @@ -1295,7 +1298,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_CANCEL_RES] = { .pc_func = nlm4svc_proc_null, .pc_decode = nlm4_svc_decode_nlm4_res, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_res), .pc_argzero = 0, .pc_ressize = 0, @@ -1305,7 +1308,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_UNLOCK_RES] = { .pc_func = nlm4svc_proc_null, .pc_decode = nlm4_svc_decode_nlm4_res, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_res), .pc_argzero = 0, .pc_ressize = 0, @@ -1315,7 +1318,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_GRANTED_RES] = { .pc_func = nlm4svc_proc_granted_res, .pc_decode = nlm4_svc_decode_nlm4_res, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_res_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1325,7 +1328,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_SM_NOTIFY] = { .pc_func = nlm4svc_proc_sm_notify, .pc_decode = nlm4_svc_decode_nlm4_notifyargs, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_notifyargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1334,8 +1337,8 @@ static const struct svc_procedure nlm4svc_procedures[24] = { }, [17] = { .pc_func = nlm4svc_proc_unused, - .pc_decode = nlm4_svc_decode_void, - .pc_encode = nlm4_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = 0, .pc_argzero = 0, .pc_ressize = 0, @@ -1344,8 +1347,8 @@ static const struct svc_procedure nlm4svc_procedures[24] = { }, [18] = { .pc_func = nlm4svc_proc_unused, - .pc_decode = nlm4_svc_decode_void, - .pc_encode = nlm4_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = 0, .pc_argzero = 0, .pc_ressize = 0, @@ -1354,8 +1357,8 @@ static const struct svc_procedure nlm4svc_procedures[24] = { }, [19] = { .pc_func = nlm4svc_proc_unused, - .pc_decode = nlm4_svc_decode_void, - .pc_encode = nlm4_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = 0, .pc_argzero = 0, .pc_ressize = 0, @@ -1395,7 +1398,7 @@ static const struct svc_procedure nlm4svc_procedures[24] = { [NLMPROC4_FREE_ALL] = { .pc_func = nlm4svc_proc_free_all, .pc_decode = nlm4_svc_decode_nlm4_notify, - .pc_encode = nlm4_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm4_notify_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1420,14 +1423,10 @@ union nlm4svc_xdrstore { struct nlm4_shareres_wrapper shareres; }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nlm4svc_call_counters[ARRAY_SIZE(nlm4svc_procedures)]); - const struct svc_version nlmsvc_version4 = { .vs_vers = 4, .vs_nproc = ARRAY_SIZE(nlm4svc_procedures), .vs_proc = nlm4svc_procedures, - .vs_count = nlm4svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize = sizeof(union nlm4svc_xdrstore), }; diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index e48d31f14a65..e628b5d35507 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -47,40 +47,6 @@ static const struct rpc_call_ops nlmsvc_grant_ops; static LIST_HEAD(nlm_blocked); static DEFINE_SPINLOCK(nlm_blocked_lock); -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) -static const char *nlmdbg_cookie2a(const struct lockd_cookie *cookie) -{ - /* - * We can get away with a static buffer because this is only called - * from lockd, which is single-threaded. - */ - static char buf[2*NLM_MAXCOOKIELEN+1]; - unsigned int i, len = sizeof(buf); - char *p = buf; - - len--; /* allow for trailing \0 */ - if (len < 3) - return "???"; - for (i = 0 ; i < cookie->len ; i++) { - if (len < 2) { - strcpy(p-3, "..."); - break; - } - sprintf(p, "%02x", cookie->data[i]); - p += 2; - len -= 2; - } - *p = '\0'; - - return buf; -} -#else -static inline const char *nlmdbg_cookie2a(const struct lockd_cookie *cookie) -{ - return "???"; -} -#endif - /* * Insert a blocked lock into the global list */ @@ -155,11 +121,12 @@ nlmsvc_lookup_block(struct nlm_file *file, struct lockd_lock *lock) spin_lock(&nlm_blocked_lock); list_for_each_entry(block, &nlm_blocked, b_list) { fl = &block->b_call->a_args.lock.fl; - dprintk("lockd: check f=%p pd=%d %Ld-%Ld ty=%d cookie=%s\n", + dprintk("lockd: check f=%p pd=%d %Ld-%Ld ty=%d cookie=%*phN\n", block->b_file, fl->c.flc_pid, (long long)fl->fl_start, (long long)fl->fl_end, fl->c.flc_type, - nlmdbg_cookie2a(&block->b_call->a_args.cookie)); + block->b_call->a_args.cookie.len, + block->b_call->a_args.cookie.data); if (block->b_file == file && nlm_compare_locks(fl, &lock->fl)) { kref_get(&block->b_count); spin_unlock(&nlm_blocked_lock); @@ -198,7 +165,8 @@ nlmsvc_find_block(struct lockd_cookie *cookie) return NULL; found: - dprintk("nlmsvc_find_block(%s): block=%p\n", nlmdbg_cookie2a(cookie), block); + dprintk("nlmsvc_find_block(%*phN): block=%p\n", + cookie->len, cookie->data, block); kref_get(&block->b_count); spin_unlock(&nlm_blocked_lock); return block; diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index 386a881b520f..a8b5195c2c6b 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -133,7 +133,7 @@ nlm3svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) { struct nlm_host *host; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return NULL; host = nlmsvc_lookup_host(rqstp, caller.data, caller.len); if (!host) @@ -924,7 +924,7 @@ static __be32 nlmsvc_proc_granted_res(struct svc_rqst *rqstp) { struct nlm_res_wrapper *argp = rqstp->rq_argp; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return rpc_success; if (nlm_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie)) @@ -1098,7 +1098,9 @@ static __be32 nlmsvc_proc_unshare(struct svc_rqst *rqstp) if (resp->xdrgen.stat) goto out; - resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh); + resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh, + argp->xdrgen.share.access, + argp->xdrgen.share.mode); nlmsvc_release_lockowner(lock); @@ -1174,8 +1176,8 @@ out: static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_NULL] = { .pc_func = nlmsvc_proc_null, - .pc_decode = nlm_svc_decode_void, - .pc_encode = nlm_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = XDR_void, .pc_argzero = 0, .pc_ressize = 0, @@ -1235,7 +1237,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_TEST_MSG] = { .pc_func = nlmsvc_proc_test_msg, .pc_decode = nlm_svc_decode_nlm_testargs, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_testargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1245,7 +1247,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_LOCK_MSG] = { .pc_func = nlmsvc_proc_lock_msg, .pc_decode = nlm_svc_decode_nlm_lockargs, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_lockargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1255,7 +1257,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_CANCEL_MSG] = { .pc_func = nlmsvc_proc_cancel_msg, .pc_decode = nlm_svc_decode_nlm_cancargs, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_cancargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1265,7 +1267,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_UNLOCK_MSG] = { .pc_func = nlmsvc_proc_unlock_msg, .pc_decode = nlm_svc_decode_nlm_unlockargs, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_unlockargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1275,7 +1277,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_GRANTED_MSG] = { .pc_func = nlmsvc_proc_granted_msg, .pc_decode = nlm_svc_decode_nlm_testargs, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_testargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1285,7 +1287,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_TEST_RES] = { .pc_func = nlmsvc_proc_null, .pc_decode = nlm_svc_decode_nlm_testres, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_testres), .pc_argzero = 0, .pc_ressize = 0, @@ -1295,7 +1297,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_LOCK_RES] = { .pc_func = nlmsvc_proc_null, .pc_decode = nlm_svc_decode_nlm_res, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_res), .pc_argzero = 0, .pc_ressize = 0, @@ -1305,7 +1307,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_CANCEL_RES] = { .pc_func = nlmsvc_proc_null, .pc_decode = nlm_svc_decode_nlm_res, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_res), .pc_argzero = 0, .pc_ressize = 0, @@ -1315,7 +1317,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_UNLOCK_RES] = { .pc_func = nlmsvc_proc_null, .pc_decode = nlm_svc_decode_nlm_res, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_res), .pc_argzero = 0, .pc_ressize = 0, @@ -1325,7 +1327,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_GRANTED_RES] = { .pc_func = nlmsvc_proc_granted_res, .pc_decode = nlm_svc_decode_nlm_res, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_res_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1335,7 +1337,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_SM_NOTIFY] = { .pc_func = nlmsvc_proc_sm_notify, .pc_decode = nlm_svc_decode_nlm_notifyargs, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_notifyargs_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1344,8 +1346,8 @@ static const struct svc_procedure nlmsvc_procedures[24] = { }, [17] = { .pc_func = nlmsvc_proc_unused, - .pc_decode = nlm_svc_decode_void, - .pc_encode = nlm_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = 0, .pc_argzero = 0, .pc_ressize = 0, @@ -1354,8 +1356,8 @@ static const struct svc_procedure nlmsvc_procedures[24] = { }, [18] = { .pc_func = nlmsvc_proc_unused, - .pc_decode = nlm_svc_decode_void, - .pc_encode = nlm_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = 0, .pc_argzero = 0, .pc_ressize = 0, @@ -1364,8 +1366,8 @@ static const struct svc_procedure nlmsvc_procedures[24] = { }, [19] = { .pc_func = nlmsvc_proc_unused, - .pc_decode = nlm_svc_decode_void, - .pc_encode = nlm_svc_encode_void, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = 0, .pc_argzero = 0, .pc_ressize = 0, @@ -1405,7 +1407,7 @@ static const struct svc_procedure nlmsvc_procedures[24] = { [NLM_FREE_ALL] = { .pc_func = nlmsvc_proc_free_all, .pc_decode = nlm_svc_decode_nlm_notify, - .pc_encode = nlm_svc_encode_void, + .pc_encode = xdrgen_svc_encode_void, .pc_argsize = sizeof(struct nlm_notify_wrapper), .pc_argzero = 0, .pc_ressize = 0, @@ -1434,25 +1436,18 @@ union nlmsvc_xdrstore { * NLMv1 defines only procedures 1 - 15. Linux lockd also implements * procedures 0 (NULL) and 16 (SM_NOTIFY). */ -static DEFINE_PER_CPU_ALIGNED(unsigned long, nlm1svc_call_counters[17]); - const struct svc_version nlmsvc_version1 = { .vs_vers = 1, .vs_nproc = 17, .vs_proc = nlmsvc_procedures, - .vs_count = nlm1svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize = sizeof(union nlmsvc_xdrstore), }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nlm3svc_call_counters[ARRAY_SIZE(nlmsvc_procedures)]); - const struct svc_version nlmsvc_version3 = { .vs_vers = 3, .vs_nproc = ARRAY_SIZE(nlmsvc_procedures), .vs_proc = nlmsvc_procedures, - .vs_count = nlm3svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize = sizeof(union nlmsvc_xdrstore), }; diff --git a/fs/lockd/svcshare.c b/fs/lockd/svcshare.c index 5ac0ec25d62d..a58b7035b58b 100644 --- a/fs/lockd/svcshare.c +++ b/fs/lockd/svcshare.c @@ -25,6 +25,25 @@ nlm_cmp_owner(struct lockd_share *share, struct xdr_netobj *oh) && !memcmp(share->s_owner.data, oh->data, oh->len); } +/* + * Recompute s_access / s_mode as the union of every (access, deny) pair + * whose bit is currently set in s_access_deny_bmap. + */ +static void nlm_recompute_share(struct lockd_share *share) +{ + u32 new_access = 0, new_mode = 0; + unsigned int i; + + for (i = 0; i < 16; i++) { + if (share->s_access_deny_bmap & BIT(i)) { + new_access |= i >> 2; + new_mode |= i & 3; + } + } + share->s_access = new_access; + share->s_mode = new_mode; +} + /** * nlmsvc_share_file - create a share * @host: Network client peer @@ -64,12 +83,13 @@ nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file, share->s_host = host; share->s_owner.data = ohdata; share->s_owner.len = oh->len; + share->s_access_deny_bmap = 0; share->s_next = file->f_shares; file->f_shares = share; update: - share->s_access = access; - share->s_mode = mode; + share->s_access_deny_bmap |= LOCKD_FSH_BIT(access, mode); + nlm_recompute_share(share); return nlm_granted; } @@ -78,12 +98,14 @@ update: * @host: Network client peer * @file: File to be unshared * @oh: Share owner handle + * @access: Access mode of the SHARE being released + * @mode: Deny mode of the SHARE being released * * Returns an NLM status code. */ __be32 nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, - struct xdr_netobj *oh) + struct xdr_netobj *oh, u32 access, u32 mode) { struct lockd_share *share, **shpp; @@ -93,8 +115,12 @@ nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, for (shpp = &file->f_shares; (share = *shpp) != NULL; shpp = &share->s_next) { if (share->s_host == host && nlm_cmp_owner(share, oh)) { - *shpp = share->s_next; - kfree(share); + share->s_access_deny_bmap &= ~LOCKD_FSH_BIT(access, mode); + nlm_recompute_share(share); + if (!share->s_access_deny_bmap) { + *shpp = share->s_next; + kfree(share); + } return nlm_granted; } } diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c index a0d1a6fbf61e..ffab76278bb2 100644 --- a/fs/lockd/svcsubs.c +++ b/fs/lockd/svcsubs.c @@ -90,22 +90,35 @@ int lock_to_openmode(struct file_lock *lock) static __be32 nlm_do_fopen(struct svc_rqst *rqstp, struct nlm_file *file, int mode) { + const struct nlmsvc_binding *ops; __be32 nlmerr = nlm__int__failed; __be32 deferred = 0; int error; int m; + rcu_read_lock(); + ops = rcu_dereference(nlmsvc_ops); + if (!ops || !try_module_get(ops->owner)) { + rcu_read_unlock(); + return nlm__int__failed; + } + rcu_read_unlock(); + for (m = O_RDONLY; m <= O_WRONLY; m++) { struct file **fp = &file->f_file[m]; if (mode != O_RDWR && mode != m) continue; - if (*fp) + if (*fp) { + module_put(ops->owner); return nlm_granted; + } - error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, m); - if (!error) + error = ops->fopen(rqstp, &file->f_handle, fp, m); + if (!error) { + module_put(ops->owner); return nlm_granted; + } dprintk("lockd: open failed (errno %d)\n", error); switch (error) { @@ -122,6 +135,7 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp, } } + module_put(ops->owner); return deferred ? deferred : nlmerr; } @@ -186,6 +200,33 @@ out_free: } /* + * Release the struct file references held by a nlm_file. + */ +static void nlm_release_files(struct nlm_file *file) +{ + const struct nlmsvc_binding *ops; + bool have_ops; + + rcu_read_lock(); + ops = rcu_dereference(nlmsvc_ops); + have_ops = ops && try_module_get(ops->owner); + rcu_read_unlock(); + + if (have_ops) { + if (file->f_file[O_RDONLY]) + ops->fclose(file->f_file[O_RDONLY]); + if (file->f_file[O_WRONLY]) + ops->fclose(file->f_file[O_WRONLY]); + module_put(ops->owner); + } else { + if (file->f_file[O_RDONLY]) + fput(file->f_file[O_RDONLY]); + if (file->f_file[O_WRONLY]) + fput(file->f_file[O_WRONLY]); + } +} + +/* * Delete a file after having released all locks, blocks and shares */ static inline void @@ -194,10 +235,7 @@ nlm_delete_file(struct nlm_file *file) nlm_debug_print_file("closing file", file); if (!hlist_unhashed(&file->f_list)) { hlist_del(&file->f_list); - if (file->f_file[O_RDONLY]) - nlmsvc_ops->fclose(file->f_file[O_RDONLY]); - if (file->f_file[O_WRONLY]) - nlmsvc_ops->fclose(file->f_file[O_WRONLY]); + nlm_release_files(file); kfree(file); } else { printk(KERN_WARNING "lockd: attempt to release unknown file!\n"); @@ -312,12 +350,10 @@ nlm_file_inuse(struct nlm_file *file) return 0; } -static void nlm_close_files(struct nlm_file *file) +static void nlm_file_release(struct nlm_file *file) { - if (file->f_file[O_RDONLY]) - nlmsvc_ops->fclose(file->f_file[O_RDONLY]); - if (file->f_file[O_WRONLY]) - nlmsvc_ops->fclose(file->f_file[O_WRONLY]); + if (!nlm_file_inuse(file)) + nlm_delete_file(file); } /* @@ -327,32 +363,41 @@ static int nlm_traverse_files(void *data, nlm_host_match_fn_t match, int (*is_failover_file)(void *data, struct nlm_file *file)) { - struct hlist_node *next; - struct nlm_file *file; + struct nlm_file *file, *next; int i, ret = 0; mutex_lock(&nlm_file_mutex); for (i = 0; i < FILE_NRHASH; i++) { - hlist_for_each_entry_safe(file, next, &nlm_files[i], f_list) { - if (is_failover_file && !is_failover_file(data, file)) - continue; + file = hlist_entry_safe(nlm_files[i].first, + struct nlm_file, f_list); + if (file) file->f_count++; - mutex_unlock(&nlm_file_mutex); - - /* Traverse locks, blocks and shares of this file - * and update file->f_locks count */ - if (nlm_inspect_file(data, file, match)) - ret = 1; + while (file) { + /* + * Pin the next neighbour before we drop the mutex + * for nlm_inspect_file(); a concurrent + * nlm_release_file() under the same mutex would + * otherwise be free to unlink and kfree it during + * the unlock window, leaving us to dereference a + * freed slab when we walked to next afterwards. + */ + next = hlist_entry_safe(file->f_list.next, + struct nlm_file, f_list); + if (next) + next->f_count++; + + if (!is_failover_file || is_failover_file(data, file)) { + mutex_unlock(&nlm_file_mutex); + + if (nlm_inspect_file(data, file, match)) + ret = 1; + + mutex_lock(&nlm_file_mutex); + } - mutex_lock(&nlm_file_mutex); file->f_count--; - /* No more references to this file. Let go of it. */ - if (list_empty(&file->f_blocks) && !file->f_locks - && !file->f_shares && !file->f_count) { - hlist_del(&file->f_list); - nlm_close_files(file); - kfree(file); - } + nlm_file_release(file); + file = next; } } mutex_unlock(&nlm_file_mutex); @@ -512,7 +557,7 @@ EXPORT_SYMBOL_GPL(nlmsvc_unlock_all_by_sb); static int nlmsvc_match_ip(void *datap, struct nlm_host *host) { - return rpc_cmp_addr(nlm_srcaddr(host), datap); + return rpc_cmp_addr(nlm_srcaddr(datap), (struct sockaddr *)host); } /** diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index ff4e9fd38e83..bc282b744f34 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -231,8 +231,9 @@ int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt) cb_info->users++; err_net: if (!cb_info->users) { + xprt_svc_shutdown_bc(xprt); svc_set_num_threads(cb_info->serv, 0, 0); - svc_destroy(&cb_info->serv); + xprt_svc_destroy_nullify_bc(xprt, &cb_info->serv); } err_create: mutex_unlock(&nfs_callback_mutex); @@ -254,6 +255,7 @@ void nfs_callback_down(int minorversion, struct net *net, struct rpc_xprt *xprt) mutex_lock(&nfs_callback_mutex); serv = cb_info->serv; + xprt_svc_shutdown_bc(xprt); nfs_callback_down_net(minorversion, serv, net); cb_info->users--; if (cb_info->users == 0) { diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c index 4382baddc9ee..eec6040556c9 100644 --- a/fs/nfs/callback_xdr.c +++ b/fs/nfs/callback_xdr.c @@ -1090,26 +1090,20 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { } }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfs4_callback_count1[ARRAY_SIZE(nfs4_callback_procedures1)]); const struct svc_version nfs4_callback_version1 = { .vs_vers = 1, .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1), .vs_proc = nfs4_callback_procedures1, - .vs_count = nfs4_callback_count1, .vs_xdrsize = NFS4_CALLBACK_XDRSIZE, .vs_dispatch = nfs_callback_dispatch, .vs_hidden = true, .vs_need_cong_ctrl = true, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfs4_callback_count4[ARRAY_SIZE(nfs4_callback_procedures1)]); const struct svc_version nfs4_callback_version4 = { .vs_vers = 4, .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1), .vs_proc = nfs4_callback_procedures1, - .vs_count = nfs4_callback_count4, .vs_xdrsize = NFS4_CALLBACK_XDRSIZE, .vs_dispatch = nfs_callback_dispatch, .vs_hidden = true, diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index dd715cdb6c04..85aa03a7b020 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -292,8 +292,22 @@ struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, cred, nfs_fh, pnf, fmode); if (!IS_ERR(localio) && nfs_uuid_add_file(uuid, nfl) < 0) { - /* Delete the cached file when racing with nfs_uuid_put() */ + /* + * Delete the cached file when racing with nfs_uuid_put(). + * Since nfl->nfs_uuid was never published via + * rcu_assign_pointer(), nfs_close_local_fh() will early-return + * and cannot clean up after us. Drop the slot's file ref and + * its paired net ref, then drop the caller-owned nfsd_file ref + * (+1) and the entry-time nfsd_net ref carried via nf->nf_net, + * and return -ENXIO so the caller never dereferences the + * now-cleared localio. + */ + struct nfsd_file __rcu *tmp = + (struct nfsd_file __force __rcu *)localio; + nfs_to_nfsd_file_put_local(pnf); + nfs_to_nfsd_file_put_local(&tmp); + localio = ERR_PTR(-ENXIO); } nfs_to_nfsd_net_put(net); diff --git a/fs/nfsd/attr4.h b/fs/nfsd/attr4.h new file mode 100644 index 000000000000..f0b51f8050b7 --- /dev/null +++ b/fs/nfsd/attr4.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NFSv4 file attributes supported by this implementation + */ + +#ifndef _LINUX_NFSD_ATTR4_H +#define _LINUX_NFSD_ATTR4_H + +#include <linux/types.h> +#include <linux/nfs4.h> + +/* + * The following attributes are not implemented by NFSD: + * ARCHIVE (deprecated anyway) + * HIDDEN (unlikely to be supported any time soon) + * MIMETYPE (unlikely to be supported any time soon) + * QUOTA_* (unlikely to be supported any time soon) + * SYSTEM (unlikely to be supported any time soon) + * TIME_BACKUP (unlikely to be supported any time soon) + */ +#define NFSD4_SUPPORTED_ATTRS_WORD0 \ +(FATTR4_WORD0_SUPPORTED_ATTRS | FATTR4_WORD0_TYPE | FATTR4_WORD0_FH_EXPIRE_TYPE \ + | FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE | FATTR4_WORD0_LINK_SUPPORT \ + | FATTR4_WORD0_SYMLINK_SUPPORT | FATTR4_WORD0_NAMED_ATTR | FATTR4_WORD0_FSID \ + | FATTR4_WORD0_UNIQUE_HANDLES | FATTR4_WORD0_LEASE_TIME | FATTR4_WORD0_RDATTR_ERROR \ + | FATTR4_WORD0_ACLSUPPORT | FATTR4_WORD0_CANSETTIME | FATTR4_WORD0_CASE_INSENSITIVE \ + | FATTR4_WORD0_CASE_PRESERVING | FATTR4_WORD0_CHOWN_RESTRICTED \ + | FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FILEID | FATTR4_WORD0_FILES_AVAIL \ + | FATTR4_WORD0_FILES_FREE | FATTR4_WORD0_FILES_TOTAL | FATTR4_WORD0_FS_LOCATIONS | FATTR4_WORD0_HOMOGENEOUS \ + | FATTR4_WORD0_MAXFILESIZE | FATTR4_WORD0_MAXLINK | FATTR4_WORD0_MAXNAME \ + | FATTR4_WORD0_MAXREAD | FATTR4_WORD0_MAXWRITE | FATTR4_WORD0_ACL) + +#define NFSD4_SUPPORTED_ATTRS_WORD1 \ +(FATTR4_WORD1_MODE | FATTR4_WORD1_NO_TRUNC | FATTR4_WORD1_NUMLINKS \ + | FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP | FATTR4_WORD1_RAWDEV \ + | FATTR4_WORD1_SPACE_AVAIL | FATTR4_WORD1_SPACE_FREE | FATTR4_WORD1_SPACE_TOTAL \ + | FATTR4_WORD1_SPACE_USED | FATTR4_WORD1_TIME_ACCESS | FATTR4_WORD1_TIME_ACCESS_SET \ + | FATTR4_WORD1_TIME_DELTA | FATTR4_WORD1_TIME_METADATA | FATTR4_WORD1_TIME_CREATE \ + | FATTR4_WORD1_TIME_MODIFY | FATTR4_WORD1_TIME_MODIFY_SET | FATTR4_WORD1_MOUNTED_ON_FILEID) + +#define NFSD4_SUPPORTED_ATTRS_WORD2 0 + +/* 4.1 */ +#ifdef CONFIG_NFSD_PNFS +#define PNFSD_SUPPORTED_ATTRS_WORD1 FATTR4_WORD1_FS_LAYOUT_TYPES +#define PNFSD_SUPPORTED_ATTRS_WORD2 \ +(FATTR4_WORD2_LAYOUT_BLKSIZE | FATTR4_WORD2_LAYOUT_TYPES) +#else +#define PNFSD_SUPPORTED_ATTRS_WORD1 0 +#define PNFSD_SUPPORTED_ATTRS_WORD2 0 +#endif /* CONFIG_NFSD_PNFS */ + +#define NFSD4_1_SUPPORTED_ATTRS_WORD0 \ + NFSD4_SUPPORTED_ATTRS_WORD0 + +#define NFSD4_1_SUPPORTED_ATTRS_WORD1 \ + (NFSD4_SUPPORTED_ATTRS_WORD1 | PNFSD_SUPPORTED_ATTRS_WORD1) + +#define NFSD4_1_SUPPORTED_ATTRS_WORD2 \ + (NFSD4_SUPPORTED_ATTRS_WORD2 | PNFSD_SUPPORTED_ATTRS_WORD2 | \ + FATTR4_WORD2_SUPPATTR_EXCLCREAT) + +/* 4.2 */ +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL +#define NFSD4_2_SECURITY_ATTRS FATTR4_WORD2_SECURITY_LABEL +#else +#define NFSD4_2_SECURITY_ATTRS 0 +#endif + +#ifdef CONFIG_NFSD_V4_POSIX_ACLS +#define NFSD4_2_POSIX_ACL_ATTRS \ + (FATTR4_WORD2_ACL_TRUEFORM | \ + FATTR4_WORD2_ACL_TRUEFORM_SCOPE | \ + FATTR4_WORD2_POSIX_DEFAULT_ACL | \ + FATTR4_WORD2_POSIX_ACCESS_ACL) +#else +#define NFSD4_2_POSIX_ACL_ATTRS 0 +#endif + +#define NFSD4_2_SUPPORTED_ATTRS_WORD2 \ + (NFSD4_1_SUPPORTED_ATTRS_WORD2 | \ + FATTR4_WORD2_MODE_UMASK | \ + FATTR4_WORD2_CLONE_BLKSIZE | \ + NFSD4_2_SECURITY_ATTRS | \ + FATTR4_WORD2_XATTR_SUPPORT | \ + FATTR4_WORD2_TIME_DELEG_ACCESS | \ + FATTR4_WORD2_TIME_DELEG_MODIFY | \ + FATTR4_WORD2_OPEN_ARGUMENTS | \ + NFSD4_2_POSIX_ACL_ATTRS) + +/* These will return ERR_INVAL if specified in GETATTR or READDIR. */ +#define NFSD_WRITEONLY_ATTRS_WORD1 \ + (FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET) + +/* + * These are the only attrs allowed in CREATE/OPEN/SETATTR. Don't add + * a writeable attribute here without also adding code to parse it to + * nfsd4_decode_fattr4(). + */ +#define NFSD_WRITEABLE_ATTRS_WORD0 \ + (FATTR4_WORD0_SIZE | FATTR4_WORD0_ACL) +#define NFSD_WRITEABLE_ATTRS_WORD1 \ + (FATTR4_WORD1_MODE | FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP \ + | FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_CREATE \ + | FATTR4_WORD1_TIME_MODIFY_SET) +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL +#define MAYBE_FATTR4_WORD2_SECURITY_LABEL \ + FATTR4_WORD2_SECURITY_LABEL +#else +#define MAYBE_FATTR4_WORD2_SECURITY_LABEL 0 +#endif +#ifdef CONFIG_NFSD_V4_POSIX_ACLS +#define MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS \ + FATTR4_WORD2_POSIX_DEFAULT_ACL | FATTR4_WORD2_POSIX_ACCESS_ACL +#else +#define MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS 0 +#endif +#define NFSD_WRITEABLE_ATTRS_WORD2 \ + (FATTR4_WORD2_MODE_UMASK \ + | MAYBE_FATTR4_WORD2_SECURITY_LABEL \ + | FATTR4_WORD2_TIME_DELEG_ACCESS \ + | FATTR4_WORD2_TIME_DELEG_MODIFY \ + | MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS \ + ) + +#define NFSD_SUPPATTR_EXCLCREAT_WORD0 \ + NFSD_WRITEABLE_ATTRS_WORD0 +/* + * we currently store the exclusive create verifier in the v_{a,m}time + * attributes so the client can't set these at create time using EXCLUSIVE4_1 + */ +#define NFSD_SUPPATTR_EXCLCREAT_WORD1 \ + (NFSD_WRITEABLE_ATTRS_WORD1 & \ + ~(FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET)) +/* + * The FATTR4_WORD2_TIME_DELEG attributes are not to be allowed for + * OPEN(create) with EXCLUSIVE4_1. It doesn't make sense to set a + * delegated timestamp on a new file. + * + * This mask includes NFSv4.2-only attributes (e.g., POSIX ACLs). + * Version filtering occurs via nfsd_suppattrs[] before this mask + * is applied, so pre-4.2 clients never see unsupported attributes. + */ +#define NFSD_SUPPATTR_EXCLCREAT_WORD2 \ + (NFSD_WRITEABLE_ATTRS_WORD2 & \ + ~(FATTR4_WORD2_TIME_DELEG_ACCESS | FATTR4_WORD2_TIME_DELEG_MODIFY)) + +extern const u32 nfsd_suppattrs[3][3]; + +static inline bool bmval_is_subset(const u32 *bm1, const u32 *bm2) +{ + return !((bm1[0] & ~bm2[0]) || + (bm1[1] & ~bm2[1]) || + (bm1[2] & ~bm2[2])); +} + +static inline bool nfsd_attrs_supported(u32 minorversion, const u32 *bmval) +{ + return bmval_is_subset(bmval, nfsd_suppattrs[minorversion]); +} + +#endif /* _LINUX_NFSD_ATTR4_H */ diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c index 4dc327e02456..5ef9a0466eda 100644 --- a/fs/nfsd/auth.c +++ b/fs/nfsd/auth.c @@ -3,6 +3,7 @@ #include <linux/sched.h> #include "nfsd.h" +#include "export.h" #include "auth.h" int nfsexp_flags(struct svc_cred *cred, struct svc_export *exp) @@ -87,3 +88,21 @@ oom: return -ENOMEM; } +/** + * nfsd_user_namespace - Get user_namespace in effect for an RPC request + * @rqstp: RPC execution context + * + * xpt_cred is set once at transport creation and never modified. The + * transport itself is reference-counted during request processing, so + * no explicit reference on the namespace is necessary. + * + * Return: the user_namespace from the transport credential, or + * init_user_ns if no credential was set. The returned namespace pointer + * is valid for the duration of the RPC request. + */ +struct user_namespace *nfsd_user_namespace(const struct svc_rqst *rqstp) +{ + const struct cred *cred = rqstp->rq_xprt->xpt_cred; + + return cred ? cred->user_ns : &init_user_ns; +} diff --git a/fs/nfsd/auth.h b/fs/nfsd/auth.h index 8c5031bbbcee..832bc957980d 100644 --- a/fs/nfsd/auth.h +++ b/fs/nfsd/auth.h @@ -8,10 +8,16 @@ #ifndef LINUX_NFSD_AUTH_H #define LINUX_NFSD_AUTH_H +struct user_namespace; +struct svc_export; +struct svc_rqst; + /* * Set the current process's fsuid/fsgid etc to those of the NFS * client user */ int nfsd_setuser(struct svc_cred *cred, struct svc_export *exp); +struct user_namespace *nfsd_user_namespace(const struct svc_rqst *rqstp); + #endif /* LINUX_NFSD_AUTH_H */ diff --git a/fs/nfsd/cache.h b/fs/nfsd/cache.h index bb7addef4a31..3bc4856e34b8 100644 --- a/fs/nfsd/cache.h +++ b/fs/nfsd/cache.h @@ -10,7 +10,8 @@ #define NFSCACHE_H #include <linux/sunrpc/svc.h> -#include "nfsd.h" + +struct nfsd_net; /* * Representation of a reply cache entry. diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 24511c3208db..b9548eb17c77 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -55,6 +55,17 @@ /* We only care about NFSD_MAY_READ/WRITE for this cache */ #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) +/* If the shrinker runs between calls to list_lru_walk_node() in + * nfsd_file_gc(), the "remaining" count will be wrong. This could + * result in premature freeing of some files. This may not matter much + * but is easy to fix with this spinlock which temporarily disables + * the shrinker. + * + * It also serializes callers of nfsd_file_dispose_list_delayed() + * against per-net shutdown. + */ +static DEFINE_SPINLOCK(nfsd_gc_lock); + static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations); @@ -62,16 +73,12 @@ static DEFINE_PER_CPU(unsigned long, nfsd_file_releases); static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age); static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); -struct nfsd_fcache_disposal { - spinlock_t lock; - struct list_head freeme; -}; - static struct kmem_cache *nfsd_file_slab; static struct kmem_cache *nfsd_file_mark_slab; static struct list_lru nfsd_file_lru; static unsigned long nfsd_file_flags; static struct fsnotify_group *nfsd_file_fsnotify_group; +static struct fsnotify_group *nfsd_dir_fsnotify_group; static struct delayed_work nfsd_filecache_laundrette; static struct rhltable nfsd_file_rhltable ____cacheline_aligned_in_smp; @@ -147,7 +154,7 @@ static void nfsd_file_mark_put(struct nfsd_file_mark *nfm) { if (refcount_dec_and_test(&nfm->nfm_ref)) { - fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group); + fsnotify_destroy_mark(&nfm->nfm_mark, nfm->nfm_mark.group); fsnotify_put_mark(&nfm->nfm_mark); } } @@ -155,37 +162,40 @@ nfsd_file_mark_put(struct nfsd_file_mark *nfm) static struct nfsd_file_mark * nfsd_file_mark_find_or_create(struct inode *inode) { - int err; - struct fsnotify_mark *mark; struct nfsd_file_mark *nfm = NULL, *new; + struct fsnotify_group *group; + struct fsnotify_mark *mark; + int err; + + group = S_ISDIR(inode->i_mode) ? nfsd_dir_fsnotify_group : nfsd_file_fsnotify_group; do { - fsnotify_group_lock(nfsd_file_fsnotify_group); - mark = fsnotify_find_inode_mark(inode, - nfsd_file_fsnotify_group); + fsnotify_group_lock(group); + mark = fsnotify_find_inode_mark(inode, group); if (mark) { nfm = nfsd_file_mark_get(container_of(mark, struct nfsd_file_mark, nfm_mark)); - fsnotify_group_unlock(nfsd_file_fsnotify_group); + fsnotify_group_unlock(group); if (nfm) { fsnotify_put_mark(mark); break; } /* Avoid soft lockup race with nfsd_file_mark_put() */ - fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group); + fsnotify_destroy_mark(mark, group); fsnotify_put_mark(mark); } else { - fsnotify_group_unlock(nfsd_file_fsnotify_group); + fsnotify_group_unlock(group); } /* allocate a new nfm */ new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL); if (!new) return NULL; - fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group); + fsnotify_init_mark(&new->nfm_mark, group); new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF; refcount_set(&new->nfm_ref, 1); + mutex_init(&new->nfm_recalc_mutex); err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0); @@ -327,8 +337,11 @@ static void nfsd_file_lru_add(struct nfsd_file *nf) refcount_inc(&nf->nf_ref); if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) trace_nfsd_file_lru_add(nf); - else - WARN_ON(1); + else { + refcount_dec(&nf->nf_ref); + WARN_ON_ONCE(1); + return; + } nfsd_file_schedule_laundrette(); } @@ -419,25 +432,31 @@ nfsd_file_dispose_list(struct list_head *dispose) } /** - * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list + * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal * @dispose: list of nfsd_files to be disposed * - * Transfers each file to the "freeme" list for its nfsd_net, to eventually - * be disposed of by the per-net garbage collector. + * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd + * thread to do the actual close. This keeps the cost of fput() in the nfsd + * threads rather than in the shrinker or GC worker. + * + * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net() + * can synchronize against them before draining the per-net dispose list. + * This guarantees nf_net is still live when we call net_generic(). */ static void nfsd_file_dispose_list_delayed(struct list_head *dispose) { - while(!list_empty(dispose)) { + lockdep_assert_held(&nfsd_gc_lock); + + while (!list_empty(dispose)) { struct nfsd_file *nf = list_first_entry(dispose, struct nfsd_file, nf_gc); struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); - struct nfsd_fcache_disposal *l = nn->fcache_disposal; struct svc_serv *serv; - spin_lock(&l->lock); - list_move_tail(&nf->nf_gc, &l->freeme); - spin_unlock(&l->lock); + spin_lock(&nn->fcache_dispose_lock); + list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list); + spin_unlock(&nn->fcache_dispose_lock); /* * The filecache laundrette is shut down after the @@ -461,21 +480,28 @@ nfsd_file_dispose_list_delayed(struct list_head *dispose) */ void nfsd_file_net_dispose(struct nfsd_net *nn) { - struct nfsd_fcache_disposal *l = nn->fcache_disposal; - - if (!list_empty(&l->freeme)) { + if (!list_empty(&nn->fcache_dispose_list)) { LIST_HEAD(dispose); int i; - spin_lock(&l->lock); - for (i = 0; i < 8 && !list_empty(&l->freeme); i++) - list_move(l->freeme.next, &dispose); - spin_unlock(&l->lock); - if (!list_empty(&l->freeme)) - /* Wake up another thread to share the work + spin_lock(&nn->fcache_dispose_lock); + for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++) + list_move(nn->fcache_dispose_list.next, &dispose); + spin_unlock(&nn->fcache_dispose_lock); + if (!list_empty(&nn->fcache_dispose_list)) { + /* + * Wake up another thread to share the work * *before* doing any actual disposing. + * + * The filecache laundrette is shut down after + * the nn->nfsd_serv pointer is cleared, but + * before the svc_serv is freed. */ - svc_wake_up(nn->nfsd_serv); + struct svc_serv *serv = nn->nfsd_serv; + + if (serv) + svc_wake_up(serv); + } nfsd_file_dispose_list(&dispose); } } @@ -552,13 +578,6 @@ nfsd_file_gc_cb(struct list_head *item, struct list_lru_one *lru, return nfsd_file_lru_cb(item, lru, arg); } -/* If the shrinker runs between calls to list_lru_walk_node() in - * nfsd_file_gc(), the "remaining" count will be wrong. This could - * result in premature freeing of some files. This may not matter much - * but is easy to fix with this spinlock which temporarily disables - * the shrinker. - */ -static DEFINE_SPINLOCK(nfsd_gc_lock); static void nfsd_file_gc(void) { @@ -581,9 +600,9 @@ nfsd_file_gc(void) remaining = 0; } } + nfsd_file_dispose_list_delayed(&dispose); spin_unlock(&nfsd_gc_lock); trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru)); - nfsd_file_dispose_list_delayed(&dispose); } static void @@ -611,9 +630,9 @@ nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc) ret = list_lru_shrink_walk(&nfsd_file_lru, sc, nfsd_file_lru_cb, &dispose); + nfsd_file_dispose_list_delayed(&dispose); spin_unlock(&nfsd_gc_lock); trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru)); - nfsd_file_dispose_list_delayed(&dispose); return ret; } @@ -686,11 +705,11 @@ nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose) } /** - * nfsd_file_close_inode - attempt a delayed close of a nfsd_file + * nfsd_file_close_inode - attempt a deferred close of a nfsd_file * @inode: inode of the file to attempt to remove * * Close out any open nfsd_files that can be reaped for @inode. The - * actual freeing is deferred to the dispose_list_delayed infrastructure. + * actual freeing is deferred to the nfsd service threads. * * This is used by the fsnotify callbacks and setlease notifier. */ @@ -699,8 +718,10 @@ nfsd_file_close_inode(struct inode *inode) { LIST_HEAD(dispose); + spin_lock(&nfsd_gc_lock); nfsd_file_queue_for_close(inode, &dispose); nfsd_file_dispose_list_delayed(&dispose); + spin_unlock(&nfsd_gc_lock); } /** @@ -812,12 +833,36 @@ nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask, return 0; } +#ifdef CONFIG_NFSD_V4 +static int +nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask, + const void *data, int data_type, struct inode *dir, + const struct qstr *name, u32 cookie, + struct fsnotify_iter_info *iter_info) +{ + return nfsd_handle_dir_event(mask, dir, data, data_type, name); +} +#else +static int +nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask, + const void *data, int data_type, struct inode *dir, + const struct qstr *name, u32 cookie, + struct fsnotify_iter_info *iter_info) +{ + return 0; +} +#endif static const struct fsnotify_ops nfsd_file_fsnotify_ops = { .handle_inode_event = nfsd_file_fsnotify_handle_event, .free_mark = nfsd_file_mark_free, }; +static const struct fsnotify_ops nfsd_dir_fsnotify_ops = { + .handle_event = nfsd_dir_fsnotify_handle_event, + .free_mark = nfsd_file_mark_free, +}; + int nfsd_file_cache_init(void) { @@ -869,8 +914,7 @@ nfsd_file_cache_init(void) goto out_shrinker; } - nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops, - 0); + nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops, 0); if (IS_ERR(nfsd_file_fsnotify_group)) { pr_err("nfsd: unable to create fsnotify group: %ld\n", PTR_ERR(nfsd_file_fsnotify_group)); @@ -879,11 +923,23 @@ nfsd_file_cache_init(void) goto out_notifier; } + nfsd_dir_fsnotify_group = fsnotify_alloc_group(&nfsd_dir_fsnotify_ops, 0); + if (IS_ERR(nfsd_dir_fsnotify_group)) { + pr_err("nfsd: unable to create fsnotify group: %ld\n", + PTR_ERR(nfsd_dir_fsnotify_group)); + ret = PTR_ERR(nfsd_dir_fsnotify_group); + nfsd_dir_fsnotify_group = NULL; + goto out_notify_group; + } + INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker); out: if (ret) clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags); return ret; +out_notify_group: + fsnotify_put_group(nfsd_file_fsnotify_group); + nfsd_file_fsnotify_group = NULL; out_notifier: lease_unregister_notifier(&nfsd_file_lease_notifier); out_shrinker: @@ -940,42 +996,14 @@ __nfsd_file_cache_purge(struct net *net) nfsd_file_dispose_list(&dispose); } -static struct nfsd_fcache_disposal * -nfsd_alloc_fcache_disposal(void) -{ - struct nfsd_fcache_disposal *l; - - l = kmalloc_obj(*l); - if (!l) - return NULL; - spin_lock_init(&l->lock); - INIT_LIST_HEAD(&l->freeme); - return l; -} - -static void -nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l) -{ - nfsd_file_dispose_list(&l->freeme); - kfree(l); -} - -static void -nfsd_free_fcache_disposal_net(struct net *net) -{ - struct nfsd_net *nn = net_generic(net, nfsd_net_id); - struct nfsd_fcache_disposal *l = nn->fcache_disposal; - - nfsd_free_fcache_disposal(l); -} - int nfsd_file_cache_start_net(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - nn->fcache_disposal = nfsd_alloc_fcache_disposal(); - return nn->fcache_disposal ? 0 : -ENOMEM; + spin_lock_init(&nn->fcache_dispose_lock); + INIT_LIST_HEAD(&nn->fcache_dispose_list); + return 0; } /** @@ -994,8 +1022,18 @@ nfsd_file_cache_purge(struct net *net) void nfsd_file_cache_shutdown_net(struct net *net) { + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + nfsd_file_cache_purge(net); - nfsd_free_fcache_disposal_net(net); + /* + * Ensure any in-progress shrinker, GC, or fsnotify/lease callback + * (all of which hold nfsd_gc_lock while calling + * nfsd_file_dispose_list_delayed()) has fully completed before + * draining the per-net dispose list. + */ + spin_lock(&nfsd_gc_lock); + spin_unlock(&nfsd_gc_lock); + nfsd_file_dispose_list(&nn->fcache_dispose_list); } void @@ -1019,6 +1057,8 @@ nfsd_file_cache_shutdown(void) rcu_barrier(); fsnotify_put_group(nfsd_file_fsnotify_group); nfsd_file_fsnotify_group = NULL; + fsnotify_put_group(nfsd_dir_fsnotify_group); + nfsd_dir_fsnotify_group = NULL; kmem_cache_destroy(nfsd_file_slab); nfsd_file_slab = NULL; fsnotify_wait_marks_destroyed(); @@ -1223,11 +1263,9 @@ out: open_file: trace_nfsd_file_alloc(nf); - if (type == S_IFREG) - nf->nf_mark = nfsd_file_mark_find_or_create(inode); - - if (type != S_IFREG || nf->nf_mark) { - if (file) { + nf->nf_mark = nfsd_file_mark_find_or_create(inode); + if (nf->nf_mark) { + if (file && (file->f_mode & FMODE_OPENED)) { get_file(file); nf->nf_file = file; status = nfs_ok; @@ -1374,12 +1412,12 @@ nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, * @rqstp: the RPC transaction being executed * @fhp: the NFS filehandle of the file just created * @may_flags: NFSD_MAY_ settings for the file - * @file: cached, already-open file (may be NULL) + * @file: cached, already-open file (may be NULL or not yet opened) * @pnf: OUT: new or found "struct nfsd_file" object * * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist, - * and @file is non-NULL, use it to instantiate a new nfsd_file instead of - * opening a new one. + * and @file has FMODE_OPENED set, use it to instantiate a new nfsd_file + * instead of opening a new one. * * Return values: * %nfs_ok - @pnf points to an nfsd_file with its reference @@ -1474,3 +1512,54 @@ int nfsd_file_cache_stats_show(struct seq_file *m, void *v) seq_printf(m, "mean age (ms): -\n"); return 0; } + +/** + * nfsd_fsnotify_recalc_mask - recalculate the fsnotify mask for a nfsd_file + * @nf: nfsd_file to recalculate the mask on + * + * When a directory nfsd_file has a delegation added or removed, that may + * change the events that nfsd requires from the VFS layer. This function + * recalculates the fsnotify mask based on the leases present. + */ +void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf) +{ + struct inode *inode = file_inode(nf->nf_file); + u32 lease_mask, set = 0, clear = 0; + struct fsnotify_mark *mark; + + /* This is only needed when adding or removing dir delegs */ + if (!S_ISDIR(inode->i_mode) || !nf->nf_mark) + return; + + mark = &nf->nf_mark->nfm_mark; + + /* + * The mark is shared by every nfsd_file on this inode, so concurrent + * delegation add/remove on the same directory can recalc it in + * parallel. Serialize the read of the lease state and the update of + * the mark so that a recalc working from a stale snapshot of the + * lease list can't clobber a concurrent recalc's update. + */ + mutex_lock(&nf->nf_mark->nfm_recalc_mutex); + + /* Set up notifications for any ignored delegation events */ + lease_mask = inode_lease_ignore_mask(inode); + + if (lease_mask & FL_IGN_DIR_CREATE) + set |= FS_CREATE | FS_MOVED_TO; + else + clear |= FS_CREATE | FS_MOVED_TO; + + if (lease_mask & FL_IGN_DIR_DELETE) + set |= FS_DELETE | FS_MOVED_FROM; + else + clear |= FS_DELETE | FS_MOVED_FROM; + + if (lease_mask & FL_IGN_DIR_RENAME) + set |= FS_RENAME; + else + clear |= FS_RENAME; + + fsnotify_modify_mark_mask(mark, set, clear); + mutex_unlock(&nf->nf_mark->nfm_recalc_mutex); +} diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h index 683b6437cacc..b224902b438d 100644 --- a/fs/nfsd/filecache.h +++ b/fs/nfsd/filecache.h @@ -26,6 +26,8 @@ struct nfsd_file_mark { struct fsnotify_mark nfm_mark; refcount_t nfm_ref; + /* serializes nfsd_fsnotify_recalc_mask() against itself */ + struct mutex nfm_recalc_mutex; }; /* @@ -86,4 +88,5 @@ __be32 nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, __be32 nfsd_file_acquire_dir(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file **pnf); int nfsd_file_cache_stats_show(struct seq_file *m, void *v); +void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf); #endif /* _FS_NFSD_FILECACHE_H */ diff --git a/fs/nfsd/flexfilelayoutxdr.c b/fs/nfsd/flexfilelayoutxdr.c index f9f7e38cba13..374e52d3064a 100644 --- a/fs/nfsd/flexfilelayoutxdr.c +++ b/fs/nfsd/flexfilelayoutxdr.c @@ -30,19 +30,24 @@ nfsd4_ff_encode_layoutget(struct xdr_stream *xdr, struct ff_idmap uid; struct ff_idmap gid; - fh_len = 4 + fl->fh.size; + fh_len = 4 + xdr_align_size(fl->fh.size); uid.len = sprintf(uid.buf, "%u", from_kuid(&init_user_ns, fl->uid)); gid.len = sprintf(gid.buf, "%u", from_kgid(&init_user_ns, fl->gid)); - /* 8 + len for recording the length, name, and padding */ - ds_len = 20 + sizeof(stateid_opaque_t) + 4 + fh_len + - 8 + uid.len + 8 + gid.len; + /* data server entry: deviceid + efficiency + stateid + fh list + + * user + group + flags + stats_collect_hint + */ + ds_len = 16 + 4 + 4 + sizeof(stateid_opaque_t) + 4 + fh_len + + 4 + xdr_align_size(uid.len) + + 4 + xdr_align_size(gid.len) + + 4 + 4; + /* mirror: ds_count + ds */ mirror_len = 4 + ds_len; - /* The layout segment */ - len = 20 + mirror_len; + /* stripe_unit + mirror_count + mirror */ + len = 12 + mirror_len; p = xdr_reserve_space(xdr, sizeof(__be32) + len); if (!p) @@ -94,7 +99,8 @@ nfsd4_ff_encode_getdeviceinfo(struct xdr_stream *xdr, } /* len + padding for two strings */ - addr_len = 16 + da->netaddr.netid_len + da->netaddr.addr_len; + addr_len = 8 + xdr_align_size(da->netaddr.netid_len) + + xdr_align_size(da->netaddr.addr_len); ver_len = 20; len = 4 + ver_len + 4 + addr_len; diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c index be710d809a3b..c458c01e9478 100644 --- a/fs/nfsd/localio.c +++ b/fs/nfsd/localio.c @@ -97,11 +97,15 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom, } nfsd_file_get(localio); again: + rcu_read_lock(); new = unrcu_pointer(cmpxchg(pnf, NULL, RCU_INITIALIZER(localio))); if (new) { /* Some other thread installed an nfsd_file */ - if (nfsd_file_get(new) == NULL) + if (nfsd_file_get(new) == NULL) { + rcu_read_unlock(); goto again; + } + rcu_read_unlock(); /* * Drop the ref we were going to install (both file and * net) and the one we were going to return (only file). @@ -110,6 +114,8 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom, nfsd_net_put(net); nfsd_file_put(localio); localio = new; + } else { + rcu_read_unlock(); } } else nfsd_net_put(net); @@ -204,14 +210,11 @@ static const struct svc_procedure localio_procedures1[] = { }; #define LOCALIO_NR_PROCEDURES ARRAY_SIZE(localio_procedures1) -static DEFINE_PER_CPU_ALIGNED(unsigned long, - localio_count[LOCALIO_NR_PROCEDURES]); const struct svc_version localio_version1 = { .vs_vers = 1, .vs_nproc = LOCALIO_NR_PROCEDURES, .vs_proc = localio_procedures1, .vs_dispatch = nfsd_dispatch, - .vs_count = localio_count, .vs_xdrsize = XDR_QUADLEN(UUID_SIZE), .vs_hidden = true, }; diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c index 6fe1325815e0..72a5b499839d 100644 --- a/fs/nfsd/lockd.c +++ b/fs/nfsd/lockd.c @@ -92,6 +92,7 @@ nlm_fclose(struct file *filp) } static const struct nlmsvc_binding nfsd_nlm_ops = { + .owner = THIS_MODULE, .fopen = nlm_fopen, /* open file for locking */ .fclose = nlm_fclose, /* close file */ }; @@ -100,11 +101,12 @@ void nfsd_lockd_init(void) { dprintk("nfsd: initializing lockd\n"); - nlmsvc_ops = &nfsd_nlm_ops; + rcu_assign_pointer(nlmsvc_ops, &nfsd_nlm_ops); } void nfsd_lockd_shutdown(void) { - nlmsvc_ops = NULL; + RCU_INIT_POINTER(nlmsvc_ops, NULL); + synchronize_rcu(); } diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c index fbee3676d253..eba8b353f412 100644 --- a/fs/nfsd/netlink.c +++ b/fs/nfsd/netlink.c @@ -225,6 +225,11 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .maxattr = NFSD_A_UNLOCK_EXPORT_PATH, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, + { + .cmd = NFSD_CMD_SERVER_STATS_GET, + .dumpit = nfsd_nl_server_stats_get_dumpit, + .flags = GENL_CMD_CAP_DUMP, + }, }; static const struct genl_multicast_group nfsd_nl_mcgrps[] = { diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h index af41aa0d4a65..027e2953db26 100644 --- a/fs/nfsd/netlink.h +++ b/fs/nfsd/netlink.h @@ -42,6 +42,8 @@ int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_unlock_ip_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_unlock_export_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_server_stats_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); enum { NFSD_NLGRP_NONE, diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 27da1a3edacb..71eebfea020d 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -28,6 +28,16 @@ struct cld_net; struct nfsd_net_cb; struct nfsd4_client_tracking_ops; +enum nfsd_net_flag { + NFSD_NET_GRACE_ENDED, + NFSD_NET_GRACE_END_FORCED, + NFSD_NET_IN_GRACE, + NFSD_NET_SOMEBODY_RECLAIMED, + NFSD_NET_TRACK_RECLAIM_COMPLETES, + NFSD_NET_UP, + NFSD_NET_LOCKD_UP, +}; + enum { /* cache misses due only to checksum comparison failures */ NFSD_STATS_PAYLOAD_MISSES, @@ -43,12 +53,17 @@ enum { NFSD_STATS_FIRST_NFS4_OP, /* count of individual nfsv4 operations */ NFSD_STATS_LAST_NFS4_OP = NFSD_STATS_FIRST_NFS4_OP + LAST_NFS4_OP, #define NFSD_STATS_NFS4_OP(op) (NFSD_STATS_FIRST_NFS4_OP + (op)) - NFSD_STATS_WDELEG_GETATTR, /* count of getattr conflict with wdeleg */ #endif NFSD_STATS_COUNTERS_NUM }; /* + * Per-netns NFSv4 callback (backchannel) per-operation counters, indexed + * directly by RFC 8881 callback opcode (OP_CB_GETATTR..OP_CB_OFFLOAD). + */ +#define NFSD_STATS_CB_OPS_NUM (OP_CB_OFFLOAD + 1) + +/* * Represents a nfsd "container". With respect to nfsv4 state tracking, the * fields of interest are the *_id_hashtbls and the *_name_tree. These track * the nfs4_client objects by either short or long form clientid. @@ -66,9 +81,9 @@ struct nfsd_net { struct cache_detail *nametoid_cache; struct lock_manager nfsd4_manager; - bool grace_ended; - bool grace_end_forced; + unsigned long flags; time64_t boot_time; + time64_t boot_time_bt; /* same instant in CLOCK_BOOTTIME */ struct dentry *nfsd_client_dir; @@ -84,6 +99,7 @@ struct nfsd_net { */ struct list_head *reclaim_str_hashtbl; int reclaim_str_hashtbl_size; + struct rw_semaphore reclaim_str_hashtbl_lock; struct list_head *conf_id_hashtbl; struct rb_root conf_name_tree; struct list_head *unconf_id_hashtbl; @@ -96,12 +112,16 @@ struct nfsd_net { * close_lru holds (open) stateowner queue ordered by nfs4_stateowner.so_time * for last close replay. * - * All of the above fields are protected by the client_mutex. + * reclaim_str_hashtbl[], reclaim_str_hashtbl_size are protected by + * reclaim_str_hashtbl_lock. + * + * All of the remaining fields are protected by the client_lock. */ struct list_head client_lru; struct list_head close_lru; - /* protects del_recall_lru and delegation hash/unhash */ + /* protects del_recall_lru and delegation hash/unhash; + * nests outside client_lock */ spinlock_t deleg_lock ____cacheline_aligned; struct list_head del_recall_lru; @@ -110,26 +130,21 @@ struct nfsd_net { struct delayed_work laundromat_work; - /* client_lock protects the client lru list and session hash table */ + /* client_lock protects the client lru list and session hash + * table; nests inside deleg_lock */ spinlock_t client_lock; /* protects blocked_locks_lru */ spinlock_t blocked_locks_lock; struct file *rec_file; - bool in_grace; const struct nfsd4_client_tracking_ops *client_tracking_ops; time64_t nfsd4_lease; time64_t nfsd4_grace; - bool somebody_reclaimed; - bool track_reclaim_completes; atomic_t nr_reclaim_complete; - bool nfsd_net_up; - bool lockd_up; - seqlock_t writeverf_lock; unsigned char writeverf[8]; @@ -190,6 +205,11 @@ struct nfsd_net { /* Per-netns stats counters */ struct percpu_counter counter[NFSD_STATS_COUNTERS_NUM]; +#ifdef CONFIG_NFSD_V4 + /* Per-netns NFSv4 callback (backchannel) per-operation counters */ + struct percpu_counter cb_counter[NFSD_STATS_CB_OPS_NUM]; +#endif + /* sunrpc svc stats */ struct svc_stat nfsd_svcstats; @@ -209,7 +229,8 @@ struct nfsd_net { /* utsname taken from the process that starts the server */ char nfsd_name[UNX_MAXNODENAME+1]; - struct nfsd_fcache_disposal *fcache_disposal; + spinlock_t fcache_dispose_lock; + struct list_head fcache_dispose_list; siphash_key_t siphash_key; diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 76305b86c1a9..190f5a001900 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -115,14 +115,19 @@ static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp) inode_lock(inode); - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS, - argp->acl_access); - if (error) - goto out_drop_lock; - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT, - argp->acl_default); - if (error) - goto out_drop_lock; + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + if (error) + goto out_drop_lock; + } inode_unlock(inode); @@ -248,22 +253,21 @@ nfsaclsvc_encode_getaclres(struct svc_rqst *rqstp, struct xdr_stream *xdr) if (!svcxdr_encode_stat(xdr, resp->status)) return false; - - if (dentry == NULL || d_really_is_negative(dentry)) - return true; - inode = d_inode(dentry); - - if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) - return false; - if (xdr_stream_encode_u32(xdr, resp->mask) < 0) - return false; - - if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, - resp->mask & NFS_ACL, 0)) - return false; - if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, - resp->mask & NFS_DFACL, NFS_ACL_DEFAULT)) - return false; + switch (resp->status) { + case nfs_ok: + inode = d_inode(dentry); + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + if (xdr_stream_encode_u32(xdr, resp->mask) < 0) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, + resp->mask & NFS_ACL, 0)) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, + resp->mask & NFS_DFACL, NFS_ACL_DEFAULT)) + return false; + break; + } return true; } @@ -384,13 +388,10 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_acl_count2[ARRAY_SIZE(nfsd_acl_procedures2)]); const struct svc_version nfsd_acl_version2 = { .vs_vers = 2, .vs_nproc = ARRAY_SIZE(nfsd_acl_procedures2), .vs_proc = nfsd_acl_procedures2, - .vs_count = nfsd_acl_count2, .vs_dispatch = nfsd_dispatch, .vs_xdrsize = NFS3_SVC_XDRSIZE, }; diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index e87731380be8..6b6b289db636 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -105,12 +105,17 @@ static __be32 nfsd3_proc_setacl(struct svc_rqst *rqstp) inode_lock(inode); - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS, - argp->acl_access); - if (error) - goto out_drop_lock; - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT, - argp->acl_default); + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + } out_drop_lock: inode_unlock(inode); @@ -273,13 +278,10 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_acl_count3[ARRAY_SIZE(nfsd_acl_procedures3)]); const struct svc_version nfsd_acl_version3 = { .vs_vers = 3, .vs_nproc = ARRAY_SIZE(nfsd_acl_procedures3), .vs_proc = nfsd_acl_procedures3, - .vs_count = nfsd_acl_count3, .vs_dispatch = nfsd_dispatch, .vs_xdrsize = NFS3_SVC_XDRSIZE, }; diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index aeda7a802bdf..0904d953d10e 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -29,6 +29,25 @@ static int nfs3_ftypes[] = { S_IFIFO, /* NF3FIFO */ }; +/* + * Reject a client-supplied atime or mtime whose nanoseconds field is out + * of range. Such a value is well-formed on the wire but is not a valid + * timespec64, and storing it verbatim can corrupt on-disk timestamps. + * tv_nsec is a long, so it is cast to unsigned long (the same width) to + * catch both an over-large value and one that became negative when an + * out-of-range u32 wire nseconds was assigned to a 32-bit long. + */ +static bool nfsd3_time_in_range(const struct iattr *iap) +{ + if ((iap->ia_valid & ATTR_ATIME_SET) && + (unsigned long)iap->ia_atime.tv_nsec >= NSEC_PER_SEC) + return false; + if ((iap->ia_valid & ATTR_MTIME_SET) && + (unsigned long)iap->ia_mtime.tv_nsec >= NSEC_PER_SEC) + return false; + return true; +} + static __be32 nfsd3_map_status(__be32 status) { switch (status) { @@ -101,9 +120,14 @@ nfsd3_proc_setattr(struct svc_rqst *rqstp) SVCFH_fmt(&argp->fh)); fh_copy(&resp->fh, &argp->fh); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } if (argp->check_guard) guardtime = &argp->guardtime; resp->status = nfsd_setattr(rqstp, &resp->fh, &attrs, guardtime); +out: resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -265,7 +289,9 @@ nfsd3_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, trace_nfsd_vfs_create(rqstp, fhp, S_IFREG, argp->name, argp->len); - if (isdotent(argp->name, argp->len)) + if (!nfsd3_time_in_range(iap)) + return nfserr_inval; + if (name_is_dot_dotdot(argp->name, argp->len)) return nfserr_exist; if (!(iap->ia_valid & ATTR_MODE)) iap->ia_mode = 0; @@ -400,8 +426,13 @@ nfsd3_proc_mkdir(struct svc_rqst *rqstp) argp->attrs.ia_valid &= ~ATTR_SIZE; fh_copy(&resp->dirfh, &argp->fh); fh_init(&resp->fh, NFS3_FHSIZE); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, S_IFDIR, 0, &resp->fh); +out: resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -415,6 +446,10 @@ nfsd3_proc_symlink(struct svc_rqst *rqstp) .na_iattr = &argp->attrs, }; + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } if (argp->tlen == 0) { resp->status = nfserr_inval; goto out; @@ -471,6 +506,11 @@ nfsd3_proc_mknod(struct svc_rqst *rqstp) goto out; } + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } + type = nfs3_ftypes[argp->ftype]; resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, type, rdev, &resp->fh); @@ -1068,13 +1108,10 @@ static const struct svc_procedure nfsd_procedures3[22] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count3[ARRAY_SIZE(nfsd_procedures3)]); const struct svc_version nfsd_version3 = { .vs_vers = 3, .vs_nproc = ARRAY_SIZE(nfsd_procedures3), .vs_proc = nfsd_procedures3, .vs_dispatch = nfsd_dispatch, - .vs_count = nfsd_count3, .vs_xdrsize = NFS3_SVC_XDRSIZE, }; diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 2ff9a991a8fb..e481804bb120 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -987,7 +987,7 @@ compose_entry_fh(struct nfsd3_readdirres *cd, struct svc_fh *fhp, dparent = cd->fh.fh_dentry; exp = cd->fh.fh_export; - if (isdotent(name, namlen)) { + if (name_is_dot_dotdot(name, namlen)) { if (namlen == 2) { dchild = dget_parent(dparent); /* diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 50827405468d..a901bbe67e03 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -39,6 +39,7 @@ #include "nfsd.h" #include "state.h" #include "netns.h" +#include "stats.h" #include "trace.h" #include "xdr4cb.h" #include "xdr4.h" @@ -108,6 +109,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap, if (!xdrgen_decode_fattr4_time_deleg_access(xdr, &access)) return -EIO; + if (access.nseconds >= NSEC_PER_SEC) + return -EIO; fattr->ncf_cb_atime.tv_sec = access.seconds; fattr->ncf_cb_atime.tv_nsec = access.nseconds; @@ -117,6 +120,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap, if (!xdrgen_decode_fattr4_time_deleg_modify(xdr, &modify)) return -EIO; + if (modify.nseconds >= NSEC_PER_SEC) + return -EIO; fattr->ncf_cb_mtime.tv_sec = modify.seconds; fattr->ncf_cb_mtime.tv_nsec = modify.nseconds; @@ -456,13 +461,20 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, const struct nfsd4_callback *cb, struct nfs4_cb_compound_hdr *hdr) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; struct nfsd4_referring_call_list *rcl; __be32 *p; if (hdr->minorversion == 0) return; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + return; + } + encode_nfs_cb_opnum4(xdr, OP_CB_SEQUENCE); encode_sessionid4(xdr, session); @@ -478,6 +490,7 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, encode_referring_call_list4(xdr, rcl); hdr->nops++; + rcu_read_unlock(); } static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) @@ -529,21 +542,32 @@ static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) static int decode_cb_sequence4resok(struct xdr_stream *xdr, struct nfsd4_callback *cb) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; int status = -ESERVERFAULT; __be32 *p; u32 seqid, slotid, target; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + cb->cb_seq_status = -NFS4ERR_BADSESSION; + return -NFS4ERR_BADSESSION; + } + /* * If the server returns different values for sessionID, slotID or * sequence number, the server is looney tunes. */ p = xdr_inline_decode(xdr, NFS4_MAX_SESSIONID_LEN + 4 + 4 + 4 + 4); - if (unlikely(p == NULL)) + if (unlikely(p == NULL)) { + rcu_read_unlock(); goto out_overflow; + } if (memcmp(p, session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN)) { dprintk("NFS: %s Invalid session id\n", __func__); + rcu_read_unlock(); goto out; } p += XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN); @@ -551,12 +575,14 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, seqid = be32_to_cpup(p++); if (seqid != session->se_cb_seq_nr[cb->cb_held_slot]) { dprintk("NFS: %s Invalid sequence number\n", __func__); + rcu_read_unlock(); goto out; } slotid = be32_to_cpup(p++); if (slotid != cb->cb_held_slot) { dprintk("NFS: %s Invalid slotid\n", __func__); + rcu_read_unlock(); goto out; } @@ -564,6 +590,7 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, target = be32_to_cpup(p++); update_cb_slot_table(session, target); + rcu_read_unlock(); status = 0; out: cb->cb_seq_status = status; @@ -865,6 +892,84 @@ static void encode_stateowner(struct xdr_stream *xdr, struct nfs4_stateowner *so xdr_encode_opaque(p, so->so_owner.data, so->so_owner.len); } +static void nfs4_xdr_enc_cb_notify(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfsd4_callback *cb = data; + struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + struct nfs4_cb_compound_hdr hdr = { + .ident = 0, + .minorversion = cb->cb_clp->cl_minorversion, + }; + struct CB_NOTIFY4args args; + unsigned int start; + + WARN_ON_ONCE(hdr.minorversion == 0); + + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + + /* + * nfsd4_cb_notify_prepare() sized the payload against a single page, + * but did not account for the compound, sequence, stateid, and + * filehandle encoded here. If the variable-length encode overflows the + * backchannel send buffer, roll back to before the operation so that a + * truncated CB_NOTIFY is never placed on the wire. + */ + start = xdr_stream_pos(xdr); + + if (xdr_stream_encode_u32(xdr, OP_CB_NOTIFY) < 0) + goto out_err; + + args.cna_stateid.seqid = dp->dl_stid.sc_stateid.si_generation; + memcpy(&args.cna_stateid.other, &dp->dl_stid.sc_stateid.si_opaque, + ARRAY_SIZE(args.cna_stateid.other)); + args.cna_fh.len = dp->dl_stid.sc_file->fi_fhandle.fh_size; + args.cna_fh.data = dp->dl_stid.sc_file->fi_fhandle.fh_raw; + args.cna_changes.count = ncn->ncn_nf_cnt; + args.cna_changes.element = ncn->ncn_nf; + if (!xdrgen_encode_CB_NOTIFY4args(xdr, &args)) + goto out_err; + + hdr.nops++; + encode_cb_nops(&hdr); + return; + +out_err: + /* + * Drop the CB_NOTIFY op and emit a valid CB_SEQUENCE-only compound so + * the client still advances its slot. Flag the failure so the done + * handler recalls the delegation and the missed notification is not + * silently lost. The flag is written here in the transmit path and read + * in the done handler; the two are serialized phases of the same + * rpc_task, so no additional barrier is needed. + */ + ncn->ncn_encode_err = true; + xdr_truncate_encode(xdr, start); + encode_cb_nops(&hdr); +} + +static int nfs4_xdr_dec_cb_notify(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + + return decode_cb_op_status(xdr, OP_CB_NOTIFY, &cb->cb_status); +} + static void nfs4_xdr_enc_cb_notify_lock(struct rpc_rqst *req, struct xdr_stream *xdr, const void *data) @@ -1026,6 +1131,7 @@ static const struct rpc_procinfo nfs4_cb_procedures[] = { #ifdef CONFIG_NFSD_PNFS PROC(CB_LAYOUT, COMPOUND, cb_layout, cb_layout), #endif + PROC(CB_NOTIFY, COMPOUND, cb_notify, cb_notify), PROC(CB_NOTIFY_LOCK, COMPOUND, cb_notify_lock, cb_notify_lock), PROC(CB_OFFLOAD, COMPOUND, cb_offload, cb_offload), PROC(CB_RECALL_ANY, COMPOUND, cb_recall_any, cb_recall_any), @@ -1150,9 +1256,8 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c } else { if (!conn->cb_xprt || !ses) return -EINVAL; - clp->cl_cb_session = ses; args.bc_xprt = conn->cb_xprt; - args.prognumber = clp->cl_cb_session->se_cb_prog; + args.prognumber = ses->se_cb_prog; args.protocol = conn->cb_xprt->xpt_class->xcl_ident | XPRT_TRANSPORT_BC; args.authflavor = ses->se_cb_sec.flavor; @@ -1170,8 +1275,10 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c return -ENOMEM; } - if (clp->cl_minorversion != 0) + if (clp->cl_minorversion != 0) { clp->cl_cb_conn.cb_xprt = conn->cb_xprt; + rcu_assign_pointer(clp->cl_cb_session, ses); + } clp->cl_cb_client = client; clp->cl_cb_cred = cred; rcu_read_lock(); @@ -1278,18 +1385,33 @@ static int grab_slot(struct nfsd4_session *ses) static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) { struct nfs4_client *clp = cb->cb_clp; - struct nfsd4_session *ses = clp->cl_cb_session; + struct nfsd4_session *ses; if (cb->cb_held_slot >= 0) return true; + + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (!ses) { + rcu_read_unlock(); + rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); + return false; + } cb->cb_held_slot = grab_slot(ses); if (cb->cb_held_slot < 0) { + rcu_read_unlock(); rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); /* Race breaker */ - cb->cb_held_slot = grab_slot(ses); + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) + cb->cb_held_slot = grab_slot(ses); + rcu_read_unlock(); if (cb->cb_held_slot < 0) return false; rpc_wake_up_queued_task(&clp->cl_cb_waitq, task); + } else { + rcu_read_unlock(); } return true; } @@ -1297,12 +1419,17 @@ static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) static void nfsd41_cb_release_slot(struct nfsd4_callback *cb) { struct nfs4_client *clp = cb->cb_clp; - struct nfsd4_session *ses = clp->cl_cb_session; + struct nfsd4_session *ses; if (cb->cb_held_slot >= 0) { - spin_lock(&ses->se_lock); - ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); - spin_unlock(&ses->se_lock); + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) { + spin_lock(&ses->se_lock); + ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); + spin_unlock(&ses->se_lock); + } + rcu_read_unlock(); cb->cb_held_slot = -1; rpc_wake_up_next(&clp->cl_cb_waitq); } @@ -1319,6 +1446,16 @@ static void nfsd41_destroy_cb(struct nfsd4_callback *cb) else clear_bit(NFSD4_CALLBACK_RUNNING, &cb->cb_flags); + /* + * Order the clear of NFSD4_CALLBACK_RUNNING above before the ->release() + * callback below. A release op may re-check producer-side state to decide + * whether to requeue itself (see nfsd4_cb_notify_release()), and that + * check must not be reordered ahead of the clear. The plain clear_bit() + * path carries no ordering; clear_and_wake_up_bit() already issues this + * barrier internally, so the extra one is harmless there. + */ + smp_mb__after_atomic(); + if (cb->cb_ops && cb->cb_ops->release) cb->cb_ops->release(cb); nfsd41_cb_inflight_end(clp); @@ -1434,22 +1571,35 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata) trace_nfsd_cb_rpc_prepare(clp); cb->cb_seq_status = 1; cb->cb_status = 0; - if (minorversion && !nfsd41_cb_get_slot(cb, task)) - return; + if (minorversion) { + if (!rcu_access_pointer(clp->cl_cb_session)) { + rpc_exit(task, -EIO); + return; + } + if (!nfsd41_cb_get_slot(cb, task)) + return; + } rpc_call_start(task); } /* Returns true if CB_COMPOUND processing should continue */ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback *cb) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; bool ret = false; if (cb->cb_held_slot < 0) goto requeue; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + goto requeue; + } + /* This is the operation status code for CB_SEQUENCE */ - trace_nfsd_cb_seq_status(task, cb); + trace_nfsd_cb_seq_status(task, cb, session); switch (cb->cb_seq_status) { case 0: /* @@ -1481,12 +1631,16 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback fallthrough; case -NFS4ERR_BADSESSION: nfsd4_mark_cb_fault(cb->cb_clp); + rcu_read_unlock(); goto requeue; case -NFS4ERR_DELAY: cb->cb_seq_status = 1; - if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) + if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) { + rcu_read_unlock(); goto requeue; + } rpc_delay(task, 2 * HZ); + rcu_read_unlock(); return false; case -NFS4ERR_SEQ_MISORDERED: case -NFS4ERR_BADSLOT: @@ -1498,11 +1652,13 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback */ nfsd4_mark_cb_fault(cb->cb_clp); cb->cb_held_slot = -1; + rcu_read_unlock(); goto retry_nowait; default: nfsd4_mark_cb_fault(cb->cb_clp); } - trace_nfsd_cb_free_slot(task, cb); + trace_nfsd_cb_free_slot(task, cb, session); + rcu_read_unlock(); nfsd41_cb_release_slot(cb); return ret; retry_nowait: @@ -1624,7 +1780,15 @@ static struct nfsd4_conn * __nfsd4_find_backchannel(struct nfs4_client *clp) * Note there isn't a lot of locking in this code; instead we depend on * the fact that it is run from clp->cl_callback_wq, which won't run two * work items at once. So, for example, clp->cl_callback_wq handles all - * access of cl_cb_client and all calls to rpc_create or rpc_shutdown_client. + * access of cl_cb_client, and all calls to rpc_create or + * rpc_shutdown_client. + * + * cl_cb_session is written only from cl_callback_wq (via + * rcu_assign_pointer) and read from rpciod under rcu_read_lock (via + * rcu_dereference) by encode_cb_sequence4args(), decode_cb_sequence4resok(), + * nfsd4_cb_sequence_done(), and the cb-slot helpers. Sessions are freed + * with kfree_rcu() so that rpciod readers in an RCU read-side critical + * section never dereference a freed session. */ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) { @@ -1676,6 +1840,7 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) nfsd4_mark_cb_down(clp); if (c) svc_xprt_put(c->cn_xprt); + rcu_assign_pointer(clp->cl_cb_session, ses); return; } } @@ -1715,7 +1880,10 @@ nfsd4_run_cb_work(struct work_struct *work) if (!test_and_clear_bit(NFSD4_CALLBACK_REQUEUE, &cb->cb_flags)) { if (cb->cb_ops && cb->cb_ops->prepare) - cb->cb_ops->prepare(cb); + if (!cb->cb_ops->prepare(cb)) { + nfsd41_destroy_cb(cb); + return; + } } cb->cb_msg.rpc_cred = clp->cl_cb_cred; @@ -1754,12 +1922,31 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, bool nfsd4_run_cb(struct nfsd4_callback *cb) { struct nfs4_client *clp = cb->cb_clp; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + const struct nfsd4_callback_ops *ops = cb->cb_ops; + u32 minorversion = clp->cl_minorversion; bool queued; nfsd41_cb_inflight_begin(clp); queued = nfsd4_queue_cb(cb); - if (!queued) + if (queued) { + if (ops) { + nfsd_stats_cb_op_inc(nn, ops->opcode); + /* + * Minorversion > 0 callbacks prepend a CB_SEQUENCE op + * (see encode_cb_sequence4args()); count it like the + * forechannel counts SEQUENCE, so it isn't perpetually + * reported as zero. CB_NULL probes (ops == NULL) carry + * no CB_SEQUENCE -- and on 4.1+ they are dropped without + * sending any RPC (see nfsd4_run_cb_work()) -- so they + * must not be counted here. + */ + if (minorversion > 0) + nfsd_stats_cb_op_inc(nn, OP_CB_SEQUENCE); + } + } else { nfsd41_cb_inflight_end(clp); + } return queued; } diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index 71ba61b5d0a3..e9faf8b78f74 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -38,6 +38,7 @@ #include <linux/slab.h> #include <linux/sunrpc/svc_xprt.h> #include <net/net_namespace.h> +#include "auth.h" #include "idmap.h" #include "nfsd.h" #include "netns.h" diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index f34320e4c2f4..22bcb6d09f70 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -247,13 +247,21 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, nfsd4_init_cb(&ls->ls_recall, clp, &nfsd4_cb_layout_ops, NFSPROC4_CLNT_CB_LAYOUT); - if (parent->sc_type == SC_TYPE_DELEG) - ls->ls_file = nfsd_file_get(fp->fi_deleg_file); - else + if (parent->sc_type == SC_TYPE_DELEG) { + rcu_read_lock(); + ls->ls_file = nfsd_file_get(rcu_dereference(fp->fi_deleg_file)); + rcu_read_unlock(); + } else { ls->ls_file = find_any_file(fp); - BUG_ON(!ls->ls_file); + } + + if (!ls->ls_file) { + nfs4_put_stid(stp); + return NULL; + } ls->ls_fenced = false; + ls->ls_fence_inflight = false; ls->ls_fence_delay = 0; INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker); @@ -652,7 +660,7 @@ nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file) } } -static void +static bool nfsd4_cb_layout_prepare(struct nfsd4_callback *cb) { struct nfs4_layout_stateid *ls = @@ -661,6 +669,7 @@ nfsd4_cb_layout_prepare(struct nfsd4_callback *cb) mutex_lock(&ls->ls_mutex); nfs4_inc_and_copy_stateid(&ls->ls_recall_sid, &ls->ls_stid); mutex_unlock(&ls->ls_mutex); + return true; } static int @@ -791,15 +800,6 @@ nfsd4_layout_fence_worker(struct work_struct *work) struct nfs4_client *clp; struct nfsd_net *nn; - /* - * The workqueue clears WORK_STRUCT_PENDING before invoking - * this callback. Re-arm immediately so that - * delayed_work_pending() returns true while the fence - * operation is in progress, preventing - * lm_breaker_timedout() from taking a duplicate reference. - */ - mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0); - spin_lock(&ls->ls_lock); if (list_empty(&ls->ls_layouts)) { spin_unlock(&ls->ls_lock); @@ -809,6 +809,9 @@ dispose: nfsd4_close_layout(ls); ls->ls_fenced = true; + spin_lock(&ls->ls_lock); + ls->ls_fence_inflight = false; + spin_unlock(&ls->ls_lock); nfs4_put_stid(&ls->ls_stid); return; } @@ -894,18 +897,26 @@ nfsd4_layout_lm_breaker_timedout(struct file_lease *fl) if ((!nfsd4_layout_ops[ls->ls_layout_type]->fence_client) || ls->ls_fenced) return true; - if (delayed_work_pending(&ls->ls_fence_work)) - return false; /* * Make sure layout has not been returned yet before - * taking a reference count on the layout stateid. + * taking a reference count on the layout stateid. The + * ls_fence_inflight flag is set together with the sc_count + * increment under ls_lock so that a fence worker invocation + * already in progress (which has cleared WORK_STRUCT_PENDING + * but not yet reached dispose:) cannot be coalesced with a + * fresh schedule that takes an extra unmatched reference. */ spin_lock(&ls->ls_lock); + if (ls->ls_fence_inflight) { + spin_unlock(&ls->ls_lock); + return false; + } if (list_empty(&ls->ls_layouts) || !refcount_inc_not_zero(&ls->ls_stid.sc_count)) { spin_unlock(&ls->ls_lock); return true; } + ls->ls_fence_inflight = true; spin_unlock(&ls->ls_lock); mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0); diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 8561540ab2db..50c07561e31f 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -42,6 +42,7 @@ #include <linux/sunrpc/addr.h> #include <linux/nfs_ssc.h> +#include "attr4.h" #include "idmap.h" #include "cache.h" #include "xdr4.h" @@ -57,7 +58,7 @@ module_param(inter_copy_offload_enable, bool, 0644); MODULE_PARM_DESC(inter_copy_offload_enable, "Enable inter server to server copy offload. Default: false"); -static void cleanup_async_copy(struct nfsd4_copy *copy); +static void cleanup_async_copy(struct nfsd4_async_copy *copy); #ifdef CONFIG_NFSD_V4_2_INTER_SSC static int nfsd4_ssc_umount_timeout = 900000; /* default to 15 mins */ @@ -259,7 +260,7 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, __be32 status; int host_err; - if (isdotent(open->op_fname, open->op_fnamelen)) + if (name_is_dot_dotdot(open->op_fname, open->op_fnamelen)) return nfserr_exist; if (!(iap->ia_valid & ATTR_MODE)) iap->ia_mode = 0; @@ -306,10 +307,6 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, goto out; } - status = fh_compose(resfhp, fhp->fh_export, child, fhp); - if (status != nfs_ok) - goto out; - v_mtime = 0; v_atime = 0; if (nfsd4_create_is_exclusive(open->op_createmode)) { @@ -335,6 +332,10 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, if (status != nfs_ok) goto out; + status = fh_compose(resfhp, fhp->fh_export, child, fhp); + if (status != nfs_ok) + goto out; + switch (open->op_createmode) { case NFS4_CREATE_UNCHECKED: if (!d_is_reg(child)) @@ -385,6 +386,10 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, open->op_created = true; fh_fill_post_attrs(fhp); + status = fh_compose(resfhp, fhp->fh_export, child, fhp); + if (status != nfs_ok) + goto out; + /* A newly created file already has a file size of zero. */ if ((iap->ia_valid & ATTR_SIZE) && (iap->ia_size == 0)) iap->ia_valid &= ~ATTR_SIZE; @@ -663,7 +668,7 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, pr_warn("nfsd4_process_open2 failed to open newly-created file: status=%u\n", be32_to_cpu(status)); if (reclaim && !status) - nn->somebody_reclaimed = true; + set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags); out: if (open->op_filp) { fput(open->op_filp); @@ -677,8 +682,6 @@ out: nfsd4_cleanup_open_state(cstate, open); nfsd4_bump_seqid(cstate, status); out_err: - posix_acl_release(open->op_dpacl); - posix_acl_release(open->op_pacl); return status; } @@ -700,6 +703,13 @@ static __be32 nfsd4_open_omfg(struct svc_rqst *rqstp, struct nfsd4_compound_stat return nfsd4_open(rqstp, cstate, &op->u); } +static void +nfsd4_open_release(union nfsd4_op_u *u) +{ + posix_acl_release(u->open.op_dpacl); + posix_acl_release(u->open.op_pacl); +} + /* * filehandle-manipulating ops. */ @@ -839,6 +849,20 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) goto out_aftermask; + /* Sanitize cr_type to avoid returning ATTRNOTSUPP. */ + switch (create->cr_type) { + case NF4LNK: + case NF4BLK: + case NF4CHR: + case NF4SOCK: + case NF4FIFO: + case NF4DIR: + break; + default: + status = nfserr_badtype; + goto out_aftermask; + } + if (create->cr_acl) { if (attrs.na_dpacl || attrs.na_pacl) { status = nfserr_inval; @@ -846,6 +870,8 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, } status = nfsd4_acl_to_attr(create->cr_type, create->cr_acl, &attrs); + if (status != nfs_ok) + goto out_aftermask; } current->fs->umask = create->cr_umask; switch (create->cr_type) { @@ -1253,7 +1279,7 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (deleg_attrs) { status = nfserr_bad_stateid; - if (st->sc_type & SC_TYPE_DELEG) { + if (st && (st->sc_type & SC_TYPE_DELEG)) { struct nfs4_delegation *dp = delegstateid(st); /* Only for *_ATTRS_DELEG flavors */ @@ -1440,13 +1466,13 @@ out: */ bool nfsd4_has_active_async_copies(struct nfs4_client *clp) { - struct nfsd4_copy *copy; + struct nfsd4_async_copy *copy; bool result = false; spin_lock(&clp->async_lock); list_for_each_entry(copy, &clp->async_copies, copies) { - if (!test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags) && - !test_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags)) { + if (!test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_copy.cp_flags) && + !test_bit(NFSD4_COPY_F_STOPPED, ©->cp_copy.cp_flags)) { result = true; break; } @@ -1462,7 +1488,7 @@ bool nfsd4_has_active_async_copies(struct nfs4_client *clp) void nfsd4_async_copy_reaper(struct nfsd_net *nn) { struct nfs4_client *clp; - struct nfsd4_copy *copy; + struct nfsd4_async_copy *copy; LIST_HEAD(reaplist); spin_lock(&nn->client_lock); @@ -1471,8 +1497,9 @@ void nfsd4_async_copy_reaper(struct nfsd_net *nn) spin_lock(&clp->async_lock); list_for_each_safe(pos, next, &clp->async_copies) { - copy = list_entry(pos, struct nfsd4_copy, copies); - if (test_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags)) { + copy = list_entry(pos, struct nfsd4_async_copy, copies); + if (test_bit(NFSD4_COPY_F_OFFLOAD_DONE, + ©->cp_copy.cp_flags)) { if (!--copy->cp_ttl) { list_del_init(©->copies); list_add(©->copies, &reaplist); @@ -1484,53 +1511,56 @@ void nfsd4_async_copy_reaper(struct nfsd_net *nn) spin_unlock(&nn->client_lock); while (!list_empty(&reaplist)) { - copy = list_first_entry(&reaplist, struct nfsd4_copy, copies); + copy = list_first_entry(&reaplist, struct nfsd4_async_copy, + copies); list_del_init(©->copies); cleanup_async_copy(copy); } } -static void nfs4_put_copy(struct nfsd4_copy *copy) +static void nfs4_put_copy(struct nfsd4_async_copy *copy) { if (!refcount_dec_and_test(©->refcount)) return; - kfree(copy->cp_src); - kfree(copy); + /* + * Drop the copy offload stateid's sole reference: removes it from + * cl_stateids and frees the async_copy via nfsd4_free_async_copy_stid(). + */ + nfs4_put_stid(©->cp_stid); } static void release_copy_files(struct nfsd4_copy *copy); -static void nfsd4_stop_copy(struct nfsd4_copy *copy) +static void nfsd4_stop_copy(struct nfsd4_async_copy *copy) { - trace_nfsd_copy_async_cancel(copy); - if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags)) { - kthread_stop(copy->copy_task); - if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_flags)) - copy->nfserr = nfs_ok; - set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); - } - + trace_nfsd_copy_async_cancel(©->cp_copy); /* - * The copy was removed from async_copies before this function - * was called, so the reaper cannot clean it up. Release files - * here regardless of who won the STOPPED race. If the thread - * set STOPPED, it has finished using the files. If STOPPED - * was set here, kthread_stop() waited for the thread to exit. + * Join the kthread before releasing its resources. The task_struct is + * pinned in nfsd4_copy(), so kthread_stop() is safe even after the + * one-shot kthread has exited. The caller already unlinked the copy, + * so this runs once per copy. */ - release_copy_files(copy); + set_bit(NFSD4_COPY_F_STOPPED, ©->cp_copy.cp_flags); + kthread_stop(copy->copy_task); + if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_copy.cp_flags)) + copy->cp_copy.nfserr = nfs_ok; + set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_copy.cp_flags); + + release_copy_files(©->cp_copy); nfs4_put_copy(copy); } -static struct nfsd4_copy *nfsd4_unhash_copy(struct nfs4_client *clp) +static struct nfsd4_async_copy *nfsd4_unhash_copy(struct nfs4_client *clp) { - struct nfsd4_copy *copy = NULL; + struct nfsd4_async_copy *copy = NULL; spin_lock(&clp->async_lock); if (!list_empty(&clp->async_copies)) { - copy = list_first_entry(&clp->async_copies, struct nfsd4_copy, - copies); + copy = list_first_entry(&clp->async_copies, + struct nfsd4_async_copy, copies); refcount_inc(©->refcount); - copy->cp_clp = NULL; + /* Pairs with smp_load_acquire() in nfsd4_send_cb_offload(). */ + smp_store_release(©->cp_copy.cp_clp, NULL); if (!list_empty(©->copies)) list_del_init(©->copies); } @@ -1540,10 +1570,13 @@ static struct nfsd4_copy *nfsd4_unhash_copy(struct nfs4_client *clp) void nfsd4_shutdown_copy(struct nfs4_client *clp) { - struct nfsd4_copy *copy; + struct nfsd4_async_copy *copy; - while ((copy = nfsd4_unhash_copy(clp)) != NULL) + while ((copy = nfsd4_unhash_copy(clp)) != NULL) { nfsd4_stop_copy(copy); + /* Reaper can't reach the unhashed copy; drop its membership ref. */ + nfs4_put_copy(copy); + } } static bool nfsd4_copy_on_sb(const struct nfsd4_copy *copy, @@ -1562,15 +1595,20 @@ static bool nfsd4_copy_on_sb(const struct nfsd4_copy *copy, * nfsd4_cancel_copy_by_sb - cancel async copy operations on @sb * @net: net namespace containing the copy operations * @sb: targeted superblock + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. */ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - struct nfsd4_copy *copy, *tmp; + struct nfsd4_async_copy *copy, *tmp; struct nfs4_client *clp; unsigned int idhashval; LIST_HEAD(to_cancel); + lockdep_assert_held(&nfsd_mutex); spin_lock(&nn->client_lock); for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) { struct list_head *head = &nn->conf_id_hashtbl[idhashval]; @@ -1579,7 +1617,7 @@ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) spin_lock(&clp->async_lock); list_for_each_entry_safe(copy, tmp, &clp->async_copies, copies) { - if (nfsd4_copy_on_sb(copy, sb)) { + if (nfsd4_copy_on_sb(©->cp_copy, sb)) { refcount_inc(©->refcount); /* * Hold a reference on the client while @@ -1591,9 +1629,9 @@ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) * survive callback flight. */ kref_get(&clp->cl_nfsdfs.cl_ref); - copy->nfserr = nfserr_admin_revoked; + copy->cp_copy.nfserr = nfserr_admin_revoked; set_bit(NFSD4_COPY_F_CB_ERROR, - ©->cp_flags); + ©->cp_copy.cp_flags); list_move(©->copies, &to_cancel); } } @@ -1603,10 +1641,12 @@ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) spin_unlock(&nn->client_lock); list_for_each_entry_safe(copy, tmp, &to_cancel, copies) { - struct nfs4_client *clp = copy->cp_clp; + struct nfs4_client *clp = copy->cp_copy.cp_clp; list_del_init(©->copies); nfsd4_stop_copy(copy); + /* Reaper can't reach the unlinked copy; drop the membership ref here. */ + nfs4_put_copy(copy); nfsd4_put_client(clp); } } @@ -1896,11 +1936,14 @@ static void nfsd4_cb_offload_release(struct nfsd4_callback *cb) { struct nfsd4_cb_offload *cbo = container_of(cb, struct nfsd4_cb_offload, co_cb); - struct nfsd4_copy *copy = - container_of(cbo, struct nfsd4_copy, cp_cb_offload); + struct nfsd4_async_copy *copy = + container_of(cbo, struct nfsd4_async_copy, cp_cb_offload); - set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags); + set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_copy.cp_flags); + /* nfs4_put_copy needs clp; cl_cb_inflight defers free_client. */ nfsd4_put_client(cb->cb_clp); + /* Drop the copy reference taken in nfsd4_send_cb_offload(). */ + nfs4_put_copy(copy); } static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, @@ -1950,6 +1993,7 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, /* See RFC 7862 p.67: */ if (bytes_total == 0) bytes_total = ULLONG_MAX; + since = READ_ONCE(dst->f_wb_err); do { /* Only async copies can be stopped here */ if (kthread_should_stop()) @@ -1965,13 +2009,14 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, } while (bytes_total > 0 && nfsd4_copy_is_async(copy)); /* for a non-zero asynchronous copy do a commit of data */ if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) { - since = READ_ONCE(dst->f_wb_err); end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1; status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0); if (!status) status = filemap_check_wb_err(dst->f_mapping, since); if (!status) set_bit(NFSD4_COPY_F_COMMITTED, ©->cp_flags); + else if (status != -EAGAIN && status != -ESTALE) + nfsd_reset_write_verifier(copy->cp_nn); } return bytes_copied; } @@ -2011,7 +2056,6 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst) if (!nfsd4_ssc_is_inter(src)) dst->nf_src = nfsd_file_get(src->nf_src); - memcpy(&dst->cp_stateid, &src->cp_stateid, sizeof(src->cp_stateid)); memcpy(dst->cp_src, src->cp_src, sizeof(struct nl4_server)); memcpy(&dst->stateid, &src->stateid, sizeof(src->stateid)); memcpy(&dst->c_fh, &src->c_fh, sizeof(src->c_fh)); @@ -2030,43 +2074,45 @@ static void release_copy_files(struct nfsd4_copy *copy) } } -static void cleanup_async_copy(struct nfsd4_copy *copy) +/* + * Called from the reaper and from nfsd4_copy()'s error path; in both + * cases the copy is already unreachable from clp->async_copies. + */ +static void cleanup_async_copy(struct nfsd4_async_copy *copy) { - nfs4_free_copy_state(copy); - release_copy_files(copy); - if (copy->cp_clp) { - spin_lock(©->cp_clp->async_lock); - if (!list_empty(©->copies)) - list_del_init(©->copies); - spin_unlock(©->cp_clp->async_lock); - } + release_copy_files(©->cp_copy); nfs4_put_copy(copy); } -static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) +static void nfsd4_send_cb_offload(struct nfsd4_async_copy *copy) { struct nfsd4_cb_offload *cbo = ©->cp_cb_offload; - struct nfs4_client *clp = copy->cp_clp; + struct nfs4_client *clp; /* - * cp_clp is NULL when called via nfsd4_shutdown_copy() during - * client destruction. Skip the callback; the client is gone. + * Pairs with smp_store_release(&cp_clp) in find_async_copy() and + * nfsd4_unhash_copy(); the set_bit/clear_bit writers are unordered. + * cp_clp is NULL once the copy was canceled; skip the callback, the + * canceling path owns the notification. */ + clp = smp_load_acquire(©->cp_copy.cp_clp); if (!clp) { - set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags); + set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_copy.cp_flags); return; } - memcpy(&cbo->co_res, ©->cp_res, sizeof(copy->cp_res)); - memcpy(&cbo->co_fh, ©->fh, sizeof(copy->fh)); - cbo->co_nfserr = copy->nfserr; + memcpy(&cbo->co_res, ©->cp_copy.cp_res, sizeof(copy->cp_copy.cp_res)); + memcpy(&cbo->co_fh, ©->cp_copy.fh, sizeof(copy->cp_copy.fh)); + cbo->co_nfserr = copy->cp_copy.nfserr; cbo->co_retries = 5; /* - * Hold a reference on the client while the callback is in flight. - * Released in nfsd4_cb_offload_release(). + * Hold the client and the copy across the in-flight callback; co_cb is + * embedded in the copy, so it must outlive the callback. Both are + * dropped in nfsd4_cb_offload_release(). */ kref_get(&clp->cl_nfsdfs.cl_ref); + refcount_inc(©->refcount); nfsd4_init_cb(&cbo->co_cb, clp, &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD); @@ -2074,7 +2120,8 @@ static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) cbo->co_referring_slotid, cbo->co_referring_seqno); trace_nfsd_cb_offload(clp, &cbo->co_res.cb_stateid, - &cbo->co_fh, copy->cp_count, copy->nfserr); + &cbo->co_fh, copy->cp_copy.cp_count, + copy->cp_copy.nfserr); nfsd4_try_run_cb(&cbo->co_cb); } @@ -2087,7 +2134,8 @@ static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) */ static int nfsd4_do_async_copy(void *data) { - struct nfsd4_copy *copy = (struct nfsd4_copy *)data; + struct nfsd4_async_copy *async = data; + struct nfsd4_copy *copy = &async->cp_copy; __be32 nfserr = nfs_ok; trace_nfsd_copy_async(copy); @@ -2118,16 +2166,20 @@ static int nfsd4_do_async_copy(void *data) do_callback: if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_flags)) copy->nfserr = nfserr; - /* The kthread exits forthwith. Ensure that a subsequent - * OFFLOAD_CANCEL won't try to kill it again. */ - set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags); - + /* + * Don't set NFSD4_COPY_F_STOPPED here: it tells a teardown caller it + * may skip kthread_stop(), which would then release nf_dst and the + * client while still in use. Only nfsd4_stop_copy() sets it, after + * joining. + */ set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); trace_nfsd_copy_async_done(copy); atomic_dec(©->cp_nn->pending_async_copies); if (copy->cp_res.wr_bytes_written > 0 && copy->attr_update) nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); - nfsd4_send_cb_offload(copy); + nfsd4_send_cb_offload(async); + /* Drop the kthread's reference (taken in nfsd4_copy()); copy may be freed after this. */ + nfs4_put_copy(async); return 0; } @@ -2136,7 +2188,7 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) { struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); - struct nfsd4_copy *async_copy = NULL; + struct nfsd4_async_copy *async_copy = NULL; struct nfsd4_copy *copy = &u->copy; struct nfsd42_write_res *result; __be32 status; @@ -2153,25 +2205,30 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, } status = nfsd4_setup_inter_ssc(rqstp, cstate, copy); if (status) { - trace_nfsd_copy_done(copy, status); - return nfserr_offload_denied; + status = nfserr_offload_denied; + goto out; } } else { trace_nfsd_copy_intra(copy); status = nfsd4_setup_intra_ssc(rqstp, cstate, copy); - if (status) { - trace_nfsd_copy_done(copy, status); - return status; - } + if (status) + goto out; } memcpy(©->fh, &cstate->current_fh.fh_handle, sizeof(struct knfsd_fh)); if (nfsd4_copy_is_async(copy)) { - async_copy = kzalloc_obj(struct nfsd4_copy); + struct task_struct *task; + + /* + * Allocate the durable async copy. Its offload stateid is a + * first-class nfs4_stid in clp->cl_stateids, returned to the + * client and freed only when the background copy is torn down. + */ + async_copy = nfs4_alloc_copy_stid(cstate->clp); if (!async_copy) goto out_err; - async_copy->cp_nn = nn; + async_copy->cp_copy.cp_nn = nn; INIT_LIST_HEAD(&async_copy->copies); refcount_set(&async_copy->refcount, 1); async_copy->cp_ttl = NFSD_COPY_INITIAL_TTL; @@ -2179,31 +2236,45 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (atomic_inc_return(&nn->pending_async_copies) > (int)rqstp->rq_pool->sp_nrthreads) goto out_dec_async_copy_err; - async_copy->cp_src = kmalloc_obj(*async_copy->cp_src); - if (!async_copy->cp_src) - goto out_dec_async_copy_err; - if (!nfs4_init_copy_state(nn, copy)) + async_copy->cp_copy.cp_src = kmalloc_obj(*async_copy->cp_copy.cp_src); + if (!async_copy->cp_copy.cp_src) goto out_dec_async_copy_err; - memcpy(&result->cb_stateid, ©->cp_stateid.cs_stid, + memcpy(&result->cb_stateid, &async_copy->cp_stid.sc_stateid, sizeof(result->cb_stateid)); - dup_copy_fields(copy, async_copy); + /* + * dup after writing cb_stateid; duplicating first would leave + * the callback stateid zeroed. + */ + dup_copy_fields(copy, &async_copy->cp_copy); if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0) - async_copy->attr_update = true; + async_copy->cp_copy.attr_update = true; memcpy(async_copy->cp_cb_offload.co_referring_sessionid.data, cstate->session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN); async_copy->cp_cb_offload.co_referring_slotid = cstate->slot->sl_index; async_copy->cp_cb_offload.co_referring_seqno = cstate->slot->sl_seqid; - async_copy->copy_task = kthread_create(nfsd4_do_async_copy, - async_copy, "%s", "copy thread"); - if (IS_ERR(async_copy->copy_task)) + task = kthread_create(nfsd4_do_async_copy, async_copy, + "%s", "copy thread"); + if (IS_ERR(task)) goto out_dec_async_copy_err; - spin_lock(&async_copy->cp_clp->async_lock); - list_add(&async_copy->copies, - &async_copy->cp_clp->async_copies); - spin_unlock(&async_copy->cp_clp->async_lock); + /* + * Pin the task_struct so kthread_stop() is safe after this + * one-shot kthread exits. Released by nfs4_put_copy(). + */ + get_task_struct(task); + async_copy->copy_task = task; + /* + * Take the kthread's ref and wake it before publishing, so the + * publisher touches async_copy no further and teardown can + * drain it. + */ + refcount_inc(&async_copy->refcount); wake_up_process(async_copy->copy_task); + spin_lock(&async_copy->cp_copy.cp_clp->async_lock); + list_add(&async_copy->copies, + &async_copy->cp_copy.cp_clp->async_copies); + spin_unlock(&async_copy->cp_copy.cp_clp->async_lock); status = nfs_ok; } else { status = nfsd4_do_copy(copy, copy->nf_src->nf_file, @@ -2235,30 +2306,39 @@ out_err: goto out; } -static struct nfsd4_copy * +static struct nfsd4_async_copy * find_async_copy_locked(struct nfs4_client *clp, stateid_t *stateid) { - struct nfsd4_copy *copy; + struct nfsd4_async_copy *copy; lockdep_assert_held(&clp->async_lock); list_for_each_entry(copy, &clp->async_copies, copies) { - if (memcmp(©->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE)) + if (memcmp(©->cp_stid.sc_stateid, stateid, NFS4_STATEID_SIZE)) continue; return copy; } return NULL; } -static struct nfsd4_copy * +static struct nfsd4_async_copy * find_async_copy(struct nfs4_client *clp, stateid_t *stateid) { - struct nfsd4_copy *copy; + struct nfsd4_async_copy *copy; spin_lock(&clp->async_lock); copy = find_async_copy_locked(clp, stateid); - if (copy) + if (copy) { refcount_inc(©->refcount); + /* + * Mirror nfsd4_unhash_copy(): unlink and clear cp_clp under + * async_lock so the reaper can't reach it. Caller drops the + * membership ref after nfsd4_stop_copy(). + */ + smp_store_release(©->cp_copy.cp_clp, NULL); + if (!list_empty(©->copies)) + list_del_init(©->copies); + } spin_unlock(&clp->async_lock); return copy; } @@ -2269,7 +2349,7 @@ nfsd4_offload_cancel(struct svc_rqst *rqstp, union nfsd4_op_u *u) { struct nfsd4_offload_status *os = &u->offload_status; - struct nfsd4_copy *copy; + struct nfsd4_async_copy *copy; struct nfs4_client *clp = cstate->clp; copy = find_async_copy(clp, &os->stateid); @@ -2277,13 +2357,47 @@ nfsd4_offload_cancel(struct svc_rqst *rqstp, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); return manage_cpntf_state(nn, &os->stateid, clp, NULL); - } else + } else { nfsd4_stop_copy(copy); + /* find_async_copy() unlinked it from the reaper; drop the membership ref. */ + nfs4_put_copy(copy); + } return nfs_ok; } static __be32 +nfsd4_set_netaddr(struct sockaddr *addr, struct nfs42_netaddr *netaddr) +{ + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + unsigned int port; + size_t ret_addr, ret_port; + + switch (addr->sa_family) { + case AF_INET: + port = ntohs(sin->sin_port); + sprintf(netaddr->netid, "tcp"); + netaddr->netid_len = 3; + break; + case AF_INET6: + port = ntohs(sin6->sin6_port); + sprintf(netaddr->netid, "tcp6"); + netaddr->netid_len = 4; + break; + default: + return nfserr_inval; + } + ret_addr = rpc_ntop(addr, netaddr->addr, sizeof(netaddr->addr)); + ret_port = snprintf(netaddr->addr + ret_addr, + RPCBIND_MAXUADDRLEN + 1 - ret_addr, + ".%u.%u", port >> 8, port & 0xff); + WARN_ON(ret_port >= RPCBIND_MAXUADDRLEN + 1 - ret_addr); + netaddr->addr_len = ret_addr + ret_port; + return 0; +} + +static __be32 nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) { @@ -2292,7 +2406,6 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct nfs4_stid *stid = NULL; struct nfs4_cpntf_state *cps; - struct nfs4_client *clp = cstate->clp; status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, &cn->cpn_src_stateid, RD_STATE, NULL, @@ -2306,12 +2419,14 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, cn->cpn_lease_time.tv_nsec = 0; status = nfserrno(-ENOMEM); + /* + * The returned cps is published and fully initialized, and carries an + * extra reference for us; drop it once we are done with it. + */ cps = nfs4_alloc_init_cpntf_state(nn, stid); if (!cps) goto out; memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.cs_stid, sizeof(stateid_t)); - memcpy(&cps->cp_p_stateid, &stid->sc_stateid, sizeof(stateid_t)); - memcpy(&cps->cp_p_clid, &clp->cl_clientid, sizeof(clientid_t)); /* For now, only return one server address in cpn_src, the * address used by the client to connect to this server. @@ -2320,10 +2435,11 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, status = nfsd4_set_netaddr((struct sockaddr *)&rqstp->rq_daddr, &cn->cpn_src->u.nl4_addr); WARN_ON_ONCE(status); - if (status) { - nfs4_put_cpntf_state(nn, cps); - goto out; - } + /* + * Drop our extra reference. The membership reference keeps the entry + * alive for a later inter-server READ, or until the laundromat reaps it. + */ + nfs4_put_cpntf_state(nn, cps); out: nfs4_put_stid(stid); return status; @@ -2357,17 +2473,17 @@ nfsd4_offload_status(struct svc_rqst *rqstp, { struct nfsd4_offload_status *os = &u->offload_status; __be32 status = nfs_ok; - struct nfsd4_copy *copy; + struct nfsd4_async_copy *copy; struct nfs4_client *clp = cstate->clp; os->completed = false; spin_lock(&clp->async_lock); copy = find_async_copy_locked(clp, &os->stateid); if (copy) { - os->count = copy->cp_res.wr_bytes_written; - if (test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags)) { + os->count = copy->cp_copy.cp_res.wr_bytes_written; + if (test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_copy.cp_flags)) { os->completed = true; - os->status = copy->nfserr; + os->status = copy->cp_copy.nfserr; } } else status = nfserr_bad_stateid; @@ -2521,12 +2637,19 @@ nfsd4_verify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, return status == nfserr_same ? nfs_ok : status; } +#define SUPPORTED_NOTIFY_MASK (BIT(NOTIFY4_CHANGE_DIR_ATTRS) | \ + BIT(NOTIFY4_REMOVE_ENTRY) | \ + BIT(NOTIFY4_ADD_ENTRY) | \ + BIT(NOTIFY4_RENAME_ENTRY) | \ + BIT(NOTIFY4_GFLAG_EXTEND)) + static __be32 nfsd4_get_dir_delegation(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) { struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation; + u32 requested = gdd->gdda_notification_types[0]; struct nfs4_delegation *dd; struct nfsd_file *nf; __be32 status; @@ -2536,6 +2659,21 @@ nfsd4_get_dir_delegation(struct svc_rqst *rqstp, return status; /* + * Offer no notifications to an order-aware client. RFC8881bis section + * 16.2.13 defines order-aware as NOTIFY4_CFLAG_ORDER being set or + * NOTIFY4_GFLAG_EXTEND being reset. Such a client expects cookie and + * previous-entry information with its notifications (e.g. 27.4.5), and + * nfsd does not track or emit directory offset information. Per + * 16.2.11.3 the alternative would be to recall the delegation, so it's + * simpler to just decline the notifications here. + */ + if (!(requested & BIT(NOTIFY4_GFLAG_EXTEND)) || + (requested & BIT(NOTIFY4_CFLAG_ORDER))) + requested = 0; + + gdd->gddr_notification[0] = requested & SUPPORTED_NOTIFY_MASK; + + /* * RFC 8881, section 18.39.3 says: * * "The server may refuse to grant the delegation. In that case, the @@ -2556,6 +2694,10 @@ nfsd4_get_dir_delegation(struct svc_rqst *rqstp, gdd->gddrnf_status = GDD4_OK; memcpy(&gdd->gddr_stateid, &dd->dl_stid.sc_stateid, sizeof(gdd->gddr_stateid)); + gdd->gddr_child_attributes[0] = dd->dl_child_attrs[0]; + gdd->gddr_child_attributes[1] = dd->dl_child_attrs[1]; + gdd->gddr_dir_attributes[0] = dd->dl_dir_attrs[0]; + gdd->gddr_dir_attributes[1] = dd->dl_dir_attrs[1]; nfs4_put_stid(&dd->dl_stid); return nfs_ok; } @@ -3119,9 +3261,22 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) op->status = nfsd4_open_omfg(rqstp, cstate, op); goto encode_op; } - if (!current_fh->fh_dentry && - !HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) { - if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + if (!current_fh->fh_dentry) { + if (HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) { + /* + * FOREIGN fh from inter-SSC PUTFH: only + * SAVEFH may proceed with a NULL fh_dentry. + * Per RFC 7862 S15.2.3, validation of a + * foreign fh is deferred to the operation + * that consumes it, and NFS4ERR_STALE is + * returned at that point. + */ + if (op->opnum != OP_SAVEFH && + !(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + op->status = nfserr_stale; + goto encode_op; + } + } else if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { op->status = nfserr_nofilehandle; goto encode_op; } @@ -3185,6 +3340,9 @@ encode_op: status = op->status; } + if (op->opdesc && op->opdesc->op_release) + op->opdesc->op_release(&op->u); + trace_nfsd_compound_status(args->client_opcnt, resp->opcnt, status, nfsd4_op_name(op->opnum)); @@ -3506,8 +3664,8 @@ static u32 nfsd4_get_dir_delegation_rsize(const struct svc_rqst *rqstp, op_encode_verifier_maxsz + op_encode_stateid_maxsz + 2 /* gddr_notification */ + - 2 /* gddr_child_attributes */ + - 2 /* gddr_dir_attributes */); + 3 /* gddr_child_attributes */ + + 3 /* gddr_dir_attributes */) * sizeof(__be32); } #ifdef CONFIG_NFSD_PNFS @@ -3684,6 +3842,7 @@ static const struct nfsd4_operation nfsd4_ops[] = { }, [OP_OPEN] = { .op_func = nfsd4_open, + .op_release = nfsd4_open_release, .op_flags = OP_HANDLES_WRONGSEC | OP_MODIFIES_SOMETHING, .op_name = "OP_OPEN", .op_rsize_bop = nfsd4_open_rsize, @@ -4103,13 +4262,10 @@ static const struct svc_procedure nfsd_procedures4[2] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count4[ARRAY_SIZE(nfsd_procedures4)]); const struct svc_version nfsd_version4 = { .vs_vers = 4, .vs_nproc = ARRAY_SIZE(nfsd_procedures4), .vs_proc = nfsd_procedures4, - .vs_count = nfsd_count4, .vs_dispatch = nfsd_dispatch, .vs_xdrsize = NFS4_SVC_XDRSIZE, .vs_rpcb_optnl = true, diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 6ea25a52d2f4..d513971fb119 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -167,7 +167,7 @@ out_end: end_creating(dentry); out: if (status == 0) { - if (nn->in_grace) + if (test_bit(NFSD_NET_IN_GRACE, &nn->flags)) __nfsd4_create_reclaim_record_grace(clp, dname, nn); vfs_fsync(nn->rec_file, 0); } else { @@ -285,10 +285,12 @@ __nfsd4_remove_reclaim_record_grace(const char *dname, int len, return; } name.len = len; + down_write(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(name, nn); - kfree(name.data); if (crp) nfs4_remove_reclaim_record(crp, nn); + up_write(&nn->reclaim_str_hashtbl_lock); + kfree(name.data); } static void @@ -317,7 +319,7 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp) nfs4_reset_creds(original_cred); if (status == 0) { vfs_fsync(nn->rec_file, 0); - if (nn->in_grace) + if (test_bit(NFSD_NET_IN_GRACE, &nn->flags)) __nfsd4_remove_reclaim_record_grace(dname, HEXDIR_LEN, nn); } @@ -373,7 +375,7 @@ nfsd4_recdir_purge_old(struct nfsd_net *nn) { int status; - nn->in_grace = false; + clear_bit(NFSD_NET_IN_GRACE, &nn->flags); if (!nn->rec_file) return; status = mnt_want_write_file(nn->rec_file); @@ -455,7 +457,7 @@ nfsd4_init_recdir(struct net *net) nfs4_reset_creds(original_cred); if (!status) - nn->in_grace = true; + set_bit(NFSD_NET_IN_GRACE, &nn->flags); return status; } @@ -484,6 +486,7 @@ nfs4_legacy_state_init(struct net *net) for (i = 0; i < CLIENT_HASH_SIZE; i++) INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]); nn->reclaim_str_hashtbl_size = 0; + init_rwsem(&nn->reclaim_str_hashtbl_lock); return 0; } @@ -598,13 +601,16 @@ nfsd4_check_legacy_client(struct nfs4_client *clp) goto out_enoent; } name.len = HEXDIR_LEN; + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(name, nn); - kfree(name.data); if (crp) { set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); crp->cr_clp = clp; - return 0; } + up_read(&nn->reclaim_str_hashtbl_lock); + kfree(name.data); + if (crp) + return 0; out_enoent: return -ENOENT; @@ -1176,6 +1182,7 @@ nfsd4_cld_check(struct nfs4_client *clp) return 0; /* look for it in the reclaim hashtable otherwise */ + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(clp->cl_name, nn); if (crp) goto found; @@ -1191,6 +1198,7 @@ nfsd4_cld_check(struct nfs4_client *clp) if (!name.data) { dprintk("%s: failed to allocate memory for name.data!\n", __func__); + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; } name.len = HEXDIR_LEN; @@ -1201,9 +1209,11 @@ nfsd4_cld_check(struct nfs4_client *clp) } #endif + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; found: crp->cr_clp = clp; + up_read(&nn->reclaim_str_hashtbl_lock); return 0; } @@ -1215,6 +1225,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) struct cld_net *cn = nn->cld_net; #endif struct nfs4_client_reclaim *crp; + unsigned int princhashlen; char *principal = NULL; /* did we already find that this client is stable? */ @@ -1222,6 +1233,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) return 0; /* look for it in the reclaim hashtable otherwise */ + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(clp->cl_name, nn); if (crp) goto found; @@ -1237,6 +1249,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) if (!name.data) { dprintk("%s: failed to allocate memory for name.data\n", __func__); + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; } name.len = HEXDIR_LEN; @@ -1247,23 +1260,31 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) } #endif + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; found: - if (crp->cr_princhash.len) { + princhashlen = crp->cr_princhash.len; + if (princhashlen) { u8 digest[SHA256_DIGEST_SIZE]; + u8 *pdata; if (clp->cl_cred.cr_raw_principal) principal = clp->cl_cred.cr_raw_principal; else if (clp->cl_cred.cr_principal) principal = clp->cl_cred.cr_principal; - if (principal == NULL) + if (principal == NULL) { + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; + } sha256(principal, strlen(principal), digest); - if (memcmp(crp->cr_princhash.data, digest, - crp->cr_princhash.len)) + pdata = crp->cr_princhash.data; + if (memcmp(pdata, digest, princhashlen)) { + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; + } } crp->cr_clp = clp; + up_read(&nn->reclaim_str_hashtbl_lock); return 0; } @@ -1362,7 +1383,8 @@ nfs4_cld_state_init(struct net *net) for (i = 0; i < CLIENT_HASH_SIZE; i++) INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]); nn->reclaim_str_hashtbl_size = 0; - nn->track_reclaim_completes = true; + init_rwsem(&nn->reclaim_str_hashtbl_lock); + set_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags); atomic_set(&nn->nr_reclaim_complete, 0); return 0; @@ -1373,7 +1395,7 @@ nfs4_cld_state_shutdown(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - nn->track_reclaim_completes = false; + clear_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags); kfree(nn->reclaim_str_hashtbl); } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index a42f34842d77..18e17232cf94 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -51,10 +51,12 @@ #include "xdr4cb.h" #include "vfs.h" #include "current_stateid.h" +#include "stats.h" #include "netns.h" #include "pnfs.h" #include "filecache.h" +#include "nfs4xdr_gen.h" #include "trace.h" #define NFSDDBG_FACILITY NFSDDBG_PROC @@ -91,6 +93,8 @@ static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_stat static void nfsd4_file_hash_remove(struct nfs4_file *fi); static void deleg_reaper(struct nfsd_net *nn); +static const struct lease_manager_operations nfsd_lease_mng_ops; + /* Locking: */ enum nfsd4_st_mutex_lock_subclass { @@ -118,12 +122,14 @@ static struct kmem_cache *file_slab; static struct kmem_cache *stateid_slab; static struct kmem_cache *deleg_slab; static struct kmem_cache *odstate_slab; +static struct kmem_cache *async_copy_slab; static void free_session(struct nfsd4_session *); static const struct nfsd4_callback_ops nfsd4_cb_recall_ops; static const struct nfsd4_callback_ops nfsd4_cb_notify_lock_ops; static const struct nfsd4_callback_ops nfsd4_cb_getattr_ops; +static const struct nfsd4_callback_ops nfsd4_cb_notify_ops; static struct workqueue_struct *laundry_wq; @@ -202,18 +208,28 @@ renew_client_locked(struct nfs4_client *clp) clp->cl_state = NFSD4_ACTIVE; } +/* + * Finish a cl_rpc_users unpin with the client_lock held. A + * revocation walk clears @renew so the client whose state it is + * revoking is not revived; every other caller renews the lease of + * a still-active client. + */ +static void __put_client_locked(struct nfs4_client *clp, bool renew) +{ + if (is_client_expired(clp)) + wake_up_all(&expiry_wq); + else if (renew) + renew_client_locked(clp); +} + static void put_client_renew_locked(struct nfs4_client *clp) { struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); lockdep_assert_held(&nn->client_lock); - if (!atomic_dec_and_test(&clp->cl_rpc_users)) - return; - if (!is_client_expired(clp)) - renew_client_locked(clp); - else - wake_up_all(&expiry_wq); + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, true); } static void put_client_renew(struct nfs4_client *clp) @@ -222,10 +238,27 @@ static void put_client_renew(struct nfs4_client *clp) if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) return; - if (!is_client_expired(clp)) - renew_client_locked(clp); - else - wake_up_all(&expiry_wq); + __put_client_locked(clp, true); + spin_unlock(&nn->client_lock); +} + +static void put_client_no_renew_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, false); +} + +static void put_client_no_renew(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) + return; + __put_client_locked(clp, false); spin_unlock(&nn->client_lock); } @@ -326,6 +359,16 @@ free_blocked_lock(struct nfsd4_blocked_lock *nbl) kref_put(&nbl->nbl_kref, free_nbl); } +/* A blocked lock's flc_owner is its nfs4_lockowner. */ +static struct nfs4_client * +nbl_client(struct nfsd4_blocked_lock *nbl) +{ + struct nfs4_lockowner *lo; + + lo = (struct nfs4_lockowner *)nbl->nbl_lock.c.flc_owner; + return lo->lo_owner.so_client; +} + static void remove_blocked_locks(struct nfs4_lockowner *lo) { @@ -355,12 +398,13 @@ remove_blocked_locks(struct nfs4_lockowner *lo) } } -static void +static bool nfsd4_cb_notify_lock_prepare(struct nfsd4_callback *cb) { struct nfsd4_blocked_lock *nbl = container_of(cb, struct nfsd4_blocked_lock, nbl_cb); locks_delete_block(&nbl->nbl_lock); + return true; } static int @@ -936,10 +980,11 @@ out_free: } /* - * Create a unique stateid_t to represent each COPY. + * Publish a COPY_NOTIFY stateid in nn->s2s_cp_stateids and link it onto the + * parent's sc_cp_list. That IDR holds only COPY_NOTIFY stateids. */ static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, - unsigned char cs_type) + struct nfs4_stid *p_stid) { int new_id; @@ -949,19 +994,57 @@ static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, idr_preload(GFP_KERNEL); spin_lock(&nn->s2s_cp_lock); new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, stid, 0, 0, GFP_NOWAIT); - stid->cs_stid.si_opaque.so_id = new_id; - stid->cs_stid.si_generation = 1; + if (new_id >= 0) { + struct nfs4_cpntf_state *cps = + container_of(stid, struct nfs4_cpntf_state, cp_stateid); + + stid->cs_stid.si_opaque.so_id = new_id; + stid->cs_stid.si_generation = 1; + /* + * Set cs_type and link onto sc_cp_list under the same lock + * that installed the IDR entry, so a concurrent + * manage_cpntf_state() sees either no entry or a fully + * linked cp_list. + */ + stid->cs_type = NFS4_COPYNOTIFY_STID; + list_add(&cps->cp_list, &p_stid->sc_cp_list); + } spin_unlock(&nn->s2s_cp_lock); idr_preload_end(); if (new_id < 0) return 0; - stid->cs_type = cs_type; return 1; } -int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy) +/* sc_free for a copy offload stateid; runs from nfs4_put_stid(). */ +static void nfsd4_free_async_copy_stid(struct nfs4_stid *stid) { - return nfs4_init_cp_state(nn, ©->cp_stateid, NFS4_COPY_STID); + struct nfsd4_async_copy *copy = + container_of(stid, struct nfsd4_async_copy, cp_stid); + + if (copy->copy_task) + put_task_struct(copy->copy_task); + kfree(copy->cp_copy.cp_src); + kmem_cache_free(async_copy_slab, copy); +} + +/* + * Allocate durable async COPY state. The offload stateid is a first-class + * nfs4_stid (SC_TYPE_COPY) in the client's cl_stateids, so it is per-client + * and uses the common refcounting/teardown. find_stateid_locked() hides it; + * OFFLOAD_CANCEL/OFFLOAD_STATUS find it via clp->async_copies. + */ +struct nfsd4_async_copy *nfs4_alloc_copy_stid(struct nfs4_client *clp) +{ + struct nfs4_stid *stid; + + stid = nfs4_alloc_stid(clp, async_copy_slab, nfsd4_free_async_copy_stid); + if (!stid) + return NULL; + stid->sc_type = SC_TYPE_COPY; + /* RFC 7862 Section 4.8: a copy offload stateid's seqid MUST NOT be 0 */ + stid->sc_stateid.si_generation = 1; + return container_of(stid, struct nfsd4_async_copy, cp_stid); } struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, @@ -972,44 +1055,86 @@ struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, cps = kzalloc_obj(struct nfs4_cpntf_state); if (!cps) return NULL; + /* So a stale list_del_init() before linking is a no-op. */ + INIT_LIST_HEAD(&cps->cp_list); cps->cpntf_time = ktime_get_boottime_seconds(); - refcount_set(&cps->cp_stateid.cs_count, 1); - if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID)) + /* + * Fully initialize the entry before nfs4_init_cp_state() publishes it, + * since a concurrent OFFLOAD_CANCEL could then free it. Take an extra + * reference for the caller (dropped with nfs4_put_cpntf_state()). + */ + memcpy(&cps->cp_p_stateid, &p_stid->sc_stateid, sizeof(stateid_t)); + memcpy(&cps->cp_p_clid, &p_stid->sc_client->cl_clientid, + sizeof(clientid_t)); + refcount_set(&cps->cp_stateid.cs_count, 2); + if (!nfs4_init_cp_state(nn, &cps->cp_stateid, p_stid)) goto out_free; - spin_lock(&nn->s2s_cp_lock); - list_add(&cps->cp_list, &p_stid->sc_cp_list); - spin_unlock(&nn->s2s_cp_lock); return cps; out_free: kfree(cps); return NULL; } -void nfs4_free_copy_state(struct nfsd4_copy *copy) +/* + * Drop the parent's reference on an already-unlinked cpntf entry. If a + * concurrent holder still owns a reference, its nfs4_put_cpntf_state() does + * the final free. + * + * nn->s2s_cp_lock must be held. + */ +static void put_cpntf_state_unlinked_locked(struct nfs4_cpntf_state *cps) +{ + WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); + WARN_ON_ONCE(!list_empty(&cps->cp_list)); + + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) + kfree(cps); +} + +/* + * Unhash from the IDR and sc_cp_list. Gated on list_empty() to avoid + * evicting a recycled so_id. + */ +static void nfsd4_unhash_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) { - struct nfsd_net *nn; + lockdep_assert_held(&nn->s2s_cp_lock); - if (copy->cp_stateid.cs_type != NFS4_COPY_STID) - return; - nn = net_generic(copy->cp_clp->net, nfsd_net_id); - spin_lock(&nn->s2s_cp_lock); - idr_remove(&nn->s2s_cp_stateids, - copy->cp_stateid.cs_stid.si_opaque.so_id); - spin_unlock(&nn->s2s_cp_lock); + if (!list_empty(&cps->cp_list)) { + list_del_init(&cps->cp_list); + idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.cs_stid.si_opaque.so_id); + } +} + +/* + * Revoke a copy-notify stateid: unlink it from the IDR and sc_cp_list first + * so no new finder can discover it, then drop the membership reference. Every + * revoke path (cancel, laundromat, drain) must use this rather than + * _free_cpntf_state_locked(), which unlinks only at refcount zero and so could + * let a second revoke free the entry under a concurrent reader. + * + * nn->s2s_cp_lock must be held. + */ +static void revoke_cpntf_state_locked(struct nfsd_net *nn, + struct nfs4_cpntf_state *cps) +{ + nfsd4_unhash_cpntf_state(nn, cps); + put_cpntf_state_unlinked_locked(cps); } static void nfs4_free_cpntf_statelist(struct net *net, struct nfs4_stid *stid) { - struct nfs4_cpntf_state *cps; + struct nfs4_cpntf_state *cps, *tmp; struct nfsd_net *nn; nn = net_generic(net, nfsd_net_id); spin_lock(&nn->s2s_cp_lock); - while (!list_empty(&stid->sc_cp_list)) { - cps = list_first_entry(&stid->sc_cp_list, - struct nfs4_cpntf_state, cp_list); - _free_cpntf_state_locked(nn, cps); - } + /* + * Revoke unlinks each entry before dropping the parent's reference, so + * the drain terminates in one pass per entry regardless of cs_count; a + * concurrent holder does the final kfree via nfs4_put_cpntf_state(). + */ + list_for_each_entry_safe(cps, tmp, &stid->sc_cp_list, cp_list) + revoke_cpntf_state_locked(nn, cps); spin_unlock(&nn->s2s_cp_lock); } @@ -1120,29 +1245,31 @@ static void block_delegations(struct knfsd_fh *fh) } static struct nfs4_delegation * -alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, - struct nfs4_clnt_odstate *odstate, u32 dl_type) +__alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, + struct nfs4_clnt_odstate *odstate, u32 dl_type, + void (*sc_free)(struct nfs4_stid *)) { struct nfs4_delegation *dp; struct nfs4_stid *stid; long n; - dprintk("NFSD alloc_init_deleg\n"); + if (delegation_blocked(&fp->fi_fhandle)) + return NULL; + n = atomic_long_inc_return(&num_delegations); if (n < 0 || n > max_delegations) goto out_dec; - if (delegation_blocked(&fp->fi_fhandle)) - goto out_dec; - stid = nfs4_alloc_stid(clp, deleg_slab, nfs4_free_deleg); + + stid = nfs4_alloc_stid(clp, deleg_slab, sc_free); if (stid == NULL) goto out_dec; - dp = delegstateid(stid); /* * delegation seqid's are never incremented. The 4.1 special * meaning of seqid 0 isn't meaningful, really, but let's avoid - * 0 anyway just for consistency and use 1: + * 0 anyway just for consistency and use 1. */ + dp = delegstateid(stid); dp->dl_stid.sc_stateid.si_generation = 1; INIT_LIST_HEAD(&dp->dl_perfile); INIT_LIST_HEAD(&dp->dl_perclnt); @@ -1152,25 +1279,85 @@ alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, dp->dl_type = dl_type; dp->dl_retries = 1; dp->dl_recalled = false; - nfsd4_init_cb(&dp->dl_recall, dp->dl_stid.sc_client, - &nfsd4_cb_recall_ops, NFSPROC4_CLNT_CB_RECALL); - nfsd4_init_cb(&dp->dl_cb_fattr.ncf_getattr, dp->dl_stid.sc_client, - &nfsd4_cb_getattr_ops, NFSPROC4_CLNT_CB_GETATTR); - dp->dl_cb_fattr.ncf_file_modified = false; get_nfs4_file(fp); dp->dl_stid.sc_file = fp; + nfsd4_init_cb(&dp->dl_recall, dp->dl_stid.sc_client, + &nfsd4_cb_recall_ops, NFSPROC4_CLNT_CB_RECALL); return dp; out_dec: atomic_long_dec(&num_delegations); return NULL; } +static struct nfs4_delegation * +alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, + struct nfs4_clnt_odstate *odstate, u32 dl_type) +{ + struct nfs4_delegation *dp; + + dp = __alloc_init_deleg(clp, fp, odstate, dl_type, nfs4_free_deleg); + if (!dp) + return NULL; + + nfsd4_init_cb(&dp->dl_cb_fattr.ncf_getattr, dp->dl_stid.sc_client, + &nfsd4_cb_getattr_ops, NFSPROC4_CLNT_CB_GETATTR); + dp->dl_cb_fattr.ncf_file_modified = false; + return dp; +} + +static void nfs4_free_dir_deleg(struct nfs4_stid *stid) +{ + struct nfs4_delegation *dp = delegstateid(stid); + struct nfsd4_cb_notify *ncn = &dp->dl_cb_notify; + int i; + + for (i = 0; i < ncn->ncn_evt_cnt; ++i) + nfsd_notify_event_put(ncn->ncn_evt[i]); + kfree(ncn->ncn_nf); + for (i = 0; i < NOTIFY4_PAGE_ARRAY_SIZE; i++) { + if (!ncn->ncn_pages[i]) + break; + put_page(ncn->ncn_pages[i]); + } + nfs4_free_deleg(stid); +} + +static struct nfs4_delegation * +alloc_init_dir_deleg(struct nfs4_client *clp, struct nfs4_file *fp) +{ + struct nfs4_delegation *dp; + struct nfsd4_cb_notify *ncn; + int npages; + + dp = __alloc_init_deleg(clp, fp, NULL, NFS4_OPEN_DELEGATE_READ, nfs4_free_dir_deleg); + if (!dp) + return NULL; + + ncn = &dp->dl_cb_notify; + + npages = alloc_pages_bulk(GFP_KERNEL, NOTIFY4_PAGE_ARRAY_SIZE, ncn->ncn_pages); + if (npages != NOTIFY4_PAGE_ARRAY_SIZE) { + nfs4_put_stid(&dp->dl_stid); + return NULL; + } + + ncn->ncn_nf = kcalloc(NOTIFY4_EVENT_QUEUE_SIZE, sizeof(*ncn->ncn_nf), GFP_KERNEL); + if (!ncn->ncn_nf) { + nfs4_put_stid(&dp->dl_stid); + return NULL; + } + spin_lock_init(&ncn->ncn_lock); + nfsd4_init_cb(&ncn->ncn_cb, dp->dl_stid.sc_client, + &nfsd4_cb_notify_ops, NFSPROC4_CLNT_CB_NOTIFY); + return dp; +} + void nfs4_put_stid(struct nfs4_stid *s) { - struct svc_export *exp = s->sc_export; struct nfs4_file *fp = s->sc_file; struct nfs4_client *clp = s->sc_client; + struct svc_export *exp; might_lock(&clp->cl_lock); @@ -1181,6 +1368,8 @@ nfs4_put_stid(struct nfs4_stid *s) idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id); if (s->sc_status & SC_STATUS_ADMIN_REVOKED) atomic_dec(&s->sc_client->cl_admin_revoked); + /* Read under cl_lock to serialize with drop_stid_export(). */ + exp = s->sc_export; nfs4_free_cpntf_statelist(clp->net, s); spin_unlock(&clp->cl_lock); s->sc_free(s); @@ -1209,7 +1398,9 @@ static void put_deleg_file(struct nfs4_file *fp) spin_lock(&fp->fi_lock); if (--fp->fi_delegees == 0) { - swap(nf, fp->fi_deleg_file); + nf = rcu_dereference_protected(fp->fi_deleg_file, + lockdep_is_held(&fp->fi_lock)); + RCU_INIT_POINTER(fp->fi_deleg_file, NULL); swap(rnf, fp->fi_rdeleg_file); } spin_unlock(&fp->fi_lock); @@ -1247,12 +1438,13 @@ static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct f static void nfs4_unlock_deleg_lease(struct nfs4_delegation *dp) { struct nfs4_file *fp = dp->dl_stid.sc_file; - struct nfsd_file *nf = fp->fi_deleg_file; + struct nfsd_file *nf = rcu_dereference_protected(fp->fi_deleg_file, 1); WARN_ON_ONCE(!fp->fi_delegees); nfsd4_finalize_deleg_timestamps(dp, nf->nf_file); kernel_setlease(nf->nf_file, F_UNLCK, NULL, (void **)&dp); + nfsd_fsnotify_recalc_mask(nf); put_deleg_file(fp); } @@ -1637,6 +1829,7 @@ static void free_ol_stateid_reaplist(struct list_head *reaplist) { struct nfs4_ol_stateid *stp; + struct svc_export *exp; struct nfs4_file *fp; might_sleep(); @@ -1646,7 +1839,10 @@ free_ol_stateid_reaplist(struct list_head *reaplist) st_locks); list_del(&stp->st_locks); fp = stp->st_stid.sc_file; + exp = stp->st_stid.sc_export; stp->st_stid.sc_free(&stp->st_stid); + if (exp) + exp_put(exp); if (fp) put_nfs4_file(fp); } @@ -1873,14 +2069,20 @@ static void revoke_one_stid(struct nfsd_net *nn, struct nfs4_client *clp, * being released. Thus nfsd will no longer prevent the filesystem from being * unmounted. * - * The clients which own the states will subsequently being notified that the + * The clients which own the states will subsequently be notified that the * states have been "admin-revoked". + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. */ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) { unsigned int idhashval; unsigned int sc_types; + lockdep_assert_held(&nfsd_mutex); + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; spin_lock(&nn->client_lock); @@ -1889,9 +2091,19 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) struct nfs4_client *clp; retry: list_for_each_entry(clp, head, cl_idhash) { - struct nfs4_stid *stid = find_one_sb_stid(clp, sb, - sc_types); + struct nfs4_stid *stid; + + /* + * force_expire_client() ignores cl_rpc_users once + * its wait_event() has passed, so pinning cannot + * keep an already-expiring client alive; the + * expiry path revokes its states instead. + */ + if (is_client_expired(clp)) + continue; + stid = find_one_sb_stid(clp, sb, sc_types); if (stid) { + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); revoke_one_stid(nn, clp, stid); nfs4_put_stid(stid); @@ -1904,6 +2116,7 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) */ nn->nfs40_last_revoke = ktime_get_boottime_seconds(); + put_client_no_renew_locked(clp); goto retry; } } @@ -1946,12 +2159,18 @@ static struct nfs4_stid *find_one_export_stid(struct nfs4_client *clp, * * Userspace (exportfs -u) sends this after removing the last client * for a path, enabling the underlying filesystem to be unmounted. + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. */ void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) { unsigned int idhashval; unsigned int sc_types; + lockdep_assert_held(&nfsd_mutex); + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; spin_lock(&nn->client_lock); @@ -1960,10 +2179,14 @@ void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) struct nfs4_client *clp; retry: list_for_each_entry(clp, head, cl_idhash) { - struct nfs4_stid *stid = find_one_export_stid( - clp, path, - sc_types); + struct nfs4_stid *stid; + + /* Skip or pin clp as in nfsd4_revoke_states(). */ + if (is_client_expired(clp)) + continue; + stid = find_one_export_stid(clp, path, sc_types); if (stid) { + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); revoke_one_stid(nn, clp, stid); nfs4_put_stid(stid); @@ -1971,6 +2194,7 @@ void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) if (clp->cl_minorversion == 0) nn->nfs40_last_revoke = ktime_get_boottime_seconds(); + put_client_no_renew_locked(clp); goto retry; } } @@ -2052,12 +2276,10 @@ gen_sessionid(struct nfsd4_session *ses) static struct shrinker *nfsd_slot_shrinker; static DEFINE_SPINLOCK(nfsd_session_list_lock); static LIST_HEAD(nfsd_session_list); -/* The sum of "target_slots-1" on every session. The shrinker can push this - * down, though it can take a little while for the memory to actually - * be freed. The "-1" is because we can never free slot 0 while the - * session is active. - */ +/* The sum of "target_slots" on every session, slot 0 included. */ static atomic_t nfsd_total_target_slots = ATOMIC_INIT(0); +/* Session count, subtracted from the sum to exclude slot 0. */ +static atomic_t nfsd_total_sessions = ATOMIC_INIT(0); static void free_session_slots(struct nfsd4_session *ses, int from) @@ -2081,26 +2303,21 @@ free_session_slots(struct nfsd4_session *ses, int from) } ses->se_fchannel.maxreqs = from; if (ses->se_target_maxslots > from) { - int new_target = from ?: 1; - atomic_sub(ses->se_target_maxslots - new_target, &nfsd_total_target_slots); - ses->se_target_maxslots = new_target; + int delta = ses->se_target_maxslots - from; + + atomic_sub(delta, &nfsd_total_target_slots); + /* Retain one slot so the session can make forward progress. */ + ses->se_target_maxslots = from ?: 1; } } -/** - * reduce_session_slots - reduce the target max-slots of a session if possible - * @ses: The session to affect - * @dec: how much to decrease the target by - * +/* * This interface can be used by a shrinker to reduce the target max-slots * for a session so that some slots can eventually be freed. * It uses spin_trylock() as it may be called in a context where another * spinlock is held that has a dependency on client_lock. As shrinkers are - * best-effort, skiping a session is client_lock is already held has no - * great coast - * - * Return value: - * The number of slots that the target was reduced by. + * best-effort, skipping a session with the client_lock already held has no + * great cost. */ static int reduce_session_slots(struct nfsd4_session *ses, int dec) @@ -2179,7 +2396,7 @@ static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fattrs, fattrs->maxreqs = i; memcpy(&new->se_fchannel, fattrs, sizeof(struct nfsd4_channel_attrs)); new->se_target_maxslots = i; - atomic_add(i - 1, &nfsd_total_target_slots); + atomic_add(i, &nfsd_total_target_slots); new->se_cb_slot_avail = ~0U; new->se_cb_highest_slot = min(battrs->maxreqs - 1, NFSD_BC_SLOT_TABLE_SIZE - 1); @@ -2295,7 +2512,7 @@ static void __free_session(struct nfsd4_session *ses) { free_session_slots(ses, 0); xa_destroy(&ses->se_slots); - kfree(ses); + kfree_rcu(ses, rcu_head); } static void free_session(struct nfsd4_session *ses) @@ -2304,21 +2521,51 @@ static void free_session(struct nfsd4_session *ses) __free_session(ses); } +/** + * nfsd_slot_shrinker_count - report reclaimable DRC slots + * @s: shrinker descriptor (unused) + * @sc: shrink control (unused) + * + * Return: a positive count of reclaimable slots, or SHRINK_EMPTY when + * there is nothing to reclaim. + */ static unsigned long -nfsd_slot_count(struct shrinker *s, struct shrink_control *sc) +nfsd_slot_shrinker_count(struct shrinker *s, struct shrink_control *sc) { - unsigned long cnt = atomic_read(&nfsd_total_target_slots); + int count; - return cnt ? cnt : SHRINK_EMPTY; + /* + * To prevent session deadlock, one slot of each session (slot 0) + * is not reclaimable while the session is active. Thus the number + * of sessions is subtracted from the total number of target slots. + */ + count = atomic_read(&nfsd_total_target_slots) - + atomic_read(&nfsd_total_sessions); + + return count > 0 ? count : SHRINK_EMPTY; } +/** + * nfsd_slot_shrinker_scan - reclaim DRC slots under memory pressure + * @s: shrinker descriptor (unused) + * @sc: shrink control; @sc->nr_to_scan bounds the sessions visited, + * @sc->nr_scanned reports how many were visited + * + * Return: the number of session slots NFSD will release. + */ static unsigned long -nfsd_slot_scan(struct shrinker *s, struct shrink_control *sc) +nfsd_slot_shrinker_scan(struct shrinker *s, struct shrink_control *sc) { struct nfsd4_session *ses; unsigned long scanned = 0; unsigned long freed = 0; + /* + * Each visited session releases at most one slot. After + * nr_to_scan sessions have been visited, the list head is + * rotated past the last visited session so the next scan + * resumes from there. + */ spin_lock(&nfsd_session_list_lock); list_for_each_entry(ses, &nfsd_session_list, se_all_sessions) { freed += reduce_session_slots(ses, 1); @@ -2360,6 +2607,7 @@ static void init_session(struct svc_rqst *rqstp, struct nfsd4_session *new, stru spin_lock(&nfsd_session_list_lock); list_add_tail(&new->se_all_sessions, &nfsd_session_list); + atomic_inc(&nfsd_total_sessions); spin_unlock(&nfsd_session_list_lock); { @@ -2433,6 +2681,7 @@ unhash_session(struct nfsd4_session *ses) spin_unlock(&ses->se_client->cl_lock); spin_lock(&nfsd_session_list_lock); list_del(&ses->se_all_sessions); + atomic_dec(&nfsd_total_sessions); spin_unlock(&nfsd_session_list_lock); } @@ -2581,7 +2830,17 @@ unhash_client_locked(struct nfs4_client *clp) spin_lock(&nfsd_session_list_lock); list_for_each_entry(ses, &clp->cl_sessions, se_perclnt) { list_del_init(&ses->se_hash); - list_del_init(&ses->se_all_sessions); + /* + * unhash_client_locked() can run more than once for a + * client; the session stays on cl_sessions across calls. + * The first pass empties se_all_sessions via + * list_del_init(), so skip the decrement on later passes + * to keep nfsd_total_sessions from being double-counted. + */ + if (!list_empty(&ses->se_all_sessions)) { + list_del_init(&ses->se_all_sessions); + atomic_dec(&nfsd_total_sessions); + } } spin_unlock(&nfsd_session_list_lock); spin_unlock(&clp->cl_lock); @@ -2641,14 +2900,24 @@ __destroy_client(struct nfs4_client *clp) release_openowner(oo); } for (i = 0; i < OWNER_HASH_SIZE; i++) { - struct nfs4_stateowner *so, *tmp; + struct nfs4_stateowner *so; - list_for_each_entry_safe(so, tmp, &clp->cl_ownerstr_hashtbl[i], - so_strhash) { + spin_lock(&clp->cl_lock); + while (!list_empty(&clp->cl_ownerstr_hashtbl[i])) { + so = list_first_entry(&clp->cl_ownerstr_hashtbl[i], + struct nfs4_stateowner, so_strhash); /* Should be no openowners at this point */ WARN_ON_ONCE(so->so_is_open_owner); + nfs4_get_stateowner(so); + unhash_lockowner_locked(lockowner(so)); + spin_unlock(&clp->cl_lock); + remove_blocked_locks(lockowner(so)); + nfs4_put_stateowner(so); + + spin_lock(&clp->cl_lock); } + spin_unlock(&clp->cl_lock); } nfsd4_return_all_client_layouts(clp); nfsd4_shutdown_copy(clp); @@ -2675,16 +2944,23 @@ static void inc_reclaim_complete(struct nfs4_client *clp) { struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); - if (!nn->track_reclaim_completes) + if (!test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags)) return; - if (!nfsd4_find_reclaim_client(clp->cl_name, nn)) + + down_read(&nn->reclaim_str_hashtbl_lock); + if (!nfsd4_find_reclaim_client(clp->cl_name, nn)) { + up_read(&nn->reclaim_str_hashtbl_lock); return; + } if (atomic_inc_return(&nn->nr_reclaim_complete) == nn->reclaim_str_hashtbl_size) { + up_read(&nn->reclaim_str_hashtbl_lock); printk(KERN_INFO "NFSD: all clients done reclaiming, ending NFSv4 grace period (net %x)\n", clp->net->ns.inum); nfsd4_end_grace(nn); + return; } + up_read(&nn->reclaim_str_hashtbl_lock); } static void expire_client(struct nfs4_client *clp) @@ -2852,6 +3128,14 @@ find_stateid_locked(struct nfs4_client *cl, stateid_t *t) ret = idr_find(&cl->cl_stateids, t->si_opaque.so_id); if (!ret || !ret->sc_type) return NULL; + /* + * Copy offload stateids live in cl_stateids only for id allocation and + * refcounting; per RFC 7862 they are not valid targets for generic + * stateid ops (FREE_STATEID, TEST_STATEID, I/O). Hide them so those + * paths return NFS4ERR_BAD_STATEID. + */ + if (ret->sc_type == SC_TYPE_COPY) + return NULL; return ret; } @@ -3140,7 +3424,8 @@ static int nfs4_show_deleg(struct seq_file *s, struct nfs4_stid *st) /* XXX: lease time, whether it's being recalled. */ spin_lock(&nf->fi_lock); - file = nf->fi_deleg_file; + file = rcu_dereference_protected(nf->fi_deleg_file, + lockdep_is_held(&nf->fi_lock)); if (file) { seq_puts(s, ", "); nfs4_show_superblock(s, file); @@ -3259,6 +3544,11 @@ static void force_expire_client(struct nfs4_client *clp) trace_nfsd_clid_admin_expired(&clp->cl_clientid); + /* + * cl_time is cleared under client_lock before the wait so a + * revocation walk pinning cl_rpc_users under it either skips + * this client or is seen by this wait_event(). + */ spin_lock(&nn->client_lock); clp->cl_time = 0; spin_unlock(&nn->client_lock); @@ -3359,6 +3649,212 @@ nfsd4_cb_getattr_release(struct nfsd4_callback *cb) nfs4_put_stid(&dp->dl_stid); } +static void nfsd_break_one_deleg(struct nfs4_delegation *dp) +{ + bool queued; + + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags)) + return; + + /* + * When called from the lease break (nfsd_break_deleg_cb()) the state + * code is serialized by the flc_lock and the lease has not been + * removed yet, so sc_count is known to be nonzero. The CB_NOTIFY + * callback paths reach here from a workqueue without the flc_lock, + * where the delegation may already be unhashed with sc_count at zero. + * Use refcount_inc_not_zero() so both cases are safe, and bail if the + * delegation is already being torn down. + */ + if (!refcount_inc_not_zero(&dp->dl_stid.sc_count)) { + clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags); + return; + } + queued = nfsd4_run_cb(&dp->dl_recall); + WARN_ON_ONCE(!queued); + if (!queued) { + refcount_dec(&dp->dl_stid.sc_count); + clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags); + } +} + +static bool +nfsd4_cb_notify_prepare(struct nfsd4_callback *cb) +{ + struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + struct nfsd_notify_event *events[NOTIFY4_EVENT_QUEUE_SIZE]; + struct xdr_buf xdr = { .buflen = PAGE_SIZE * NOTIFY4_PAGE_ARRAY_SIZE, + .pages = ncn->ncn_pages }; + int limit = NOTIFY4_EVENT_QUEUE_SIZE; + struct xdr_stream stream; + struct nfsd_file *nf; + bool error = false; + int count, i; + + /* Save a slot for dir attr update if requested */ + if (dp->dl_notify_mask & BIT(NOTIFY4_CHANGE_DIR_ATTRS)) + --limit; + + /* Clear any failure recorded by a previous transmit. */ + ncn->ncn_encode_err = false; + + xdr_init_encode_pages(&stream, &xdr); + + spin_lock(&ncn->ncn_lock); + count = ncn->ncn_evt_cnt; + + /* spurious queueing? */ + if (count == 0) { + spin_unlock(&ncn->ncn_lock); + return false; + } + + memcpy(events, ncn->ncn_evt, sizeof(*events) * count); + ncn->ncn_evt_cnt = 0; + spin_unlock(&ncn->ncn_lock); + + /* + * We can't keep up! Drop the queued events and recall. The queue must + * be drained here: out_recall leaves ncn_evt_cnt at 0, so the release + * op won't see leftover events and requeue this callback forever. + */ + if (count > limit) { + for (i = 0; i < count; ++i) + nfsd_notify_event_put(events[i]); + goto out_recall; + } + + rcu_read_lock(); + nf = nfsd_file_get(rcu_dereference(dp->dl_stid.sc_file->fi_deleg_file)); + rcu_read_unlock(); + if (!nf) { + for (i = 0; i < count; ++i) + nfsd_notify_event_put(events[i]); + goto out_recall; + } + + for (i = 0; i < count; ++i) { + struct nfsd_notify_event *nne = events[i]; + + if (!error) { + u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp)); + u8 *p; + + if (!maskp) { + error = true; + goto put_event; + } + + p = nfsd4_encode_notify_event(&stream, nne, dp, nf, maskp); + if (!p) { + pr_notice("Could not generate CB_NOTIFY from fsnotify mask 0x%x\n", + nne->ne_mask); + error = true; + goto put_event; + } + + ncn->ncn_nf[i].notify_mask.count = 1; + ncn->ncn_nf[i].notify_mask.element = maskp; + ncn->ncn_nf[i].notify_vals.data = p; + ncn->ncn_nf[i].notify_vals.len = (u8 *)stream.p - p; + } +put_event: + nfsd_notify_event_put(nne); + } + if (!error && (dp->dl_notify_mask & BIT(NOTIFY4_CHANGE_DIR_ATTRS))) { + u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp)); + u8 *p; + + if (maskp) + p = nfsd4_encode_dir_attr_change(&stream, dp, nf); + else + p = ERR_PTR(-ENOBUFS); + + if (IS_ERR(p)) { + /* + * The client asked to be told about dir attr changes + * but the change could not be encoded. RFC 8881 + * s10.9.4 requires the server to recall the delegation + * rather than drop a requested notification, so fall + * through to recall. A NULL return instead means there + * were no attributes to report, so omit the event in + * that case. + */ + error = true; + } else if (p) { + *maskp = BIT(NOTIFY4_CHANGE_DIR_ATTRS); + ncn->ncn_nf[count].notify_mask.count = 1; + ncn->ncn_nf[count].notify_mask.element = maskp; + ncn->ncn_nf[count].notify_vals.data = p; + ncn->ncn_nf[count].notify_vals.len = (u8 *)stream.p - p; + ++count; + } + } + if (!error) { + ncn->ncn_nf_cnt = count; + nfsd_file_put(nf); + return true; + } + nfsd_file_put(nf); +out_recall: + nfsd_break_one_deleg(dp); + return false; +} + +static int +nfsd4_cb_notify_done(struct nfsd4_callback *cb, + struct rpc_task *task) +{ + struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + + if (dp->dl_stid.sc_status) + return 1; + + /* + * The CB_NOTIFY op overflowed the send buffer and was dropped from the + * compound. The notification is lost, so recall the delegation rather + * than leaving the client unaware of the directory change. + */ + if (ncn->ncn_encode_err) { + nfsd_break_one_deleg(dp); + return 1; + } + + switch (task->tk_status) { + case -NFS4ERR_DELAY: + rpc_delay(task, 2 * HZ); + return 0; + default: + /* For any other hard error, recall the deleg */ + nfsd_break_one_deleg(dp); + fallthrough; + case 0: + return 1; + } +} + +static void nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn); + +static void +nfsd4_cb_notify_release(struct nfsd4_callback *cb) +{ + struct nfsd4_cb_notify *ncn = + container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = + container_of(ncn, struct nfs4_delegation, dl_cb_notify); + + /* + * Drain events that arrived while this callback was in flight, but + * don't requeue against a revoked delegation: there's no point in + * notifying a client that no longer holds it, and doing so can pin the + * stid and spin the workqueue. + */ + if (!dp->dl_stid.sc_status && READ_ONCE(ncn->ncn_evt_cnt) > 0) + nfsd4_run_cb_notify(ncn); + nfs4_put_stid(&dp->dl_stid); +} + static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = { .done = nfsd4_cb_recall_any_done, .release = nfsd4_cb_recall_any_release, @@ -3371,6 +3867,13 @@ static const struct nfsd4_callback_ops nfsd4_cb_getattr_ops = { .opcode = OP_CB_GETATTR, }; +static const struct nfsd4_callback_ops nfsd4_cb_notify_ops = { + .prepare = nfsd4_cb_notify_prepare, + .done = nfsd4_cb_notify_done, + .release = nfsd4_cb_notify_release, + .opcode = OP_CB_NOTIFY, +}; + static void nfs4_cb_getattr(struct nfs4_cb_fattr *ncf) { struct nfs4_delegation *dp = @@ -3414,7 +3917,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name, clp->cl_time = ktime_get_boottime_seconds(); copy_verf(clp, verf); memcpy(&clp->cl_addr, sa, sizeof(struct sockaddr_storage)); - clp->cl_cb_session = NULL; + RCU_INIT_POINTER(clp->cl_cb_session, NULL); clp->net = net; clp->cl_nfsd_dentry = nfsd_client_mkdir( nn, &clp->cl_nfsdfs, @@ -4496,6 +4999,19 @@ static void nfsd4_construct_sequence_response(struct nfsd4_session *session, seq->status_flags |= SEQ4_STATUS_ADMIN_STATE_REVOKED; } +static bool nfsd4_slots_inuse(struct nfsd4_session *ses, int from) +{ + int i; + + for (i = from; i < ses->se_fchannel.maxreqs; i++) { + struct nfsd4_slot *slot = xa_load(&ses->se_slots, i); + + if (slot->sl_flags & NFSD4_SLOT_INUSE) + return true; + } + return false; +} + __be32 nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) @@ -4575,7 +5091,9 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (session->se_target_maxslots < session->se_fchannel.maxreqs && slot->sl_generation == session->se_slot_gen && - seq->maxslots <= session->se_target_maxslots) + seq->maxslots <= session->se_target_maxslots && + seq->slotid < session->se_target_maxslots && + !nfsd4_slots_inuse(session, session->se_target_maxslots)) /* Client acknowledged our reduce maxreqs */ free_session_slots(session, session->se_target_maxslots); @@ -4608,15 +5126,26 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, * gently try to allocate another 20%. This allows * fairly quick growth without grossly over-shooting what * the client might use. + * + * Bound that growth by the service's thread ceiling: + * slots beyond the nfsd thread count cannot raise this + * client's throughput, only deepen its backlog. Cap each + * session independently, since a session cannot use + * another's slots; a shared budget would let idle sessions + * pin an active client small. Compare against the + * configured maximum, not the running thread count, so a + * client resuming from idle can grow back before the pool + * scales up. */ if (seq->slotid == session->se_fchannel.maxreqs - 1 && - session->se_target_maxslots >= session->se_fchannel.maxreqs && - session->se_fchannel.maxreqs < NFSD_MAX_SLOTS_PER_SESSION) { + session->se_target_maxslots >= session->se_fchannel.maxreqs) { int s = session->se_fchannel.maxreqs; - int cnt = DIV_ROUND_UP(s, 5); + int ceiling = min_t(int, NFSD_MAX_SLOTS_PER_SESSION, + svc_serv_maxthreads(rqstp->rq_server)); + int cnt = min(DIV_ROUND_UP(s, 5), ceiling - s); void *prev_slot; - do { + while (cnt-- > 0) { /* * GFP_NOWAIT both allows allocation under a * spinlock, and only succeeds if there is @@ -4624,13 +5153,14 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, */ slot = nfsd4_alloc_slot(&session->se_fchannel, s, GFP_NOWAIT); + if (!slot) + break; prev_slot = xa_load(&session->se_slots, s); - if (xa_is_value(prev_slot) && slot) { + if (xa_is_value(prev_slot)) { slot->sl_seqid = xa_to_value(prev_slot); slot->sl_flags |= NFSD4_SLOT_REUSED; } - if (slot && - !xa_is_err(xa_store(&session->se_slots, s, slot, + if (!xa_is_err(xa_store(&session->se_slots, s, slot, GFP_NOWAIT))) { s += 1; session->se_fchannel.maxreqs = s; @@ -4639,9 +5169,9 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, session->se_target_maxslots = s; } else { kfree(slot); - slot = NULL; + break; } - } while (slot && --cnt > 0); + } } out: @@ -4922,7 +5452,7 @@ static void nfsd4_file_init(const struct svc_fh *fh, struct nfs4_file *fp) INIT_LIST_HEAD(&fp->fi_delegations); INIT_LIST_HEAD(&fp->fi_clnt_odstate); fh_copy_shallow(&fp->fi_fhandle, &fh->fh_handle); - fp->fi_deleg_file = NULL; + RCU_INIT_POINTER(fp->fi_deleg_file, NULL); fp->fi_rdeleg_file = NULL; fp->fi_had_conflict = false; fp->fi_share_deny = 0; @@ -4946,6 +5476,7 @@ nfsd4_free_slabs(void) kmem_cache_destroy(stateid_slab); kmem_cache_destroy(deleg_slab); kmem_cache_destroy(odstate_slab); + kmem_cache_destroy(async_copy_slab); } int @@ -4972,8 +5503,13 @@ nfsd4_init_slabs(void) odstate_slab = KMEM_CACHE(nfs4_clnt_odstate, 0); if (odstate_slab == NULL) goto out_free_deleg_slab; + async_copy_slab = KMEM_CACHE(nfsd4_async_copy, 0); + if (async_copy_slab == NULL) + goto out_free_odstate_slab; return 0; +out_free_odstate_slab: + kmem_cache_destroy(odstate_slab); out_free_deleg_slab: kmem_cache_destroy(deleg_slab); out_free_stateid_slab: @@ -5018,8 +5554,6 @@ nfsd4_init_leases_net(struct nfsd_net *nn) nn->nfsd4_lease = 90; /* default lease time */ nn->nfsd4_grace = 90; - nn->somebody_reclaimed = false; - nn->track_reclaim_completes = false; nn->clverifier_counter = get_random_u32(); nn->clientid_base = get_random_u32(); nn->clientid_counter = nn->clientid_base + 1; @@ -5172,6 +5706,7 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) case SC_TYPE_DELEG: dp = delegstateid(s); list_del_init(&dp->dl_recall_lru); + s->sc_status |= SC_STATUS_FREED; spin_unlock(&cl->cl_lock); nfs4_put_stid(s); break; @@ -5528,7 +6063,7 @@ bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode) return timeo > 0; } -static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb) +static bool nfsd4_cb_recall_prepare(struct nfsd4_callback *cb) { struct nfs4_delegation *dp = cb_to_delegation(cb); struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net, @@ -5549,6 +6084,7 @@ static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb) list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru); } spin_unlock(&nn->deleg_lock); + return true; } static int nfsd4_cb_recall_done(struct nfsd4_callback *cb, @@ -5598,27 +6134,6 @@ static const struct nfsd4_callback_ops nfsd4_cb_recall_ops = { .opcode = OP_CB_RECALL, }; -static void nfsd_break_one_deleg(struct nfs4_delegation *dp) -{ - bool queued; - - if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags)) - return; - - /* - * We're assuming the state code never drops its reference - * without first removing the lease. Since we're in this lease - * callback (and since the lease code is serialized by the - * flc_lock) we know the server hasn't removed the lease yet, and - * we know it's safe to take a reference. - */ - refcount_inc(&dp->dl_stid.sc_count); - queued = nfsd4_run_cb(&dp->dl_recall); - WARN_ON_ONCE(!queued); - if (!queued) - refcount_dec(&dp->dl_stid.sc_count); -} - /* Called from break_lease() with flc_lock held. */ static bool nfsd_break_deleg_cb(struct file_lease *fl) @@ -5664,6 +6179,10 @@ static bool nfsd_breaker_owns_lease(struct file_lease *fl) struct svc_rqst *rqst; struct nfs4_client *clp; + /* Only nfsd leases */ + if (fl->fl_lmops != &nfsd_lease_mng_ops) + return false; + rqst = nfsd_current_rqst(); if (!nfsd_v4client(rqst)) return false; @@ -6043,7 +6562,22 @@ static bool nfsd4_cb_channel_good(struct nfs4_client *clp) return clp->cl_minorversion && clp->cl_cb_state == NFSD4_CB_UNKNOWN; } -static struct file_lease *nfs4_alloc_init_lease(struct nfs4_delegation *dp) +static unsigned int +nfsd_notify_to_ignore(u32 notify) +{ + unsigned int mask = 0; + + if (notify & BIT(NOTIFY4_REMOVE_ENTRY)) + mask |= FL_IGN_DIR_DELETE; + if (notify & BIT(NOTIFY4_ADD_ENTRY)) + mask |= FL_IGN_DIR_CREATE; + if (notify & BIT(NOTIFY4_RENAME_ENTRY)) + mask |= FL_IGN_DIR_RENAME; + + return mask; +} + +static struct file_lease *nfs4_alloc_init_lease(struct nfs4_delegation *dp, u32 notify) { struct file_lease *fl; @@ -6051,11 +6585,11 @@ static struct file_lease *nfs4_alloc_init_lease(struct nfs4_delegation *dp) if (!fl) return NULL; fl->fl_lmops = &nfsd_lease_mng_ops; - fl->c.flc_flags = FL_DELEG; + fl->c.flc_flags = FL_DELEG | nfsd_notify_to_ignore(notify); fl->c.flc_type = deleg_is_read(dp->dl_type) ? F_RDLCK : F_WRLCK; fl->c.flc_owner = (fl_owner_t)dp; fl->c.flc_pid = current->tgid; - fl->c.flc_file = dp->dl_stid.sc_file->fi_deleg_file->nf_file; + fl->c.flc_file = rcu_dereference_protected(dp->dl_stid.sc_file->fi_deleg_file, 1)->nf_file; return fl; } @@ -6063,7 +6597,7 @@ static int nfsd4_check_conflicting_opens(struct nfs4_client *clp, struct nfs4_file *fp) { struct nfs4_ol_stateid *st; - struct file *f = fp->fi_deleg_file->nf_file; + struct file *f = rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file; struct inode *ino = file_inode(f); int writes; @@ -6140,7 +6674,7 @@ nfsd4_verify_deleg_dentry(struct nfsd4_open *open, struct nfs4_file *fp, exp_put(exp); dput(child); - if (child != file_dentry(fp->fi_deleg_file->nf_file)) + if (child != file_dentry(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file)) return -EAGAIN; return 0; @@ -6246,8 +6780,9 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, status = -EAGAIN; else if (nfsd4_verify_setuid_write(open, nf)) status = -EAGAIN; - else if (!fp->fi_deleg_file) { - fp->fi_deleg_file = nf; + else if (!rcu_dereference_protected(fp->fi_deleg_file, + lockdep_is_held(&fp->fi_lock))) { + rcu_assign_pointer(fp->fi_deleg_file, nf); /* increment early to prevent fi_deleg_file from being * cleared */ fp->fi_delegees = 1; @@ -6268,11 +6803,11 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, if (stp->st_stid.sc_export) dp->dl_stid.sc_export = exp_get(stp->st_stid.sc_export); - fl = nfs4_alloc_init_lease(dp); + fl = nfs4_alloc_init_lease(dp, 0); if (!fl) goto out_clnt_odstate; - status = kernel_setlease(fp->fi_deleg_file->nf_file, + status = kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file, fl->c.flc_type, &fl, NULL); if (fl) locks_free_lease(fl); @@ -6293,7 +6828,7 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, * Now that the deleg is set, check again to ensure that nothing * raced in and changed the mode while we weren't looking. */ - status = nfsd4_verify_setuid_write(open, fp->fi_deleg_file); + status = nfsd4_verify_setuid_write(open, rcu_dereference_protected(fp->fi_deleg_file, 1)); if (status) goto out_unlock; @@ -6314,7 +6849,8 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, return dp; out_unlock: - kernel_setlease(fp->fi_deleg_file->nf_file, F_UNLCK, NULL, (void **)&dp); + kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file, + F_UNLCK, NULL, (void **)&dp); out_clnt_odstate: put_clnt_odstate(dp->dl_clnt_odstate); nfs4_put_stid(&dp->dl_stid); @@ -6471,8 +7007,9 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, memcpy(&open->op_delegate_stateid, &dp->dl_stid.sc_stateid, sizeof(dp->dl_stid.sc_stateid)); if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) { - struct file *f = dp->dl_stid.sc_file->fi_deleg_file->nf_file; + struct file *f; + f = rcu_dereference_protected(dp->dl_stid.sc_file->fi_deleg_file, 1)->nf_file; if (!nfsd4_add_rdaccess_to_wrdeleg(rqstp, open, fh, stp) || !nfs4_delegation_stat(dp, currentfh, &stat)) { nfs4_put_stid(&dp->dl_stid); @@ -6728,12 +7265,21 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, static void nfsd4_end_grace(struct nfsd_net *nn) { - /* do nothing if grace period already ended */ - if (nn->grace_ended) + /* + * nfsd4_end_grace() can be entered concurrently from the + * laundromat workqueue and from an nfsd compound thread + * handling RECLAIM_COMPLETE. Without serialization, both + * callers can observe NFSD_NET_GRACE_ENDED clear and proceed + * into nfsd4_record_grace_done(). For tracking ops whose + * grace_done drains reclaim_str_hashtbl, that results in + * list corruption and a double free of every + * nfs4_client_reclaim entry. Use an atomic test-and-set so + * exactly one caller proceeds. + */ + if (test_and_set_bit(NFSD_NET_GRACE_ENDED, &nn->flags)) return; trace_nfsd_grace_complete(nn); - nn->grace_ended = true; /* * If the server goes down again right now, an NFSv4 * client will still be allowed to reclaim after it comes back up, @@ -6774,10 +7320,10 @@ bool nfsd4_force_end_grace(struct nfsd_net *nn) { if (!nn->client_tracking_ops) return false; - if (READ_ONCE(nn->grace_ended)) + if (test_bit(NFSD_NET_GRACE_ENDED, &nn->flags)) return false; /* laundromat_work must be initialised now, though it might be disabled */ - WRITE_ONCE(nn->grace_end_forced, true); + set_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags); /* mod_delayed_work() doesn't queue work after * nfs4_state_shutdown_net() has called disable_delayed_work_sync() */ @@ -6791,18 +7337,22 @@ bool nfsd4_force_end_grace(struct nfsd_net *nn) */ static bool clients_still_reclaiming(struct nfsd_net *nn) { - time64_t double_grace_period_end = nn->boot_time + + time64_t double_grace_period_end = nn->boot_time_bt + 2 * nn->nfsd4_lease; - if (READ_ONCE(nn->grace_end_forced)) + if (test_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags)) return false; - if (nn->track_reclaim_completes && - atomic_read(&nn->nr_reclaim_complete) == - nn->reclaim_str_hashtbl_size) - return false; - if (!nn->somebody_reclaimed) + if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags)) { + int size; + + down_read(&nn->reclaim_str_hashtbl_lock); + size = nn->reclaim_str_hashtbl_size; + up_read(&nn->reclaim_str_hashtbl_lock); + if (atomic_read(&nn->nr_reclaim_complete) == size) + return false; + } + if (!test_and_clear_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags)) return false; - nn->somebody_reclaimed = false; /* * If we've given them *two* lease times to reclaim, and they're * still not done, give up: @@ -6859,30 +7409,36 @@ static void nfsd4_ssc_shutdown_umount(struct nfsd_net *nn) static void nfsd4_ssc_expire_umount(struct nfsd_net *nn) { bool do_wakeup = false; - struct nfsd4_ssc_umount_item *ni = NULL; - struct nfsd4_ssc_umount_item *tmp; + struct nfsd4_ssc_umount_item *ni; +restart: spin_lock(&nn->nfsd_ssc_lock); - list_for_each_entry_safe(ni, tmp, &nn->nfsd_ssc_mount_list, nsui_list) { - if (time_after(jiffies, ni->nsui_expire)) { - if (refcount_read(&ni->nsui_refcnt) > 1) - continue; + list_for_each_entry(ni, &nn->nfsd_ssc_mount_list, nsui_list) { + if (!time_after(jiffies, ni->nsui_expire)) + break; + if (refcount_read(&ni->nsui_refcnt) > 1) + continue; - /* mark being unmount */ - ni->nsui_busy = true; - spin_unlock(&nn->nfsd_ssc_lock); - mntput(ni->nsui_vfsmount); - spin_lock(&nn->nfsd_ssc_lock); + /* Prevent concurrent setup during unmount */ + ni->nsui_busy = true; + spin_unlock(&nn->nfsd_ssc_lock); + mntput(ni->nsui_vfsmount); + spin_lock(&nn->nfsd_ssc_lock); - /* waiters need to start from begin of list */ - list_del(&ni->nsui_list); - kfree(ni); + /* Force concurrent scanners to restart */ + list_del(&ni->nsui_list); + kfree(ni); - /* wakeup ssc_connect waiters */ - do_wakeup = true; - continue; - } - break; + /* wakeup ssc_connect waiters */ + do_wakeup = true; + /* + * Concurrent nfsd4_ssc_cancel_dul() can free any item + * on the list under nfsd_ssc_lock while mntput() runs + * above. Restart from the head; the list is short and + * the expire worker is periodic, so this is cheap. + */ + spin_unlock(&nn->nfsd_ssc_lock); + goto restart; } if (do_wakeup) wake_up_all(&nn->nfsd_ssc_waitq); @@ -7032,16 +7588,20 @@ retry: if (atomic_read(&clp->cl_admin_revoked) == 0) continue; + if (is_client_expired(clp)) + continue; spin_lock(&clp->cl_lock); idr_for_each_entry_ul(&clp->cl_stateids, stid, tmp, id) if (stid->sc_status & SC_STATUS_ADMIN_REVOKED) { refcount_inc(&stid->sc_count); + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); /* this function drops ->cl_lock */ nfsd4_drop_revoked_stid(stid); nfs4_put_stid(stid); spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); goto retry; } spin_unlock(&clp->cl_lock); @@ -7062,6 +7622,7 @@ nfs4_laundromat(struct nfsd_net *nn) .new_timeo = nn->nfsd4_lease }; struct nfs4_cpntf_state *cps; + struct nfs4_client *clp; copy_stateid_t *cps_t; int i; @@ -7072,11 +7633,11 @@ nfs4_laundromat(struct nfsd_net *nn) nfsd4_end_grace(nn); spin_lock(&nn->s2s_cp_lock); + /* s2s_cp_stateids holds only COPY_NOTIFY stateids */ idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) { cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); - if (cps->cp_stateid.cs_type == NFS4_COPYNOTIFY_STID && - state_expired(<, cps->cpntf_time)) - _free_cpntf_state_locked(nn, cps); + if (state_expired(<, cps->cpntf_time)) + revoke_cpntf_state_locked(nn, cps); } spin_unlock(&nn->s2s_cp_lock); nfsd4_async_copy_reaper(nn); @@ -7090,6 +7651,18 @@ nfs4_laundromat(struct nfsd_net *nn) dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); if (!state_expired(<, dp->dl_time)) break; + clp = dp->dl_stid.sc_client; + spin_lock(&nn->client_lock); + if (is_client_expired(clp)) { + spin_unlock(&nn->client_lock); + continue; + } + /* + * Pin without reviving: get_client_locked() would + * flip a courtesy client back to NFSD4_ACTIVE. + */ + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); refcount_inc(&dp->dl_stid.sc_count); unhash_delegation_locked(dp, SC_STATUS_REVOKED); list_add(&dp->dl_recall_lru, &reaplist); @@ -7098,8 +7671,10 @@ nfs4_laundromat(struct nfsd_net *nn) while (!list_empty(&reaplist)) { dp = list_first_entry(&reaplist, struct nfs4_delegation, dl_recall_lru); + clp = dp->dl_stid.sc_client; list_del_init(&dp->dl_recall_lru); revoke_delegation(dp); + put_client_no_renew(clp); } spin_lock(&nn->client_lock); @@ -7109,11 +7684,16 @@ nfs4_laundromat(struct nfsd_net *nn) if (!state_expired(<, oo->oo_time)) break; list_del_init(&oo->oo_close_lru); + clp = oo->oo_owner.so_client; + if (is_client_expired(clp)) + continue; stp = oo->oo_last_closed_stid; oo->oo_last_closed_stid = NULL; + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); nfs4_put_stid(&stp->st_stid); spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); } spin_unlock(&nn->client_lock); @@ -7129,22 +7709,29 @@ nfs4_laundromat(struct nfsd_net *nn) * indefinitely once the lock does become free. */ BUG_ON(!list_empty(&reaplist)); + spin_lock(&nn->client_lock); spin_lock(&nn->blocked_locks_lock); - while (!list_empty(&nn->blocked_locks_lru)) { - nbl = list_first_entry(&nn->blocked_locks_lru, - struct nfsd4_blocked_lock, nbl_lru); + list_for_each_safe(pos, next, &nn->blocked_locks_lru) { + nbl = list_entry(pos, struct nfsd4_blocked_lock, nbl_lru); if (!state_expired(<, nbl->nbl_time)) break; + clp = nbl_client(nbl); + if (is_client_expired(clp)) + continue; + atomic_inc(&clp->cl_rpc_users); list_move(&nbl->nbl_lru, &reaplist); list_del_init(&nbl->nbl_list); } spin_unlock(&nn->blocked_locks_lock); + spin_unlock(&nn->client_lock); while (!list_empty(&reaplist)) { nbl = list_first_entry(&reaplist, struct nfsd4_blocked_lock, nbl_lru); + clp = nbl_client(nbl); list_del_init(&nbl->nbl_lru); free_blocked_lock(nbl); + put_client_no_renew(clp); } #ifdef CONFIG_NFSD_V4_2_INTER_SSC /* service the server-to-server copy delayed unmount list */ @@ -7195,12 +7782,12 @@ deleg_reaper(struct nfsd_net *nn) continue; if (atomic_read(&clp->cl_delegs_in_recall)) continue; - if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) - continue; if (ktime_get_boottime_seconds() - clp->cl_ra_time < 5) continue; if (clp->cl_cb_state != NFSD4_CB_UP) continue; + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) + continue; /* release in nfsd4_cb_recall_any_release */ kref_get(&clp->cl_nfsdfs.cl_ref); @@ -7463,16 +8050,14 @@ nfs4_check_file(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfs4_stid *s, out: return status; } -static void -_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) + +static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) { WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); - if (!refcount_dec_and_test(&cps->cp_stateid.cs_count)) - return; - list_del(&cps->cp_list); - idr_remove(&nn->s2s_cp_stateids, - cps->cp_stateid.cs_stid.si_opaque.so_id); - kfree(cps); + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) { + nfsd4_unhash_cpntf_state(nn, cps); + kfree(cps); + } } /* * A READ from an inter server to server COPY will have a @@ -7489,18 +8074,25 @@ __be32 manage_cpntf_state(struct nfsd_net *nn, stateid_t *st, if (st->si_opaque.so_clid.cl_id != nn->s2s_cp_cl_id) return nfserr_bad_stateid; spin_lock(&nn->s2s_cp_lock); + /* s2s_cp_stateids holds only COPY_NOTIFY stateids */ cps_t = idr_find(&nn->s2s_cp_stateids, st->si_opaque.so_id); if (cps_t) { state = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); - if (state->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID) { + if (!clp) { + refcount_inc(&state->cp_stateid.cs_count); + } else if (memcmp(&clp->cl_clientid, &state->cp_p_clid, + sizeof(clientid_t))) { + /* + * OFFLOAD_CANCEL: only the creating client may cancel. + * so_id is guessable, so without this check any client + * could free another's cpntf state. + */ state = NULL; goto unlock; + } else { + revoke_cpntf_state_locked(nn, state); } - if (!clp) - refcount_inc(&state->cp_stateid.cs_count); - else - _free_cpntf_state_locked(nn, state); } unlock: spin_unlock(&nn->s2s_cp_lock); @@ -7787,7 +8379,7 @@ retry: return status; stp = openlockstateid(s); if (nfsd4_cstate_assign_replay(cstate, stp->st_stateowner) == -EAGAIN) { - nfs4_put_stateowner(stp->st_stateowner); + nfs4_put_stid(&stp->st_stid); goto retry; } @@ -8036,6 +8628,10 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) goto put_stateid; + status = nfs4_check_fh(&cstate->current_fh, &dp->dl_stid); + if (status) + goto put_stateid; + trace_nfsd_deleg_return(stateid); destroy_delegation(dp); smp_mb__after_atomic(); @@ -8506,6 +9102,9 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, status = nfserr_no_grace; if (!locks_in_grace(net) && lock->lk_reclaim) goto out; + if (lock->lk_reclaim && + test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags)) + goto out; if (lock->lk_reclaim) flags |= FL_RECLAIM; @@ -8542,10 +9141,11 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out; } - if (lock->lk_type & (NFS4_READW_LT | NFS4_WRITEW_LT) && - nfsd4_has_session(cstate) && - locks_can_async_lock(nf->nf_file->f_op)) - flags |= FL_SLEEP; + if ((lock->lk_type == NFS4_READW_LT || + lock->lk_type == NFS4_WRITEW_LT) && + nfsd4_has_session(cstate) && + locks_can_async_lock(nf->nf_file->f_op)) + flags |= FL_SLEEP; nbl = find_or_allocate_block(lock_sop, &fp->fi_fhandle, nn); if (!nbl) { @@ -8587,7 +9187,7 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, nfs4_inc_and_copy_stateid(&lock->lk_resp_stateid, &lock_stp->st_stid); status = 0; if (lock->lk_reclaim) - nn->somebody_reclaimed = true; + set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags); break; case FILE_LOCK_DEFERRED: kref_put(&nbl->nbl_kref, free_nbl); @@ -8963,9 +9563,13 @@ bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn) { struct nfs4_client_reclaim *crp; + bool found; + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(name, nn); - return (crp && crp->cr_clp); + found = (crp && crp->cr_clp); + up_read(&nn->reclaim_str_hashtbl_lock); + return found; } /* @@ -8978,10 +9582,39 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, unsigned int strhashval; struct nfs4_client_reclaim *crp; + down_write(&nn->reclaim_str_hashtbl_lock); + + /* + * A reclaim record for this client name may already exist (for + * example, populated at boot from the recovery directory before + * an in-grace RECLAIM_COMPLETE or an nfsdcld downcall delivers + * the same name). Dedup here so reclaim_str_hashtbl_size stays + * equal to the number of distinct client names; inc_reclaim_complete + * relies on that equality to end the grace period via the fast path. + */ + crp = nfsd4_find_reclaim_client(name, nn); + if (crp) { + if (princhash.len && crp->cr_princhash.len == 0) { + void *pdata = kmemdup(princhash.data, princhash.len, + GFP_KERNEL); + if (pdata) { + crp->cr_princhash.data = pdata; + crp->cr_princhash.len = princhash.len; + } else { + dprintk("%s: failed to allocate memory for princhash.data!\n", + __func__); + crp = NULL; + } + } + up_write(&nn->reclaim_str_hashtbl_lock); + return crp; + } + name.data = kmemdup(name.data, name.len, GFP_KERNEL); if (!name.data) { dprintk("%s: failed to allocate memory for name.data!\n", __func__); + up_write(&nn->reclaim_str_hashtbl_lock); return NULL; } if (princhash.len) { @@ -8990,6 +9623,7 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, dprintk("%s: failed to allocate memory for princhash.data!\n", __func__); kfree(name.data); + up_write(&nn->reclaim_str_hashtbl_lock); return NULL; } } else @@ -9009,6 +9643,7 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, kfree(name.data); kfree(princhash.data); } + up_write(&nn->reclaim_str_hashtbl_lock); return crp; } @@ -9028,6 +9663,7 @@ nfs4_release_reclaim(struct nfsd_net *nn) struct nfs4_client_reclaim *crp = NULL; int i; + down_write(&nn->reclaim_str_hashtbl_lock); for (i = 0; i < CLIENT_HASH_SIZE; i++) { while (!list_empty(&nn->reclaim_str_hashtbl[i])) { crp = list_entry(nn->reclaim_str_hashtbl[i].next, @@ -9036,6 +9672,7 @@ nfs4_release_reclaim(struct nfsd_net *nn) } } WARN_ON_ONCE(nn->reclaim_str_hashtbl_size); + up_write(&nn->reclaim_str_hashtbl_lock); } /* @@ -9113,8 +9750,9 @@ static int nfs4_state_create_net(struct net *net) nn->conf_name_tree = RB_ROOT; nn->unconf_name_tree = RB_ROOT; nn->boot_time = ktime_get_real_seconds(); - nn->grace_ended = false; - nn->grace_end_forced = false; + nn->boot_time_bt = ktime_get_boottime_seconds(); + clear_bit(NFSD_NET_GRACE_ENDED, &nn->flags); + clear_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags); nn->nfsd4_manager.block_opens = true; INIT_LIST_HEAD(&nn->nfsd4_manager.list); INIT_LIST_HEAD(&nn->client_lru); @@ -9200,7 +9838,8 @@ nfs4_state_start_net(struct net *net) nfsd4_client_tracking_init(net); /* safe for laundromat to run now */ enable_delayed_work(&nn->laundromat_work); - if (nn->track_reclaim_completes && nn->reclaim_str_hashtbl_size == 0) + if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags) && + nn->reclaim_str_hashtbl_size == 0) goto skip_grace; printk(KERN_INFO "NFSD: starting %lld-second grace period (net %x)\n", nn->nfsd4_grace, net->ns.inum); @@ -9231,8 +9870,8 @@ nfs4_state_start(void) rhltable_destroy(&nfs4_file_rhltable); return -ENOMEM; } - nfsd_slot_shrinker->count_objects = nfsd_slot_count; - nfsd_slot_shrinker->scan_objects = nfsd_slot_scan; + nfsd_slot_shrinker->count_objects = nfsd_slot_shrinker_count; + nfsd_slot_shrinker->scan_objects = nfsd_slot_shrinker_scan; shrinker_register(nfsd_slot_shrinker); set_max_delegations(); @@ -9483,7 +10122,6 @@ __be32 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, struct nfs4_delegation **pdp) { - struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct nfsd_thread_local_info *ntli = rqstp->rq_private; struct file_lock_context *ctx; struct nfs4_delegation *dp = NULL; @@ -9523,7 +10161,6 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, return 0; } - nfsd_stats_wdeleg_getattr_inc(nn); refcount_inc(&dp->dl_stid.sc_count); ncf = &dp->dl_cb_fattr; nfs4_cb_getattr(&dp->dl_cb_fattr); @@ -9569,6 +10206,30 @@ out_status: return status; } +#define GDD_WORD0_CHILD_ATTRS (FATTR4_WORD0_TYPE | \ + FATTR4_WORD0_CHANGE | \ + FATTR4_WORD0_SIZE | \ + FATTR4_WORD0_FILEID | \ + FATTR4_WORD0_FILEHANDLE) + +#define GDD_WORD1_CHILD_ATTRS (FATTR4_WORD1_MODE | \ + FATTR4_WORD1_NUMLINKS | \ + FATTR4_WORD1_RAWDEV | \ + FATTR4_WORD1_SPACE_USED | \ + FATTR4_WORD1_TIME_ACCESS | \ + FATTR4_WORD1_TIME_METADATA | \ + FATTR4_WORD1_TIME_MODIFY | \ + FATTR4_WORD1_TIME_CREATE) + +#define GDD_WORD0_DIR_ATTRS (FATTR4_WORD0_CHANGE | \ + FATTR4_WORD0_SIZE) + +#define GDD_WORD1_DIR_ATTRS (FATTR4_WORD1_NUMLINKS | \ + FATTR4_WORD1_SPACE_USED | \ + FATTR4_WORD1_TIME_ACCESS | \ + FATTR4_WORD1_TIME_METADATA | \ + FATTR4_WORD1_TIME_MODIFY) + /** * nfsd_get_dir_deleg - attempt to get a directory delegation * @cstate: compound state @@ -9576,8 +10237,7 @@ out_status: * @nf: nfsd_file opened on the directory * * Given a GET_DIR_DELEGATION request @gdd, attempt to acquire a delegation - * on the directory to which @nf refers. Note that this does not set up any - * sort of async notifications for the delegation. + * on the directory to which @nf refers. */ struct nfs4_delegation * nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, @@ -9614,8 +10274,9 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, /* existing delegation? */ if (nfs4_delegation_exists(clp, fp)) { status = -EAGAIN; - } else if (!fp->fi_deleg_file) { - fp->fi_deleg_file = nfsd_file_get(nf); + } else if (!rcu_dereference_protected(fp->fi_deleg_file, + lockdep_is_held(&fp->fi_lock))) { + rcu_assign_pointer(fp->fi_deleg_file, nfsd_file_get(nf)); fp->fi_delegees = 1; } else { ++fp->fi_delegees; @@ -9630,14 +10291,24 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, /* Try to set up the lease */ status = -ENOMEM; - dp = alloc_init_deleg(clp, fp, NULL, NFS4_OPEN_DELEGATE_READ); + dp = alloc_init_dir_deleg(clp, fp); if (!dp) goto out_delegees; if (cstate->current_fh.fh_export) dp->dl_stid.sc_export = exp_get(cstate->current_fh.fh_export); - fl = nfs4_alloc_init_lease(dp); + /* + * NB: gddr_notification[0] represents the notifications that + * will be granted to the client + */ + dp->dl_notify_mask = gdd->gddr_notification[0]; + dp->dl_child_attrs[0] = gdd->gdda_child_attributes[0] & GDD_WORD0_CHILD_ATTRS; + dp->dl_child_attrs[1] = gdd->gdda_child_attributes[1] & GDD_WORD1_CHILD_ATTRS; + dp->dl_dir_attrs[0] = gdd->gdda_dir_attributes[0] & GDD_WORD0_DIR_ATTRS; + dp->dl_dir_attrs[1] = gdd->gdda_dir_attributes[1] & GDD_WORD1_DIR_ATTRS; + + fl = nfs4_alloc_init_lease(dp, dp->dl_notify_mask); if (!fl) goto out_put_stid; @@ -9663,11 +10334,22 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, if (!status) { put_nfs4_file(fp); + nfsd_fsnotify_recalc_mask(nf); return dp; } - /* Something failed. Drop the lease and clean up the stid */ - kernel_setlease(fp->fi_deleg_file->nf_file, F_UNLCK, NULL, (void **)&dp); + /* + * Something failed after the lease was set. Drop the lease and clean + * up the stid. The lease's flc_file is the fi_deleg_file (see + * nfs4_alloc_init_lease()), which is not necessarily this client's + * @nf when an earlier client already holds a delegation on @fp. + * generic_delete_lease() matches on flc_file, so unlock against + * fi_deleg_file or the lease will be leaked (and later freed with the + * stid, leading to a use-after-free when it's eventually broken). + */ + kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file, + F_UNLCK, NULL, (void **)&dp); + nfsd_fsnotify_recalc_mask(nf); out_put_stid: nfs4_put_stid(&dp->dl_stid); out_delegees: @@ -9703,3 +10385,170 @@ void nfsd_update_cmtime_attr(struct file *f, unsigned int flags) MINOR(inode->i_sb->s_dev), inode->i_ino, ret); } + +static void +nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn) +{ + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags)) + return; + + if (!refcount_inc_not_zero(&dp->dl_stid.sc_count)) + clear_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags); + else + nfsd4_run_cb(&ncn->ncn_cb); +} + +static struct nfsd_notify_event * +alloc_nfsd_notify_event(u32 mask, const struct qstr *q, struct dentry *dentry, + struct inode *target) +{ + struct nfsd_notify_event *ne; + struct name_snapshot newname; + u32 newnamelen = 0; + + /* + * For a rename, @q is the old name and the live dentry carries the new + * name. Snapshot the new name now, while it is guaranteed to describe + * this event: the dentry can be renamed again before the CB_NOTIFY work + * runs, which would corrupt a late read in nfsd4_encode_notify_event(). + */ + if (mask & FS_RENAME) { + take_dentry_name_snapshot(&newname, dentry); + newnamelen = newname.name.len; + } + + ne = kmalloc(struct_size(ne, ne_name, q->len + 1 + + (newnamelen ? newnamelen + 1 : 0)), GFP_NOFS); + if (!ne) + goto out; + + memcpy(ne->ne_name, q->name, q->len); + ne->ne_name[q->len] = '\0'; + ne->ne_namelen = q->len; + + ne->ne_newnamelen = newnamelen; + if (newnamelen) { + char *p = nfsd_notify_event_newname(ne); + + memcpy(p, newname.name.name, newnamelen); + p[newnamelen] = '\0'; + } + + refcount_set(&ne->ne_ref, 1); + ne->ne_mask = mask; + ne->ne_dentry = dget(dentry); + ne->ne_target = target; + if (ne->ne_target) + ihold(ne->ne_target); +out: + if (mask & FS_RENAME) + release_dentry_name_snapshot(&newname); + return ne; +} + +static bool +should_notify_deleg(u32 mask, struct file_lease *fl) +{ + /* Don't notify the client generating the event */ + if (nfsd_breaker_owns_lease(fl)) + return false; + + /* Skip if this event wasn't ignored by the lease */ + if ((mask & FS_DELETE) && !(fl->c.flc_flags & FL_IGN_DIR_DELETE)) + return false; + if ((mask & FS_CREATE) && !(fl->c.flc_flags & FL_IGN_DIR_CREATE)) + return false; + if ((mask & FS_RENAME) && !(fl->c.flc_flags & FL_IGN_DIR_RENAME)) + return false; + + return true; +} + +static void +nfsd_recall_all_dir_delegs(const struct inode *dir) +{ + struct file_lock_context *ctx = locks_inode_context(dir); + struct file_lock_core *flc; + + spin_lock(&ctx->flc_lock); + list_for_each_entry(flc, &ctx->flc_lease, flc_list) { + struct file_lease *fl = container_of(flc, struct file_lease, c); + + if (fl->fl_lmops == &nfsd_lease_mng_ops) + nfsd_break_deleg_cb(fl); + } + spin_unlock(&ctx->flc_lock); +} + +int +nfsd_handle_dir_event(u32 mask, const struct inode *dir, const void *data, + int data_type, const struct qstr *name) +{ + struct dentry *dentry = fsnotify_data_dentry(data, data_type); + struct inode *target = fsnotify_data_rename_target(data, data_type); + struct file_lock_context *ctx; + struct file_lock_core *flc; + struct nfsd_notify_event *evt; + + trace_nfsd_handle_dir_event(mask, dir, name); + + /* Normalize cross-dir rename events to create/delete */ + if (mask & FS_MOVED_FROM) { + mask &= ~FS_MOVED_FROM; + mask |= FS_DELETE; + } + if (mask & FS_MOVED_TO) { + mask &= ~FS_MOVED_TO; + mask |= FS_CREATE; + } + + /* + * FS_RENAME fires on the source directory even for a cross-dir + * rename, where the moved entry now lives under a different parent. + * NOTIFY4_RENAME_ENTRY describes an in-place rename, so reporting it + * here would advertise a name absent from this directory. + */ + if ((mask & FS_RENAME) && dentry && d_inode(dentry->d_parent) != dir) + mask &= ~FS_RENAME; + + /* Don't do anything if this is not an expected event */ + if (!(mask & (FS_CREATE|FS_DELETE|FS_RENAME))) + return 0; + + ctx = locks_inode_context(dir); + if (!ctx || list_empty(&ctx->flc_lease)) + return 0; + + evt = alloc_nfsd_notify_event(mask, name, dentry, target); + if (!evt) { + nfsd_recall_all_dir_delegs(dir); + return 0; + } + + spin_lock(&ctx->flc_lock); + list_for_each_entry(flc, &ctx->flc_lease, flc_list) { + struct file_lease *fl = container_of(flc, struct file_lease, c); + struct nfs4_delegation *dp = flc->flc_owner; + struct nfsd4_cb_notify *ncn = &dp->dl_cb_notify; + + if (!should_notify_deleg(mask, fl)) + continue; + + spin_lock(&ncn->ncn_lock); + if (ncn->ncn_evt_cnt >= NOTIFY4_EVENT_QUEUE_SIZE) { + /* We're generating notifications too fast. Recall. */ + spin_unlock(&ncn->ncn_lock); + nfsd_break_deleg_cb(fl); + continue; + } + ncn->ncn_evt[ncn->ncn_evt_cnt++] = nfsd_notify_event_get(evt); + spin_unlock(&ncn->ncn_lock); + + nfsd4_run_cb_notify(ncn); + } + spin_unlock(&ctx->flc_lock); + nfsd_notify_event_put(evt); + return 0; +} diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index e17488a911f7..606ddcb085c0 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -47,6 +47,8 @@ #include <uapi/linux/xattr.h> +#include "attr4.h" +#include "auth.h" #include "idmap.h" #include "acl.h" #include "xdr4.h" @@ -98,7 +100,7 @@ check_filename(char *str, int len) return nfserr_inval; if (len > NFS4_MAXNAMLEN) return nfserr_nametoolong; - if (isdotent(str, len)) + if (name_is_dot_dotdot(str, len)) return nfserr_badname; for (i = 0; i < len; i++) if (str[i] == '/') @@ -244,7 +246,7 @@ nfsd4_decode_nfstime4(struct nfsd4_compoundargs *argp, struct timespec64 *tv) return nfserr_bad_xdr; p = xdr_decode_hyper(p, &tv->tv_sec); tv->tv_nsec = be32_to_cpup(p++); - if (tv->tv_nsec >= (u32)1000000000) + if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) return nfserr_inval; return nfs_ok; } @@ -449,9 +451,18 @@ nfsd4_decode_posixacl(struct nfsd4_compoundargs *argp, struct posix_acl **acl) if (xdr_stream_decode_u32(argp->xdr, &count) < 0) return nfserr_bad_xdr; + /* + * The NFSv4 POSIX ACL draft doesn't define a max number of ACE's, but + * the NFSACL spec does. For NFSv4, cap the number of entries to the v3 + * limit, as we want to ensure that ACLs set via NFSv4 POSIX ACL + * extensions are retrievable via NFSACL. + */ + if (count > NFS_ACL_MAX_ENTRIES) + return nfserr_inval; + *acl = posix_acl_alloc(count, GFP_KERNEL); if (*acl == NULL) - return nfserr_resource; + return nfserr_jukebox; (*acl)->a_count = count; for (ace = (*acl)->a_entries; ace < (*acl)->a_entries + count; ace++) { @@ -628,6 +639,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen, if (!xdrgen_decode_fattr4_time_deleg_access(argp->xdr, &access)) return nfserr_bad_xdr; + if (access.nseconds >= NSEC_PER_SEC) + return nfserr_inval; iattr->ia_atime.tv_sec = access.seconds; iattr->ia_atime.tv_nsec = access.nseconds; iattr->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET | ATTR_DELEG; @@ -637,6 +650,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen, if (!xdrgen_decode_fattr4_time_deleg_modify(argp->xdr, &modify)) return nfserr_bad_xdr; + if (modify.nseconds >= NSEC_PER_SEC) + return nfserr_inval; iattr->ia_mtime.tv_sec = modify.seconds; iattr->ia_mtime.tv_nsec = modify.nseconds; iattr->ia_ctime.tv_sec = modify.seconds; @@ -955,6 +970,10 @@ nfsd4_decode_create(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) case NF4LNK: if (xdr_stream_decode_u32(argp->xdr, &create->cr_datalen) < 0) return nfserr_bad_xdr; + if (create->cr_datalen == 0) + return nfserr_inval; + if (create->cr_datalen > NFS4_MAXPATHLEN) + return nfserr_nametoolong; p = xdr_inline_decode(argp->xdr, create->cr_datalen); if (!p) return nfserr_bad_xdr; @@ -2106,6 +2125,7 @@ static __be32 nfsd4_decode_nl4_server(struct nfsd4_compoundargs *argp, { struct nfs42_netaddr *naddr; __be32 *p; + u32 str_len; if (xdr_stream_decode_u32(argp->xdr, &ns->nl4_type) < 0) return nfserr_bad_xdr; @@ -2135,6 +2155,18 @@ static __be32 nfsd4_decode_nl4_server(struct nfsd4_compoundargs *argp, return nfserr_bad_xdr; memcpy(naddr->addr, p, naddr->addr_len); break; + case NL4_NAME: + case NL4_URL: + /* + * Well-formed XDR, but only NL4_NETADDR is supported. Consume + * the utf8str_cis to keep the stream aligned, then return + * NFS4ERR_NOTSUPP rather than the misleading NFS4ERR_BADXDR. + */ + if (xdr_stream_decode_u32(argp->xdr, &str_len) < 0) + return nfserr_bad_xdr; + if (!xdr_inline_decode(argp->xdr, str_len)) + return nfserr_bad_xdr; + return nfserr_notsupp; default: return nfserr_bad_xdr; } @@ -2702,7 +2734,7 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp) } static __be32 nfsd4_encode_nfs_fh4(struct xdr_stream *xdr, - struct knfsd_fh *fh_handle) + const struct knfsd_fh *fh_handle) { return nfsd4_encode_opaque(xdr, fh_handle->fh_raw, fh_handle->fh_size); } @@ -3142,9 +3174,9 @@ out_resource: struct nfsd4_fattr_args { struct svc_rqst *rqstp; - struct svc_fh *fhp; struct svc_export *exp; struct dentry *dentry; + struct knfsd_fh fhandle; struct kstat stat; struct kstatfs statfs; struct nfs4_acl *acl; @@ -3260,7 +3292,7 @@ static __be32 nfsd4_encode_fattr4_change(struct xdr_stream *xdr, { const struct svc_export *exp = args->exp; - if (unlikely(exp->ex_flags & NFSEXP_V4ROOT)) { + if (exp && unlikely(exp->ex_flags & NFSEXP_V4ROOT)) { u32 flush_time = convert_to_wallclock(exp->cd->flush_time); if (xdr_stream_encode_u32(xdr, flush_time) != XDR_UNIT) @@ -3292,7 +3324,7 @@ static __be32 nfsd4_encode_fattr4_fsid(struct xdr_stream *xdr, xdr_encode_hyper(p, NFS4_REFERRAL_FSID_MINOR); return nfs_ok; } - switch (fsid_source(args->fhp)) { + switch (fsid_source_fh(&args->fhandle, args->exp)) { case FSIDSOURCE_FSID: p = xdr_encode_hyper(p, (u64)args->exp->ex_fsid); xdr_encode_hyper(p, (u64)0); @@ -3389,7 +3421,7 @@ static __be32 nfsd4_encode_fattr4_homogeneous(struct xdr_stream *xdr, static __be32 nfsd4_encode_fattr4_filehandle(struct xdr_stream *xdr, const struct nfsd4_fattr_args *args) { - return nfsd4_encode_nfs_fh4(xdr, &args->fhp->fh_handle); + return nfsd4_encode_nfs_fh4(xdr, &args->fhandle); } static __be32 nfsd4_encode_fattr4_fileid(struct xdr_stream *xdr, @@ -3882,6 +3914,22 @@ static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = { #endif }; +static __be32 +nfsd4_encode_attr_vals(struct xdr_stream *xdr, u32 *attrmask, struct nfsd4_fattr_args *args) +{ + DECLARE_BITMAP(attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); + unsigned long bit; + __be32 status; + + bitmap_from_arr32(attr_bitmap, attrmask, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); + for_each_set_bit(bit, attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)) { + status = nfsd4_enc_fattr4_encode_ops[bit](xdr, args); + if (status != nfs_ok) + return status; + } + return nfs_ok; +} + /* * Note: @fhp can be NULL; in this case, we might have to compose the filehandle * ourselves. @case_cache is NULL for callers that encode a single dentry @@ -3895,7 +3943,6 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, int ignore_crossmnt, struct nfsd_case_attrs_cache *case_cache) { - DECLARE_BITMAP(attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); struct nfs4_delegation *dp = NULL; struct nfsd4_fattr_args args; struct svc_fh *tempfh = NULL; @@ -3910,7 +3957,6 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, .mnt = exp->ex_path.mnt, .dentry = dentry, }; - unsigned long bit; WARN_ON_ONCE(bmval[1] & NFSD_WRITEONLY_ATTRS_WORD1); WARN_ON_ONCE(!nfsd_attrs_supported(minorversion, bmval)); @@ -3988,19 +4034,22 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, if (err) goto out_nfserr; } - if ((attrmask[0] & (FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FSID)) && - !fhp) { - tempfh = kmalloc_obj(struct svc_fh); - status = nfserr_jukebox; - if (!tempfh) - goto out; - fh_init(tempfh, NFS4_FHSIZE); - status = fh_compose(tempfh, exp, dentry, NULL); - if (status) - goto out; - args.fhp = tempfh; - } else - args.fhp = fhp; + + if ((attrmask[0] & (FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FSID))) { + if (!fhp) { + tempfh = kmalloc_obj(struct svc_fh); + status = nfserr_jukebox; + if (!tempfh) + goto out; + fh_init(tempfh, NFS4_FHSIZE); + status = fh_compose(tempfh, exp, dentry, NULL); + if (status) + goto out; + fhp = tempfh; + } + fh_copy_shallow(&args.fhandle, &fhp->fh_handle); + } + if (attrmask[0] & (FATTR4_WORD0_CASE_INSENSITIVE | FATTR4_WORD0_CASE_PRESERVING)) { /* @@ -4124,27 +4173,22 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, #endif /* CONFIG_NFSD_V4_POSIX_ACLS */ /* attrmask */ - status = nfsd4_encode_bitmap4(xdr, attrmask[0], attrmask[1], - attrmask[2]); + status = nfsd4_encode_bitmap4(xdr, attrmask[0], attrmask[1], attrmask[2]); if (status) goto out; /* attr_vals */ attrlen_offset = xdr->buf->len; - if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT))) - goto out_resource; - bitmap_from_arr32(attr_bitmap, attrmask, - ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); - for_each_set_bit(bit, attr_bitmap, - ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)) { - status = nfsd4_enc_fattr4_encode_ops[bit](xdr, &args); - if (status != nfs_ok) - goto out; + if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT))) { + status = nfserr_resource; + goto out; } - attrlen = cpu_to_be32(xdr->buf->len - attrlen_offset - XDR_UNIT); - write_bytes_to_xdr_buf(xdr->buf, attrlen_offset, &attrlen, XDR_UNIT); - status = nfs_ok; + status = nfsd4_encode_attr_vals(xdr, attrmask, &args); + if (status == nfs_ok) { + attrlen = cpu_to_be32(xdr->buf->len - attrlen_offset - XDR_UNIT); + write_bytes_to_xdr_buf(xdr->buf, attrlen_offset, &attrlen, XDR_UNIT); + } out: #ifdef CONFIG_NFSD_V4_POSIX_ACLS if (args.dpacl) @@ -4167,9 +4211,286 @@ out: out_nfserr: status = nfserrno(err); goto out; -out_resource: - status = nfserr_resource; - goto out; +} + +static bool +setup_notify_fhandle(struct dentry *dentry, struct nfs4_delegation *dp, + struct nfsd_file *nf, struct nfsd4_fattr_args *args) +{ + struct nfs4_file *fi = dp->dl_stid.sc_file; + struct nfs4_client *clp = dp->dl_stid.sc_client; + int fileid_type, fsid_len, maxsize, flags = 0; + struct knfsd_fh *fhp = &args->fhandle; + struct inode *inode = d_inode(dentry); + struct inode *parent = NULL; + struct svc_export *exp; + struct fid *fid; + bool ret = false; + + /* + * drop_stid_export() can clear sc_export under cl_lock and drop its + * reference when the delegation is admin-revoked, concurrently with + * this callback. Grab our own reference under cl_lock so the export + * can be neither NULL-raced nor freed while we encode. + */ + spin_lock(&clp->cl_lock); + exp = dp->dl_stid.sc_export; + if (exp) + exp_get(exp); + spin_unlock(&clp->cl_lock); + + fsid_len = key_len(fi->fi_fhandle.fh_fsid_type); + fhp->fh_size = 4 + fsid_len; + + /* Copy first 4 bytes + fsid */ + memcpy(&fhp->fh_raw, &fi->fi_fhandle.fh_raw, fhp->fh_size); + + fid = (struct fid *)(fh_fsid(fhp) + fsid_len/4); + maxsize = (NFS4_FHSIZE - fhp->fh_size)/4; + + /* + * Subtree-checking exports need a connectable filehandle so the + * parent can be resolved at decode time. Derive this from the + * delegation's export rather than the shared nfs4_file, which may + * have been initialized under a different export. + */ + if (exp && !(exp->ex_flags & NFSEXP_NOSUBTREECHECK) && + !S_ISDIR(inode->i_mode)) { + parent = d_inode(nf->nf_file->f_path.dentry); + flags = EXPORT_FH_CONNECTABLE; + } + + fileid_type = exportfs_encode_inode_fh(inode, fid, &maxsize, parent, flags); + if (fileid_type < 0 || fileid_type == FILEID_INVALID) + goto out; + + fhp->fh_fileid_type = fileid_type; + fhp->fh_size += maxsize * 4; + + if (exp && (exp->ex_flags & NFSEXP_SIGN_FH)) + if (!fh_append_mac(fhp, NFS4_FHSIZE, exp->cd->net)) + goto out; + + ret = true; +out: + if (exp) + exp_put(exp); + return ret; +} + +#define CB_NOTIFY_STATX_REQUEST_MASK (STATX_BASIC_STATS | \ + STATX_BTIME | \ + STATX_CHANGE_COOKIE) + +static bool +nfsd4_setup_notify_entry4(struct notify_entry4 *ne, struct xdr_stream *xdr, + struct dentry *dentry, struct nfs4_delegation *dp, + struct nfsd_file *nf, char *name, u32 namelen) +{ + struct path path = nf->nf_file->f_path; + struct nfsd4_fattr_args args = { }; + const u32 *reqmask; + uint32_t *attrmask; + __be32 status; + bool parent; + int ret; + + /* Reserve space for attrmask */ + attrmask = xdr_reserve_space(xdr, 3 * sizeof(uint32_t)); + if (!attrmask) + return false; + + ne->ne_file.data = name; + ne->ne_file.len = namelen; + ne->ne_attrs.attrmask.element = attrmask; + + parent = (dentry == path.dentry); + path.dentry = dentry; + reqmask = parent ? dp->dl_dir_attrs : dp->dl_child_attrs; + + /* + * A NULL or negative dentry has no attributes to report (expected, + * e.g. for the old entry of a rename or an entry already removed). + * The client may also have been granted the notification while + * requesting no attributes for this entry. Both cases encode an + * empty attribute set rather than failing: the vfs_getattr() and + * nfsd4_encode_attr_vals() failures below recall the delegation, so + * a case with nothing to fetch must short-circuit ahead of them. + */ + if (!path.dentry || !d_inode(path.dentry) || + (!reqmask[0] && !reqmask[1])) { + attrmask[0] = 0; + attrmask[1] = 0; + attrmask[2] = 0; + ne->ne_attrs.attr_vals.data = NULL; + ne->ne_attrs.attr_vals.len = 0; + ne->ne_attrs.attrmask.count = 1; + return true; + } + + /* + * It is possible that the client was granted a delegation when a file + * was created. Note that we don't issue a CB_GETATTR here since stale + * attributes are presumably ok. + */ + ret = vfs_getattr(&path, &args.stat, CB_NOTIFY_STATX_REQUEST_MASK, AT_STATX_SYNC_AS_STAT); + if (ret) + return false; + + args.change_attr = nfsd4_change_attribute(&args.stat); + + if (parent) { + attrmask[0] = dp->dl_dir_attrs[0]; + attrmask[1] = dp->dl_dir_attrs[1]; + } else { + attrmask[0] = dp->dl_child_attrs[0]; + attrmask[1] = dp->dl_child_attrs[1]; + + if (!setup_notify_fhandle(dentry, dp, nf, &args)) + attrmask[0] &= ~FATTR4_WORD0_FILEHANDLE; + + if (!(args.stat.result_mask & STATX_BTIME)) + attrmask[1] &= ~FATTR4_WORD1_TIME_CREATE; + } + attrmask[2] = 0; + + ne->ne_attrs.attrmask.count = 2; + ne->ne_attrs.attr_vals.data = (u8 *)xdr->p; + + status = nfsd4_encode_attr_vals(xdr, attrmask, &args); + if (status != nfs_ok) + return false; + + ne->ne_attrs.attr_vals.len = (u8 *)xdr->p - ne->ne_attrs.attr_vals.data; + return true; +} + +/** + * nfsd4_encode_notify_event - encode a notify + * @xdr: stream to which to encode the fattr4 + * @nne: nfsd_notify_event to encode + * @dp: delegation where the event occurred + * @nf: nfsd_file on which event occurred + * @notify_mask: pointer to word where notification mask should be set + * + * Encode @nne into @xdr. The matching bit in @notify_mask is set on + * success. + * + * Return: pointer to the start of the encoded event, or NULL if the + * event could not be encoded. + */ +u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *nne, + struct nfs4_delegation *dp, struct nfsd_file *nf, + u32 *notify_mask) +{ + u8 *p = NULL; + + *notify_mask = 0; + + if (nne->ne_mask & FS_DELETE) { + struct notify_remove4 nr = { }; + + if (!nfsd4_setup_notify_entry4(&nr.nrm_old_entry, xdr, nne->ne_dentry, dp, + nf, nne->ne_name, nne->ne_namelen)) + goto out_err; + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_remove4(xdr, &nr)) + goto out_err; + *notify_mask |= BIT(NOTIFY4_REMOVE_ENTRY); + } else if (nne->ne_mask & FS_CREATE) { + struct notify_add4 na = { }; + struct notify_remove4 old = { }; + + if (!nfsd4_setup_notify_entry4(&na.nad_new_entry, xdr, nne->ne_dentry, dp, + nf, nne->ne_name, nne->ne_namelen)) + goto out_err; + + /* If a file was overwritten, report it in nad_old_entry */ + if (nne->ne_target) { + if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr, + NULL, dp, nf, + nne->ne_name, nne->ne_namelen)) + goto out_err; + na.nad_old_entry.count = 1; + na.nad_old_entry.element = &old; + } + + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_add4(xdr, &na)) + goto out_err; + + *notify_mask |= BIT(NOTIFY4_ADD_ENTRY); + } else if (nne->ne_mask & FS_RENAME) { + struct notify_rename4 nr = { }; + struct notify_remove4 old = { }; + char *newname = nfsd_notify_event_newname(nne); + + /* Don't send any attributes in the old_entry since they're the same in new */ + if (!nfsd4_setup_notify_entry4(&nr.nrn_old_entry.nrm_old_entry, xdr, + NULL, dp, nf, nne->ne_name, + nne->ne_namelen)) + goto out_err; + + if (!nfsd4_setup_notify_entry4(&nr.nrn_new_entry.nad_new_entry, xdr, + nne->ne_dentry, dp, nf, newname, + nne->ne_newnamelen)) + goto out_err; + + /* If a file was overwritten, report it in nad_old_entry */ + if (nne->ne_target) { + if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr, + NULL, dp, nf, newname, + nne->ne_newnamelen)) + goto out_err; + nr.nrn_new_entry.nad_old_entry.count = 1; + nr.nrn_new_entry.nad_old_entry.element = &old; + } + + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_rename4(xdr, &nr)) + goto out_err; + *notify_mask |= BIT(NOTIFY4_RENAME_ENTRY); + } + return p; +out_err: + pr_warn("nfsd: unable to marshal notify event to xdr stream\n"); + return NULL; +} + +/** + * nfsd4_encode_dir_attr_change + * @xdr: stream to which to encode the fattr4 + * @dp: delegation where the event occurred + * @nf: nfsd_file opened on the directory + * + * Encode a dir attr change event. + * + * Return: a pointer to the start of the encoded event on success; NULL + * if there were no requested attributes to report, in which case the + * caller should omit the event; or an ERR_PTR if the event was requested + * but could not be marshalled into @xdr, in which case the caller should + * recall the delegation. + */ +u8 *nfsd4_encode_dir_attr_change(struct xdr_stream *xdr, struct nfs4_delegation *dp, + struct nfsd_file *nf) +{ + struct dentry *dentry = nf->nf_file->f_path.dentry; + struct notify_attr4 na = { }; + u8 *p; + + /* RFC 8881 s10.4.3: ne_file must be a zero-length string for dir attrs */ + if (!nfsd4_setup_notify_entry4(&na.na_changed_entry, xdr, + dentry, dp, nf, "", 0)) + return ERR_PTR(-ENOBUFS); + + /* No requested attributes to report; omit the event */ + if (!na.na_changed_entry.ne_attrs.attr_vals.len) + return NULL; + + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_attr4(xdr, &na)) + return ERR_PTR(-ENOBUFS); + return p; } static void svcxdr_init_encode_from_buffer(struct xdr_stream *xdr, @@ -4323,7 +4644,7 @@ nfsd4_encode_entry4(void *ccdv, const char *name, int namlen, __be32 nfserr = nfserr_toosmall; /* In nfsv4, "." and ".." never make it onto the wire.. */ - if (name && isdotent(name, namlen)) { + if (name && name_is_dot_dotdot(name, namlen)) { cd->common.err = nfs_ok; return 0; } @@ -6390,9 +6711,6 @@ status: write_bytes_to_xdr_buf(xdr->buf, op_status_offset, &op->status, XDR_UNIT); release: - if (opdesc && opdesc->op_release) - opdesc->op_release(&op->u); - /* * Account for pages consumed while encoding this operation. * The xdr_stream primitives don't manage rq_next_page. @@ -6424,9 +6742,12 @@ void nfsd4_release_compoundargs(struct svc_rqst *rqstp) { struct nfsd4_compoundargs *args = rqstp->rq_argp; + args->opcnt = 0; if (args->ops != args->iops) { - vfree(args->ops); + void *old_ops = args->ops; + args->ops = args->iops; + kvfree_rcu_mightsleep(old_ops); } while (args->to_free) { struct svcxdr_tmpbuf *tb = args->to_free; diff --git a/fs/nfsd/nfs4xdr_gen.c b/fs/nfsd/nfs4xdr_gen.c index 824497051b87..d77835033a8c 100644 --- a/fs/nfsd/nfs4xdr_gen.c +++ b/fs/nfsd/nfs4xdr_gen.c @@ -1,16 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x -// XDR specification modification time: Thu Jan 8 23:12:07 2026 +// XDR specification modification time: Tue Jun 30 11:57:21 2026 #include <linux/sunrpc/svc.h> #include "nfs4xdr_gen.h" static bool __maybe_unused -xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr) +xdrgen_decode_int32_t(struct xdr_stream *xdr, int32_t *ptr) { - return xdrgen_decode_hyper(xdr, ptr); + return xdrgen_decode_int(xdr, ptr); } static bool __maybe_unused @@ -20,6 +20,154 @@ xdrgen_decode_uint32_t(struct xdr_stream *xdr, uint32_t *ptr) } static bool __maybe_unused +xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr) +{ + return xdrgen_decode_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_uint64_t(struct xdr_stream *xdr, uint64_t *ptr) +{ + return xdrgen_decode_unsigned_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfsstat4(struct xdr_stream *xdr, nfsstat4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NFS4_OK: + case NFS4ERR_PERM: + case NFS4ERR_NOENT: + case NFS4ERR_IO: + case NFS4ERR_NXIO: + case NFS4ERR_ACCESS: + case NFS4ERR_EXIST: + case NFS4ERR_XDEV: + case NFS4ERR_NOTDIR: + case NFS4ERR_ISDIR: + case NFS4ERR_INVAL: + case NFS4ERR_FBIG: + case NFS4ERR_NOSPC: + case NFS4ERR_ROFS: + case NFS4ERR_MLINK: + case NFS4ERR_NAMETOOLONG: + case NFS4ERR_NOTEMPTY: + case NFS4ERR_DQUOT: + case NFS4ERR_STALE: + case NFS4ERR_BADHANDLE: + case NFS4ERR_BAD_COOKIE: + case NFS4ERR_NOTSUPP: + case NFS4ERR_TOOSMALL: + case NFS4ERR_SERVERFAULT: + case NFS4ERR_BADTYPE: + case NFS4ERR_DELAY: + case NFS4ERR_SAME: + case NFS4ERR_DENIED: + case NFS4ERR_EXPIRED: + case NFS4ERR_LOCKED: + case NFS4ERR_GRACE: + case NFS4ERR_FHEXPIRED: + case NFS4ERR_SHARE_DENIED: + case NFS4ERR_WRONGSEC: + case NFS4ERR_CLID_INUSE: + case NFS4ERR_RESOURCE: + case NFS4ERR_MOVED: + case NFS4ERR_NOFILEHANDLE: + case NFS4ERR_MINOR_VERS_MISMATCH: + case NFS4ERR_STALE_CLIENTID: + case NFS4ERR_STALE_STATEID: + case NFS4ERR_OLD_STATEID: + case NFS4ERR_BAD_STATEID: + case NFS4ERR_BAD_SEQID: + case NFS4ERR_NOT_SAME: + case NFS4ERR_LOCK_RANGE: + case NFS4ERR_SYMLINK: + case NFS4ERR_RESTOREFH: + case NFS4ERR_LEASE_MOVED: + case NFS4ERR_ATTRNOTSUPP: + case NFS4ERR_NO_GRACE: + case NFS4ERR_RECLAIM_BAD: + case NFS4ERR_RECLAIM_CONFLICT: + case NFS4ERR_BADXDR: + case NFS4ERR_LOCKS_HELD: + case NFS4ERR_OPENMODE: + case NFS4ERR_BADOWNER: + case NFS4ERR_BADCHAR: + case NFS4ERR_BADNAME: + case NFS4ERR_BAD_RANGE: + case NFS4ERR_LOCK_NOTSUPP: + case NFS4ERR_OP_ILLEGAL: + case NFS4ERR_DEADLOCK: + case NFS4ERR_FILE_OPEN: + case NFS4ERR_ADMIN_REVOKED: + case NFS4ERR_CB_PATH_DOWN: + case NFS4ERR_BADIOMODE: + case NFS4ERR_BADLAYOUT: + case NFS4ERR_BAD_SESSION_DIGEST: + case NFS4ERR_BADSESSION: + case NFS4ERR_BADSLOT: + case NFS4ERR_COMPLETE_ALREADY: + case NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case NFS4ERR_DELEG_ALREADY_WANTED: + case NFS4ERR_BACK_CHAN_BUSY: + case NFS4ERR_LAYOUTTRYLATER: + case NFS4ERR_LAYOUTUNAVAILABLE: + case NFS4ERR_NOMATCHING_LAYOUT: + case NFS4ERR_RECALLCONFLICT: + case NFS4ERR_UNKNOWN_LAYOUTTYPE: + case NFS4ERR_SEQ_MISORDERED: + case NFS4ERR_SEQUENCE_POS: + case NFS4ERR_REQ_TOO_BIG: + case NFS4ERR_REP_TOO_BIG: + case NFS4ERR_REP_TOO_BIG_TO_CACHE: + case NFS4ERR_RETRY_UNCACHED_REP: + case NFS4ERR_UNSAFE_COMPOUND: + case NFS4ERR_TOO_MANY_OPS: + case NFS4ERR_OP_NOT_IN_SESSION: + case NFS4ERR_HASH_ALG_UNSUPP: + case NFS4ERR_CLIENTID_BUSY: + case NFS4ERR_PNFS_IO_HOLE: + case NFS4ERR_SEQ_FALSE_RETRY: + case NFS4ERR_BAD_HIGH_SLOT: + case NFS4ERR_DEADSESSION: + case NFS4ERR_ENCR_ALG_UNSUPP: + case NFS4ERR_PNFS_NO_LAYOUT: + case NFS4ERR_NOT_ONLY_OP: + case NFS4ERR_WRONG_CRED: + case NFS4ERR_WRONG_TYPE: + case NFS4ERR_DIRDELEG_UNAVAIL: + case NFS4ERR_REJECT_DELEG: + case NFS4ERR_RETURNCONFLICT: + case NFS4ERR_DELEG_REVOKED: + case NFS4ERR_PARTNER_NOTSUPP: + case NFS4ERR_PARTNER_NO_AUTH: + case NFS4ERR_UNION_NOTSUPP: + case NFS4ERR_OFFLOAD_DENIED: + case NFS4ERR_WRONG_LFS: + case NFS4ERR_BADLABEL: + case NFS4ERR_OFFLOAD_NO_REQS: + case NFS4ERR_NOXATTR: + case NFS4ERR_XATTR2BIG: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_attrlist4(struct xdr_stream *xdr, attrlist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + +static bool __maybe_unused xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr) { if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) @@ -31,6 +179,24 @@ xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr) } static bool __maybe_unused +xdrgen_decode_verifier4(struct xdr_stream *xdr, verifier4 *ptr) +{ + return xdr_stream_decode_opaque_fixed(xdr, ptr, NFS4_VERIFIER_SIZE) == 0; +} + +static bool __maybe_unused +xdrgen_decode_nfs_cookie4(struct xdr_stream *xdr, nfs_cookie4 *ptr) +{ + return xdrgen_decode_uint64_t(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfs_fh4(struct xdr_stream *xdr, nfs_fh4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, NFS4_FHSIZE); +} + +static bool __maybe_unused xdrgen_decode_utf8string(struct xdr_stream *xdr, utf8string *ptr) { return xdrgen_decode_opaque(xdr, ptr, 0); @@ -55,6 +221,29 @@ xdrgen_decode_utf8str_mixed(struct xdr_stream *xdr, utf8str_mixed *ptr) } static bool __maybe_unused +xdrgen_decode_component4(struct xdr_stream *xdr, component4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_linktext4(struct xdr_stream *xdr, linktext4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_pathname4(struct xdr_stream *xdr, pathname4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_component4(xdr, &ptr->element[i])) + return false; + return true; +} + +static bool __maybe_unused xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr) { if (!xdrgen_decode_int64_t(xdr, &ptr->seconds)) @@ -65,6 +254,26 @@ xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr) } static bool __maybe_unused +xdrgen_decode_fattr4(struct xdr_stream *xdr, struct fattr4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->attrmask)) + return false; + if (!xdrgen_decode_attrlist4(xdr, &ptr->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_stateid4(struct xdr_stream *xdr, struct stateid4 *ptr) +{ + if (!xdrgen_decode_uint32_t(xdr, &ptr->seqid)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->other, 12) < 0) + return false; + return true; +} + +static bool __maybe_unused xdrgen_decode_fattr4_offline(struct xdr_stream *xdr, fattr4_offline *ptr) { return xdrgen_decode_bool(xdr, ptr); @@ -366,9 +575,171 @@ xdrgen_decode_fattr4_posix_access_acl(struct xdr_stream *xdr, fattr4_posix_acces */ static bool __maybe_unused -xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value) +xdrgen_decode_notify_type4(struct xdr_stream *xdr, notify_type4 *ptr) { - return xdrgen_encode_hyper(xdr, value); + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NOTIFY4_CHANGE_CHILD_ATTRS: + case NOTIFY4_CHANGE_DIR_ATTRS: + case NOTIFY4_REMOVE_ENTRY: + case NOTIFY4_ADD_ENTRY: + case NOTIFY4_RENAME_ENTRY: + case NOTIFY4_CHANGE_COOKIE_VERIFIER: + case NOTIFY4_GFLAG_EXTEND: + case NOTIFY4_AUFLAG_VALID: + case NOTIFY4_AUFLAG_USER: + case NOTIFY4_AUFLAG_GROUP: + case NOTIFY4_AUFLAG_OTHER: + case NOTIFY4_CHANGE_AUTH: + case NOTIFY4_CFLAG_ORDER: + case NOTIFY4_AUFLAG_GANOW: + case NOTIFY4_AUFLAG_GALATER: + case NOTIFY4_CHANGE_GA: + case NOTIFY4_CHANGE_AMASK: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_entry4(struct xdr_stream *xdr, struct notify_entry4 *ptr) +{ + if (!xdrgen_decode_component4(xdr, &ptr->ne_file)) + return false; + if (!xdrgen_decode_fattr4(xdr, &ptr->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_prev_entry4(struct xdr_stream *xdr, struct prev_entry4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->pe_prev_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->pe_prev_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nrm_old_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nrm_old_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->nad_old_entry.count) < 0) + return false; + if (ptr->nad_old_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_old_entry.count; i++) + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nad_old_entry.element[i])) + return false; + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nad_new_entry)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_new_entry_cookie.count) < 0) + return false; + if (ptr->nad_new_entry_cookie.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_new_entry_cookie.count; i++) + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nad_new_entry_cookie.element[i])) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_prev_entry.count) < 0) + return false; + if (ptr->nad_prev_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_prev_entry.count; i++) + if (!xdrgen_decode_prev_entry4(xdr, &ptr->nad_prev_entry.element[i])) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->nad_last_entry)) + return false; + return true; +} + +bool +xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->na_changed_entry)) + return false; + return true; +} + +bool +xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr) +{ + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nrn_old_entry)) + return false; + if (!xdrgen_decode_notify_add4(xdr, &ptr->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_verifier4(struct xdr_stream *xdr, struct notify_verifier4 *ptr) +{ + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_old_cookieverf)) + return false; + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notifylist4(struct xdr_stream *xdr, notifylist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + +static bool __maybe_unused +xdrgen_decode_notify4(struct xdr_stream *xdr, struct notify4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->notify_mask)) + return false; + if (!xdrgen_decode_notifylist4(xdr, &ptr->notify_vals)) + return false; + return true; +} + +bool +xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr) +{ + if (!xdrgen_decode_stateid4(xdr, &ptr->cna_stateid)) + return false; + if (!xdrgen_decode_nfs_fh4(xdr, &ptr->cna_fh)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->cna_changes.count) < 0) + return false; + for (u32 i = 0; i < ptr->cna_changes.count; i++) + if (!xdrgen_decode_notify4(xdr, &ptr->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_CB_NOTIFY4res(struct xdr_stream *xdr, struct CB_NOTIFY4res *ptr) +{ + if (!xdrgen_decode_nfsstat4(xdr, &ptr->cnr_status)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_int32_t(struct xdr_stream *xdr, const int32_t value) +{ + return xdrgen_encode_int(xdr, value); } static bool __maybe_unused @@ -378,6 +749,30 @@ xdrgen_encode_uint32_t(struct xdr_stream *xdr, const uint32_t value) } static bool __maybe_unused +xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value) +{ + return xdrgen_encode_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_uint64_t(struct xdr_stream *xdr, const uint64_t value) +{ + return xdrgen_encode_unsigned_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfsstat4(struct xdr_stream *xdr, nfsstat4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_attrlist4(struct xdr_stream *xdr, const attrlist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value) { if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) @@ -389,6 +784,24 @@ xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value) } static bool __maybe_unused +xdrgen_encode_verifier4(struct xdr_stream *xdr, const verifier4 value) +{ + return xdr_stream_encode_opaque_fixed(xdr, value, NFS4_VERIFIER_SIZE) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_nfs_cookie4(struct xdr_stream *xdr, const nfs_cookie4 value) +{ + return xdrgen_encode_uint64_t(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfs_fh4(struct xdr_stream *xdr, const nfs_fh4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused xdrgen_encode_utf8string(struct xdr_stream *xdr, const utf8string value) { return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; @@ -413,6 +826,29 @@ xdrgen_encode_utf8str_mixed(struct xdr_stream *xdr, const utf8str_mixed value) } static bool __maybe_unused +xdrgen_encode_component4(struct xdr_stream *xdr, const component4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_linktext4(struct xdr_stream *xdr, const linktext4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_pathname4(struct xdr_stream *xdr, const pathname4 value) +{ + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) + if (!xdrgen_encode_component4(xdr, value.element[i])) + return false; + return true; +} + +static bool __maybe_unused xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value) { if (!xdrgen_encode_int64_t(xdr, value->seconds)) @@ -423,6 +859,26 @@ xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value) } static bool __maybe_unused +xdrgen_encode_fattr4(struct xdr_stream *xdr, const struct fattr4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->attrmask)) + return false; + if (!xdrgen_encode_attrlist4(xdr, value->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_stateid4(struct xdr_stream *xdr, const struct stateid4 *value) +{ + if (!xdrgen_encode_uint32_t(xdr, value->seqid)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->other, 12) < 0) + return false; + return true; +} + +static bool __maybe_unused xdrgen_encode_fattr4_offline(struct xdr_stream *xdr, const fattr4_offline value) { return xdrgen_encode_bool(xdr, value); @@ -567,3 +1023,137 @@ xdrgen_encode_fattr4_posix_access_acl(struct xdr_stream *xdr, const fattr4_posix return false; return true; } + +static bool __maybe_unused +xdrgen_encode_notify_type4(struct xdr_stream *xdr, notify_type4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_notify_entry4(struct xdr_stream *xdr, const struct notify_entry4 *value) +{ + if (!xdrgen_encode_component4(xdr, value->ne_file)) + return false; + if (!xdrgen_encode_fattr4(xdr, &value->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_prev_entry4(struct xdr_stream *xdr, const struct prev_entry4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->pe_prev_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->pe_prev_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->nrm_old_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->nrm_old_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value) +{ + if (value->nad_old_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_old_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_old_entry.count; i++) + if (!xdrgen_encode_notify_remove4(xdr, &value->nad_old_entry.element[i])) + return false; + if (!xdrgen_encode_notify_entry4(xdr, &value->nad_new_entry)) + return false; + if (value->nad_new_entry_cookie.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_new_entry_cookie.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_new_entry_cookie.count; i++) + if (!xdrgen_encode_nfs_cookie4(xdr, value->nad_new_entry_cookie.element[i])) + return false; + if (value->nad_prev_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_prev_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_prev_entry.count; i++) + if (!xdrgen_encode_prev_entry4(xdr, &value->nad_prev_entry.element[i])) + return false; + if (!xdrgen_encode_bool(xdr, value->nad_last_entry)) + return false; + return true; +} + +bool +xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->na_changed_entry)) + return false; + return true; +} + +bool +xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value) +{ + if (!xdrgen_encode_notify_remove4(xdr, &value->nrn_old_entry)) + return false; + if (!xdrgen_encode_notify_add4(xdr, &value->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_verifier4(struct xdr_stream *xdr, const struct notify_verifier4 *value) +{ + if (!xdrgen_encode_verifier4(xdr, value->nv_old_cookieverf)) + return false; + if (!xdrgen_encode_verifier4(xdr, value->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notifylist4(struct xdr_stream *xdr, const notifylist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_notify4(struct xdr_stream *xdr, const struct notify4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->notify_mask)) + return false; + if (!xdrgen_encode_notifylist4(xdr, value->notify_vals)) + return false; + return true; +} + +bool +xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value) +{ + if (!xdrgen_encode_stateid4(xdr, &value->cna_stateid)) + return false; + if (!xdrgen_encode_nfs_fh4(xdr, value->cna_fh)) + return false; + if (xdr_stream_encode_u32(xdr, value->cna_changes.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->cna_changes.count; i++) + if (!xdrgen_encode_notify4(xdr, &value->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_CB_NOTIFY4res(struct xdr_stream *xdr, const struct CB_NOTIFY4res *value) +{ + if (!xdrgen_encode_nfsstat4(xdr, value->cnr_status)) + return false; + return true; +} diff --git a/fs/nfsd/nfs4xdr_gen.h b/fs/nfsd/nfs4xdr_gen.h index 1c487f1a11ab..21ca82078615 100644 --- a/fs/nfsd/nfs4xdr_gen.h +++ b/fs/nfsd/nfs4xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ -/* XDR specification modification time: Thu Jan 8 23:12:07 2026 */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ #ifndef _LINUX_XDRGEN_NFS4_1_DECL_H #define _LINUX_XDRGEN_NFS4_1_DECL_H @@ -21,10 +21,13 @@ bool xdrgen_encode_fattr4_time_deleg_access(struct xdr_stream *xdr, const fattr4 bool xdrgen_decode_fattr4_time_deleg_modify(struct xdr_stream *xdr, fattr4_time_deleg_modify *ptr); bool xdrgen_encode_fattr4_time_deleg_modify(struct xdr_stream *xdr, const fattr4_time_deleg_modify *value); + bool xdrgen_decode_aclmodel4(struct xdr_stream *xdr, aclmodel4 *ptr); bool xdrgen_encode_aclmodel4(struct xdr_stream *xdr, aclmodel4 value); + bool xdrgen_decode_aclscope4(struct xdr_stream *xdr, aclscope4 *ptr); bool xdrgen_encode_aclscope4(struct xdr_stream *xdr, aclscope4 value); + bool xdrgen_decode_posixacetag4(struct xdr_stream *xdr, posixacetag4 *ptr); bool xdrgen_encode_posixacetag4(struct xdr_stream *xdr, posixacetag4 value); @@ -32,4 +35,19 @@ bool xdrgen_decode_posixaceperm4(struct xdr_stream *xdr, posixaceperm4 *ptr); bool xdrgen_encode_posixaceperm4(struct xdr_stream *xdr, const posixaceperm4 value); +bool xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr); +bool xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value); + +bool xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr); +bool xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value); + +bool xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr); +bool xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value); + +bool xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr); +bool xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value); + +bool xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr); +bool xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value); + #endif /* _LINUX_XDRGEN_NFS4_1_DECL_H */ diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index 154468ceccdc..c7db532c8523 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -19,6 +19,8 @@ #include <net/checksum.h> #include "nfsd.h" +#include "netns.h" +#include "stats.h" #include "cache.h" #include "trace.h" @@ -200,14 +202,14 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) nn->nfsd_reply_cache_shrinker->seeks = 1; nn->nfsd_reply_cache_shrinker->private_data = nn; - shrinker_register(nn->nfsd_reply_cache_shrinker); - for (i = 0; i < hashsize; i++) { INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head); spin_lock_init(&nn->drc_hashtbl[i].cache_lock); } nn->drc_hashsize = hashsize; + shrinker_register(nn->nfsd_reply_cache_shrinker); + return 0; out_shrinker: kvfree(nn->drc_hashtbl); @@ -275,7 +277,7 @@ nfsd_prune_bucket_locked(struct nfsd_net *nn, struct nfsd_drc_bucket *b, nfsd_cacherep_unlink_locked(nn, b, rp); list_add(&rp->c_lru, dispose); - if (max && ++freed > max) + if (max && ++freed >= max) break; } } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index fa92e31d19d6..adb032b7311a 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -23,6 +23,8 @@ #include "idmap.h" #include "nfsd.h" +#include "netns.h" +#include "stats.h" #include "cache.h" #include "state.h" #include "netns.h" @@ -296,14 +298,15 @@ static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size) * 2. Is that directory a mount point, or * 3. Is that directory the root of an exported file system? */ - nfsd4_cancel_copy_by_sb(netns(file), path.dentry->d_sb); error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); mutex_lock(&nfsd_mutex); nn = net_generic(netns(file), nfsd_net_id); - if (nn->nfsd_serv) + if (test_bit(NFSD_NET_UP, &nn->flags)) { + nfsd4_cancel_copy_by_sb(netns(file), path.dentry->d_sb); nfsd4_revoke_states(nn, path.dentry->d_sb); - else + } else { error = -EINVAL; + } mutex_unlock(&nfsd_mutex); path_put(&path); @@ -420,6 +423,7 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size) char *mesg = buf; int rv; struct net *net = netns(file); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); if (size > 0) { int newthreads; @@ -430,7 +434,10 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size) return -EINVAL; trace_nfsd_ctl_threads(net, newthreads); mutex_lock(&nfsd_mutex); - rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL); + if (newthreads > 0 || nn->nfsd_serv != NULL) + rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL); + else + rv = 0; mutex_unlock(&nfsd_mutex); if (rv < 0) return rv; @@ -1111,7 +1118,7 @@ static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size) } return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%c\n", - nn->grace_ended ? 'Y' : 'N'); + test_bit(NFSD_NET_GRACE_ENDED, &nn->flags) ? 'Y' : 'N'); } #endif @@ -1414,8 +1421,8 @@ static int create_proc_exports_entry(void) unsigned int nfsd_net_id; struct nfsd_genl_rqstp { - struct sockaddr rq_daddr; - struct sockaddr rq_saddr; + struct sockaddr_storage rq_daddr; + struct sockaddr_storage rq_saddr; unsigned long rq_flags; ktime_t rq_stime; __be32 rq_xid; @@ -1448,9 +1455,9 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, nla_put_s64(skb, NFSD_A_RPC_STATUS_SERVICE_TIME, ktime_to_us(genl_rqstp->rq_stime), NFSD_A_RPC_STATUS_PAD)) - return -ENOBUFS; + goto out_cancel; - switch (genl_rqstp->rq_saddr.sa_family) { + switch (genl_rqstp->rq_saddr.ss_family) { case AF_INET: { const struct sockaddr_in *s_in, *d_in; @@ -1464,7 +1471,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, s_in->sin_port) || nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, d_in->sin_port)) - return -ENOBUFS; + goto out_cancel; break; } case AF_INET6: { @@ -1480,7 +1487,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, s_in->sin6_port) || nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, d_in->sin6_port)) - return -ENOBUFS; + goto out_cancel; break; } } @@ -1488,10 +1495,14 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, for (i = 0; i < genl_rqstp->rq_opcnt; i++) if (nla_put_u32(skb, NFSD_A_RPC_STATUS_COMPOUND_OPS, genl_rqstp->rq_opnum[i])) - return -ENOBUFS; + goto out_cancel; genlmsg_end(skb, hdr); return 0; + +out_cancel: + genlmsg_cancel(skb, hdr); + return -ENOBUFS; } /** @@ -1517,20 +1528,30 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, rcu_read_lock(); - for (i = 0; i < nn->nfsd_serv->sv_nrpools; i++) { + for (i = 0; i < svc_serv_nrpools(nn->nfsd_serv); i++) { struct svc_rqst *rqstp; + long thread_skip = 0; if (i < cb->args[0]) /* already consumed */ continue; + /* + * The saved thread index only applies to the pool the dump + * was resumed in. Subsequent pools must start from thread 0, + * otherwise their first cb->args[1] threads are silently + * skipped. + */ + if (i == cb->args[0]) + thread_skip = cb->args[1]; + rqstp_index = 0; list_for_each_entry_rcu(rqstp, &nn->nfsd_serv->sv_pools[i].sp_all_threads, rq_all) { - struct nfsd_genl_rqstp genl_rqstp; + struct nfsd_genl_rqstp genl_rqstp = {}; unsigned int status_counter; - if (rqstp_index++ < cb->args[1]) /* already consumed */ + if (rqstp_index++ < thread_skip) /* already consumed */ continue; /* * Acquire rq_status_counter before parsing the rqst @@ -1551,9 +1572,9 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, genl_rqstp.rq_stime = rqstp->rq_stime; genl_rqstp.rq_opcnt = 0; memcpy(&genl_rqstp.rq_daddr, svc_daddr(rqstp), - sizeof(struct sockaddr)); + sizeof(struct sockaddr_storage)); memcpy(&genl_rqstp.rq_saddr, svc_addr(rqstp), - sizeof(struct sockaddr)); + sizeof(struct sockaddr_storage)); #ifdef CONFIG_NFSD_V4 if (rqstp->rq_vers == NFS4_VERSION && @@ -1572,17 +1593,26 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, #endif /* CONFIG_NFSD_V4 */ /* - * Acquire rq_status_counter before reporting the rqst - * fields to the user. + * Read-side load-load fence: order the field reads + * above before the counter re-read below, mirroring + * the smp_rmb() in the standard seqcount retry. The + * begin-side smp_load_acquire() above pairs with the + * smp_store_release() in nfsd_dispatch(). */ - if (smp_load_acquire(&rqstp->rq_status_counter) != - status_counter) + smp_rmb(); + if (READ_ONCE(rqstp->rq_status_counter) != status_counter) continue; ret = nfsd_genl_rpc_status_compose_msg(skb, cb, &genl_rqstp); - if (ret) + if (ret) { + if (skb->len) { + cb->args[0] = i; + cb->args[1] = rqstp_index - 1; + ret = skb->len; + } goto out; + } } } @@ -1944,6 +1974,60 @@ err_free_msg: } /** + * nfsd_nl_validate_listeners - sanity-check the listener list from userland + * @info: netlink metadata and command arguments + * + * Walk every NFSD_A_SERVER_SOCK_ADDR attribute and confirm that each entry + * is well-formed: it parses against the policy, carries both an address and + * a transport name, and the address is long enough for its family. Doing + * this up front lets the callers below assume every entry is valid and + * guarantees we make no changes when the request is malformed. + * + * Return: 0 if every entry is valid, or a negative errno otherwise. + */ +static int nfsd_nl_validate_listeners(struct genl_info *info) +{ + const struct nlattr *attr; + int rem; + + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_SOCK_ADDR, info->nlhdr, + GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + struct sockaddr *sa; + int err; + + err = nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack); + if (err < 0) + return err; + + if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) + return -EINVAL; + + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(sa->sa_family)) + return -EINVAL; + + switch (sa->sa_family) { + case AF_INET: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in)) + return -EINVAL; + break; + case AF_INET6: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in6)) + return -EINVAL; + break; + default: + return -EAFNOSUPPORT; + } + } + + return 0; +} + +/** * nfsd_nl_listener_set_doit - set the nfs running sockets * @skb: reply buffer * @info: netlink metadata and command arguments @@ -1961,6 +2045,15 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) bool delete = false; int err, rem; + /* + * Validate the entire listener list before making any changes, so a + * malformed request fails cleanly without creating a serv or touching + * the existing listeners. + */ + err = nfsd_nl_validate_listeners(info); + if (err) + return err; + mutex_lock(&nfsd_mutex); err = nfsd_create_serv(net); @@ -1987,16 +2080,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) const char *xcl_name; struct sockaddr *sa; + /* validated up front in nfsd_nl_validate_listeners() */ if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, nfsd_sock_nl_policy, info->extack) < 0) continue; - if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) - continue; - - if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) - continue; - xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); sa = nla_data(tb[NFSD_A_SOCK_ADDR]); @@ -2048,16 +2136,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) struct sockaddr *sa; int ret; + /* validated up front in nfsd_nl_validate_listeners() */ if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, nfsd_sock_nl_policy, info->extack) < 0) continue; - if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) - continue; - - if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) - continue; - xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); sa = nla_data(tb[NFSD_A_SOCK_ADDR]); @@ -2248,6 +2331,255 @@ int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) return 0; } +/* Emit a single server-proc-entry nest: { op, count }. */ +static int nfsd_nl_put_proc_entry(struct sk_buff *skb, int attr, + u32 op, u64 count) +{ + struct nlattr *nest; + + nest = nla_nest_start(skb, attr); + if (!nest) + return -EMSGSIZE; + if (nla_put_u32(skb, NFSD_A_SERVER_PROC_ENTRY_OP, op) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_PROC_ENTRY_COUNT, + count, NFSD_A_SERVER_PROC_ENTRY_PAD)) { + nla_nest_cancel(skb, nest); + return -EMSGSIZE; + } + nla_nest_end(skb, nest); + return 0; +} + +/* Emit the scalar server-stats counters. Only ever called on a fresh skb. */ +static int nfsd_nl_server_stats_scalars(struct sk_buff *skb, + struct nfsd_net *nn, + struct svc_stat *statp) +{ + if (nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_RC_HITS, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_HITS]), + NFSD_A_SERVER_STATS_PAD) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_RC_MISSES, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_MISSES]), + NFSD_A_SERVER_STATS_PAD) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_RC_NOCACHE, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_NOCACHE]), + NFSD_A_SERVER_STATS_PAD)) + return -EMSGSIZE; + + if (nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_FH_STALE, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_FH_STALE]), + NFSD_A_SERVER_STATS_PAD)) + return -EMSGSIZE; + + if (nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_IO_READ, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_READ]), + NFSD_A_SERVER_STATS_PAD) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_IO_WRITE, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_WRITE]), + NFSD_A_SERVER_STATS_PAD)) + return -EMSGSIZE; + + if (nla_put_u32(skb, NFSD_A_SERVER_STATS_NETCNT, statp->netcnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_NETUDPCNT, statp->netudpcnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_NETTCPCNT, statp->nettcpcnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_NETTCPCONN, statp->nettcpconn)) + return -EMSGSIZE; + + if (nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCCNT, statp->rpccnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCBADFMT, statp->rpcbadfmt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCBADAUTH, statp->rpcbadauth) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCBADCLNT, statp->rpcbadclnt)) + return -EMSGSIZE; + + return 0; +} + +/* + * Emit per-version procedure counts for one NFS version, resuming at *idx. + * Returns 0 when the version has been fully emitted (or is not present), or + * -EMSGSIZE when @skb filled up, leaving *idx at the entry still to emit. + */ +static int nfsd_nl_server_stats_proc(struct sk_buff *skb, + struct svc_stat *statp, + struct svc_program *prog, + unsigned int ver, int attr, int *idx) +{ + unsigned long __percpu *counts; + unsigned int nproc; + + if (!statp->vs_count || ver >= prog->pg_nvers || + !prog->pg_vers[ver] || !statp->vs_count[ver]) + return 0; + + counts = statp->vs_count[ver]; + nproc = prog->pg_vers[ver]->vs_nproc; + + for (; *idx < nproc; (*idx)++) { + unsigned long count = 0; + int cpu; + + for_each_possible_cpu(cpu) + count += per_cpu(counts[*idx], cpu); + + if (!count) + continue; + if (nfsd_nl_put_proc_entry(skb, attr, *idx, count)) + return -EMSGSIZE; + } + + return 0; +} + +#ifdef CONFIG_NFSD_V4 +/* + * Emit NFSv4 per-operation counts, resuming at *idx. Same return convention + * as nfsd_nl_server_stats_proc(). + */ +static int nfsd_nl_server_stats_nfs4ops(struct sk_buff *skb, + struct nfsd_net *nn, int *idx) +{ + for (; *idx <= LAST_NFS4_OP; (*idx)++) { + u64 cnt = percpu_counter_sum_positive( + &nn->counter[NFSD_STATS_NFS4_OP(*idx)]); + + if (!cnt) + continue; + if (nfsd_nl_put_proc_entry(skb, NFSD_A_SERVER_STATS_PROC4OPS_OPS, + *idx, cnt)) + return -EMSGSIZE; + } + + return 0; +} + +/* + * Emit NFSv4 callback (backchannel) per-operation counts, resuming at *idx, + * which counts from OP_CB_GETATTR. Same return convention as + * nfsd_nl_server_stats_proc(). + */ +static int nfsd_nl_server_stats_cbops(struct sk_buff *skb, + struct nfsd_net *nn, int *idx) +{ + int op; + + for (op = OP_CB_GETATTR + *idx; op <= OP_CB_OFFLOAD; op++, (*idx)++) { + u64 cnt = percpu_counter_sum_positive(&nn->cb_counter[op]); + + if (!cnt) + continue; + if (nfsd_nl_put_proc_entry(skb, NFSD_A_SERVER_STATS_PROC4CB_OPS, + op, cnt)) + return -EMSGSIZE; + } + + return 0; +} +#endif + +/* Sections of the server-stats dump, emitted in order across messages. */ +enum { + NFSD_SERVER_STATS_SCALARS = 0, + NFSD_SERVER_STATS_PROC2, + NFSD_SERVER_STATS_PROC3, + NFSD_SERVER_STATS_PROC4, + NFSD_SERVER_STATS_PROC4CB, + NFSD_SERVER_STATS_PROC4OPS, + NFSD_SERVER_STATS_DONE, +}; + +/** + * nfsd_nl_server_stats_get_dumpit - dump NFS server statistics + * @skb: reply buffer + * @cb: netlink metadata and command arguments + * + * The server-stats object is emitted across one or more netlink messages. + * cb->args[0] tracks the current section and cb->args[1] the entry index + * within it, so a section that does not fit in the current message is resumed + * in the next one. The scalar counters are small and emitted once, in the + * first message; userspace merges the attributes from every message. + * + * Returns the size of the reply or a negative errno. + */ +int nfsd_nl_server_stats_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct net *net = sock_net(skb->sk); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_stat *statp = &nn->nfsd_svcstats; + struct svc_program *prog = statp->program; + int section = cb->args[0]; + int idx = cb->args[1]; + void *hdr; + + if (section >= NFSD_SERVER_STATS_DONE) + return 0; + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, + NLM_F_MULTI, NFSD_CMD_SERVER_STATS_GET); + if (!hdr) + return -ENOBUFS; + + /* Scalar stats fit easily and are emitted in the first message. */ + if (section == NFSD_SERVER_STATS_SCALARS) { + if (nfsd_nl_server_stats_scalars(skb, nn, statp)) + goto err_cancel; + section = NFSD_SERVER_STATS_PROC2; + idx = 0; + } + + /* + * Emit as many of the remaining sections as fit. A section returning + * -EMSGSIZE means the message is full: close it and resume from the + * same section/index on the next call with a fresh skb. Each entry is + * small enough to fit in a fresh skb, so forward progress is assured. + */ + while (section < NFSD_SERVER_STATS_DONE) { + int ret = 0; + + switch (section) { + case NFSD_SERVER_STATS_PROC2: + ret = nfsd_nl_server_stats_proc(skb, statp, prog, 2, + NFSD_A_SERVER_STATS_PROC2_OPS, &idx); + break; + case NFSD_SERVER_STATS_PROC3: + ret = nfsd_nl_server_stats_proc(skb, statp, prog, 3, + NFSD_A_SERVER_STATS_PROC3_OPS, &idx); + break; + case NFSD_SERVER_STATS_PROC4: + ret = nfsd_nl_server_stats_proc(skb, statp, prog, 4, + NFSD_A_SERVER_STATS_PROC4_OPS, &idx); + break; +#ifdef CONFIG_NFSD_V4 + case NFSD_SERVER_STATS_PROC4CB: + ret = nfsd_nl_server_stats_cbops(skb, nn, &idx); + break; + case NFSD_SERVER_STATS_PROC4OPS: + ret = nfsd_nl_server_stats_nfs4ops(skb, nn, &idx); + break; +#endif + } + + if (ret == -EMSGSIZE) + goto out; + if (ret) + goto err_cancel; + + section++; + idx = 0; + } + +out: + genlmsg_end(skb, hdr); + cb->args[0] = section; + cb->args[1] = idx; + return skb->len; + +err_cancel: + genlmsg_cancel(skb, hdr); + return -EMSGSIZE; +} + int nfsd_cache_notify(struct cache_detail *cd, struct cache_head *h, u32 cache_type) { struct genlmsghdr *hdr; @@ -2343,7 +2675,7 @@ int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb, error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); mutex_lock(&nfsd_mutex); - if (nn->nfsd_serv) { + if (test_bit(NFSD_NET_UP, &nn->flags)) { nfsd4_cancel_copy_by_sb(net, path.dentry->d_sb); nfsd4_revoke_states(nn, path.dentry->d_sb); } else { @@ -2390,7 +2722,7 @@ int nfsd_nl_unlock_export_doit(struct sk_buff *skb, struct genl_info *info) return error; mutex_lock(&nfsd_mutex); - if (nn->nfsd_serv) { + if (test_bit(NFSD_NET_UP, &nn->flags)) { nfsd_file_close_export(net, &path); nfsd4_revoke_export_states(nn, &path); } else @@ -2431,11 +2763,21 @@ static __net_init int nfsd_net_init(struct net *net) if (retval) goto out_repcache_error; +#ifdef CONFIG_NFSD_V4 + retval = percpu_counter_init_many(nn->cb_counter, 0, GFP_KERNEL, + NFSD_STATS_CB_OPS_NUM); + if (retval) + goto out_cb_counter_error; +#endif + memset(&nn->nfsd_svcstats, 0, sizeof(nn->nfsd_svcstats)); nn->nfsd_svcstats.program = &nfsd_programs[0]; + retval = svc_stat_alloc_counts(&nn->nfsd_svcstats); + if (retval) + goto out_proc_error; if (!nfsd_proc_stat_init(net)) { retval = -ENOMEM; - goto out_proc_error; + goto out_svcstats_error; } for (i = 0; i < sizeof(nn->nfsd_versions); i++) @@ -2453,7 +2795,13 @@ static __net_init int nfsd_net_init(struct net *net) #endif return 0; +out_svcstats_error: + svc_stat_free_counts(&nn->nfsd_svcstats); out_proc_error: +#ifdef CONFIG_NFSD_V4 + percpu_counter_destroy_many(nn->cb_counter, NFSD_STATS_CB_OPS_NUM); +out_cb_counter_error: +#endif percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM); out_repcache_error: nfsd_idmap_shutdown(net); @@ -2493,6 +2841,10 @@ static __net_exit void nfsd_net_exit(struct net *net) kfree_sensitive(nn->fh_key); nfsd_net_cb_shutdown(nn); nfsd_proc_stat_shutdown(net); + svc_stat_free_counts(&nn->nfsd_svcstats); +#ifdef CONFIG_NFSD_V4 + percpu_counter_destroy_many(nn->cb_counter, NFSD_STATS_CB_OPS_NUM); +#endif percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM); nfsd_idmap_shutdown(net); nfsd_export_shutdown(net); @@ -2512,11 +2864,12 @@ static int __init init_nfsd(void) { int retval; - nfsd_debugfs_init(); - retval = nfsd4_init_slabs(); if (retval) return retval; + + nfsd_debugfs_init(); + retval = nfsd4_init_pnfs(); if (retval) goto out_free_slabs; @@ -2561,8 +2914,8 @@ out_free_lockd: out_free_pnfs: nfsd4_exit_pnfs(); out_free_slabs: - nfsd4_free_slabs(); nfsd_debugfs_exit(); + nfsd4_free_slabs(); return retval; } @@ -2577,9 +2930,9 @@ static void __exit exit_nfsd(void) unregister_pernet_subsys(&nfsd_net_ops); nfsd_drc_slab_free(); nfsd_lockd_shutdown(); - nfsd4_free_slabs(); nfsd4_exit_pnfs(); nfsd_debugfs_exit(); + nfsd4_free_slabs(); } MODULE_AUTHOR("Olaf Kirch <okir@monad.swb.de>"); diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 11bce03b9031..76a69d9a4e73 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -18,13 +18,9 @@ #include <linux/nfs4.h> #include <linux/sunrpc/svc.h> #include <linux/sunrpc/svc_xprt.h> -#include <linux/sunrpc/msg_prot.h> -#include <linux/sunrpc/addr.h> #include <uapi/linux/nfsd/debug.h> -#include "export.h" - #undef ifdebug #ifdef CONFIG_SUNRPC_DEBUG # define ifdebug(flag) if (nfsd_debug & NFSDDBG_##flag) @@ -40,9 +36,6 @@ #define NFSD_SUPPORTED_MINOR_VERSION 2 bool nfsd_support_version(int vers); -#include "netns.h" -#include "stats.h" - /* * Default and maximum payload size (NFS READ or WRITE), in bytes. * The maximum is an implementation limit. @@ -52,10 +45,6 @@ enum { NFSSVC_MAXBLKSIZE = RPCSVC_MAXPAYLOAD, }; -struct readdir_cd { - __be32 err; /* 0, nfserr, or nfserr_eof */ -}; - /* Maximum number of operations per session compound */ #define NFSD_MAX_OPS_PER_COMPOUND 200 @@ -101,6 +90,8 @@ struct nfsdfs_client { void (*cl_release)(struct kref *kref); }; +struct nfsd_net; + struct nfsdfs_client *get_nfsdfs_client(struct inode *); struct dentry *nfsd_client_mkdir(struct nfsd_net *nn, struct nfsdfs_client *ncl, u32 id, @@ -126,8 +117,6 @@ extern const struct svc_version nfsd_acl_version3; extern const struct svc_version localio_version1; #endif -struct nfsd_net; - enum vers_op {NFSD_SET, NFSD_CLEAR, NFSD_TEST, NFSD_AVAIL }; int nfsd_vers(struct nfsd_net *nn, int vers, enum vers_op change); int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change); @@ -162,12 +151,6 @@ static inline int nfsd_v4client(struct svc_rqst *rq) { return rq && rq->rq_prog == NFS_PROGRAM && rq->rq_vers == 4; } -static inline struct user_namespace * -nfsd_user_namespace(const struct svc_rqst *rqstp) -{ - const struct cred *cred = rqstp->rq_xprt->xpt_cred; - return cred ? cred->user_ns : &init_user_ns; -} /* * NFSv4 State @@ -332,13 +315,14 @@ void nfsd_lockd_shutdown(void); #define nfserr_noxattr cpu_to_be32(NFS4ERR_NOXATTR) /* - * Error codes for internal use. We use enum to choose numbers that are - * not already assigned, then covert to be32 resulting in a number that - * cannot conflict with any existing be32 nfserr value. + * Error codes for internal use. These are based at an impossible + * nfsstat4 value so that, once converted to be32, they cannot conflict + * with any value defined by the protocol (compare the nlm__int__* codes + * in fs/lockd/lockd.h). */ enum { /* end-of-file indicator in readdir */ - NFSERR_EOF = NFS4ERR_FIRST_FREE, + NFSERR_EOF = 30000, #define nfserr_eof cpu_to_be32(NFSERR_EOF) /* replay detected */ @@ -356,9 +340,6 @@ enum { #define nfserr_symlink_not_dir cpu_to_be32(NFSERR_SYMLINK_NOT_DIR) }; -/* Check for dir entries '.' and '..' */ -#define isdotent(n, l) (l < 3 && n[0] == '.' && (l == 1 || n[1] == '.')) - #ifdef CONFIG_NFSD_V4 /* before processing a COMPOUND operation, we have to check that there @@ -386,187 +367,6 @@ enum { #define NFSD_DELEGRETURN_TIMEOUT (HZ / 34) /* 30ms */ #define NFSD_CB_GETATTR_TIMEOUT NFSD_DELEGRETURN_TIMEOUT -/* - * The following attributes are not implemented by NFSD: - * ARCHIVE (deprecated anyway) - * HIDDEN (unlikely to be supported any time soon) - * MIMETYPE (unlikely to be supported any time soon) - * QUOTA_* (will be supported in a forthcoming patch) - * SYSTEM (unlikely to be supported any time soon) - * TIME_BACKUP (unlikely to be supported any time soon) - */ -#define NFSD4_SUPPORTED_ATTRS_WORD0 \ -(FATTR4_WORD0_SUPPORTED_ATTRS | FATTR4_WORD0_TYPE | FATTR4_WORD0_FH_EXPIRE_TYPE \ - | FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE | FATTR4_WORD0_LINK_SUPPORT \ - | FATTR4_WORD0_SYMLINK_SUPPORT | FATTR4_WORD0_NAMED_ATTR | FATTR4_WORD0_FSID \ - | FATTR4_WORD0_UNIQUE_HANDLES | FATTR4_WORD0_LEASE_TIME | FATTR4_WORD0_RDATTR_ERROR \ - | FATTR4_WORD0_ACLSUPPORT | FATTR4_WORD0_CANSETTIME | FATTR4_WORD0_CASE_INSENSITIVE \ - | FATTR4_WORD0_CASE_PRESERVING | FATTR4_WORD0_CHOWN_RESTRICTED \ - | FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FILEID | FATTR4_WORD0_FILES_AVAIL \ - | FATTR4_WORD0_FILES_FREE | FATTR4_WORD0_FILES_TOTAL | FATTR4_WORD0_FS_LOCATIONS | FATTR4_WORD0_HOMOGENEOUS \ - | FATTR4_WORD0_MAXFILESIZE | FATTR4_WORD0_MAXLINK | FATTR4_WORD0_MAXNAME \ - | FATTR4_WORD0_MAXREAD | FATTR4_WORD0_MAXWRITE | FATTR4_WORD0_ACL) - -#define NFSD4_SUPPORTED_ATTRS_WORD1 \ -(FATTR4_WORD1_MODE | FATTR4_WORD1_NO_TRUNC | FATTR4_WORD1_NUMLINKS \ - | FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP | FATTR4_WORD1_RAWDEV \ - | FATTR4_WORD1_SPACE_AVAIL | FATTR4_WORD1_SPACE_FREE | FATTR4_WORD1_SPACE_TOTAL \ - | FATTR4_WORD1_SPACE_USED | FATTR4_WORD1_TIME_ACCESS | FATTR4_WORD1_TIME_ACCESS_SET \ - | FATTR4_WORD1_TIME_DELTA | FATTR4_WORD1_TIME_METADATA | FATTR4_WORD1_TIME_CREATE \ - | FATTR4_WORD1_TIME_MODIFY | FATTR4_WORD1_TIME_MODIFY_SET | FATTR4_WORD1_MOUNTED_ON_FILEID) - -#define NFSD4_SUPPORTED_ATTRS_WORD2 0 - -/* 4.1 */ -#ifdef CONFIG_NFSD_PNFS -#define PNFSD_SUPPORTED_ATTRS_WORD1 FATTR4_WORD1_FS_LAYOUT_TYPES -#define PNFSD_SUPPORTED_ATTRS_WORD2 \ -(FATTR4_WORD2_LAYOUT_BLKSIZE | FATTR4_WORD2_LAYOUT_TYPES) -#else -#define PNFSD_SUPPORTED_ATTRS_WORD1 0 -#define PNFSD_SUPPORTED_ATTRS_WORD2 0 -#endif /* CONFIG_NFSD_PNFS */ - -#define NFSD4_1_SUPPORTED_ATTRS_WORD0 \ - NFSD4_SUPPORTED_ATTRS_WORD0 - -#define NFSD4_1_SUPPORTED_ATTRS_WORD1 \ - (NFSD4_SUPPORTED_ATTRS_WORD1 | PNFSD_SUPPORTED_ATTRS_WORD1) - -#define NFSD4_1_SUPPORTED_ATTRS_WORD2 \ - (NFSD4_SUPPORTED_ATTRS_WORD2 | PNFSD_SUPPORTED_ATTRS_WORD2 | \ - FATTR4_WORD2_SUPPATTR_EXCLCREAT) - -/* 4.2 */ -#ifdef CONFIG_NFSD_V4_SECURITY_LABEL -#define NFSD4_2_SECURITY_ATTRS FATTR4_WORD2_SECURITY_LABEL -#else -#define NFSD4_2_SECURITY_ATTRS 0 -#endif - -#ifdef CONFIG_NFSD_V4_POSIX_ACLS -#define NFSD4_2_POSIX_ACL_ATTRS \ - (FATTR4_WORD2_ACL_TRUEFORM | \ - FATTR4_WORD2_ACL_TRUEFORM_SCOPE | \ - FATTR4_WORD2_POSIX_DEFAULT_ACL | \ - FATTR4_WORD2_POSIX_ACCESS_ACL) -#else -#define NFSD4_2_POSIX_ACL_ATTRS 0 -#endif - -#define NFSD4_2_SUPPORTED_ATTRS_WORD2 \ - (NFSD4_1_SUPPORTED_ATTRS_WORD2 | \ - FATTR4_WORD2_MODE_UMASK | \ - FATTR4_WORD2_CLONE_BLKSIZE | \ - NFSD4_2_SECURITY_ATTRS | \ - FATTR4_WORD2_XATTR_SUPPORT | \ - FATTR4_WORD2_TIME_DELEG_ACCESS | \ - FATTR4_WORD2_TIME_DELEG_MODIFY | \ - FATTR4_WORD2_OPEN_ARGUMENTS | \ - NFSD4_2_POSIX_ACL_ATTRS) - -extern const u32 nfsd_suppattrs[3][3]; - -static inline __be32 nfsd4_set_netaddr(struct sockaddr *addr, - struct nfs42_netaddr *netaddr) -{ - struct sockaddr_in *sin = (struct sockaddr_in *)addr; - struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; - unsigned int port; - size_t ret_addr, ret_port; - - switch (addr->sa_family) { - case AF_INET: - port = ntohs(sin->sin_port); - sprintf(netaddr->netid, "tcp"); - netaddr->netid_len = 3; - break; - case AF_INET6: - port = ntohs(sin6->sin6_port); - sprintf(netaddr->netid, "tcp6"); - netaddr->netid_len = 4; - break; - default: - return nfserr_inval; - } - ret_addr = rpc_ntop(addr, netaddr->addr, sizeof(netaddr->addr)); - ret_port = snprintf(netaddr->addr + ret_addr, - RPCBIND_MAXUADDRLEN + 1 - ret_addr, - ".%u.%u", port >> 8, port & 0xff); - WARN_ON(ret_port >= RPCBIND_MAXUADDRLEN + 1 - ret_addr); - netaddr->addr_len = ret_addr + ret_port; - return 0; -} - -static inline bool bmval_is_subset(const u32 *bm1, const u32 *bm2) -{ - return !((bm1[0] & ~bm2[0]) || - (bm1[1] & ~bm2[1]) || - (bm1[2] & ~bm2[2])); -} - -static inline bool nfsd_attrs_supported(u32 minorversion, const u32 *bmval) -{ - return bmval_is_subset(bmval, nfsd_suppattrs[minorversion]); -} - -/* These will return ERR_INVAL if specified in GETATTR or READDIR. */ -#define NFSD_WRITEONLY_ATTRS_WORD1 \ - (FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET) - -/* - * These are the only attrs allowed in CREATE/OPEN/SETATTR. Don't add - * a writeable attribute here without also adding code to parse it to - * nfsd4_decode_fattr(). - */ -#define NFSD_WRITEABLE_ATTRS_WORD0 \ - (FATTR4_WORD0_SIZE | FATTR4_WORD0_ACL) -#define NFSD_WRITEABLE_ATTRS_WORD1 \ - (FATTR4_WORD1_MODE | FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP \ - | FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_CREATE \ - | FATTR4_WORD1_TIME_MODIFY_SET) -#ifdef CONFIG_NFSD_V4_SECURITY_LABEL -#define MAYBE_FATTR4_WORD2_SECURITY_LABEL \ - FATTR4_WORD2_SECURITY_LABEL -#else -#define MAYBE_FATTR4_WORD2_SECURITY_LABEL 0 -#endif -#ifdef CONFIG_NFSD_V4_POSIX_ACLS -#define MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS \ - FATTR4_WORD2_POSIX_DEFAULT_ACL | FATTR4_WORD2_POSIX_ACCESS_ACL -#else -#define MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS 0 -#endif -#define NFSD_WRITEABLE_ATTRS_WORD2 \ - (FATTR4_WORD2_MODE_UMASK \ - | MAYBE_FATTR4_WORD2_SECURITY_LABEL \ - | FATTR4_WORD2_TIME_DELEG_ACCESS \ - | FATTR4_WORD2_TIME_DELEG_MODIFY \ - | MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS \ - ) - -#define NFSD_SUPPATTR_EXCLCREAT_WORD0 \ - NFSD_WRITEABLE_ATTRS_WORD0 -/* - * we currently store the exclusive create verifier in the v_{a,m}time - * attributes so the client can't set these at create time using EXCLUSIVE4_1 - */ -#define NFSD_SUPPATTR_EXCLCREAT_WORD1 \ - (NFSD_WRITEABLE_ATTRS_WORD1 & \ - ~(FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET)) -/* - * The FATTR4_WORD2_TIME_DELEG attributes are not to be allowed for - * OPEN(create) with EXCLUSIVE4_1. It doesn't make sense to set a - * delegated timestamp on a new file. - * - * This mask includes NFSv4.2-only attributes (e.g., POSIX ACLs). - * Version filtering occurs via nfsd_suppattrs[] before this mask - * is applied, so pre-4.2 clients never see unsupported attributes. - */ -#define NFSD_SUPPATTR_EXCLCREAT_WORD2 \ - (NFSD_WRITEABLE_ATTRS_WORD2 & \ - ~(FATTR4_WORD2_TIME_DELEG_ACCESS | FATTR4_WORD2_TIME_DELEG_MODIFY)) - extern int nfsd4_is_junction(struct dentry *dentry); extern int register_cld_notifier(void); extern void unregister_cld_notifier(void); diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 429ca5c6ec08..c7c60c35bdfc 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -13,6 +13,8 @@ #include <linux/sunrpc/svcauth_gss.h> #include <crypto/utils.h> #include "nfsd.h" +#include "netns.h" +#include "stats.h" #include "vfs.h" #include "auth.h" #include "trace.h" @@ -70,10 +72,8 @@ nfsd_mode_check(struct dentry *dentry, umode_t requested) if (requested == 0) /* the caller doesn't care */ return nfs_ok; if (mode == requested) { - if (mode == S_IFDIR && !d_can_lookup(dentry)) { - WARN_ON_ONCE(1); + if (mode == S_IFDIR && !d_can_lookup(dentry)) return nfserr_notdir; - } return nfs_ok; } if (mode == S_IFLNK) { @@ -144,16 +144,15 @@ static inline __be32 check_pseudo_root(struct dentry *dentry, /* Size of a file handle MAC, in 4-octet words */ #define FH_MAC_WORDS (sizeof(__le64) / 4) -static bool fh_append_mac(struct svc_fh *fhp, struct net *net) +bool fh_append_mac(struct knfsd_fh *fh, int fh_maxsize, struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - struct knfsd_fh *fh = &fhp->fh_handle; siphash_key_t *fh_key = nn->fh_key; __le64 hash; if (!fh_key) goto out_no_key; - if (fh->fh_size + sizeof(hash) > fhp->fh_maxsize) + if (fh->fh_size + sizeof(hash) > fh_maxsize) goto out_no_space; hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size, fh_key)); @@ -167,7 +166,7 @@ out_no_key: out_no_space: pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_size %zu would be greater than fh_maxsize %d.\n", - fh->fh_size + sizeof(hash), fhp->fh_maxsize); + fh->fh_size + sizeof(hash), fh_maxsize); return false; } @@ -344,15 +343,19 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net, if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC) fhp->fh_no_wcc = true; fhp->fh_64bit_cookies = true; - if (exp->ex_flags & NFSEXP_V4ROOT) + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); goto out; + } break; case NFS_FHSIZE: fhp->fh_no_wcc = true; if (EX_WGATHER(exp)) fhp->fh_use_wgather = true; - if (exp->ex_flags & NFSEXP_V4ROOT) + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); goto out; + } } fhp->fh_dentry = dentry; @@ -562,7 +565,8 @@ static void _fh_update(struct svc_fh *fhp, struct svc_export *exp, fhp->fh_handle.fh_size += maxsize * 4; if (exp->ex_flags & NFSEXP_SIGN_FH) - if (!fh_append_mac(fhp, exp->cd->net)) + if (!fh_append_mac(&fhp->fh_handle, fhp->fh_maxsize, + exp->cd->net)) fhp->fh_handle.fh_fileid_type = FILEID_INVALID; } else { fhp->fh_handle.fh_fileid_type = FILEID_ROOT; @@ -892,19 +896,20 @@ char * SVCFH_fmt(struct svc_fh *fhp) return buf; } -enum fsid_source fsid_source(const struct svc_fh *fhp) +enum fsid_source fsid_source_fh(const struct knfsd_fh *fh, + struct svc_export *exp) { - if (fhp->fh_handle.fh_version != 1) + if (fh->fh_version != 1) return FSIDSOURCE_DEV; - switch(fhp->fh_handle.fh_fsid_type) { + switch (fh->fh_fsid_type) { case FSID_DEV: case FSID_ENCODE_DEV: case FSID_MAJOR_MINOR: - if (exp_sb(fhp->fh_export)->s_type->fs_flags & FS_REQUIRES_DEV) + if (exp_sb(exp)->s_type->fs_flags & FS_REQUIRES_DEV) return FSIDSOURCE_DEV; break; case FSID_NUM: - if (fhp->fh_export->ex_flags & NFSEXP_FSID) + if (exp->ex_flags & NFSEXP_FSID) return FSIDSOURCE_FSID; break; default: @@ -913,13 +918,18 @@ enum fsid_source fsid_source(const struct svc_fh *fhp) /* either a UUID type filehandle, or the filehandle doesn't * match the export. */ - if (fhp->fh_export->ex_flags & NFSEXP_FSID) + if (exp->ex_flags & NFSEXP_FSID) return FSIDSOURCE_FSID; - if (fhp->fh_export->ex_uuid) + if (exp->ex_uuid) return FSIDSOURCE_UUID; return FSIDSOURCE_DEV; } +enum fsid_source fsid_source(const struct svc_fh *fhp) +{ + return fsid_source_fh(&fhp->fh_handle, fhp->fh_export); +} + /** * nfsd4_change_attribute - Generate an NFSv4 change_attribute value * @stat: inode attributes diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h index 5ef7191f8ad8..cdeb5eea65a8 100644 --- a/fs/nfsd/nfsfh.h +++ b/fs/nfsd/nfsfh.h @@ -131,6 +131,8 @@ enum fsid_source { FSIDSOURCE_FSID, FSIDSOURCE_UUID, }; +extern enum fsid_source fsid_source_fh(const struct knfsd_fh *fh, + struct svc_export *exp); extern enum fsid_source fsid_source(const struct svc_fh *fhp); @@ -226,6 +228,7 @@ __be32 fh_getattr(const struct svc_fh *fhp, struct kstat *stat); __be32 fh_compose(struct svc_fh *, struct svc_export *, struct dentry *, struct svc_fh *); __be32 fh_update(struct svc_fh *); void fh_put(struct svc_fh *); +bool fh_append_mac(struct knfsd_fh *fh, int fh_maxsize, struct net *net); static __inline__ struct svc_fh * fh_copy(struct svc_fh *dst, const struct svc_fh *src) diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 8873033d1e82..e2b5f8a241be 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -82,6 +82,7 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) .na_iattr = iap, }; struct svc_fh *fhp; + int hosterr; dprintk("nfsd: SETATTR %s, valid=%x, size=%ld\n", SVCFH_fmt(&argp->fh), @@ -117,6 +118,12 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) if (resp->status != nfs_ok) goto out; + hosterr = fh_want_write(fhp); + if (hosterr) { + resp->status = nfserrno(hosterr); + goto out; + } + if (delta < 0) delta = -delta; if (delta < MAX_TOUCH_TIME_ERROR && @@ -298,7 +305,7 @@ nfsd_proc_create(struct svc_rqst *rqstp) /* Check for NFSD_MAY_WRITE in nfsd_create if necessary */ resp->status = nfserr_exist; - if (isdotent(argp->name, argp->len)) + if (name_is_dot_dotdot(argp->name, argp->len)) goto done; hosterr = fh_want_write(dirfhp); if (hosterr) { @@ -838,13 +845,10 @@ static const struct svc_procedure nfsd_procedures2[18] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count2[ARRAY_SIZE(nfsd_procedures2)]); const struct svc_version nfsd_version2 = { .vs_vers = 2, .vs_nproc = ARRAY_SIZE(nfsd_procedures2), .vs_proc = nfsd_procedures2, - .vs_count = nfsd_count2, .vs_dispatch = nfsd_dispatch, .vs_xdrsize = NFS2_SVC_XDRSIZE, }; diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 4f1ab3222a4d..2edf716ea022 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -237,15 +237,21 @@ static void nfsd_net_free(struct percpu_ref *ref) */ #define NFSD_MAXSERVS 8192 +/** + * nfsd_nrthreads - report a namespace's configured nfsd thread count + * @net: network namespace to query + * + * Return: the configured thread ceiling, or 0 when no service runs. + */ int nfsd_nrthreads(struct net *net) { - int i, rv = 0; + int rv = 0; struct nfsd_net *nn = net_generic(net, nfsd_net_id); + /* nfsd_mutex keeps nn->nfsd_serv valid across the read. */ mutex_lock(&nfsd_mutex); if (nn->nfsd_serv) - for (i = 0; i < nn->nfsd_serv->sv_nrpools; ++i) - rv += nn->nfsd_serv->sv_pools[i].sp_nrthrmax; + rv = svc_serv_maxthreads(nn->nfsd_serv); mutex_unlock(&nfsd_mutex); return rv; } @@ -351,7 +357,7 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) struct nfsd_net *nn = net_generic(net, nfsd_net_id); int ret; - if (nn->nfsd_net_up) + if (test_bit(NFSD_NET_UP, &nn->flags)) return 0; ret = nfsd_startup_generic(); @@ -364,11 +370,11 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) goto out_socks; } - if (nfsd_needs_lockd(nn) && !nn->lockd_up) { + if (nfsd_needs_lockd(nn) && !test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { ret = lockd_up(net, cred); if (ret) goto out_socks; - nn->lockd_up = true; + set_bit(NFSD_NET_LOCKD_UP, &nn->flags); } ret = nfsd_file_cache_start_net(net); @@ -386,7 +392,7 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) if (ret) goto out_reply_cache; - nn->nfsd_net_up = true; + set_bit(NFSD_NET_UP, &nn->flags); return 0; out_reply_cache: @@ -394,9 +400,9 @@ out_reply_cache: out_filecache: nfsd_file_cache_shutdown_net(net); out_lockd: - if (nn->lockd_up) { + if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { lockd_down(net); - nn->lockd_up = false; + clear_bit(NFSD_NET_LOCKD_UP, &nn->flags); } out_socks: nfsd_shutdown_generic(); @@ -407,7 +413,7 @@ static void nfsd_shutdown_net(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - if (nn->nfsd_net_up) { + if (test_bit(NFSD_NET_UP, &nn->flags)) { percpu_ref_kill_and_confirm(&nn->nfsd_net_ref, nfsd_net_done); wait_for_completion(&nn->nfsd_net_confirm_done); @@ -415,18 +421,18 @@ static void nfsd_shutdown_net(struct net *net) nfs4_state_shutdown_net(net); nfsd_reply_cache_shutdown(nn); nfsd_file_cache_shutdown_net(net); - if (nn->lockd_up) { + if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { lockd_down(net); - nn->lockd_up = false; + clear_bit(NFSD_NET_LOCKD_UP, &nn->flags); } wait_for_completion(&nn->nfsd_net_free_done); } percpu_ref_exit(&nn->nfsd_net_ref); - if (nn->nfsd_net_up) + if (test_bit(NFSD_NET_UP, &nn->flags)) nfsd_shutdown_generic(); - nn->nfsd_net_up = false; + clear_bit(NFSD_NET_UP, &nn->flags); } static DEFINE_SPINLOCK(nfsd_notifier_lock); @@ -649,7 +655,7 @@ int nfsd_nrpools(struct net *net) if (nn->nfsd_serv == NULL) return 0; else - return nn->nfsd_serv->sv_nrpools; + return svc_serv_nrpools(nn->nfsd_serv); } int nfsd_get_nrthreads(int n, int *nthreads, struct net *net) @@ -659,7 +665,7 @@ int nfsd_get_nrthreads(int n, int *nthreads, struct net *net) int i; if (serv) - for (i = 0; i < serv->sv_nrpools && i < n; i++) + for (i = 0; i < svc_serv_nrpools(serv) && i < n; i++) nthreads[i] = serv->sv_pools[i].sp_nrthrmax; return 0; } @@ -693,8 +699,8 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) if (n == 1) return svc_set_num_threads(nn->nfsd_serv, nn->min_threads, nthreads[0]); - if (n > nn->nfsd_serv->sv_nrpools) - n = nn->nfsd_serv->sv_nrpools; + if (n > svc_serv_nrpools(nn->nfsd_serv)) + n = svc_serv_nrpools(nn->nfsd_serv); /* enforce a global maximum number of threads */ tot = 0; @@ -725,7 +731,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) } /* Anything undefined in array is considered to be 0 */ - for (i = n; i < nn->nfsd_serv->sv_nrpools; ++i) { + for (i = n; i < svc_serv_nrpools(nn->nfsd_serv); ++i) { err = svc_set_pool_threads(nn->nfsd_serv, &nn->nfsd_serv->sv_pools[i], 0, 0); @@ -815,7 +821,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp, ret->mismatch.lovers = NFSD_ACL_NRVERS; for (i = NFSD_ACL_MINVERS; i < NFSD_ACL_NRVERS; i++) { - if (nfsd_support_acl_version(rqstp->rq_vers) && + if (nfsd_support_acl_version(i) && nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.lovers = i; break; @@ -825,7 +831,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp, return rpc_prog_unavail; ret->mismatch.hivers = NFSD_ACL_MINVERS; for (i = NFSD_ACL_NRVERS - 1; i >= NFSD_ACL_MINVERS; i--) { - if (nfsd_support_acl_version(rqstp->rq_vers) && + if (nfsd_support_acl_version(i) && nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.hivers = i; break; @@ -960,6 +966,20 @@ nfsd(void *vrqstp) return 0; } +/* + * Set rq_status_counter back to an even value, indicating that the rqstp + * fields are no longer meaningful to a lockless reader. This pairs with the + * odd-valued store made once the request has been decoded, and must run on + * every return path that follows it so that the seq-lock like protocol used + * by nfsd_nl_rpc_status_get_dumpit() is not left permanently odd. The store + * also advances the counter so a concurrent reader detects the transition. + */ +static void nfsd_status_counter_set_idle(struct svc_rqst *rqstp) +{ + smp_store_release(&rqstp->rq_status_counter, + (rqstp->rq_status_counter | 1) + 1); +} + /** * nfsd_dispatch - Process an NFS or NFSACL or LOCALIO Request * @rqstp: incoming request @@ -1022,14 +1042,9 @@ int nfsd_dispatch(struct svc_rqst *rqstp) if (!proc->pc_encode(rqstp, &rqstp->rq_res_stream)) goto out_encode_err; - /* - * Release rq_status_counter setting it to an even value after the rpc - * request has been properly processed. - */ - smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter + 1); - nfsd_cache_update(rqstp, rp, ntli->ntli_cachetype, nfs_reply); out_cached_reply: + nfsd_status_counter_set_idle(rqstp); return 1; out_decode_err: @@ -1040,12 +1055,14 @@ out_decode_err: out_update_drop: nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL); out_dropit: + nfsd_status_counter_set_idle(rqstp); return 0; out_encode_err: trace_nfsd_cant_encode_err(rqstp); nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL); *statp = rpc_system_err; + nfsd_status_counter_set_idle(rqstp); return 1; } diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index ae71e0621317..019f0cc971a7 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -5,11 +5,23 @@ * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> */ +#include <linux/filelock.h> + #include "vfs.h" #include "xdr.h" #include "auth.h" /* + * Sun convention: a sattr time-useconds field of one full second (an + * otherwise out-of-range value) means "set this time to the current + * server time." It's needed to make permissions checks for the "touch" + * program across NFSv2 mounts work correctly. See description of + * sattr in section 6.1 of "NFS Illustrated" by Brent Callaghan, + * Addison-Wesley, ISBN 0-201-32750-5 + */ +#define NFS2_SATTR_SET_TO_SERVER_TIME (1000000) + +/* * Mapping of S_IF* types to NFS file types */ static const u32 nfs_ftypes[] = { @@ -172,27 +184,29 @@ svcxdr_decode_sattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, tmp1 = be32_to_cpup(p++); tmp2 = be32_to_cpup(p++); if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + /* + * Range test here to prevent the multiplication from + * wrapping to a valid (but incorrect) value on 32-bit + * platforms. + */ + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; iap->ia_atime.tv_sec = tmp1; iap->ia_atime.tv_nsec = tmp2 * NSEC_PER_USEC; + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) + iap->ia_valid &= ~ATTR_ATIME_SET; } tmp1 = be32_to_cpup(p++); tmp2 = be32_to_cpup(p++); if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; iap->ia_mtime.tv_sec = tmp1; iap->ia_mtime.tv_nsec = tmp2 * NSEC_PER_USEC; - /* - * Passing the invalid value useconds=1000000 for mtime - * is a Sun convention for "set both mtime and atime to - * current server time". It's needed to make permissions - * checks for the "touch" program across v2 mounts to - * Solaris and Irix boxes work correctly. See description of - * sattr in section 6.1 of "NFS Illustrated" by - * Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5 - */ - if (tmp2 == 1000000) + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) iap->ia_valid &= ~(ATTR_ATIME_SET|ATTR_MTIME_SET); } diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index dec83e92650d..2d00a411c663 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -36,9 +36,12 @@ #define _NFSD4_STATE_H #include <crypto/md5.h> + +#include <linux/filelock.h> #include <linux/idr.h> #include <linux/refcount.h> #include <linux/sunrpc/svc_xprt.h> + #include "nfsfh.h" #include "nfsd.h" @@ -59,7 +62,6 @@ typedef struct { typedef struct { stateid_t cs_stid; -#define NFS4_COPY_STID 1 #define NFS4_COPYNOTIFY_STID 2 unsigned char cs_type; refcount_t cs_count; @@ -98,9 +100,9 @@ struct nfsd4_callback { }; struct nfsd4_callback_ops { - void (*prepare)(struct nfsd4_callback *); - int (*done)(struct nfsd4_callback *, struct rpc_task *); - void (*release)(struct nfsd4_callback *); + bool (*prepare)(struct nfsd4_callback *cb); + int (*done)(struct nfsd4_callback *cb, struct rpc_task *task); + void (*release)(struct nfsd4_callback *cb); uint32_t opcode; }; @@ -121,6 +123,7 @@ struct nfs4_stid { #define SC_TYPE_LOCK BIT(1) #define SC_TYPE_DELEG BIT(2) #define SC_TYPE_LAYOUT BIT(3) +#define SC_TYPE_COPY BIT(4) unsigned short sc_type; /* nn->deleg_lock protects sc_status for delegation stateids. @@ -191,6 +194,66 @@ struct nfs4_cb_fattr { }; /* + * FIXME: the current backchannel encoder can't handle a send buffer longer + * than a single page (see bc_malloc/bc_free). + */ +#define NOTIFY4_EVENT_QUEUE_SIZE 3 +#define NOTIFY4_PAGE_ARRAY_SIZE 1 + +struct nfsd_notify_event { + refcount_t ne_ref; // refcount + u32 ne_mask; // FS_* mask from fsnotify callback + struct dentry *ne_dentry; // dentry reference to target + struct inode *ne_target; // inode overwritten by rename, or NULL + u32 ne_namelen; // length of ne_name (old name for a rename) + u32 ne_newnamelen; // length of new name (rename only), else 0 + char ne_name[]; // entry name, then new name (rename only) +}; + +/* + * For a rename, the new name is snapshotted at event-alloc time and stored + * immediately after the (NUL-terminated) old name in ne_name[]. ne_dentry can + * be renamed again before the CB_NOTIFY work runs, so the new name must not be + * read from the live dentry at encode time. + */ +static inline char *nfsd_notify_event_newname(struct nfsd_notify_event *ne) +{ + return ne->ne_name + ne->ne_namelen + 1; +} + +static inline struct nfsd_notify_event *nfsd_notify_event_get(struct nfsd_notify_event *ne) +{ + refcount_inc(&ne->ne_ref); + return ne; +} + +static inline void nfsd_notify_event_put(struct nfsd_notify_event *ne) +{ + if (refcount_dec_and_test(&ne->ne_ref)) { + iput(ne->ne_target); + dput(ne->ne_dentry); + kfree(ne); + } +} + +/* + * Represents a directory delegation. The callback is for handling CB_NOTIFYs. + * As notifications from fsnotify come in, allocate a new event, take the ncn_lock, + * and add it to the ncn_evt queue. The CB_NOTIFY prepare handler will take the + * lock, clean out the list and process it. + */ +struct nfsd4_cb_notify { + spinlock_t ncn_lock; // protects the evt queue and count + int ncn_evt_cnt; // count of events in ncn_evt + int ncn_nf_cnt; // count of valid entries in ncn_nf + struct nfsd_notify_event *ncn_evt[NOTIFY4_EVENT_QUEUE_SIZE]; // list of events + struct page *ncn_pages[NOTIFY4_PAGE_ARRAY_SIZE]; // for encoding + struct notify4 *ncn_nf; // array of notify4's to be sent + bool ncn_encode_err; // did encoding fail? + struct nfsd4_callback ncn_cb; // notify4 callback +}; + +/* * Represents a delegation stateid. The nfs4_client holds references to these * and they are put when it is being destroyed or when the delegation is * returned by the client: @@ -226,13 +289,22 @@ struct nfs4_delegation { bool dl_written; bool dl_setattr; - /* for CB_GETATTR */ - struct nfs4_cb_fattr dl_cb_fattr; + union { + /* for CB_GETATTR */ + struct nfs4_cb_fattr dl_cb_fattr; + /* for CB_NOTIFY */ + struct nfsd4_cb_notify dl_cb_notify; + }; /* For delegated timestamps */ struct timespec64 dl_atime; struct timespec64 dl_mtime; struct timespec64 dl_ctime; + + /* For dir delegations */ + u32 dl_notify_mask; + u32 dl_child_attrs[2]; + u32 dl_dir_attrs[2]; }; static inline bool deleg_is_read(u32 dl_type) @@ -384,6 +456,7 @@ struct nfsd4_session { u16 se_slot_gen; bool se_dead; u32 se_target_maxslots; + struct rcu_head rcu_head; }; /* formatted contents of nfs4_sessionid */ @@ -496,7 +569,7 @@ struct nfs4_client { #define NFSD4_CB_FAULT 3 int cl_cb_state; struct nfsd4_callback cl_cb_null; - struct nfsd4_session *cl_cb_session; + struct nfsd4_session __rcu *cl_cb_session; /* for all client information that callback code might need: */ spinlock_t cl_lock; @@ -691,7 +764,7 @@ struct nfs4_file { */ atomic_t fi_access[2]; u32 fi_share_deny; - struct nfsd_file *fi_deleg_file; + struct nfsd_file __rcu *fi_deleg_file; struct nfsd_file *fi_rdeleg_file; int fi_delegees; struct knfsd_fh fi_fhandle; @@ -754,6 +827,7 @@ struct nfs4_layout_stateid { struct delayed_work ls_fence_work; unsigned int ls_fence_delay; bool ls_fenced; + bool ls_fence_inflight; }; static inline struct nfs4_layout_stateid *layoutstateid(struct nfs4_stid *s) @@ -774,6 +848,7 @@ enum nfsd4_cb_op { NFSPROC4_CLNT_CB_NOTIFY_LOCK, NFSPROC4_CLNT_CB_RECALL_ANY, NFSPROC4_CLNT_CB_GETATTR, + NFSPROC4_CLNT_CB_NOTIFY, }; /* Returns true iff a is later than b: */ @@ -800,6 +875,7 @@ struct nfsd4_blocked_lock { struct nfsd4_compound_state; struct nfsd_net; struct nfsd4_copy; +struct nfsd4_async_copy; extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct svc_fh *fhp, @@ -811,8 +887,7 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, struct nfs4_stid **s, struct nfsd_net *nn); struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab, void (*sc_free)(struct nfs4_stid *)); -int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy); -void nfs4_free_copy_state(struct nfsd4_copy *copy); +struct nfsd4_async_copy *nfs4_alloc_copy_stid(struct nfs4_client *clp); struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, struct nfs4_stid *p_stid); void nfs4_put_stid(struct nfs4_stid *s); @@ -848,6 +923,8 @@ void nfsd_update_cmtime_attr(struct file *f, unsigned int flags); extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, struct nfsd_net *nn); extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn); +int nfsd_handle_dir_event(u32 mask, const struct inode *dir, const void *data, + int data_type, const struct qstr *name); void put_nfs4_file(struct nfs4_file *fi); extern void nfs4_put_cpntf_state(struct nfsd_net *nn, diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c index f7eaf95e20fc..f16559813292 100644 --- a/fs/nfsd/stats.c +++ b/fs/nfsd/stats.c @@ -26,6 +26,8 @@ #include <net/net_namespace.h> #include "nfsd.h" +#include "netns.h" +#include "stats.h" static int nfsd_show(struct seq_file *seq, void *v) { @@ -63,7 +65,7 @@ static int nfsd_show(struct seq_file *seq, void *v) percpu_counter_sum_positive(&nn->counter[NFSD_STATS_NFS4_OP(i)])); } seq_printf(seq, "\nwdeleg_getattr %lld", - percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR])); + percpu_counter_sum_positive(&nn->cb_counter[OP_CB_GETATTR])); seq_putc(seq, '\n'); #endif diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h index e4efb0e4e56d..aabfbb1a9c71 100644 --- a/fs/nfsd/stats.h +++ b/fs/nfsd/stats.h @@ -10,22 +10,45 @@ #include <uapi/linux/nfsd/stats.h> #include <linux/percpu_counter.h> +#include "export.h" +#include "netns.h" + struct proc_dir_entry *nfsd_proc_stat_init(struct net *net); void nfsd_proc_stat_shutdown(struct net *net); +/** + * nfsd_stats_rc_hits_inc - Count a duplicate reply cache hit + * @nn: target network namespace + * + * These reply cache counters are updated once per RPC. Readers use + * percpu_counter_sum_positive(), so local batching does not affect + * read accuracy. + */ static inline void nfsd_stats_rc_hits_inc(struct nfsd_net *nn) { - percpu_counter_inc(&nn->counter[NFSD_STATS_RC_HITS]); + percpu_counter_add_local(&nn->counter[NFSD_STATS_RC_HITS], 1); } +/** + * nfsd_stats_rc_misses_inc - Count a duplicate reply cache miss + * @nn: target network namespace + * + * See nfsd_stats_rc_hits_inc() for batching rationale. + */ static inline void nfsd_stats_rc_misses_inc(struct nfsd_net *nn) { - percpu_counter_inc(&nn->counter[NFSD_STATS_RC_MISSES]); + percpu_counter_add_local(&nn->counter[NFSD_STATS_RC_MISSES], 1); } +/** + * nfsd_stats_rc_nocache_inc - Count a request not cached in the reply cache + * @nn: target network namespace + * + * See nfsd_stats_rc_hits_inc() for batching rationale. + */ static inline void nfsd_stats_rc_nocache_inc(struct nfsd_net *nn) { - percpu_counter_inc(&nn->counter[NFSD_STATS_RC_NOCACHE]); + percpu_counter_add_local(&nn->counter[NFSD_STATS_RC_NOCACHE], 1); } static inline void nfsd_stats_fh_stale_inc(struct nfsd_net *nn, @@ -36,20 +59,42 @@ static inline void nfsd_stats_fh_stale_inc(struct nfsd_net *nn, percpu_counter_inc(&exp->ex_stats->counter[EXP_STATS_FH_STALE]); } +/** + * nfsd_stats_io_read_add - Count number of bytes for an NFS READ + * @nn: target network namespace + * @exp: target export + * @amount: byte count + * + * These counters are updated on every READ request. Readers use + * percpu_counter_sum_positive(), so local batching does not affect + * read accuracy. + */ static inline void nfsd_stats_io_read_add(struct nfsd_net *nn, struct svc_export *exp, s64 amount) { - percpu_counter_add(&nn->counter[NFSD_STATS_IO_READ], amount); + percpu_counter_add_local(&nn->counter[NFSD_STATS_IO_READ], amount); if (exp && exp->ex_stats) - percpu_counter_add(&exp->ex_stats->counter[EXP_STATS_IO_READ], amount); + percpu_counter_add_local(&exp->ex_stats->counter[EXP_STATS_IO_READ], + amount); } +/** + * nfsd_stats_io_write_add - Count number of bytes for an NFS WRITE + * @nn: target network namespace + * @exp: target export + * @amount: byte count + * + * These counters are updated on every WRITE request. Readers use + * percpu_counter_sum_positive(), so local batching does not affect + * read accuracy. + */ static inline void nfsd_stats_io_write_add(struct nfsd_net *nn, struct svc_export *exp, s64 amount) { - percpu_counter_add(&nn->counter[NFSD_STATS_IO_WRITE], amount); + percpu_counter_add_local(&nn->counter[NFSD_STATS_IO_WRITE], amount); if (exp && exp->ex_stats) - percpu_counter_add(&exp->ex_stats->counter[EXP_STATS_IO_WRITE], amount); + percpu_counter_add_local(&exp->ex_stats->counter[EXP_STATS_IO_WRITE], + amount); } static inline void nfsd_stats_payload_misses_inc(struct nfsd_net *nn) @@ -57,20 +102,39 @@ static inline void nfsd_stats_payload_misses_inc(struct nfsd_net *nn) percpu_counter_inc(&nn->counter[NFSD_STATS_PAYLOAD_MISSES]); } +/** + * nfsd_stats_drc_mem_usage_add - Add memory used by a cache item + * @nn: target network namespace + * @amount: byte count + * + * percpu_counter_add_local() keeps updates on the per-CPU fast + * path. The sole reader, percpu_counter_sum_positive(), sums the + * per-CPU deltas, so batching locally does not lose accuracy. + */ static inline void nfsd_stats_drc_mem_usage_add(struct nfsd_net *nn, s64 amount) { - percpu_counter_add(&nn->counter[NFSD_STATS_DRC_MEM_USAGE], amount); + percpu_counter_add_local(&nn->counter[NFSD_STATS_DRC_MEM_USAGE], + amount); } +/** + * nfsd_stats_drc_mem_usage_sub - Subtract memory used by a cache item + * @nn: target network namespace + * @amount: byte count + * + * See nfsd_stats_drc_mem_usage_add() for batching rationale. + */ static inline void nfsd_stats_drc_mem_usage_sub(struct nfsd_net *nn, s64 amount) { - percpu_counter_sub(&nn->counter[NFSD_STATS_DRC_MEM_USAGE], amount); + percpu_counter_sub_local(&nn->counter[NFSD_STATS_DRC_MEM_USAGE], + amount); } #ifdef CONFIG_NFSD_V4 -static inline void nfsd_stats_wdeleg_getattr_inc(struct nfsd_net *nn) +static inline void nfsd_stats_cb_op_inc(struct nfsd_net *nn, u32 opcode) { - percpu_counter_inc(&nn->counter[NFSD_STATS_WDELEG_GETATTR]); + if (opcode >= OP_CB_GETATTR && opcode <= OP_CB_OFFLOAD) + percpu_counter_inc(&nn->cb_counter[opcode]); } #endif #endif /* _NFSD_STATS_H */ diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 1c5a1e50f946..7d7a1483109a 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -12,12 +12,14 @@ #include <linux/sunrpc/clnt.h> #include <linux/sunrpc/xprt.h> #include <trace/misc/fs.h> +#include <trace/misc/fsnotify.h> #include <trace/misc/nfs.h> #include <trace/misc/sunrpc.h> #include "export.h" #include "nfsfh.h" #include "xdr4.h" +#include "netns.h" #define NFSD_TRACE_PROC_CALL_FIELDS(r) \ __field(unsigned int, netns_ino) \ @@ -271,7 +273,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify, TP_CONDITION(rqstp != NULL), TP_STRUCT__entry( __field(unsigned int, netns_ino) - __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) __field(u32, xid) __field(u32, fh_hash) @@ -310,7 +312,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify_err, TP_CONDITION(rqstp != NULL && error), TP_STRUCT__entry( __field(unsigned int, netns_ino) - __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) __field(u32, xid) __field(u32, fh_hash) @@ -1377,6 +1379,28 @@ TRACE_EVENT(nfsd_file_fsnotify_handle_event, __entry->nlink, __entry->mode, __entry->mask) ); +TRACE_EVENT(nfsd_handle_dir_event, + TP_PROTO(u32 mask, const struct inode *dir, const struct qstr *name), + TP_ARGS(mask, dir, name), + TP_STRUCT__entry( + __field(u32, mask) + __field(dev_t, s_dev) + __field(u64, i_ino) + __string_len(name, name ? name->name : NULL, + name ? name->len : 0) + ), + TP_fast_assign( + __entry->mask = mask; + __entry->s_dev = dir ? dir->i_sb->s_dev : 0; + __entry->i_ino = dir ? dir->i_ino : 0; + __assign_str(name); + ), + TP_printk("inode=0x%x:0x%x:0x%llx mask=%s name=%s", + MAJOR(__entry->s_dev), MINOR(__entry->s_dev), + __entry->i_ino, show_fsnotify_mask(__entry->mask), + __get_str(name)) +); + DECLARE_EVENT_CLASS(nfsd_file_gc_class, TP_PROTO( const struct nfsd_file *nf @@ -1677,6 +1701,7 @@ TRACE_EVENT(nfsd_cb_setup_err, { OP_CB_RECALL, "CB_RECALL" }, \ { OP_CB_LAYOUTRECALL, "CB_LAYOUTRECALL" }, \ { OP_CB_RECALL_ANY, "CB_RECALL_ANY" }, \ + { OP_CB_NOTIFY, "CB_NOTIFY" }, \ { OP_CB_NOTIFY_LOCK, "CB_NOTIFY_LOCK" }, \ { OP_CB_OFFLOAD, "CB_OFFLOAD" }) @@ -1727,9 +1752,10 @@ DEFINE_NFSD_CB_LIFETIME_EVENT(bc_shutdown); TRACE_EVENT(nfsd_cb_seq_status, TP_PROTO( const struct rpc_task *task, - const struct nfsd4_callback *cb + const struct nfsd4_callback *cb, + const struct nfsd4_session *session ), - TP_ARGS(task, cb), + TP_ARGS(task, cb, session), TP_STRUCT__entry( __field(unsigned int, task_id) __field(unsigned int, client_id) @@ -1741,8 +1767,6 @@ TRACE_EVENT(nfsd_cb_seq_status, __field(int, seq_status) ), TP_fast_assign( - const struct nfs4_client *clp = cb->cb_clp; - const struct nfsd4_session *session = clp->cl_cb_session; const struct nfsd4_sessionid *sid = (struct nfsd4_sessionid *)&session->se_sessionid; @@ -1768,9 +1792,10 @@ TRACE_EVENT(nfsd_cb_seq_status, TRACE_EVENT(nfsd_cb_free_slot, TP_PROTO( const struct rpc_task *task, - const struct nfsd4_callback *cb + const struct nfsd4_callback *cb, + const struct nfsd4_session *session ), - TP_ARGS(task, cb), + TP_ARGS(task, cb, session), TP_STRUCT__entry( __field(unsigned int, task_id) __field(unsigned int, client_id) @@ -1781,8 +1806,6 @@ TRACE_EVENT(nfsd_cb_free_slot, __field(u32, slot_seqno) ), TP_fast_assign( - const struct nfs4_client *clp = cb->cb_clp; - const struct nfsd4_session *session = clp->cl_cb_session; const struct nfsd4_sessionid *sid = (struct nfsd4_sessionid *)&session->se_sessionid; diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 1e89c7ff9493..8923a9910a08 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -43,6 +43,8 @@ #endif /* CONFIG_NFSD_V4 */ #include "nfsd.h" +#include "netns.h" +#include "stats.h" #include "vfs.h" #include "filecache.h" #include "trace.h" @@ -139,16 +141,17 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, err = follow_down(&path, follow_flags); if (err < 0) goto out; + if (path.mnt == exp->ex_path.mnt && path.dentry == dentry && nfsd_mountpoint(dentry, exp) == 2) { /* This is only a mountpoint in some other namespace */ - path_put(&path); goto out; } exp2 = rqst_exp_get_by_name(rqstp, &path); if (IS_ERR(exp2)) { err = PTR_ERR(exp2); + exp2 = NULL; /* * We normally allow NFS clients to continue * "underneath" a mountpoint that is not exported. @@ -158,10 +161,7 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, */ if (err == -ENOENT && !(exp->ex_flags & NFSEXP_V4ROOT)) err = 0; - path_put(&path); - goto out; - } - if (nfsd_v4client(rqstp) || + } else if (nfsd_v4client(rqstp) || (exp->ex_flags & NFSEXP_CROSSMOUNT) || EX_NOHIDE(exp2)) { /* successfully crossed mount point */ /* @@ -175,9 +175,10 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, *expp = exp2; exp2 = exp; } - path_put(&path); - exp_put(exp2); out: + path_put(&path); + if (exp2) + exp_put(exp2); return err; } @@ -256,7 +257,7 @@ nfsd_lookup_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp, exp = exp_get(fhp->fh_export); /* Lookup the name, but don't follow links */ - if (isdotent(name, len)) { + if (name_is_dot_dotdot(name, len)) { if (len==1) dentry = dget(dparent); else if (dparent != exp->ex_path.dentry) @@ -419,21 +420,22 @@ nfsd_sanitize_attrs(struct inode *inode, struct iattr *iap) } static __be32 -nfsd_get_write_access(struct svc_rqst *rqstp, struct svc_fh *fhp, - struct iattr *iap) +nfsd_may_truncate(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct iattr *iap) { struct inode *inode = d_inode(fhp->fh_dentry); - if (iap->ia_size < inode->i_size) { - __be32 err; + if (iap->ia_size >= i_size_read(inode)) + return nfs_ok; - err = nfsd_permission(&rqstp->rq_cred, - fhp->fh_export, fhp->fh_dentry, - NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); - if (err) - return err; - } - return nfserrno(get_write_access(inode)); + return nfsd_permission(&rqstp->rq_cred, fhp->fh_export, fhp->fh_dentry, + NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); +} + +static __be32 +nfsd_get_write_access(struct svc_fh *fhp) +{ + return nfserrno(get_write_access(d_inode(fhp->fh_dentry))); } static int __nfsd_setattr(struct dentry *dentry, struct iattr *iap) @@ -560,12 +562,17 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, * setattr call. */ if (size_change) { - err = nfsd_get_write_access(rqstp, fhp, iap); + err = nfsd_get_write_access(fhp); if (err) return err; } inode_lock(inode); + if (size_change) { + err = nfsd_may_truncate(rqstp, fhp, iap); + if (err) + goto out_unlock; + } err = fh_fill_pre_attrs(fhp); if (err) goto out_unlock; @@ -1374,6 +1381,7 @@ nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct file *file = nf->nf_file; unsigned int nsegs, i; ssize_t host_err; + size_t expected; nsegs = nfsd_write_dio_iters_init(nf, rqstp->rq_bvec, nvecs, kiocb, *cnt, segments); @@ -1395,11 +1403,13 @@ nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp, kiocb->ki_flags |= IOCB_DONTCACHE; } + expected = iov_iter_count(&segments[i].iter); + host_err = vfs_iocb_iter_write(file, kiocb, &segments[i].iter); if (host_err < 0) return host_err; *cnt += host_err; - if (host_err < segments[i].iter.count) + if (host_err < (ssize_t)expected) break; /* partial write */ } @@ -1876,7 +1886,7 @@ nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp, trace_nfsd_vfs_create(rqstp, fhp, type, fname, flen); - if (isdotent(fname, flen)) + if (name_is_dot_dotdot(fname, flen)) return nfserr_exist; err = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_NOP); @@ -1978,7 +1988,7 @@ nfsd_symlink(struct svc_rqst *rqstp, struct svc_fh *fhp, if (!flen || path[0] == '\0') goto out; err = nfserr_exist; - if (isdotent(fname, flen)) + if (name_is_dot_dotdot(fname, flen)) goto out; err = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_CREATE); @@ -2055,7 +2065,7 @@ nfsd_link(struct svc_rqst *rqstp, struct svc_fh *ffhp, if (!len) goto out; err = nfserr_exist; - if (isdotent(name, len)) + if (name_is_dot_dotdot(name, len)) goto out; err = nfs_ok; @@ -2166,7 +2176,8 @@ nfsd_rename(struct svc_rqst *rqstp, struct svc_fh *ffhp, char *fname, int flen, tdentry = tfhp->fh_dentry; err = nfserr_perm; - if (!flen || isdotent(fname, flen) || !tlen || isdotent(tname, tlen)) + if (!flen || name_is_dot_dotdot(fname, flen) || + !tlen || name_is_dot_dotdot(tname, tlen)) goto out; err = nfserr_xdev; @@ -2288,7 +2299,7 @@ nfsd_unlink(struct svc_rqst *rqstp, struct svc_fh *fhp, int type, trace_nfsd_vfs_unlink(rqstp, fhp, fname, flen); err = nfserr_acces; - if (!flen || isdotent(fname, flen)) + if (!flen || name_is_dot_dotdot(fname, flen)) goto out; err = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_REMOVE); if (err) diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index e09ea04a51b9..4af2ff9e9dfe 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -9,7 +9,6 @@ #include <linux/fs.h> #include <linux/posix_acl.h> #include "nfsfh.h" -#include "nfsd.h" /* * Flags for nfsd_permission @@ -45,6 +44,10 @@ struct nfsd_file; */ typedef int (*nfsd_filldir_t)(void *, const char *, int, loff_t, u64, unsigned); +struct readdir_cd { + __be32 err; /* nfs_ok, nfserr, or nfserr_eof */ +}; + /* nfsd/vfs.c */ struct nfsd_attrs { struct iattr *na_iattr; /* input */ diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h index 852f71580bd0..df540c940cef 100644 --- a/fs/nfsd/xdr.h +++ b/fs/nfsd/xdr.h @@ -7,6 +7,7 @@ #include <linux/vfs.h> #include "nfsd.h" #include "nfsfh.h" +#include "vfs.h" struct nfsd_fhandle { struct svc_fh fh; diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index a7c9714b0b0e..344203874b4c 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -9,6 +9,7 @@ #define _LINUX_NFSD_XDR3_H #include "xdr.h" +#include "vfs.h" struct nfsd3_sattrargs { struct svc_fh fh; diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 85574b2a139a..c7eda5bc833b 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -38,7 +38,7 @@ #define _LINUX_NFSD_XDR4_H #include "state.h" -#include "nfsd.h" +#include "vfs.h" #define NFSD4_MAX_TAGLEN 128 #define XDR_LEN(n) (((n) + 3) & ~3) @@ -759,28 +759,38 @@ struct nfsd4_copy { struct nfsd42_write_res cp_res; struct knfsd_fh fh; - /* offload callback */ - struct nfsd4_cb_offload cp_cb_offload; - struct nfs4_client *cp_clp; struct nfsd_file *nf_src; struct nfsd_file *nf_dst; bool attr_update; - copy_stateid_t cp_stateid; - - struct list_head copies; - struct task_struct *copy_task; - refcount_t refcount; - unsigned int cp_ttl; - struct nfsd4_ssc_umount_item *ss_nsui; struct nfs_fh c_fh; nfs4_stateid stateid; struct nfsd_net *cp_nn; }; +/* + * Durable state for an async (background) server-side COPY. + * + * struct nfsd4_copy is transient: it lives in the COMPOUND argument buffer + * and is reused once the op returns. An async COPY outlives the COMPOUND + * (worker kthread, reaper linkage, CB_OFFLOAD), so its params and result are + * snapshotted into the embedded cp_copy and it never points into the request + * buffer. + */ +struct nfsd4_async_copy { + struct nfs4_stid cp_stid; /* SC_TYPE_COPY, in cl_stateids */ + struct nfsd4_copy cp_copy; /* operation params + result */ + + struct list_head copies; /* nfs4_client.async_copies */ + struct task_struct *copy_task; + refcount_t refcount; + unsigned int cp_ttl; + struct nfsd4_cb_offload cp_cb_offload; +}; + static inline void nfsd4_copy_set_sync(struct nfsd4_copy *copy, bool sync) { if (sync) @@ -970,6 +980,11 @@ __be32 nfsd4_encode_fattr_to_buf(__be32 **p, int words, struct svc_fh *fhp, struct svc_export *exp, struct dentry *dentry, u32 *bmval, struct svc_rqst *, int ignore_crossmnt); +u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *nne, + struct nfs4_delegation *dd, struct nfsd_file *nf, + u32 *notify_mask); +u8 *nfsd4_encode_dir_attr_change(struct xdr_stream *xdr, struct nfs4_delegation *dp, + struct nfsd_file *nf); extern __be32 nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *, union nfsd4_op_u *u); extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp, diff --git a/fs/nfsd/xdr4cb.h b/fs/nfsd/xdr4cb.h index f4e29c0c701c..b06d0170d7c4 100644 --- a/fs/nfsd/xdr4cb.h +++ b/fs/nfsd/xdr4cb.h @@ -33,6 +33,18 @@ cb_sequence_dec_sz + \ op_dec_sz) +#define NFS4_enc_cb_notify_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + enc_stateid_sz + \ + enc_nfs4_fh_sz + \ + 1 + \ + NOTIFY4_EVENT_QUEUE_SIZE * \ + (2 + (NFS4_OPAQUE_LIMIT >> 2))) + +#define NFS4_dec_cb_notify_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz) + #define NFS4_enc_cb_notify_lock_sz (cb_compound_enc_hdr_sz + \ cb_sequence_enc_sz + \ 2 + 1 + \ diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index a3555bebad63..a208a7ec1692 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -599,9 +599,9 @@ static struct fanotify_event *fanotify_alloc_perm_event(const void *data, pevent->hdr.pad = 0; pevent->hdr.len = 0; pevent->state = FAN_EVENT_INIT; + pevent->watchdog_cnt = 0; pevent->path = *path; - /* NULL ppos means no range info */ - pevent->ppos = range ? &range->pos : NULL; + pevent->pos = range ? range->pos : FANOTIFY_NO_RANGE; pevent->count = range ? range->count : 0; path_get(path); diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h index a0619e7694d5..3710543dbf82 100644 --- a/fs/notify/fanotify/fanotify.h +++ b/fs/notify/fanotify/fanotify.h @@ -428,6 +428,8 @@ FANOTIFY_ME(struct fanotify_event *event) return container_of(event, struct fanotify_mnt_event, fae); } +#define FANOTIFY_NO_RANGE ((loff_t)-1) + /* * Structure for permission fanotify events. It gets allocated and freed in * fanotify_handle_event() since we wait there for user response. When the @@ -438,7 +440,7 @@ FANOTIFY_ME(struct fanotify_event *event) struct fanotify_perm_event { struct fanotify_event fae; struct path path; - const loff_t *ppos; /* optional file range info */ + loff_t pos; /* FANOTIFY_NO_RANGE if unavailable */ size_t count; u32 response; /* userspace answer to the event */ unsigned short state; /* state of the event */ @@ -468,7 +470,7 @@ static inline bool fanotify_event_has_access_range(struct fanotify_event *event) if (!(event->mask & FANOTIFY_PRE_CONTENT_EVENTS)) return false; - return FANOTIFY_PERM(event)->ppos; + return FANOTIFY_PERM(event)->pos != FANOTIFY_NO_RANGE; } static inline struct fanotify_event *FANOTIFY_E(struct fsnotify_event *fse) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index b604e3da58ad..a32c6634d592 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -112,7 +112,12 @@ static DECLARE_DELAYED_WORK(perm_group_work, perm_group_watchdog); static void perm_group_watchdog_schedule(void) { - schedule_delayed_work(&perm_group_work, secs_to_jiffies(perm_group_timeout)); + int timeout = READ_ONCE(perm_group_timeout); + + if (!timeout) + return; + + schedule_delayed_work(&perm_group_work, secs_to_jiffies(timeout)); } static void perm_group_watchdog(struct work_struct *work) @@ -675,12 +680,9 @@ static size_t copy_range_info_to_user(struct fanotify_event *event, if (WARN_ON_ONCE(info_len > count)) return -EFAULT; - if (WARN_ON_ONCE(!pevent->ppos)) - return -EINVAL; - info.hdr.info_type = FAN_EVENT_INFO_TYPE_RANGE; info.hdr.len = info_len; - info.offset = *(pevent->ppos); + info.offset = pevent->pos; info.count = pevent->count; if (copy_to_user(buf, &info, info_len)) @@ -1145,11 +1147,13 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar { struct fsnotify_group *group; struct fsnotify_event *fsn_event; + unsigned int info_mode; void __user *p; int ret = -ENOTTY; size_t send_len = 0; group = file->private_data; + info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); p = (void __user *) arg; @@ -1157,7 +1161,8 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar case FIONREAD: spin_lock(&group->notification_lock); list_for_each_entry(fsn_event, &group->notification_list, list) - send_len += FAN_EVENT_METADATA_LEN; + send_len += fanotify_event_len(info_mode, + FANOTIFY_E(fsn_event)); spin_unlock(&group->notification_lock); ret = put_user(send_len, (int __user *) p); break; @@ -1316,16 +1321,18 @@ static bool fanotify_mark_update_flags(struct fsnotify_mark *fsn_mark, static bool fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark, __u32 mask, unsigned int fan_flags) { + __u32 old_mask; bool recalc; spin_lock(&fsn_mark->lock); - if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS)) + if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS)) { + old_mask = fsn_mark->mask; fsn_mark->mask |= mask; - else + recalc = old_mask != fsn_mark->mask; + } else { fsn_mark->ignore_mask |= mask; - - recalc = fsnotify_calc_mask(fsn_mark) & - ~fsnotify_conn_mask(fsn_mark->connector); + recalc = true; + } recalc |= fanotify_mark_update_flags(fsn_mark, fan_flags); spin_unlock(&fsn_mark->lock); diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index ed37491c1618..5f19c24ec187 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -539,7 +539,6 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, { struct fsnotify_mark *fsn_mark; struct inotify_inode_mark *i_mark; - __u32 old_mask, new_mask; int replace = !(arg & IN_MASK_ADD); int create = (arg & IN_MASK_CREATE); int ret; @@ -555,27 +554,15 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); spin_lock(&fsn_mark->lock); - old_mask = fsn_mark->mask; if (replace) { fsn_mark->mask = 0; fsn_mark->flags &= ~INOTIFY_MARK_FLAGS; } fsn_mark->mask |= inotify_arg_to_mask(inode, arg); fsn_mark->flags |= inotify_arg_to_flags(arg); - new_mask = fsn_mark->mask; spin_unlock(&fsn_mark->lock); - if (old_mask != new_mask) { - /* more bits in old than in new? */ - int dropped = (old_mask & ~new_mask); - /* more bits in this fsn_mark than the inode's mask? */ - int do_inode = (new_mask & ~READ_ONCE(inode->i_fsnotify_mask)); - - /* update the inode with this new fsn_mark */ - if (dropped || do_inode) - fsnotify_recalc_mask(fsn_mark->connector); - - } + fsnotify_recalc_mask(fsn_mark->connector); /* return the wd */ ret = i_mark->wd; diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index c621a4c582f9..b1c315206ffa 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -278,7 +278,7 @@ int attr_make_nonresident(struct ntfs_inode *ni, struct ATTRIB *attr, next = Add2Ptr(attr, asize); aoff = PtrOffset(rec, attr); rsize = le32_to_cpu(attr->res.data_size); - is_data = attr->type == ATTR_DATA && !attr->name_len; + is_data = attr->type == ATTR_DATA; /* len - how many clusters required to store 'rsize' bytes */ if (is_attr_compressed(attr)) { @@ -433,6 +433,7 @@ int attr_set_size_ex(struct ntfs_inode *ni, enum ATTR_TYPE type, struct ATTRIB **ret, bool no_da) { int err = 0; + struct ntfs_inode *nb = ni->base; struct ntfs_sb_info *sbi = ni->mi.sbi; u8 cluster_bits = sbi->cluster_bits; bool is_mft = ni->mi.rno == MFT_REC_MFT && type == ATTR_DATA && @@ -703,8 +704,8 @@ pack_runs: goto again; } - if (!ni->attr_list.size) { - err = ni_create_attr_list(ni); + if (!nb->attr_list.size) { + err = ni_create_attr_list(nb); /* In case of error layout of records is not changed. */ if (err) goto undo_2; @@ -877,8 +878,7 @@ ok1: if (ret) *ret = attr_b; - if (((type == ATTR_DATA && !name_len) || - (type == ATTR_ALLOC && name == I30_NAME))) { + if ((type == ATTR_DATA || (type == ATTR_ALLOC && name == I30_NAME))) { /* Update inode_set_bytes. */ if (attr_b->non_res && inode_get_bytes(&ni->vfs_inode) != new_alloc) { @@ -1001,7 +1001,6 @@ int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, struct ATTRIB *attr, *attr_b; struct ATTR_LIST_ENTRY *le, *le_b; struct mft_inode *mi, *mi_b; - struct page *page; CLST hint, svcn, to_alloc, evcn1, next_svcn, asize, end, vcn0; CLST alloc, evcn; unsigned fr; @@ -1026,7 +1025,8 @@ again: step = 0; le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); if (!attr_b) { err = -ENOENT; goto out; @@ -1036,11 +1036,15 @@ again: u32 data_size = le32_to_cpu(attr_b->res.data_size); *lcn = RESIDENT_LCN; *len = data_size; - if (res && data_size) { - page = alloc_page(GFP_KERNEL); - if (!page) { - err = -ENOMEM; - } else { + if (res) { + *res = NULL; + if (data_size) { + struct page *page = alloc_page(GFP_KERNEL); + if (!page) { + err = -ENOMEM; + goto out; + } + *res = page_address(page); memcpy(*res, resident_data(attr_b), data_size); } @@ -1067,7 +1071,8 @@ again: mi = mi_b; if (le_b && (vcn < svcn || evcn1 <= vcn)) { - attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, &mi); if (!attr) { err = -EINVAL; @@ -1140,8 +1145,9 @@ again: if (vcn < svcn || evcn1 <= vcn) { struct ATTRIB *attr2; /* Load runs for truncated vcn. */ - attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, NULL, - 0, &vcn, &mi); + attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, &vcn, &mi); if (!attr2) { err = -EINVAL; goto out; @@ -1155,8 +1161,9 @@ again: if (vcn0 < svcn || evcn1 <= vcn0) { struct ATTRIB *attr2; - attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, NULL, - 0, &vcn0, &mi); + attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, &vcn0, &mi); if (!attr2) { err = -EINVAL; goto out; @@ -1269,8 +1276,9 @@ repack: goto undo1; /* Layout of records is changed. */ le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, - 0, NULL, &mi_b); + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); if (!attr_b) { err = -ENOENT; goto out; @@ -1300,7 +1308,8 @@ repack: svcn = evcn1; /* Estimate next attribute. */ - attr = ni_find_attr(ni, attr, &le, ATTR_DATA, NULL, 0, &svcn, &mi); + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &svcn, &mi); if (!attr) { /* Insert new attribute segment. */ @@ -1333,7 +1342,8 @@ repack: goto out; } - attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, NULL, 0, &le->id); + attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &le->id); if (!attr) { err = -EINVAL; goto out; @@ -1362,9 +1372,10 @@ repack: ins_ext: if (evcn1 > next_svcn) { - err = ni_insert_nonresident(ni, ATTR_DATA, NULL, 0, run, - next_svcn, evcn1 - next_svcn, - attr_b->flags, &attr, &mi, NULL); + err = ni_insert_nonresident(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 - next_svcn, attr_b->flags, + &attr, &mi, NULL); if (err) goto out; } @@ -1398,7 +1409,8 @@ int attr_data_write_resident(struct ntfs_inode *ni, struct folio *folio) struct ATTRIB *attr; u32 data_size; - attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, &mi); + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi); if (!attr) return -EINVAL; @@ -1515,7 +1527,7 @@ int attr_wof_frame_info(struct ntfs_inode *ni, struct ATTRIB *attr, u8 bytes_per_off; char *addr; struct folio *folio; - int i, err; + int i, err = 0; __le32 *off32; __le64 *off64; @@ -1775,7 +1787,8 @@ int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, u64 total_size, valid_size, data_size; le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); if (!attr_b) return -ENOENT; @@ -1798,7 +1811,8 @@ int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, goto out; } else { le = le_b; - attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, &mi); if (!attr) { err = -EINVAL; @@ -1885,8 +1899,9 @@ repack: goto out; /* Layout of records is changed. */ le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, - 0, NULL, &mi_b); + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); if (!attr_b) { err = -ENOENT; goto out; @@ -1902,7 +1917,8 @@ repack: svcn = evcn1; /* Estimate next attribute. */ - attr = ni_find_attr(ni, attr, &le, ATTR_DATA, NULL, 0, &svcn, &mi); + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &svcn, &mi); if (attr) { CLST alloc = bytes_to_cluster( @@ -1931,7 +1947,8 @@ repack: goto out; } - attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, NULL, 0, + attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &le->id); if (!attr) { err = -EINVAL; @@ -1962,9 +1979,10 @@ repack: } ins_ext: if (evcn1 > next_svcn) { - err = ni_insert_nonresident(ni, ATTR_DATA, NULL, 0, run, - next_svcn, evcn1 - next_svcn, - attr_b->flags, &attr, &mi, NULL); + err = ni_insert_nonresident(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 - next_svcn, attr_b->flags, + &attr, &mi, NULL); if (err) goto out; } @@ -2007,7 +2025,8 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) return 0; le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); if (!attr_b) return -ENOENT; @@ -2037,7 +2056,8 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) /* Simple truncate file at 'vbo'. */ truncate_setsize(&ni->vfs_inode, vbo); - err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, vbo, + err = attr_set_size(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &ni->file.run, vbo, &valid_size, true); if (!err && valid_size < ni->i_valid) @@ -2061,7 +2081,8 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) /* * The requested range is full in delayed clusters. */ - err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, run, + err = attr_set_size_ex(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, i_size - bytes, NULL, false, NULL, true); goto out; @@ -2074,7 +2095,8 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) /* Layout of records maybe changed. */ le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, &mi_b); if (!attr_b || !attr_b->non_res) { err = -ENOENT; @@ -2105,7 +2127,8 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) } le = le_b; - attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, &mi); + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &vcn, &mi); if (!attr) { err = -EINVAL; goto out; @@ -2169,7 +2192,8 @@ check_seg: next_svcn = le64_to_cpu(attr->nres.evcn) + 1; if (next_svcn + eat + done < evcn1) { err = ni_insert_nonresident( - ni, ATTR_DATA, NULL, 0, run, next_svcn, + ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, evcn1 - eat - next_svcn, a_flags, &attr, &mi, &le); if (err) @@ -2209,7 +2233,8 @@ check_seg: /* Look for required attribute. */ attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, - NULL, 0, &le->id); + ni->file.ads.name, + ni->file.ads.len, &le->id); if (!attr) { err = -EINVAL; goto out; @@ -2232,7 +2257,8 @@ next_attr: if (!attr_b) { le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, &mi_b); if (!attr_b) { err = -ENOENT; @@ -2293,7 +2319,8 @@ int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size) return 0; le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); if (!attr_b) return -ENOENT; @@ -2364,7 +2391,8 @@ int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size) goto bad_inode; } else { le = le_b; - attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, &mi); if (!attr) { err = -EINVAL; @@ -2416,10 +2444,10 @@ int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size) next_svcn = le64_to_cpu(attr->nres.evcn) + 1; if (next_svcn < evcn1) { /* Insert new attribute segment. */ - err = ni_insert_nonresident(ni, ATTR_DATA, NULL, 0, run, - next_svcn, - evcn1 - next_svcn, a_flags, - &attr, &mi, &le); + err = ni_insert_nonresident( + ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 - next_svcn, a_flags, &attr, &mi, &le); if (err) goto undo_punch; @@ -2454,7 +2482,8 @@ done: goto out; if (!attr_b) { - attr_b = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, + attr_b = ni_find_attr(ni, NULL, NULL, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, &mi_b); if (!attr_b) { err = -EINVAL; @@ -2512,7 +2541,8 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) return 0; le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); if (!attr_b) return -ENOENT; @@ -2559,11 +2589,13 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) down_write(&ni->file.run_lock); if (!attr_b->non_res) { - err = attr_set_size(ni, ATTR_DATA, NULL, 0, run, - data_size + bytes, NULL, false); + err = attr_set_size(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, data_size + bytes, + NULL, false); le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, &mi_b); if (!attr_b) { err = -EINVAL; @@ -2604,7 +2636,8 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) goto bad_inode; } else { le = le_b; - attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, &mi); if (!attr) { err = -EINVAL; @@ -2647,12 +2680,14 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) } if (next_svcn < evcn1 + len) { - err = ni_insert_nonresident(ni, ATTR_DATA, NULL, 0, run, - next_svcn, evcn1 + len - next_svcn, - a_flags, NULL, NULL, NULL); + err = ni_insert_nonresident(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 + len - next_svcn, a_flags, + NULL, NULL, NULL); le_b = NULL; - attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, &mi_b); if (!attr_b) { err = -EINVAL; @@ -2709,7 +2744,8 @@ undo_insert_range: goto bad_inode; } else { le = le_b; - attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, NULL, 0, &vcn, + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, &mi); if (!attr) { goto bad_inode; @@ -2754,7 +2790,8 @@ int attr_force_nonresident(struct ntfs_inode *ni) struct ATTR_LIST_ENTRY *le = NULL; struct mft_inode *mi; - attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, &mi); + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi); if (!attr) { _ntfs_bad_inode(&ni->vfs_inode); return -ENOENT; diff --git a/fs/ntfs3/attrlist.c b/fs/ntfs3/attrlist.c index 270a29323530..8710560cf294 100644 --- a/fs/ntfs3/attrlist.c +++ b/fs/ntfs3/attrlist.c @@ -19,6 +19,7 @@ static inline bool al_is_valid_le(const struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le) { + ni = ni->base; if (!le || !ni->attr_list.le || !ni->attr_list.size) return false; @@ -28,6 +29,7 @@ static inline bool al_is_valid_le(const struct ntfs_inode *ni, void al_destroy(struct ntfs_inode *ni) { + ni = ni->base; run_close(&ni->attr_list.run); kvfree(ni->attr_list.le); ni->attr_list.le = NULL; @@ -47,6 +49,7 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) size_t lsize; void *le = NULL; + ni = ni->base; if (ni->attr_list.size) return 0; @@ -199,6 +202,7 @@ struct ATTR_LIST_ENTRY *al_find_ex(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *ret = NULL; u32 type_in = le32_to_cpu(type); + ni = ni->base; while ((le = al_enumerate(ni, le))) { u64 le_vcn; int diff = le32_to_cpu(le->type) - type_in; @@ -256,6 +260,7 @@ static struct ATTR_LIST_ENTRY *al_find_le_to_insert(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le = NULL, *prev; u32 type_in = le32_to_cpu(type); + ni = ni->base; /* List entries are sorted by type, name and VCN. */ while ((le = al_enumerate(ni, prev = le))) { int diff = le32_to_cpu(le->type) - type_in; @@ -305,6 +310,7 @@ int al_add_le(struct ntfs_inode *ni, enum ATTR_TYPE type, const __le16 *name, u64 new_size; typeof(ni->attr_list) *al = &ni->attr_list; + ni = ni->base; /* * Compute the size of the new 'le' */ @@ -374,8 +380,10 @@ bool al_remove_le(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le) { u16 size; size_t off; - typeof(ni->attr_list) *al = &ni->attr_list; + typeof(ni->attr_list) *al; + ni = ni->base; + al = &ni->attr_list; if (!al_is_valid_le(ni, le)) return false; @@ -395,7 +403,10 @@ int al_update(struct ntfs_inode *ni, int sync) { int err; struct ATTRIB *attr; - typeof(ni->attr_list) *al = &ni->attr_list; + typeof(ni->attr_list) *al; + + ni = ni->base; + al = &ni->attr_list; if (!al->dirty || !al->size) return 0; diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index 873d52233003..eb9152e9fa22 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -25,6 +25,11 @@ int ntfs_utf16_to_nls(struct ntfs_sb_info *sbi, const __le16 *name, u32 len, static_assert(sizeof(wchar_t) == sizeof(__le16)); + if (buf_len <= 0) + return -EINVAL; + + buf_len -= 1; + if (!nls) { /* UTF-16 -> UTF-8 */ ret = utf16s_to_utf8s((wchar_t *)name, len, UTF16_LITTLE_ENDIAN, @@ -179,7 +184,7 @@ int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, struct cpu_str *uni, u32 max_ulen, enum utf16_endian endian) { - int ret, slen; + int ret, slen, i; const u8 *end; struct nls_table *nls = sbi->options->nls; u16 *uname = uni->name; @@ -189,50 +194,83 @@ int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, if (!nls) { /* utf8 -> utf16 */ ret = _utf8s_to_utf16s(name, name_len, endian, uname, max_ulen); - uni->len = ret; - return ret; - } - - for (ret = 0, end = name + name_len; name < end; ret++, name += slen) { - if (ret >= max_ulen) - return -ENAMETOOLONG; + } else { + for (ret = 0, end = name + name_len; name < end; + ret++, name += slen) { + if (ret >= max_ulen) + return -ENAMETOOLONG; - slen = nls->char2uni(name, end - name, uname + ret); - if (!slen) - return -EINVAL; - if (slen < 0) - return slen; - } + slen = nls->char2uni(name, end - name, uname + ret); + if (!slen) + return -EINVAL; + if (slen < 0) + return slen; + } #ifdef __BIG_ENDIAN - if (endian == UTF16_LITTLE_ENDIAN) { - int i = ret; + if (endian == UTF16_LITTLE_ENDIAN) { + i = ret; - while (i--) { - __cpu_to_le16s(uname); - uname++; + while (i--) { + __cpu_to_le16s(uname); + uname++; + } } - } #else - if (endian == UTF16_BIG_ENDIAN) { - int i = ret; + if (endian == UTF16_BIG_ENDIAN) { + i = ret; - while (i--) { - __cpu_to_be16s(uname); - uname++; + while (i--) { + __cpu_to_be16s(uname); + uname++; + } } - } #endif + } uni->len = ret; + uni->ads_len = 0; + if (ret > 0 && sbi->options->ads) { + uname = uni->name; + /* Find delimiter in range [1 : ret-2). */ + for (i = 1; i + 1 < ret; i++) { + if (uname[i] == ':') { + uni->ads_len = ret - i - 1; + uni->len = i; + uname[i] = 0; + ret = i; + + uname += i + 1; + i = uni->ads_len; + /* Return ADS name as little endian. Always */ +#ifdef __BIG_ENDIAN + if (endian == UTF16_LITTLE_ENDIAN) { + while (i--) { + __cpu_to_le16s(uname); + uname++; + } + } +#else + if (endian == UTF16_BIG_ENDIAN) { + while (i--) { + __cpu_to_be16s(uname); + uname++; + } + } +#endif + break; + } + } + } + return ret; } /* * dir_search_u - Helper function. */ -struct inode *dir_search_u(struct inode *dir, const struct cpu_str *uni, - struct ntfs_fnd *fnd) +struct inode *dir_search_flags(struct inode *dir, const struct cpu_str *uni, + struct ntfs_fnd *fnd, u32 flags) { int err = 0; struct super_block *sb = dir->i_sb; @@ -262,7 +300,7 @@ struct inode *dir_search_u(struct inode *dir, const struct cpu_str *uni, goto out; } - inode = ntfs_iget5(sb, &e->ref, uni); + inode = ntfs_iget5_flags(sb, &e->ref, uni, flags); if (!IS_ERR(inode) && is_bad_inode(inode)) { iput(inode); err = -EINVAL; @@ -273,6 +311,12 @@ out: return err == -ENOENT ? NULL : err ? ERR_PTR(err) : inode; } +static inline bool de_fname_fits(const struct NTFS_DE *e, u32 e_size, + const struct ATTR_FILE_NAME *fname) +{ + return sizeof(struct NTFS_DE) + fname_full_size(fname) <= e_size; +} + /* * returns false if 'ctx' if full */ @@ -281,7 +325,7 @@ static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi, u8 *name, struct dir_context *ctx) { const struct ATTR_FILE_NAME *fname; - unsigned long ino; + u64 ino; int name_len; u32 dt_type; @@ -305,15 +349,13 @@ static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi, if (sbi->options->nohidden && (fname->dup.fa & FILE_ATTRIBUTE_HIDDEN)) return true; - if (sizeof(struct NTFS_DE) + - offsetof(struct ATTR_FILE_NAME, name) + - fname->name_len * sizeof(short) > le16_to_cpu(e->size)) + if (!de_fname_fits(e, le16_to_cpu(e->size), fname)) return true; name_len = ntfs_utf16_to_nls(sbi, fname->name, fname->name_len, name, PATH_MAX); if (name_len <= 0) { - ntfs_warn(sbi->sb, "failed to convert name for inode %lx.", + ntfs_warn(sbi->sb, "failed to convert name for inode %llx.", ino); return true; } @@ -576,6 +618,23 @@ out: return err; } +/* + * Return fname when @e passes the same checks as ntfs_dir_emit() before + * exposing an entry (valid key, non-DOS, fname fits in e->size). + */ +static inline const struct ATTR_FILE_NAME * +de_countable_fname(const struct NTFS_DE *e, u32 e_size) +{ + const struct ATTR_FILE_NAME *fname; + + fname = de_get_fname(e); + if (!fname || fname->type == FILE_NAME_DOS || + !de_fname_fits(e, e_size, fname)) + return NULL; + + return fname; +} + static int ntfs_dir_count(struct inode *dir, bool *is_empty, size_t *dirs, size_t *files) { @@ -615,13 +674,10 @@ static int ntfs_dir_count(struct inode *dir, bool *is_empty, size_t *dirs, if (de_is_last(e)) break; - fname = de_get_fname(e); + fname = de_countable_fname(e, e_size); if (!fname) continue; - if (fname->type == FILE_NAME_DOS) - continue; - if (is_empty) { *is_empty = false; if (!dirs && !files) diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index d601f088618c..2abf334bfa0c 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -753,7 +753,9 @@ int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, setattr_copy(idmap, inode, attr); - if (mode != inode->i_mode) { + if (!is_ni_base(ni)) { + ia_valid &= ~ATTR_SIZE; + } else if (mode != inode->i_mode) { err = ntfs_acl_chmod(idmap, dentry); if (err) goto out; @@ -820,15 +822,31 @@ static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) size_t bytes = iov_iter_count(iter); loff_t valid, i_size, vbo, end; unsigned int dio_flags; - ssize_t err; + ssize_t ret; - err = check_read_restriction(inode); - if (err) - return err; + ret = check_read_restriction(inode); + if (ret) + return ret; if (!bytes) return 0; /* skip atime */ + if (ni->file.ads.len == ARRAY_SIZE(QUERY_STREAMS) && + !memcmp(ni->file.ads.name, QUERY_STREAMS, sizeof(QUERY_STREAMS))) { + /* Query ADS. */ + if (unlikely(iocb->ki_flags & IOCB_DIRECT)) { + ntfs_inode_warn( + inode, + "direct I/O for streams is not supported"); + return -EOPNOTSUPP; + } + + inode_lock_shared(inode); + ret = ni_query_ads(ni, &iocb->ki_pos, iter); + inode_unlock_shared(inode); + return ret; + } + if (is_compressed(ni)) { if (iocb->ki_flags & IOCB_DIRECT) { ntfs_inode_warn( @@ -867,17 +885,17 @@ static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) if (ni->file.run_da.count) { /* Direct I/O is not compatible with delalloc. */ - err = ni_allocate_da_blocks(ni); - if (err) + ret = ni_allocate_da_blocks(ni); + if (ret) goto out; } - err = iomap_dio_rw(iocb, iter, &ntfs_iomap_ops, NULL, dio_flags, + ret = iomap_dio_rw(iocb, iter, &ntfs_iomap_ops, NULL, dio_flags, NULL, 0); - if (err <= 0) + if (ret <= 0) goto out; - end = vbo + err; + end = vbo + ret; if (valid < end) { size_t to_zero = end - valid; /* Fix iter. */ @@ -889,35 +907,36 @@ static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) bytes = i_size - vbo; iov_iter_zero(bytes, iter); iocb->ki_pos += bytes; - err = bytes; + ret = bytes; } out: inode_unlock_shared(inode); - file_accessed(iocb->ki_filp); - return err; + file_accessed(file); + return ret; } /* * ntfs_file_splice_read - file_operations::splice_read */ -static ssize_t ntfs_file_splice_read(struct file *in, loff_t *ppos, +static ssize_t ntfs_file_splice_read(struct file *file, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) { - struct inode *inode = file_inode(in); - ssize_t err; + struct inode *inode = file_inode(file); + struct ntfs_inode *ni = ntfs_i(inode); + ssize_t ret; - err = check_read_restriction(inode); - if (err) - return err; + ret = check_read_restriction(inode); + if (ret) + return ret; - if (is_compressed(ntfs_i(inode))) { + if (is_compressed(ni)) { /* Turn off readahead for compressed files. */ - in->f_ra.ra_pages = 0; + file->f_ra.ra_pages = 0; } - return filemap_splice_read(in, ppos, pipe, len, flags); + return filemap_splice_read(file, ppos, pipe, len, flags); } /* @@ -1420,7 +1439,8 @@ static int ntfs_file_release(struct inode *inode, struct file *file) down_write(&ni->file.run_lock); /* Deallocate preallocated. */ - err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, + err = attr_set_size_ex(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &ni->file.run, inode->i_size, &ni->i_valid, false, NULL, true); diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 2b49bc077558..bead01a953f3 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -132,6 +132,13 @@ void ni_clear(struct ntfs_inode *ni) ni->file.offs_folio = NULL; } #endif + kfree(ni->file.ads.name); + ni->file.ads.name = NULL; + } + + if (ni->base && ni->base != ni) { + iput(&ni->base->vfs_inode); + ni->base = NULL; } mi_clear(&ni->mi); @@ -145,6 +152,7 @@ int ni_load_mi_ex(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi) int err; struct mft_inode *r; + ni = ni->base; r = ni_find_mi(ni, rno); if (r) goto out; @@ -169,8 +177,9 @@ out: int ni_load_mi(struct ntfs_inode *ni, const struct ATTR_LIST_ENTRY *le, struct mft_inode **mi) { - CLST rno; + u64 rno; + ni = ni->base; if (!le) { *mi = &ni->mi; return 0; @@ -197,6 +206,7 @@ struct ATTRIB *ni_find_attr(struct ntfs_inode *ni, struct ATTRIB *attr, struct ATTR_LIST_ENTRY *le; struct mft_inode *m; + ni = ni->base; if (!ni->attr_list.size || (!name_len && (type == ATTR_LIST || type == ATTR_STD))) { if (le_o) @@ -257,6 +267,7 @@ struct ATTRIB *ni_enum_attr_ex(struct ntfs_inode *ni, struct ATTRIB *attr, struct mft_inode *mi2; struct ATTR_LIST_ENTRY *le2; + ni = ni->base; /* Do we have an attribute list? */ if (!ni->attr_list.size) { *le = NULL; @@ -290,6 +301,7 @@ int ni_load_all_mi(struct ntfs_inode *ni) int err; struct ATTR_LIST_ENTRY *le; + ni = ni->base; if (!ni->attr_list.size) return 0; @@ -316,6 +328,7 @@ bool ni_add_subrecord(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi) { struct mft_inode *m; + ni = ni->base; m = kzalloc_obj(struct mft_inode, GFP_NOFS); if (!m) return false; @@ -348,6 +361,7 @@ int ni_remove_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, u32 type_in; int diff; + ni = ni->base; if (base_only || type == ATTR_LIST || !ni->attr_list.size) { attr = mi_find_attr(ni, &ni->mi, NULL, type, name, name_len, id); @@ -417,6 +431,7 @@ ni_ins_new_attr(struct ntfs_inode *ni, struct mft_inode *mi, bool le_added = false; struct MFT_REF ref; + ni = ni->base; mi_get_ref(mi, &ref); if (type != ATTR_LIST && !le && ni->attr_list.size) { @@ -618,6 +633,7 @@ static int ni_try_remove_attr_list(struct ntfs_inode *ni) struct MFT_REC *mrec; __le16 id; + ni = ni->base; if (!ni->attr_list.dirty) return 0; @@ -763,15 +779,29 @@ int ni_create_attr_list(struct ntfs_inode *ni) u32 free_b, nb, to_free, rs; u16 sz; + ni = ni->base; is_mft = ni->mi.rno == MFT_REC_MFT; rec = ni->mi.mrec; rs = sbi->record_size; /* - * Skip estimating exact memory requirement. - * Looks like one record_size is always enough. + * Compute the exact size of the attribute list. Each attribute in the + * record yields one ATTR_LIST_ENTRY of le_size(name_len) bytes. The + * minimum on-disk attribute is SIZEOF_RESIDENT (0x18) bytes, but an + * unnamed one expands to le_size(0) (0x20) here, so a record crafted + * with many such attributes needs more than a single record_size; the + * previous fixed kzalloc(record_size) could therefore be overflowed by + * an attacker-controlled record. */ - le = kzalloc(al_aligned(rs), GFP_NOFS); + lsize = 0; + attr = NULL; + while ((attr = mi_enum_attr(ni, &ni->mi, attr))) + lsize += le_size(attr->name_len); + + if (!lsize) + return -EINVAL; + + le = kzalloc(al_aligned(lsize), GFP_NOFS); if (!le) return -ENOMEM; @@ -781,7 +811,6 @@ int ni_create_attr_list(struct ntfs_inode *ni) attr = NULL; nb = 0; free_b = 0; - attr = NULL; for (; (attr = mi_enum_attr(ni, &ni->mi, attr)); le = Add2Ptr(le, sz)) { sz = le_size(attr->name_len); @@ -905,6 +934,7 @@ static int ni_ins_attr_ext(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le, bool is_mft, is_mft_data; struct ntfs_sb_info *sbi = ni->mi.sbi; + ni = ni->base; is_mft = ni->mi.rno == MFT_REC_MFT; is_mft_data = is_mft && type == ATTR_DATA && !name_len; @@ -1057,6 +1087,7 @@ static int ni_insert_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, __le16 id; u16 t16; + ni = ni->base; is_mft = ni->mi.rno == MFT_REC_MFT; rec = ni->mi.mrec; @@ -1203,6 +1234,7 @@ static int ni_expand_mft_list(struct ntfs_inode *ni) struct mft_inode *mi, *mi_min, *mi_new; struct ntfs_sb_info *sbi = ni->mi.sbi; + ni = ni->base; /* Find the nearest MFT. */ mft_min = 0; mft_new = 0; @@ -1335,6 +1367,7 @@ int ni_expand_list(struct ntfs_inode *ni) bool is_mft = ni->mi.rno == MFT_REC_MFT; struct MFT_REF ref; + ni = ni->base; mi_get_ref(&ni->mi, &ref); le = NULL; @@ -1407,6 +1440,7 @@ int ni_insert_nonresident(struct ntfs_inode *ni, enum ATTR_TYPE type, u32 run_size, asize; struct ntfs_sb_info *sbi = ni->mi.sbi; + ni = ni->base; /* Estimate packed size (run_buf=NULL). */ err = run_pack(run, svcn, len, NULL, sbi->max_bytes_per_attr - run_off, &plen); @@ -1476,6 +1510,7 @@ int ni_insert_resident(struct ntfs_inode *ni, u32 data_size, u32 asize = SIZEOF_RESIDENT + name_size + ALIGN(data_size, 8); struct ATTRIB *attr; + ni = ni->base; err = ni_insert_attr(ni, type, name, name_len, asize, SIZEOF_RESIDENT, 0, &attr, mi, le); if (err) @@ -1507,6 +1542,7 @@ int ni_insert_resident(struct ntfs_inode *ni, u32 data_size, void ni_remove_attr_le(struct ntfs_inode *ni, struct ATTRIB *attr, struct mft_inode *mi, struct ATTR_LIST_ENTRY *le) { + ni = ni->base; mi_remove_attr(ni, mi, attr); if (le) @@ -1531,6 +1567,7 @@ int ni_delete_all(struct ntfs_inode *ni) bool nt3 = is_ntfs3(sbi); struct MFT_REF ref; + ni = ni->base; while ((attr = ni_enum_attr_ex(ni, attr, &le, NULL))) { if (!nt3 || attr->name_len) { ; @@ -1608,6 +1645,7 @@ struct ATTR_FILE_NAME *ni_fname_name(struct ntfs_inode *ni, struct ATTRIB *attr = NULL; struct ATTR_FILE_NAME *fname; + ni = ni->base; if (le) *le = NULL; @@ -1649,6 +1687,7 @@ struct ATTR_FILE_NAME *ni_fname_type(struct ntfs_inode *ni, u8 name_type, struct ATTR_FILE_NAME *fname; *le = NULL; + ni = ni->base; if (name_type == FILE_NAME_POSIX) return NULL; @@ -1679,6 +1718,7 @@ int ni_new_attr_flags(struct ntfs_inode *ni, enum FILE_ATTRIBUTE new_fa) __le16 new_aflags; u32 new_asize; + ni = ni->base; attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, &mi); if (!attr) return -EINVAL; @@ -1760,6 +1800,7 @@ enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, u16 len; typeof(rp->CompressReparseBuffer) *cmpr; + ni = ni->base; /* Try to estimate reparse point. */ if (!attr->non_res) { rp = resident_data_ex(attr, sizeof(struct REPARSE_DATA_BUFFER)); @@ -1988,6 +2029,7 @@ int ni_decompress_file(struct ntfs_inode *ni) struct mft_inode *mi; int err; + ni = ni->base; /* Clusters for decompressed data. */ cend = bytes_to_cluster(sbi, i_size); @@ -2249,6 +2291,7 @@ int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, struct ATTRIB *attr; CLST frame, clst_data; + ni = ni->base; /* * To simplify decompress algorithm do vmap for source * and target pages. @@ -2443,6 +2486,15 @@ int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, err = unc_size; else if (!unc_size || unc_size > frame_size) err = -EINVAL; + else if (unc_size < frame_size) { + /* + * Partial decompress: zero the [unc_size, frame_size) + * tail. decompress_lznt() leaves it untouched, so + * without this the freshly vmapped pages would expose + * uninitialized kernel memory to userspace. + */ + memset(frame_mem + unc_size, 0, frame_size - unc_size); + } } if (!err && valid_size < frame_vbo + frame_size) { size_t ok = valid_size - frame_vbo; @@ -2498,6 +2550,7 @@ int ni_write_frame(struct ntfs_inode *ni, struct page **pages, size_t compr_size, ondisk_size; struct lznt *lznt; + ni = ni->base; attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, &mi); if (!attr) { err = -ENOENT; @@ -2623,6 +2676,7 @@ int ni_remove_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, u16 de_key_size = le16_to_cpu(de->key_size); u8 name_type; + ni = ni->base; *undo_step = 0; /* Find name in record. */ @@ -2685,6 +2739,7 @@ bool ni_remove_name_undo(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, struct ATTRIB *attr; u16 de_key_size; + ni = ni->base; switch (undo_step) { case 4: de_key_size = le16_to_cpu(de2->key_size); @@ -2735,6 +2790,7 @@ int ni_add_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, struct ATTR_FILE_NAME *de_name = (struct ATTR_FILE_NAME *)(de + 1); u16 de_key_size = le16_to_cpu(de->key_size); + ni = ni->base; if (sbi->options->windows_names && !valid_windows_name(sbi, (struct le_str *)&de_name->name_len)) return -EINVAL; @@ -2782,6 +2838,7 @@ int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, int err; struct NTFS_DE *de2 = NULL; int undo = 0; + const int way = 1; /* Hope compiler removes below 'else'. */ /* * There are two possible ways to rename: @@ -2793,29 +2850,30 @@ int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, * Second way may result to bad inode if we can't add new name * and then can't restore (add) old name. */ - - /* - * Way 1 - Add new + remove old. - */ - err = ni_add_name(new_dir_ni, ni, new_de); - if (!err) { + if (way == 1) { + /* + * Way 1 - Add new + remove old. + */ + err = ni_add_name(new_dir_ni, ni, new_de); + if (!err) { + err = ni_remove_name(dir_ni, ni, de, &de2, &undo); + if (err && + ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)) + _ntfs_bad_inode(&ni->vfs_inode); + } + } else { + /* + * Way 2 - Remove old + add new. + */ err = ni_remove_name(dir_ni, ni, de, &de2, &undo); - if (err && ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)) - _ntfs_bad_inode(&ni->vfs_inode); + if (!err) { + err = ni_add_name(new_dir_ni, ni, new_de); + if (err && + !ni_remove_name_undo(dir_ni, ni, de, de2, undo)) + _ntfs_bad_inode(&ni->vfs_inode); + } } - /* - * Way 2 - Remove old + add new. - */ - /* - * err = ni_remove_name(dir_ni, ni, de, &de2, &undo); - * if (!err) { - * err = ni_add_name(new_dir_ni, ni, new_de); - * if (err && !ni_remove_name_undo(dir_ni, ni, de, de2, undo)) - * *is_bad = true; - * } - */ - return err; } @@ -2919,7 +2977,6 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) break; } } - } vbo = (u64)vcn << cluster_bits; @@ -2939,6 +2996,9 @@ int ni_write_parents(struct ntfs_inode *ni, int sync) struct ntfs_sb_info *sbi = ni->mi.sbi; struct super_block *sb = sbi->sb; + if (!is_ni_base(ni)) + return 0; + while ((attr = ni_find_attr(ni, attr, &le, ATTR_NAME, NULL, 0, NULL, NULL))) { struct inode *dir; @@ -2961,8 +3021,8 @@ int ni_write_parents(struct ntfs_inode *ni, int sync) if (IS_ERR(dir)) { ntfs_inode_warn( &ni->vfs_inode, - "failed to open parent directory r=%lx to write", - (long)ino_get(&fname->home)); + "failed to open parent directory r=%llx to write", + (u64)ino_get(&fname->home)); continue; } @@ -3081,8 +3141,8 @@ static bool ni_update_parent(struct ntfs_inode *ni, struct NTFS_DUP_INFO *dup, if (IS_ERR(dir)) { ntfs_inode_warn( &ni->vfs_inode, - "failed to open parent directory r=%lx to update", - (long)ino_get(&fname->home)); + "failed to open parent directory r=%llx to update", + (u64)ino_get(&fname->home)); continue; } @@ -3121,6 +3181,9 @@ int ni_write_inode(struct inode *inode, int sync, const char *hint) if (is_bad_inode(inode) || sb_rdonly(sb)) return 0; + if (!is_ni_base(ni)) + return 0; + /* Avoid any operation if inode is bad. */ if (unlikely(is_bad_ni(ni))) return -EINVAL; @@ -3305,10 +3368,84 @@ int ni_allocate_da_blocks_locked(struct ntfs_inode *ni) /* * Normal file allocates clusters in 'attr_set_size' */ - err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, + err = attr_set_size_ex(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &ni->file.run, ni->vfs_inode.i_size, &ni->i_valid, false, NULL, true); } return err; } + +/* + * Helper function to read ADS. + * bytes = iov_iter_count(iter) is not 0. Checked by caller; + */ +ssize_t ni_query_ads(struct ntfs_inode *ni, loff_t *pos, struct iov_iter *iter) +{ + ssize_t ret = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + size_t bytes = iov_iter_count(iter); + loff_t end = *pos + bytes; + char *buf = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + u64 vbo = 0; + struct ATTRIB *attr; + size_t done, used; + int err; + + /* + * Enumerate ADS. + */ + ni = ni->base; + attr = NULL; + while ((attr = ni_enum_attr_ex(ni, attr, &le, NULL))) { + if (attr->type != ATTR_DATA || !attr->name_len) + continue; + + if (!buf) { + buf = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!buf) + return -ENOMEM; + } + + /* attr - named DATA attribute (ADS). */ + err = ntfs_utf16_to_nls(sbi, attr_name(attr), attr->name_len, + buf, PAGE_SIZE); + if (err < 0) { + ret = err; + break; + } + + /* + * err is the length of ADS name in bytes. + * Copy pseudo data interval [vbo, err + 1). + * into 'iter': [*pos, bytes) + */ + /* Add \n as streams name separator. */ + buf[err++] = '\n'; + + if (vbo >= end) { + iov_iter_zero(bytes, iter); + break; + } + + if (vbo + err > *pos) { + size_t off = *pos - vbo; + used = err - off; + done = copy_to_iter(Add2Ptr(buf, off), min(used, bytes), + iter); + } else { + done = 0; + } + + ret += done; + *pos += done; + if (done >= bytes) + break; + bytes -= done; + vbo += err; + } + kfree(buf); + return ret; +} diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index f038c799e7ac..ed50c1d0c23e 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -648,6 +648,14 @@ static inline void *enum_rstbl(struct RESTART_TABLE *t, void *c) } /* + * dp_range_ok - true if [j, j + count) fits in a page_lcns[cap] array. + */ +static inline bool dp_range_ok(size_t j, u32 count, u32 cap) +{ + return j < cap && count <= cap - j; +} + +/* * find_dp - Search for a @vcn in Dirty Page Table. */ static inline struct DIR_PAGE_ENTRY *find_dp(struct RESTART_TABLE *dptbl, @@ -789,6 +797,20 @@ static bool check_rstbl(const struct RESTART_TABLE *rt, size_t bytes) return true; } +static bool check_dp_table(const struct RESTART_TABLE *dptbl) +{ + u32 rsize = le16_to_cpu(dptbl->size); + struct DIR_PAGE_ENTRY *dp = NULL; + + while ((dp = enum_rstbl((struct RESTART_TABLE *)dptbl, dp))) { + if (struct_size(dp, page_lcns, le32_to_cpu(dp->lcns_follow)) > + rsize) + return false; + } + + return true; +} + /* * free_rsttbl_idx - Free a previously allocated index a Restart Table. */ @@ -853,6 +875,9 @@ static inline struct RESTART_TABLE *extend_rsttbl(struct RESTART_TABLE *tbl, u32 used = le16_to_cpu(tbl->used); struct RESTART_TABLE *rt; + if (used + add > U16_MAX) + return NULL; + rt = init_rsttbl(esize, used + add); if (!rt) return NULL; @@ -2276,7 +2301,15 @@ static int read_log_rec_buf(struct ntfs_log *log, */ for (;;) { bool usa_error; - u32 tail = log->page_size - off; + u32 tail; + + /* off comes from the on-disk restart area; bound it. */ + if (off > log->page_size) { + err = -EINVAL; + goto out; + } + + tail = log->page_size - off; if (tail >= data_len) tail = data_len; @@ -2613,7 +2646,6 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) const bool has_subnode = hdr_has_subnode(hdr); __le16 mask; u32 min_de, de_off, used, total; - const struct NTFS_DE *e; if (has_subnode) { min_de = sizeof(struct NTFS_DE) + sizeof(u64); @@ -2632,8 +2664,8 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) return false; } - e = (const struct NTFS_DE *)((const u8 *)hdr + de_off); for (;;) { + const struct NTFS_DE *e = Add2Ptr(hdr, de_off); u16 esize = le16_to_cpu(e->size); u16 key_size = le16_to_cpu(e->key_size); u16 data_size; @@ -2649,7 +2681,6 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) if (de_is_last(e)) { if (key_size) return false; - break; } @@ -2658,7 +2689,6 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) return false; de_off += esize; - e = (const struct NTFS_DE *)((const u8 *)hdr + de_off); } return true; @@ -3544,8 +3574,7 @@ move_data: * bound here so the memmove cannot reach past the entry. */ if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || - le16_to_cpu(e->view.data_off) + dlen > - le16_to_cpu(e->size)) + le16_to_cpu(e->view.data_off) + dlen > le16_to_cpu(e->size)) goto dirty_vol; memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); @@ -3756,8 +3785,7 @@ move_data: /* See UpdateRecordDataRoot for the rationale. */ if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || - le16_to_cpu(e->view.data_off) + dlen > - le16_to_cpu(e->size)) + le16_to_cpu(e->view.data_off) + dlen > le16_to_cpu(e->size)) goto dirty_vol; memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); @@ -4295,6 +4323,11 @@ check_dirty_page_table: goto out; } + if (!check_dp_table(rt)) { + err = -EINVAL; + goto out; + } + dptbl = kmemdup(rt, t32, GFP_NOFS); if (!dptbl) { err = -ENOMEM; @@ -4653,11 +4686,11 @@ copy_lcns: } /* - * find_dp() only validates that target_vcn is the first - * cluster covered by dp. The walk through lrh->lcns_follow - * further entries must stay within the allocated - * dp->page_lcns[] array, which is sized by dp->lcns_follow. - */ + * find_dp() only validates that target_vcn is the first + * cluster covered by dp. The walk through lrh->lcns_follow + * further entries must stay within the allocated + * dp->page_lcns[] array, which is sized by dp->lcns_follow. + */ if (le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn) + t16 > le32_to_cpu(dp->lcns_follow)) { err = -EINVAL; @@ -5087,6 +5120,13 @@ find_dirty_page: /* Shorten length by any Lcns which were deleted. */ saved_len = dlen; + if (!dp_range_ok(le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn), + le16_to_cpu(lrh->lcns_follow), + le32_to_cpu(dp->lcns_follow))) { + err = -EINVAL; + goto out; + } + for (i = le16_to_cpu(lrh->lcns_follow); i; i--) { size_t j; u32 alen, voff; diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index bc7469d0a34d..97c04ab2763a 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -88,6 +88,13 @@ const __le16 SQ_NAME[2] = { const __le16 SR_NAME[2] = { cpu_to_le16('$'), cpu_to_le16('R'), }; +const __le16 QUERY_STREAMS[13] = { + cpu_to_le16('q'), cpu_to_le16('u'), cpu_to_le16('e'), cpu_to_le16('r'), + cpu_to_le16('y'), cpu_to_le16('_'), cpu_to_le16('s'), cpu_to_le16('t'), + cpu_to_le16('r'), cpu_to_le16('e'), cpu_to_le16('a'), cpu_to_le16('m'), + cpu_to_le16('s'), +}; + #ifdef CONFIG_NTFS3_LZX_XPRESS const __le16 WOF_NAME[17] = { @@ -122,7 +129,6 @@ static const __le16 COM_NAME[3] = { static const __le16 LPT_NAME[3] = { cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), }; - // clang-format on /* @@ -236,7 +242,7 @@ int ntfs_extend_init(struct ntfs_sb_info *sbi) } /* Try to find $ObjId */ - inode2 = dir_search_u(inode, &NAME_OBJID, NULL); + inode2 = dir_search(inode, &NAME_OBJID); if (inode2 && !IS_ERR(inode2)) { if (is_bad_inode(inode2)) { iput(inode2); @@ -247,21 +253,21 @@ int ntfs_extend_init(struct ntfs_sb_info *sbi) } /* Try to find $Quota */ - inode2 = dir_search_u(inode, &NAME_QUOTA, NULL); + inode2 = dir_search(inode, &NAME_QUOTA); if (inode2 && !IS_ERR(inode2)) { sbi->quota_no = inode2->i_ino; iput(inode2); } /* Try to find $Reparse */ - inode2 = dir_search_u(inode, &NAME_REPARSE, NULL); + inode2 = dir_search(inode, &NAME_REPARSE); if (inode2 && !IS_ERR(inode2)) { sbi->reparse.ni = ntfs_i(inode2); sbi->reparse_no = inode2->i_ino; } /* Try to find $UsnJrnl */ - inode2 = dir_search_u(inode, &NAME_USNJRNL, NULL); + inode2 = dir_search(inode, &NAME_USNJRNL); if (inode2 && !IS_ERR(inode2)) { sbi->usn_jrnl_no = inode2->i_ino; iput(inode2); @@ -475,7 +481,7 @@ bool ntfs_check_free_space(struct ntfs_sb_info *sbi, CLST clen, CLST mlen, avail = free - (zlen + clen); - /* + /* * When delalloc is active then keep in mind some reserved space. * The worst case: 1 mft record per each ~500 clusters. */ @@ -1705,6 +1711,8 @@ struct ntfs_inode *ntfs_new_inode(struct ntfs_sb_info *sbi, CLST rno, goto out; } + ni->base = ni; + out: if (err) { make_bad_inode(inode); @@ -2302,8 +2310,8 @@ int ntfs_reparse_init(struct ntfs_sb_info *sbi) goto out; } - root_r = resident_data(attr); - if (root_r->type != ATTR_ZERO || + root_r = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root_r || root_r->type != ATTR_ZERO || root_r->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; @@ -2340,8 +2348,8 @@ int ntfs_objid_init(struct ntfs_sb_info *sbi) goto out; } - root = resident_data(attr); - if (root->type != ATTR_ZERO || + root = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root || root->type != ATTR_ZERO || root->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; @@ -2664,6 +2672,12 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) if (err < 0) goto out; + if (uni->ads_len) { + /* Undo delimiter parse */ + uni->len += uni->ads_len + 1; + uni->ads_len = 0; + } + uni_bytes = uni->len * sizeof(u16); if (uni_bytes > NTFS_LABEL_MAX_LENGTH * sizeof(u16)) { ntfs_warn(sbi->sb, "new label is too long"); diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 2b439ac04356..689712d3463d 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -612,8 +612,8 @@ static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr, static bool index_hdr_check(const struct INDEX_HDR *hdr, u32 bytes) { const bool has_subnode = hdr_has_subnode(hdr); - const u16 min_size = sizeof(struct NTFS_DE) + - (has_subnode ? sizeof(u64) : 0); + const u16 min_size = + sizeof(struct NTFS_DE) + (has_subnode ? sizeof(u64) : 0); u32 end = le32_to_cpu(hdr->used); u32 tot = le32_to_cpu(hdr->total); u32 off = le32_to_cpu(hdr->de_off); @@ -1325,6 +1325,7 @@ pop_level: /* Pop one level. */ if (n) { fnd_pop(fnd); + kfree(n->index); kfree(n); } @@ -2131,8 +2132,7 @@ static struct indx_node *indx_find_buffer(struct ntfs_index *indx, if (err) return ERR_PTR(err); - r = indx_find_buffer(indx, ni, root, vbn, n, - depth + 1); + r = indx_find_buffer(indx, ni, root, vbn, n, depth + 1); if (r) return r; } diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 6ffe99da4d2a..56b4f6469a28 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -18,12 +18,16 @@ #include "ntfs.h" #include "ntfs_fs.h" +struct IGET5_PARAM { + const struct MFT_REF *ref; + const struct cpu_str *name; +}; + /* * ntfs_read_mft - Read record and parse MFT. */ -static struct inode *ntfs_read_mft(struct inode *inode, - const struct cpu_str *name, - const struct MFT_REF *ref) +static int ntfs_read_mft(struct inode *inode, const struct cpu_str *name, + const struct MFT_REF *ref) { int err = 0; struct ntfs_inode *ni = ntfs_i(inode); @@ -36,7 +40,7 @@ static struct inode *ntfs_read_mft(struct inode *inode, bool is_match = false; bool is_root = false; bool is_dir; - unsigned long ino = inode->i_ino; + u64 ino = inode->i_ino; u32 rp_fa = 0, asize, t32; u16 roff, rsize, names = 0, links = 0; const struct ATTR_FILE_NAME *fname = NULL; @@ -46,6 +50,7 @@ static struct inode *ntfs_read_mft(struct inode *inode, struct MFT_REC *rec; struct runs_tree *run; struct timespec64 ts; + const __le16 *aname; inode->i_op = NULL; /* Setup 'uid' and 'gid' */ @@ -79,7 +84,7 @@ static struct inode *ntfs_read_mft(struct inode *inode, ; } else if (ref->seq != rec->seq) { err = -EINVAL; - ntfs_err(sb, "MFT: r=%lx, expect seq=%x instead of %x!", ino, + ntfs_err(sb, "MFT: r=%llx, expect seq=%x instead of %x!", ino, le16_to_cpu(ref->seq), le16_to_cpu(rec->seq)); goto out; } else if (!is_rec_inuse(rec)) { @@ -127,10 +132,16 @@ next_attr: if (le && le->vcn) { /* This is non primary attribute segment. Ignore if not MFT. */ - if (ino != MFT_REC_MFT || attr->type != ATTR_DATA) + if (ino != MFT_REC_MFT) + goto next_attr; + + if (attr->type == ATTR_DATA) + run = &ni->file.run; + else if (attr->type == ATTR_BITMAP) + run = &sbi->mft.bitmap.run; + else goto next_attr; - run = &ni->file.run; asize = le32_to_cpu(attr->size); goto attr_unpack_run; } @@ -138,6 +149,7 @@ next_attr: roff = attr->non_res ? 0 : le16_to_cpu(attr->res.data_off); rsize = attr->non_res ? 0 : le32_to_cpu(attr->res.data_size); asize = le32_to_cpu(attr->size); + aname = attr_name(attr); /* * Really this check was done in 'ni_enum_attr_ex' -> ... 'mi_enum_attr'. @@ -224,10 +236,10 @@ next_attr: if (attr->name_len && ((ino != MFT_REC_BADCLUST || !attr->non_res || attr->name_len != ARRAY_SIZE(BAD_NAME) || - memcmp(attr_name(attr), BAD_NAME, sizeof(BAD_NAME))) && + memcmp(aname, BAD_NAME, sizeof(BAD_NAME))) && (ino != MFT_REC_SECURE || !attr->non_res || attr->name_len != ARRAY_SIZE(SDS_NAME) || - memcmp(attr_name(attr), SDS_NAME, sizeof(SDS_NAME))))) { + memcmp(aname, SDS_NAME, sizeof(SDS_NAME))))) { /* File contains stream attribute. Ignore it. */ goto next_attr; } @@ -247,14 +259,11 @@ next_attr: else ni->std_fa &= ~FILE_ATTRIBUTE_ENCRYPTED; - if (!attr->non_res) { - ni->i_valid = inode->i_size = rsize; - inode_set_bytes(inode, rsize); - } - mode = S_IFREG | (0777 & sbi->options->fs_fmask_inv); if (!attr->non_res) { + ni->i_valid = inode->i_size = rsize; + inode_set_bytes(inode, rsize); ni->ni_flags |= NI_FLAG_RESIDENT; goto next_attr; } @@ -495,16 +504,136 @@ end_enum: if (ino == MFT_REC_MFT && !sb->s_root) sbi->mft.ni = NULL; - unlock_new_inode(inode); - - return inode; + return 0; out: if (ino == MFT_REC_MFT && !sb->s_root) sbi->mft.ni = NULL; - iget_failed(inode); - return ERR_PTR(err); + return err; +} + +/* + * ntfs_init_ads_node + * + * This function scans base inode for given ADS. + * And init inode associated with this ADS + */ +static int ntfs_init_ads_node(struct inode *inode, const __le16 *ads_name, + u8 ads_len, u32 flags) +{ + int err = -EINVAL; + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_inode *nb = ni->base; + struct ntfs_sb_info *sbi = nb->mi.sbi; + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr = NULL; + u16 roff, asize; + u64 svcn; + + if (nb->ni_flags & NI_FLAG_DIR) + return -EINVAL; /* no ADS for directories. */ + + ni->mi.sbi = sbi; + ni->mi.rno = inode->i_ino; + + if (ads_len == ARRAY_SIZE(QUERY_STREAMS) && + !memcmp(ads_name, QUERY_STREAMS, sizeof(QUERY_STREAMS))) { + goto ok; /* use goto to reduce tab pressure. */ + } + + /* Enumerate all attributes in record. */ + while ((attr = ni_enum_attr_ex(nb, attr, &le, NULL))) { + if (attr->type == ATTR_DATA && attr->name_len && + ads_len == attr->name_len && + !memcmp(ads_name, attr_name(attr), ads_len * sizeof(u16))) { + /* We have found the ADS to open. */ + break; + } + } + + if (!attr) { + if (!(flags & LOOKUP_CREATE)) { + /* Do not create ADS. */ + return -ENOENT; + } + + /* Create new ADS. */ + err = ni_insert_resident(nb, 0, ATTR_DATA, ads_name, ads_len, + &attr, NULL, NULL); + if (err) { + /* Looks like the only reasons: ENOSPC/ENOMEM .*/ + return err; + } + } + + if (is_attr_sparsed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_SPARSE_FILE; + else + ni->std_fa &= ~FILE_ATTRIBUTE_SPARSE_FILE; + + if (is_attr_compressed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_COMPRESSED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_COMPRESSED; + + if (is_attr_encrypted(attr)) + ni->std_fa |= FILE_ATTRIBUTE_ENCRYPTED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_ENCRYPTED; + + if (!attr->non_res) { + ni->ni_flags |= NI_FLAG_RESIDENT; + ni->i_valid = inode->i_size = le32_to_cpu(attr->res.data_size); + inode_set_bytes(inode, inode->i_size); + goto ok; + } + + inode_set_bytes(inode, attr_ondisk_size(attr)); + ni->i_valid = le64_to_cpu(attr->nres.valid_size); + inode->i_size = le64_to_cpu(attr->nres.data_size); + + if (!attr->nres.alloc_size) + goto ok; + + roff = le16_to_cpu(attr->nres.run_off); + asize = le32_to_cpu(attr->size); + + if (roff > asize) { + /* This case should be checked in mi_enum_attr */ + return -EINVAL; + } + + svcn = le64_to_cpu(attr->nres.svcn); + err = run_unpack_ex(&ni->file.run, sbi, ni->mi.rno, svcn, + le64_to_cpu(attr->nres.evcn), svcn, + Add2Ptr(attr, roff), asize - roff); + if (err < 0) { + /* run_unpack_ex marks volume dirty, if logical error. */ + return err; + } + +ok: + /* Keep ADS name (little endian). */ + ni->file.ads.name = kmemdup(ads_name, ads_len * sizeof(u16), GFP_NOFS); + if (!ni->file.ads.name) + return -ENOMEM; + ni->file.ads.len = ads_len; + + set_nlink(inode, 1); + + init_rwsem(&ni->file.run_lock); + /* Most fields are the same as the base's? */ + inode->i_op = nb->vfs_inode.i_op; + inode->i_fop = nb->vfs_inode.i_fop; + inode->i_mapping->a_ops = nb->vfs_inode.i_mapping->a_ops; + inode->i_flags = nb->vfs_inode.i_flags; + inode->i_mode = nb->vfs_inode.i_mode; + inode->i_uid = nb->vfs_inode.i_uid; + inode->i_gid = nb->vfs_inode.i_gid; + inode->i_generation = nb->vfs_inode.i_generation; + + return 0; } /* @@ -514,44 +643,120 @@ out: */ static int ntfs_test_inode(struct inode *inode, void *data) { - struct MFT_REF *ref = data; + const struct IGET5_PARAM *ig5 = data; + struct ntfs_inode *ni; + const struct cpu_str *name; - return ino_get(ref) == inode->i_ino; + if (ino_get(ig5->ref) != inode->i_ino) + return 0; + + ni = ntfs_i(inode); + + if (ni->ni_flags & NI_FLAG_DIR) { + /* No ads for directories. */ + return 1; + } + + name = ig5->name; + if (!name || !name->ads_len) { + if (!ni->file.ads.len) { + /* default file (not ads) match. */ + return 1; + } + } else if (ni->file.ads.len == name->ads_len && + !memcmp(ni->file.ads.name, &name->name[name->len + 1], + name->ads_len * sizeof(u16))) { + /* ads name match. */ + return 1; + } + + return 0; } static int ntfs_set_inode(struct inode *inode, void *data) { - const struct MFT_REF *ref = data; + const struct IGET5_PARAM *ig5 = data; - inode->i_ino = ino_get(ref); + inode->i_ino = ino_get(ig5->ref); return 0; } -struct inode *ntfs_iget5(struct super_block *sb, const struct MFT_REF *ref, - const struct cpu_str *name) +struct inode *ntfs_iget5_flags(struct super_block *sb, + const struct MFT_REF *ref, + const struct cpu_str *name, u32 flags) { - struct inode *inode; + int err; + /* Pack params to pass in iget5_locked. */ + struct IGET5_PARAM ig5 = { ref, name }; + u64 ino = ino_get(ref); + struct inode *inode, *base = NULL; + bool ads = name && name->ads_len; + struct ntfs_inode *ni; + + if (ads) { + /* First get base inode */ + base = ntfs_iget5_flags(sb, ref, NULL, 0); + if (IS_ERR(base)) + return base; + } + + inode = iget5_locked(sb, ino, ntfs_test_inode, ntfs_set_inode, &ig5); + if (unlikely(!inode)) { + err = -ENOMEM; + goto out; + } - inode = iget5_locked(sb, ino_get(ref), ntfs_test_inode, ntfs_set_inode, - (void *)ref); - if (unlikely(!inode)) - return ERR_PTR(-ENOMEM); + ni = ntfs_i(inode); /* If this is a freshly allocated inode, need to read it now. */ - if (inode_state_read_once(inode) & I_NEW) - inode = ntfs_read_mft(inode, name, ref); - else if (ref->seq != ntfs_i(inode)->mi.mrec->seq) { + if (inode_state_read_once(inode) & I_NEW) { + if (!base) { + /* default inode. generic file/dir. */ + ni->base = ni; + } else { + /* inode + ads */ + ni->base = ntfs_i(base); + base = NULL; /* keep reference incremented (instead of ihold). */ + } + + if (ads) { + /* base record is loaded. Init ads node. */ + err = ntfs_init_ads_node( + inode, (__le16 *)&name->name[name->len + 1], + name->ads_len, flags); + } else { + err = ntfs_read_mft(inode, name, ref); + } + + if (!err) { + unlock_new_inode(inode); + } else { + iget_failed(inode); + /* Do not mark volume dirty if ADS not found. */ + if (ads) + goto out; + } + } else if (!ads && ref->seq != ni->mi.mrec->seq) { /* * Sequence number is not expected. * Looks like inode was reused but caller uses the old reference */ iput(inode); - inode = ERR_PTR(-ESTALE); + err = -ESTALE; + } else { + err = 0; } - if (IS_ERR(inode)) + if (err) ntfs_set_state(sb->s_fs_info, NTFS_DIRTY_ERROR); +out: + if (base) + iput(base); + + if (err) + return ERR_PTR(err); + return inode; } @@ -606,15 +811,17 @@ static void ntfs_iomap_read_end_io(struct bio *bio) } static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter, - struct iomap_read_folio_ctx *ctx) + struct iomap_read_folio_ctx *ctx) { iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io); } +// clang-format off static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { .read_folio_range = iomap_bio_read_folio_range, .submit_read = ntfs_iomap_bio_submit_read, }; +// clang-format on static int ntfs_read_folio(struct file *file, struct folio *folio) { @@ -698,8 +905,8 @@ int ntfs_set_size(struct inode *inode, u64 new_size) ni->i_valid = new_size; /* last 'true' means keep preallocated. */ - err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, - &ni->i_valid, true); + err = attr_set_size(ni, ATTR_DATA, ni->file.ads.name, ni->file.ads.len, + &ni->file.run, new_size, &ni->i_valid, true); up_write(&ni->file.run_lock); ni_unlock(ni); @@ -793,7 +1000,8 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (lcn == RESIDENT_LCN) { if (offset >= clen) { - __free_page(virt_to_page(res)); + if (res) + __free_page(virt_to_page(res)); if (flags & IOMAP_REPORT) { /* special code for report. */ return -ENOENT; @@ -884,7 +1092,8 @@ static int ntfs_iomap_end(struct inode *inode, loff_t pos, loff_t length, struct ATTRIB *attr; struct mft_inode *mi; - attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, &mi); if (!attr || attr->non_res) { err = -EINVAL; @@ -1206,6 +1415,15 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, if (!fnd) ni_lock_dir(dir_ni); + if (sbi->options->ads) { + const char *ads = strchr(name->name + 1, ':'); + if (ads && ads[1]) { + ntfs_warn(sb, "failed to create ads"); + err = -EINVAL; + goto out1; + } + } + dir_root = indx_get_root(&dir_ni->dir, dir_ni, NULL, NULL); if (!dir_root) { err = -EINVAL; diff --git a/fs/ntfs3/lznt.c b/fs/ntfs3/lznt.c index f818d9785004..5dcb7674790c 100644 --- a/fs/ntfs3/lznt.c +++ b/fs/ntfs3/lznt.c @@ -240,8 +240,10 @@ static inline ssize_t decompress_chunk(u8 *unc, u8 *unc_end, const u8 *cmpr, if (up - unc > LZNT_CHUNK_SIZE) return -EINVAL; /* Correct index */ - while (index < ARRAY_SIZE(s_max_off) - 1 && unc + s_max_off[index] < up) + while (index < ARRAY_SIZE(s_max_off) - 1 && + unc + s_max_off[index] < up) { index += 1; + } /* Check the current flag for zero. */ if (!(ch & (1 << bit))) { diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c index 5ab45d17333e..ec59bbabd3c5 100644 --- a/fs/ntfs3/namei.c +++ b/fs/ntfs3/namei.c @@ -22,7 +22,7 @@ int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name, { int err; struct NTFS_DE *e = buf; - u16 data_size; + u16 data_size, real_size, aligned_size; struct ATTR_FILE_NAME *fname = (struct ATTR_FILE_NAME *)(e + 1); #ifndef CONFIG_NTFS3_64BIT_CLUSTER @@ -53,7 +53,12 @@ int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name, fname->type = FILE_NAME_POSIX; data_size = fname_full_size(fname); - e->size = cpu_to_le16(ALIGN(data_size, 8) + sizeof(struct NTFS_DE)); + real_size = data_size + sizeof(struct NTFS_DE); + aligned_size = ALIGN(data_size, 8) + sizeof(struct NTFS_DE); + if (aligned_size > real_size) + memset((char *)buf + real_size, 0, aligned_size - real_size); + + e->size = cpu_to_le16(aligned_size); e->key_size = cpu_to_le16(data_size); e->flags = 0; e->res = 0; @@ -73,21 +78,23 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, int err; if (!uni) - inode = ERR_PTR(-ENOMEM); - else { - err = ntfs_nls_to_utf16(ni->mi.sbi, dentry->d_name.name, - dentry->d_name.len, uni, NTFS_NAME_LEN, - UTF16_HOST_ENDIAN); - if (err < 0) - inode = ERR_PTR(err); - else { - ni_lock_dir(ni); - inode = dir_search_u(dir, uni, NULL); - ni_unlock(ni); - } + return ERR_PTR(-ENOMEM); + + err = ntfs_nls_to_utf16(ni->mi.sbi, dentry->d_name.name, + dentry->d_name.len, uni, NTFS_NAME_LEN, + UTF16_HOST_ENDIAN); + + if (err < 0) { kfree(uni); + return ERR_PTR(err); } + ni_lock_dir(ni); + inode = dir_search_flags(dir, uni, NULL, flags); + ni_unlock(ni); + + kfree(uni); + /* * Check for a null pointer * If the MFT record of ntfs inode is not a base record, inode->i_op can be NULL. @@ -95,7 +102,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, */ if (!IS_ERR_OR_NULL(inode) && !inode->i_op) { iput(inode); - inode = ERR_PTR(-EINVAL); + return ERR_PTR(-EINVAL); } return d_splice_alias(inode, dentry); @@ -168,7 +175,9 @@ static int ntfs_link(struct dentry *ode, struct inode *dir, struct dentry *de) */ static int ntfs_unlink(struct inode *dir, struct dentry *dentry) { - struct ntfs_inode *ni = ntfs_i(dir); + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); int err; /* Avoid any operation if inode is bad. */ @@ -178,11 +187,21 @@ static int ntfs_unlink(struct inode *dir, struct dentry *dentry) if (unlikely(ntfs3_forced_shutdown(dir->i_sb))) return -EIO; - ni_lock_dir(ni); - - err = ntfs_unlink_inode(dir, dentry); + if (likely(is_ni_base(ni))) { + ni_lock_dir(dir_ni); + /* Remove general file/dir. */ + err = ntfs_unlink_inode(dir, dentry); + ni_unlock(dir_ni); + } else { + ni_lock(ni); + /* Remove ADS. */ + err = ni_remove_attr(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, false, NULL); + ni_unlock(ni); - ni_unlock(ni); + if (!err) + drop_nlink(inode); + } return err; } @@ -268,6 +287,11 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, 1024); static_assert(PATH_MAX >= 4 * 1024); + if (!is_ni_base(ni)) { + /* No rename for ADS. */ + return -EOPNOTSUPP; + } + /* Avoid any operation if inode is bad. */ if (unlikely(is_bad_ni(ni))) return -EINVAL; diff --git a/fs/ntfs3/ntfs.h b/fs/ntfs3/ntfs.h index 892f13e65d42..4589b16329c9 100644 --- a/fs/ntfs3/ntfs.h +++ b/fs/ntfs3/ntfs.h @@ -58,7 +58,7 @@ struct GUID { */ struct cpu_str { u8 len; - u8 unused; + u8 ads_len; u16 name[]; }; @@ -170,6 +170,7 @@ extern const __le16 SDH_NAME[4]; extern const __le16 SO_NAME[2]; extern const __le16 SQ_NAME[2]; extern const __le16 SR_NAME[2]; +extern const __le16 QUERY_STREAMS[13]; extern const __le16 BAD_NAME[4]; extern const __le16 SDS_NAME[4]; diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index d98d7e474476..5811d89d67b3 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -110,6 +110,7 @@ struct ntfs_mount_options { unsigned prealloc : 1; /* Preallocate space when file is growing. */ unsigned nocase : 1; /* case insensitive. */ unsigned delalloc : 1; /* delay allocation. */ + unsigned ads : 1; /* ads support. */ }; /* Special value to unpack and deallocate. */ @@ -401,7 +402,7 @@ struct ntfs_inode { struct rw_semaphore run_lock; /* Unpacked runs from just one record. */ struct runs_tree run; - /* + /* * Pairs [vcn, len] for all delay allocated clusters. * Normal file always contains delayed clusters in one fragment. * TODO: use 2 CLST per pair instead of 3. @@ -410,6 +411,11 @@ struct ntfs_inode { #ifdef CONFIG_NTFS3_LZX_XPRESS struct folio *offs_folio; #endif + /* Alternative data stream */ + struct { + __le16 *name; + u8 len; + } ads; } file; }; @@ -421,6 +427,7 @@ struct ntfs_inode { } attr_list; size_t ni_flags; // NI_FLAG_XXX + struct ntfs_inode *base; /* ADS: points to base inode. Other: this. */ struct inode vfs_inode; }; @@ -444,6 +451,11 @@ enum REPARSE_SIGN { REPARSE_LINK = 3 }; +static inline bool is_ni_base(const struct ntfs_inode *ni) +{ + return ni == ni->base; +} + /* Functions from attrib.c */ int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, struct runs_tree *run_da, CLST vcn, CLST lcn, @@ -526,8 +538,14 @@ int ntfs_utf16_to_nls(struct ntfs_sb_info *sbi, const __le16 *name, u32 len, int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, struct cpu_str *uni, u32 max_ulen, enum utf16_endian endian); -struct inode *dir_search_u(struct inode *dir, const struct cpu_str *uni, - struct ntfs_fnd *fnd); +struct inode *dir_search_flags(struct inode *dir, const struct cpu_str *uni, + struct ntfs_fnd *fnd, u32 flags); +static inline struct inode *dir_search(struct inode *dir, + const struct cpu_str *uni) +{ + return dir_search_flags(dir, uni, NULL, 0); +} + bool dir_is_empty(struct inode *dir); extern const struct file_operations ntfs_dir_operations; @@ -622,6 +640,7 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data); int ni_write_parents(struct ntfs_inode *ni, int sync); int ni_allocate_da_blocks(struct ntfs_inode *ni); int ni_allocate_da_blocks_locked(struct ntfs_inode *ni); +ssize_t ni_query_ads(struct ntfs_inode *ni, loff_t *pos, struct iov_iter *iter); /* Globals from fslog.c */ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes); @@ -679,7 +698,6 @@ static inline int ntfs_read_bh(struct ntfs_sb_info *sbi, { return ntfs_read_bh_ra(sbi, run, vbo, rhdr, bytes, nb, NULL); } - int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, u32 bytes, struct ntfs_buffers *nb); int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, @@ -772,8 +790,15 @@ int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi, const struct NTFS_DUP_INFO *dup, int sync); /* Globals from inode.c */ -struct inode *ntfs_iget5(struct super_block *sb, const struct MFT_REF *ref, - const struct cpu_str *name); +struct inode *ntfs_iget5_flags(struct super_block *sb, + const struct MFT_REF *ref, + const struct cpu_str *name, u32 flags); +static inline struct inode *ntfs_iget5(struct super_block *sb, + const struct MFT_REF *ref, + const struct cpu_str *name) +{ + return ntfs_iget5_flags(sb, ref, name, 0); +} int ntfs_set_size(struct inode *inode, u64 new_size); int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc); int ntfs_sync_inode(struct inode *inode); @@ -886,8 +911,8 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, #else #define run_unpack_ex run_unpack #endif -int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, - u64 *highest_vcn); +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn); int run_clone(const struct runs_tree *run, struct runs_tree *new_run); bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done); CLST run_len(const struct runs_tree *run); @@ -1219,27 +1244,27 @@ static inline void mi_clear(struct mft_inode *mi) static inline void ni_lock(struct ntfs_inode *ni) { - mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_NORMAL); + mutex_lock_nested(&ni->base->ni_lock, NTFS_INODE_MUTEX_NORMAL); } static inline void ni_lock_dir(struct ntfs_inode *ni) { - mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_PARENT); + mutex_lock_nested(&ni->base->ni_lock, NTFS_INODE_MUTEX_PARENT); } static inline void ni_lock_dir2(struct ntfs_inode *ni) { - mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_PARENT2); + mutex_lock_nested(&ni->base->ni_lock, NTFS_INODE_MUTEX_PARENT2); } static inline void ni_unlock(struct ntfs_inode *ni) { - mutex_unlock(&ni->ni_lock); + mutex_unlock(&ni->base->ni_lock); } static inline int ni_trylock(struct ntfs_inode *ni) { - return mutex_trylock(&ni->ni_lock); + return mutex_trylock(&ni->base->ni_lock); } static inline int attr_load_runs_attr(struct ntfs_inode *ni, diff --git a/fs/ntfs3/record.c b/fs/ntfs3/record.c index 32bdb034c2a3..4f12ce15b03b 100644 --- a/fs/ntfs3/record.c +++ b/fs/ntfs3/record.c @@ -202,7 +202,7 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, u32 used = le32_to_cpu(rec->used); u32 t32, off, asize, prev_type; u16 t16; - u64 data_size, alloc_size, tot_size; + u64 svcn, evcn, data_size, alloc_size, tot_size; if (!attr) { u32 total = le32_to_cpu(rec->total); @@ -310,10 +310,38 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, if (t32 && le16_to_cpu(attr->name_off) + t32 > t16) goto out; - /* Check start/end vcn. */ - if (le64_to_cpu(attr->nres.svcn) > le64_to_cpu(attr->nres.evcn) + 1) + /* + * Check start/end vcn. svcn == 0 with evcn == -1 (U64_MAX) is the + * sentinel for an empty non-resident attribute (no allocated + * clusters) and must be accepted: "svcn > evcn + 1" tolerates it, + * since "(u64)-1 + 1" is 0 and "0 > 0" is false. + * + * For a non-empty attribute evcn is a cluster index and must lie + * within the volume (sbi->used.bitmap.nbits, set up in + * ntfs_init_from_boot() before any caller of mi_enum_attr() runs). + * Bounding evcn also prevents a malformed value close to U64_MAX + * from slipping through the near-wrap "evcn + 1" upper bound. + */ + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + if (svcn > evcn + 1) goto out; + if (is_attr_ext(attr)) { + /* sparsed/compressed attribute. */ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + /* No limits. */ +#else + /* Check evcn fits into 32 bits. */ + if (evcn != U64_MAX && evcn >= (1ull << 32)) + goto out; +#endif + } else { + /* Check out of volume for normal attribute. */ + if (evcn != U64_MAX && evcn >= mi->sbi->used.bitmap.nbits) + goto out; + } + data_size = le64_to_cpu(attr->nres.data_size); if (le64_to_cpu(attr->nres.valid_size) > data_size) goto out; diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 3ebf0154eda3..6e3ef89fc666 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -1265,8 +1265,8 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, * Return the highest vcn from a mapping pairs array * it used while replaying log file. */ -int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, - u64 *highest_vcn) +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn) { const u8 *run_last = run_buf + run_buf_size; u64 vcn64 = vcn; @@ -1279,7 +1279,7 @@ int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, if (size_size > 8 || offset_size > 8) return -EINVAL; - if (run_buf + size_size + offset_size > run_last) + if (run_buf + size_size + offset_size > run_last) return -EINVAL; len = run_unpack_s64(run_buf, size_size, 0); @@ -1357,7 +1357,8 @@ bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done) if (r_end > end) { /* Remove a middle part, split. */ CLST tail_lcn = r->lcn == SPARSE_LCN ? - SPARSE_LCN : (r->lcn + (end - r->vcn)); + SPARSE_LCN : + (r->lcn + (end - r->vcn)); *done += len; r->len = d; diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 3305fe406cb2..f4a42a0c73a4 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -23,6 +23,7 @@ * allocated_size - Total size of clusters allocated for non-resident content * total_size - Actual size of allocated clusters for sparse or compressed attributes * - Constraint: valid_size <= data_size <= allocated_size + * ADS - Alternative data stream: Named data attribute (0x80) * * WSL - Windows Subsystem for Linux * https://docs.microsoft.com/en-us/windows/wsl/file-permissions @@ -65,6 +66,7 @@ #include <linux/minmax.h> #include <linux/module.h> #include <linux/nls.h> +#include <linux/overflow.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/statfs.h> @@ -271,6 +273,8 @@ enum Opt { Opt_nocase, Opt_delalloc, Opt_delalloc_bool, + Opt_ads, + Opt_ads_bool, Opt_err, }; @@ -297,6 +301,8 @@ static const struct fs_parameter_spec ntfs_fs_parameters[] = { fsparam_flag("nocase", Opt_nocase), fsparam_flag("delalloc", Opt_delalloc), fsparam_bool("delalloc", Opt_delalloc_bool), + fsparam_flag("ads", Opt_ads), + fsparam_bool("ads", Opt_ads_bool), {} }; // clang-format on @@ -420,6 +426,12 @@ static int ntfs_fs_parse_param(struct fs_context *fc, case Opt_delalloc_bool: opts->delalloc = result.boolean; break; + case Opt_ads: + opts->ads = 1; + break; + case Opt_ads_bool: + opts->ads = result.boolean; + break; default: /* Should not be here unless we forget add case. */ return -EINVAL; @@ -791,6 +803,8 @@ static int ntfs_show_options(struct seq_file *m, struct dentry *root) seq_puts(m, ",nocase"); if (opts->delalloc) seq_puts(m, ",delalloc"); + if (opts->ads) + seq_puts(m, ",ads"); return 0; } @@ -957,7 +971,7 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, struct ntfs_sb_info *sbi = sb->s_fs_info; int err; u32 mb, gb, boot_sector_size, sct_per_clst, record_size; - u64 sectors, clusters, mlcn, mlcn2, dev_size0; + u64 sectors, clusters, mlcn, mlcn2, mft_pos, mft2_pos, dev_size0; struct NTFS_BOOT *boot; struct buffer_head *bh; struct MFT_REC *rec; @@ -1026,7 +1040,15 @@ read_boot: mlcn2 = le64_to_cpu(boot->mft2_clst); sectors = le64_to_cpu(boot->sectors_per_volume); - if (mlcn * sct_per_clst >= sectors || mlcn2 * sct_per_clst >= sectors) { + /* + * Convert mlcn/mlcn2 to sector positions before comparing with + * 'sectors'. All three are u64 values that come from the boot + * sector, so use check_mul_overflow() to keep a wraparound from + * silently bypassing the comparison. + */ + if (check_mul_overflow(mlcn, (u64)sct_per_clst, &mft_pos) || + check_mul_overflow(mlcn2, (u64)sct_per_clst, &mft2_pos) || + mft_pos >= sectors || mft2_pos >= sectors) { ntfs_err( sb, "%s: start of MFT 0x%llx (0x%llx) is out of volume 0x%llx.", @@ -1189,7 +1211,7 @@ read_boot: #ifdef CONFIG_NTFS3_64BIT_CLUSTER if (clusters >= (1ull << (64 - cluster_bits))) sbi->maxbytes = -1; - sbi->maxbytes_sparse = -1; + sbi->maxbytes_sparse = MAX_LFS_FILESIZE; sb->s_maxbytes = MAX_LFS_FILESIZE; #else /* Maximum size for sparse file. */ @@ -1458,7 +1480,10 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) Add2Ptr(a, roff), le32_to_cpu(a->size) - roff); if (err < 0) { - ntfs_err(sb, "Failed to unpack $MFT bitmap extent (%d).", err); + ntfs_err( + sb, + "Failed to unpack $MFT bitmap extent (%d).", + err); goto put_inode_out; } err = 0; @@ -1866,9 +1891,9 @@ static int ntfs_init_fs_context(struct fs_context *fc) /* Default options. */ opts->fs_uid = current_uid(); opts->fs_gid = current_gid(); - opts->fs_fmask_inv = ~current_umask(); - opts->fs_dmask_inv = ~current_umask(); + opts->fs_fmask_inv = opts->fs_dmask_inv = ~current_umask(); opts->prealloc = 1; + opts->ads = 1; #ifdef CONFIG_NTFS3_FS_POSIX_ACL /* Set the default value 'acl' */ @@ -1928,7 +1953,6 @@ static struct file_system_type ntfs_fs_type = { .kill_sb = ntfs3_kill_sb, .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, }; - // clang-format on static int __init init_ntfs_fs(void) diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index 04814dd29375..594ef6860b93 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -146,26 +146,29 @@ static int ntfs_read_ea(struct ntfs_inode *ni, struct EA_FULL **ea, for (off = 0; off < size; off += ea_size) { const struct EA_FULL *ef = Add2Ptr(ea_p, off); u32 bytes = size - off; + size_t need; /* Check if we can use field ea->size. */ if (bytes < sizeof(ef->size)) goto out1; + /* Check if we can use fields ef->name_len and ef->elength. */ + if (bytes < offsetof(struct EA_FULL, name)) + goto out1; + + /* Size needed to hold this record's name and value. */ + need = struct_size(ef, name, + 1 + ef->name_len + le16_to_cpu(ef->elength)); + if (ef->size) { ea_size = le32_to_cpu(ef->size); - if (ea_size > bytes) + /* ef->size must fit the list and cover the record. */ + if (ea_size > bytes || ea_size < need) goto out1; continue; } - /* Check if we can use fields ef->name_len and ef->elength. */ - if (bytes < offsetof(struct EA_FULL, name)) - goto out1; - - ea_size = ALIGN(struct_size(ef, name, - 1 + ef->name_len + - le16_to_cpu(ef->elength)), - 4); + ea_size = ALIGN(need, 4); if (ea_size > bytes) goto out1; } @@ -660,7 +663,6 @@ static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap, inode->i_mode = old_mode; goto out; } - inode->i_mode = mode; } set_cached_acl(inode, type, acl); inode_set_ctime_current(inode); diff --git a/fs/orangefs/devorangefs-req.c b/fs/orangefs/devorangefs-req.c index 33ee8cb32f83..e5c60da7e677 100644 --- a/fs/orangefs/devorangefs-req.c +++ b/fs/orangefs/devorangefs-req.c @@ -474,6 +474,7 @@ static ssize_t orangefs_devreq_write_iter(struct kiocb *iocb, op->downcall.trailer_size, iter)) { gossip_err("%s: failed to copy trailer.\n", __func__); vfree(op->downcall.trailer_buf); + op->downcall.trailer_buf = NULL; goto Efault; } diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index 7143b64b5b25..cd3273c88e03 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -652,8 +652,8 @@ vm_fault_t orangefs_page_mkwrite(struct vm_fault *vmf) wr = folio_get_private(folio); if (uid_eq(wr->uid, current_fsuid()) && gid_eq(wr->gid, current_fsgid())) { - wr->pos = page_offset(vmf->page); - wr->len = PAGE_SIZE; + wr->pos = folio_pos(folio); + wr->len = folio_size(folio); goto okay; } else { if (orangefs_launder_folio(folio)) { @@ -667,8 +667,8 @@ vm_fault_t orangefs_page_mkwrite(struct vm_fault *vmf) ret = VM_FAULT_LOCKED|VM_FAULT_RETRY; goto out; } - wr->pos = page_offset(vmf->page); - wr->len = PAGE_SIZE; + wr->pos = folio_pos(folio); + wr->len = folio_size(folio); wr->uid = current_fsuid(); wr->gid = current_fsgid(); folio_attach_private(folio, wr); diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c index 69bd73a2b556..9f94919a6bc6 100644 --- a/fs/orangefs/orangefs-debugfs.c +++ b/fs/orangefs/orangefs-debugfs.c @@ -569,6 +569,7 @@ static int orangefs_prepare_cdm_array(char *debug_array_string) cds_delimiter = strchr(cds_head, '\n'); *cds_delimiter = '\0'; + cds_head = skip_spaces(cds_head); keyword_len = strcspn(cds_head, " "); cdm_array[i].keyword = kzalloc(keyword_len + 1, GFP_KERNEL); diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c index b6d116302de4..885fd3bd5a3d 100644 --- a/fs/orangefs/xattr.c +++ b/fs/orangefs/xattr.c @@ -72,11 +72,6 @@ static struct orangefs_cached_xattr *find_cached_xattr(struct inode *inode, if (hlist_empty(h)) return NULL; hlist_for_each_entry_safe(cx, tmp, h, node) { -/* if (!time_before(jiffies, cx->timeout)) { - hlist_del(&cx->node); - kfree(cx); - continue; - }*/ if (!strcmp(cx->key, key)) return cx; } diff --git a/fs/quota/Kconfig b/fs/quota/Kconfig index 818083a36bef..68eb441328ef 100644 --- a/fs/quota/Kconfig +++ b/fs/quota/Kconfig @@ -25,16 +25,6 @@ config QUOTA_NETLINK_INTERFACE hardlimit, etc.) will be reported through netlink interface. If unsure, say Y. -config PRINT_QUOTA_WARNING - bool "Print quota warnings to console (OBSOLETE)" - depends on QUOTA && BROKEN - default y - help - If you say Y here, quota warnings (about exceeding softlimit, reaching - hardlimit, etc.) will be printed to the process' controlling terminal. - Note that this behavior is currently deprecated and may go away in - future. Please use notification via netlink socket instead. - config QUOTA_DEBUG bool "Additional quota sanity checks" depends on QUOTA diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 9850de3955d3..204afc5e984b 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -153,7 +153,7 @@ void __quota_error(struct super_block *sb, const char *func, } EXPORT_SYMBOL(__quota_error); -#if defined(CONFIG_QUOTA_DEBUG) || defined(CONFIG_PRINT_QUOTA_WARNING) +#ifdef CONFIG_QUOTA_DEBUG static char *quotatypes[] = INITQFNAMES; #endif static struct quota_format_type *quota_formats; /* List of registered formats */ @@ -1208,72 +1208,6 @@ static int warning_issued(struct dquot *dquot, const int warntype) return test_and_set_bit(flag, &dquot->dq_flags); } -#ifdef CONFIG_PRINT_QUOTA_WARNING -static int flag_print_warnings = 1; - -static int need_print_warning(struct dquot_warn *warn) -{ - if (!flag_print_warnings) - return 0; - - switch (warn->w_dq_id.type) { - case USRQUOTA: - return uid_eq(current_fsuid(), warn->w_dq_id.uid); - case GRPQUOTA: - return in_group_p(warn->w_dq_id.gid); - case PRJQUOTA: - return 1; - } - return 0; -} - -/* Print warning to user which exceeded quota */ -static void print_warning(struct dquot_warn *warn) -{ - char *msg = NULL; - struct tty_struct *tty; - int warntype = warn->w_type; - - if (warntype == QUOTA_NL_IHARDBELOW || - warntype == QUOTA_NL_ISOFTBELOW || - warntype == QUOTA_NL_BHARDBELOW || - warntype == QUOTA_NL_BSOFTBELOW || !need_print_warning(warn)) - return; - - tty = get_current_tty(); - if (!tty) - return; - tty_write_message(tty, warn->w_sb->s_id); - if (warntype == QUOTA_NL_ISOFTWARN || warntype == QUOTA_NL_BSOFTWARN) - tty_write_message(tty, ": warning, "); - else - tty_write_message(tty, ": write failed, "); - tty_write_message(tty, quotatypes[warn->w_dq_id.type]); - switch (warntype) { - case QUOTA_NL_IHARDWARN: - msg = " file limit reached.\r\n"; - break; - case QUOTA_NL_ISOFTLONGWARN: - msg = " file quota exceeded too long.\r\n"; - break; - case QUOTA_NL_ISOFTWARN: - msg = " file quota exceeded.\r\n"; - break; - case QUOTA_NL_BHARDWARN: - msg = " block limit reached.\r\n"; - break; - case QUOTA_NL_BSOFTLONGWARN: - msg = " block quota exceeded too long.\r\n"; - break; - case QUOTA_NL_BSOFTWARN: - msg = " block quota exceeded.\r\n"; - break; - } - tty_write_message(tty, msg); - tty_kref_put(tty); -} -#endif - static void prepare_warning(struct dquot_warn *warn, struct dquot *dquot, int warntype) { @@ -1296,9 +1230,7 @@ static void flush_warnings(struct dquot_warn *warn) for (i = 0; i < MAXQUOTAS; i++) { if (warn[i].w_type == QUOTA_NL_NOWARN) continue; -#ifdef CONFIG_PRINT_QUOTA_WARNING - print_warning(&warn[i]); -#endif + quota_send_warning(warn[i].w_dq_id, warn[i].w_sb->s_dev, warn[i].w_type); } @@ -3008,15 +2940,6 @@ static const struct ctl_table fs_dqstats_table[] = { .mode = 0444, .proc_handler = do_proc_dqstats, }, -#ifdef CONFIG_PRINT_QUOTA_WARNING - { - .procname = "warnings", - .data = &flag_print_warnings, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif }; static int __init dquot_init(void) diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c index cc6dc6e1d84d..30cec5600149 100644 --- a/fs/udf/balloc.c +++ b/fs/udf/balloc.c @@ -502,6 +502,8 @@ static int udf_table_prealloc_blocks(struct super_block *sb, int8_t etype = -1; struct udf_inode_info *iinfo; int ret = 0; + /* AED block freed by udf_delete_aext(), released after unlock */ + struct kernel_lb_addr freed = { .partitionReferenceNum = 0xFFFF }; if (first_block >= sbi->s_partmaps[partition].s_partition_len) return 0; @@ -541,7 +543,7 @@ static int udf_table_prealloc_blocks(struct super_block *sb, udf_write_aext(table, &epos, &eloc, (etype << 30) | elen, 1); } else - udf_delete_aext(table, epos); + udf_delete_aext(table, epos, &freed); } else { alloc_count = 0; } @@ -552,6 +554,8 @@ err_out: if (alloc_count) udf_add_free_space(sb, partition, -alloc_count); mutex_unlock(&sbi->s_alloc_mutex); + if (freed.partitionReferenceNum != 0xFFFF) + udf_free_blocks(sb, table, &freed, 0, 1); return alloc_count; } @@ -560,6 +564,8 @@ static udf_pblk_t udf_table_new_block(struct super_block *sb, uint32_t goal, int *err) { struct udf_sb_info *sbi = UDF_SB(sb); + /* AED block freed by udf_delete_aext(), released after unlock */ + struct kernel_lb_addr freed = { .partitionReferenceNum = 0xFFFF }; uint32_t spread = 0xFFFFFFFF, nspread = 0xFFFFFFFF; udf_pblk_t newblock = 0; uint32_t adsize; @@ -643,12 +649,14 @@ static udf_pblk_t udf_table_new_block(struct super_block *sb, if (goal_elen) udf_write_aext(table, &goal_epos, &goal_eloc, goal_elen, 1); else - udf_delete_aext(table, goal_epos); + udf_delete_aext(table, goal_epos, &freed); brelse(goal_epos.bh); udf_add_free_space(sb, partition, -1); mutex_unlock(&sbi->s_alloc_mutex); + if (freed.partitionReferenceNum != 0xFFFF) + udf_free_blocks(sb, table, &freed, 0, 1); *err = 0; return newblock; } diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 68c6c2ba8ed1..e45e546a739a 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -334,65 +334,6 @@ const struct address_space_operations udf_aops = { .migrate_folio = buffer_migrate_folio, }; -/* - * Expand file stored in ICB to a normal one-block-file - * - * This function requires i_mutex held - */ -int udf_expand_file_adinicb(struct inode *inode) -{ - struct folio *folio; - struct udf_inode_info *iinfo = UDF_I(inode); - int err; - - WARN_ON_ONCE(!inode_is_locked(inode)); - if (!iinfo->i_lenAlloc) { - down_write(&iinfo->i_data_sem); - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - up_write(&iinfo->i_data_sem); - mark_inode_dirty(inode); - return 0; - } - - folio = __filemap_get_folio(inode->i_mapping, 0, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); - if (IS_ERR(folio)) - return PTR_ERR(folio); - - if (!folio_test_uptodate(folio)) - udf_adinicb_read_folio(folio); - down_write(&iinfo->i_data_sem); - memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, - iinfo->i_lenAlloc); - iinfo->i_lenAlloc = 0; - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - folio_mark_dirty(folio); - folio_unlock(folio); - up_write(&iinfo->i_data_sem); - err = filemap_fdatawrite(inode->i_mapping); - if (err) { - /* Restore everything back so that we don't lose data... */ - folio_lock(folio); - down_write(&iinfo->i_data_sem); - memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, - folio, 0, inode->i_size); - folio_unlock(folio); - iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; - iinfo->i_lenAlloc = inode->i_size; - up_write(&iinfo->i_data_sem); - } - folio_put(folio); - mark_inode_dirty(inode); - - return err; -} - #define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ #define UDF_MAP_NOPREALLOC 0x02 /* Do not preallocate blocks */ @@ -453,6 +394,76 @@ out_read: return ret; } +/* + * Expand file stored in ICB to a normal one-block-file + * + * This function requires i_mutex held + */ +int udf_expand_file_adinicb(struct inode *inode) +{ + struct folio *folio; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_map_rq map = { + .lblk = 0, + .iflags = UDF_MAP_CREATE, + }; + int err; + + WARN_ON_ONCE(!inode_is_locked(inode)); + if (!iinfo->i_lenAlloc) { + down_write(&iinfo->i_data_sem); + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + mark_inode_dirty(inode); + return 0; + } + + folio = __filemap_get_folio(inode->i_mapping, 0, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!folio_test_uptodate(folio)) + udf_adinicb_read_folio(folio); + down_write(&iinfo->i_data_sem); + memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, + iinfo->i_lenAlloc); + iinfo->i_lenAlloc = 0; + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + + /* Allocate the block underlying the data */ + err = udf_map_block(inode, &map); + if (err < 0) + goto restore; + + folio_mark_dirty(folio); + folio_unlock(folio); + err = filemap_fdatawrite(inode->i_mapping); + if (err) { + /* Restore everything back so that we don't lose data... */ + folio_lock(folio); +restore: + down_write(&iinfo->i_data_sem); + memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, + folio, 0, inode->i_size); + iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; + iinfo->i_lenAlloc = inode->i_size; + up_write(&iinfo->i_data_sem); + folio_unlock(folio); + } + folio_put(folio); + mark_inode_dirty(inode); + + return err; +} + static int __udf_get_block(struct inode *inode, sector_t block, struct buffer_head *bh_result, int flags) { @@ -529,7 +540,7 @@ static int udf_do_extend_file(struct inode *inode, sb->s_blocksize - 1) & ~(sb->s_blocksize - 1)); iinfo->i_lenExtents = (iinfo->i_lenExtents + sb->s_blocksize - 1) & - ~(sb->s_blocksize - 1); + ~((u64)sb->s_blocksize - 1); } add = 0; @@ -1199,7 +1210,7 @@ static int udf_update_extents(struct inode *inode, struct kernel_long_ad *laarr, if (startnum > endnum) { for (i = 0; i < (startnum - endnum); i++) - udf_delete_aext(inode, *epos); + udf_delete_aext(inode, *epos, NULL); } else if (startnum < endnum) { for (i = 0; i < (endnum - startnum); i++) { err = udf_insert_aext(inode, *epos, @@ -1467,6 +1478,10 @@ reread: iinfo->i_lenAlloc = le32_to_cpu( ((struct unallocSpaceEntry *)bh->b_data)-> lengthAllocDescs); + if (iinfo->i_lenAlloc > bs - sizeof(struct unallocSpaceEntry)) { + ret = -EFSCORRUPTED; + goto out; + } ret = udf_alloc_i_data(inode, bs - sizeof(struct unallocSpaceEntry)); if (ret) @@ -1474,6 +1489,7 @@ reread: memcpy(iinfo->i_data, bh->b_data + sizeof(struct unallocSpaceEntry), bs - sizeof(struct unallocSpaceEntry)); + brelse(bh); return 0; } @@ -2297,6 +2313,13 @@ int udf_current_aext(struct inode *inode, struct extent_position *epos, return -EINVAL; } + if (eloc->partitionReferenceNum >= UDF_SB(inode->i_sb)->s_partitions) { + udf_debug("invalid partition reference %u (partitions %u)\n", + eloc->partitionReferenceNum, + UDF_SB(inode->i_sb)->s_partitions); + return -EFSCORRUPTED; + } + return 1; } @@ -2326,7 +2349,8 @@ static int udf_insert_aext(struct inode *inode, struct extent_position epos, return ret; } -int8_t udf_delete_aext(struct inode *inode, struct extent_position epos) +int8_t udf_delete_aext(struct inode *inode, struct extent_position epos, + struct kernel_lb_addr *freed) { struct extent_position oepos; int adsize; @@ -2376,7 +2400,19 @@ int8_t udf_delete_aext(struct inode *inode, struct extent_position epos) elen = 0; if (epos.bh != oepos.bh) { - udf_free_blocks(inode->i_sb, inode, &epos.block, 0, 1); + /* + * The block that held the now-empty allocation extent must be + * returned to free space. When the caller already holds + * s_alloc_mutex (the space-table allocator in balloc.c), + * freeing it inline would recurse through udf_free_blocks() + * into udf_table_free_blocks() and deadlock re-acquiring + * s_alloc_mutex. In that case report the block to the caller, + * which frees it after dropping the lock. + */ + if (freed) + *freed = epos.block; + else + udf_free_blocks(inode->i_sb, inode, &epos.block, 0, 1); udf_write_aext(inode, &oepos, &eloc, elen, 1); udf_write_aext(inode, &oepos, &eloc, elen, 1); if (!oepos.bh) { diff --git a/fs/udf/partition.c b/fs/udf/partition.c index 2b85c9501bed..ad8dcedca263 100644 --- a/fs/udf/partition.c +++ b/fs/udf/partition.c @@ -55,7 +55,7 @@ uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block, map = &sbi->s_partmaps[partition]; vdata = &map->s_type_specific.s_virtual; - if (block > vdata->s_num_entries) { + if (block >= vdata->s_num_entries) { udf_debug("Trying to access block beyond end of VAT (%u max %u)\n", block, vdata->s_num_entries); return 0xFFFFFFFF; diff --git a/fs/udf/super.c b/fs/udf/super.c index e7e9f2a0d24e..2ba5973ef4dd 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -2055,6 +2055,17 @@ static int udf_load_vrs(struct super_block *sb, struct udf_options *uopt, return 0; } +static void udf_mark_buffer_dirty(struct buffer_head *bh) +{ + /* + * We set buffer uptodate unconditionally here to avoid spurious + * warnings from mark_buffer_dirty() when previous EIO has marked + * the buffer as !uptodate + */ + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); +} + static void udf_finalize_lvid(struct logicalVolIntegrityDesc *lvid) { struct timespec64 ts; @@ -2090,7 +2101,7 @@ static void udf_open_lvid(struct super_block *sb) UDF_SET_FLAG(sb, UDF_FLAG_INCONSISTENT); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make opening of filesystem visible on the media immediately */ @@ -2123,14 +2134,8 @@ static void udf_close_lvid(struct super_block *sb) if (!UDF_QUERY_FLAG(sb, UDF_FLAG_INCONSISTENT)) lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_CLOSE); - /* - * We set buffer uptodate unconditionally here to avoid spurious - * warnings from mark_buffer_dirty() when previous EIO has marked - * the buffer as !uptodate - */ - set_buffer_uptodate(bh); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make closing of filesystem visible on the media immediately */ @@ -2412,7 +2417,7 @@ static int udf_sync_fs(struct super_block *sb, int wait) * Blockdevice will be synced later so we don't have to submit * the buffer for IO */ - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; } mutex_unlock(&sbi->s_alloc_mutex); diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index fe03745d09b1..a05d1888a2ba 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -36,6 +36,8 @@ static int udf_pc_to_char(struct super_block *sb, unsigned char *from, /* Reserve one byte for terminating \0 */ tolen--; while (elen < fromlen) { + if (fromlen - elen < sizeof(struct pathComponent)) + return -EIO; pc = (struct pathComponent *)(from + elen); elen += sizeof(struct pathComponent); switch (pc->componentType) { diff --git a/fs/udf/truncate.c b/fs/udf/truncate.c index 41b2bfd30449..0990f94b8551 100644 --- a/fs/udf/truncate.c +++ b/fs/udf/truncate.c @@ -159,7 +159,7 @@ void udf_discard_prealloc(struct inode *inode) if (etype == (EXT_NOT_RECORDED_ALLOCATED >> 30)) { lbcount -= elen; - udf_delete_aext(inode, prev_epos); + udf_delete_aext(inode, prev_epos, NULL); udf_free_blocks(inode->i_sb, inode, &eloc, 0, DIV_ROUND_UP(elen, bsize)); } diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 86dc2d6a2ef1..7d5a1981434e 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -170,7 +170,8 @@ extern int udf_add_aext(struct inode *, struct extent_position *, struct kernel_lb_addr *, uint32_t, int); extern void udf_write_aext(struct inode *, struct extent_position *, struct kernel_lb_addr *, uint32_t, int); -extern int8_t udf_delete_aext(struct inode *, struct extent_position); +extern int8_t udf_delete_aext(struct inode *, struct extent_position, + struct kernel_lb_addr *); extern int udf_next_aext(struct inode *inode, struct extent_position *epos, struct kernel_lb_addr *eloc, uint32_t *elen, int8_t *etype, int inc); diff --git a/include/linux/fs.h b/include/linux/fs.h index 8e9bc9dda0cb..072d8cd09a0b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2419,6 +2419,21 @@ static inline void super_set_sysfs_name_generic(struct super_block *sb, const ch extern void ihold(struct inode * inode); extern void iput(struct inode *); void iput_not_last(struct inode *); + +/** + * iput_if_not_last - drop an inode reference only if it is not the last one + * @inode: inode to put + * + * Returns true if the reference was dropped, false if this was the last + * reference and the caller must arrange for final iput() in a safe context. + */ +static inline bool __must_check iput_if_not_last(struct inode *inode) +{ + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); + VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); + return atomic_add_unless(&inode->i_count, -1, 1); +} + int inode_update_time(struct inode *inode, enum fs_update_time type, unsigned int flags); int generic_update_time(struct inode *inode, enum fs_update_time type, diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index b68561187e90..1b42fe47c26b 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -510,11 +510,12 @@ struct jbd2_journal_handle int h_err; /* Flags [no locking] */ - unsigned int h_sync: 1; - unsigned int h_reserved: 1; - unsigned int h_aborted: 1; - unsigned int h_type: 8; - unsigned int h_line_no: 16; + unsigned char h_sync: 1; + unsigned char h_reserved: 1; + unsigned char h_aborted: 1; + unsigned char h_invalid: 1; + unsigned char h_type; + unsigned short h_line_no; unsigned long h_start_jiffies; unsigned int h_requested_credits; diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h index b614e0deea72..db8207d4059f 100644 --- a/include/linux/lockd/bind.h +++ b/include/linux/lockd/bind.h @@ -16,17 +16,23 @@ struct svc_rqst; struct rpc_task; struct rpc_clnt; struct super_block; +struct module; -/* - * This is the set of functions for lockd->nfsd communication +/** + * struct nlmsvc_binding - lockd -> nfsd callback table + * @owner: module that provides this binding. + * @fopen: open a file by NFS file handle on behalf of an NLM request. + * @fclose: close a file that was previously opened via @fopen. + * Implementations MUST be semantically equivalent to fput(). */ struct nlmsvc_binding { + struct module *owner; int (*fopen)(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp, int flags); void (*fclose)(struct file *filp); }; -extern const struct nlmsvc_binding *nlmsvc_ops; +extern const struct nlmsvc_binding __rcu *nlmsvc_ops; /* * Similar to nfs_client_initdata, but without the NFS-specific diff --git a/include/linux/nd.h b/include/linux/nd.h index fa099e295f78..62988000e7a7 100644 --- a/include/linux/nd.h +++ b/include/linux/nd.h @@ -110,7 +110,7 @@ static inline struct nd_namespace_common *to_ndns(struct device *dev) /** * struct nd_namespace_io - device representation of a persistent memory range - * @dev: namespace device created by the nd region driver + * @common: namespace device core infrastructure created by the nd region driver * @res: struct resource conversion of a NFIT SPA table * @size: cached resource_size(@res) for fast path size checks * @addr: virtual address to access the namespace range @@ -158,8 +158,11 @@ static inline struct nd_namespace_pmem *to_nd_namespace_pmem(const struct device * @offset: namespace-relative starting offset * @buf: buffer to fill * @size: transfer length + * @flags: process (0) or atomic (1) context * * @buf is up-to-date upon return from this routine. + * + * Returns: %0 on success or a negative error code on failure */ static inline int nvdimm_read_bytes(struct nd_namespace_common *ndns, resource_size_t offset, void *buf, size_t size, @@ -174,11 +177,14 @@ static inline int nvdimm_read_bytes(struct nd_namespace_common *ndns, * @offset: namespace-relative starting offset * @buf: buffer to drain * @size: transfer length + * @flags: process (0) or atomic (1) context * * NVDIMM Namepaces disks do not implement sectors internally. Depending on * the @ndns, the contents of @buf may be in cpu cache, platform buffers, * or on backing memory media upon return from this routine. Flushing * to media is handled internal to the @ndns driver, if at all. + * + * Returns: %0 on success or a negative error code on failure */ static inline int nvdimm_write_bytes(struct nd_namespace_common *ndns, resource_size_t offset, void *buf, size_t size, diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index d87be1f25273..44e5e9fa12e1 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -171,133 +171,6 @@ Needs to be updated if more operations are defined in future.*/ #define LAST_NFS42_OP OP_REMOVEXATTR #define LAST_NFS4_OP LAST_NFS42_OP -enum nfsstat4 { - NFS4_OK = 0, - NFS4ERR_PERM = 1, - NFS4ERR_NOENT = 2, - NFS4ERR_IO = 5, - NFS4ERR_NXIO = 6, - NFS4ERR_ACCESS = 13, - NFS4ERR_EXIST = 17, - NFS4ERR_XDEV = 18, - /* Unused/reserved 19 */ - NFS4ERR_NOTDIR = 20, - NFS4ERR_ISDIR = 21, - NFS4ERR_INVAL = 22, - NFS4ERR_FBIG = 27, - NFS4ERR_NOSPC = 28, - NFS4ERR_ROFS = 30, - NFS4ERR_MLINK = 31, - NFS4ERR_NAMETOOLONG = 63, - NFS4ERR_NOTEMPTY = 66, - NFS4ERR_DQUOT = 69, - NFS4ERR_STALE = 70, - NFS4ERR_BADHANDLE = 10001, - NFS4ERR_BAD_COOKIE = 10003, - NFS4ERR_NOTSUPP = 10004, - NFS4ERR_TOOSMALL = 10005, - NFS4ERR_SERVERFAULT = 10006, - NFS4ERR_BADTYPE = 10007, - NFS4ERR_DELAY = 10008, - NFS4ERR_SAME = 10009, - NFS4ERR_DENIED = 10010, - NFS4ERR_EXPIRED = 10011, - NFS4ERR_LOCKED = 10012, - NFS4ERR_GRACE = 10013, - NFS4ERR_FHEXPIRED = 10014, - NFS4ERR_SHARE_DENIED = 10015, - NFS4ERR_WRONGSEC = 10016, - NFS4ERR_CLID_INUSE = 10017, - NFS4ERR_RESOURCE = 10018, - NFS4ERR_MOVED = 10019, - NFS4ERR_NOFILEHANDLE = 10020, - NFS4ERR_MINOR_VERS_MISMATCH = 10021, - NFS4ERR_STALE_CLIENTID = 10022, - NFS4ERR_STALE_STATEID = 10023, - NFS4ERR_OLD_STATEID = 10024, - NFS4ERR_BAD_STATEID = 10025, - NFS4ERR_BAD_SEQID = 10026, - NFS4ERR_NOT_SAME = 10027, - NFS4ERR_LOCK_RANGE = 10028, - NFS4ERR_SYMLINK = 10029, - NFS4ERR_RESTOREFH = 10030, - NFS4ERR_LEASE_MOVED = 10031, - NFS4ERR_ATTRNOTSUPP = 10032, - NFS4ERR_NO_GRACE = 10033, - NFS4ERR_RECLAIM_BAD = 10034, - NFS4ERR_RECLAIM_CONFLICT = 10035, - NFS4ERR_BADXDR = 10036, - NFS4ERR_LOCKS_HELD = 10037, - NFS4ERR_OPENMODE = 10038, - NFS4ERR_BADOWNER = 10039, - NFS4ERR_BADCHAR = 10040, - NFS4ERR_BADNAME = 10041, - NFS4ERR_BAD_RANGE = 10042, - NFS4ERR_LOCK_NOTSUPP = 10043, - NFS4ERR_OP_ILLEGAL = 10044, - NFS4ERR_DEADLOCK = 10045, - NFS4ERR_FILE_OPEN = 10046, - NFS4ERR_ADMIN_REVOKED = 10047, - NFS4ERR_CB_PATH_DOWN = 10048, - - /* nfs41 */ - NFS4ERR_BADIOMODE = 10049, - NFS4ERR_BADLAYOUT = 10050, - NFS4ERR_BAD_SESSION_DIGEST = 10051, - NFS4ERR_BADSESSION = 10052, - NFS4ERR_BADSLOT = 10053, - NFS4ERR_COMPLETE_ALREADY = 10054, - NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, - NFS4ERR_DELEG_ALREADY_WANTED = 10056, - NFS4ERR_BACK_CHAN_BUSY = 10057, /* backchan reqs outstanding */ - NFS4ERR_LAYOUTTRYLATER = 10058, - NFS4ERR_LAYOUTUNAVAILABLE = 10059, - NFS4ERR_NOMATCHING_LAYOUT = 10060, - NFS4ERR_RECALLCONFLICT = 10061, - NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, - NFS4ERR_SEQ_MISORDERED = 10063, /* unexpected seq.id in req */ - NFS4ERR_SEQUENCE_POS = 10064, /* [CB_]SEQ. op not 1st op */ - NFS4ERR_REQ_TOO_BIG = 10065, /* request too big */ - NFS4ERR_REP_TOO_BIG = 10066, /* reply too big */ - NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, /* rep. not all cached */ - NFS4ERR_RETRY_UNCACHED_REP = 10068, /* retry & rep. uncached */ - NFS4ERR_UNSAFE_COMPOUND = 10069, /* retry/recovery too hard */ - NFS4ERR_TOO_MANY_OPS = 10070, /* too many ops in [CB_]COMP */ - NFS4ERR_OP_NOT_IN_SESSION = 10071, /* op needs [CB_]SEQ. op */ - NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp. */ - /* Error 10073 is unused. */ - NFS4ERR_CLIENTID_BUSY = 10074, /* clientid has state */ - NFS4ERR_PNFS_IO_HOLE = 10075, /* IO to _SPARSE file hole */ - NFS4ERR_SEQ_FALSE_RETRY = 10076, /* retry not original */ - NFS4ERR_BAD_HIGH_SLOT = 10077, /* sequence arg bad */ - NFS4ERR_DEADSESSION = 10078, /* persistent session dead */ - NFS4ERR_ENCR_ALG_UNSUPP = 10079, /* SSV alg mismatch */ - NFS4ERR_PNFS_NO_LAYOUT = 10080, /* direct I/O with no layout */ - NFS4ERR_NOT_ONLY_OP = 10081, /* bad compound */ - NFS4ERR_WRONG_CRED = 10082, /* permissions:state change */ - NFS4ERR_WRONG_TYPE = 10083, /* current operation mismatch */ - NFS4ERR_DIRDELEG_UNAVAIL = 10084, /* no directory delegation */ - NFS4ERR_REJECT_DELEG = 10085, /* on callback */ - NFS4ERR_RETURNCONFLICT = 10086, /* outstanding layoutreturn */ - NFS4ERR_DELEG_REVOKED = 10087, /* deleg./layout revoked */ - - /* nfs42 */ - NFS4ERR_PARTNER_NOTSUPP = 10088, - NFS4ERR_PARTNER_NO_AUTH = 10089, - NFS4ERR_UNION_NOTSUPP = 10090, - NFS4ERR_OFFLOAD_DENIED = 10091, - NFS4ERR_WRONG_LFS = 10092, - NFS4ERR_BADLABEL = 10093, - NFS4ERR_OFFLOAD_NO_REQS = 10094, - - /* xattr (RFC8276) */ - NFS4ERR_NOXATTR = 10095, - NFS4ERR_XATTR2BIG = 10096, - - /* can be used for internal errors */ - NFS4ERR_FIRST_FREE -}; - /* error codes for internal client use */ #define NFS4ERR_RESET_TO_MDS 12001 #define NFS4ERR_RESET_TO_PNFS 12002 diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h index 98939cb664cf..59d0cc889beb 100644 --- a/include/linux/sunrpc/bc_xprt.h +++ b/include/linux/sunrpc/bc_xprt.h @@ -32,6 +32,7 @@ int xprt_setup_bc(struct rpc_xprt *xprt, unsigned int min_reqs); void xprt_destroy_bc(struct rpc_xprt *xprt, unsigned int max_reqs); void xprt_free_bc_rqst(struct rpc_rqst *req); unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt); +void xprt_svc_shutdown_bc(struct rpc_xprt *xprt); void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv); /* @@ -71,6 +72,10 @@ static inline void xprt_free_bc_request(struct rpc_rqst *req) { } +static inline void xprt_svc_shutdown_bc(struct rpc_xprt *xprt) +{ +} + static inline void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv) { svc_destroy(serv); diff --git a/include/linux/sunrpc/stats.h b/include/linux/sunrpc/stats.h index 3ce1550d1beb..087ade905e29 100644 --- a/include/linux/sunrpc/stats.h +++ b/include/linux/sunrpc/stats.h @@ -37,9 +37,15 @@ struct svc_stat { rpcbadfmt, rpcbadauth, rpcbadclnt; + + /* Per-version per-procedure call counts (per-cpu, per-netns) */ + unsigned long __percpu **vs_count; }; struct net; +int svc_stat_alloc_counts(struct svc_stat *statp); +void svc_stat_free_counts(struct svc_stat *statp); + #ifdef CONFIG_PROC_FS int rpc_proc_init(struct net *); void rpc_proc_exit(struct net *); diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 4be6204f6630..2db1b9ec5658 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -85,7 +85,6 @@ struct svc_serv { char * sv_name; /* service name */ - unsigned int sv_nrpools; /* number of thread pools */ bool sv_is_pooled; /* is this a pooled service? */ struct svc_pool * sv_pools; /* array of thread pools */ int (*sv_threadfn)(void *data); @@ -408,7 +407,6 @@ struct svc_version { u32 vs_vers; /* version number */ u32 vs_nproc; /* number of procedures */ const struct svc_procedure *vs_proc; /* per-procedure info */ - unsigned long __percpu *vs_count; /* call counts */ u32 vs_xdrsize; /* xdrsize needed for this version */ /* Don't register with rpcbind */ @@ -469,6 +467,7 @@ int svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool, unsigned int min_threads, unsigned int max_threads); int svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads, unsigned int nrservs); +unsigned int svc_serv_maxthreads(const struct svc_serv *serv); int svc_pool_stats_open(struct svc_info *si, struct file *file); void svc_process(struct svc_rqst *rqstp); void svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp); @@ -479,6 +478,7 @@ void svc_wake_up(struct svc_serv *); void svc_reserve(struct svc_rqst *rqstp, int space); void svc_pool_wake_idle_thread(struct svc_pool *pool); struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv); +unsigned int svc_serv_nrpools(const struct svc_serv *serv); char * svc_print_addr(struct svc_rqst *, char *, size_t); const char * svc_proc_name(const struct svc_rqst *rqstp); int svc_encode_result_payload(struct svc_rqst *rqstp, diff --git a/include/linux/sunrpc/svc_rdma_pcl.h b/include/linux/sunrpc/svc_rdma_pcl.h index 7516ad0fae80..6346d8cf2587 100644 --- a/include/linux/sunrpc/svc_rdma_pcl.h +++ b/include/linux/sunrpc/svc_rdma_pcl.h @@ -97,7 +97,7 @@ pcl_next_chunk(const struct svc_rdma_pcl *pcl, struct svc_rdma_chunk *chunk) */ #define pcl_for_each_segment(pos, chunk) \ for (pos = &(chunk)->ch_segments[0]; \ - pos <= &(chunk)->ch_segments[(chunk)->ch_segcount - 1]; \ + pos < &(chunk)->ch_segments[(chunk)->ch_segcount]; \ pos++) /** @@ -119,6 +119,8 @@ extern bool pcl_alloc_call(struct svc_rdma_recv_ctxt *rctxt, __be32 *p); extern bool pcl_alloc_read(struct svc_rdma_recv_ctxt *rctxt, __be32 *p); extern bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, struct svc_rdma_pcl *pcl, __be32 *p); +extern bool pcl_check_read_chunk_positions(struct svc_rdma_recv_ctxt *rctxt, + unsigned int inline_len); extern int pcl_process_nonpayloads(const struct svc_rdma_pcl *pcl, const struct xdr_buf *xdr, int (*actor)(const struct xdr_buf *, diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h index a723fb1da9c8..c9033eb1c829 100644 --- a/include/linux/sunrpc/xdrgen/_builtins.h +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -296,4 +296,36 @@ xdrgen_encode_opaque(struct xdr_stream *xdr, opaque val) return true; } +struct svc_rqst; + +/** + * xdrgen_svc_decode_void - Decode a void argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +static inline bool +xdrgen_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_decode_void(xdr); +} + +/** + * xdrgen_svc_encode_void - Encode a void result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +static inline bool +xdrgen_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_encode_void(xdr); +} + #endif /* _SUNRPC_XDRGEN__BUILTINS_H_ */ diff --git a/include/linux/sunrpc/xdrgen/_defs.h b/include/linux/sunrpc/xdrgen/_defs.h index 20c7270aa64d..8f3776ef3229 100644 --- a/include/linux/sunrpc/xdrgen/_defs.h +++ b/include/linux/sunrpc/xdrgen/_defs.h @@ -25,6 +25,8 @@ typedef struct { #define XDR_void (0) #define XDR_bool (1) +#define XDR_short (1) +#define XDR_unsigned_short (1) #define XDR_int (1) #define XDR_unsigned_int (1) #define XDR_long (1) diff --git a/include/linux/sunrpc/xdrgen/nfs4_1.h b/include/linux/sunrpc/xdrgen/nfs4_1.h index 4ac54bdbd335..bd3289a1d66a 100644 --- a/include/linux/sunrpc/xdrgen/nfs4_1.h +++ b/include/linux/sunrpc/xdrgen/nfs4_1.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ -/* XDR specification modification time: Thu Jan 8 23:12:07 2026 */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ #ifndef _LINUX_XDRGEN_NFS4_1_DEF_H #define _LINUX_XDRGEN_NFS4_1_DEF_H @@ -9,15 +9,149 @@ #include <linux/types.h> #include <linux/sunrpc/xdrgen/_defs.h> -typedef s64 int64_t; +typedef s32 int32_t; typedef u32 uint32_t; +typedef s64 int64_t; + +typedef u64 uint64_t; + +enum { NFS4_VERIFIER_SIZE = 8 }; + +enum { NFS4_FHSIZE = 128 }; + +enum nfsstat4 { + NFS4_OK = 0, + NFS4ERR_PERM = 1, + NFS4ERR_NOENT = 2, + NFS4ERR_IO = 5, + NFS4ERR_NXIO = 6, + NFS4ERR_ACCESS = 13, + NFS4ERR_EXIST = 17, + NFS4ERR_XDEV = 18, + NFS4ERR_NOTDIR = 20, + NFS4ERR_ISDIR = 21, + NFS4ERR_INVAL = 22, + NFS4ERR_FBIG = 27, + NFS4ERR_NOSPC = 28, + NFS4ERR_ROFS = 30, + NFS4ERR_MLINK = 31, + NFS4ERR_NAMETOOLONG = 63, + NFS4ERR_NOTEMPTY = 66, + NFS4ERR_DQUOT = 69, + NFS4ERR_STALE = 70, + NFS4ERR_BADHANDLE = 10001, + NFS4ERR_BAD_COOKIE = 10003, + NFS4ERR_NOTSUPP = 10004, + NFS4ERR_TOOSMALL = 10005, + NFS4ERR_SERVERFAULT = 10006, + NFS4ERR_BADTYPE = 10007, + NFS4ERR_DELAY = 10008, + NFS4ERR_SAME = 10009, + NFS4ERR_DENIED = 10010, + NFS4ERR_EXPIRED = 10011, + NFS4ERR_LOCKED = 10012, + NFS4ERR_GRACE = 10013, + NFS4ERR_FHEXPIRED = 10014, + NFS4ERR_SHARE_DENIED = 10015, + NFS4ERR_WRONGSEC = 10016, + NFS4ERR_CLID_INUSE = 10017, + NFS4ERR_RESOURCE = 10018, + NFS4ERR_MOVED = 10019, + NFS4ERR_NOFILEHANDLE = 10020, + NFS4ERR_MINOR_VERS_MISMATCH = 10021, + NFS4ERR_STALE_CLIENTID = 10022, + NFS4ERR_STALE_STATEID = 10023, + NFS4ERR_OLD_STATEID = 10024, + NFS4ERR_BAD_STATEID = 10025, + NFS4ERR_BAD_SEQID = 10026, + NFS4ERR_NOT_SAME = 10027, + NFS4ERR_LOCK_RANGE = 10028, + NFS4ERR_SYMLINK = 10029, + NFS4ERR_RESTOREFH = 10030, + NFS4ERR_LEASE_MOVED = 10031, + NFS4ERR_ATTRNOTSUPP = 10032, + NFS4ERR_NO_GRACE = 10033, + NFS4ERR_RECLAIM_BAD = 10034, + NFS4ERR_RECLAIM_CONFLICT = 10035, + NFS4ERR_BADXDR = 10036, + NFS4ERR_LOCKS_HELD = 10037, + NFS4ERR_OPENMODE = 10038, + NFS4ERR_BADOWNER = 10039, + NFS4ERR_BADCHAR = 10040, + NFS4ERR_BADNAME = 10041, + NFS4ERR_BAD_RANGE = 10042, + NFS4ERR_LOCK_NOTSUPP = 10043, + NFS4ERR_OP_ILLEGAL = 10044, + NFS4ERR_DEADLOCK = 10045, + NFS4ERR_FILE_OPEN = 10046, + NFS4ERR_ADMIN_REVOKED = 10047, + NFS4ERR_CB_PATH_DOWN = 10048, + NFS4ERR_BADIOMODE = 10049, + NFS4ERR_BADLAYOUT = 10050, + NFS4ERR_BAD_SESSION_DIGEST = 10051, + NFS4ERR_BADSESSION = 10052, + NFS4ERR_BADSLOT = 10053, + NFS4ERR_COMPLETE_ALREADY = 10054, + NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, + NFS4ERR_DELEG_ALREADY_WANTED = 10056, + NFS4ERR_BACK_CHAN_BUSY = 10057, + NFS4ERR_LAYOUTTRYLATER = 10058, + NFS4ERR_LAYOUTUNAVAILABLE = 10059, + NFS4ERR_NOMATCHING_LAYOUT = 10060, + NFS4ERR_RECALLCONFLICT = 10061, + NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, + NFS4ERR_SEQ_MISORDERED = 10063, + NFS4ERR_SEQUENCE_POS = 10064, + NFS4ERR_REQ_TOO_BIG = 10065, + NFS4ERR_REP_TOO_BIG = 10066, + NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, + NFS4ERR_RETRY_UNCACHED_REP = 10068, + NFS4ERR_UNSAFE_COMPOUND = 10069, + NFS4ERR_TOO_MANY_OPS = 10070, + NFS4ERR_OP_NOT_IN_SESSION = 10071, + NFS4ERR_HASH_ALG_UNSUPP = 10072, + NFS4ERR_CLIENTID_BUSY = 10074, + NFS4ERR_PNFS_IO_HOLE = 10075, + NFS4ERR_SEQ_FALSE_RETRY = 10076, + NFS4ERR_BAD_HIGH_SLOT = 10077, + NFS4ERR_DEADSESSION = 10078, + NFS4ERR_ENCR_ALG_UNSUPP = 10079, + NFS4ERR_PNFS_NO_LAYOUT = 10080, + NFS4ERR_NOT_ONLY_OP = 10081, + NFS4ERR_WRONG_CRED = 10082, + NFS4ERR_WRONG_TYPE = 10083, + NFS4ERR_DIRDELEG_UNAVAIL = 10084, + NFS4ERR_REJECT_DELEG = 10085, + NFS4ERR_RETURNCONFLICT = 10086, + NFS4ERR_DELEG_REVOKED = 10087, + NFS4ERR_PARTNER_NOTSUPP = 10088, + NFS4ERR_PARTNER_NO_AUTH = 10089, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096, +}; + +typedef enum nfsstat4 nfsstat4; + +typedef opaque attrlist4; + typedef struct { u32 count; uint32_t *element; } bitmap4; +typedef u8 verifier4[NFS4_VERIFIER_SIZE]; + +typedef uint64_t nfs_cookie4; + +typedef opaque nfs_fh4; + typedef opaque utf8string; typedef utf8string utf8str_cis; @@ -26,11 +160,30 @@ typedef utf8string utf8str_cs; typedef utf8string utf8str_mixed; +typedef utf8str_cs component4; + +typedef utf8str_cs linktext4; + +typedef struct { + u32 count; + component4 *element; +} pathname4; + struct nfstime4 { int64_t seconds; uint32_t nseconds; }; +struct fattr4 { + bitmap4 attrmask; + attrlist4 attr_vals; +}; + +struct stateid4 { + uint32_t seqid; + u8 other[12]; +}; + typedef bool fattr4_offline; enum { FATTR4_OFFLINE = 83 }; @@ -216,11 +369,109 @@ enum { FATTR4_POSIX_DEFAULT_ACL = 91 }; enum { FATTR4_POSIX_ACCESS_ACL = 92 }; -#define NFS4_int64_t_sz \ - (XDR_hyper) +enum notify_type4 { + NOTIFY4_CHANGE_CHILD_ATTRS = 0, + NOTIFY4_CHANGE_DIR_ATTRS = 1, + NOTIFY4_REMOVE_ENTRY = 2, + NOTIFY4_ADD_ENTRY = 3, + NOTIFY4_RENAME_ENTRY = 4, + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5, + NOTIFY4_GFLAG_EXTEND = 6, + NOTIFY4_AUFLAG_VALID = 7, + NOTIFY4_AUFLAG_USER = 8, + NOTIFY4_AUFLAG_GROUP = 9, + NOTIFY4_AUFLAG_OTHER = 10, + NOTIFY4_CHANGE_AUTH = 11, + NOTIFY4_CFLAG_ORDER = 12, + NOTIFY4_AUFLAG_GANOW = 13, + NOTIFY4_AUFLAG_GALATER = 14, + NOTIFY4_CHANGE_GA = 15, + NOTIFY4_CHANGE_AMASK = 16, +}; + +typedef enum notify_type4 notify_type4; + +struct notify_entry4 { + component4 ne_file; + struct fattr4 ne_attrs; +}; + +struct prev_entry4 { + struct notify_entry4 pe_prev_entry; + nfs_cookie4 pe_prev_entry_cookie; +}; + +struct notify_remove4 { + struct notify_entry4 nrm_old_entry; + nfs_cookie4 nrm_old_entry_cookie; +}; + +struct notify_add4 { + struct { + u32 count; + struct notify_remove4 *element; + } nad_old_entry; + struct notify_entry4 nad_new_entry; + struct { + u32 count; + nfs_cookie4 *element; + } nad_new_entry_cookie; + struct { + u32 count; + struct prev_entry4 *element; + } nad_prev_entry; + bool nad_last_entry; +}; + +struct notify_attr4 { + struct notify_entry4 na_changed_entry; +}; + +struct notify_rename4 { + struct notify_remove4 nrn_old_entry; + struct notify_add4 nrn_new_entry; +}; + +struct notify_verifier4 { + verifier4 nv_old_cookieverf; + verifier4 nv_new_cookieverf; +}; + +typedef opaque notifylist4; + +struct notify4 { + bitmap4 notify_mask; + notifylist4 notify_vals; +}; + +struct CB_NOTIFY4args { + struct stateid4 cna_stateid; + nfs_fh4 cna_fh; + struct { + u32 count; + struct notify4 *element; + } cna_changes; +}; + +struct CB_NOTIFY4res { + nfsstat4 cnr_status; +}; + +#define NFS4_int32_t_sz \ + (XDR_int) #define NFS4_uint32_t_sz \ (XDR_unsigned_int) +#define NFS4_int64_t_sz \ + (XDR_hyper) +#define NFS4_uint64_t_sz \ + (XDR_unsigned_hyper) +#define NFS4_nfsstat4_sz (XDR_int) +#define NFS4_attrlist4_sz (XDR_unsigned_int) #define NFS4_bitmap4_sz (XDR_unsigned_int) +#define NFS4_verifier4_sz (XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define NFS4_nfs_cookie4_sz \ + (NFS4_uint64_t_sz) +#define NFS4_nfs_fh4_sz (XDR_unsigned_int + XDR_QUADLEN(NFS4_FHSIZE)) #define NFS4_utf8string_sz (XDR_unsigned_int) #define NFS4_utf8str_cis_sz \ (NFS4_utf8string_sz) @@ -228,8 +479,17 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_utf8string_sz) #define NFS4_utf8str_mixed_sz \ (NFS4_utf8string_sz) +#define NFS4_component4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_linktext4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_pathname4_sz (XDR_unsigned_int) #define NFS4_nfstime4_sz \ (NFS4_int64_t_sz + NFS4_uint32_t_sz) +#define NFS4_fattr4_sz \ + (NFS4_bitmap4_sz + NFS4_attrlist4_sz) +#define NFS4_stateid4_sz \ + (NFS4_uint32_t_sz + XDR_QUADLEN(12)) #define NFS4_fattr4_offline_sz \ (XDR_bool) #define NFS4_open_arguments4_sz \ @@ -259,5 +519,27 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_aclscope4_sz) #define NFS4_fattr4_posix_default_acl_sz (XDR_unsigned_int) #define NFS4_fattr4_posix_access_acl_sz (XDR_unsigned_int) +#define NFS4_notify_type4_sz (XDR_int) +#define NFS4_notify_entry4_sz \ + (NFS4_component4_sz + NFS4_fattr4_sz) +#define NFS4_prev_entry4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_remove4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_add4_sz \ + (XDR_unsigned_int + (1 * (NFS4_notify_remove4_sz)) + NFS4_notify_entry4_sz + XDR_unsigned_int + (1 * (NFS4_nfs_cookie4_sz)) + XDR_unsigned_int + (1 * (NFS4_prev_entry4_sz)) + XDR_bool) +#define NFS4_notify_attr4_sz \ + (NFS4_notify_entry4_sz) +#define NFS4_notify_rename4_sz \ + (NFS4_notify_remove4_sz + NFS4_notify_add4_sz) +#define NFS4_notify_verifier4_sz \ + (NFS4_verifier4_sz + NFS4_verifier4_sz) +#define NFS4_notifylist4_sz (XDR_unsigned_int) +#define NFS4_notify4_sz \ + (NFS4_bitmap4_sz + NFS4_notifylist4_sz) +#define NFS4_CB_NOTIFY4args_sz \ + (NFS4_stateid4_sz + NFS4_nfs_fh4_sz + XDR_unsigned_int) +#define NFS4_CB_NOTIFY4res_sz \ + (NFS4_nfsstat4_sz) #endif /* _LINUX_XDRGEN_NFS4_1_DEF_H */ diff --git a/include/linux/sunrpc/xdrgen/nlm3.h b/include/linux/sunrpc/xdrgen/nlm3.h index 897e7d91807c..0fc627031d8a 100644 --- a/include/linux/sunrpc/xdrgen/nlm3.h +++ b/include/linux/sunrpc/xdrgen/nlm3.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ -/* XDR specification modification time: Thu Apr 23 10:56:34 2026 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM3_DEF_H #define _LINUX_XDRGEN_NLM3_DEF_H diff --git a/include/linux/sunrpc/xdrgen/nlm4.h b/include/linux/sunrpc/xdrgen/nlm4.h index e95e8f105624..77860a3d1c1e 100644 --- a/include/linux/sunrpc/xdrgen/nlm4.h +++ b/include/linux/sunrpc/xdrgen/nlm4.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ -/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM4_DEF_H #define _LINUX_XDRGEN_NLM4_DEF_H diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 6c1438f6a4d3..6ecfab97c1a9 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -1613,9 +1613,9 @@ TRACE_EVENT(btrfs_sync_log_enter, __entry->log_transid_committed = data_race(root->log_transid_committed); __entry->log_committing = - atomic_read(&root->log_commit[ctx->log_transid % 2]); + data_race(root->log_commit[ctx->log_transid % 2]); __entry->log_committing_prev = - atomic_read(&root->log_commit[(ctx->log_transid + 1) % 2]); + data_race(root->log_commit[(ctx->log_transid + 1) % 2]); __entry->log_writers = atomic_read(&root->log_writers); ), diff --git a/include/uapi/linux/nfs4.h b/include/uapi/linux/nfs4.h index 4273e0249fcb..289205b53a08 100644 --- a/include/uapi/linux/nfs4.h +++ b/include/uapi/linux/nfs4.h @@ -17,11 +17,9 @@ #include <linux/types.h> #define NFS4_BITMAP_SIZE 3 -#define NFS4_VERIFIER_SIZE 8 #define NFS4_STATEID_SEQID_SIZE 4 #define NFS4_STATEID_OTHER_SIZE 12 #define NFS4_STATEID_SIZE (NFS4_STATEID_SEQID_SIZE + NFS4_STATEID_OTHER_SIZE) -#define NFS4_FHSIZE 128 #define NFS4_MAXPATHLEN PATH_MAX #define NFS4_MAXNAMLEN NAME_MAX #define NFS4_OPAQUE_LIMIT 1024 diff --git a/include/uapi/linux/nfsd_netlink.h b/include/uapi/linux/nfsd_netlink.h index f5b75d5caba9..87da1d0bb21e 100644 --- a/include/uapi/linux/nfsd_netlink.h +++ b/include/uapi/linux/nfsd_netlink.h @@ -226,6 +226,41 @@ enum { }; enum { + NFSD_A_SERVER_PROC_ENTRY_OP = 1, + NFSD_A_SERVER_PROC_ENTRY_COUNT, + NFSD_A_SERVER_PROC_ENTRY_PAD, + + __NFSD_A_SERVER_PROC_ENTRY_MAX, + NFSD_A_SERVER_PROC_ENTRY_MAX = (__NFSD_A_SERVER_PROC_ENTRY_MAX - 1) +}; + +enum { + NFSD_A_SERVER_STATS_RC_HITS = 1, + NFSD_A_SERVER_STATS_RC_MISSES, + NFSD_A_SERVER_STATS_RC_NOCACHE, + NFSD_A_SERVER_STATS_PAD, + NFSD_A_SERVER_STATS_FH_STALE, + NFSD_A_SERVER_STATS_IO_READ, + NFSD_A_SERVER_STATS_IO_WRITE, + NFSD_A_SERVER_STATS_NETCNT, + NFSD_A_SERVER_STATS_NETUDPCNT, + NFSD_A_SERVER_STATS_NETTCPCNT, + NFSD_A_SERVER_STATS_NETTCPCONN, + NFSD_A_SERVER_STATS_RPCCNT, + NFSD_A_SERVER_STATS_RPCBADFMT, + NFSD_A_SERVER_STATS_RPCBADAUTH, + NFSD_A_SERVER_STATS_RPCBADCLNT, + NFSD_A_SERVER_STATS_PROC2_OPS, + NFSD_A_SERVER_STATS_PROC3_OPS, + NFSD_A_SERVER_STATS_PROC4_OPS, + NFSD_A_SERVER_STATS_PROC4OPS_OPS, + NFSD_A_SERVER_STATS_PROC4CB_OPS, + + __NFSD_A_SERVER_STATS_MAX, + NFSD_A_SERVER_STATS_MAX = (__NFSD_A_SERVER_STATS_MAX - 1) +}; + +enum { NFSD_CMD_RPC_STATUS_GET = 1, NFSD_CMD_THREADS_SET, NFSD_CMD_THREADS_GET, @@ -244,6 +279,7 @@ enum { NFSD_CMD_UNLOCK_IP, NFSD_CMD_UNLOCK_FILESYSTEM, NFSD_CMD_UNLOCK_EXPORT, + NFSD_CMD_SERVER_STATS_GET, __NFSD_CMD_MAX, NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1) diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 9d3fb6848f40..8ddc65e894da 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -2072,7 +2072,11 @@ gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred, goto unwrap_failed; opaque_len = be32_to_cpup(p++); offset = (u8 *)(p) - (u8 *)head->iov_base; - if (offset + opaque_len > rcv_buf->len) + if (offset > rcv_buf->len) + goto unwrap_failed; + if (opaque_len > rcv_buf->len - offset) + goto unwrap_failed; + if (opaque_len <= GSS_KRB5_TOK_HDR_LEN) goto unwrap_failed; maj_stat = gss_unwrap(ctx->gc_gss_ctx, offset, diff --git a/net/sunrpc/auth_gss/gss_krb5_unseal.c b/net/sunrpc/auth_gss/gss_krb5_unseal.c index b5fb70419faa..4d12d49434c2 100644 --- a/net/sunrpc/auth_gss/gss_krb5_unseal.c +++ b/net/sunrpc/auth_gss/gss_krb5_unseal.c @@ -89,6 +89,9 @@ gss_krb5_verify_mic_v2(struct krb5_ctx *ctx, struct xdr_buf *message_buffer, dprintk("RPC: %s\n", __func__); + if (read_token->len < GSS_KRB5_TOK_HDR_LEN + cksum_len) + return GSS_S_DEFECTIVE_TOKEN; + memcpy(&be16_ptr, (char *) ptr, 2); if (be16_to_cpu(be16_ptr) != KG2_TOK_MIC) return GSS_S_DEFECTIVE_TOKEN; diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c index ac4b32df42b9..d3f61c4b5a13 100644 --- a/net/sunrpc/auth_gss/gss_krb5_wrap.c +++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c @@ -73,6 +73,8 @@ static void _rotate_left(struct xdr_buf *buf, unsigned int shift) int shifted = 0; int this_shift; + if (!buf->len) + return; shift %= buf->len; while (shifted < shift) { this_shift = min(shift - shifted, LOCAL_BUF_LEN); @@ -85,6 +87,8 @@ static void rotate_left(u32 base, struct xdr_buf *buf, unsigned int shift) { struct xdr_buf subbuf; + if (buf->len <= base) + return; xdr_buf_subsegment(buf, &subbuf, base, buf->len - base); _rotate_left(&subbuf, shift); } @@ -154,6 +158,9 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, dprintk("RPC: %s\n", __func__); + if (len - offset <= GSS_KRB5_TOK_HDR_LEN) + return GSS_S_DEFECTIVE_TOKEN; + ptr = buf->head[0].iov_base + offset; if (be16_to_cpu(*((__be16 *)ptr)) != KG2_TOK_WRAP) @@ -220,14 +227,16 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, * head buffer space rather than that actually occupied. */ movelen = min_t(unsigned int, buf->head[0].iov_len, len); + if (movelen < offset + GSS_KRB5_TOK_HDR_LEN + headskip) + return GSS_S_DEFECTIVE_TOKEN; movelen -= offset + GSS_KRB5_TOK_HDR_LEN + headskip; - BUG_ON(offset + GSS_KRB5_TOK_HDR_LEN + headskip + movelen > - buf->head[0].iov_len); memmove(ptr, ptr + GSS_KRB5_TOK_HDR_LEN + headskip, movelen); buf->head[0].iov_len -= GSS_KRB5_TOK_HDR_LEN + headskip; buf->len = len - (GSS_KRB5_TOK_HDR_LEN + headskip); /* Trim off the trailing "extra count" and checksum blob */ + if (ec + GSS_KRB5_TOK_HDR_LEN + tailskip > buf->len - offset) + return GSS_S_DEFECTIVE_TOKEN; xdr_buf_trim(buf, ec + GSS_KRB5_TOK_HDR_LEN + tailskip); *align = XDR_QUADLEN(GSS_KRB5_TOK_HDR_LEN + headskip); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.c b/net/sunrpc/auth_gss/gss_rpc_upcall.c index 0fa4778620d9..b7f70b1adb18 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.c +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.c @@ -121,12 +121,6 @@ out: return result; } -void init_gssp_clnt(struct sunrpc_net *sn) -{ - mutex_init(&sn->gssp_lock); - sn->gssp_clnt = NULL; -} - int set_gssp_clnt(struct net *net) { struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.h b/net/sunrpc/auth_gss/gss_rpc_upcall.h index 31e96344167e..b3c2b2b90798 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.h +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.h @@ -29,7 +29,6 @@ int gssp_accept_sec_context_upcall(struct net *net, struct gssp_upcall_data *data); void gssp_free_upcall_data(struct gssp_upcall_data *data); -void init_gssp_clnt(struct sunrpc_net *); int set_gssp_clnt(struct net *); void clear_gssp_clnt(struct sunrpc_net *); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index fceee648d545..0549edae1ebe 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -222,7 +222,8 @@ static int gssx_dec_linux_creds(struct xdr_stream *xdr, return 0; out_free_groups: - groups_free(creds->cr_group_info); + put_group_info(creds->cr_group_info); + creds->cr_group_info = NULL; return err; } @@ -230,6 +231,7 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, struct gssx_option_array *oa) { struct svc_cred *creds; + bool creds_decoded = false; u32 count, i; __be32 *p; int err; @@ -242,12 +244,12 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; /* we recognize only 1 currently: CREDS_VALUE */ - oa->count = 1; - oa->data = kmalloc_obj(struct gssx_option); if (!oa->data) return -ENOMEM; + oa->count = 1; + creds = kzalloc_obj(struct svc_cred); if (!creds) { err = -ENOMEM; @@ -280,9 +282,14 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, if (length == sizeof(CREDS_VALUE) && memcmp(p, CREDS_VALUE, sizeof(CREDS_VALUE)) == 0) { /* We have creds here. parse them */ + if (creds_decoded) { + err = -EINVAL; + goto free_creds; + } err = gssx_dec_linux_creds(xdr, creds); if (err) goto free_creds; + creds_decoded = true; oa->data[0].value.len = 1; /* presence */ } else { /* consume uninteresting buffer */ @@ -294,8 +301,10 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; free_creds: + free_svc_cred(creds); kfree(creds); free_oa: + oa->count = 0; kfree(oa->data); oa->data = NULL; return err; diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index d14209031e18..967e9d53080d 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -949,6 +949,8 @@ svcauth_gss_unwrap_priv(struct svc_rqst *rqstp, u32 seq, struct gss_ctx *ctx) } if (len > xdr_stream_remaining(xdr)) goto unwrap_failed; + if (len <= GSS_KRB5_TOK_HDR_LEN) + goto unwrap_failed; offset = xdr_stream_pos(xdr); saved_len = buf->len; @@ -1466,7 +1468,6 @@ static int create_use_gss_proxy_proc_entry(struct net *net) &use_gss_proxy_proc_ops, net); if (!*p) return -ENOMEM; - init_gssp_clnt(sn); return 0; } @@ -1573,6 +1574,9 @@ svcauth_gss_decode_credbody(struct xdr_stream *xdr, u32 body_len; __be32 *p; + /* Early-return paths leave deterministic state, not stale residue. */ + memset(gc, 0, sizeof(*gc)); + p = xdr_inline_decode(xdr, XDR_UNIT); if (!p) return false; @@ -1942,6 +1946,8 @@ svcauth_gss_release(struct svc_rqst *rqstp) if (!gsd) goto out; + if (rqstp->rq_auth_stat != rpc_auth_ok) + goto out; gc = &gsd->clcred; if (gc->gc_proc != RPC_GSS_PROC_DATA) goto out; diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c index 0ffa4d01a938..1482b06e0f38 100644 --- a/net/sunrpc/backchannel_rqst.c +++ b/net/sunrpc/backchannel_rqst.c @@ -25,20 +25,39 @@ unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt) } /* - * Helper function to nullify backchannel server pointer in transport. - * We need to synchronize setting the pointer to NULL (done so after - * the backchannel server is shutdown) with the usage of that pointer - * by the backchannel request processing routines - * xprt_complete_bc_request() and rpcrdma_bc_receive_call(). + * Close the backchannel producer side, drain any requests still + * queued on sv_cb_list, then destroy the callback service. */ void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv) { - spin_lock(&xprt->bc_pa_lock); + struct svc_serv *bc_serv = *serv; + struct rpc_rqst *req; + + xprt_svc_shutdown_bc(xprt); + while ((req = lwq_dequeue(&bc_serv->sv_cb_list, struct rpc_rqst, + rq_bc_list)) != NULL) { + atomic_dec(&req->rq_xprt->bc_slot_count); + xprt_free_bc_request(req); + } svc_destroy(serv); +} +EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc); + +/* + * Clear the backchannel server pointer in the transport. The NULL + * store is serialized under bc_pa_lock against readers of + * xprt->bc_serv in xprt_complete_bc_request() and + * rpcrdma_bc_receive_call(). Clearing it before the callback service + * is stopped prevents a producer from enqueueing onto a service that + * is being torn down. + */ +void xprt_svc_shutdown_bc(struct rpc_xprt *xprt) +{ + spin_lock(&xprt->bc_pa_lock); xprt->bc_serv = NULL; spin_unlock(&xprt->bc_pa_lock); } -EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc); +EXPORT_SYMBOL_GPL(xprt_svc_shutdown_bc); /* * Helper routines that track the number of preallocation elements @@ -393,7 +412,12 @@ void xprt_enqueue_bc_request(struct rpc_rqst *req) if (bc_serv) { lwq_enqueue(&req->rq_bc_list, &bc_serv->sv_cb_list); svc_pool_wake_idle_thread(&bc_serv->sv_pools[0]); + spin_unlock(&xprt->bc_pa_lock); + return; } spin_unlock(&xprt->bc_pa_lock); + + atomic_dec(&xprt->bc_slot_count); + xprt_free_bc_request(req); } EXPORT_SYMBOL_GPL(xprt_enqueue_bc_request); diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 391037f15292..1bc04109d213 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -430,10 +430,9 @@ void sunrpc_destroy_cache_detail(struct cache_detail *cd) list_del_init(&cd->others); spin_unlock(&cd->hash_lock); spin_unlock(&cache_list_lock); - if (list_empty(&cache_list)) { - /* module must be being unloaded so its safe to kill the worker */ - cancel_delayed_work_sync(&cache_cleaner); - } + cancel_delayed_work_sync(&cache_cleaner); + if (!list_empty(&cache_list)) + queue_delayed_work(system_power_efficient_wq, &cache_cleaner, 0); } EXPORT_SYMBOL_GPL(sunrpc_destroy_cache_detail); diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c index 7093e18ac26c..d08711bee18e 100644 --- a/net/sunrpc/stats.c +++ b/net/sunrpc/stats.c @@ -108,7 +108,7 @@ void svc_seq_show(struct seq_file *seq, const struct svc_stat *statp) for (j = 0; j < vers->vs_nproc; j++) { count = 0; for_each_possible_cpu(k) - count += per_cpu(vers->vs_count[j], k); + count += per_cpu(statp->vs_count[i][j], k); seq_printf(seq, " %lu", count); } seq_putc(seq, '\n'); diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c index ab88ce46afb5..1a3884a0376a 100644 --- a/net/sunrpc/sunrpc_syms.c +++ b/net/sunrpc/sunrpc_syms.c @@ -57,6 +57,7 @@ static __net_init int sunrpc_init_net(struct net *net) INIT_LIST_HEAD(&sn->all_clients); spin_lock_init(&sn->rpc_client_lock); spin_lock_init(&sn->rpcb_clnt_lock); + mutex_init(&sn->gssp_lock); return 0; err_pipefs: diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index ae9ec4bf34f7..8297bad2b177 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -38,82 +38,36 @@ static void svc_unregister(const struct svc_serv *serv, struct net *net); -#define SVC_POOL_DEFAULT SVC_POOL_GLOBAL - -/* - * Mode for mapping cpus to pools. - */ -enum { - SVC_POOL_AUTO = -1, /* choose one of the others */ - SVC_POOL_GLOBAL, /* no mapping, just a single global pool - * (legacy & UP mode) */ - SVC_POOL_PERCPU, /* one pool per cpu */ - SVC_POOL_PERNODE /* one pool per numa node */ -}; - /* - * Structure for mapping cpus to pools and vice versa. + * Structure for mapping nodes to pools and vice versa. * Setup once during sunrpc initialisation. */ struct svc_pool_map { int count; /* How many svc_servs use us */ - int mode; /* Note: int not enum to avoid - * warnings about "enumeration value - * not handled in switch" */ unsigned int npools; - unsigned int *pool_to; /* maps pool id to cpu or node */ - unsigned int *to_pool; /* maps cpu or node to pool id */ + unsigned int *pool_to; /* maps pool id to node */ + unsigned int *to_pool; /* maps node to pool id */ }; -static struct svc_pool_map svc_pool_map = { - .mode = SVC_POOL_DEFAULT -}; +static struct svc_pool_map svc_pool_map; static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */ -static int -__param_set_pool_mode(const char *val, struct svc_pool_map *m) -{ - int err, mode; - - mutex_lock(&svc_pool_map_mutex); - - err = 0; - if (!strncmp(val, "auto", 4)) - mode = SVC_POOL_AUTO; - else if (!strncmp(val, "global", 6)) - mode = SVC_POOL_GLOBAL; - else if (!strncmp(val, "percpu", 6)) - mode = SVC_POOL_PERCPU; - else if (!strncmp(val, "pernode", 7)) - mode = SVC_POOL_PERNODE; - else - err = -EINVAL; - - if (err) - goto out; - - if (m->count == 0) - m->mode = mode; - else if (mode != m->mode) - err = -EBUSY; -out: - mutex_unlock(&svc_pool_map_mutex); - return err; -} - -static int -param_set_pool_mode(const char *val, const struct kernel_param *kp) -{ - struct svc_pool_map *m = kp->arg; - - return __param_set_pool_mode(val, m); -} +/* + * Pool modes that were historically accepted. They no longer select + * anything: the pool mode is always pernode. The names are retained + * only so that writing a previously-valid value still succeeds. + */ +static const char * const pool_mode_names[] = { + "auto", "global", "percpu", "pernode", +}; int sunrpc_set_pool_mode(const char *val) { - return __param_set_pool_mode(val, &svc_pool_map); + int idx = sysfs_match_string(pool_mode_names, val); + + return idx < 0 ? idx : 0; } EXPORT_SYMBOL(sunrpc_set_pool_mode); @@ -122,84 +76,32 @@ EXPORT_SYMBOL(sunrpc_set_pool_mode); * @buf: where to write the current pool_mode * @size: size of @buf * - * Grab the current pool_mode from the svc_pool_map and write - * the resulting string to @buf. Returns the number of characters + * Write the pool_mode string to @buf. Returns the number of characters * written to @buf (a'la snprintf()). */ int sunrpc_get_pool_mode(char *buf, size_t size) { - struct svc_pool_map *m = &svc_pool_map; - - switch (m->mode) - { - case SVC_POOL_AUTO: - return snprintf(buf, size, "auto"); - case SVC_POOL_GLOBAL: - return snprintf(buf, size, "global"); - case SVC_POOL_PERCPU: - return snprintf(buf, size, "percpu"); - case SVC_POOL_PERNODE: - return snprintf(buf, size, "pernode"); - default: - return snprintf(buf, size, "%d", m->mode); - } + return snprintf(buf, size, "pernode"); } EXPORT_SYMBOL(sunrpc_get_pool_mode); static int -param_get_pool_mode(char *buf, const struct kernel_param *kp) +param_set_pool_mode(const char *val, const struct kernel_param *kp) { - char str[16]; - int len; - - len = sunrpc_get_pool_mode(str, ARRAY_SIZE(str)); - - /* Ensure we have room for newline and NUL */ - len = min_t(int, len, ARRAY_SIZE(str) - 2); - - /* tack on the newline */ - str[len] = '\n'; - str[len + 1] = '\0'; - - return sysfs_emit(buf, "%s", str); + pr_notice_once("sunrpc: the pool_mode module parameter is deprecated and no longer has any effect; the pool mode is always 'pernode'\n"); + return sunrpc_set_pool_mode(val); } -module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode, - &svc_pool_map, 0644); - -/* - * Detect best pool mapping mode heuristically, - * according to the machine's topology. - */ static int -svc_pool_map_choose_mode(void) +param_get_pool_mode(char *buf, const struct kernel_param *kp) { - unsigned int node; - - if (nr_online_nodes > 1) { - /* - * Actually have multiple NUMA nodes, - * so split pools on NUMA node boundaries - */ - return SVC_POOL_PERNODE; - } - - node = first_online_node; - if (nr_cpus_node(node) > 2) { - /* - * Non-trivial SMP, or CONFIG_NUMA on - * non-NUMA hardware, e.g. with a generic - * x86_64 kernel on Xeons. In this case we - * want to divide the pools on cpu boundaries. - */ - return SVC_POOL_PERCPU; - } - - /* default: one global pool */ - return SVC_POOL_GLOBAL; + return sysfs_emit(buf, "pernode\n"); } +module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode, + NULL, 0644); + /* * Allocate the to_pool[] and pool_to[] arrays. * Returns 0 on success or an errno. @@ -224,35 +126,7 @@ fail: } /* - * Initialise the pool map for SVC_POOL_PERCPU mode. - * Returns number of pools or <0 on error. - */ -static int -svc_pool_map_init_percpu(struct svc_pool_map *m) -{ - unsigned int maxpools = nr_cpu_ids; - unsigned int pidx = 0; - unsigned int cpu; - int err; - - err = svc_pool_map_alloc_arrays(m, maxpools); - if (err) - return err; - - for_each_online_cpu(cpu) { - BUG_ON(pidx >= maxpools); - m->to_pool[cpu] = pidx; - m->pool_to[pidx] = cpu; - pidx++; - } - /* cpus brought online later all get mapped to pool0, sorry */ - - return pidx; -}; - - -/* - * Initialise the pool map for SVC_POOL_PERNODE mode. + * Initialise the pool map for one pool per NUMA node. * Returns number of pools or <0 on error. */ static int @@ -281,17 +155,16 @@ svc_pool_map_init_pernode(struct svc_pool_map *m) /* - * Add a reference to the global map of cpus to pools (and + * Add a reference to the global map of nodes to pools (and * vice versa) if pools are in use. * Initialise the map if we're the first user. - * Returns the number of pools. If this is '1', no reference - * was taken. + * Returns the number of pools, or 0 on failure. */ static unsigned int svc_pool_map_get(void) { struct svc_pool_map *m = &svc_pool_map; - int npools = -1; + int npools; mutex_lock(&svc_pool_map_mutex); if (m->count++) { @@ -299,22 +172,11 @@ svc_pool_map_get(void) return m->npools; } - if (m->mode == SVC_POOL_AUTO) - m->mode = svc_pool_map_choose_mode(); - - switch (m->mode) { - case SVC_POOL_PERCPU: - npools = svc_pool_map_init_percpu(m); - break; - case SVC_POOL_PERNODE: - npools = svc_pool_map_init_pernode(m); - break; - } - + npools = svc_pool_map_init_pernode(m); if (npools <= 0) { - /* default, or memory allocation failure */ - npools = 1; - m->mode = SVC_POOL_GLOBAL; + m->count = 0; + mutex_unlock(&svc_pool_map_mutex); + return 0; } m->npools = npools; mutex_unlock(&svc_pool_map_mutex); @@ -322,7 +184,7 @@ svc_pool_map_get(void) } /* - * Drop a reference to the global map of cpus to pools. + * Drop a reference to the global map of nodes to pools. * When the last reference is dropped, the map data is * freed; this allows the sysadmin to change the pool. */ @@ -346,14 +208,9 @@ static int svc_pool_map_get_node(unsigned int pidx) { const struct svc_pool_map *m = &svc_pool_map; - if (m->count) { - if (m->mode == SVC_POOL_PERCPU) - return cpu_to_node(m->pool_to[pidx]); - if (m->mode == SVC_POOL_PERNODE) - return m->pool_to[pidx]; - } - return numa_mem_id(); + return m->pool_to[pidx]; } + /* * Set the given thread's cpus_allowed mask so that it * will only run on cpus in the given pool. @@ -365,57 +222,80 @@ svc_pool_map_set_cpumask(struct task_struct *task, unsigned int pidx) unsigned int node = m->pool_to[pidx]; /* - * The caller checks for sv_nrpools > 1, which + * The caller checks for more than one pool, which * implies that we've been initialized. */ WARN_ON_ONCE(m->count == 0); if (m->count == 0) return; - switch (m->mode) { - case SVC_POOL_PERCPU: - { - set_cpus_allowed_ptr(task, cpumask_of(node)); - break; - } - case SVC_POOL_PERNODE: - { - set_cpus_allowed_ptr(task, cpumask_of_node(node)); - break; - } - } + set_cpus_allowed_ptr(task, cpumask_of_node(node)); } /** + * svc_serv_nrpools - number of thread pools backing a service + * @serv: An RPC service + * + * Pooled services all share the global svc_pool_map, so their pool count + * is svc_pool_map.npools. Unpooled services have a single pool. Reading + * npools without svc_pool_map_mutex is safe: a pooled service holds a map + * reference for its whole lifetime, so npools is stable once set. + * + * Return value: + * The number of pools in @serv + */ +unsigned int svc_serv_nrpools(const struct svc_serv *serv) +{ + return serv->sv_is_pooled ? svc_pool_map.npools : 1; +} +EXPORT_SYMBOL_GPL(svc_serv_nrpools); + +/** * svc_pool_for_cpu - Select pool to run a thread on this cpu * @serv: An RPC service * - * Use the active CPU and the svc_pool_map's mode setting to - * select the svc thread pool to use. Once initialized, the - * svc_pool_map does not change. + * Use the active CPU and the svc_pool_map to select the svc thread + * pool to use. Once initialized, the svc_pool_map does not change. * * Return value: * A pointer to an svc_pool */ struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv) { + unsigned int nrpools = svc_serv_nrpools(serv); struct svc_pool_map *m = &svc_pool_map; - int cpu = raw_smp_processor_id(); - unsigned int pidx = 0; + unsigned int pidx, i; - if (serv->sv_nrpools <= 1) + if (nrpools <= 1) return serv->sv_pools; - switch (m->mode) { - case SVC_POOL_PERCPU: - pidx = m->to_pool[cpu]; - break; - case SVC_POOL_PERNODE: - pidx = m->to_pool[cpu_to_node(cpu)]; - break; + /* + * It's possible to have a pool with no threads. Userland can just set + * things up this way directly. Also, when threads are autodistributed + * they are spread evenly across the pools, but when there are fewer + * threads than pools some pools can end up with none. + * + * A transport enqueued on a threadless pool would never be picked up, + * since each thread only services its own pool. Fall back to the next + * populated pool, trading NUMA locality for a guarantee that the + * transport is serviced. + */ + pidx = m->to_pool[cpu_to_node(raw_smp_processor_id())]; + for (i = 0; i < nrpools; i++) { + struct svc_pool *pool = &serv->sv_pools[pidx]; + + /* This is set under the service mutex and rarely ever + * changes. A data race here is harmless. + */ + if (data_race(pool->sp_nrthreads)) + return pool; + + if (++pidx >= nrpools) + pidx = 0; } - return &serv->sv_pools[pidx % serv->sv_nrpools]; + /* No pool has any threads; nothing can service the transport. */ + return &serv->sv_pools[pidx]; } static int svc_rpcb_setup(struct svc_serv *serv, struct net *net) @@ -476,6 +356,35 @@ __svc_init_bc(struct svc_serv *serv) } #endif +static int svc_pool_init_counters(struct svc_pool *pool) +{ + int err; + + err = percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); + if (err) + return err; + err = percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); + if (err) + goto err_sockets; + err = percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (err) + goto err_threads; + return 0; + +err_threads: + percpu_counter_destroy(&pool->sp_sockets_queued); +err_sockets: + percpu_counter_destroy(&pool->sp_messages_arrived); + return err; +} + +static void svc_pool_destroy_counters(struct svc_pool *pool) +{ + percpu_counter_destroy(&pool->sp_messages_arrived); + percpu_counter_destroy(&pool->sp_sockets_queued); + percpu_counter_destroy(&pool->sp_threads_woken); +} + /* * Create an RPC service */ @@ -521,15 +430,13 @@ __svc_create(struct svc_program *prog, int nprogs, struct svc_stat *stats, __svc_init_bc(serv); - serv->sv_nrpools = npools; - serv->sv_pools = - kzalloc_objs(struct svc_pool, serv->sv_nrpools); + serv->sv_pools = kzalloc_objs(struct svc_pool, npools); if (!serv->sv_pools) { kfree(serv); return NULL; } - for (i = 0; i < serv->sv_nrpools; i++) { + for (i = 0; i < npools; i++) { struct svc_pool *pool = &serv->sv_pools[i]; dprintk("svc: initialising pool %u for %s\n", @@ -540,12 +447,18 @@ __svc_create(struct svc_program *prog, int nprogs, struct svc_stat *stats, INIT_LIST_HEAD(&pool->sp_all_threads); init_llist_head(&pool->sp_idle_threads); - percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (svc_pool_init_counters(pool)) + goto out_err; } return serv; + +out_err: + while (i--) + svc_pool_destroy_counters(&serv->sv_pools[i]); + kfree(serv->sv_pools); + kfree(serv); + return NULL; } /** @@ -582,6 +495,9 @@ struct svc_serv *svc_create_pooled(struct svc_program *prog, struct svc_serv *serv; unsigned int npools = svc_pool_map_get(); + if (!npools) + return NULL; + serv = __svc_create(prog, nprogs, stats, bufsize, npools, threadfn); if (!serv) goto out_err; @@ -618,16 +534,15 @@ svc_destroy(struct svc_serv **servp) cache_clean_deferred(serv); - if (serv->sv_is_pooled) - svc_pool_map_put(); - - for (i = 0; i < serv->sv_nrpools; i++) { + for (i = 0; i < svc_serv_nrpools(serv); i++) { struct svc_pool *pool = &serv->sv_pools[i]; - percpu_counter_destroy(&pool->sp_messages_arrived); - percpu_counter_destroy(&pool->sp_sockets_queued); - percpu_counter_destroy(&pool->sp_threads_woken); + svc_pool_destroy_counters(pool); } + + if (serv->sv_is_pooled) + svc_pool_map_put(); + kfree(serv->sv_pools); kfree(serv); } @@ -683,6 +598,15 @@ svc_release_buffer(struct svc_rqst *rqstp) } } +static void svc_rqst_free_rcu(struct rcu_head *head) +{ + struct svc_rqst *rqstp = container_of(head, struct svc_rqst, rq_rcu_head); + + kfree(rqstp->rq_resp); + kfree(rqstp->rq_argp); + kfree(rqstp); +} + static void svc_rqst_free(struct svc_rqst *rqstp) { @@ -691,10 +615,8 @@ svc_rqst_free(struct svc_rqst *rqstp) svc_release_buffer(rqstp); if (rqstp->rq_scratch_folio) folio_put(rqstp->rq_scratch_folio); - kfree(rqstp->rq_resp); - kfree(rqstp->rq_argp); kfree(rqstp->rq_auth_data); - kfree_rcu(rqstp, rq_rcu_head); + call_rcu(&rqstp->rq_rcu_head, svc_rqst_free_rcu); } static struct svc_rqst * @@ -711,7 +633,9 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) rqstp->rq_server = serv; rqstp->rq_pool = pool; - rqstp->rq_scratch_folio = __folio_alloc_node(GFP_KERNEL, 0, node); + rqstp->rq_scratch_folio = __folio_alloc_node(GFP_KERNEL, 0, + node == NUMA_NO_NODE ? + numa_mem_id() : node); if (!rqstp->rq_scratch_folio) goto out_enomem; @@ -800,7 +724,15 @@ int svc_new_thread(struct svc_serv *serv, struct svc_pool *pool) int node; int err = 0; - node = svc_pool_map_get_node(pool->sp_id); + /* + * Only pooled services hold a reference to the pool map, so only they + * may consult it. Unpooled services (e.g. lockd, the NFS callback) + * leave placement to the allocator. + */ + if (serv->sv_is_pooled) + node = svc_pool_map_get_node(pool->sp_id); + else + node = NUMA_NO_NODE; rqstp = svc_prepare_thread(serv, pool, node); if (!rqstp) @@ -813,7 +745,7 @@ int svc_new_thread(struct svc_serv *serv, struct svc_pool *pool) } rqstp->rq_task = task; - if (serv->sv_nrpools > 1) + if (svc_serv_nrpools(serv) > 1) svc_pool_map_set_cpumask(task, pool->sp_id); svc_sock_update_bufs(serv); @@ -922,6 +854,12 @@ EXPORT_SYMBOL_GPL(svc_set_pool_threads); * are multiple pools then the new threads or victims will be distributed * evenly among them. * + * When @nrservs is non-zero but smaller than the number of pools, even + * distribution would leave some pools empty. Since each pool maps to a + * NUMA node and only services transports steered to that node, every + * pool is instead guaranteed at least one thread. The resulting total + * may therefore exceed @nrservs. + * * Caller must ensure mutual exclusion between this and server startup or * shutdown. * @@ -933,11 +871,22 @@ int svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads, unsigned int nrservs) { - unsigned int base = nrservs / serv->sv_nrpools; - unsigned int remain = nrservs % serv->sv_nrpools; + unsigned int nrpools = svc_serv_nrpools(serv); + unsigned int base = nrservs / nrpools; + unsigned int remain = nrservs % nrpools; int i, err = 0; - for (i = 0; i < serv->sv_nrpools; ++i) { + /* + * Don't let a pool sit empty while threads are being + * auto-distributed: a transport steered to its node would have + * nothing to service it. Every pool maps to a CPU-bearing node, + * so hand each one a thread. This may push the total above + * @nrservs. + */ + if (base == 0 && nrservs != 0) + remain = nrpools; + + for (i = 0; i < nrpools; ++i) { struct svc_pool *pool = &serv->sv_pools[i]; int threads = base; @@ -955,6 +904,29 @@ svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads, EXPORT_SYMBOL_GPL(svc_set_num_threads); /** + * svc_serv_maxthreads - report a service's configured thread ceiling + * @serv: RPC service to query + * + * A pooled service sizes its threads dynamically, so the number of + * threads running at any moment tracks recent load rather than the + * service's capacity. The per-pool maximum is the stable figure a + * consumer should size against. + * + * The caller must keep @serv valid for the duration of the call. + * + * Return: the sum of every pool's maximum thread count. + */ +unsigned int svc_serv_maxthreads(const struct svc_serv *serv) +{ + unsigned int i, max = 0; + + for (i = 0; i < svc_serv_nrpools(serv); i++) + max += data_race(serv->sv_pools[i].sp_nrthrmax); + return max; +} +EXPORT_SYMBOL_GPL(svc_serv_maxthreads); + +/** * svc_rqst_replace_page - Replace one page in rq_respages[] * @rqstp: svc_rqst with pages to replace * @page: replacement page @@ -1372,8 +1344,13 @@ svc_generic_init_request(struct svc_rqst *rqstp, memset(rqstp->rq_argp, 0, procp->pc_argzero); memset(rqstp->rq_resp, 0, procp->pc_ressize); - /* Bump per-procedure stats counter */ - this_cpu_inc(versp->vs_count[rqstp->rq_proc]); + /* Bump per-net per-procedure stats counter */ + if (rqstp->rq_server->sv_stats && + rqstp->rq_server->sv_stats->program == progp && + rqstp->rq_server->sv_stats->vs_count && + rqstp->rq_server->sv_stats->vs_count[rqstp->rq_vers]) + this_cpu_inc(rqstp->rq_server->sv_stats->vs_count + [rqstp->rq_vers][rqstp->rq_proc]); ret->dispatch = versp->vs_dispatch; return rpc_success; @@ -1386,6 +1363,60 @@ err_bad_proc: } EXPORT_SYMBOL_GPL(svc_generic_init_request); +/** + * svc_stat_alloc_counts - allocate per-netns per-version call count arrays + * @statp: svc_stat whose vs_count arrays should be allocated + * + * statp->program must be set before calling this. + * + * Returns zero on success, or a negative errno otherwise. + */ +int svc_stat_alloc_counts(struct svc_stat *statp) +{ + struct svc_program *prog = statp->program; + unsigned int i; + + statp->vs_count = kcalloc(prog->pg_nvers, + sizeof(unsigned long __percpu *), + GFP_KERNEL); + if (!statp->vs_count) + return -ENOMEM; + + for (i = 0; i < prog->pg_nvers; i++) { + if (!prog->pg_vers[i]) + continue; + statp->vs_count[i] = __alloc_percpu(prog->pg_vers[i]->vs_nproc * + sizeof(unsigned long), + sizeof(unsigned long)); + if (!statp->vs_count[i]) + goto err; + } + return 0; +err: + svc_stat_free_counts(statp); + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(svc_stat_alloc_counts); + +/** + * svc_stat_free_counts - free per-netns per-version call count arrays + * @statp: svc_stat whose vs_count arrays should be freed + */ +void svc_stat_free_counts(struct svc_stat *statp) +{ + struct svc_program *prog = statp->program; + unsigned int i; + + if (!statp->vs_count) + return; + + for (i = 0; i < prog->pg_nvers; i++) + free_percpu(statp->vs_count[i]); + kfree(statp->vs_count); + statp->vs_count = NULL; +} +EXPORT_SYMBOL_GPL(svc_stat_free_counts); + /* * Common routine for processing the RPC request. */ diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 63d1002e63e7..40040af588fb 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -1188,7 +1188,7 @@ static void svc_clean_up_xprts(struct svc_serv *serv, struct net *net) struct svc_xprt *xprt; int i; - for (i = 0; i < serv->sv_nrpools; i++) { + for (i = 0; i < svc_serv_nrpools(serv); i++) { struct svc_pool *pool = &serv->sv_pools[i]; struct llist_node *q, **t1, *t2; @@ -1517,7 +1517,7 @@ static void *svc_pool_stats_start(struct seq_file *m, loff_t *pos) return SEQ_START_TOKEN; if (!si->serv) return NULL; - return pidx > si->serv->sv_nrpools ? NULL + return pidx > svc_serv_nrpools(si->serv) ? NULL : &si->serv->sv_pools[pidx - 1]; } @@ -1535,7 +1535,7 @@ static void *svc_pool_stats_next(struct seq_file *m, void *p, loff_t *pos) pool = &serv->sv_pools[0]; } else { unsigned int pidx = (pool - &serv->sv_pools[0]); - if (pidx < serv->sv_nrpools-1) + if (pidx < svc_serv_nrpools(serv) - 1) pool = &serv->sv_pools[pidx+1]; else pool = NULL; diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c index 64a2658faddb..aebd97e7f66c 100644 --- a/net/sunrpc/svcauth_unix.c +++ b/net/sunrpc/svcauth_unix.c @@ -133,7 +133,7 @@ static void ip_map_init(struct cache_head *cnew, struct cache_head *citem) struct ip_map *new = container_of(cnew, struct ip_map, h); struct ip_map *item = container_of(citem, struct ip_map, h); - strcpy(new->m_class, item->m_class); + strscpy(new->m_class, item->m_class); new->m_addr = item->m_addr; } static void update(struct cache_head *cnew, struct cache_head *citem) @@ -296,7 +296,7 @@ static struct ip_map *__ip_map_lookup(struct cache_detail *cd, char *class, struct ip_map ip; struct cache_head *ch; - strcpy(ip.m_class, class); + strscpy(ip.m_class, class); ip.m_addr = *addr; ch = sunrpc_cache_lookup_rcu(cd, &ip.h, hash_str(class, IP_HASHBITS) ^ diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index fa6a30b5f046..cb2ef428651f 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -2049,7 +2049,7 @@ void xdr_buf_trim(struct xdr_buf *buf, unsigned int len) trim -= cur; } fix_len: - buf->len -= (len - trim); + buf->len -= min_t(unsigned int, buf->len, len - trim); } EXPORT_SYMBOL_GPL(xdr_buf_trim); diff --git a/net/sunrpc/xprtrdma/ib_client.c b/net/sunrpc/xprtrdma/ib_client.c index de49ad02053d..188f7a13397f 100644 --- a/net/sunrpc/xprtrdma/ib_client.c +++ b/net/sunrpc/xprtrdma/ib_client.c @@ -51,7 +51,11 @@ static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device) * to be invoked when the device is removed, unless this notification * is unregistered first. * - * On failure, a negative errno is returned. + * On failure, a negative errno is returned. rn->rn_done is left + * NULL on every failure path (it is armed before xa_alloc but + * cleared again if xa_alloc fails), so the @rn may safely be + * passed to rpcrdma_rn_unregister() without a separate + * registered/unregistered flag in the caller. */ int rpcrdma_rn_register(struct ib_device *device, struct rpcrdma_notification *rn, @@ -62,10 +66,21 @@ int rpcrdma_rn_register(struct ib_device *device, if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags)) return -ENETUNREACH; - if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) + /* + * Arm rn_done before xa_alloc() publishes @rn: once @rn is + * visible in rd_xa, a concurrent rpcrdma_remove_one() can + * call rn->rn_done(), so the pointer must already be set. + * + * Restore NULL if xa_alloc() fails. rn_done doubles as the + * registration sentinel for rpcrdma_rn_unregister(); a stale + * value would unregister an @rn that was never inserted. + */ + rn->rn_done = done; + if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) { + rn->rn_done = NULL; return -ENOMEM; + } kref_get(&rd->rd_kref); - rn->rn_done = done; trace_rpcrdma_client_register(device, rn); return 0; } @@ -83,6 +98,10 @@ static void rpcrdma_rn_release(struct kref *kref) * rpcrdma_rn_unregister - stop device removal notifications * @device: monitored device * @rn: notification object that no longer wishes to be notified + * + * It is safe to call this on an @rn whose registration never + * completed or failed; rn_done == NULL is treated as + * never-registered and the call is a no-op. */ void rpcrdma_rn_unregister(struct ib_device *device, struct rpcrdma_notification *rn) @@ -92,6 +111,21 @@ void rpcrdma_rn_unregister(struct ib_device *device, if (!rd) return; + /* + * rn_done is the registration sentinel: rpcrdma_rn_register + * leaves it NULL on every failure path, clearing it again if + * xa_alloc fails, so a non-NULL rn_done marks a completed + * registration. A NULL rn_done means this notification was + * never registered (or its registration failed) or has + * already been unregistered, and the call is a no-op. + * Without this guard, rn_index == 0 from a kzalloc'd + * parent would erase another caller's slot 0 and underflow + * rd_kref. + */ + if (!rn->rn_done) + return; + rn->rn_done = NULL; + trace_rpcrdma_client_unregister(device, rn); xa_erase(&rd->rd_xa, rn->rn_index); kref_put(&rd->rd_kref, rpcrdma_rn_release); diff --git a/net/sunrpc/xprtrdma/svc_rdma_pcl.c b/net/sunrpc/xprtrdma/svc_rdma_pcl.c index 1f8f7dad8b6f..8623722790f2 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_pcl.c +++ b/net/sunrpc/xprtrdma/svc_rdma_pcl.c @@ -149,9 +149,6 @@ bool pcl_alloc_call(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) * cl_count is updated to be the number of chunks (ie. * unique position values) in the Read list. * %false: Memory allocation failed. - * - * TODO: - * - Check for chunk range overlaps */ bool pcl_alloc_read(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) { @@ -213,7 +210,6 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk = pcl_alloc_chunk(segcount, 0); if (!chunk) return false; - list_add_tail(&chunk->ch_list, &pcl->cl_chunks); for (j = 0; j < segcount; j++) { segment = &chunk->ch_segments[j]; @@ -225,7 +221,66 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk->ch_length += segment->rs_length; chunk->ch_segcount++; } + list_add_tail(&chunk->ch_list, &pcl->cl_chunks); + } + return true; +} + +/** + * pcl_check_read_chunk_positions - Validate Read chunk positions + * @rctxt: Ingress receive context with populated chunk lists + * @inline_len: Length of the inline RPC body after the transport header + * + * Read chunk positions are offsets in the unreduced XDR stream + * (RFC 8166 Section 3.4.4), so each position includes the + * cumulative length of preceding Read chunks. This function + * subtracts those lengths to recover the inline-body offset + * before comparing against @inline_len or the Call chunk length. + * + * Rejects frames where a Read chunk's inline-body offset exceeds + * the bound, where adjacent Read chunks overlap, or where any + * single chunk length exceeds the page budget. + * + * Return values: + * %true: Read chunk positions and lengths are valid + * %false: Malformed chunk list detected + */ +bool pcl_check_read_chunk_positions(struct svc_rdma_recv_ctxt *rctxt, + unsigned int inline_len) +{ + unsigned int max_len, bound, total_read; + struct svc_rdma_chunk *chunk, *next; + + max_len = rctxt->rc_maxpages << PAGE_SHIFT; + + if (!pcl_is_empty(&rctxt->rc_call_pcl)) { + chunk = pcl_first_chunk(&rctxt->rc_call_pcl); + if (chunk->ch_length > max_len) + return false; + bound = chunk->ch_length; + } else { + bound = inline_len; + } + + if (pcl_is_empty(&rctxt->rc_read_pcl)) + return true; + + total_read = 0; + pcl_for_each_chunk(chunk, &rctxt->rc_read_pcl) { + if (chunk->ch_position - total_read > bound) + return false; + if (chunk->ch_length > max_len) + return false; + + next = pcl_next_chunk(&rctxt->rc_read_pcl, chunk); + if (!next) + break; + + if (chunk->ch_position + chunk->ch_length > next->ch_position) + return false; + total_read += chunk->ch_length; } + return true; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index fe9bf0371b6e..fdfed1be97da 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -440,11 +440,14 @@ static void svc_rdma_build_arg_xdr(struct svc_rqst *rqstp, * to the first byte past the Read list. rc_read_pcl and * rc_call_pcl cl_count fields are set to the number of * Read segments in the list. - * %false: Read list is corrupt. @rctxt's xdr_stream is left in an - * unknown state. + * %false: Read list is corrupt or exceeds the page budget. @rctxt's + * xdr_stream is left in an unknown state. */ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) { + unsigned int maxlen = rctxt->rc_maxpages << PAGE_SHIFT; + unsigned int total_len = 0; + rctxt->rc_call_pcl.cl_count = 0; rctxt->rc_read_pcl.cl_count = 0; while (xdr_item_is_present(p)) { @@ -458,6 +461,11 @@ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) xdr_decode_read_segment(p, &position, &handle, &length, &offset); + if (length > maxlen) + return false; + total_len += length; + if (PAGE_ALIGN(total_len) > maxlen) + return false; if (position) { if (position & 3) return false; @@ -508,10 +516,13 @@ static bool xdr_check_write_chunk(struct svc_rdma_recv_ctxt *rctxt) return false; /* Before trusting the segcount value enough to use it in - * a computation, perform a simple range check. This is an - * arbitrary but sensible limit (ie, not architectural). + * a computation, perform a simple range check. A zero + * segcount describes no remote buffer and is rejected so + * downstream consumers never see a degenerate ch_segcount==0 + * chunk. The upper bound is an arbitrary but sensible limit + * (ie, not architectural). */ - if (unlikely(segcount > rctxt->rc_maxpages)) + if (segcount == 0 || unlikely(segcount > rctxt->rc_maxpages)) return false; p = xdr_inline_decode(&rctxt->rc_stream, @@ -719,6 +730,9 @@ static int svc_rdma_xdr_decode_req(struct xdr_buf *rq_arg, rq_arg->head[0].iov_base = rctxt->rc_stream.p; hdr_len = xdr_stream_pos(&rctxt->rc_stream); + if (!pcl_check_read_chunk_positions(rctxt, + rq_arg->head[0].iov_len - hdr_len)) + goto out_inval; rq_arg->head[0].iov_len -= hdr_len; rq_arg->len -= hdr_len; trace_svcrdma_decode_rqst(rctxt, rdma_argp, hdr_len); diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index f7fd22cc4a59..9aaaade99e6e 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -795,7 +795,7 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, len -= seg_len; if (len && ((head->rc_curpage + 1) > rqstp->rq_maxpages)) - goto out_overrun; + goto out_put; } ret = svc_rdma_rw_ctx_init(rdma, ctxt, segment->rs_offset, @@ -809,7 +809,8 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, cc->cc_sqecount += ret; return 0; -out_overrun: +out_put: + svc_rdma_put_rw_ctxt(rdma, ctxt); trace_svcrdma_page_overrun_err(&cc->cc_cid, head->rc_curpage); return -EINVAL; } @@ -847,7 +848,7 @@ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp, * svc_rdma_copy_inline_range - Copy part of the inline content into pages * @rqstp: RPC transaction context * @head: context for ongoing I/O - * @offset: offset into the Receive buffer of region to copy + * @offset: offset into the inline content of region to copy * @remaining: length of region to copy * * Take a page at a time from rqstp->rq_pages and copy the inline @@ -864,9 +865,13 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, unsigned int offset, unsigned int remaining) { - unsigned char *dst, *src = head->rc_recv_buf; + unsigned char *dst, *src = head->rc_saved_arg.head[0].iov_base; + unsigned int inline_len = head->rc_saved_arg.head[0].iov_len; unsigned int page_no, numpages; + if (offset > inline_len || remaining > inline_len - offset) + return -EINVAL; + numpages = PAGE_ALIGN(head->rc_pageoff + remaining) >> PAGE_SHIFT; for (page_no = 0; page_no < numpages; page_no++) { unsigned int page_len; @@ -917,9 +922,10 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, { const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int inline_len, start, length; int ret; + inline_len = head->rc_saved_arg.head[0].iov_len; start = 0; chunk = pcl_first_chunk(pcl); length = chunk->ch_position; @@ -937,6 +943,8 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, break; start += length; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_copy_inline_range(rqstp, head, start, length); if (ret < 0) @@ -944,7 +952,9 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, } start += length; - length = head->rc_byte_len - start; + if (start > inline_len) + return -EINVAL; + length = inline_len - start; return svc_rdma_copy_inline_range(rqstp, head, start, length); } @@ -969,8 +979,12 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, static int svc_rdma_read_data_item(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { - return svc_rdma_build_read_chunk(rqstp, head, - pcl_first_chunk(&head->rc_read_pcl)); + struct svc_rdma_chunk *chunk = pcl_first_chunk(&head->rc_read_pcl); + + if (chunk->ch_position > head->rc_saved_arg.head[0].iov_len) + return -EINVAL; + + return svc_rdma_build_read_chunk(rqstp, head, chunk); } /** @@ -996,17 +1010,20 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, const struct svc_rdma_segment *segment; int ret; + if (!length) + return 0; + ret = -EINVAL; pcl_for_each_segment(segment, chunk) { struct svc_rdma_segment dummy; - if (offset > segment->rs_length) { + if (offset >= segment->rs_length) { offset -= segment->rs_length; continue; } dummy.rs_handle = segment->rs_handle; - dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; + dummy.rs_length = min_t(u32, length, segment->rs_length - offset); dummy.rs_offset = segment->rs_offset + offset; ret = svc_rdma_build_read_segment(rqstp, head, &dummy); @@ -1015,6 +1032,8 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, head->rc_readbytes += dummy.rs_length; length -= dummy.rs_length; + if (!length) + break; offset = 0; } return ret; @@ -1039,14 +1058,17 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, pcl_first_chunk(&head->rc_call_pcl); const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int call_len, start, length; int ret; if (pcl_is_empty(pcl)) return svc_rdma_build_read_chunk(rqstp, head, call_chunk); + call_len = call_chunk->ch_length; start = 0; chunk = pcl_first_chunk(pcl); + if (chunk->ch_position > call_len) + return -EINVAL; length = chunk->ch_position; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1063,6 +1085,10 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, break; start += length; + if (next->ch_position > call_len) + return -EINVAL; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1071,7 +1097,9 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, } start += length; - length = call_chunk->ch_length - start; + if (start > call_len) + return -EINVAL; + length = call_len - start; return svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); } diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 7f6d17bf8c1f..c09659b17351 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -825,20 +825,21 @@ static int svc_rdma_xb_count_sges(const struct xdr_buf *xdr, } /** - * svc_rdma_pull_up_needed - Determine whether to use pull-up + * svc_rdma_check_pull_up - Determine whether to use pull-up * @rdma: controlling transport * @sctxt: send_ctxt for the Send WR * @write_pcl: Write chunk list provided by client * @xdr: xdr_buf containing RPC message to transmit * * Returns: - * %true if pull-up must be used - * %false otherwise + * %1 if pull-up must be used + * %0 if pull-up is not needed + * %-E2BIG if the reply is too large to be pulled up */ -static bool svc_rdma_pull_up_needed(const struct svcxprt_rdma *rdma, - const struct svc_rdma_send_ctxt *sctxt, - const struct svc_rdma_pcl *write_pcl, - const struct xdr_buf *xdr) +static int svc_rdma_check_pull_up(const struct svcxprt_rdma *rdma, + const struct svc_rdma_send_ctxt *sctxt, + const struct svc_rdma_pcl *write_pcl, + const struct xdr_buf *xdr) { /* Resources needed for the transport header */ struct svc_rdma_pullup_data args = { @@ -850,11 +851,22 @@ static bool svc_rdma_pull_up_needed(const struct svcxprt_rdma *rdma, ret = pcl_process_nonpayloads(write_pcl, xdr, svc_rdma_xb_count_sges, &args); if (ret < 0) - return false; + return 0; if (args.pd_length < RPCRDMA_PULLUP_THRESH) - return true; - return args.pd_num_sges >= rdma->sc_max_send_sges; + return 1; + if (args.pd_num_sges < rdma->sc_max_send_sges) + return 0; + + /* + * The reply has too many SGEs to Send inline, so it has to be + * linearized into sc_xprt_buf. That buffer holds only + * sc_max_req_size bytes, so a larger reply cannot be pulled up. + * RFC 8166 Section 4.5.3 requires responding with ERR_CHUNK. + */ + if (args.pd_length > rdma->sc_max_req_size) + return -E2BIG; + return 1; } /** @@ -910,7 +922,7 @@ static int svc_rdma_xb_linearize(const struct xdr_buf *xdr, * Assemble the elements of @xdr into the transport header buffer. * * Assumptions: - * pull_up_needed has determined that @xdr will fit in the buffer. + * check_pull_up has determined that @xdr will fit in the buffer. * * Returns: * %0 if pull-up was successful @@ -945,6 +957,7 @@ static int svc_rdma_pull_up_reply_msg(const struct svcxprt_rdma *rdma, * * Returns: * %0 if DMA mapping was successful. + * %-E2BIG if the reply is too large to be pulled up * %-EMSGSIZE if a buffer manipulation problem occurred * %-EIO if DMA mapping failed * @@ -960,6 +973,7 @@ int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, .md_rdma = rdma, .md_ctxt = sctxt, }; + int ret; /* Set up the (persistently-mapped) transport header SGE. */ sctxt->sc_send_wr.num_sge = 1; @@ -974,7 +988,10 @@ int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, /* For pull-up, svc_rdma_send() will sync the transport header. * No additional DMA mapping is necessary. */ - if (svc_rdma_pull_up_needed(rdma, sctxt, write_pcl, xdr)) + ret = svc_rdma_check_pull_up(rdma, sctxt, write_pcl, xdr); + if (ret < 0) + return ret; + if (ret) return svc_rdma_pull_up_reply_msg(rdma, sctxt, write_pcl, xdr); return pcl_process_nonpayloads(write_pcl, xdr, @@ -1162,7 +1179,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) &rctxt->rc_reply_pcl, sctxt, &rqstp->rq_res); if (ret < 0) - goto reply_chunk; + goto send_err; rc_size = ret; } @@ -1183,10 +1200,10 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) ret = svc_rdma_send_reply_msg(rdma, sctxt, rctxt, rqstp); if (ret < 0) - goto put_ctxt; + goto send_err; return 0; -reply_chunk: +send_err: if (ret != -E2BIG && ret != -EINVAL) goto put_ctxt; diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 7ca71741106b..093371f9d245 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -43,6 +43,7 @@ */ #include <linux/interrupt.h> +#include <linux/module.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/spinlock.h> @@ -245,12 +246,16 @@ svc_rdma_parse_connect_private(struct svcxprt_rdma *newxprt, * structure for the listening endpoint. * * This function creates a new xprt for the new connection and enqueues it on - * the accept queue for the listent xprt. When the listen thread is kicked, it + * the accept queue for the listen xprt. When the listen thread is kicked, it * will call the recvfrom method on the listen xprt which will accept the new * connection. + * + * Return values: + * %0: Do not destroy @new_cma_id + * %1: Destroy @new_cma_id (allocation failure) */ -static void handle_connect_req(struct rdma_cm_id *new_cma_id, - struct rdma_conn_param *param) +static int handle_connect_req(struct rdma_cm_id *new_cma_id, + struct rdma_conn_param *param) { struct svcxprt_rdma *listen_xprt = new_cma_id->context; struct svcxprt_rdma *newxprt; @@ -260,7 +265,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, listen_xprt->sc_xprt.xpt_net, ibdev_to_node(new_cma_id->device)); if (!newxprt) - return; + return 1; newxprt->sc_cm_id = new_cma_id; new_cma_id->context = newxprt; svc_rdma_parse_connect_private(newxprt, param); @@ -294,6 +299,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, set_bit(XPT_CONN, &listen_xprt->sc_xprt.xpt_flags); svc_xprt_enqueue(&listen_xprt->sc_xprt); + return 0; } /** @@ -317,14 +323,14 @@ static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id, switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: - handle_connect_req(cma_id, &event->param.conn); - break; + return handle_connect_req(cma_id, &event->param.conn); case RDMA_CM_EVENT_ADDR_CHANGE: listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net, sap, cma_xprt); if (IS_ERR(listen_id)) { pr_err("Listener dead, address change failed for device %s\n", cma_id->device->name); + cma_xprt->sc_cm_id = NULL; } else cma_xprt->sc_cm_id = listen_id; return 1; @@ -387,7 +393,13 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, listen_id = svc_rdma_create_listen_id(net, sa, cma_xprt); if (IS_ERR(listen_id)) { - kfree(cma_xprt); + /* _svc_xprt_create() acquired one module reference and + * puts it on xpo_create failure. svc_xprt_free() puts + * a second one when the kref drops to zero. Take a + * compensating reference so both puts are balanced. + */ + __module_get(cma_xprt->sc_xprt.xpt_class->xcl_owner); + svc_xprt_put(&cma_xprt->sc_xprt); return ERR_CAST(listen_id); } cma_xprt->sc_cm_id = listen_id; @@ -598,13 +610,26 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) return &newxprt->sc_xprt; errout: - /* Take a reference in case the DTO handler runs */ - svc_xprt_get(&newxprt->sc_xprt); - if (newxprt->sc_qp && !IS_ERR(newxprt->sc_qp)) - ib_destroy_qp(newxprt->sc_qp); - rdma_destroy_id(newxprt->sc_cm_id); - rpcrdma_rn_unregister(dev, &newxprt->sc_rn); - /* This call to put will destroy the transport */ + /* + * Drop the kref_init birth reference. svc_xprt_free will + * dispatch xpo_free = svc_rdma_free, which tears down sc_qp, + * sc_sq_cq, sc_rq_cq, and sc_pd under existing IS_ERR/NULL + * guards, and sc_rn under the rn_done sentinel guard inside + * rpcrdma_rn_unregister. + * + * sc_cm_id is destroyed unconditionally by svc_rdma_free; that + * is safe here because sc_cm_id is non-NULL by caller invariant + * on every path that reaches this errout: handle_connect_req + * installs newxprt->sc_cm_id before queueing the new xprt for + * accept, and svc_rdma_accept has already dereferenced it above + * the first goto errout. + * + * svc_handle_xprt() drops its pre-acquired module reference when + * ->xpo_accept() returns NULL. Take a replacement reference before + * freeing @newxprt, because svc_xprt_free() drops the module + * reference associated with @newxprt. + */ + __module_get(newxprt->sc_xprt.xpt_class->xcl_owner); svc_xprt_put(&newxprt->sc_xprt); return NULL; } @@ -614,7 +639,8 @@ static void svc_rdma_detach(struct svc_xprt *xprt) struct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); - rdma_disconnect(rdma->sc_cm_id); + if (rdma->sc_cm_id) + rdma_disconnect(rdma->sc_cm_id); /* * Most close paths go through svc_rdma_xprt_deferred_close(), @@ -634,10 +660,15 @@ static void svc_rdma_free(struct svc_xprt *xprt) { struct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); - struct ib_device *device = rdma->sc_cm_id->device; + struct ib_device *device; might_sleep(); + if (!rdma->sc_cm_id) + goto out_free; + + device = rdma->sc_cm_id->device; + /* This blocks until the Completion Queues are empty */ if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) ib_drain_qp(rdma->sc_qp); @@ -662,11 +693,13 @@ static void svc_rdma_free(struct svc_xprt *xprt) if (rdma->sc_pd && !IS_ERR(rdma->sc_pd)) ib_dealloc_pd(rdma->sc_pd); + if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags)) + rpcrdma_rn_unregister(device, &rdma->sc_rn); + /* Destroy the CM ID */ rdma_destroy_id(rdma->sc_cm_id); - if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags)) - rpcrdma_rn_unregister(device, &rdma->sc_rn); +out_free: kfree(rdma); } diff --git a/tools/net/sunrpc/xdrgen/generators/program.py b/tools/net/sunrpc/xdrgen/generators/program.py index c0cb3f6d3319..37f9655c83fe 100644 --- a/tools/net/sunrpc/xdrgen/generators/program.py +++ b/tools/net/sunrpc/xdrgen/generators/program.py @@ -38,6 +38,8 @@ def emit_version_declarations( arguments = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: + if procedure.argument.type_name == "void": + continue arguments[procedure.argument.type_name] = None if len(arguments) > 0: print("") @@ -48,6 +50,8 @@ def emit_version_declarations( results = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: + if procedure.result.type_name == "void": + continue results[procedure.result.type_name] = None if len(results) > 0: print("") @@ -63,6 +67,8 @@ def emit_version_argument_decoders( arguments = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: + if procedure.argument.type_name == "void": + continue arguments[procedure.argument.type_name] = None template = environment.get_template("decoder/argument.j2") @@ -105,6 +111,8 @@ def emit_version_result_encoders( results = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: + if procedure.result.type_name == "void": + continue results[procedure.result.type_name] = None template = environment.get_template("encoder/result.j2") diff --git a/tools/net/sunrpc/xdrgen/subcmds/declarations.py b/tools/net/sunrpc/xdrgen/subcmds/declarations.py index ed83d48d1f68..f187611466d7 100644 --- a/tools/net/sunrpc/xdrgen/subcmds/declarations.py +++ b/tools/net/sunrpc/xdrgen/subcmds/declarations.py @@ -21,16 +21,15 @@ from generators.union import XdrUnionGenerator from xdr_ast import transform_parse_tree, _RpcProgram, Specification from xdr_ast import _XdrEnum, _XdrPointer, _XdrTypedef, _XdrStruct, _XdrUnion +from xdr_ast import XdrSemanticError from xdr_parse import xdr_parser, set_xdr_annotate from xdr_parse import make_error_handler, XdrParseError -from xdr_parse import handle_transform_error +from xdr_parse import handle_transform_error, handle_semantic_error logger.setLevel(logging.INFO) -def emit_header_declarations( - root: Specification, language: str, peer: str -) -> None: +def emit_header_declarations(root: Specification, language: str, peer: str) -> None: """Emit header declarations""" for definition in root.definitions: if isinstance(definition.value, _XdrEnum): @@ -68,6 +67,9 @@ def subcmd(args: Namespace) -> int: except VisitError as e: handle_transform_error(e, source, args.filename) return 1 + except XdrSemanticError as e: + handle_semantic_error(e, source, args.filename) + return 1 gen = XdrHeaderTopGenerator(args.language, args.peer) gen.emit_declaration(args.filename, ast) diff --git a/tools/net/sunrpc/xdrgen/subcmds/definitions.py b/tools/net/sunrpc/xdrgen/subcmds/definitions.py index a48ca0549382..77b666943a11 100644 --- a/tools/net/sunrpc/xdrgen/subcmds/definitions.py +++ b/tools/net/sunrpc/xdrgen/subcmds/definitions.py @@ -21,12 +21,12 @@ from generators.typedef import XdrTypedefGenerator from generators.struct import XdrStructGenerator from generators.union import XdrUnionGenerator -from xdr_ast import transform_parse_tree, Specification +from xdr_ast import transform_parse_tree, Specification, XdrSemanticError from xdr_ast import _RpcProgram, _XdrConstant, _XdrEnum, _XdrPassthru, _XdrPointer from xdr_ast import _XdrTypedef, _XdrStruct, _XdrUnion from xdr_parse import xdr_parser, set_xdr_annotate from xdr_parse import make_error_handler, XdrParseError -from xdr_parse import handle_transform_error +from xdr_parse import handle_transform_error, handle_semantic_error logger.setLevel(logging.INFO) @@ -94,6 +94,9 @@ def subcmd(args: Namespace) -> int: except VisitError as e: handle_transform_error(e, source, args.filename) return 1 + except XdrSemanticError as e: + handle_semantic_error(e, source, args.filename) + return 1 gen = XdrHeaderTopGenerator(args.language, args.peer) gen.emit_definition(args.filename, ast) diff --git a/tools/net/sunrpc/xdrgen/subcmds/lint.py b/tools/net/sunrpc/xdrgen/subcmds/lint.py index e1da49632e62..b4ea0f55f079 100644 --- a/tools/net/sunrpc/xdrgen/subcmds/lint.py +++ b/tools/net/sunrpc/xdrgen/subcmds/lint.py @@ -11,8 +11,8 @@ from lark import logger from lark.exceptions import VisitError from xdr_parse import xdr_parser, make_error_handler, XdrParseError -from xdr_parse import handle_transform_error -from xdr_ast import transform_parse_tree +from xdr_parse import handle_transform_error, handle_semantic_error +from xdr_ast import transform_parse_tree, XdrSemanticError logger.setLevel(logging.DEBUG) @@ -34,5 +34,8 @@ def subcmd(args: Namespace) -> int: except VisitError as e: handle_transform_error(e, source, args.filename) return 1 + except XdrSemanticError as e: + handle_semantic_error(e, source, args.filename) + return 1 return 0 diff --git a/tools/net/sunrpc/xdrgen/subcmds/source.py b/tools/net/sunrpc/xdrgen/subcmds/source.py index 27e8767b1b58..56eba34d8eb3 100644 --- a/tools/net/sunrpc/xdrgen/subcmds/source.py +++ b/tools/net/sunrpc/xdrgen/subcmds/source.py @@ -21,11 +21,11 @@ from generators.union import XdrUnionGenerator from xdr_ast import transform_parse_tree, _RpcProgram, Specification from xdr_ast import _XdrAst, _XdrEnum, _XdrPassthru, _XdrPointer -from xdr_ast import _XdrStruct, _XdrTypedef, _XdrUnion +from xdr_ast import _XdrStruct, _XdrTypedef, _XdrUnion, XdrSemanticError from xdr_parse import xdr_parser, set_xdr_annotate, set_xdr_enum_validation from xdr_parse import make_error_handler, XdrParseError -from xdr_parse import handle_transform_error +from xdr_parse import handle_transform_error, handle_semantic_error logger.setLevel(logging.INFO) @@ -123,6 +123,9 @@ def subcmd(args: Namespace) -> int: except VisitError as e: handle_transform_error(e, source, args.filename) return 1 + except XdrSemanticError as e: + handle_semantic_error(e, source, args.filename) + return 1 match args.peer: case "server": generate_server_source(args.filename, ast, args.language) diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 index c7ae506076bb..d1405c7c5354 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 @@ -1,3 +1,4 @@ {# SPDX-License-Identifier: GPL-2.0 #} + bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr); bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, {{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 index cf65b71eaef3..7ddc2bf3edac 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 index 1d477c2d197a..5bf00070ae95 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 index 19b219dd276d..096d553b2a1e 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 @@ -11,9 +11,6 @@ */ bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_stream *xdr) { -{% if argument == 'void' %} - return xdrgen_decode_void(xdr); -{% else %} {% if argument in structs %} struct {{ argument }} *argp = rqstp->rq_argp; {% else %} @@ -21,5 +18,4 @@ bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_ {% endif %} return xdrgen_decode_{{ argument }}(xdr, argp); -{% endif %} } diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 index 746592cfda56..4243d91966fd 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 @@ -11,9 +11,6 @@ */ bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_stream *xdr) { -{% if result == 'void' %} - return xdrgen_encode_void(xdr); -{% else %} {% if result in structs %} struct {{ result }} *resp = rqstp->rq_resp; @@ -23,5 +20,4 @@ bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_st return xdrgen_encode_{{ result }}(xdr, *resp); {% endif %} -{% endif %} } diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 index cf65b71eaef3..7ddc2bf3edac 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 index 1d477c2d197a..5bf00070ae95 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 index 5fc1937ba774..19ee759d70c6 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 @@ -1,9 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} } u; }; -{%- if name in public_apis %} - - -bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); -bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *ptr); -{%- endif -%} diff --git a/tools/net/sunrpc/xdrgen/tests/bad-procedure-number-negative.x b/tools/net/sunrpc/xdrgen/tests/bad-procedure-number-negative.x new file mode 100644 index 000000000000..33ed272c3ce2 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/tests/bad-procedure-number-negative.x @@ -0,0 +1,20 @@ +/* + * NEGATIVE TEST CASE -- xdrgen must REJECT this specification. + * + * RFC 5531 assigns only unsigned constants to program, version, and + * procedure numbers (Section 12.3). This spec gives a procedure a + * negative number, which the front end must reject. + * + * Expected diagnostic: + * negative procedure number -5 in version 'BADVERS' + * + * The tests directory has no automated runner; exercise by hand: + * ./xdrgen definitions tests/bad-procedure-number-negative.x (must fail) + */ + +program BADPROG { + version BADVERS { + void BADPROC_NULL(void) = 0; + void BADPROC_FOO(void) = -5; + } = 1; +} = 100000; diff --git a/tools/net/sunrpc/xdrgen/tests/bad-procedure-number-too-large.x b/tools/net/sunrpc/xdrgen/tests/bad-procedure-number-too-large.x new file mode 100644 index 000000000000..521581c57358 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/tests/bad-procedure-number-too-large.x @@ -0,0 +1,20 @@ +/* + * NEGATIVE TEST CASE -- xdrgen must REJECT this specification. + * + * RFC 5531 encodes program, version, and procedure numbers as unsigned + * 32-bit integers (Section 9). This spec gives a procedure a number one + * past the 32-bit maximum, which the front end must reject. + * + * Expected diagnostic: + * procedure number 4294967296 in version 'BADVERS' exceeds 4294967295 + * + * The tests directory has no automated runner; exercise by hand: + * ./xdrgen definitions tests/bad-procedure-number-too-large.x (must fail) + */ + +program BADPROG { + version BADVERS { + void BADPROC_NULL(void) = 0; + void BADPROC_FOO(void) = 4294967296; + } = 1; +} = 100000; diff --git a/tools/net/sunrpc/xdrgen/tests/bad-program-number-negative.x b/tools/net/sunrpc/xdrgen/tests/bad-program-number-negative.x new file mode 100644 index 000000000000..f7b71ee07f6c --- /dev/null +++ b/tools/net/sunrpc/xdrgen/tests/bad-program-number-negative.x @@ -0,0 +1,19 @@ +/* + * NEGATIVE TEST CASE -- xdrgen must REJECT this specification. + * + * RFC 5531 assigns only unsigned constants to program, version, and + * procedure numbers (Section 12.3). This spec gives the program a + * negative number, which the front end must reject. + * + * Expected diagnostic: + * negative program number -100000 in program 'BADPROG' + * + * The tests directory has no automated runner; exercise by hand: + * ./xdrgen definitions tests/bad-program-number-negative.x (must fail) + */ + +program BADPROG { + version BADVERS { + void BADPROC_NULL(void) = 0; + } = 1; +} = -100000; diff --git a/tools/net/sunrpc/xdrgen/tests/bad-program-number-too-large.x b/tools/net/sunrpc/xdrgen/tests/bad-program-number-too-large.x new file mode 100644 index 000000000000..c761584e712f --- /dev/null +++ b/tools/net/sunrpc/xdrgen/tests/bad-program-number-too-large.x @@ -0,0 +1,19 @@ +/* + * NEGATIVE TEST CASE -- xdrgen must REJECT this specification. + * + * RFC 5531 encodes program, version, and procedure numbers as unsigned + * 32-bit integers (Section 9). This spec gives the program a number one + * past the 32-bit maximum, which the front end must reject. + * + * Expected diagnostic: + * program number 4294967296 in program 'BADPROG' exceeds 4294967295 + * + * The tests directory has no automated runner; exercise by hand: + * ./xdrgen definitions tests/bad-program-number-too-large.x (must fail) + */ + +program BADPROG { + version BADVERS { + void BADPROC_NULL(void) = 0; + } = 1; +} = 4294967296; diff --git a/tools/net/sunrpc/xdrgen/tests/bad-version-number-negative.x b/tools/net/sunrpc/xdrgen/tests/bad-version-number-negative.x new file mode 100644 index 000000000000..dd9c773435c0 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/tests/bad-version-number-negative.x @@ -0,0 +1,19 @@ +/* + * NEGATIVE TEST CASE -- xdrgen must REJECT this specification. + * + * RFC 5531 assigns only unsigned constants to program, version, and + * procedure numbers (Section 12.3). This spec gives the version a + * negative number, which the front end must reject. + * + * Expected diagnostic: + * negative version number -1 in program 'BADPROG' + * + * The tests directory has no automated runner; exercise by hand: + * ./xdrgen definitions tests/bad-version-number-negative.x (must fail) + */ + +program BADPROG { + version BADVERS { + void BADPROC_NULL(void) = 0; + } = -1; +} = 100000; diff --git a/tools/net/sunrpc/xdrgen/tests/bad-version-number-too-large.x b/tools/net/sunrpc/xdrgen/tests/bad-version-number-too-large.x new file mode 100644 index 000000000000..dd44f6eed564 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/tests/bad-version-number-too-large.x @@ -0,0 +1,19 @@ +/* + * NEGATIVE TEST CASE -- xdrgen must REJECT this specification. + * + * RFC 5531 encodes program, version, and procedure numbers as unsigned + * 32-bit integers (Section 9). This spec gives the version a number one + * past the 32-bit maximum, which the front end must reject. + * + * Expected diagnostic: + * version number 4294967296 in program 'BADPROG' exceeds 4294967295 + * + * The tests directory has no automated runner; exercise by hand: + * ./xdrgen definitions tests/bad-version-number-too-large.x (must fail) + */ + +program BADPROG { + version BADVERS { + void BADPROC_NULL(void) = 0; + } = 4294967296; +} = 100000; diff --git a/tools/net/sunrpc/xdrgen/xdr_ast.py b/tools/net/sunrpc/xdrgen/xdr_ast.py index 14bff9477473..9dab8bc545b0 100644 --- a/tools/net/sunrpc/xdrgen/xdr_ast.py +++ b/tools/net/sunrpc/xdrgen/xdr_ast.py @@ -5,7 +5,7 @@ import sys from typing import List -from dataclasses import dataclass +from dataclasses import dataclass, KW_ONLY from lark import ast_utils, Transformer from lark.tree import Meta @@ -65,6 +65,16 @@ max_widths = { class _XdrAst(ast_utils.Ast): """Base class for the XDR abstract syntax tree""" + # Source position of the construct's declared identifier, when + # the transformer records one, so semantic diagnostics can point + # at the exact declaration. The KW_ONLY marker makes the fields + # keyword-only, so they never disturb the positional child + # ordering lark uses to build each node; 0 means the position was + # not recorded. + _: KW_ONLY + line: int = 0 + column: int = 0 + @dataclass class _XdrIdentifier(_XdrAst): @@ -488,7 +498,7 @@ class _RpcProcedure(_XdrAst): """RPC procedure definition""" name: str - number: str + number: int argument: _XdrTypeSpecifier result: _XdrTypeSpecifier @@ -498,7 +508,7 @@ class _RpcVersion(_XdrAst): """RPC version definition""" name: str - number: str + number: int procedures: List[_RpcProcedure] @@ -507,7 +517,7 @@ class _RpcProgram(_XdrAst): """RPC program definition""" name: str - number: str + number: int versions: List[_RpcVersion] @@ -543,7 +553,8 @@ class ParseToAst(Transformer): def identifier(self, children): """Instantiate one _XdrIdentifier object""" - return _XdrIdentifier(children[0].value) + token = children[0] + return _XdrIdentifier(token.value, line=token.line, column=token.column) def value(self, children): """Instantiate one _XdrValue object""" @@ -573,84 +584,103 @@ class ParseToAst(Transformer): def constant_def(self, children): """Instantiate one _XdrConstant object""" - name = children[0].symbol + ident = children[0] value = children[1].value - return _XdrConstant(name, value) + return _XdrConstant(ident.symbol, value, line=ident.line, column=ident.column) def enum(self, children): """Instantiate one _XdrEnum object""" - enum_name = children[0].symbol + name_ident = children[0] i = 0 enumerators = [] body = children[1] while i < len(body.children): - name = body.children[i].symbol + ident = body.children[i] value = body.children[i + 1].value - enumerators.append(_XdrEnumerator(name, value)) + enumerators.append( + _XdrEnumerator( + ident.symbol, value, line=ident.line, column=ident.column + ) + ) i = i + 2 - return _XdrEnum(enum_name, enumerators) + return _XdrEnum( + name_ident.symbol, + enumerators, + line=name_ident.line, + column=name_ident.column, + ) def fixed_length_opaque(self, children): """Instantiate one _XdrFixedLengthOpaque declaration object""" - name = children[0].symbol + ident = children[0] size = children[1].value - return _XdrFixedLengthOpaque(name, size) + return _XdrFixedLengthOpaque( + ident.symbol, size, line=ident.line, column=ident.column + ) def variable_length_opaque(self, children): """Instantiate one _XdrVariableLengthOpaque declaration object""" - name = children[0].symbol + ident = children[0] if children[1] is not None: maxsize = children[1].value else: maxsize = "0" - return _XdrVariableLengthOpaque(name, maxsize) + return _XdrVariableLengthOpaque( + ident.symbol, maxsize, line=ident.line, column=ident.column + ) def string(self, children): """Instantiate one _XdrString declaration object""" - name = children[0].symbol + ident = children[0] if children[1] is not None: maxsize = children[1].value else: maxsize = "0" - return _XdrString(name, maxsize) + return _XdrString(ident.symbol, maxsize, line=ident.line, column=ident.column) def fixed_length_array(self, children): """Instantiate one _XdrFixedLengthArray declaration object""" spec = children[0] - name = children[1].symbol + ident = children[1] size = children[2].value - return _XdrFixedLengthArray(name, spec, size) + return _XdrFixedLengthArray( + ident.symbol, spec, size, line=ident.line, column=ident.column + ) def variable_length_array(self, children): """Instantiate one _XdrVariableLengthArray declaration object""" spec = children[0] - name = children[1].symbol + ident = children[1] if children[2] is not None: maxsize = children[2].value else: maxsize = "0" - return _XdrVariableLengthArray(name, spec, maxsize) + return _XdrVariableLengthArray( + ident.symbol, spec, maxsize, line=ident.line, column=ident.column + ) def optional_data(self, children): """Instantiate one _XdrOptionalData declaration object""" spec = children[0] - name = children[1].symbol + ident = children[1] - return _XdrOptionalData(name, spec) + return _XdrOptionalData( + ident.symbol, spec, line=ident.line, column=ident.column + ) def basic(self, children): """Instantiate one _XdrBasic object""" spec = children[0] - name = children[1].symbol + ident = children[1] - return _XdrBasic(name, spec) + return _XdrBasic(ident.symbol, spec, line=ident.line, column=ident.column) def void(self, children): """Instantiate one _XdrVoid declaration object""" @@ -659,17 +689,19 @@ class ParseToAst(Transformer): def struct(self, children): """Instantiate one _XdrStruct object""" - name = children[0].symbol + ident = children[0] + name = ident.symbol fields = children[1].children + pos = {"line": ident.line, "column": ident.column} last_field = fields[-1] if ( isinstance(last_field, _XdrOptionalData) and name == last_field.spec.type_name ): - return _XdrPointer(name, fields) + return _XdrPointer(name, fields, **pos) - return _XdrStruct(name, fields) + return _XdrStruct(name, fields, **pos) def typedef(self, children): """Instantiate one _XdrTypedef object""" @@ -694,39 +726,57 @@ class ParseToAst(Transformer): def union(self, children): """Instantiate one _XdrUnion object""" - name = children[0].symbol + ident = children[0] body = children[1] discriminant = body.children[0].children[0] cases = body.children[1:-1] default = body.children[-1] - return _XdrUnion(name, discriminant, cases, default) + return _XdrUnion( + ident.symbol, + discriminant, + cases, + default, + line=ident.line, + column=ident.column, + ) def procedure_def(self, children): """Instantiate one _RpcProcedure object""" result = children[0] - name = children[1].symbol + ident = children[1] argument = children[2] number = children[3].value - return _RpcProcedure(name, number, argument, result) + return _RpcProcedure( + ident.symbol, + number, + argument, + result, + line=ident.line, + column=ident.column, + ) def version_def(self, children): """Instantiate one _RpcVersion object""" - name = children[0].symbol + ident = children[0] number = children[-1].value procedures = children[1:-1] - return _RpcVersion(name, number, procedures) + return _RpcVersion( + ident.symbol, number, procedures, line=ident.line, column=ident.column + ) def program_def(self, children): """Instantiate one _RpcProgram object""" - name = children[0].symbol + ident = children[0] number = children[-1].value versions = children[1:-1] - return _RpcProgram(name, number, versions) + return _RpcProgram( + ident.symbol, number, versions, line=ident.line, column=ident.column + ) def pragma_def(self, children): """Instantiate one _Pragma object""" @@ -764,7 +814,9 @@ def _merge_consecutive_passthru(definitions: List[Definition]) -> List[Definitio lines = [definitions[i].value.content] meta = definitions[i].meta j = i + 1 - while j < len(definitions) and isinstance(definitions[j].value, _XdrPassthru): + while j < len(definitions) and isinstance( + definitions[j].value, _XdrPassthru + ): lines.append(definitions[j].value.content) j += 1 merged = _XdrPassthru("\n".join(lines)) @@ -776,10 +828,174 @@ def _merge_consecutive_passthru(definitions: List[Definition]) -> List[Definitio return result +def _meta_line(meta) -> int: + """Return the 1-based source line for a node's meta, or 0 if unknown""" + try: + return meta.line + except AttributeError: + return 0 + + +class XdrSemanticError(Exception): + """A specification that parses but violates an XDR semantic rule. + + Detection lives in the language-independent front end because a + duplicate name is malformed XDR regardless of the output language. + """ + + def __init__(self, message: str, meta): + super().__init__(message) + self.message = message + self.line = _meta_line(meta) + self.column = getattr(meta, "column", 0) + + +def _introduced_names(value): + """Yield (name, node) for each identifier a definition introduces.""" + if isinstance(value, (_XdrStruct, _XdrUnion, _XdrPointer)): + yield value.name, value + elif isinstance(value, _XdrEnum): + yield value.name, value + for enumerator in value.enumerators: + yield enumerator.name, enumerator + elif isinstance(value, _XdrTypedef): + yield value.declaration.name, value.declaration + elif isinstance(value, _XdrConstant): + yield value.name, value + elif isinstance(value, _RpcProgram): + yield value.name, value + + +def _check_rpc_scope_names(program: "_RpcProgram") -> None: + """Enforce RFC 5531 Section 12.3 scoping within an RPC program. + + A version name and number are unique within the program and a + procedure name and number are unique within its version. + """ + version_names = set() + version_numbers = set() + for version in program.versions: + if version.name in version_names: + raise XdrSemanticError( + f"duplicate version name '{version.name}'" + f" in program '{program.name}'", + version, + ) + version_names.add(version.name) + if version.number in version_numbers: + raise XdrSemanticError( + f"duplicate version number {version.number}" + f" in program '{program.name}'", + version, + ) + version_numbers.add(version.number) + procedure_names = set() + procedure_numbers = set() + for procedure in version.procedures: + if procedure.name in procedure_names: + raise XdrSemanticError( + f"duplicate procedure name '{procedure.name}'" + f" in version '{version.name}'", + procedure, + ) + procedure_names.add(procedure.name) + if procedure.number in procedure_numbers: + raise XdrSemanticError( + f"duplicate procedure number {procedure.number}" + f" in version '{version.name}'", + procedure, + ) + procedure_numbers.add(procedure.number) + + +def check_duplicate_definitions(root: "Specification") -> None: + """Reject a spec that declares an identifier more than once. + + RFC 4506 Section 6.4 places constant and type identifiers in a + single name space that must be unique within a specification. + RFC 5531 Section 12.3 adds RPC program names to that name space + and scopes version names and numbers to their program and + procedure names and numbers to their version. + """ + seen = {} + for definition in root.definitions: + for name, node in _introduced_names(definition.value): + where = node if node.line else definition.meta + first = seen.get(name) + if first is not None: + raise XdrSemanticError( + f"duplicate identifier '{name}'" + f" (first declared at line {_meta_line(first)})", + where, + ) + seen[name] = where + if isinstance(definition.value, _RpcProgram): + _check_rpc_scope_names(definition.value) + + +# RFC 5531 (Section 9) encodes program, version, and procedure numbers +# as unsigned 32-bit integers, so each must fall within [0, 2**32 - 1]. +_RPC_NUMBER_MAX = 2**32 - 1 + + +def _check_rpc_number(kind: str, number: int, scope: str, meta) -> None: + """Reject one RPC number that is negative or wider than 32 bits.""" + if number < 0: + raise XdrSemanticError( + f"negative {kind} number {number} {scope}", + meta, + ) + if number > _RPC_NUMBER_MAX: + raise XdrSemanticError( + f"{kind} number {number} {scope} exceeds {_RPC_NUMBER_MAX}", + meta, + ) + + +def check_rpc_number_range(root: "Specification") -> None: + """Reject an out-of-range program, version, or procedure number. + + RFC 5531 assigns only unsigned constants to program, version, and + procedure numbers (Section 12.3) and encodes each as an unsigned + 32-bit integer (Section 9). RFC 4506 Section 6.2 permits a signed + decimal constant for XDR constants in general and sets no ceiling on + magnitude, so the grammar accepts an out-of-range value; the range + is enforced here instead. The parser retains no per-version or + per-procedure source location, so a violation is reported against the + program definition. + """ + for definition in root.definitions: + program = definition.value + if not isinstance(program, _RpcProgram): + continue + _check_rpc_number( + "program", + program.number, + f"in program '{program.name}'", + definition.meta, + ) + for version in program.versions: + _check_rpc_number( + "version", + version.number, + f"in program '{program.name}'", + definition.meta, + ) + for procedure in version.procedures: + _check_rpc_number( + "procedure", + procedure.number, + f"in version '{version.name}'", + definition.meta, + ) + + def transform_parse_tree(parse_tree): """Transform productions into an abstract syntax tree""" ast = transformer.transform(parse_tree) ast.definitions = _merge_consecutive_passthru(ast.definitions) + check_duplicate_definitions(ast) + check_rpc_number_range(ast) return ast diff --git a/tools/net/sunrpc/xdrgen/xdr_parse.py b/tools/net/sunrpc/xdrgen/xdr_parse.py index 241e96c1fdd9..78298553ee78 100644 --- a/tools/net/sunrpc/xdrgen/xdr_parse.py +++ b/tools/net/sunrpc/xdrgen/xdr_parse.py @@ -63,6 +63,22 @@ def get_xdr_enum_validation() -> bool: return enum_validation +def format_source_caret(line_text: str, column: int) -> list[str]: + """Render an offending source line with a caret beneath a column. + + Args: + line_text: The raw source line containing the error + column: 1-based column of the offending token within line_text + + Returns: + Output lines for the diagnostic: a blank separator, the source + line with tabs expanded, and a caret aligned under the column. + """ + expanded = line_text.expandtabs() + caret = len(line_text[: column - 1].expandtabs()) + return ["", f" {expanded}", f" {' ' * caret}^"] + + def make_error_handler(source: str, filename: str) -> Callable[[UnexpectedInput], bool]: """Create an error handler that reports the first parse error and aborts. @@ -110,10 +126,7 @@ def make_error_handler(source: str, filename: str) -> Callable[[UnexpectedInput] msg_parts.append(str(e).split("\n")[0]) # Show the offending line with a caret pointing to the error - msg_parts.append("") - msg_parts.append(f" {line_text}") - prefix = line_text[: column - 1].expandtabs() - msg_parts.append(f" {' ' * len(prefix)}^") + msg_parts.extend(format_source_caret(line_text, column)) sys.stderr.write("\n".join(msg_parts) + "\n") raise XdrParseError() @@ -151,10 +164,27 @@ def handle_transform_error(e: VisitError, source: str, filename: str) -> None: # Show the offending line with a caret pointing to the error if line_text: - msg_parts.append("") - msg_parts.append(f" {line_text}") - prefix = line_text[: column - 1].expandtabs() - msg_parts.append(f" {' ' * len(prefix)}^") + msg_parts.extend(format_source_caret(line_text, column)) + + sys.stderr.write("\n".join(msg_parts) + "\n") + + +def handle_semantic_error(e, source: str, filename: str) -> None: + """Report a semantic error (e.g., a duplicate name) with context. + + Args: + e: The XdrSemanticError carrying message and source position + source: The XDR source text being parsed + filename: The name of the file being parsed + """ + lines = source.splitlines() + line_num = getattr(e, "line", 0) + column = getattr(e, "column", 0) + line_text = lines[line_num - 1] if 0 < line_num <= len(lines) else "" + + msg_parts = [f"{filename}:{line_num}:{column}: semantic error", e.message] + if line_text: + msg_parts.extend(format_source_caret(line_text, column)) sys.stderr.write("\n".join(msg_parts) + "\n") diff --git a/tools/testing/nvdimm/test/ndtest.c b/tools/testing/nvdimm/test/ndtest.c index 8e3b6be53839..2051ad5d4882 100644 --- a/tools/testing/nvdimm/test/ndtest.c +++ b/tools/testing/nvdimm/test/ndtest.c @@ -376,8 +376,7 @@ static void *ndtest_alloc_resource(struct ndtest_priv *p, size_t size, buf_err: if (__dma && size >= DIMM_SIZE) gen_pool_free(ndtest_pool, __dma, size); - if (buf) - vfree(buf); + vfree(buf); kfree(res); return NULL; diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index f87e9f251d13..009fe107b0d7 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -1644,8 +1644,7 @@ static void *__test_alloc(struct nfit_test *t, size_t size, dma_addr_t *dma, err: if (*dma && size >= DIMM_SIZE) gen_pool_free(nfit_pool, *dma, size); - if (buf) - vfree(buf); + vfree(buf); kfree(nfit_res); return NULL; } |