diff options
Diffstat (limited to 'fs/ext4')
36 files changed, 3322 insertions, 1390 deletions
diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile index 72206a292676..3f9fc0eb8eca 100644 --- a/fs/ext4/Makefile +++ b/fs/ext4/Makefile @@ -14,7 +14,8 @@ ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \ ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o -ext4-inode-test-objs += inode-test.o -obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-inode-test.o +ext4-test-objs += inode-test.o mballoc-test.o \ + extents-test.o hash-test.o +obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-test.o ext4-$(CONFIG_FS_VERITY) += verity.o ext4-$(CONFIG_FS_ENCRYPTION) += crypto.o diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c index 8040c731b3e4..52f4c5169f91 100644 --- a/fs/ext4/balloc.c +++ b/fs/ext4/balloc.c @@ -331,9 +331,13 @@ struct ext4_group_info *ext4_get_group_info(struct super_block *sb, if (unlikely(group >= EXT4_SB(sb)->s_groups_count)) return NULL; + if (unlikely(!EXT4_SB(sb)->s_group_info)) + return NULL; indexv = group >> (EXT4_DESC_PER_BLOCK_BITS(sb)); indexh = group & ((EXT4_DESC_PER_BLOCK(sb)) - 1); grp_info = sbi_array_rcu_deref(EXT4_SB(sb), s_group_info, indexv); + if (unlikely(!grp_info)) + return NULL; return grp_info[indexh]; } diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c index cf0a0970c095..1a0fccb084ef 100644 --- a/fs/ext4/crypto.c +++ b/fs/ext4/crypto.c @@ -144,7 +144,13 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, if (inode->i_ino == EXT4_ROOT_INO) return -EPERM; - if (WARN_ON_ONCE(IS_DAX(inode) && i_size_read(inode))) + /* + * For new encrypted inodes, S_DAX is never set in the first place. + * + * For existing inodes, this is called only on empty directories. ext4 + * never sets S_DAX on directories. + */ + if (WARN_ON_ONCE(IS_DAX(inode))) return -EINVAL; if (ext4_test_inode_flag(inode, EXT4_INODE_DAX)) @@ -163,21 +169,25 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, */ if (handle) { - res = ext4_xattr_set_handle(handle, inode, - EXT4_XATTR_INDEX_ENCRYPTION, - EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, - ctx, len, 0); - if (!res) { - ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - ext4_clear_inode_state(inode, - EXT4_STATE_MAY_INLINE_DATA); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ - ext4_set_inode_flags(inode, false); - } - return res; + /* + * __ext4_new_inode() should have already set the encrypt flag + * on the inode and avoided enabling inline data. + */ + if (WARN_ON_ONCE(!IS_ENCRYPTED(inode))) + return -EINVAL; + if (WARN_ON_ONCE(ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))) + return -EINVAL; + /* + * Since the inode is new it is ok to pass the + * XATTR_CREATE flag. This is necessary to match the + * remaining journal credits check in the set_handle + * function with the credits allocated for the new + * inode. + */ + return ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, XATTR_CREATE); } res = dquot_initialize(inode); @@ -198,10 +208,7 @@ retry: ctx, len, 0); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ + /* Update inode->i_flags to set S_ENCRYPTED. */ ext4_set_inode_flags(inode, false); res = ext4_mark_inode_dirty(handle, inode); if (res) @@ -229,7 +236,7 @@ static bool ext4_has_stable_inodes(struct super_block *sb) const struct fscrypt_operations ext4_cryptops = { .inode_info_offs = (int)offsetof(struct ext4_inode_info, i_crypt_info) - (int)offsetof(struct ext4_inode_info, vfs_inode), - .needs_bounce_pages = 1, + .is_block_based = 1, .has_32bit_inodes = 1, .supports_subblock_data_units = 1, .legacy_key_prefix = "ext4:", diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c index 28b2a3deb954..8d7b81e6948e 100644 --- a/fs/ext4/dir.c +++ b/fs/ext4/dir.c @@ -138,6 +138,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) struct buffer_head *bh = NULL; struct fscrypt_str fstr = FSTR_INIT(NULL, 0); struct dir_private_info *info = file->private_data; + bool has_csum = ext4_has_feature_metadata_csum(sb); err = fscrypt_prepare_readdir(inode); if (err) @@ -149,7 +150,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) return err; /* Can we just clear INDEX flag to ignore htree information? */ - if (!ext4_has_feature_metadata_csum(sb)) { + if (!has_csum) { /* * We don't set the inode dirty flag since it's not * critical that it gets flushed back to the disk. @@ -235,7 +236,10 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) * dirent right now. Scan from the start of the block * to make sure. */ if (!inode_eq_iversion(inode, info->cookie)) { - for (i = 0; i < sb->s_blocksize && i < offset; ) { + for (i = 0; + i <= sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode) && + i < offset;) { de = (struct ext4_dir_entry_2 *) (bh->b_data + i); /* It's too expensive to do a full @@ -257,6 +261,17 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) info->cookie = inode_query_iversion(inode); } + if (unlikely(offset < sb->s_blocksize && + offset > sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode))) { + EXT4_ERROR_FILE(file, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, size=%lu", + "directory entry too close to block end", + offset, sb->s_blocksize); + ctx->pos = round_up(ctx->pos, sb->s_blocksize); + goto next_block; + } + while (ctx->pos < inode->i_size && offset < sb->s_blocksize) { de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); @@ -312,6 +327,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) ctx->pos += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); } +next_block: if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode)) goto done; brelse(bh); @@ -535,7 +551,7 @@ static int call_filldir(struct file *file, struct dir_context *ctx, struct super_block *sb = inode->i_sb; if (!fname) { - ext4_msg(sb, KERN_ERR, "%s:%d: inode #%lu: comm %s: " + ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: comm %s: " "called with null fname?!?", __func__, __LINE__, inode->i_ino, current->comm); return 0; diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 293f698b7042..724a27e8be61 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -28,7 +28,6 @@ #include <linux/seqlock.h> #include <linux/mutex.h> #include <linux/timer.h> -#include <linux/wait.h> #include <linux/sched/signal.h> #include <linux/blockgroup_lock.h> #include <linux/percpu_counter.h> @@ -92,7 +91,7 @@ */ #ifdef CONFIG_EXT4_DEBUG #define ext_debug(ino, fmt, ...) \ - pr_debug("[%s/%d] EXT4-fs (%s): ino %lu: (%s, %d): %s:" fmt, \ + pr_debug("[%s/%d] EXT4-fs (%s): ino %llu: (%s, %d): %s:" fmt, \ current->comm, task_pid_nr(current), \ ino->i_sb->s_id, ino->i_ino, __FILE__, __LINE__, \ __func__, ##__VA_ARGS__) @@ -335,7 +334,7 @@ struct ext4_io_submit { #define EXT4_MAX_BLOCK_SIZE 65536 #define EXT4_MIN_BLOCK_LOG_SIZE 10 #define EXT4_MAX_BLOCK_LOG_SIZE 16 -#define EXT4_MAX_CLUSTER_LOG_SIZE 30 +#define EXT4_MAX_CLUSTER_LOG_SIZE 28 #ifdef __KERNEL__ # define EXT4_BLOCK_SIZE(s) ((s)->s_blocksize) #else @@ -1016,14 +1015,32 @@ do { \ * than the first * I_DATA_SEM_QUOTA - Used for quota inodes only * I_DATA_SEM_EA - Used for ea_inodes only + * I_DATA_SEM_JOURNAL - Used for journal inode only */ enum { I_DATA_SEM_NORMAL = 0, I_DATA_SEM_OTHER, I_DATA_SEM_QUOTA, - I_DATA_SEM_EA + I_DATA_SEM_EA, + I_DATA_SEM_JOURNAL }; +struct ext4_fc_inode_snap; + +/* + * Snapshot failure reasons for ext4_fc_lock_updates tracepoint. + * Keep these stable for tooling. + */ +enum ext4_fc_snap_err { + EXT4_FC_SNAP_ERR_NONE = 0, + EXT4_FC_SNAP_ERR_ES_MISS, + EXT4_FC_SNAP_ERR_ES_DELAYED, + EXT4_FC_SNAP_ERR_ES_OTHER, + EXT4_FC_SNAP_ERR_INODES_CAP, + EXT4_FC_SNAP_ERR_RANGES_CAP, + EXT4_FC_SNAP_ERR_NOMEM, + EXT4_FC_SNAP_ERR_INODE_LOC, +}; /* * fourth extended file system inode data in memory @@ -1053,8 +1070,14 @@ struct ext4_inode_info { * between readers of EAs and writers of regular file data, so * instead we synchronize on xattr_sem when reading or changing * EAs. + * + * EA inodes (EXT4_EA_INODE_FL) do not use xattr_sem; they reuse + * the space for deferred iput linkage. */ - struct rw_semaphore xattr_sem; + union { + struct rw_semaphore xattr_sem; + struct llist_node i_ea_iput_node; + }; /* * Inodes with EXT4_STATE_ORPHAN_FILE use i_orphan_idx. Otherwise @@ -1080,10 +1103,23 @@ struct ext4_inode_info { /* End of lblk range that needs to be committed in this fast commit */ ext4_lblk_t i_fc_lblk_len; - spinlock_t i_raw_lock; /* protects updates to the raw inode */ + /* + * Commit-time fast commit snapshots. + * + * i_fc_snap is installed and freed under sbi->s_fc_lock. The fast + * commit log writing path reads the snapshot under sbi->s_fc_lock while + * serializing fast commit TLVs. + * + * The snapshot lifetime is bounded by EXT4_STATE_FC_COMMITTING and the + * corresponding cleanup / eviction paths. + * + * i_fc_snap points to per-inode snapshot data for fast commit: + * - a raw inode snapshot for EXT4_FC_TAG_INODE + * - data range records for EXT4_FC_TAG_{ADD,DEL}_RANGE + */ + struct ext4_fc_inode_snap *i_fc_snap; - /* Fast commit wait queue for this inode */ - wait_queue_head_t i_fc_wait; + spinlock_t i_raw_lock; /* protects updates to the raw inode */ /* * Protect concurrent accesses on i_fc_lblk_start, i_fc_lblk_len @@ -1121,6 +1157,7 @@ struct ext4_inode_info { struct rw_semaphore i_data_sem; struct inode vfs_inode; struct jbd2_inode *jinode; + struct mapping_metadata_bhs *i_metadata_bhs; /* * File creation time. Its function is same as that of @@ -1520,6 +1557,36 @@ struct ext4_orphan_info { }; /* + * Ext4 fast commit snapshot statistics. + * + * These are best-effort counters intended for debugging / performance + * introspection; they are not exact under concurrent updates. + */ +struct ext4_fc_snap_stats { + atomic64_t lock_updates_ns_total; + atomic64_t lock_updates_ns_max; + atomic64_t lock_updates_samples; + + atomic64_t snap_inodes; + atomic64_t snap_ranges; + + atomic64_t snap_fail_es_miss; + atomic64_t snap_fail_es_delayed; + atomic64_t snap_fail_es_other; + + atomic64_t snap_fail_inodes_cap; + atomic64_t snap_fail_ranges_cap; + atomic64_t snap_fail_nomem; + atomic64_t snap_fail_inode_loc; + + /* + * Missing inode snapshots during log writing should never happen. + * Keep this counter to help catch unexpected regressions. + */ + atomic64_t snap_fail_no_snap; +}; + +/* * fourth extended-fs super-block data in memory */ struct ext4_sb_info { @@ -1570,6 +1637,7 @@ struct ext4_sb_info { struct proc_dir_entry *s_proc; struct kobject s_kobj; struct completion s_kobj_unregister; + struct mutex s_error_notify_mutex; /* protects sysfs_notify vs kobject_del */ struct super_block *s_sb; struct buffer_head *s_mmp_bh; @@ -1708,6 +1776,11 @@ struct ext4_sb_info { struct ext4_es_stats s_es_stats; struct mb_cache *s_ea_block_cache; struct mb_cache *s_ea_inode_cache; + + /* Deferred iput for EA inodes to avoid lock ordering issues */ + struct llist_head s_ea_inode_to_free; + struct delayed_work s_ea_inode_work; + spinlock_t s_es_lock ____cacheline_aligned_in_smp; /* Journal triggers for checksum computation */ @@ -1792,6 +1865,7 @@ struct ext4_sb_info { struct mutex s_fc_lock; struct buffer_head *s_fc_bh; struct ext4_fc_stats s_fc_stats; + struct ext4_fc_snap_stats s_fc_snap_stats; tid_t s_fc_ineligible_tid; #ifdef CONFIG_EXT4_DEBUG int s_fc_debug_max_replay; @@ -1974,6 +2048,7 @@ enum { EXT4_STATE_FC_COMMITTING, /* Fast commit ongoing */ EXT4_STATE_FC_FLUSHING_DATA, /* Fast commit flushing data */ EXT4_STATE_ORPHAN_FILE, /* Inode orphaned in orphan file */ + EXT4_STATE_FC_REQUEUE, /* Inode modified during fast commit */ }; #define EXT4_INODE_BIT_FNS(name, field, offset) \ @@ -2002,6 +2077,8 @@ EXT4_INODE_BIT_FNS(flag, flags, 0) static inline int ext4_test_inode_state(struct inode *inode, int bit); static inline void ext4_set_inode_state(struct inode *inode, int bit); static inline void ext4_clear_inode_state(struct inode *inode, int bit); +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode); +static inline int ext4_inode_state_wait_bit(int bit); #if (BITS_PER_LONG < 64) EXT4_INODE_BIT_FNS(state, state_flags, 0) @@ -2017,6 +2094,24 @@ static inline void ext4_clear_state_flags(struct ext4_inode_info *ei) /* We depend on the fact that callers will set i_flags */ } #endif + +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode) +{ +#if (BITS_PER_LONG < 64) + return &EXT4_I(inode)->i_state_flags; +#else + return &EXT4_I(inode)->i_flags; +#endif +} + +static inline int ext4_inode_state_wait_bit(int bit) +{ +#if (BITS_PER_LONG < 64) + return bit; +#else + return bit + 32; +#endif +} #else /* Assume that user mode programs are passing in an ext4fs superblock, not * a kernel struct super_block. This will allow us to call the feature-test @@ -2042,6 +2137,17 @@ static inline bool ext4_inode_orphan_tracked(struct inode *inode) !list_empty(&EXT4_I(inode)->i_orphan); } +static inline struct mapping_metadata_bhs *ext4_i_metadata_bhs( + struct inode *inode) +{ + /* + * i_metadata_bhs is set in ext4_inode_attach_mmb() using cmpxchg(). + * We use READ_ONCE when accessing i_metadata_bhs to make sure we get + * consistent view for all accesses. + */ + return READ_ONCE(EXT4_I(inode)->i_metadata_bhs); +} + /* * Codes for operating systems */ @@ -2961,7 +3067,7 @@ extern unsigned long ext4_count_dirs(struct super_block *); extern void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap); extern int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, int barrier); -extern void ext4_end_bitmap_read(struct buffer_head *bh, int uptodate); +void ext4_end_bitmap_read(struct bio *bio); /* fast_commit.c */ int ext4_fc_info_show(struct seq_file *seq, void *v); @@ -2974,7 +3080,8 @@ void __ext4_fc_track_unlink(handle_t *handle, struct inode *inode, void __ext4_fc_track_link(handle_t *handle, struct inode *inode, struct dentry *dentry); void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry); -void ext4_fc_track_link(handle_t *handle, struct dentry *dentry); +void ext4_fc_track_link(handle_t *handle, struct inode *inode, + struct dentry *dentry); void __ext4_fc_track_create(handle_t *handle, struct inode *inode, struct dentry *dentry); void ext4_fc_track_create(handle_t *handle, struct dentry *dentry); @@ -3052,14 +3159,15 @@ int do_journal_get_write_access(handle_t *handle, struct inode *inode, struct buffer_head *bh); void ext4_set_inode_mapping_order(struct inode *inode); #define FALL_BACK_TO_NONDELALLOC 1 -#define CONVERT_INLINE_DATA 2 +#define EXT4_WRITE_DATA_INLINE 2 typedef enum { EXT4_IGET_NORMAL = 0, EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */ EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */ EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */ - EXT4_IGET_EA_INODE = 0x0008 /* Inode should contain an EA value */ + EXT4_IGET_EA_INODE = 0x0008, /* Inode should contain an EA value */ + EXT4_IGET_NOWAIT = 0x0010 /* Non-blocking lookup (skip if freeing) */ } ext4_iget_flags; extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, @@ -3070,6 +3178,7 @@ extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, __ext4_iget((sb), (ino), (flags), __func__, __LINE__) extern int ext4_write_inode(struct inode *, struct writeback_control *); +extern int ext4_sync_inode_metadata(struct inode *, struct writeback_control *); extern int ext4_setattr(struct mnt_idmap *, struct dentry *, struct iattr *); extern u32 ext4_dio_alignment(struct inode *inode); @@ -3081,8 +3190,9 @@ extern int ext4_file_getattr(struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int); extern void ext4_dirty_inode(struct inode *, int); extern int ext4_change_inode_journal_flag(struct inode *, int); -extern int ext4_get_inode_loc(struct inode *, struct ext4_iloc *); -extern int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, +int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc); +int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc); +int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, struct ext4_iloc *iloc); extern int ext4_inode_attach_jinode(struct inode *inode); extern int ext4_can_truncate(struct inode *inode); @@ -3098,9 +3208,13 @@ extern int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode); extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks); extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks); extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks, - int pextents); -extern int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode, - loff_t lstart, loff_t lend); + int pextents, int alloc_extents); +extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end); + +#define EXT4_PARTIAL_ZERO_START 0x1 +#define EXT4_PARTIAL_ZERO_END 0x2 +extern int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, + loff_t length, unsigned int *partial_zeroed); extern vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf); extern qsize_t *ext4_get_reserved_space(struct inode *inode); extern int ext4_get_projid(struct inode *inode, kprojid_t *projid); @@ -3184,10 +3298,10 @@ extern struct buffer_head *ext4_sb_bread_unmovable(struct super_block *sb, sector_t block); extern struct buffer_head *ext4_sb_bread_nofail(struct super_block *sb, sector_t block); -extern void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail); -extern int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail); +void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); +int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); extern int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait); extern void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block); extern int ext4_seq_options_show(struct seq_file *seq, void *offset); @@ -3229,7 +3343,7 @@ extern void __dump_mmp_msg(struct super_block *, struct mmp_struct *mmp, extern __printf(7, 8) void __ext4_grp_locked_error(const char *, unsigned int, struct super_block *, ext4_group_t, - unsigned long, ext4_fsblk_t, + u64, ext4_fsblk_t, const char *, ...); #define EXT4_ERROR_INODE(inode, fmt, a...) \ @@ -3552,7 +3666,13 @@ struct ext4_group_info { #define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \ (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state))) #define EXT4_MB_GRP_TEST_AND_SET_READ(grp) \ - (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state))) + (ext4_mb_grp_test_and_set_read((grp))) + +static inline int ext4_mb_grp_test_and_set_read(struct ext4_group_info *grp) +{ + return (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state) || + test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state)); +} #define EXT4_MAX_CONTENTION 8 #define EXT4_CONTENTION_THRESHOLD 2 @@ -3661,7 +3781,7 @@ extern int ext4_generic_write_inline_data(struct address_space *mapping, struct inode *inode, loff_t pos, unsigned len, struct folio **foliop, - void **fsdata, bool da); + bool da); extern int ext4_try_add_inline_entry(handle_t *handle, struct ext4_filename *fname, struct inode *dir, struct inode *inode); @@ -3719,7 +3839,7 @@ extern int ext4_handle_dirty_dirblock(handle_t *handle, struct inode *inode, extern int __ext4_unlink(struct inode *dir, const struct qstr *d_name, struct inode *inode, struct dentry *dentry); extern int __ext4_link(struct inode *dir, struct inode *inode, - struct dentry *dentry); + const struct qstr *d_name, struct dentry *dentry); #define S_SHIFT 12 static const unsigned char ext4_type_by_mode[(S_IFMT >> S_SHIFT) + 1] = { @@ -3742,8 +3862,8 @@ static inline void ext4_set_de_type(struct super_block *sb, /* readpages.c */ int ext4_read_folio(struct file *file, struct folio *folio); void ext4_readahead(struct readahead_control *rac); -extern int __init ext4_init_post_read_processing(void); -extern void ext4_exit_post_read_processing(void); +int __init ext4_init_verity_caches(void); +void ext4_exit_verity_caches(void); /* symlink.c */ extern const struct inode_operations ext4_encrypted_symlink_inode_operations; @@ -3793,7 +3913,8 @@ extern void ext4_ext_release(struct super_block *); extern long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len); extern int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, - loff_t offset, ssize_t len); + loff_t offset, ssize_t len, + ext4_lblk_t *converted); extern int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, loff_t offset, ssize_t len); extern int ext4_convert_unwritten_io_end_vec(handle_t *handle, @@ -3858,7 +3979,7 @@ extern void ext4_io_submit_init(struct ext4_io_submit *io, struct writeback_control *wbc); extern void ext4_end_io_rsv_work(struct work_struct *work); extern void ext4_io_submit(struct ext4_io_submit *io); -int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *page, +void ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *page, size_t len); extern struct ext4_io_end_vec *ext4_alloc_io_end_vec(ext4_io_end_t *io_end); extern struct ext4_io_end_vec *ext4_last_io_end_vec(ext4_io_end_t *io_end); @@ -3920,6 +4041,9 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) extern const struct iomap_ops ext4_iomap_ops; extern const struct iomap_ops ext4_iomap_report_ops; +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap, struct iomap *srcmap); + static inline int ext4_buffer_uptodate(struct buffer_head *bh) { /* @@ -3944,6 +4068,11 @@ static inline bool ext4_inode_can_atomic_write(struct inode *inode) extern int ext4_block_write_begin(handle_t *handle, struct folio *folio, loff_t pos, unsigned len, get_block_t *get_block); + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +#define EXPORT_SYMBOL_FOR_EXT4_TEST(sym) \ + EXPORT_SYMBOL_FOR_MODULES(sym, "ext4-test") +#endif #endif /* __KERNEL__ */ #endif /* _EXT4_H */ diff --git a/fs/ext4/ext4_extents.h b/fs/ext4/ext4_extents.h index c484125d963f..ebaf7cc42430 100644 --- a/fs/ext4/ext4_extents.h +++ b/fs/ext4/ext4_extents.h @@ -264,5 +264,17 @@ static inline void ext4_idx_store_pblock(struct ext4_extent_idx *ix, 0xffff); } +extern int __ext4_ext_dirty(const char *where, unsigned int line, + handle_t *handle, struct inode *inode, + struct ext4_ext_path *path); +extern int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex); +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +extern int ext4_ext_space_root_idx_test(struct inode *inode, int check); +extern struct ext4_ext_path *ext4_split_convert_extents_test( + handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, + struct ext4_ext_path *path, + int flags, unsigned int *allocated); +#endif #endif /* _EXT4_EXTENTS */ diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c index 05e5946ed9b3..53ddedb52a6f 100644 --- a/fs/ext4/ext4_jbd2.c +++ b/fs/ext4/ext4_jbd2.c @@ -33,14 +33,22 @@ int ext4_inode_journal_mode(struct inode *inode) static handle_t *ext4_get_nojournal(void) { handle_t *handle = current->journal_info; - unsigned long ref_cnt = (unsigned long)handle; - BUG_ON(ref_cnt >= EXT4_NOJOURNAL_MAX_REF_COUNT); - - ref_cnt++; - handle = (handle_t *)ref_cnt; - - current->journal_info = handle; + BUG_ON(handle && !handle->h_invalid); + + if (!handle) { + handle = jbd2_alloc_handle(GFP_NOFS); + if (!handle) + return ERR_PTR(-ENOMEM); + handle->h_invalid = 1; + /* + * This is done by start_this_handle() if journalling + * is enabled. + */ + handle->saved_alloc_context = memalloc_nofs_save(); + current->journal_info = handle; + } + handle->h_ref++; return handle; } @@ -48,14 +56,14 @@ static handle_t *ext4_get_nojournal(void) /* Decrement the non-pointer handle value */ static void ext4_put_nojournal(handle_t *handle) { - unsigned long ref_cnt = (unsigned long)handle; - - BUG_ON(ref_cnt == 0); + BUG_ON(handle->h_ref == 0); - ref_cnt--; - handle = (handle_t *)ref_cnt; - - current->journal_info = handle; + handle->h_ref--; + if (handle->h_ref == 0) { + memalloc_nofs_restore(handle->saved_alloc_context); + jbd2_free_handle(handle); + current->journal_info = NULL; + } } /* @@ -350,6 +358,21 @@ int __ext4_journal_get_create_access(const char *where, unsigned int line, return 0; } +static void ext4_inode_attach_mmb(struct inode *inode) +{ + struct mapping_metadata_bhs *mmb; + + /* + * It's difficult to handle failure when marking buffer dirty without + * leaving filesystem corrupted + */ + mmb = kmalloc_obj(*mmb, GFP_NOFS | __GFP_NOFAIL | __GFP_ACCOUNT); + mmb_init(mmb, &inode->i_data); + /* Someone swapped another mmb before us? */ + if (cmpxchg(&EXT4_I(inode)->i_metadata_bhs, NULL, mmb)) + kfree(mmb); +} + int __ext4_handle_dirty_metadata(const char *where, unsigned int line, handle_t *handle, struct inode *inode, struct buffer_head *bh) @@ -389,10 +412,13 @@ int __ext4_handle_dirty_metadata(const char *where, unsigned int line, err); } } else { - if (inode) - mark_buffer_dirty_inode(bh, inode); - else + if (inode) { + if (!ext4_i_metadata_bhs(inode)) + ext4_inode_attach_mmb(inode); + mmb_mark_buffer_dirty(bh, ext4_i_metadata_bhs(inode)); + } else { mark_buffer_dirty(bh); + } if (inode && inode_needs_sync(inode)) { sync_dirty_buffer(bh); if (buffer_req(bh) && !buffer_uptodate(bh)) { diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h index 63d17c5201b5..2fbf48b3dfe2 100644 --- a/fs/ext4/ext4_jbd2.h +++ b/fs/ext4/ext4_jbd2.h @@ -182,15 +182,11 @@ handle_t *__ext4_journal_start_sb(struct inode *inode, struct super_block *sb, int rsv_blocks, int revoke_creds); int __ext4_journal_stop(const char *where, unsigned int line, handle_t *handle); -#define EXT4_NOJOURNAL_MAX_REF_COUNT ((unsigned long) 4096) - /* Note: Do not use this for NULL handles. This is only to determine if * a properly allocated handle is using a journal or not. */ static inline int ext4_handle_valid(handle_t *handle) { - if ((unsigned long)handle < EXT4_NOJOURNAL_MAX_REF_COUNT) - return 0; - return 1; + return (handle && !handle->h_invalid); } static inline void ext4_handle_sync(handle_t *handle) diff --git a/fs/ext4/extents-test.c b/fs/ext4/extents-test.c index 7c4690eb7dad..c3836ecb89f9 100644 --- a/fs/ext4/extents-test.c +++ b/fs/ext4/extents-test.c @@ -37,6 +37,7 @@ #include <kunit/test.h> #include <kunit/static_stub.h> +#include <linux/fs_context.h> #include <linux/gfp_types.h> #include <linux/stddef.h> @@ -125,25 +126,32 @@ struct kunit_ext_test_param { struct kunit_ext_data_state exp_data_state[3]; }; -static void ext_kill_sb(struct super_block *sb) +static int ext_init_fs_context(struct fs_context *fc) { - generic_shutdown_super(sb); + return 0; } -static int ext_set(struct super_block *sb, void *data) +static int ext_set(struct super_block *sb, struct fs_context *fc) { - return 0; + return set_anon_super_fc(sb, fc); } static struct file_system_type ext_fs_type = { - .name = "extents test", - .kill_sb = ext_kill_sb, + .name = "extents test", + .init_fs_context = ext_init_fs_context, + .kill_sb = kill_anon_super, }; static void extents_kunit_exit(struct kunit *test) { - struct ext4_sb_info *sbi = k_ctx.k_ei->vfs_inode.i_sb->s_fs_info; + struct ext4_sb_info *sbi; + if (!k_ctx.k_ei) + return; + + sbi = k_ctx.k_ei->vfs_inode.i_sb->s_fs_info; + ext4_es_unregister_shrinker(sbi); + deactivate_super(sbi->s_sb); kfree(sbi); kfree(k_ctx.k_ei); kfree(k_ctx.k_data); @@ -217,39 +225,50 @@ static int extents_kunit_init(struct kunit *test) struct ext4_inode_info *ei; struct inode *inode; struct super_block *sb; + struct fs_context *fc; struct ext4_sb_info *sbi = NULL; struct kunit_ext_test_param *param = (struct kunit_ext_test_param *)(test->param_value); int err; - sb = sget(&ext_fs_type, NULL, ext_set, 0, NULL); - if (IS_ERR(sb)) - return PTR_ERR(sb); - - sb->s_blocksize = 4096; - sb->s_blocksize_bits = 12; - sbi = kzalloc_obj(struct ext4_sb_info); if (sbi == NULL) return -ENOMEM; + fc = fs_context_for_mount(&ext_fs_type, 0); + if (IS_ERR(fc)) { + kfree(sbi); + return PTR_ERR(fc); + } + sb = sget_fc(fc, NULL, ext_set); + put_fs_context(fc); + if (IS_ERR(sb)) { + kfree(sbi); + return PTR_ERR(sb); + } + sbi->s_sb = sb; sb->s_fs_info = sbi; + sb->s_blocksize = 4096; + sb->s_blocksize_bits = 12; + if (!param || !param->disable_zeroout) sbi->s_extent_max_zeroout_kb = 32; + err = ext4_es_register_shrinker(sbi); + if (err) + goto out_deactivate; + /* setup the mock inode */ k_ctx.k_ei = kzalloc_obj(struct ext4_inode_info); - if (k_ctx.k_ei == NULL) - return -ENOMEM; + if (k_ctx.k_ei == NULL) { + err = -ENOMEM; + goto out; + } ei = k_ctx.k_ei; inode = &ei->vfs_inode; - err = ext4_es_register_shrinker(sbi); - if (err) - return err; - ext4_es_init_tree(&ei->i_es_tree); rwlock_init(&ei->i_es_lock); INIT_LIST_HEAD(&ei->i_es_list); @@ -264,8 +283,10 @@ static int extents_kunit_init(struct kunit *test) inode->i_sb = sb; k_ctx.k_data = kzalloc(EXT_DATA_LEN * 4096, GFP_KERNEL); - if (k_ctx.k_data == NULL) - return -ENOMEM; + if (k_ctx.k_data == NULL) { + err = -ENOMEM; + goto out; + } /* * set the data area to a junk value @@ -280,8 +301,8 @@ static int extents_kunit_init(struct kunit *test) eh->eh_depth = 0; eh->eh_entries = cpu_to_le16(1); eh->eh_magic = EXT4_EXT_MAGIC; - eh->eh_max = - cpu_to_le16(ext4_ext_space_root_idx(&k_ctx.k_ei->vfs_inode, 0)); + eh->eh_max = cpu_to_le16(ext4_ext_space_root_idx_test( + &k_ctx.k_ei->vfs_inode, 0)); eh->eh_generation = 0; /* @@ -307,7 +328,23 @@ static int extents_kunit_init(struct kunit *test) kunit_activate_static_stub(test, ext4_ext_zeroout, ext4_ext_zeroout_stub); kunit_activate_static_stub(test, ext4_issue_zeroout, ext4_issue_zeroout_stub); + up_write(&sb->s_umount); + return 0; + +out: + kfree(k_ctx.k_ei); + k_ctx.k_ei = NULL; + + kfree(k_ctx.k_data); + k_ctx.k_data = NULL; + + ext4_es_unregister_shrinker(sbi); +out_deactivate: + deactivate_locked_super(sb); + kfree(sbi); + + return err; } /* @@ -384,8 +421,8 @@ static void test_split_convert(struct kunit *test) switch (param->type) { case TEST_SPLIT_CONVERT: - path = ext4_split_convert_extents(NULL, inode, &map, path, - param->split_flags, NULL); + path = ext4_split_convert_extents_test(NULL, inode, &map, + path, param->split_flags, NULL); break; case TEST_CREATE_BLOCKS: ext4_map_create_blocks_helper(test, inode, &map, param->split_flags); diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index ae3804f36535..76038b6c3655 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -184,9 +184,9 @@ static int ext4_ext_get_access(handle_t *handle, struct inode *inode, * - ENOMEM * - EIO */ -static int __ext4_ext_dirty(const char *where, unsigned int line, - handle_t *handle, struct inode *inode, - struct ext4_ext_path *path) +int __ext4_ext_dirty(const char *where, unsigned int line, + handle_t *handle, struct inode *inode, + struct ext4_ext_path *path) { int err; @@ -1736,6 +1736,13 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, err = ext4_ext_get_access(handle, inode, path + k); if (err) return err; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + return -EFSCORRUPTED; + } path[k].p_idx->ei_block = border; err = ext4_ext_dirty(handle, inode, path + k); if (err) @@ -1748,6 +1755,14 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, err = ext4_ext_get_access(handle, inode, path + k); if (err) goto clean; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + err = -EFSCORRUPTED; + goto clean; + } path[k].p_idx->ei_block = border; err = ext4_ext_dirty(handle, inode, path + k); if (err) @@ -2412,9 +2427,17 @@ int ext4_ext_index_trans_blocks(struct inode *inode, int extents) */ if (extents <= 1) index = (EXT4_MAX_EXTENT_DEPTH * 2) + extents; - else - index = (EXT4_MAX_EXTENT_DEPTH * 3) + - DIV_ROUND_UP(extents, ext4_ext_space_block(inode, 0)); + else { + int ext_max = ext4_ext_space_block(inode, 0); + + index = EXT4_MAX_EXTENT_DEPTH * 3; + /* + * Modified extents need not start at the beginning of the + * leaf. Already two extents may need two leaf block + * modifications... + */ + index += DIV_ROUND_UP(extents + ext_max - 1, ext_max); + } return index; } @@ -3144,7 +3167,7 @@ static void ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex) } /* FIXME!! we need to try to merge to left or right after zero-out */ -static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex) +int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex) { ext4_fsblk_t ee_pblock; unsigned int ee_len; @@ -3239,6 +3262,9 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, insert_err = PTR_ERR(path); err = 0; + if (insert_err != -ENOSPC && insert_err != -EDQUOT && + insert_err != -ENOMEM) + goto out_path; /* * Get a new path to try to zeroout or fix the extent length. @@ -3250,18 +3276,25 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, */ path = ext4_find_extent(inode, ee_block, NULL, flags | EXT4_EX_NOFAIL); if (IS_ERR(path)) { - EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld", - split, PTR_ERR(path)); + EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %pe", + split, path); goto out_path; } + depth = ext_depth(inode); + ex = path[depth].p_ext; + if (!ex) { + EXT4_ERROR_INODE(inode, + "bad extent address lblock: %lu, depth: %d pblock %llu", + (unsigned long)ee_block, depth, path[depth].p_block); + err = -EFSCORRUPTED; + goto out; + } + err = ext4_ext_get_access(handle, inode, path + depth); if (err) goto out; - depth = ext_depth(inode); - ex = path[depth].p_ext; - fix_extent_len: ex->ee_len = orig_ex.ee_len; err = ext4_ext_dirty(handle, inode, path + path->p_depth); @@ -3363,7 +3396,7 @@ static int ext4_split_extent_zeroout(handle_t *handle, struct inode *inode, ext4_ext_mark_initialized(ex); - ext4_ext_dirty(handle, inode, path + depth); + err = ext4_ext_dirty(handle, inode, path + depth); if (err) return err; @@ -4457,9 +4490,13 @@ got_allocated_blocks: path = ext4_ext_insert_extent(handle, inode, path, &newex, flags); if (IS_ERR(path)) { err = PTR_ERR(path); - if (allocated_clusters) { + /* + * Gracefully handle out of space conditions. If the filesystem + * is inconsistent, we'll just leak allocated blocks to avoid + * causing even more damage. + */ + if (allocated_clusters && (err == -EDQUOT || err == -ENOSPC)) { int fb_flags = 0; - /* * free data blocks we just allocated. * not a good idea to call discard here directly, @@ -4542,30 +4579,47 @@ retry_remove_space: return err; } -static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset, - ext4_lblk_t len, loff_t new_size, - int flags) +/* + * Pre-allocate blocks for the range [@offset, @offset + @len). Allocated + * blocks are marked as unwritten by default. If EXT4_GET_BLOCKS_ZERO is + * set, the allocated blocks are zeroed on disk and their extents are + * converted to written state. + * + * When @new_size is nonzero, the caller intends to extend the file, and + * the file size should be updated to the end of the allocated blocks. + * + * Allocation may partially succeed due to some non-fatal issues. In that + * case, i_disksize (and i_size) is advanced up to the successfully + * processed portion of the range. + * + * Return 0 on success, or a negative error code on failure or partial + * failure. + */ +static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len, + loff_t new_size, int flags) { struct inode *inode = file_inode(file); handle_t *handle; int ret = 0, ret2 = 0, ret3 = 0; int retries = 0; int depth = 0; + ext4_lblk_t len_lblk; struct ext4_map_blocks map; unsigned int credits; - loff_t epos, old_size = i_size_read(inode); + loff_t epos = 0, old_size = i_size_read(inode); unsigned int blkbits = inode->i_blkbits; bool alloc_zero = false; + bool orphan = false; BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)); - map.m_lblk = offset; - map.m_len = len; + map.m_lblk = offset >> blkbits; + map.m_len = len_lblk = EXT4_MAX_BLOCKS(len, offset, blkbits); /* * Don't normalize the request if it can fit in one extent so * that it doesn't get unnecessarily split into multiple * extents. */ - if (len <= EXT_UNWRITTEN_MAX_LEN) + if (len_lblk <= EXT_UNWRITTEN_MAX_LEN) flags |= EXT4_GET_BLOCKS_NO_NORMALIZE; /* @@ -4582,16 +4636,23 @@ static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset, /* * credits to insert 1 extent into extent tree */ - credits = ext4_chunk_trans_blocks(inode, len); + credits = ext4_chunk_trans_blocks(inode, len_lblk); depth = ext_depth(inode); + /* Zero to the end of the block containing i_size */ + if (new_size > old_size) { + ret = ext4_block_zero_eof(inode, old_size, LLONG_MAX); + if (ret) + return ret; + } + retry: - while (len) { + while (len_lblk) { /* * Recalculate credits when extent tree depth changes. */ if (depth != ext_depth(inode)) { - credits = ext4_chunk_trans_blocks(inode, len); + credits = ext4_chunk_trans_blocks(inode, len_lblk); depth = ext_depth(inode); } @@ -4603,7 +4664,7 @@ retry: } ret = ext4_map_blocks(handle, inode, &map, flags); if (ret <= 0) { - ext4_debug("inode #%lu: block %u: len %u: " + ext4_debug("inode #%llu: block %u: len %u: " "ext4_ext_map_blocks returned %d", inode->i_ino, map.m_lblk, map.m_len, ret); @@ -4611,50 +4672,105 @@ retry: ext4_journal_stop(handle); break; } + ext4_update_inode_fsync_trans(handle, inode, 1); + ret = ext4_journal_stop(handle); + if (unlikely(ret)) + break; + /* * allow a full retry cycle for any remaining allocations */ retries = 0; - epos = EXT4_LBLK_TO_B(inode, map.m_lblk + ret); - inode_set_ctime_current(inode); - if (new_size) { - if (epos > new_size) - epos = new_size; - if (ext4_update_inode_size(inode, epos) & 0x1) - inode_set_mtime_to_ts(inode, - inode_get_ctime(inode)); - if (epos > old_size) { - pagecache_isize_extended(inode, old_size, epos); - ext4_zero_partial_blocks(handle, inode, - old_size, epos - old_size); - } - } - ret2 = ext4_mark_inode_dirty(handle, inode); - ext4_update_inode_fsync_trans(handle, inode, 1); - ret3 = ext4_journal_stop(handle); - ret2 = ret3 ? ret3 : ret2; - if (unlikely(ret2)) - break; if (alloc_zero && (map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) { - ret2 = ext4_issue_zeroout(inode, map.m_lblk, map.m_pblk, - map.m_len); - if (likely(!ret2)) - ret2 = ext4_convert_unwritten_extents(NULL, - inode, (loff_t)map.m_lblk << blkbits, - (loff_t)map.m_len << blkbits); - if (ret2) + ext4_lblk_t converted; + + WARN_ON_ONCE(map.m_lblk + map.m_len > + EXT4_B_TO_LBLK(inode, new_size ?: old_size)); + + ret = ext4_issue_zeroout(inode, map.m_lblk, map.m_pblk, + map.m_len); + if (unlikely(ret)) break; + + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, + credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + break; + } + + ret = ext4_convert_unwritten_extents(handle, + inode, (loff_t)map.m_lblk << blkbits, + (loff_t)map.m_len << blkbits, + &converted); + if (ret) + map.m_len = converted; + + /* + * If blocks beyond i_disksize are converted, add + * the inode to the orphan list and advance the epos. + */ + if (new_size && converted) { + ret2 = ext4_orphan_add(handle, inode); + ret = ret ? ret : ret2; + orphan = true; + } + + ret3 = ext4_journal_stop(handle); + ret = ret ? ret : ret3; } - map.m_lblk += ret; - map.m_len = len = len - ret; + map.m_lblk += map.m_len; + map.m_len = len_lblk = len_lblk - map.m_len; + epos = EXT4_LBLK_TO_B(inode, map.m_lblk); + if (ret) + break; } + if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) goto retry; - return ret > 0 ? ret2 : ret; + if (!epos || !new_size) + return ret; + + /* + * Allocate blocks, update the file size to match the size of the + * already successfully allocated blocks. + */ + if (epos > new_size) + epos = new_size; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 2); + if (IS_ERR(handle)) { + /* + * The conversion has successfully completed. Not much to + * do with the error here so just cleanup the orphan list + * and hope for the best. + */ + if (orphan && inode->i_nlink) + ext4_orphan_del(NULL, inode); + ret2 = PTR_ERR(handle); + goto out; + } + + ext4_update_inode_size(inode, epos); + if (orphan && inode->i_nlink) + ext4_orphan_del(handle, inode); + + ret2 = ext4_mark_inode_dirty(handle, inode); + ext4_update_inode_fsync_trans(handle, inode, 1); + ret3 = ext4_journal_stop(handle); + ret2 = ret3 ? ret3 : ret2; + + if (epos > old_size) + pagecache_isize_extended(inode, old_size, epos); +out: + if (ret2) + ext4_std_error(inode->i_sb, ret2); + + return ret ? ret : ret2; } static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len); @@ -4666,12 +4782,11 @@ static long ext4_zero_range(struct file *file, loff_t offset, { struct inode *inode = file_inode(file); handle_t *handle = NULL; - loff_t new_size = 0; + loff_t align_start, align_end, new_size = 0; loff_t end = offset + len; - ext4_lblk_t start_lblk, end_lblk; unsigned int blocksize = i_blocksize(inode); - unsigned int blkbits = inode->i_blkbits; - int ret, flags, credits; + unsigned int partial_zeroed = 0; + int ret, flags; trace_ext4_zero_range(inode, offset, len, mode); WARN_ON_ONCE(!inode_is_locked(inode)); @@ -4689,13 +4804,16 @@ static long ext4_zero_range(struct file *file, loff_t offset, } flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; - /* Preallocate the range including the unaligned edges */ + /* + * Preallocate the range including the unaligned edges, and zero + * out partial blocks if they already contain data. + */ if (!IS_ALIGNED(offset | end, blocksize)) { - ext4_lblk_t alloc_lblk = offset >> blkbits; - ext4_lblk_t len_lblk = EXT4_MAX_BLOCKS(len, offset, blkbits); - - ret = ext4_alloc_file_blocks(file, alloc_lblk, len_lblk, - new_size, flags); + ret = ext4_alloc_file_blocks(file, offset, len, new_size, + flags); + if (!ret) + ret = ext4_zero_partial_blocks(inode, offset, len, + &partial_zeroed); if (ret) return ret; } @@ -4710,18 +4828,32 @@ static long ext4_zero_range(struct file *file, loff_t offset, return ret; /* Zero range excluding the unaligned edges */ - start_lblk = EXT4_B_TO_LBLK(inode, offset); - end_lblk = end >> blkbits; - if (end_lblk > start_lblk) { - ext4_lblk_t zero_blks = end_lblk - start_lblk; + align_start = round_up(offset, blocksize); + align_end = round_down(end, blocksize); + /* + * In WRITE_ZEROES mode, edges that were not partial-zeroed (clean + * unwritten or hole) must be allocated and zeroed as whole blocks. + * Expand the aligned range outward to cover them. + */ + if (mode & FALLOC_FL_WRITE_ZEROES) { + if (!IS_ALIGNED(offset, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_START)) + align_start = round_down(offset, blocksize); + if (!IS_ALIGNED(end, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_END)) + align_end = round_up(end, blocksize); + } + + if (align_end > align_start) { if (mode & FALLOC_FL_WRITE_ZEROES) flags = EXT4_GET_BLOCKS_CREATE_ZERO | EXT4_EX_NOCACHE; else flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN | EXT4_EX_NOCACHE); - ret = ext4_alloc_file_blocks(file, start_lblk, zero_blks, - new_size, flags); + ret = ext4_alloc_file_blocks(file, align_start, + align_end - align_start, new_size, + flags); if (ret) return ret; } @@ -4730,24 +4862,27 @@ static long ext4_zero_range(struct file *file, loff_t offset, return ret; /* - * In worst case we have to writeout two nonadjacent unwritten - * blocks and update the inode + * In FALLOC_FL_WRITE_ZEROES mode, edges that have been partially + * zeroed must be written back to ensure the entire zeroed range + * is converted to the written state. In SYNC mode, writeback is + * also required to persist the zeroed data to disk. */ - credits = (2 * ext4_ext_index_trans_blocks(inode, 2)) + 1; - if (ext4_should_journal_data(inode)) - credits += 2; - handle = ext4_journal_start(inode, EXT4_HT_MISC, credits); + if (partial_zeroed && + ((mode & FALLOC_FL_WRITE_ZEROES) || + (file->f_flags & O_SYNC) || IS_SYNC(inode))) { + ret = filemap_write_and_wait_range(inode->i_mapping, offset, + end - 1); + if (ret) + return ret; + } + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); if (IS_ERR(handle)) { ret = PTR_ERR(handle); ext4_std_error(inode->i_sb, ret); return ret; } - /* Zero out partial block at the edges of the range */ - ret = ext4_zero_partial_blocks(handle, inode, offset, len); - if (ret) - goto out_handle; - if (new_size) ext4_update_inode_size(inode, new_size); ret = ext4_mark_inode_dirty(handle, inode); @@ -4755,7 +4890,7 @@ static long ext4_zero_range(struct file *file, loff_t offset, goto out_handle; ext4_update_inode_fsync_trans(handle, inode, 1); - if (file->f_flags & O_SYNC) + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) ext4_handle_sync(handle); out_handle: @@ -4769,15 +4904,11 @@ static long ext4_do_fallocate(struct file *file, loff_t offset, struct inode *inode = file_inode(file); loff_t end = offset + len; loff_t new_size = 0; - ext4_lblk_t start_lblk, len_lblk; int ret; trace_ext4_fallocate_enter(inode, offset, len, mode); WARN_ON_ONCE(!inode_is_locked(inode)); - start_lblk = offset >> inode->i_blkbits; - len_lblk = EXT4_MAX_BLOCKS(len, offset, inode->i_blkbits); - /* We only support preallocation for extent-based files only. */ if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { ret = -EOPNOTSUPP; @@ -4792,17 +4923,19 @@ static long ext4_do_fallocate(struct file *file, loff_t offset, goto out; } - ret = ext4_alloc_file_blocks(file, start_lblk, len_lblk, new_size, + ret = ext4_alloc_file_blocks(file, offset, len, new_size, EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT); if (ret) goto out; - if (file->f_flags & O_SYNC && EXT4_SB(inode->i_sb)->s_journal) { + if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && + EXT4_SB(inode->i_sb)->s_journal) { ret = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, EXT4_I(inode)->i_sync_tid); } out: - trace_ext4_fallocate_exit(inode, offset, len_lblk, ret); + trace_ext4_fallocate_exit(inode, offset, + EXT4_MAX_BLOCKS(len, offset, inode->i_blkbits), ret); return ret; } @@ -4938,7 +5071,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * it can tell if the extent in the cache is a split extent. * But for now let's assume pextents as 2 always. */ - credits = ext4_meta_trans_blocks(inode, max_blocks, 2); + credits = ext4_meta_trans_blocks(inode, max_blocks, 2, 0); } if (credits) { @@ -4955,7 +5088,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, ret = ext4_map_blocks(handle, inode, &map, flags); if (ret != max_blocks) ext4_msg(inode->i_sb, KERN_INFO, - "inode #%lu: block %u: len %u: " + "inode #%llu: block %u: len %u: " "split block mapping found for atomic write, " "ret = %d", inode->i_ino, map.m_lblk, @@ -4974,7 +5107,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, if (ret <= 0 || ret2) ext4_warning(inode->i_sb, - "inode #%lu: block %u: len %u: " + "inode #%llu: block %u: len %u: " "returned %d or %d", inode->i_ino, map.m_lblk, map.m_len, ret, ret2); @@ -4988,21 +5121,26 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * all unwritten extents within this range will be converted to * written extents. * - * This function is called from the direct IO end io call back - * function, to convert the fallocated extents after IO is completed. - * Returns 0 on success. + * This function is called from the direct/buffered I/O end io call back + * function and FALLOC_FL_WRITE_ZEROES, to convert the fallocated + * unwritten extents after data I/O is completed. + * + * Returns 0 on full success, or a negative error code on partial + * success or failure. The number of blocks converted is returned via + * @converted. */ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, - loff_t offset, ssize_t len) + loff_t offset, ssize_t len, + ext4_lblk_t *converted) { - unsigned int max_blocks; + ext4_lblk_t max_blocks, conv_blocks = 0; int ret = 0, ret2 = 0, ret3 = 0; struct ext4_map_blocks map; unsigned int blkbits = inode->i_blkbits; unsigned int credits = 0; map.m_lblk = offset >> blkbits; - max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); + map.m_len = max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); if (!handle) { /* @@ -5010,9 +5148,8 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, */ credits = ext4_chunk_trans_blocks(inode, max_blocks); } - while (ret >= 0 && ret < max_blocks) { - map.m_lblk += ret; - map.m_len = (max_blocks -= ret); + + while (max_blocks) { if (credits) { handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, credits); @@ -5029,23 +5166,34 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, ret = ext4_map_blocks(handle, inode, &map, EXT4_GET_BLOCKS_IO_CONVERT_EXT | EXT4_EX_NOCACHE); - if (ret <= 0) + if (ret <= 0) { ext4_warning(inode->i_sb, - "inode #%lu: block %u: len %u: " - "ext4_ext_map_blocks returned %d", - inode->i_ino, map.m_lblk, - map.m_len, ret); + "inode #%llu: block %u: len %u: ext4_map_blocks returned %d", + inode->i_ino, map.m_lblk, map.m_len, ret); + if (unlikely(ret == 0)) + ret = -EINVAL; + } else { + conv_blocks += map.m_len; + } + ret2 = ext4_mark_inode_dirty(handle, inode); if (credits) { ret3 = ext4_journal_stop(handle); if (unlikely(ret3)) ret2 = ret3; } - - if (ret <= 0 || ret2) + ret = ret < 0 ? ret : ret2; + if (ret) break; + + map.m_lblk += map.m_len; + map.m_len = (max_blocks -= map.m_len); } - return ret > 0 ? ret2 : ret; + /* Converted some or all blocks successfully? */ + if (converted) + *converted = conv_blocks; + + return ret; } int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end) @@ -5068,7 +5216,7 @@ int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end) list_for_each_entry(io_end_vec, &io_end->list_vec, list) { ret = ext4_convert_unwritten_extents(handle, io_end->inode, io_end_vec->offset, - io_end_vec->size); + io_end_vec->size, NULL); if (ret) break; } @@ -5133,8 +5281,10 @@ static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset, return error; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_xattr_next, ext4_iomap_xattr_begin); + static const struct iomap_ops ext4_iomap_xattr_ops = { - .iomap_begin = ext4_iomap_xattr_begin, + .iomap_next = ext4_iomap_xattr_next, }; static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len) @@ -5569,7 +5719,7 @@ static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len) goto out_handle; ext4_update_inode_fsync_trans(handle, inode, 1); - if (IS_SYNC(inode)) + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) ext4_handle_sync(handle); out_handle: @@ -5693,7 +5843,7 @@ static int ext4_insert_range(struct file *file, loff_t offset, loff_t len) goto out_handle; ext4_update_inode_fsync_trans(handle, inode, 1); - if (IS_SYNC(inode)) + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) ext4_handle_sync(handle); out_handle: @@ -6238,6 +6388,33 @@ out: return 0; } -#ifdef CONFIG_EXT4_KUNIT_TESTS -#include "extents-test.c" +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +int ext4_ext_space_root_idx_test(struct inode *inode, int check) +{ + return ext4_ext_space_root_idx(inode, check); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_space_root_idx_test); + +struct ext4_ext_path *ext4_split_convert_extents_test(handle_t *handle, + struct inode *inode, struct ext4_map_blocks *map, + struct ext4_ext_path *path, int flags, + unsigned int *allocated) +{ + return ext4_split_convert_extents(handle, inode, map, path, + flags, allocated); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_split_convert_extents_test); + +EXPORT_SYMBOL_FOR_EXT4_TEST(__ext4_ext_dirty); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_zeroout); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_register_shrinker); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_unregister_shrinker); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_create_blocks); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_init_tree); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_lookup_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_insert_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_insert_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_find_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_issue_zeroout); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_query_blocks); #endif diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c index a1538bac51c6..6e4a191e8219 100644 --- a/fs/ext4/extents_status.c +++ b/fs/ext4/extents_status.c @@ -214,7 +214,7 @@ static void ext4_es_print_tree(struct inode *inode) struct ext4_es_tree *tree; struct rb_node *node; - printk(KERN_DEBUG "status extents for inode %lu:", inode->i_ino); + printk(KERN_DEBUG "status extents for inode %llu:", inode->i_ino); tree = &EXT4_I(inode)->i_es_tree; node = rb_first(&tree->root); while (node) { @@ -703,7 +703,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode, if (!ext4_es_is_written(es) && !ext4_es_is_unwritten(es)) { if (in_range(es->es_lblk, ee_block, ee_len)) { pr_warn("ES insert assertion failed for " - "inode: %lu we can find an extent " + "inode: %llu we can find an extent " "at block [%d/%d/%llu/%c], but we " "want to add a delayed/hole extent " "[%d/%d/%llu/%x]\n", @@ -721,7 +721,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode, */ if (es->es_lblk < ee_block || ext4_es_pblock(es) != ee_start + es->es_lblk - ee_block) { - pr_warn("ES insert assertion failed for inode: %lu " + pr_warn("ES insert assertion failed for inode: %llu " "ex_status [%d/%d/%llu/%c] != " "es_status [%d/%d/%llu/%c]\n", inode->i_ino, ee_block, ee_len, ee_start, @@ -731,7 +731,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode, } if (ee_status ^ es_status) { - pr_warn("ES insert assertion failed for inode: %lu " + pr_warn("ES insert assertion failed for inode: %llu " "ex_status [%d/%d/%llu/%c] != " "es_status [%d/%d/%llu/%c]\n", inode->i_ino, ee_block, ee_len, ee_start, @@ -744,7 +744,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode, * that we don't want to add an written/unwritten extent. */ if (!ext4_es_is_delayed(es) && !ext4_es_is_hole(es)) { - pr_warn("ES insert assertion failed for inode: %lu " + pr_warn("ES insert assertion failed for inode: %llu " "can't find an extent at block %d but we want " "to add a written/unwritten extent " "[%d/%d/%llu/%x]\n", inode->i_ino, @@ -779,7 +779,7 @@ static void ext4_es_insert_extent_ind_check(struct inode *inode, * We want to add a delayed/hole extent but this * block has been allocated. */ - pr_warn("ES insert assertion failed for inode: %lu " + pr_warn("ES insert assertion failed for inode: %llu " "We can find blocks but we want to add a " "delayed/hole extent [%d/%d/%llu/%x]\n", inode->i_ino, es->es_lblk, es->es_len, @@ -788,13 +788,13 @@ static void ext4_es_insert_extent_ind_check(struct inode *inode, } else if (ext4_es_is_written(es)) { if (retval != es->es_len) { pr_warn("ES insert assertion failed for " - "inode: %lu retval %d != es_len %d\n", + "inode: %llu retval %d != es_len %d\n", inode->i_ino, retval, es->es_len); return; } if (map.m_pblk != ext4_es_pblock(es)) { pr_warn("ES insert assertion failed for " - "inode: %lu m_pblk %llu != " + "inode: %llu m_pblk %llu != " "es_pblk %llu\n", inode->i_ino, map.m_pblk, ext4_es_pblock(es)); @@ -809,7 +809,7 @@ static void ext4_es_insert_extent_ind_check(struct inode *inode, } } else if (retval == 0) { if (ext4_es_is_written(es)) { - pr_warn("ES insert assertion failed for inode: %lu " + pr_warn("ES insert assertion failed for inode: %llu " "We can't find the block but we want to add " "a written extent [%d/%d/%llu/%x]\n", inode->i_ino, es->es_lblk, es->es_len, @@ -919,7 +919,7 @@ void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk, if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) return; - es_debug("add [%u/%u) %llu %x %d to extent status tree of inode %lu\n", + es_debug("add [%u/%u) %llu %x %d to extent status tree of inode %llu\n", lblk, len, pblk, status, delalloc_reserve_used, inode->i_ino); if (!len) @@ -1631,7 +1631,7 @@ void ext4_es_remove_extent(struct inode *inode, ext4_lblk_t lblk, if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) return; - es_debug("remove [%u/%u) from extent status tree of inode %lu\n", + es_debug("remove [%u/%u) from extent status tree of inode %llu\n", lblk, len, inode->i_ino); if (!len) @@ -1821,7 +1821,7 @@ int ext4_seq_es_shrinker_info_show(struct seq_file *seq, void *v) seq_printf(seq, " %lu shrunk objects\n", es_stats->es_stats_shrunk); if (inode_cnt) seq_printf(seq, - "maximum:\n %lu inode (%u objects, %u reclaimable)\n" + "maximum:\n %llu inode (%u objects, %u reclaimable)\n" " %llu us max scan time\n", max->vfs_inode.i_ino, max->i_es_all_nr, max->i_es_shk_nr, div_u64(es_stats->es_stats_max_scan_time, 1000)); @@ -1998,7 +1998,7 @@ static void ext4_print_pending_tree(struct inode *inode) struct rb_node *node; struct pending_reservation *pr; - printk(KERN_DEBUG "pending reservations for inode %lu:", inode->i_ino); + printk(KERN_DEBUG "pending reservations for inode %llu:", inode->i_ino); tree = &EXT4_I(inode)->i_pending_tree; node = rb_first(&tree->root); while (node) { @@ -2214,7 +2214,7 @@ void ext4_es_insert_delayed_extent(struct inode *inode, ext4_lblk_t lblk, if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) return; - es_debug("add [%u/%u) delayed to extent status tree of inode %lu\n", + es_debug("add [%u/%u) delayed to extent status tree of inode %llu\n", lblk, len, inode->i_ino); if (!len) return; diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index f575751f1cae..0cac890cf370 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -13,6 +13,7 @@ #include "mballoc.h" #include <linux/lockdep.h> +#include <linux/wait_bit.h> /* * Ext4 Fast Commits * ----------------- @@ -55,21 +56,22 @@ * deleted while it is being flushed. * [2] Flush data buffers to disk and clear "EXT4_STATE_FC_FLUSHING_DATA" * state. - * [3] Lock the journal by calling jbd2_journal_lock_updates. This ensures that - * all the exsiting handles finish and no new handles can start. - * [4] Mark all the fast commit eligible inodes as undergoing fast commit - * by setting "EXT4_STATE_FC_COMMITTING" state. - * [5] Unlock the journal by calling jbd2_journal_unlock_updates. This allows - * starting of new handles. If new handles try to start an update on - * any of the inodes that are being committed, ext4_fc_track_inode() - * will block until those inodes have finished the fast commit. + * [3] Lock the journal by calling jbd2_journal_lock_updates(). This ensures + * that all the existing handles finish and no new handles can start. + * [4] Mark all the fast commit eligible inodes as undergoing fast commit by + * setting "EXT4_STATE_FC_COMMITTING" state, and snapshot the inode state + * needed for log writing. + * [5] Unlock the journal by calling jbd2_journal_unlock_updates(). This allows + * starting of new handles. Updates to inodes being fast committed are + * tracked for requeue rather than blocking. * [6] Commit all the directory entry updates in the fast commit space. - * [7] Commit all the changed inodes in the fast commit space and clear - * "EXT4_STATE_FC_COMMITTING" for these inodes. + * [7] Commit all the changed inodes in the fast commit space. * [8] Write tail tag (this tag ensures the atomicity, please read the following * section for more details). + * [9] Clear "EXT4_STATE_FC_COMMITTING" and wake up waiters in + * ext4_fc_cleanup(). * - * All the inode updates must be enclosed within jbd2_jounrnal_start() + * All the inode updates must be enclosed within jbd2_journal_start() * and jbd2_journal_stop() similar to JBD2 journaling. * * Fast Commit Ineligibility @@ -182,23 +184,24 @@ #include <trace/events/ext4.h> static struct kmem_cache *ext4_fc_dentry_cachep; +static struct kmem_cache *ext4_fc_range_cachep; -static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate) -{ - BUFFER_TRACE(bh, ""); - if (uptodate) { - ext4_debug("%s: Block %lld up-to-date", - __func__, bh->b_blocknr); - set_buffer_uptodate(bh); - } else { - ext4_debug("%s: Block %lld not up-to-date", - __func__, bh->b_blocknr); - clear_buffer_uptodate(bh); - } +/* + * Avoid spending unbounded time/memory snapshotting highly fragmented files + * under jbd2_journal_lock_updates(). If we exceed this limit, fall back to + * full commit. + */ +#define EXT4_FC_SNAPSHOT_MAX_INODES 1024 +#define EXT4_FC_SNAPSHOT_MAX_RANGES 2048 - unlock_buffer(bh); +static inline void ext4_fc_set_snap_err(int *snap_err, int err) +{ + if (snap_err && *snap_err == EXT4_FC_SNAP_ERR_NONE) + *snap_err = err; } +static void ext4_fc_free_inode_snap(struct inode *inode); + static inline void ext4_fc_reset_inode(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); @@ -213,9 +216,10 @@ void ext4_fc_init_inode(struct inode *inode) ext4_fc_reset_inode(inode); ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING); + ext4_clear_inode_state(inode, EXT4_STATE_FC_REQUEUE); INIT_LIST_HEAD(&ei->i_fc_list); INIT_LIST_HEAD(&ei->i_fc_dilist); - init_waitqueue_head(&ei->i_fc_wait); + ei->i_fc_snap = NULL; } static bool ext4_fc_disabled(struct super_block *sb) @@ -224,6 +228,56 @@ static bool ext4_fc_disabled(struct super_block *sb) (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)); } +static bool ext4_fc_eligible(struct super_block *sb) +{ + return !ext4_fc_disabled(sb) && + !(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)); +} + +/* + * Wait for an inode fast-commit state bit to clear while dropping the + * fast-commit lock around schedule(). + */ +static void ext4_fc_wait_inode_state(struct inode *inode, int bit, + int *alloc_ctx) +{ + wait_queue_head_t *wq; + unsigned long *wait_word = ext4_inode_state_wait_word(inode); + int wait_bit = ext4_inode_state_wait_bit(bit); + + while (ext4_test_inode_state(inode, bit)) { + DEFINE_WAIT_BIT(wait, wait_word, wait_bit); + + wq = bit_waitqueue(wait_word, wait_bit); + prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); + if (ext4_test_inode_state(inode, bit)) { + ext4_fc_unlock(inode->i_sb, *alloc_ctx); + schedule(); + *alloc_ctx = ext4_fc_lock(inode->i_sb); + } + finish_wait(wq, &wait.wq_entry); + } +} + +static inline void ext4_fc_wake_inode_state(struct inode *inode, int bit) +{ + wake_up_bit(ext4_inode_state_wait_word(inode), + ext4_inode_state_wait_bit(bit)); +} + +static void ext4_fc_snap_stats_update_max(atomic64_t *stat, u64 value) +{ + u64 old = atomic64_read(stat); + + while (value > old) { + u64 prev = atomic64_cmpxchg(stat, old, value); + + if (prev == old) + break; + old = prev; + } +} + /* * Remove inode from fast commit list. If the inode is being committed * we wait until inode commit is done. @@ -232,7 +286,6 @@ void ext4_fc_del(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_dentry_update *fc_dentry; - wait_queue_head_t *wq; int alloc_ctx; if (ext4_fc_disabled(inode->i_sb)) @@ -240,59 +293,43 @@ void ext4_fc_del(struct inode *inode) alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { + ext4_fc_free_inode_snap(inode); ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } /* - * Since ext4_fc_del is called from ext4_evict_inode while having a - * handle open, there is no need for us to wait here even if a fast - * commit is going on. That is because, if this inode is being - * committed, ext4_mark_inode_dirty would have waited for inode commit - * operation to finish before we come here. So, by the time we come - * here, inode's EXT4_STATE_FC_COMMITTING would have been cleared. So, - * we shouldn't see EXT4_STATE_FC_COMMITTING to be set on this inode - * here. - * - * We may come here without any handles open in the "no_delete" case of - * ext4_evict_inode as well. However, if that happens, we first mark the - * file system as fast commit ineligible anyway. So, even in that case, - * it is okay to remove the inode from the fc list. + * Wait for ongoing fast commit to finish. We cannot remove the inode + * from fast commit lists while it is being committed. If we wake from + * FC_FLUSHING_DATA, re-check FC_COMMITTING before deleting because the + * commit thread sets FC_COMMITTING only after clearing FLUSHING_DATA. */ - WARN_ON(ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING) - && !ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)); - while (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { -#if (BITS_PER_LONG < 64) - DEFINE_WAIT_BIT(wait, &ei->i_state_flags, - EXT4_STATE_FC_FLUSHING_DATA); - wq = bit_waitqueue(&ei->i_state_flags, - EXT4_STATE_FC_FLUSHING_DATA); -#else - DEFINE_WAIT_BIT(wait, &ei->i_flags, - EXT4_STATE_FC_FLUSHING_DATA); - wq = bit_waitqueue(&ei->i_flags, - EXT4_STATE_FC_FLUSHING_DATA); -#endif - prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); - if (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { - ext4_fc_unlock(inode->i_sb, alloc_ctx); - schedule(); - alloc_ctx = ext4_fc_lock(inode->i_sb); - } - finish_wait(wq, &wait.wq_entry); + for (;;) { + ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_COMMITTING, + &alloc_ctx); + + if (!ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) + break; + + ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA, + &alloc_ctx); } + + ext4_fc_free_inode_snap(inode); list_del_init(&ei->i_fc_list); /* - * Since this inode is getting removed, let's also remove all FC - * dentry create references, since it is not needed to log it anyways. + * Since this inode is getting removed, let's also remove all FC dentry + * create references, since it is not needed to log it anyways. */ if (list_empty(&ei->i_fc_dilist)) { ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } - fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist); + fc_dentry = list_first_entry(&ei->i_fc_dilist, + struct ext4_fc_dentry_update, + fcd_dilist); WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT); list_del_init(&fc_dentry->fcd_list); list_del_init(&fc_dentry->fcd_dilist); @@ -320,7 +357,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl if (ext4_fc_disabled(sb)) return; - if (handle && !IS_ERR(handle)) + if (!IS_ERR_OR_NULL(handle)) tid = handle->h_transaction->t_tid; else { read_lock(&sbi->s_journal->j_state_lock); @@ -364,6 +401,8 @@ static int ext4_fc_track_template( tid = handle->h_transaction->t_tid; spin_lock(&ei->i_fc_lock); + if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) + ext4_set_inode_state(inode, EXT4_STATE_FC_REQUEUE); if (tid == ei->i_sync_tid) { update = true; } else { @@ -473,13 +512,8 @@ void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry) { struct inode *inode = d_inode(dentry); - if (ext4_fc_disabled(inode->i_sb)) - return; - - if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) - return; - - __ext4_fc_track_unlink(handle, inode, dentry); + if (ext4_fc_eligible(inode->i_sb)) + __ext4_fc_track_unlink(handle, inode, dentry); } void __ext4_fc_track_link(handle_t *handle, @@ -496,17 +530,11 @@ void __ext4_fc_track_link(handle_t *handle, trace_ext4_fc_track_link(handle, inode, dentry, ret); } -void ext4_fc_track_link(handle_t *handle, struct dentry *dentry) +void ext4_fc_track_link(handle_t *handle, struct inode *inode, + struct dentry *dentry) { - struct inode *inode = d_inode(dentry); - - if (ext4_fc_disabled(inode->i_sb)) - return; - - if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) - return; - - __ext4_fc_track_link(handle, inode, dentry); + if (ext4_fc_eligible(inode->i_sb)) + __ext4_fc_track_link(handle, inode, dentry); } void __ext4_fc_track_create(handle_t *handle, struct inode *inode, @@ -527,13 +555,8 @@ void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) { struct inode *inode = d_inode(dentry); - if (ext4_fc_disabled(inode->i_sb)) - return; - - if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) - return; - - __ext4_fc_track_create(handle, inode, dentry); + if (ext4_fc_eligible(inode->i_sb)) + __ext4_fc_track_create(handle, inode, dentry); } /* __track_fn for inode tracking */ @@ -550,49 +573,26 @@ static int __track_inode(handle_t *handle, struct inode *inode, void *arg, void ext4_fc_track_inode(handle_t *handle, struct inode *inode) { - struct ext4_inode_info *ei = EXT4_I(inode); - wait_queue_head_t *wq; int ret; if (S_ISDIR(inode->i_mode)) return; - if (ext4_fc_disabled(inode->i_sb)) - return; - if (ext4_should_journal_data(inode)) { ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_INODE_JOURNAL_DATA, handle); return; } - if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) + if (!ext4_fc_eligible(inode->i_sb)) return; /* - * If we come here, we may sleep while waiting for the inode to - * commit. We shouldn't be holding i_data_sem when we go to sleep since - * the commit path needs to grab the lock while committing the inode. + * Fast commit snapshots inode state at commit time, so there's no need + * to wait for EXT4_STATE_FC_COMMITTING here. If the inode is already + * on the commit queue, ext4_fc_cleanup() will requeue it for the new + * transaction once the current commit finishes. */ - lockdep_assert_not_held(&ei->i_data_sem); - - while (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) { -#if (BITS_PER_LONG < 64) - DEFINE_WAIT_BIT(wait, &ei->i_state_flags, - EXT4_STATE_FC_COMMITTING); - wq = bit_waitqueue(&ei->i_state_flags, - EXT4_STATE_FC_COMMITTING); -#else - DEFINE_WAIT_BIT(wait, &ei->i_flags, - EXT4_STATE_FC_COMMITTING); - wq = bit_waitqueue(&ei->i_flags, - EXT4_STATE_FC_COMMITTING); -#endif - prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); - if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) - schedule(); - finish_wait(wq, &wait.wq_entry); - } /* * From this point on, this inode will not be committed either @@ -616,7 +616,7 @@ static int __track_range(handle_t *handle, struct inode *inode, void *arg, (struct __track_range_args *)arg; if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) { - ext4_debug("Special inode %ld being modified\n", inode->i_ino); + ext4_debug("Special inode %llu being modified\n", inode->i_ino); return -ECANCELED; } @@ -644,10 +644,7 @@ void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t star if (S_ISDIR(inode->i_mode)) return; - if (ext4_fc_disabled(inode->i_sb)) - return; - - if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) + if (!ext4_fc_eligible(inode->i_sb)) return; if (ext4_has_inline_data(inode)) { @@ -675,8 +672,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) lock_buffer(bh); set_buffer_dirty(bh); set_buffer_uptodate(bh); - bh->b_end_io = ext4_end_buffer_io_sync; - submit_bh(REQ_OP_WRITE | write_flags, bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); EXT4_SB(sb)->s_fc_bh = NULL; } @@ -845,6 +841,21 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, return true; } +struct ext4_fc_range { + struct list_head list; + u16 tag; + ext4_lblk_t lblk; + ext4_lblk_t len; + ext4_fsblk_t pblk; + bool unwritten; +}; + +struct ext4_fc_inode_snap { + struct list_head data_list; + unsigned int inode_len; + u8 inode_buf[]; +}; + /* * Writes inode in the fast commit space under TLV with tag @tag. * Returns 0 on success, error on failure. @@ -852,21 +863,27 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, static int ext4_fc_write_inode(struct inode *inode, u32 *crc) { struct ext4_inode_info *ei = EXT4_I(inode); - int inode_len = EXT4_GOOD_OLD_INODE_SIZE; - int ret; - struct ext4_iloc iloc; + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; struct ext4_fc_inode fc_inode; struct ext4_fc_tl tl; u8 *dst; + u8 *src; + int inode_len; + int ret; - ret = ext4_get_inode_loc(inode, &iloc); - if (ret) - return ret; + if (!snap) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } - if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) - inode_len = EXT4_INODE_SIZE(inode->i_sb); - else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) - inode_len += ei->i_extra_isize; + src = snap->inode_buf; + inode_len = snap->inode_len; + if (!src || inode_len == 0) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } fc_inode.fc_ino = cpu_to_le32(inode->i_ino); tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); @@ -882,10 +899,9 @@ static int ext4_fc_write_inode(struct inode *inode, u32 *crc) dst += EXT4_FC_TAG_BASE_LEN; memcpy(dst, &fc_inode, sizeof(fc_inode)); dst += sizeof(fc_inode); - memcpy(dst, (u8 *)ext4_raw_inode(&iloc), inode_len); + memcpy(dst, src, inode_len); ret = 0; err: - brelse(iloc.bh); return ret; } @@ -895,76 +911,244 @@ err: */ static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc) { - ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size; struct ext4_inode_info *ei = EXT4_I(inode); - struct ext4_map_blocks map; + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; struct ext4_fc_add_range fc_ext; struct ext4_fc_del_range lrange; struct ext4_extent *ex; - int ret; + struct ext4_fc_range *range; + + if (!snap) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } + + list_for_each_entry(range, &snap->data_list, list) { + if (range->tag == EXT4_FC_TAG_DEL_RANGE) { + lrange.fc_ino = cpu_to_le32(inode->i_ino); + lrange.fc_lblk = cpu_to_le32(range->lblk); + lrange.fc_len = cpu_to_le32(range->len); + if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, + sizeof(lrange), (u8 *)&lrange, crc)) + return -ENOSPC; + continue; + } + + fc_ext.fc_ino = cpu_to_le32(inode->i_ino); + ex = (struct ext4_extent *)&fc_ext.fc_ex; + ex->ee_block = cpu_to_le32(range->lblk); + ex->ee_len = cpu_to_le16(range->len); + ext4_ext_store_pblock(ex, range->pblk); + if (range->unwritten) + ext4_ext_mark_unwritten(ex); + else + ext4_ext_mark_initialized(ex); + + if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, + sizeof(fc_ext), (u8 *)&fc_ext, crc)) + return -ENOSPC; + } + + return 0; +} + +static void ext4_fc_free_ranges(struct list_head *head) +{ + struct ext4_fc_range *range, *range_n; + + list_for_each_entry_safe(range, range_n, head, list) { + list_del(&range->list); + kmem_cache_free(ext4_fc_range_cachep, range); + } +} + +static void ext4_fc_free_inode_snap(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + + if (!snap) + return; + + ext4_fc_free_ranges(&snap->data_list); + kfree(snap); + ei->i_fc_snap = NULL; +} + +static int ext4_fc_snapshot_inode_data(struct inode *inode, + struct list_head *ranges, + unsigned int nr_ranges_total, + unsigned int *nr_rangesp, + int *snap_err) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + ext4_lblk_t start_lblk, end_lblk, cur_lblk; + unsigned int nr_ranges = 0; spin_lock(&ei->i_fc_lock); if (ei->i_fc_lblk_len == 0) { spin_unlock(&ei->i_fc_lock); + if (nr_rangesp) + *nr_rangesp = 0; return 0; } - old_blk_size = ei->i_fc_lblk_start; - new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; + start_lblk = ei->i_fc_lblk_start; + end_lblk = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; ei->i_fc_lblk_len = 0; spin_unlock(&ei->i_fc_lock); - cur_lblk_off = old_blk_size; - ext4_debug("will try writing %d to %d for inode %ld\n", - cur_lblk_off, new_blk_size, inode->i_ino); + cur_lblk = start_lblk; + ext4_debug("snapshot data ranges %u-%u for inode %llu\n", + start_lblk, end_lblk, + (unsigned long long)inode->i_ino); + + while (cur_lblk <= end_lblk) { + struct extent_status es; + struct ext4_fc_range *range; + ext4_lblk_t len; + u64 remaining = (u64)end_lblk - cur_lblk + 1; + + if (!ext4_es_lookup_extent(inode, cur_lblk, NULL, &es, NULL)) { + atomic64_inc(&stats->snap_fail_es_miss); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_MISS); + return -EAGAIN; + } - while (cur_lblk_off <= new_blk_size) { - map.m_lblk = cur_lblk_off; - map.m_len = new_blk_size - cur_lblk_off + 1; - ret = ext4_map_blocks(NULL, inode, &map, - EXT4_GET_BLOCKS_IO_SUBMIT | - EXT4_EX_NOCACHE); - if (ret < 0) - return -ECANCELED; + if (ext4_es_is_delayed(&es)) { + atomic64_inc(&stats->snap_fail_es_delayed); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_ES_DELAYED); + return -EAGAIN; + } - if (map.m_len == 0) { - cur_lblk_off++; + len = es.es_len - (cur_lblk - es.es_lblk); + if (len > remaining) + len = remaining; + if (len == 0) { + cur_lblk++; continue; } - if (ret == 0) { - lrange.fc_ino = cpu_to_le32(inode->i_ino); - lrange.fc_lblk = cpu_to_le32(map.m_lblk); - lrange.fc_len = cpu_to_le32(map.m_len); - if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, - sizeof(lrange), (u8 *)&lrange, crc)) - return -ENOSPC; + if (nr_ranges_total + nr_ranges >= EXT4_FC_SNAPSHOT_MAX_RANGES) { + atomic64_inc(&stats->snap_fail_ranges_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_RANGES_CAP); + return -E2BIG; + } + + range = kmem_cache_alloc(ext4_fc_range_cachep, GFP_NOFS); + if (!range) { + atomic64_inc(&stats->snap_fail_nomem); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); + return -ENOMEM; + } + nr_ranges++; + + range->lblk = cur_lblk; + range->len = len; + range->pblk = 0; + range->unwritten = false; + + if (ext4_es_is_hole(&es)) { + range->tag = EXT4_FC_TAG_DEL_RANGE; + } else if (ext4_es_is_written(&es) || + ext4_es_is_unwritten(&es)) { + unsigned int max; + + range->tag = EXT4_FC_TAG_ADD_RANGE; + range->pblk = ext4_es_pblock(&es) + + (cur_lblk - es.es_lblk); + range->unwritten = ext4_es_is_unwritten(&es); + + max = range->unwritten ? EXT_UNWRITTEN_MAX_LEN : + EXT_INIT_MAX_LEN; + if (range->len > max) + range->len = max; } else { - unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ? - EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN; - - /* Limit the number of blocks in one extent */ - map.m_len = min(max, map.m_len); - - fc_ext.fc_ino = cpu_to_le32(inode->i_ino); - ex = (struct ext4_extent *)&fc_ext.fc_ex; - ex->ee_block = cpu_to_le32(map.m_lblk); - ex->ee_len = cpu_to_le16(map.m_len); - ext4_ext_store_pblock(ex, map.m_pblk); - if (map.m_flags & EXT4_MAP_UNWRITTEN) - ext4_ext_mark_unwritten(ex); - else - ext4_ext_mark_initialized(ex); - if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, - sizeof(fc_ext), (u8 *)&fc_ext, crc)) - return -ENOSPC; + kmem_cache_free(ext4_fc_range_cachep, range); + atomic64_inc(&stats->snap_fail_es_other); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_OTHER); + return -EAGAIN; } - cur_lblk_off += map.m_len; + INIT_LIST_HEAD(&range->list); + list_add_tail(&range->list, ranges); + + if ((u64)range->len > (u64)end_lblk - cur_lblk) + break; + + cur_lblk += range->len; } + if (nr_rangesp) + *nr_rangesp = nr_ranges; return 0; } +static int ext4_fc_snapshot_inode(struct inode *inode, + unsigned int nr_ranges_total, + unsigned int *nr_rangesp, int *snap_err) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + struct ext4_fc_inode_snap *snap; + int inode_len = EXT4_GOOD_OLD_INODE_SIZE; + struct ext4_iloc iloc; + LIST_HEAD(ranges); + unsigned int nr_ranges = 0; + int ret; + int alloc_ctx; + + ret = ext4_get_inode_loc_noio(inode, &iloc); + if (ret) { + atomic64_inc(&stats->snap_fail_inode_loc); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_INODE_LOC); + return ret; + } + + if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) + inode_len = EXT4_INODE_SIZE(inode->i_sb); + else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) + inode_len += ei->i_extra_isize; + + snap = kmalloc_flex(*snap, inode_buf, inode_len, GFP_NOFS); + if (!snap) { + atomic64_inc(&stats->snap_fail_nomem); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); + brelse(iloc.bh); + return -ENOMEM; + } + INIT_LIST_HEAD(&snap->data_list); + snap->inode_len = inode_len; + + memcpy(snap->inode_buf, (u8 *)ext4_raw_inode(&iloc), inode_len); + brelse(iloc.bh); + + ret = ext4_fc_snapshot_inode_data(inode, &ranges, nr_ranges_total, + &nr_ranges, snap_err); + if (ret) { + kfree(snap); + ext4_fc_free_ranges(&ranges); + return ret; + } + + alloc_ctx = ext4_fc_lock(inode->i_sb); + ext4_fc_free_inode_snap(inode); + ei->i_fc_snap = snap; + list_splice_tail_init(&ranges, &snap->data_list); + ext4_fc_unlock(inode->i_sb, alloc_ctx); + + atomic64_inc(&stats->snap_inodes); + atomic64_add(nr_ranges, &stats->snap_ranges); + if (nr_rangesp) + *nr_rangesp = nr_ranges; + return 0; +} /* Flushes data of all the inodes in the commit queue. */ static int ext4_fc_flush_data(journal_t *journal) @@ -975,13 +1159,13 @@ static int ext4_fc_flush_data(journal_t *journal) int ret = 0; list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { - ret = jbd2_submit_inode_data(journal, ei->jinode); + ret = jbd2_submit_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; } list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { - ret = jbd2_wait_inode_data(journal, ei->jinode); + ret = jbd2_wait_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; } @@ -1015,6 +1199,11 @@ static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc) */ if (list_empty(&fc_dentry->fcd_dilist)) continue; + /* + * For EXT4_FC_TAG_CREAT, fcd_dilist is linked on the created + * inode's i_fc_dilist list (kept singular), so we can recover the + * inode through it. + */ ei = list_first_entry(&fc_dentry->fcd_dilist, struct ext4_inode_info, i_fc_dilist); inode = &ei->vfs_inode; @@ -1039,17 +1228,114 @@ static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc) return 0; } -static int ext4_fc_perform_commit(journal_t *journal) +static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, + struct inode ***inodesp, + unsigned int *nr_inodesp); + +static int ext4_fc_snapshot_inodes(journal_t *journal, struct inode **inodes, + unsigned int inodes_size, + unsigned int *nr_inodesp, + unsigned int *nr_rangesp, + int *snap_err) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_inode_info *iter; + struct ext4_fc_dentry_update *fc_dentry; + unsigned int i = 0; + unsigned int idx; + unsigned int nr_ranges = 0; + int ret = 0; + int alloc_ctx; + + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + if (i >= inodes_size) { + atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_INODES_CAP); + ret = -E2BIG; + goto unlock; + } + inodes[i++] = &iter->vfs_inode; + } + + list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { + struct ext4_inode_info *ei; + struct inode *inode; + + if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) + continue; + if (list_empty(&fc_dentry->fcd_dilist)) + continue; + + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, i_fc_dilist); + inode = &ei->vfs_inode; + if (!list_empty(&ei->i_fc_list)) + continue; + + if (i >= inodes_size) { + atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_INODES_CAP); + ret = -E2BIG; + goto unlock; + } + /* + * Create-only inodes may only be referenced via fcd_dilist and + * not appear on s_fc_q[MAIN]. They may hit the last iput while + * we are snapshotting, but inode eviction calls ext4_fc_del(), + * which waits for FC_COMMITTING to clear. Mark them FC_COMMITTING + * so the inode stays pinned and the snapshot stays valid until + * ext4_fc_cleanup(). + */ + ext4_set_inode_state(inode, EXT4_STATE_FC_COMMITTING); + inodes[i++] = inode; + } +unlock: + ext4_fc_unlock(sb, alloc_ctx); + + if (ret) + return ret; + + for (idx = 0; idx < i; idx++) { + unsigned int inode_ranges = 0; + + ret = ext4_fc_snapshot_inode(inodes[idx], nr_ranges, + &inode_ranges, snap_err); + if (ret) + break; + nr_ranges += inode_ranges; + } + + if (nr_inodesp) + *nr_inodesp = idx; + if (nr_rangesp) + *nr_rangesp = nr_ranges; + return ret; +} + +static int ext4_fc_perform_commit(journal_t *journal, tid_t commit_tid) { struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; struct ext4_inode_info *iter; struct ext4_fc_head head; struct inode *inode; + struct inode **inodes; + unsigned int inodes_size; + unsigned int snap_inodes = 0; + unsigned int snap_ranges = 0; + int snap_err = EXT4_FC_SNAP_ERR_NONE; struct blk_plug plug; int ret = 0; u32 crc = 0; int alloc_ctx; + ktime_t lock_start; + u64 locked_ns; /* * Step 1: Mark all inodes on s_fc_q[MAIN] with @@ -1075,11 +1361,8 @@ static int ext4_fc_perform_commit(journal_t *journal) list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); -#if (BITS_PER_LONG < 64) - wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_FLUSHING_DATA); -#else - wake_up_bit(&iter->i_flags, EXT4_STATE_FC_FLUSHING_DATA); -#endif + ext4_fc_wake_inode_state(&iter->vfs_inode, + EXT4_STATE_FC_FLUSHING_DATA); } /* @@ -1097,13 +1380,23 @@ static int ext4_fc_perform_commit(journal_t *journal) if (ret) return ret; + ret = ext4_fc_alloc_snapshot_inodes(sb, &inodes, &inodes_size); + if (ret) { + if (ret == -E2BIG) + atomic64_inc(&snap_stats->snap_fail_inodes_cap); + else if (ret == -ENOMEM) + atomic64_inc(&snap_stats->snap_fail_nomem); + return ret; + } /* Step 4: Mark all inodes as being committed. */ jbd2_journal_lock_updates(journal); + lock_start = ktime_get(); /* * The journal is now locked. No more handles can start and all the - * previous handles are now drained. We now mark the inodes on the - * commit queue as being committed. + * previous handles are now drained. Snapshotting happens in this + * window so log writing can consume only stable snapshots without + * doing logical-to-physical mapping. */ alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { @@ -1111,7 +1404,22 @@ static int ext4_fc_perform_commit(journal_t *journal) EXT4_STATE_FC_COMMITTING); } ext4_fc_unlock(sb, alloc_ctx); + + ret = ext4_fc_snapshot_inodes(journal, inodes, inodes_size, + &snap_inodes, &snap_ranges, &snap_err); jbd2_journal_unlock_updates(journal); + locked_ns = ktime_to_ns(ktime_sub(ktime_get(), lock_start)); + atomic64_add(locked_ns, &snap_stats->lock_updates_ns_total); + atomic64_inc(&snap_stats->lock_updates_samples); + ext4_fc_snap_stats_update_max(&snap_stats->lock_updates_ns_max, + locked_ns); + if (trace_ext4_fc_lock_updates_enabled()) + trace_call__ext4_fc_lock_updates(sb, commit_tid, locked_ns, + snap_inodes, snap_ranges, + ret, snap_err); + kvfree(inodes); + if (ret) + return ret; /* * Step 5: If file system device is different from journal device, @@ -1165,6 +1473,64 @@ out: return ret; } +static unsigned int ext4_fc_count_snapshot_inodes(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_inode_info *iter; + struct ext4_fc_dentry_update *fc_dentry; + unsigned int nr_inodes = 0; + int alloc_ctx; + + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) + nr_inodes++; + + list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { + struct ext4_inode_info *ei; + + if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) + continue; + if (list_empty(&fc_dentry->fcd_dilist)) + continue; + + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, i_fc_dilist); + if (!list_empty(&ei->i_fc_list)) + continue; + + nr_inodes++; + } + ext4_fc_unlock(sb, alloc_ctx); + + return nr_inodes; +} + +static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, + struct inode ***inodesp, + unsigned int *nr_inodesp) +{ + unsigned int nr_inodes = ext4_fc_count_snapshot_inodes(sb); + struct inode **inodes; + + *inodesp = NULL; + *nr_inodesp = 0; + + if (!nr_inodes) + return 0; + + if (nr_inodes > EXT4_FC_SNAPSHOT_MAX_INODES) + return -E2BIG; + + inodes = kvzalloc_objs(*inodes, nr_inodes, GFP_NOFS); + if (!inodes) + return -ENOMEM; + + *inodesp = inodes; + *nr_inodesp = nr_inodes; + return 0; +} + static void ext4_fc_update_stats(struct super_block *sb, int status, u64 commit_time, int nblks, tid_t commit_tid) { @@ -1255,9 +1621,12 @@ restart_fc: journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO; set_task_ioprio(current, journal_ioprio); fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; - ret = ext4_fc_perform_commit(journal); + ret = ext4_fc_perform_commit(journal, commit_tid); if (ret < 0) { - status = EXT4_FC_STATUS_FAILED; + if (ret == -EAGAIN || ret == -E2BIG || ret == -ECANCELED) + status = EXT4_FC_STATUS_INELIGIBLE; + else + status = EXT4_FC_STATUS_FAILED; goto fallback; } nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; @@ -1304,45 +1673,66 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) alloc_ctx = ext4_fc_lock(sb); while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { + bool requeue; + ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], struct ext4_inode_info, i_fc_list); list_del_init(&ei->i_fc_list); + ext4_fc_free_inode_snap(&ei->vfs_inode); + spin_lock(&ei->i_fc_lock); + if (full) + requeue = !tid_geq(tid, ei->i_sync_tid); + else + requeue = ext4_test_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_REQUEUE); + if (!requeue) + ext4_fc_reset_inode(&ei->vfs_inode); + ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_REQUEUE); ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); - if (tid_geq(tid, ei->i_sync_tid)) { - ext4_fc_reset_inode(&ei->vfs_inode); - } else if (full) { - /* - * We are called after a full commit, inode has been - * modified while the commit was running. Re-enqueue - * the inode into STAGING, which will then be splice - * back into MAIN. This cannot happen during - * fastcommit because the journal is locked all the - * time in that case (and tid doesn't increase so - * tid check above isn't reliable). - */ + spin_unlock(&ei->i_fc_lock); + if (requeue) list_add_tail(&ei->i_fc_list, &sbi->s_fc_q[FC_Q_STAGING]); - } /* * Make sure clearing of EXT4_STATE_FC_COMMITTING is * visible before we send the wakeup. Pairs with implicit - * barrier in prepare_to_wait() in ext4_fc_track_inode(). + * barrier in prepare_to_wait() in ext4_fc_del(). */ smp_mb(); -#if (BITS_PER_LONG < 64) - wake_up_bit(&ei->i_state_flags, EXT4_STATE_FC_COMMITTING); -#else - wake_up_bit(&ei->i_flags, EXT4_STATE_FC_COMMITTING); -#endif + ext4_fc_wake_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); } while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], - struct ext4_fc_dentry_update, - fcd_list); + struct ext4_fc_dentry_update, + fcd_list); list_del_init(&fc_dentry->fcd_list); + if (fc_dentry->fcd_op == EXT4_FC_TAG_CREAT && + !list_empty(&fc_dentry->fcd_dilist)) { + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, + i_fc_dilist); + ext4_fc_free_inode_snap(&ei->vfs_inode); + spin_lock(&ei->i_fc_lock); + ext4_clear_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_REQUEUE); + ext4_clear_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + spin_unlock(&ei->i_fc_lock); + /* + * Make sure clearing of EXT4_STATE_FC_COMMITTING is + * visible before we send the wakeup. Pairs with + * implicit barrier in prepare_to_wait() in + * ext4_fc_del(). + */ + smp_mb(); + ext4_fc_wake_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + } list_del_init(&fc_dentry->fcd_dilist); release_dentry_name_snapshot(&fc_dentry->fcd_name); @@ -1446,7 +1836,6 @@ static int ext4_fc_replay_link_internal(struct super_block *sb, struct inode *inode) { struct inode *dir = NULL; - struct dentry *dentry_dir = NULL, *dentry_inode = NULL; struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len); int ret = 0; @@ -1457,21 +1846,7 @@ static int ext4_fc_replay_link_internal(struct super_block *sb, goto out; } - dentry_dir = d_obtain_alias(dir); - if (IS_ERR(dentry_dir)) { - ext4_debug("Failed to obtain dentry"); - dentry_dir = NULL; - goto out; - } - - dentry_inode = d_alloc(dentry_dir, &qstr_dname); - if (!dentry_inode) { - ext4_debug("Inode dentry not created."); - ret = -ENOMEM; - goto out; - } - - ret = __ext4_link(dir, inode, dentry_inode); + ret = __ext4_link(dir, inode, &qstr_dname, NULL); /* * It's possible that link already existed since data blocks * for the dir in question got persisted before we crashed OR @@ -1485,16 +1860,8 @@ static int ext4_fc_replay_link_internal(struct super_block *sb, ret = 0; out: - if (dentry_dir) { - d_drop(dentry_dir); - dput(dentry_dir); - } else if (dir) { + if (dir) iput(dir); - } - if (dentry_inode) { - d_drop(dentry_inode); - dput(dentry_inode); - } return ret; } @@ -1613,19 +1980,21 @@ static int ext4_fc_replay_inode(struct super_block *sb, /* Immediately update the inode on disk. */ ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); if (ret) - goto out; + goto out_brelse; ret = sync_dirty_buffer(iloc.bh); if (ret) - goto out; + goto out_brelse; ret = ext4_mark_inode_used(sb, ino); if (ret) - goto out; + goto out_brelse; /* Given that we just wrote the inode on disk, this SHOULD succeed. */ inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode not found."); - return -EFSCORRUPTED; + inode = NULL; + ret = -EFSCORRUPTED; + goto out_brelse; } /* @@ -1642,13 +2011,14 @@ static int ext4_fc_replay_inode(struct super_block *sb, ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode)); ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); sync_dirty_buffer(iloc.bh); +out_brelse: brelse(iloc.bh); out: iput(inode); if (!ret) blkdev_issue_flush(sb->s_bdev); - return 0; + return ret; } /* @@ -1756,8 +2126,7 @@ int ext4_fc_record_regions(struct super_block *sb, int ino, } /* Replay add range tag */ -static int ext4_fc_replay_add_range(struct super_block *sb, - struct ext4_fc_tl_mem *tl, u8 *val) +static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) { struct ext4_fc_add_range fc_add_ex; struct ext4_extent newex, *ex; @@ -1792,7 +2161,7 @@ static int ext4_fc_replay_add_range(struct super_block *sb, cur = start; remaining = len; - ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n", + ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %llu\n", start, start_pblk, len, ext4_ext_is_unwritten(ex), inode->i_ino); @@ -1808,8 +2177,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, if (ret == 0) { /* Range is not mapped */ path = ext4_find_extent(inode, cur, path, 0); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } memset(&newex, 0, sizeof(newex)); newex.ee_block = cpu_to_le32(cur); ext4_ext_store_pblock( @@ -1821,8 +2193,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, path = ext4_ext_insert_extent(NULL, inode, path, &newex, 0); up_write((&EXT4_I(inode)->i_data_sem)); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } goto next; } @@ -1869,16 +2244,16 @@ next: } ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); + ret = 0; out: ext4_free_ext_path(path); iput(inode); - return 0; + return ret; } /* Replay DEL_RANGE tag */ static int -ext4_fc_replay_del_range(struct super_block *sb, - struct ext4_fc_tl_mem *tl, u8 *val) +ext4_fc_replay_del_range(struct super_block *sb, u8 *val) { struct inode *inode; struct ext4_fc_del_range lrange; @@ -1903,7 +2278,7 @@ ext4_fc_replay_del_range(struct super_block *sb, if (ret) goto out; - ext4_debug("DEL_RANGE, inode %ld, lblk %d, len %d\n", + ext4_debug("DEL_RANGE, inode %llu, lblk %d, len %d\n", inode->i_ino, le32_to_cpu(lrange.fc_lblk), le32_to_cpu(lrange.fc_len)); while (remaining > 0) { @@ -1933,9 +2308,10 @@ ext4_fc_replay_del_range(struct super_block *sb, ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); ext4_mark_inode_dirty(NULL, inode); + ret = 0; out: iput(inode); - return 0; + return ret; } static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) @@ -2248,13 +2624,13 @@ static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, ret = ext4_fc_replay_unlink(sb, &tl, val); break; case EXT4_FC_TAG_ADD_RANGE: - ret = ext4_fc_replay_add_range(sb, &tl, val); + ret = ext4_fc_replay_add_range(sb, val); break; case EXT4_FC_TAG_CREAT: ret = ext4_fc_replay_create(sb, &tl, val); break; case EXT4_FC_TAG_DEL_RANGE: - ret = ext4_fc_replay_del_range(sb, &tl, val); + ret = ext4_fc_replay_del_range(sb, val); break; case EXT4_FC_TAG_INODE: ret = ext4_fc_replay_inode(sb, &tl, val); @@ -2316,11 +2692,26 @@ int ext4_fc_info_show(struct seq_file *seq, void *v) { struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private); struct ext4_fc_stats *stats = &sbi->s_fc_stats; + struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; + u64 lock_avg_ns = 0; + u64 lock_updates_samples; + u64 lock_updates_ns_total; + u64 lock_updates_ns_max; int i; if (v != SEQ_START_TOKEN) return 0; + lock_updates_samples = + atomic64_read(&snap_stats->lock_updates_samples); + lock_updates_ns_total = + atomic64_read(&snap_stats->lock_updates_ns_total); + lock_updates_ns_max = + atomic64_read(&snap_stats->lock_updates_ns_max); + if (lock_updates_samples) + lock_avg_ns = div64_u64(lock_updates_ns_total, + lock_updates_samples); + seq_printf(seq, "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n", stats->fc_num_commits, stats->fc_ineligible_commits, @@ -2331,6 +2722,23 @@ int ext4_fc_info_show(struct seq_file *seq, void *v) seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i], stats->fc_ineligible_reason_count[i]); + seq_printf(seq, + "Snapshot stats:\n%llu inodes\n%llu ranges\n%lluus lock_updates_avg\n%lluus lock_updates_max\n", + atomic64_read(&snap_stats->snap_inodes), + atomic64_read(&snap_stats->snap_ranges), + div_u64(lock_avg_ns, 1000), + div_u64(lock_updates_ns_max, 1000)); + seq_printf(seq, + "Snapshot failures:\n%llu es_miss\n%llu es_delayed\n%llu es_other\n%llu inodes_cap\n%llu ranges_cap\n%llu nomem\n%llu inode_loc\n%llu no_snap\n", + atomic64_read(&snap_stats->snap_fail_es_miss), + atomic64_read(&snap_stats->snap_fail_es_delayed), + atomic64_read(&snap_stats->snap_fail_es_other), + atomic64_read(&snap_stats->snap_fail_inodes_cap), + atomic64_read(&snap_stats->snap_fail_ranges_cap), + atomic64_read(&snap_stats->snap_fail_nomem), + atomic64_read(&snap_stats->snap_fail_inode_loc), + atomic64_read(&snap_stats->snap_fail_no_snap)); + return 0; } @@ -2339,13 +2747,20 @@ int __init ext4_fc_init_dentry_cache(void) ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update, SLAB_RECLAIM_ACCOUNT); - if (ext4_fc_dentry_cachep == NULL) + if (!ext4_fc_dentry_cachep) + return -ENOMEM; + + ext4_fc_range_cachep = KMEM_CACHE(ext4_fc_range, SLAB_RECLAIM_ACCOUNT); + if (!ext4_fc_range_cachep) { + kmem_cache_destroy(ext4_fc_dentry_cachep); return -ENOMEM; + } return 0; } void ext4_fc_destroy_dentry_cache(void) { + kmem_cache_destroy(ext4_fc_range_cachep); kmem_cache_destroy(ext4_fc_dentry_cachep); } diff --git a/fs/ext4/file.c b/fs/ext4/file.c index f1dc5ce791a7..374b4bc25bd5 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -27,7 +27,6 @@ #include <linux/dax.h> #include <linux/filelock.h> #include <linux/quotaops.h> -#include <linux/pagevec.h> #include <linux/uio.h> #include <linux/mman.h> #include <linux/backing-dev.h> @@ -92,7 +91,9 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) return generic_file_read_iter(iocb, to); } - ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); + ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin); + if (ret == -ENOTBLK) + ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); @@ -214,31 +215,60 @@ ext4_extending_io(struct inode *inode, loff_t offset, size_t len) return false; } -/* Is IO overwriting allocated or initialized blocks? */ -static bool ext4_overwrite_io(struct inode *inode, - loff_t pos, loff_t len, bool *unwritten) +/* + * Does an unaligned DIO write require partial block zeroing? + * + * Partial block zeroing is performed only for the head and tail blocks + * when they are partially covered by the write and the underlying extent + * is a hole or unwritten. Middle blocks (fully covered by the write) + * are written as whole blocks without zeroing. + * + * When zeroing is required, two concurrent unaligned DIO writes to the + * same partial block can race and corrupt each other's data, so the + * caller must take the exclusive i_rwsem and drain in-flight DIO. When + * zeroing is not required, shared lock is safe -- block allocation and + * unwritten conversion for middle blocks are protected by i_data_sem + * and inode_dio_begin(). + */ +static bool ext4_dio_needs_zeroing(struct inode *inode, loff_t pos, loff_t len) { struct ext4_map_blocks map; unsigned int blkbits = inode->i_blkbits; - int err, blklen; + unsigned long blockmask = inode->i_sb->s_blocksize - 1; + bool head_partial, tail_partial; + ext4_lblk_t head_lblk, tail_lblk; + int err; if (pos + len > i_size_read(inode)) - return false; + return true; - map.m_lblk = pos >> blkbits; - map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits); - blklen = map.m_len; + head_partial = (pos & blockmask) != 0; + tail_partial = ((pos + len) & blockmask) != 0; + head_lblk = pos >> blkbits; + tail_lblk = (pos + len - 1) >> blkbits; + + /* Check the head partial block. */ + if (head_partial) { + map.m_lblk = head_lblk; + map.m_len = tail_lblk - head_lblk + 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + /* If this mapping already covers the tail block, we're done. */ + if (!tail_partial || map.m_lblk + err > tail_lblk) + return false; + } - err = ext4_map_blocks(NULL, inode, &map, 0); - if (err != blklen) - return false; - /* - * 'err==len' means that all of the blocks have been preallocated, - * regardless of whether they have been initialized or not. We need to - * check m_flags to distinguish the unwritten extents. - */ - *unwritten = !(map.m_flags & EXT4_MAP_MAPPED); - return true; + /* Check the tail partial block. */ + if (tail_partial) { + map.m_lblk = tail_lblk; + map.m_len = 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + } + + return false; } static ssize_t ext4_generic_write_checks(struct kiocb *iocb, @@ -271,15 +301,32 @@ static ssize_t ext4_generic_write_checks(struct kiocb *iocb, static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from) { + struct inode *inode = file_inode(iocb->ki_filp); + loff_t old_size = i_size_read(inode); ssize_t ret, count; count = ext4_generic_write_checks(iocb, from); if (count <= 0) return count; - ret = file_modified(iocb->ki_filp); + ret = kiocb_modified(iocb); if (ret) return ret; + + /* + * If the position is beyond the EOF, it is necessary to zero out the + * partial block that beyond the existing EOF, as it may contains + * stale data written through mmap. + */ + if (iocb->ki_pos > old_size && !ext4_verity_in_progress(inode)) { + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + + ret = ext4_block_zero_eof(inode, old_size, iocb->ki_pos); + if (ret) + return ret; + } + return count; } @@ -293,6 +340,13 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; inode_lock(inode); + + /* + * Prevent concurrent direct I/O and buffered I/O to the same file + * range. Wait for in-flight DIO to finish before dirtying pages. + */ + inode_dio_wait(inode); + ret = ext4_write_checks(iocb, from); if (ret <= 0) goto out; @@ -384,7 +438,8 @@ static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size, error = ext4_convert_unwritten_extents_atomic(NULL, inode, pos, size); else if (!error && size && flags & IOMAP_DIO_UNWRITTEN) - error = ext4_convert_unwritten_extents(NULL, inode, pos, size); + error = ext4_convert_unwritten_extents(NULL, inode, pos, size, + NULL); if (error) return error; /* @@ -412,16 +467,28 @@ static const struct iomap_dio_ops ext4_dio_write_ops = { * condition requires an exclusive inode lock. If yes, then we restart the * whole operation by releasing the shared lock and acquiring exclusive lock. * - * - For unaligned_io we never take shared lock as it may cause data corruption - * when two unaligned IO tries to modify the same block e.g. while zeroing. + * The decision is layered, evaluated in this order: * - * - For extending writes case we don't take the shared lock, since it requires - * updating inode i_disksize and/or orphan handling with exclusive lock. + * 1. If kiocb_modified() needs to update security info (!IS_NOSEC), upgrade + * to the exclusive lock -- the security update itself requires it, + * regardless of whether the write extends the file or is aligned. * - * - shared locking will only be true mostly with overwrites, including - * initialized blocks and unwritten blocks. + * 2. If the write extends i_size or i_disksize, upgrade to the exclusive + * lock to safely update i_disksize and the orphan list, regardless of + * alignment. * - * - Otherwise we will switch to exclusive i_rwsem lock. + * 3. Otherwise, for aligned non-extending writes, shared lock is always + * sufficient regardless of extent state (written, unwritten, or hole). + * truncate/punch_hole cannot run while we hold the shared i_rwsem + * (they need it exclusively); after we release it, inode_dio_begin() + * keeps their inode_dio_wait() blocked until in-flight bios complete. + * i_data_sem serializes concurrent extent tree modifications. + * + * 4. Otherwise, the write is unaligned and non-extending. Shared lock is + * safe unless the DIO layer needs to perform partial block zeroing -- + * i.e. the head or tail partial block sits on a hole or unwritten + * extent. In that case upgrade to the exclusive lock and drain + * in-flight DIO to avoid races with concurrent partial block zeroing. */ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, bool *ilock_shared, bool *extend, @@ -432,7 +499,7 @@ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, loff_t offset; size_t count; ssize_t ret; - bool overwrite, unaligned_io, unwritten; + bool needs_zeroing = false; restart: ret = ext4_generic_write_checks(iocb, from); @@ -442,24 +509,22 @@ restart: offset = iocb->ki_pos; count = ret; - unaligned_io = ext4_unaligned_io(inode, from, offset); *extend = ext4_extending_io(inode, offset, count); - overwrite = ext4_overwrite_io(inode, offset, count, &unwritten); /* - * Determine whether we need to upgrade to an exclusive lock. This is - * required to change security info in file_modified(), for extending - * I/O, any form of non-overwrite I/O, and unaligned I/O to unwritten - * extents (as partial block zeroing may be required). + * For unaligned writes, check whether partial block zeroing will be + * needed. If so, exclusive lock is required to serialize against + * concurrent DIO that could race with the zeroing. * - * Note that unaligned writes are allowed under shared lock so long as - * they are pure overwrites. Otherwise, concurrent unaligned writes risk - * data corruption due to partial block zeroing in the dio layer, and so - * the I/O must occur exclusively. + * For aligned writes we skip this check entirely since allocation + * under shared lock is safe. */ + if (ext4_unaligned_io(inode, from, offset)) + needs_zeroing = ext4_dio_needs_zeroing(inode, offset, count); + + /* Determine whether we need to upgrade to an exclusive lock. */ if (*ilock_shared && - ((!IS_NOSEC(inode) || *extend || !overwrite || - (unaligned_io && unwritten)))) { + (!IS_NOSEC(inode) || *extend || needs_zeroing)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; @@ -473,21 +538,28 @@ restart: /* * Now that locking is settled, determine dio flags and exclusivity * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce - * behavior already. The inode lock is already held exclusive if the - * write is non-overwrite or extending, so drain all outstanding dio and - * set the force wait dio flag. + * behavior already. When holding the exclusive lock for a write that + * needs partial block zeroing or is extending the file, we must wait + * for the I/O to complete synchronously: + * + * - needs_zeroing: drain in-flight DIO whose end_io could race with + * our partial block zeroing, and force synchronous completion so we + * don't leave in-flight zeroing bios for the next writer to drain. + * + * - extend: the caller must update i_disksize after I/O completion, + * which requires the data to be on disk first. */ - if (!*ilock_shared && (unaligned_io || *extend)) { + if (!*ilock_shared && (needs_zeroing || *extend)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; } - if (unaligned_io && (!overwrite || unwritten)) + if (needs_zeroing) inode_dio_wait(inode); *dio_flags = IOMAP_DIO_FORCE_WAIT; } - ret = file_modified(file); + ret = kiocb_modified(iocb); if (ret < 0) goto out; @@ -656,6 +728,11 @@ ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) count = iov_iter_count(from); if (offset + count > EXT4_I(inode)->i_disksize) { + if (iocb->ki_flags & IOCB_NOWAIT) { + ret = -EAGAIN; + goto out; + } + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); if (IS_ERR(handle)) { ret = PTR_ERR(handle); diff --git a/fs/ext4/fsync.c b/fs/ext4/fsync.c index e476c6de3074..2999c2cc8fcf 100644 --- a/fs/ext4/fsync.c +++ b/fs/ext4/fsync.c @@ -68,9 +68,6 @@ static int ext4_sync_parent(struct inode *inode) * through ext4_evict_inode()) and so we are safe to flush * metadata blocks and the inode. */ - ret = sync_mapping_buffers(inode->i_mapping); - if (ret) - break; ret = sync_inode_metadata(inode, 1); if (ret) break; @@ -85,9 +82,11 @@ static int ext4_fsync_nojournal(struct file *file, loff_t start, loff_t end, struct inode *inode = file->f_inode; int ret; - ret = generic_buffers_fsync_noflush(file, start, end, datasync); - if (!ret) - ret = ext4_sync_parent(inode); + ret = sync_inode_metadata(inode, 1); + if (ret) + return ret; + ret = ext4_sync_parent(inode); + if (test_opt(inode->i_sb, BARRIER)) *needs_barrier = true; @@ -143,6 +142,10 @@ int ext4_sync_file(struct file *file, loff_t start, loff_t end, int datasync) if (sb_rdonly(inode->i_sb)) goto out; + ret = file_write_and_wait_range(file, start, end); + if (ret) + goto out; + if (!EXT4_SB(inode->i_sb)->s_journal) { ret = ext4_fsync_nojournal(file, start, end, datasync, &needs_barrier); @@ -151,10 +154,6 @@ int ext4_sync_file(struct file *file, loff_t start, loff_t end, int datasync) goto out; } - ret = file_write_and_wait_range(file, start, end); - if (ret) - goto out; - /* * The caller's filemap_fdatawrite()/wait will sync the data. * Metadata is in the journal, we wait for proper transaction to diff --git a/fs/ext4/hash-test.c b/fs/ext4/hash-test.c new file mode 100644 index 000000000000..49b0d874c833 --- /dev/null +++ b/fs/ext4/hash-test.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for ext4 directory hash computation. + */ + +#include <kunit/test.h> +#include <kunit/resource.h> +#include <linux/fs.h> +#include <linux/stddef.h> +#include <linux/string.h> +#include <linux/unicode.h> +#include "ext4.h" + +static void ext4_hash_init_fake_dir(struct inode *dir, struct super_block *sb) +{ + memset(sb, 0, sizeof(*sb)); + memset(dir, 0, sizeof(*dir)); + dir->i_sb = sb; + strscpy(sb->s_id, "kunit-ext4", sizeof(sb->s_id)); +} + +static void ext4_hash_init_fake_dir_with_sbi(struct inode *dir, + struct super_block *sb, + struct ext4_sb_info *sbi) +{ + ext4_hash_init_fake_dir(dir, sb); + memset(sbi, 0, sizeof(*sbi)); + sb->s_fs_info = sbi; + sbi->s_sb = sb; +} + +#ifdef CONFIG_FS_ENCRYPTION +static const struct fscrypt_operations ext4_hash_test_cryptops = { + .inode_info_offs = + (int)offsetof(struct ext4_inode_info, i_crypt_info) - + (int)offsetof(struct ext4_inode_info, vfs_inode), +}; +#endif + +static void ext4_hash_init_fake_ext4_dir(struct ext4_inode_info *ei, + struct super_block *sb, + struct ext4_sb_info *sbi) +{ + struct inode *dir = &ei->vfs_inode; + + memset(sb, 0, sizeof(*sb)); + memset(ei, 0, sizeof(*ei)); + memset(sbi, 0, sizeof(*sbi)); + + strscpy(sb->s_id, "kunit-ext4", sizeof(sb->s_id)); + sb->s_fs_info = sbi; + sbi->s_sb = sb; + + dir->i_sb = sb; + dir->i_mode = S_IFDIR; + +#ifdef CONFIG_FS_ENCRYPTION + fscrypt_set_ops(sb, &ext4_hash_test_cryptops); +#endif +} + +struct ext4_dirhash_test_case { + const char *name; + u32 hash_version; + const char *input; + int len; + u32 seed[4]; + bool use_seed; + u32 expected_hash; + u32 expected_minor_hash; +}; + +static const struct ext4_dirhash_test_case ext4_dirhash_test_cases[] = { + { + .name = "legacy_abc", + .hash_version = DX_HASH_LEGACY, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0x75afd992, + .expected_minor_hash = 0x00000000, + }, + { + .name = "legacy_unsigned_abc", + .hash_version = DX_HASH_LEGACY_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0x75afd992, + .expected_minor_hash = 0x00000000, + }, + { + .name = "half_md4_abc", + .hash_version = DX_HASH_HALF_MD4, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xd196a868, + .expected_minor_hash = 0xc420eb28, + }, + { + .name = "half_md4_unsigned_abc", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xd196a868, + .expected_minor_hash = 0xc420eb28, + }, + { + .name = "tea_abc", + .hash_version = DX_HASH_TEA, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xb1435ec4, + .expected_minor_hash = 0x3f7eaa0e, + }, + { + .name = "tea_unsigned_abc", + .hash_version = DX_HASH_TEA_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xb1435ec4, + .expected_minor_hash = 0x3f7eaa0e, + }, + { + .name = "empty_half_md4", + .hash_version = DX_HASH_HALF_MD4, + .input = "", + .len = 0, + .use_seed = false, + .expected_hash = 0xefcdab88, + .expected_minor_hash = 0x98badcfe, + }, + { + .name = "half_md4_31bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "1234567890123456789012345678901", + .len = 31, + .use_seed = false, + .expected_hash = 0xc4db1f78, + .expected_minor_hash = 0xea23921b, + }, + { + .name = "half_md4_32bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "12345678901234567890123456789012", + .len = 32, + .use_seed = false, + .expected_hash = 0xfa6cc63e, + .expected_minor_hash = 0x2f77bd1c, + }, + { + .name = "half_md4_33bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "123456789012345678901234567890123", + .len = 33, + .use_seed = false, + .expected_hash = 0xdc0c2dec, + .expected_minor_hash = 0x5ca23365, + }, + { + .name = "half_md4_unsigned_31bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "1234567890123456789012345678901", + .len = 31, + .use_seed = false, + .expected_hash = 0xc4db1f78, + .expected_minor_hash = 0xea23921b, + }, + { + .name = "half_md4_unsigned_32bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "12345678901234567890123456789012", + .len = 32, + .use_seed = false, + .expected_hash = 0xfa6cc63e, + .expected_minor_hash = 0x2f77bd1c, + }, + { + .name = "half_md4_unsigned_33bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "123456789012345678901234567890123", + .len = 33, + .use_seed = false, + .expected_hash = 0xdc0c2dec, + .expected_minor_hash = 0x5ca23365, + }, + { + .name = "tea_15bytes", + .hash_version = DX_HASH_TEA, + .input = "123456789abcdef", + .len = 15, + .use_seed = false, + .expected_hash = 0xa562903a, + .expected_minor_hash = 0x6174a00f, + }, + { + .name = "tea_16bytes", + .hash_version = DX_HASH_TEA, + .input = "1234567890abcdef", + .len = 16, + .use_seed = false, + .expected_hash = 0x8449f258, + .expected_minor_hash = 0x49a16d46, + }, + { + .name = "tea_17bytes", + .hash_version = DX_HASH_TEA, + .input = "123456789abcdefgh", + .len = 17, + .use_seed = false, + .expected_hash = 0xf32ec10c, + .expected_minor_hash = 0x58ceae61, + }, + { + .name = "half_md4_seeded", + .hash_version = DX_HASH_HALF_MD4, + .input = "same-name", + .len = 9, + .seed = { 0x11111111, 0x22222222, 0x33333333, 0x44444444 }, + .use_seed = true, + .expected_hash = 0x8aebf604, + .expected_minor_hash = 0x66ce48fe, + }, + { + .name = "half_md4_non_ascii_signed", + .hash_version = DX_HASH_HALF_MD4, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x8bab0498, + .expected_minor_hash = 0xc326632d, + }, + { + .name = "half_md4_non_ascii_unsigned", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0xbc48596e, + .expected_minor_hash = 0xde0fad41, + }, + { + .name = "tea_non_ascii_signed", + .hash_version = DX_HASH_TEA, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x21e3a154, + .expected_minor_hash = 0x90112c3d, + }, + { + .name = "tea_non_ascii_unsigned", + .hash_version = DX_HASH_TEA_UNSIGNED, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x9b648616, + .expected_minor_hash = 0x011dd507, + }, +}; + +static void test_ext4fs_dirhash_vectors(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + int i; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + for (i = 0; i < ARRAY_SIZE(ext4_dirhash_test_cases); i++) { + const struct ext4_dirhash_test_case *tc = + &ext4_dirhash_test_cases[i]; + struct dx_hash_info hinfo; + int ret; + + memset(&hinfo, 0, sizeof(hinfo)); + hinfo.hash_version = tc->hash_version; + hinfo.seed = tc->use_seed ? (u32 *)tc->seed : NULL; + + ret = ext4fs_dirhash(dir, tc->input, tc->len, &hinfo); + + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "case=%s", tc->name); + KUNIT_EXPECT_EQ_MSG(test, hinfo.hash, tc->expected_hash, + "case=%s", tc->name); + KUNIT_EXPECT_EQ_MSG(test, hinfo.minor_hash, + tc->expected_minor_hash, + "case=%s", tc->name); + } +} + +static void test_ext4fs_dirhash_seed_changes_result(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + u32 seed[4] = { 0x11111111, 0x22222222, 0x33333333, 0x44444444 }; + struct dx_hash_info plain = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info seeded = { + .hash_version = DX_HASH_HALF_MD4, + .seed = seed, + }; + int ret_plain, ret_seeded; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + ret_plain = ext4fs_dirhash(dir, "same-name", 9, &plain); + ret_seeded = ext4fs_dirhash(dir, "same-name", 9, &seeded); + + KUNIT_ASSERT_EQ(test, ret_plain, 0); + KUNIT_ASSERT_EQ(test, ret_seeded, 0); + + KUNIT_EXPECT_TRUE(test, + plain.hash != seeded.hash || + plain.minor_hash != seeded.minor_hash); +} + +static void test_ext4fs_dirhash_invalid_version_returns_einval(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + struct ext4_sb_info *sbi; + struct dx_hash_info hinfo = { + .hash = 0xdeadbeef, + .minor_hash = 0xcafebabe, + .hash_version = DX_HASH_LAST + 1, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + ext4_hash_init_fake_dir_with_sbi(dir, sb, sbi); + + ret = ext4fs_dirhash(dir, "abc", 3, &hinfo); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, hinfo.hash, 0); + KUNIT_EXPECT_EQ(test, hinfo.minor_hash, 0); +} + +static void test_ext4fs_dirhash_siphash_without_key_returns_einval(struct kunit *test) +{ + struct super_block *sb; + struct ext4_inode_info *ei; + struct inode *dir; + struct ext4_sb_info *sbi; + struct dx_hash_info hinfo = { + .hash_version = DX_HASH_SIPHASH, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + ext4_hash_init_fake_ext4_dir(ei, sb, sbi); + dir = &ei->vfs_inode; + + ret = ext4fs_dirhash(dir, "name", strlen("name"), &hinfo); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_ext4fs_dirhash_signed_unsigned_differ_on_nonascii(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + static const char input[] = "\x80\xff\x81\xfe\101bc"; + struct dx_hash_info legacy_signed = { + .hash_version = DX_HASH_LEGACY, + }; + struct dx_hash_info legacy_unsigned = { + .hash_version = DX_HASH_LEGACY_UNSIGNED, + }; + struct dx_hash_info md4_signed = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info md4_unsigned = { + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + }; + struct dx_hash_info tea_signed = { + .hash_version = DX_HASH_TEA, + }; + struct dx_hash_info tea_unsigned = { + .hash_version = DX_HASH_TEA_UNSIGNED, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &legacy_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &legacy_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NE(test, legacy_signed.hash, legacy_unsigned.hash); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &md4_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &md4_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, + md4_signed.hash != md4_unsigned.hash || + md4_signed.minor_hash != md4_unsigned.minor_hash); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &tea_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &tea_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, + tea_signed.hash != tea_unsigned.hash || + tea_signed.minor_hash != tea_unsigned.minor_hash); +} + +#if IS_ENABLED(CONFIG_UNICODE) +KUNIT_DEFINE_ACTION_WRAPPER(utf8_unload_action, utf8_unload, + struct unicode_map *); +static void test_ext4fs_dirhash_casefolded_names_hash_consistently(struct kunit *test) +{ + struct super_block *sb; + struct ext4_inode_info *ei; + struct ext4_sb_info *sbi; + struct unicode_map *um; + struct dx_hash_info h1 = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info h2 = { + .hash_version = DX_HASH_HALF_MD4, + }; + int ret, ret1, ret2; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + um = utf8_load(UTF8_LATEST); + if (IS_ERR(um)) { + kunit_skip(test, "utf8_load(UTF8_LATEST) failed: %pe", + um); + return; + } + + ret = kunit_add_action_or_reset(test, utf8_unload_action, um); + KUNIT_ASSERT_EQ(test, ret, 0); + + ext4_hash_init_fake_ext4_dir(ei, sb, sbi); + sb->s_encoding = um; + ei->vfs_inode.i_flags |= S_CASEFOLD; + + KUNIT_ASSERT_TRUE(test, IS_CASEFOLDED(&ei->vfs_inode)); + + ret1 = ext4fs_dirhash(&ei->vfs_inode, "Alpha", 5, &h1); + ret2 = ext4fs_dirhash(&ei->vfs_inode, "aLPHa", 5, &h2); + + KUNIT_ASSERT_EQ(test, ret1, 0); + KUNIT_ASSERT_EQ(test, ret2, 0); + KUNIT_EXPECT_EQ(test, h1.hash, h2.hash); + KUNIT_EXPECT_EQ(test, h1.minor_hash, h2.minor_hash); +} + +static void test_ext4fs_dirhash_casefold_fallback(struct kunit *test) +{ + struct super_block *sb_cf, *sb_plain; + struct ext4_inode_info *ei; + struct ext4_sb_info *sbi; + struct inode *plain_dir; + struct unicode_map *um; + static const char invalid_utf8[] = "\xc3\x28"; + struct dx_hash_info folded_dir = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info plain = { + .hash_version = DX_HASH_HALF_MD4, + }; + int ret, ret_cf, ret_plain; + + sb_cf = kunit_kzalloc(test, sizeof(*sb_cf), GFP_KERNEL); + sb_plain = kunit_kzalloc(test, sizeof(*sb_plain), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + plain_dir = kunit_kzalloc(test, sizeof(*plain_dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb_cf); + KUNIT_ASSERT_NOT_NULL(test, sb_plain); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + KUNIT_ASSERT_NOT_NULL(test, plain_dir); + + um = utf8_load(UTF8_LATEST); + if (IS_ERR(um)) { + kunit_skip(test, "utf8_load(UTF8_LATEST) failed: %pe", + um); + return; + } + + ret = kunit_add_action_or_reset(test, utf8_unload_action, um); + KUNIT_ASSERT_EQ(test, ret, 0); + + ext4_hash_init_fake_ext4_dir(ei, sb_cf, sbi); + sb_cf->s_encoding = um; + ei->vfs_inode.i_flags |= S_CASEFOLD; + + KUNIT_ASSERT_TRUE(test, IS_CASEFOLDED(&ei->vfs_inode)); + + ext4_hash_init_fake_dir(plain_dir, sb_plain); + + ret_cf = ext4fs_dirhash(&ei->vfs_inode, invalid_utf8, + sizeof(invalid_utf8) - 1, &folded_dir); + ret_plain = ext4fs_dirhash(plain_dir, invalid_utf8, + sizeof(invalid_utf8) - 1, &plain); + + KUNIT_ASSERT_EQ(test, ret_cf, 0); + KUNIT_ASSERT_EQ(test, ret_plain, 0); + KUNIT_EXPECT_EQ(test, folded_dir.hash, plain.hash); + KUNIT_EXPECT_EQ(test, folded_dir.minor_hash, plain.minor_hash); +} +#endif + +static struct kunit_case ext4_hash_test_cases[] = { + KUNIT_CASE(test_ext4fs_dirhash_vectors), + KUNIT_CASE(test_ext4fs_dirhash_seed_changes_result), + KUNIT_CASE(test_ext4fs_dirhash_invalid_version_returns_einval), + KUNIT_CASE(test_ext4fs_dirhash_siphash_without_key_returns_einval), + KUNIT_CASE(test_ext4fs_dirhash_signed_unsigned_differ_on_nonascii), +#if IS_ENABLED(CONFIG_UNICODE) + KUNIT_CASE(test_ext4fs_dirhash_casefolded_names_hash_consistently), + KUNIT_CASE(test_ext4fs_dirhash_casefold_fallback), +#endif + {} +}; + +static struct kunit_suite ext4_hash_test_suite = { + .name = "ext4_hash", + .test_cases = ext4_hash_test_cases, +}; + +kunit_test_suites(&ext4_hash_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/fs/ext4/hash.c b/fs/ext4/hash.c index 48483cd015d3..978bd92da0ad 100644 --- a/fs/ext4/hash.c +++ b/fs/ext4/hash.c @@ -9,6 +9,7 @@ #include <linux/unicode.h> #include <linux/compiler.h> #include <linux/bitops.h> +#include <linux/unaligned.h> #include "ext4.h" #define DELTA 0x9E3779B9 @@ -141,21 +142,28 @@ static void str2hashbuf_signed(const char *msg, int len, __u32 *buf, int num) pad = (__u32)len | ((__u32)len << 8); pad |= pad << 16; - val = pad; if (len > num*4) len = num * 4; - for (i = 0; i < len; i++) { - val = ((int) scp[i]) + (val << 8); - if ((i % 4) == 3) { - *buf++ = val; - val = pad; - num--; - } + + while (len >= 4) { + val = ((__u32)scp[0] << 24) + ((__u32)scp[1] << 16) + ((__u32)scp[2] << 8) + scp[3]; + *buf++ = val; + scp += 4; + len -= 4; + num--; } + + val = pad; + + for (i = 0; i < len; i++) + val = scp[i] + (val << 8); + if (--num >= 0) *buf++ = val; + while (--num >= 0) *buf++ = pad; + } static void str2hashbuf_unsigned(const char *msg, int len, __u32 *buf, int num) @@ -167,21 +175,28 @@ static void str2hashbuf_unsigned(const char *msg, int len, __u32 *buf, int num) pad = (__u32)len | ((__u32)len << 8); pad |= pad << 16; - val = pad; if (len > num*4) len = num * 4; - for (i = 0; i < len; i++) { - val = ((int) ucp[i]) + (val << 8); - if ((i % 4) == 3) { - *buf++ = val; - val = pad; - num--; - } + + while (len >= 4) { + val = get_unaligned_be32(ucp); + *buf++ = val; + ucp += 4; + len -= 4; + num--; } + + val = pad; + + for (i = 0; i < len; i++) + val = ucp[i] + (val << 8); + if (--num >= 0) *buf++ = val; + while (--num >= 0) *buf++ = pad; + } /* @@ -205,8 +220,7 @@ static int __ext4fs_dirhash(const struct inode *dir, const char *name, int len, const char *p; int i; __u32 in[8], buf[4]; - void (*str2hashbuf)(const char *, int, __u32 *, int) = - str2hashbuf_signed; + bool use_unsigned = false; /* Initialize the default seed for the hash checksum functions */ buf[0] = 0x67452301; @@ -232,12 +246,15 @@ static int __ext4fs_dirhash(const struct inode *dir, const char *name, int len, hash = dx_hack_hash_signed(name, len); break; case DX_HASH_HALF_MD4_UNSIGNED: - str2hashbuf = str2hashbuf_unsigned; + use_unsigned = true; fallthrough; case DX_HASH_HALF_MD4: p = name; while (len > 0) { - (*str2hashbuf)(p, len, in, 8); + if (use_unsigned) + str2hashbuf_unsigned(p, len, in, 8); + else + str2hashbuf_signed(p, len, in, 8); half_md4_transform(buf, in); len -= 32; p += 32; @@ -246,12 +263,15 @@ static int __ext4fs_dirhash(const struct inode *dir, const char *name, int len, hash = buf[1]; break; case DX_HASH_TEA_UNSIGNED: - str2hashbuf = str2hashbuf_unsigned; + use_unsigned = true; fallthrough; case DX_HASH_TEA: p = name; while (len > 0) { - (*str2hashbuf)(p, len, in, 4); + if (use_unsigned) + str2hashbuf_unsigned(p, len, in, 4); + else + str2hashbuf_signed(p, len, in, 4); TEA_transform(buf, in); len -= 16; p += 16; @@ -321,3 +341,7 @@ opaque_seq: #endif return __ext4fs_dirhash(dir, name, len, hinfo); } + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4fs_dirhash); +#endif diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index b20a1bf866ab..a5831fc536db 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -66,14 +66,16 @@ void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap) memset(bitmap + (i >> 3), 0xff, (end_bit - i) >> 3); } -void ext4_end_bitmap_read(struct buffer_head *bh, int uptodate) +void ext4_end_bitmap_read(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + if (uptodate) { set_buffer_uptodate(bh); set_bitmap_uptodate(bh); } unlock_buffer(bh); - put_bh(bh); } static int ext4_validate_inode_bitmap(struct super_block *sb, @@ -252,14 +254,14 @@ void ext4_free_inode(handle_t *handle, struct inode *inode) "nonexistent device\n", __func__, __LINE__); return; } - if (icount_read(inode) > 1) { - ext4_msg(sb, KERN_ERR, "%s:%d: inode #%lu: count=%d", + if (icount_read_once(inode) > 1) { + ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: count=%d", __func__, __LINE__, inode->i_ino, - icount_read(inode)); + icount_read_once(inode)); return; } if (inode->i_nlink) { - ext4_msg(sb, KERN_ERR, "%s:%d: inode #%lu: nlink=%d\n", + ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: nlink=%d\n", __func__, __LINE__, inode->i_ino, inode->i_nlink); return; } @@ -631,7 +633,7 @@ static int find_group_other(struct super_block *sb, struct inode *parent, * * So add our directory's i_ino into the starting point for the hash. */ - *group = (*group + parent->i_ino) % ngroups; + *group = (*group + (unsigned int)parent->i_ino) % ngroups; /* * Use a quadratic hash to find a group with a free inode and some free @@ -686,6 +688,12 @@ static int recently_deleted(struct super_block *sb, ext4_group_t group, int ino) if (unlikely(!gdp)) return 0; + /* Inode was never used in this filesystem? */ + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT) || + ino >= EXT4_INODES_PER_GROUP(sb) - ext4_itable_unused_count(sb, gdp))) + return 0; + bh = sb_find_get_block(sb, ext4_inode_table(sb, gdp) + (ino / inodes_per_block)); if (!bh || !buffer_uptodate(bh)) @@ -989,6 +997,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, err = fscrypt_prepare_new_inode(dir, inode, &encrypt); if (err) goto out; + if (encrypt) + i_flags |= EXT4_ENCRYPT_FL; } err = dquot_initialize(inode); @@ -1275,7 +1285,7 @@ got: * twice. */ err = -EIO; - ext4_error(sb, "failed to insert inode %lu: doubly allocated?", + ext4_error(sb, "failed to insert inode %llu: doubly allocated?", inode->i_ino); ext4_mark_group_bitmap_corrupted(sb, group, EXT4_GROUP_INFO_IBITMAP_CORRUPT); @@ -1298,6 +1308,8 @@ got: ei->i_extra_isize = sbi->s_want_extra_isize; ei->i_inline_off = 0; if (ext4_has_feature_inline_data(sb) && + /* Encrypted inodes cannot have inline data */ + !(ei->i_flags & EXT4_ENCRYPT_FL) && (!(ei->i_flags & (EXT4_DAX_FL|EXT4_EA_INODE_FL)) || S_ISDIR(mode))) ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); ret = inode; @@ -1344,7 +1356,7 @@ got: goto fail_free_drop; } - ext4_debug("allocating inode %lu\n", inode->i_ino); + ext4_debug("allocating inode %llu\n", inode->i_ino); trace_ext4_allocate_inode(inode, dir, mode); brelse(inode_bitmap_bh); return ret; diff --git a/fs/ext4/indirect.c b/fs/ext4/indirect.c index da76353b3a57..5aec759eed70 100644 --- a/fs/ext4/indirect.c +++ b/fs/ext4/indirect.c @@ -102,7 +102,7 @@ static int ext4_block_to_path(struct inode *inode, offsets[n++] = i_block & (ptrs - 1); final = ptrs; } else { - ext4_warning(inode->i_sb, "block %lu > max in inode %lu", + ext4_warning(inode->i_sb, "block %lu > max in inode %llu", i_block + direct_blocks + indirect_blocks + double_blocks, inode->i_ino); } diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 1f6bc05593df..ceee69a66482 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -22,8 +22,7 @@ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, - struct inode *inode, - void **fsdata); + struct inode *inode); static int ext4_get_inline_size(struct inode *inode) { @@ -119,7 +118,7 @@ int ext4_get_max_inline_size(struct inode *inode) error = ext4_get_inode_loc(inode, &iloc); if (error) { ext4_error_inode_err(inode, __func__, __LINE__, 0, -error, - "can't get inode location %lu", + "can't get inode location %llu", inode->i_ino); return 0; } @@ -512,7 +511,7 @@ static int ext4_read_inline_folio(struct inode *inode, struct folio *folio) BUG_ON(folio->index); if (!EXT4_I(inode)->i_inline_off) { - ext4_warning(inode->i_sb, "inode %lu doesn't have inline data.", + ext4_warning(inode->i_sb, "inode %llu doesn't have inline data.", inode->i_ino); goto out; } @@ -522,7 +521,15 @@ static int ext4_read_inline_folio(struct inode *inode, struct folio *folio) goto out; len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode)); - BUG_ON(len > PAGE_SIZE); + + if (len > PAGE_SIZE) { + ext4_error_inode(inode, __func__, __LINE__, 0, + "inline size %zu exceeds PAGE_SIZE", len); + ret = -EFSCORRUPTED; + brelse(iloc.bh); + goto out; + } + kaddr = kmap_local_folio(folio, 0); ret = ext4_read_inline_data(inode, kaddr, len, &iloc); kaddr = folio_zero_tail(folio, len, kaddr + len); @@ -689,7 +696,7 @@ int ext4_generic_write_inline_data(struct address_space *mapping, struct inode *inode, loff_t pos, unsigned len, struct folio **foliop, - void **fsdata, bool da) + bool da) { int ret; handle_t *handle; @@ -720,7 +727,7 @@ retry_journal: return ext4_convert_inline_data_to_extent(mapping, inode); } - ret = ext4_da_convert_inline_data_to_extent(mapping, inode, fsdata); + ret = ext4_da_convert_inline_data_to_extent(mapping, inode); if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) goto retry_journal; @@ -780,7 +787,7 @@ int ext4_try_to_write_inline_data(struct address_space *mapping, if (pos + len > ext4_get_max_inline_size(inode)) return ext4_convert_inline_data_to_extent(mapping, inode); return ext4_generic_write_inline_data(mapping, inode, pos, len, - foliop, NULL, false); + foliop, false); } int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, @@ -804,7 +811,19 @@ int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, goto out; } ext4_write_lock_xattr(inode, &no_expand); - BUG_ON(!ext4_has_inline_data(inode)); + /* + * We could have raced with ext4_page_mkwrite() converting + * the inode and clearing the inline data flag, so we just + * release resources and retry the whole write. + */ + if (unlikely(!ext4_has_inline_data(inode))) { + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + return 0; + } /* * ei->i_inline_off may have changed since @@ -875,8 +894,7 @@ out: * need to start the journal since the file's metadata isn't changed now. */ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, - struct inode *inode, - void **fsdata) + struct inode *inode) { int ret = 0, inline_size; struct folio *folio; @@ -914,7 +932,6 @@ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, folio_mark_dirty(folio); folio_mark_uptodate(folio); ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); - *fsdata = (void *)CONVERT_INLINE_DATA; out: up_read(&EXT4_I(inode)->xattr_sem); @@ -934,7 +951,7 @@ void ext4_show_inline_dir(struct inode *dir, struct buffer_head *bh, struct ext4_dir_entry_2 *de = inline_start; void *dlimit = inline_start + inline_size; - trace_printk("inode %lu\n", dir->i_ino); + trace_printk("inode %llu\n", dir->i_ino); offset = 0; while ((void *)de < dlimit) { de_len = ext4_rec_len_from_disk(de->rec_len, inline_size); @@ -1071,7 +1088,7 @@ static void ext4_restore_inline_data(handle_t *handle, struct inode *inode, ret = ext4_create_inline_data(handle, inode, inline_size); if (ret) { ext4_msg(inode->i_sb, KERN_EMERG, - "error restoring inline_data for inode -- potential data loss! (inode %lu, error %d)", + "error restoring inline_data for inode -- potential data loss! (inode %llu, error %d)", inode->i_ino, ret); return; } @@ -1446,6 +1463,8 @@ int ext4_read_inline_dir(struct file *file, /* for other entry, the real offset in * the buf has to be tuned accordingly. */ + if (i + ext4_dir_rec_len(1, NULL) > extra_size) + break; de = (struct ext4_dir_entry_2 *) (dir_buf + i - extra_offset); /* It's too expensive to do a full @@ -1480,10 +1499,17 @@ int ext4_read_inline_dir(struct file *file, continue; } + /* + * de lives at dir_buf + ctx->pos - extra_offset, within the + * kmalloc(inline_size) buffer. Make sure its header fits before + * ext4_check_dir_entry() dereferences de->rec_len. + */ + if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size) + goto out; de = (struct ext4_dir_entry_2 *) (dir_buf + ctx->pos - extra_offset); if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, - extra_size, ctx->pos)) + inline_size, ctx->pos)) goto out; if (le32_to_cpu(de->inode)) { if (!dir_emit(ctx, de->name, de->name_len, @@ -1740,7 +1766,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data) err = ext4_get_inode_loc(dir, &iloc); if (err) { EXT4_ERROR_INODE_ERR(dir, -err, - "error %d getting inode %lu block", + "error %d getting inode %llu block", err, dir->i_ino); return false; } @@ -1755,7 +1781,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data) de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block; if (!le32_to_cpu(de->inode)) { ext4_warning(dir->i_sb, - "bad inline directory (dir #%lu) - no `..'", + "bad inline directory (dir #%llu) - no `..'", dir->i_ino); goto out; } @@ -1769,7 +1795,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data) iloc.bh, inline_pos, inline_size, offset)) { ext4_warning(dir->i_sb, - "bad inline directory (dir #%lu) - " + "bad inline directory (dir #%llu) - " "inode %u, rec_len %u, name_len %d" "inline size %d", dir->i_ino, le32_to_cpu(de->inode), diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 396dc3a5d16b..26f0f9714f03 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -29,7 +29,7 @@ #include <linux/string.h> #include <linux/buffer_head.h> #include <linux/writeback.h> -#include <linux/pagevec.h> +#include <linux/folio_batch.h> #include <linux/mpage.h> #include <linux/rmap.h> #include <linux/namei.h> @@ -128,6 +128,8 @@ void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, static inline int ext4_begin_ordered_truncate(struct inode *inode, loff_t new_size) { + struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode); + trace_ext4_begin_ordered_truncate(inode, new_size); /* * If jinode is zero, then we never opened the file for @@ -135,10 +137,10 @@ static inline int ext4_begin_ordered_truncate(struct inode *inode, * jbd2_journal_begin_ordered_truncate() since there's no * outstanding writes we need to flush. */ - if (!EXT4_I(inode)->jinode) + if (!jinode) return 0; return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), - EXT4_I(inode)->jinode, + jinode, new_size); } @@ -174,7 +176,6 @@ void ext4_evict_inode(struct inode *inode) * (xattr block freeing), bitmap, group descriptor (inode freeing) */ int extra_credits = 6; - struct ext4_xattr_inode_array *ea_inode_array = NULL; bool freeze_protected = false; trace_ext4_evict_inode(inode); @@ -184,8 +185,20 @@ void ext4_evict_inode(struct inode *inode) if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL) ext4_evict_ea_inode(inode); if (inode->i_nlink) { - truncate_inode_pages_final(&inode->i_data); + struct mapping_metadata_bhs *mmb; + + /* + * If there's dirty page will lead to data loss, user + * could see stale data. + */ + if (unlikely(!ext4_emergency_state(inode->i_sb) && + mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY))) + ext4_warning_inode(inode, "data will be lost"); + truncate_inode_pages_final(&inode->i_data); + mmb = ext4_i_metadata_bhs(inode); + if (mmb) + mmb_sync(mmb); goto no_delete; } @@ -252,6 +265,7 @@ void ext4_evict_inode(struct inode *inode) if (ext4_inode_is_fast_symlink(inode)) memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data)); inode->i_size = 0; + ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); err = ext4_mark_inode_dirty(handle, inode); if (err) { ext4_warning(inode->i_sb, @@ -262,15 +276,14 @@ void ext4_evict_inode(struct inode *inode) err = ext4_truncate(inode); if (err) { ext4_error_err(inode->i_sb, -err, - "couldn't truncate inode %lu (err %d)", + "couldn't truncate inode %llu (err %d)", inode->i_ino, err); goto stop_handle; } } /* Remove xattr references. */ - err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array, - extra_credits); + err = ext4_xattr_delete_inode(handle, inode, extra_credits); if (err) { ext4_warning(inode->i_sb, "xattr delete (err %d)", err); stop_handle: @@ -278,7 +291,6 @@ stop_handle: ext4_orphan_del(NULL, inode); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); goto no_delete; } @@ -308,7 +320,6 @@ stop_handle: ext4_journal_stop(handle); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); return; no_delete: /* @@ -342,7 +353,7 @@ void ext4_da_update_reserve_space(struct inode *inode, spin_lock(&ei->i_block_reservation_lock); trace_ext4_da_update_reserve_space(inode, used, quota_claim); if (unlikely(used > ei->i_reserved_data_blocks)) { - ext4_warning(inode->i_sb, "%s: ino %lu, used %d " + ext4_warning(inode->i_sb, "%s: ino %llu, used %d " "with only %d reserved data blocks", __func__, inode->i_ino, used, ei->i_reserved_data_blocks); @@ -405,7 +416,10 @@ int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, KUNIT_STATIC_STUB_REDIRECT(ext4_issue_zeroout, inode, lblk, pblk, len); if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) - return fscrypt_zeroout_range(inode, lblk, pblk, len); + return fscrypt_zeroout_range(inode, + (loff_t)lblk << inode->i_blkbits, + pblk << (inode->i_blkbits - SECTOR_SHIFT), + (u64)len << inode->i_blkbits); ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS); if (ret > 0) @@ -475,7 +489,7 @@ static void ext4_map_blocks_es_recheck(handle_t *handle, if (es_map->m_lblk != map->m_lblk || es_map->m_flags != map->m_flags || es_map->m_pblk != map->m_pblk) { - printk("ES cache assertion failed for inode: %lu " + printk("ES cache assertion failed for inode: %llu " "es_cached ex [%d/%d/%llu/%x] != " "found ex [%d/%d/%llu/%x] retval %d flags %x\n", inode->i_ino, es_map->m_lblk, es_map->m_len, @@ -515,7 +529,7 @@ static int ext4_map_query_blocks_next_in_leaf(handle_t *handle, if (unlikely(retval != map2.m_len)) { ext4_warning(inode->i_sb, "ES len assertion failed for inode " - "%lu: retval %d != map->m_len %d", + "%llu: retval %d != map->m_len %d", inode->i_ino, retval, map2.m_len); WARN_ON(1); } @@ -563,7 +577,7 @@ int ext4_map_query_blocks(handle_t *handle, struct inode *inode, if (unlikely(retval != map->m_len)) { ext4_warning(inode->i_sb, "ES len assertion failed for inode " - "%lu: retval %d != map->m_len %d", + "%llu: retval %d != map->m_len %d", inode->i_ino, retval, map->m_len); WARN_ON(1); } @@ -630,7 +644,7 @@ int ext4_map_create_blocks(handle_t *handle, struct inode *inode, if (unlikely(retval != map->m_len)) { ext4_warning(inode->i_sb, - "ES len assertion failed for inode %lu: " + "ES len assertion failed for inode %llu: " "retval %d != map->m_len %d", inode->i_ino, retval, map->m_len); WARN_ON(1); @@ -937,7 +951,7 @@ int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, { int ret = 0; - ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n", + ext4_debug("ext4_get_block_unwritten: inode %llu, create flag %d\n", inode->i_ino, create); ret = _ext4_get_block(inode, iblock, bh_result, EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT); @@ -1167,6 +1181,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, int nr_wait = 0; int i; bool should_journal_data = ext4_should_journal_data(inode); + bool folio_uptodate = folio_test_uptodate(folio); BUG_ON(!folio_test_locked(folio)); BUG_ON(to > folio_size(folio)); @@ -1178,13 +1193,13 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, head = create_empty_buffers(folio, blocksize, 0); block = EXT4_PG_TO_LBLK(inode, folio->index); - for (bh = head, block_start = 0; bh != head || !block_start; + for (bh = head, block_start = 0; + block_start < to || (!folio_uptodate && bh != head); block++, block_start = block_end, bh = bh->b_this_page) { block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { - if (folio_test_uptodate(folio)) { + if (folio_uptodate) set_buffer_uptodate(bh); - } continue; } if (WARN_ON_ONCE(buffer_new(bh))) @@ -1205,7 +1220,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, if (should_journal_data) do_journal_get_write_access(handle, inode, bh); - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { /* * Unlike __block_write_begin() we leave * dirtying of new uptodate buffers to @@ -1222,7 +1237,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, continue; } } - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { set_buffer_uptodate(bh); continue; } @@ -1247,17 +1262,6 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, from, to); else folio_zero_new_buffers(folio, from, to); - } else if (fscrypt_inode_uses_fs_layer_crypto(inode)) { - for (i = 0; i < nr_wait; i++) { - int err2; - - err2 = fscrypt_decrypt_pagecache_blocks(folio, - blocksize, bh_offset(wait[i])); - if (err2) { - clear_buffer_uptodate(wait[i]); - err = err2; - } - } } return err; @@ -1287,6 +1291,8 @@ static int ext4_write_begin(const struct kiocb *iocb, if (unlikely(ret)) return ret; + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + trace_ext4_write_begin(inode, pos, len); /* * Reserve one block more for addition to orphan list in case @@ -1301,8 +1307,10 @@ static int ext4_write_begin(const struct kiocb *iocb, foliop); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } /* @@ -1420,9 +1428,6 @@ static int write_end_fn(handle_t *handle, struct inode *inode, /* * We need to pick up the new inode size which generic_commit_write gave us * `iocb` can be NULL - eg, when called from page_symlink(). - * - * ext4 never places buffers on inode->i_mapping->i_private_list. metadata - * buffers are managed internally. */ static int ext4_write_end(const struct kiocb *iocb, struct address_space *mapping, @@ -1438,8 +1443,7 @@ static int ext4_write_end(const struct kiocb *iocb, trace_ext4_write_end(inode, pos, len, copied); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1456,10 +1460,9 @@ static int ext4_write_end(const struct kiocb *iocb, folio_unlock(folio); folio_put(folio); - if (old_size < pos && !verity) { + if (old_size < pos && !verity) pagecache_isize_extended(inode, old_size, pos); - ext4_zero_partial_blocks(handle, inode, old_size, pos - old_size); - } + /* * Don't mark the inode dirty under folio lock. First, it unnecessarily * makes the holding time of folio lock longer. Second, it forces lock @@ -1549,7 +1552,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1574,10 +1577,8 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, folio_unlock(folio); folio_put(folio); - if (old_size < pos && !verity) { + if (old_size < pos && !verity) pagecache_isize_extended(inode, old_size, pos); - ext4_zero_partial_blocks(handle, inode, old_size, pos - old_size); - } if (size_changed) { ret2 = ext4_mark_inode_dirty(handle, inode); @@ -1659,7 +1660,7 @@ void ext4_da_release_space(struct inode *inode, int to_free) * harmless to return without any action. */ ext4_warning(inode->i_sb, "ext4_da_release_space: " - "ino %lu, to_free %d with only %d reserved " + "ino %llu, to_free %d with only %d reserved " "data blocks", inode->i_ino, to_free, ei->i_reserved_data_blocks); WARN_ON(1); @@ -1747,8 +1748,22 @@ static void mpage_release_unused_pages(struct mpage_da_data *mpd, BUG_ON(!folio_test_locked(folio)); BUG_ON(folio_test_writeback(folio)); if (invalidate) { - if (folio_mapped(folio)) + if (folio_mapped(folio)) { folio_clear_dirty_for_io(folio); + /* + * Unmap folio from page + * tables to prevent + * subsequent accesses through + * stale PTEs. This ensures + * future accesses trigger new + * page faults rather than + * reusing the invalidated + * folio. + */ + unmap_mapping_pages(folio->mapping, + folio->index, + folio_nr_pages(folio), false); + } block_invalidate_folio(folio, 0, folio_size(folio)); folio_clear_uptodate(folio); @@ -2051,11 +2066,10 @@ static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio) folio_unlock(folio); } -static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) +static void mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) { size_t len; loff_t size; - int err; WARN_ON_ONCE(folio_pos(folio) != mpd->start_pos); folio_clear_dirty_for_io(folio); @@ -2077,9 +2091,7 @@ static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) if (folio_pos(folio) + len > size && !ext4_verity_in_progress(mpd->inode)) len = size & (len - 1); - err = ext4_bio_write_folio(&mpd->io_submit, folio, len); - - return err; + ext4_bio_write_folio(&mpd->io_submit, folio, len); } #define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay)) @@ -2156,8 +2168,7 @@ static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk, * accumulated extent of buffers to map or add buffers in the page to the * extent of buffers to map. The function returns 1 if the caller can continue * by processing the next page, 0 if it should stop adding buffers to the - * extent to map because we cannot extend it anymore. It can also return value - * < 0 in case of error during IO submission. + * extent to map because we cannot extend it anymore. */ static int mpage_process_page_bufs(struct mpage_da_data *mpd, struct buffer_head *head, @@ -2165,7 +2176,6 @@ static int mpage_process_page_bufs(struct mpage_da_data *mpd, ext4_lblk_t lblk) { struct inode *inode = mpd->inode; - int err; ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1) >> inode->i_blkbits; @@ -2188,9 +2198,7 @@ static int mpage_process_page_bufs(struct mpage_da_data *mpd, } while (lblk++, (bh = bh->b_this_page) != head); /* So far everything mapped? Submit the page for IO. */ if (mpd->map.m_len == 0) { - err = mpage_submit_folio(mpd, head->b_folio); - if (err < 0) - return err; + mpage_submit_folio(mpd, head->b_folio); mpage_folio_done(mpd, head->b_folio); } if (lblk >= blocks) { @@ -2320,9 +2328,7 @@ static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd) if (err < 0 || map_bh) goto out; /* Page fully mapped - let IO run! */ - err = mpage_submit_folio(mpd, folio); - if (err < 0) - goto out; + mpage_submit_folio(mpd, folio); mpage_folio_done(mpd, folio); } folio_batch_release(&fbatch); @@ -2395,7 +2401,6 @@ static int mpage_submit_partial_folio(struct mpage_da_data *mpd) struct inode *inode = mpd->inode; struct folio *folio; loff_t pos; - int ret; folio = filemap_get_folio(inode->i_mapping, mpd->start_pos >> PAGE_SHIFT); @@ -2410,9 +2415,7 @@ static int mpage_submit_partial_folio(struct mpage_da_data *mpd) !folio_contains(folio, pos >> PAGE_SHIFT))) return -EINVAL; - ret = mpage_submit_folio(mpd, folio); - if (ret) - goto out; + mpage_submit_folio(mpd, folio); /* * Update start_pos to prevent this folio from being released in * mpage_release_unused_pages(), it will be reset to the aligned folio @@ -2421,10 +2424,9 @@ static int mpage_submit_partial_folio(struct mpage_da_data *mpd) * entire folio has finished processing. */ mpd->start_pos = pos; -out: folio_unlock(folio); folio_put(folio); - return ret; + return 0; } /* @@ -2491,7 +2493,7 @@ static int mpage_map_and_submit_extent(handle_t *handle, } ext4_msg(sb, KERN_CRIT, "Delayed block allocation failed for " - "inode %lu at logical offset %llu with" + "inode %llu at logical offset %llu with" " max blocks %u with error %d", inode->i_ino, (unsigned long long)map->m_lblk, @@ -2535,7 +2537,7 @@ update_disksize: err2 = ext4_mark_inode_dirty(handle, inode); if (err2) { ext4_error_err(inode->i_sb, -err2, - "Failed to mark inode %lu dirty", + "Failed to mark inode %llu dirty", inode->i_ino); } if (!err) @@ -2670,13 +2672,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * page is already under writeback and we are not doing * a data integrity writeback, skip the page */ - if (!folio_test_dirty(folio) || - (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || unlikely(folio->mapping != mapping)) { folio_unlock(folio); continue; } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } folio_wait_writeback(folio); BUG_ON(folio_test_writeback(folio)); @@ -2711,9 +2725,8 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * through a pin. */ if (!mpd->can_map) { - err = mpage_submit_folio(mpd, folio); - if (err < 0) - goto out; + mpage_submit_folio(mpd, folio); + err = 0; /* Pending dirtying of journalled data? */ if (folio_test_checked(folio)) { err = mpage_journal_page_buffers(handle, @@ -2828,10 +2841,10 @@ static int ext4_do_writepages(struct mpage_da_data *mpd) if (ext4_should_dioread_nolock(inode)) { int bpf = ext4_journal_blocks_per_folio(inode); /* - * We may need to convert up to one extent per block in - * the folio and we may dirty the inode. + * We may need to convert up to one extent per block in the + * folio. */ - rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf); + rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); } if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) @@ -2909,7 +2922,7 @@ retry: if (IS_ERR(handle)) { ret = PTR_ERR(handle); ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: " - "%ld pages, ino %lu; err %d", __func__, + "%ld pages, ino %llu; err %d", __func__, wbc->nr_to_write, inode->i_ino, ret); /* Release allocated io_end */ ext4_put_io_end(mpd->io_submit.io_end); @@ -3031,17 +3044,23 @@ static int ext4_writepages(struct address_space *mapping, int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode) { + loff_t range_start, range_end; struct writeback_control wbc = { .sync_mode = WB_SYNC_ALL, .nr_to_write = LONG_MAX, - .range_start = jinode->i_dirty_start, - .range_end = jinode->i_dirty_end, }; struct mpage_da_data mpd = { .inode = jinode->i_vfs_inode, .wbc = &wbc, .can_map = 0, }; + + if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end)) + return 0; + + wbc.range_start = range_start; + wbc.range_end = range_end; + return ext4_do_writepages(&mpd); } @@ -3128,11 +3147,13 @@ static int ext4_da_write_begin(const struct kiocb *iocb, if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { ret = ext4_generic_write_inline_data(mapping, inode, pos, len, - foliop, fsdata, true); + foliop, true); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } retry: @@ -3196,7 +3217,7 @@ static int ext4_da_do_write_end(struct address_space *mapping, struct inode *inode = mapping->host; loff_t old_size = inode->i_size; bool disksize_changed = false; - loff_t new_i_size, zero_len = 0; + loff_t new_i_size; handle_t *handle; if (unlikely(!folio_buffers(folio))) { @@ -3240,19 +3261,15 @@ static int ext4_da_do_write_end(struct address_space *mapping, folio_unlock(folio); folio_put(folio); - if (pos > old_size) { + if (pos > old_size) pagecache_isize_extended(inode, old_size, pos); - zero_len = pos - old_size; - } - if (!disksize_changed && !zero_len) + if (!disksize_changed) return copied; - handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); if (IS_ERR(handle)) return PTR_ERR(handle); - if (zero_len) - ext4_zero_partial_blocks(handle, inode, old_size, zero_len); ext4_mark_inode_dirty(handle, inode); ext4_journal_stop(handle); @@ -3265,17 +3282,15 @@ static int ext4_da_write_end(const struct kiocb *iocb, struct folio *folio, void *fsdata) { struct inode *inode = mapping->host; - int write_mode = (int)(unsigned long)fsdata; + unsigned long write_mode = (unsigned long)fsdata; - if (write_mode == FALL_BACK_TO_NONDELALLOC) + if (write_mode & FALL_BACK_TO_NONDELALLOC) return ext4_write_end(iocb, mapping, pos, len, copied, folio, fsdata); trace_ext4_da_write_end(inode, pos, len, copied); - if (write_mode != CONVERT_INLINE_DATA && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && - ext4_has_inline_data(inode)) + if (write_mode & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -3426,6 +3441,7 @@ static bool ext4_release_folio(struct folio *folio, gfp_t wait) static bool ext4_inode_datasync_dirty(struct inode *inode) { journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; + struct mapping_metadata_bhs *mmb; if (journal) { if (jbd2_transaction_committed(journal, @@ -3436,8 +3452,9 @@ static bool ext4_inode_datasync_dirty(struct inode *inode) return true; } + mmb = ext4_i_metadata_bhs(inode); /* Any metadata buffers to write? */ - if (!list_empty(&inode->i_mapping->i_private_list)) + if (mmb && mmb_has_buffers(mmb)) return true; return inode_state_read_once(inode) & I_DIRTY_DATASYNC; } @@ -3647,6 +3664,9 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, int ret, dio_credits, m_flags = 0, retries = 0; bool force_commit = false; + if (flags & IOMAP_NOWAIT) + return -EAGAIN; + /* * Trim the mapping request to the maximum value that we can map at * once for direct I/O. @@ -3668,8 +3688,8 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, return ret; if (map->m_len < orig_mlen) { map->m_len = orig_mlen; - dio_credits = ext4_meta_trans_blocks(inode, orig_mlen, - map->m_len); + dio_credits = ext4_meta_trans_blocks(inode, map->m_len, + map->m_len, 0); } else { dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); @@ -3745,7 +3765,7 @@ retry: } -static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned flags, struct iomap *iomap, struct iomap *srcmap) { int ret; @@ -3803,9 +3823,9 @@ static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return ret; out: /* - * When inline encryption is enabled, sometimes I/O to an encrypted file - * has to be broken up to guarantee DUN contiguity. Handle this by - * limiting the length of the mapping returned. + * Sometimes I/O to an encrypted file has to be broken up to guarantee + * DUN contiguity. Handle this by limiting the length of the mapping + * returned. */ map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); @@ -3824,8 +3844,10 @@ out: return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); + const struct iomap_ops ext4_iomap_ops = { - .iomap_begin = ext4_iomap_begin, + .iomap_next = ext4_iomap_next, }; static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, @@ -3879,8 +3901,10 @@ set_iomap: return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); + const struct iomap_ops ext4_iomap_report_ops = { - .iomap_begin = ext4_iomap_begin_report, + .iomap_next = ext4_iomap_next_report, }; /* @@ -4001,22 +4025,25 @@ void ext4_set_aops(struct inode *inode) * because it might have data in pagecache (eg, if called from ext4_zero_range, * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a * racing writeback can come later and flush the stale pagecache to disk. + * + * Return the loaded bh if it actually needs zeroing - in written, dirty + * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or + * a clean unwritten block). */ -static int __ext4_block_zero_page_range(handle_t *handle, - struct address_space *mapping, loff_t from, loff_t length) +static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) { unsigned int offset, blocksize, pos; ext4_lblk_t iblock; - struct inode *inode = mapping->host; + struct address_space *mapping = inode->i_mapping; struct buffer_head *bh; struct folio *folio; int err = 0; folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + FGP_WRITEBEGIN | FGP_ACCESSED, mapping_gfp_constraint(mapping, ~__GFP_FS)); if (IS_ERR(folio)) - return PTR_ERR(folio); + return ERR_CAST(folio); blocksize = inode->i_sb->s_blocksize; @@ -4040,8 +4067,15 @@ static int __ext4_block_zero_page_range(handle_t *handle, } if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "unmapped"); - ext4_get_block(inode, iblock, bh, 0); - /* unmapped? It's a hole - nothing to do */ + err = ext4_get_block(inode, iblock, bh, 0); + if (err < 0) + goto unlock; + /* + * It's a hole or a clean unwritten block - nothing to do. + * Note that a lookup-only get_block (without + * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean + * unwritten extents. + */ if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "still unmapped"); goto unlock; @@ -4056,59 +4090,93 @@ static int __ext4_block_zero_page_range(handle_t *handle, err = ext4_read_bh_lock(bh, 0, true); if (err) goto unlock; - if (fscrypt_inode_uses_fs_layer_crypto(inode)) { - /* We expect the key to be set. */ - BUG_ON(!fscrypt_has_encryption_key(inode)); - err = fscrypt_decrypt_pagecache_blocks(folio, - blocksize, - bh_offset(bh)); - if (err) { - clear_buffer_uptodate(bh); - goto unlock; - } - } } - if (ext4_should_journal_data(inode)) { - BUFFER_TRACE(bh, "get write access"); - err = ext4_journal_get_write_access(handle, inode->i_sb, bh, - EXT4_JTR_NONE); - if (err) - goto unlock; - } - folio_zero_range(folio, offset, length); + return bh; + +unlock: + folio_unlock(folio); + folio_put(folio); + return err ? ERR_PTR(err) : NULL; +} + +static int ext4_block_do_zero_range(struct inode *inode, loff_t from, + loff_t length, bool *did_zero, + bool *zero_written) +{ + struct buffer_head *bh; + struct folio *folio; + + bh = ext4_load_tail_bh(inode, from); + if (IS_ERR_OR_NULL(bh)) + return PTR_ERR_OR_ZERO(bh); + + folio = bh->b_folio; + folio_zero_range(folio, offset_in_folio(folio, from), length); BUFFER_TRACE(bh, "zeroed end of block"); - if (ext4_should_journal_data(inode)) { - err = ext4_dirty_journalled_data(handle, bh); - } else { - mark_buffer_dirty(bh); - /* - * Only the written block requires ordered data to prevent - * exposing stale data. - */ - if (!buffer_unwritten(bh) && !buffer_delay(bh) && - ext4_should_order_data(inode)) - err = ext4_jbd2_inode_add_write(handle, inode, from, - length); + mark_buffer_dirty(bh); + if (did_zero) + *did_zero = true; + if (zero_written && !buffer_unwritten(bh) && !buffer_delay(bh)) + *zero_written = true; + + folio_unlock(folio); + folio_put(folio); + return 0; +} + +static int ext4_block_journalled_zero_range(struct inode *inode, loff_t from, + loff_t length, bool *did_zero) +{ + struct buffer_head *bh; + struct folio *folio; + handle_t *handle; + int err; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + bh = ext4_load_tail_bh(inode, from); + if (IS_ERR_OR_NULL(bh)) { + err = PTR_ERR_OR_ZERO(bh); + goto out_handle; } + folio = bh->b_folio; -unlock: + BUFFER_TRACE(bh, "get write access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (err) + goto out; + + folio_zero_range(folio, offset_in_folio(folio, from), length); + BUFFER_TRACE(bh, "zeroed end of block"); + + err = ext4_dirty_journalled_data(handle, bh); + if (err) + goto out; + + if (did_zero) + *did_zero = true; +out: folio_unlock(folio); folio_put(folio); +out_handle: + ext4_journal_stop(handle); return err; } /* - * ext4_block_zero_page_range() zeros out a mapping of length 'length' - * starting from file offset 'from'. The range to be zero'd must - * be contained with in one block. If the specified range exceeds - * the end of the block it will be shortened to end of the block - * that corresponds to 'from' + * Zeros out a mapping of length 'length' starting from file offset + * 'from'. The range to be zero'd must be contained with in one block. + * If the specified range exceeds the end of the block it will be + * shortened to end of the block that corresponds to 'from'. */ -static int ext4_block_zero_page_range(handle_t *handle, - struct address_space *mapping, loff_t from, loff_t length) +static int ext4_block_zero_range(struct inode *inode, + loff_t from, loff_t length, bool *did_zero, + bool *zero_written) { - struct inode *inode = mapping->host; unsigned blocksize = inode->i_sb->s_blocksize; unsigned int max = blocksize - (from & (blocksize - 1)); @@ -4120,43 +4188,97 @@ static int ext4_block_zero_page_range(handle_t *handle, length = max; if (IS_DAX(inode)) { - return dax_zero_range(inode, from, length, NULL, + return dax_zero_range(inode, from, length, did_zero, &ext4_iomap_ops); + } else if (ext4_should_journal_data(inode)) { + return ext4_block_journalled_zero_range(inode, from, length, + did_zero); } - return __ext4_block_zero_page_range(handle, mapping, from, length); + return ext4_block_do_zero_range(inode, from, length, did_zero, + zero_written); } /* - * ext4_block_truncate_page() zeroes out a mapping from file offset `from' - * up to the end of the block which corresponds to `from'. - * This required during truncate. We need to physically zero the tail end - * of that block so it doesn't yield old data if the file is later grown. + * Zero out a mapping from file offset 'from' up to the end of the block + * which corresponds to 'from' or to the given 'end' inside this block. + * This required during truncate up and performing append writes. We need + * to physically zero the tail end of that block so it doesn't yield old + * data if the file is grown. */ -static int ext4_block_truncate_page(handle_t *handle, - struct address_space *mapping, loff_t from) +int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) { - unsigned length; - unsigned blocksize; - struct inode *inode = mapping->host; + unsigned int blocksize = i_blocksize(inode); + unsigned int offset; + loff_t length = end - from; + bool did_zero = false; + bool zero_written = false; + int err; + offset = from & (blocksize - 1); + if (!offset || from >= end) + return 0; + /* + * Inline data has no tail block to zero out. Note that a race with + * ext4_page_mkwrite() converting inline data to an extent without + * holding i_rwsem is safe, as that path zeroes the full block before + * copying in the inline data. + */ + if (ext4_has_inline_data(inode)) + return 0; /* If we are processing an encrypted inode during orphan list handling */ if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) return 0; - blocksize = i_blocksize(inode); - length = blocksize - (from & (blocksize - 1)); + if (length > blocksize - offset) + length = blocksize - offset; + + err = ext4_block_zero_range(inode, from, length, + &did_zero, &zero_written); + if (err) + return err; + /* + * It's necessary to order zeroed data before update i_disksize when + * truncating up or performing an append write, because there might be + * exposing stale on-disk data which may caused by concurrent post-EOF + * mmap write during folio writeback. + */ + if (ext4_should_order_data(inode) && + did_zero && zero_written && !IS_DAX(inode)) { + handle_t *handle; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + err = ext4_jbd2_inode_add_write(handle, inode, from, length); + ext4_journal_stop(handle); + if (err) + return err; + } - return ext4_block_zero_page_range(handle, mapping, from, length); + return 0; } -int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode, - loff_t lstart, loff_t length) +/* + * Zero out the unaligned head and tail of the [lstart, lstart+length) + * range. + * + * On return, @partial_zeroed records which edges actually got + * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if + * the head/tail block got actually partially zeroed (in written, dirty + * unwritten or delalloc state). Cleared if the head/tail block is a + * hole or a clean unwritten block, in which case there is nothing that + * needs zeroing. When the head and tail land in the same block, both + * bits are set together on a successful zeroing. + */ +int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, + unsigned int *partial_zeroed) { struct super_block *sb = inode->i_sb; - struct address_space *mapping = inode->i_mapping; unsigned partial_start, partial_end; ext4_fsblk_t start, end; loff_t byte_end = (lstart + length - 1); + bool did_zero = false; int err = 0; partial_start = lstart & (sb->s_blocksize - 1); @@ -4168,22 +4290,32 @@ int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode, /* Handle partial zero within the single block */ if (start == end && (partial_start || (partial_end != sb->s_blocksize - 1))) { - err = ext4_block_zero_page_range(handle, mapping, - lstart, length); + err = ext4_block_zero_range(inode, lstart, length, &did_zero, + NULL); + if (did_zero) + *partial_zeroed |= (EXT4_PARTIAL_ZERO_START | + EXT4_PARTIAL_ZERO_END); return err; } /* Handle partial zero out on the start of the range */ if (partial_start) { - err = ext4_block_zero_page_range(handle, mapping, - lstart, sb->s_blocksize); + err = ext4_block_zero_range(inode, lstart, sb->s_blocksize, + &did_zero, NULL); if (err) return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_START; } /* Handle partial zero out on the end of the range */ - if (partial_end != sb->s_blocksize - 1) - err = ext4_block_zero_page_range(handle, mapping, - byte_end - partial_end, - partial_end + 1); + if (partial_end != sb->s_blocksize - 1) { + did_zero = false; + err = ext4_block_zero_range(inode, byte_end - partial_end, + partial_end + 1, &did_zero, NULL); + if (err) + return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_END; + } return err; } @@ -4332,6 +4464,7 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) loff_t end = offset + length; handle_t *handle; unsigned int credits; + unsigned int partial_zeroed = 0; int ret; trace_ext4_punch_hole(inode, offset, length, 0); @@ -4358,17 +4491,6 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) end = max_end; length = end - offset; - /* - * Attach jinode to inode for jbd2 if we do any zeroing of partial - * block. - */ - if (!IS_ALIGNED(offset | end, sb->s_blocksize)) { - ret = ext4_inode_attach_jinode(inode); - if (ret < 0) - return ret; - } - - ret = ext4_update_disksize_before_punch(inode, offset, length); if (ret) return ret; @@ -4378,8 +4500,18 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) if (ret) return ret; + ret = ext4_zero_partial_blocks(inode, offset, length, &partial_zeroed); + if (ret) + return ret; + if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) { + ret = filemap_write_and_wait_range(inode->i_mapping, offset, + end - 1); + if (ret) + return ret; + } + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) - credits = ext4_chunk_trans_extent(inode, 2); + credits = ext4_chunk_trans_extent(inode, 0); else credits = ext4_blocks_for_truncate(inode); handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); @@ -4389,10 +4521,6 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) return ret; } - ret = ext4_zero_partial_blocks(handle, inode, offset, length); - if (ret) - goto out_handle; - /* If there are blocks to remove, do it */ start_lblk = EXT4_B_TO_LBLK(inode, offset); end_lblk = end >> inode->i_blkbits; @@ -4429,7 +4557,7 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) goto out_handle; ext4_update_inode_fsync_trans(handle, inode, 1); - if (IS_SYNC(inode)) + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) ext4_handle_sync(handle); out_handle: ext4_journal_stop(handle); @@ -4451,8 +4579,13 @@ int ext4_inode_attach_jinode(struct inode *inode) spin_unlock(&inode->i_lock); return -ENOMEM; } - ei->jinode = jinode; - jbd2_journal_init_jbd_inode(ei->jinode, inode); + jbd2_journal_init_jbd_inode(jinode, inode); + /* + * Publish ->jinode only after it is fully initialized so that + * readers never observe a partially initialized jbd2_inode. + */ + smp_wmb(); + WRITE_ONCE(ei->jinode, jinode); jinode = NULL; } spin_unlock(&inode->i_lock); @@ -4495,7 +4628,6 @@ int ext4_truncate(struct inode *inode) unsigned int credits; int err = 0, err2; handle_t *handle; - struct address_space *mapping = inode->i_mapping; /* * There is a possibility that we're either freeing the inode @@ -4525,6 +4657,11 @@ int ext4_truncate(struct inode *inode) err = ext4_inode_attach_jinode(inode); if (err) goto out_trace; + + /* Zero to the end of the block containing i_size */ + err = ext4_block_zero_eof(inode, inode->i_size, LLONG_MAX); + if (err) + goto out_trace; } if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) @@ -4538,9 +4675,6 @@ int ext4_truncate(struct inode *inode) goto out_trace; } - if (inode->i_size & (inode->i_sb->s_blocksize - 1)) - ext4_block_truncate_page(handle, mapping, inode->i_size); - /* * We add the inode to the orphan list, so that if this * truncate spans multiple transactions, and we crash, we will @@ -4922,6 +5056,57 @@ int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) return ret; } +/* + * ext4_get_inode_loc_noio() is a best-effort variant of ext4_get_inode_loc(). + * It looks up the inode table block in the buffer cache and returns -EAGAIN if + * the block is not present or not uptodate, without starting any I/O. + */ +int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc) +{ + struct super_block *sb = inode->i_sb; + struct ext4_group_desc *gdp; + struct buffer_head *bh; + ext4_fsblk_t block; + int inodes_per_block, inode_offset; + unsigned long ino = inode->i_ino; + + iloc->bh = NULL; + if (ino < EXT4_ROOT_INO || + ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)) + return -EFSCORRUPTED; + + iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); + if (!gdp) + return -EIO; + + /* Figure out the offset within the block group inode table. */ + inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; + inode_offset = ((ino - 1) % EXT4_INODES_PER_GROUP(sb)); + iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); + + block = ext4_inode_table(sb, gdp); + if (block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block) || + block >= ext4_blocks_count(EXT4_SB(sb)->s_es)) { + ext4_error(sb, + "Invalid inode table block %llu in block_group %u", + block, iloc->block_group); + return -EFSCORRUPTED; + } + block += inode_offset / inodes_per_block; + + bh = sb_find_get_block(sb, block); + if (!bh) + return -EAGAIN; + if (!ext4_buffer_uptodate(bh)) { + brelse(bh); + return -EAGAIN; + } + + iloc->bh = bh; + return 0; +} + int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, struct ext4_iloc *iloc) @@ -5115,6 +5300,20 @@ void ext4_set_inode_mapping_order(struct inode *inode) mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); } +static int ext4_iget_match(struct inode *inode, u64 ino, void *data) +{ + if (inode->i_ino != ino) + return 0; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { + spin_unlock(&inode->i_lock); + return -1; + } + __iget(inode); + spin_unlock(&inode->i_lock); + return 1; +} + struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ext4_iget_flags flags, const char *function, unsigned int line) @@ -5143,9 +5342,24 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, return ERR_PTR(-EFSCORRUPTED); } - inode = iget_locked(sb, ino); - if (!inode) - return ERR_PTR(-ENOMEM); + if (flags & EXT4_IGET_NOWAIT) { + inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); + if (!inode) + return ERR_PTR(-ENOENT); + + if (inode_state_read_once(inode) & I_NEW) + wait_on_new_inode(inode); + + if (unlikely(inode_unhashed(inode))) { + iput(inode); + return ERR_PTR(-ENOENT); + } + } else { + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + } + if (!(inode_state_read_once(inode) & I_NEW)) { ret = check_igot_inode(inode, flags, function, line); if (ret) { @@ -5401,18 +5615,36 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, inode->i_op = &ext4_encrypted_symlink_inode_operations; } else if (ext4_inode_is_fast_symlink(inode)) { inode->i_op = &ext4_fast_symlink_inode_operations; - if (inode->i_size == 0 || - inode->i_size >= sizeof(ei->i_data) || - strnlen((char *)ei->i_data, inode->i_size + 1) != - inode->i_size) { - ext4_error_inode(inode, function, line, 0, - "invalid fast symlink length %llu", - (unsigned long long)inode->i_size); - ret = -EFSCORRUPTED; - goto bad_inode; + + /* + * Orphan cleanup can see inodes with i_size == 0 + * and i_data uninitialized. Skip size checks in + * that case. This is safe because the first thing + * ext4_evict_inode() does for fast symlinks is + * clearing of i_data and i_size. + */ + if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) { + if (inode->i_nlink != 0) { + ext4_error_inode(inode, function, line, 0, + "invalid orphan symlink nlink %d", + inode->i_nlink); + ret = -EFSCORRUPTED; + goto bad_inode; + } + } else { + if (inode->i_size == 0 || + inode->i_size >= sizeof(ei->i_data) || + strnlen((char *)ei->i_data, inode->i_size + 1) != + inode->i_size) { + ext4_error_inode(inode, function, line, 0, + "invalid fast symlink length %llu", + (unsigned long long)inode->i_size); + ret = -EFSCORRUPTED; + goto bad_inode; + } + inode_set_cached_link(inode, (char *)ei->i_data, + inode->i_size); } - inode_set_cached_link(inode, (char *)ei->i_data, - inode->i_size); } else { inode->i_op = &ext4_symlink_inode_operations; } @@ -5622,6 +5854,10 @@ out_brelse: * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL * writeback. * + * For nojournal mode all the work is done in ext4_sync_inode_metadata() + * because inode content is already copied into raw inode buffer and inode + * is marked with I_METADATA_WRITEBACK. + * * Note that we are absolutely dependent upon all inode dirtiers doing the * right thing: they *must* call mark_inode_dirty() after dirtying info in * which we are interested. @@ -5647,42 +5883,54 @@ int ext4_write_inode(struct inode *inode, struct writeback_control *wbc) if (unlikely(err)) return err; - if (EXT4_SB(inode->i_sb)->s_journal) { - if (ext4_journal_current_handle()) { - ext4_debug("called recursively, non-PF_MEMALLOC!\n"); - dump_stack(); - return -EIO; - } + if (!EXT4_SB(inode->i_sb)->s_journal) + return 0; - /* - * No need to force transaction in WB_SYNC_NONE mode. Also - * ext4_sync_fs() will force the commit after everything is - * written. - */ - if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) - return 0; + if (ext4_journal_current_handle()) { + ext4_debug("called recursively, non-PF_MEMALLOC!\n"); + dump_stack(); + return -EIO; + } - err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, + /* + * No need to force transaction in WB_SYNC_NONE mode. Also + * ext4_sync_fs() will force the commit after everything is + * written. + */ + if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) + return 0; + + return ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, EXT4_I(inode)->i_sync_tid); - } else { - struct ext4_iloc iloc; +} + +int ext4_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc) +{ + struct ext4_iloc iloc; + struct mapping_metadata_bhs *mmb; + int err; - err = __ext4_get_inode_loc_noinmem(inode, &iloc); + /* We should only get here in nojournal mode */ + if (WARN_ON_ONCE(EXT4_SB(inode->i_sb)->s_journal)) + return -EFSCORRUPTED; + + err = __ext4_get_inode_loc_noinmem(inode, &iloc); + if (err) + return err; + mmb = READ_ONCE(EXT4_I(inode)->i_metadata_bhs); + if (mmb) { + err = mmb_sync(mmb); if (err) - return err; - /* - * sync(2) will flush the whole buffer cache. No need to do - * it here separately for each inode. - */ - if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) - sync_dirty_buffer(iloc.bh); - if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) { - ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO, - "IO error syncing inode"); - err = -EIO; - } - brelse(iloc.bh); + goto out; + } + sync_dirty_buffer(iloc.bh); + if (buffer_write_io_error(iloc.bh)) { + ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO, + "IO error syncing inode"); + err = -EIO; } +out: + brelse(iloc.bh); return err; } @@ -5849,6 +6097,18 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (attr->ia_size == inode->i_size) inc_ivers = false; + /* + * If file has inline data but new size exceeds inline capacity, + * convert to extent-based storage first to prevent inconsistent + * state (inline flag set but size exceeds inline capacity). + */ + if (ext4_has_inline_data(inode) && + attr->ia_size > EXT4_I(inode)->i_inline_size) { + error = ext4_convert_inline_data(inode); + if (error) + goto err_out; + } + if (shrink) { if (ext4_should_order_data(inode)) { error = ext4_begin_ordered_truncate(inode, @@ -5880,15 +6140,6 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, goto out_mmap_sem; } - handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); - if (IS_ERR(handle)) { - error = PTR_ERR(handle); - goto out_mmap_sem; - } - if (ext4_handle_valid(handle) && shrink) { - error = ext4_orphan_add(handle, inode); - orphan = 1; - } /* * Update c/mtime and tail zero the EOF folio on * truncate up. ext4_truncate() handles the shrink case @@ -5897,9 +6148,22 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (!shrink) { inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - if (oldsize & (inode->i_sb->s_blocksize - 1)) - ext4_block_truncate_page(handle, - inode->i_mapping, oldsize); + if (oldsize & (inode->i_sb->s_blocksize - 1)) { + error = ext4_block_zero_eof(inode, + oldsize, LLONG_MAX); + if (error) + goto out_mmap_sem; + } + } + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); + if (IS_ERR(handle)) { + error = PTR_ERR(handle); + goto out_mmap_sem; + } + if (ext4_handle_valid(handle) && shrink) { + error = ext4_orphan_add(handle, inode); + orphan = 1; } if (shrink) @@ -5994,11 +6258,8 @@ u32 ext4_dio_alignment(struct inode *inode) return 0; if (ext4_has_inline_data(inode)) return 0; - if (IS_ENCRYPTED(inode)) { - if (!fscrypt_dio_supported(inode)) - return 0; + if (IS_ENCRYPTED(inode)) return i_blocksize(inode); - } return 1; /* use the iomap defaults */ } @@ -6017,11 +6278,7 @@ int ext4_getattr(struct mnt_idmap *idmap, const struct path *path, stat->btime.tv_nsec = ei->i_crtime.tv_nsec; } - /* - * Return the DIO alignment restrictions if requested. We only return - * this information when requested, since on encrypted files it might - * take a fair bit of work to get if the file wasn't opened recently. - */ + /* Return the DIO alignment restrictions if requested. */ if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { u32 dio_align = ext4_dio_alignment(inode); @@ -6118,17 +6375,17 @@ static int ext4_index_trans_blocks(struct inode *inode, int lblocks, } /* - * Account for index blocks, block groups bitmaps and block group - * descriptor blocks if modify datablocks and index blocks - * worse case, the indexs blocks spread over different block groups - * - * If datablocks are discontiguous, they are possible to spread over - * different block groups too. If they are contiguous, with flexbg, - * they could still across block group boundary. - * - * Also account for superblock, inode, quota and xattr blocks + * Calculate number of credits needed in a transaction to: + * * Allocate data blocks from @alloc_extents different groups - note that + * with flexbg a single physical extent can span multiple groups but + * single mballoc request only returns extent within one group. + * * Allocate metatadata (extent tree blocks, indirect blocks) to store + * pointers to @pextents data extents having @lblocks in total. + * * Modify extent tree / indirect block tree, inode, superblock, quota + * tracking, xattr blocks */ -int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) +int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, + int alloc_extents) { ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); int gdpblocks; @@ -6145,7 +6402,7 @@ int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) * Now let's see how many group bitmaps and group descriptors need * to account */ - groups = idxblocks + pextents; + groups = idxblocks + alloc_extents; gdpblocks = groups; if (groups > ngroups) groups = ngroups; @@ -6171,7 +6428,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) { int ret; - ret = ext4_meta_trans_blocks(inode, nrblocks, 1); + ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); /* Account for data blocks for journalled mode */ if (ext4_should_journal_data(inode)) ret += nrblocks; @@ -6189,7 +6446,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) */ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) { - return ext4_meta_trans_blocks(inode, nrblocks, 1); + return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); } /* @@ -6199,9 +6456,10 @@ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) int ext4_mark_iloc_dirty(handle_t *handle, struct inode *inode, struct ext4_iloc *iloc) { + struct super_block *sb = inode->i_sb; int err = 0; - err = ext4_emergency_state(inode->i_sb); + err = ext4_emergency_state(sb); if (unlikely(err)) { put_bh(iloc->bh); return err; @@ -6214,6 +6472,24 @@ int ext4_mark_iloc_dirty(handle_t *handle, /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */ err = ext4_do_update_inode(handle, inode, iloc); put_bh(iloc->bh); + /* + * Mark that there's metadata writeout pending for the inode so that it + * gets properly flushed on fsync(2) and similar. We don't bother for + * fastcommit replay as that flushes the whole bdev afterwards anyway. + * It is faster this way and we avoid entering fs writeback paths which + * aren't fully initialized yet. + */ + if (!ext4_handle_valid(handle) && + !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) { + /* + * Inode didn't need to go through dirtying, make sure it is + * attached to wb so that writeback can handle it. + */ + spin_lock(&inode->i_lock); + inode_attach_wb(inode, NULL); + spin_unlock(&inode->i_lock); + set_inode_metadata_writeback(inode); + } return err; } @@ -6322,6 +6598,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode, return -EOVERFLOW; /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + + /* * In nojournal mode, we can immediately attempt to expand * the inode. When journaled, we first need to obtain extra * buffer credits since we may write into the EA block diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 3ae9cb50a0c0..c8387e6a2c6e 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -477,7 +477,7 @@ static long swap_inode_boot_loader(struct super_block *sb, if (err < 0) { /* No need to update quota information. */ ext4_warning(inode->i_sb, - "couldn't mark inode #%lu dirty (err %d)", + "couldn't mark inode #%llu dirty (err %d)", inode->i_ino, err); /* Revert all changes: */ swap_inode_data(inode, inode_bl); @@ -493,7 +493,7 @@ static long swap_inode_boot_loader(struct super_block *sb, if (err < 0) { /* No need to update quota information. */ ext4_warning(inode_bl->i_sb, - "couldn't mark inode #%lu dirty (err %d)", + "couldn't mark inode #%llu dirty (err %d)", inode_bl->i_ino, err); goto revert; } @@ -830,11 +830,17 @@ int ext4_force_shutdown(struct super_block *sb, u32 flags) bdev_thaw(sb->s_bdev); break; case EXT4_GOING_FLAGS_LOGFLUSH: + /* + * Call ext4_force_commit() before setting EXT4_FLAGS_SHUTDOWN. + * This is because in data=ordered mode, journal commit + * triggers data writeback which fails if shutdown is already + * set, causing the journal to be aborted prematurely before + * the commit succeeds. + */ + (void) ext4_force_commit(sb); set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); - if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) { - (void) ext4_force_commit(sb); + if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) jbd2_journal_abort(sbi->s_journal, -ESHUTDOWN); - } break; case EXT4_GOING_FLAGS_NOLOGFLUSH: set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); @@ -1650,6 +1656,9 @@ group_extend_out: if (!(fd_file(donor)->f_mode & FMODE_WRITE)) return -EBADF; + if (file_inode(filp)->i_sb != file_inode(fd_file(donor))->i_sb) + return -EXDEV; + err = mnt_want_write_file(filp); if (err) return err; diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index 9fbdf6a09489..d31780075c21 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -5,9 +5,11 @@ #include <kunit/test.h> #include <kunit/static_stub.h> +#include <linux/fs_context.h> #include <linux/random.h> #include "ext4.h" +#include "mballoc.h" struct mbt_grp_ctx { struct buffer_head bitmap_bh; @@ -57,14 +59,15 @@ static const struct super_operations mbt_sops = { .free_inode = mbt_free_inode, }; -static void mbt_kill_sb(struct super_block *sb) +static int mbt_init_fs_context(struct fs_context *fc) { - generic_shutdown_super(sb); + return 0; } static struct file_system_type mbt_fs_type = { .name = "mballoc test", - .kill_sb = mbt_kill_sb, + .init_fs_context = mbt_init_fs_context, + .kill_sb = kill_anon_super, }; static int mbt_mb_init(struct super_block *sb) @@ -72,7 +75,7 @@ static int mbt_mb_init(struct super_block *sb) ext4_fsblk_t block; int ret; - /* needed by ext4_mb_init->bdev_nonrot(sb->s_bdev) */ + /* needed by ext4_mb_init->bdev_rot(sb->s_bdev) */ sb->s_bdev = kzalloc_obj(*sb->s_bdev); if (sb->s_bdev == NULL) return -ENOMEM; @@ -126,22 +129,28 @@ static void mbt_mb_release(struct super_block *sb) kfree(sb->s_bdev); } -static int mbt_set(struct super_block *sb, void *data) +static int mbt_set(struct super_block *sb, struct fs_context *fc) { - return 0; + return set_anon_super_fc(sb, fc); } static struct super_block *mbt_ext4_alloc_super_block(void) { struct mbt_ext4_super_block *fsb; struct super_block *sb; + struct fs_context *fc; struct ext4_sb_info *sbi; fsb = kzalloc_obj(*fsb); if (fsb == NULL) return NULL; - sb = sget(&mbt_fs_type, NULL, mbt_set, 0, NULL); + fc = fs_context_for_mount(&mbt_fs_type, 0); + if (IS_ERR(fc)) + goto out; + + sb = sget_fc(fc, NULL, mbt_set); + put_fs_context(fc); if (IS_ERR(sb)) goto out; @@ -336,7 +345,7 @@ ext4_mb_mark_context_stub(handle_t *handle, struct super_block *sb, bool state, if (state) mb_set_bits(bitmap_bh->b_data, blkoff, len); else - mb_clear_bits(bitmap_bh->b_data, blkoff, len); + mb_clear_bits_test(bitmap_bh->b_data, blkoff, len); return 0; } @@ -361,7 +370,6 @@ static int mbt_kunit_init(struct kunit *test) return ret; } - test->priv = sb; kunit_activate_static_stub(test, ext4_read_block_bitmap_nowait, ext4_read_block_bitmap_nowait_stub); @@ -382,6 +390,8 @@ static int mbt_kunit_init(struct kunit *test) return -ENOMEM; } + test->priv = sb; + return 0; } @@ -389,6 +399,9 @@ static void mbt_kunit_exit(struct kunit *test) { struct super_block *sb = (struct super_block *)test->priv; + if (!sb) + return; + mbt_mb_release(sb); mbt_ctx_release(sb); mbt_ext4_free_super_block(sb); @@ -413,14 +426,14 @@ static void test_new_blocks_simple(struct kunit *test) /* get block at goal */ ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ar.goal, found, "failed to alloc block at goal, expected %llu found %llu", ar.goal, found); /* get block after goal in goal group */ ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ar.goal + EXT4_C2B(sbi, 1), found, "failed to alloc block after goal in goal group, expected %llu found %llu", ar.goal + 1, found); @@ -428,7 +441,7 @@ static void test_new_blocks_simple(struct kunit *test) /* get block after goal group */ mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ext4_group_first_block_no(sb, goal_group + 1), found, "failed to alloc block after goal group, expected %llu found %llu", @@ -438,7 +451,7 @@ static void test_new_blocks_simple(struct kunit *test) for (i = goal_group; i < ext4_get_groups_count(sb); i++) mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_EQ_MSG(test, ext4_group_first_block_no(sb, 0) + EXT4_C2B(sbi, 1), found, "failed to alloc block before goal group, expected %llu found %llu", @@ -448,7 +461,7 @@ static void test_new_blocks_simple(struct kunit *test) for (i = 0; i < ext4_get_groups_count(sb); i++) mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); ar.goal = ext4_group_first_block_no(sb, goal_group); - found = ext4_mb_new_blocks_simple(&ar, &err); + found = ext4_mb_new_blocks_simple_test(&ar, &err); KUNIT_ASSERT_NE_MSG(test, err, 0, "unexpectedly get block when no block is available"); } @@ -492,16 +505,16 @@ validate_free_blocks_simple(struct kunit *test, struct super_block *sb, continue; bitmap = mbt_ctx_bitmap(sb, i); - bit = mb_find_next_zero_bit(bitmap, max, 0); + bit = mb_find_next_zero_bit_test(bitmap, max, 0); KUNIT_ASSERT_EQ_MSG(test, bit, max, "free block on unexpected group %d", i); } bitmap = mbt_ctx_bitmap(sb, goal_group); - bit = mb_find_next_zero_bit(bitmap, max, 0); + bit = mb_find_next_zero_bit_test(bitmap, max, 0); KUNIT_ASSERT_EQ(test, bit, start); - bit = mb_find_next_bit(bitmap, max, bit + 1); + bit = mb_find_next_bit_test(bitmap, max, bit + 1); KUNIT_ASSERT_EQ(test, bit, start + len); } @@ -524,7 +537,7 @@ test_free_blocks_simple_range(struct kunit *test, ext4_group_t goal_group, block = ext4_group_first_block_no(sb, goal_group) + EXT4_C2B(sbi, start); - ext4_free_blocks_simple(inode, block, len); + ext4_free_blocks_simple_test(inode, block, len); validate_free_blocks_simple(test, sb, goal_group, start, len); mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); } @@ -566,15 +579,15 @@ test_mark_diskspace_used_range(struct kunit *test, bitmap = mbt_ctx_bitmap(sb, TEST_GOAL_GROUP); memset(bitmap, 0, sb->s_blocksize); - ret = ext4_mb_mark_diskspace_used(ac, NULL); + ret = ext4_mb_mark_diskspace_used_test(ac, NULL); KUNIT_ASSERT_EQ(test, ret, 0); max = EXT4_CLUSTERS_PER_GROUP(sb); - i = mb_find_next_bit(bitmap, max, 0); + i = mb_find_next_bit_test(bitmap, max, 0); KUNIT_ASSERT_EQ(test, i, start); - i = mb_find_next_zero_bit(bitmap, max, i + 1); + i = mb_find_next_zero_bit_test(bitmap, max, i + 1); KUNIT_ASSERT_EQ(test, i, start + len); - i = mb_find_next_bit(bitmap, max, i + 1); + i = mb_find_next_bit_test(bitmap, max, i + 1); KUNIT_ASSERT_EQ(test, max, i); } @@ -617,54 +630,54 @@ static void mbt_generate_buddy(struct super_block *sb, void *buddy, max = EXT4_CLUSTERS_PER_GROUP(sb); bb_h = buddy + sbi->s_mb_offsets[1]; - off = mb_find_next_zero_bit(bb, max, 0); + off = mb_find_next_zero_bit_test(bb, max, 0); grp->bb_first_free = off; while (off < max) { grp->bb_counters[0]++; grp->bb_free++; - if (!(off & 1) && !mb_test_bit(off + 1, bb)) { + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { grp->bb_free++; grp->bb_counters[0]--; - mb_clear_bit(off >> 1, bb_h); + mb_clear_bit_test(off >> 1, bb_h); grp->bb_counters[1]++; grp->bb_largest_free_order = 1; off++; } - off = mb_find_next_zero_bit(bb, max, off + 1); + off = mb_find_next_zero_bit_test(bb, max, off + 1); } for (order = 1; order < MB_NUM_ORDERS(sb) - 1; order++) { bb = buddy + sbi->s_mb_offsets[order]; bb_h = buddy + sbi->s_mb_offsets[order + 1]; max = max >> 1; - off = mb_find_next_zero_bit(bb, max, 0); + off = mb_find_next_zero_bit_test(bb, max, 0); while (off < max) { - if (!(off & 1) && !mb_test_bit(off + 1, bb)) { + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { mb_set_bits(bb, off, 2); grp->bb_counters[order] -= 2; - mb_clear_bit(off >> 1, bb_h); + mb_clear_bit_test(off >> 1, bb_h); grp->bb_counters[order + 1]++; grp->bb_largest_free_order = order + 1; off++; } - off = mb_find_next_zero_bit(bb, max, off + 1); + off = mb_find_next_zero_bit_test(bb, max, off + 1); } } max = EXT4_CLUSTERS_PER_GROUP(sb); - off = mb_find_next_zero_bit(bitmap, max, 0); + off = mb_find_next_zero_bit_test(bitmap, max, 0); while (off < max) { grp->bb_fragments++; - off = mb_find_next_bit(bitmap, max, off + 1); + off = mb_find_next_bit_test(bitmap, max, off + 1); if (off + 1 >= max) break; - off = mb_find_next_zero_bit(bitmap, max, off + 1); + off = mb_find_next_zero_bit_test(bitmap, max, off + 1); } } @@ -706,11 +719,10 @@ do_test_generate_buddy(struct kunit *test, struct super_block *sb, void *bitmap, /* needed by validation in ext4_mb_generate_buddy */ ext4_grp->bb_free = mbt_grp->bb_free; memset(ext4_buddy, 0xff, sb->s_blocksize); - ext4_mb_generate_buddy(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, + ext4_mb_generate_buddy_test(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, ext4_grp); - KUNIT_ASSERT_EQ(test, memcmp(mbt_buddy, ext4_buddy, sb->s_blocksize), - 0); + KUNIT_ASSERT_MEMEQ(test, mbt_buddy, ext4_buddy, sb->s_blocksize); mbt_validate_group_info(test, mbt_grp, ext4_grp); } @@ -760,7 +772,7 @@ test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, ex.fe_group = TEST_GOAL_GROUP; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_mark_used(e4b, &ex); + mb_mark_used_test(e4b, &ex); ext4_unlock_group(sb, TEST_GOAL_GROUP); mb_set_bits(bitmap, start, len); @@ -769,10 +781,9 @@ test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, memset(buddy, 0xff, sb->s_blocksize); for (i = 0; i < MB_NUM_ORDERS(sb); i++) grp->bb_counters[i] = 0; - ext4_mb_generate_buddy(sb, buddy, bitmap, 0, grp); + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); - KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), - 0); + KUNIT_ASSERT_MEMEQ(test, buddy, e4b->bd_buddy, sb->s_blocksize); mbt_validate_group_info(test, grp, e4b->bd_info); } @@ -798,7 +809,7 @@ static void test_mb_mark_used(struct kunit *test) bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); - ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); KUNIT_ASSERT_EQ(test, ret, 0); grp->bb_free = EXT4_CLUSTERS_PER_GROUP(sb); @@ -809,7 +820,7 @@ static void test_mb_mark_used(struct kunit *test) test_mb_mark_used_range(test, &e4b, ranges[i].start, ranges[i].len, bitmap, buddy, grp); - ext4_mb_unload_buddy(&e4b); + ext4_mb_unload_buddy_test(&e4b); } static void @@ -825,19 +836,18 @@ test_mb_free_blocks_range(struct kunit *test, struct ext4_buddy *e4b, return; ext4_lock_group(sb, e4b->bd_group); - mb_free_blocks(NULL, e4b, start, len); + mb_free_blocks_test(NULL, e4b, start, len); ext4_unlock_group(sb, e4b->bd_group); - mb_clear_bits(bitmap, start, len); + mb_clear_bits_test(bitmap, start, len); /* bypass bb_free validatoin in ext4_mb_generate_buddy */ grp->bb_free += len; memset(buddy, 0xff, sb->s_blocksize); for (i = 0; i < MB_NUM_ORDERS(sb); i++) grp->bb_counters[i] = 0; - ext4_mb_generate_buddy(sb, buddy, bitmap, 0, grp); + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); - KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), - 0); + KUNIT_ASSERT_MEMEQ(test, buddy, e4b->bd_buddy, sb->s_blocksize); mbt_validate_group_info(test, grp, e4b->bd_info); } @@ -865,7 +875,7 @@ static void test_mb_free_blocks(struct kunit *test) bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); - ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); KUNIT_ASSERT_EQ(test, ret, 0); ex.fe_start = 0; @@ -873,7 +883,7 @@ static void test_mb_free_blocks(struct kunit *test) ex.fe_group = TEST_GOAL_GROUP; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_mark_used(&e4b, &ex); + mb_mark_used_test(&e4b, &ex); ext4_unlock_group(sb, TEST_GOAL_GROUP); grp->bb_free = 0; @@ -886,7 +896,7 @@ static void test_mb_free_blocks(struct kunit *test) test_mb_free_blocks_range(test, &e4b, ranges[i].start, ranges[i].len, bitmap, buddy, grp); - ext4_mb_unload_buddy(&e4b); + ext4_mb_unload_buddy_test(&e4b); } #define COUNT_FOR_ESTIMATE 100000 @@ -904,7 +914,7 @@ static void test_mb_mark_used_cost(struct kunit *test) if (sb->s_blocksize > PAGE_SIZE) kunit_skip(test, "blocksize exceeds pagesize"); - ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); KUNIT_ASSERT_EQ(test, ret, 0); ex.fe_group = TEST_GOAL_GROUP; @@ -918,7 +928,7 @@ static void test_mb_mark_used_cost(struct kunit *test) ex.fe_start = ranges[i].start; ex.fe_len = ranges[i].len; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_mark_used(&e4b, &ex); + mb_mark_used_test(&e4b, &ex); ext4_unlock_group(sb, TEST_GOAL_GROUP); } end = jiffies; @@ -929,14 +939,14 @@ static void test_mb_mark_used_cost(struct kunit *test) continue; ext4_lock_group(sb, TEST_GOAL_GROUP); - mb_free_blocks(NULL, &e4b, ranges[i].start, + mb_free_blocks_test(NULL, &e4b, ranges[i].start, ranges[i].len); ext4_unlock_group(sb, TEST_GOAL_GROUP); } } kunit_info(test, "costed jiffies %lu\n", all); - ext4_mb_unload_buddy(&e4b); + ext4_mb_unload_buddy_test(&e4b); } static const struct mbt_ext4_block_layout mbt_test_layouts[] = { diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 20e9fdaf4301..06171a11db12 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -1199,6 +1199,8 @@ static int ext4_mb_scan_groups(struct ext4_allocation_context *ac) /* searching for the right group start from the goal value specified */ start = ac->ac_g_ex.fe_group; + if (start >= ngroups) + start = 0; ac->ac_prefetch_grp = start; ac->ac_prefetch_nr = 0; @@ -2266,7 +2268,7 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, folio_get(ac->ac_buddy_folio); /* store last allocated for subsequent stream allocation */ if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { - int hash = ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; + int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; WRITE_ONCE(sbi->s_mb_last_groups[hash], ac->ac_f_ex.fe_group); } @@ -2443,8 +2445,12 @@ int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, return 0; err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); - if (err) + if (err) { + if (EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info) && + !(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) + return 0; return err; + } ext4_lock_group(ac->ac_sb, group); if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) @@ -2855,8 +2861,6 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, blk_start_plug(&plug); while (nr-- > 0) { - struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, - NULL); struct ext4_group_info *grp = ext4_get_group_info(sb, group); /* @@ -2866,14 +2870,17 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, * prefetch once, so we avoid getblk() call, which can * be expensive. */ - if (gdp && grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && - EXT4_MB_GRP_NEED_INIT(grp) && - ext4_free_group_clusters(sb, gdp) > 0 ) { - bh = ext4_read_block_bitmap_nowait(sb, group, true); - if (bh && !IS_ERR(bh)) { - if (!buffer_uptodate(bh) && cnt) - (*cnt)++; - brelse(bh); + if (grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && + EXT4_MB_GRP_NEED_INIT(grp)) { + struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, NULL); + + if (gdp && ext4_free_group_clusters(sb, gdp) > 0) { + bh = ext4_read_block_bitmap_nowait(sb, group, true); + if (!IS_ERR_OR_NULL(bh)) { + if (!buffer_uptodate(bh) && cnt) + (*cnt)++; + brelse(bh); + } } } if (++group >= ngroups) @@ -3032,7 +3039,7 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) /* if stream allocation is enabled, use global goal */ if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { - int hash = ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; + int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; ac->ac_g_ex.fe_group = READ_ONCE(sbi->s_mb_last_groups[hash]); ac->ac_g_ex.fe_start = -1; @@ -3580,9 +3587,7 @@ err_freebuddy: rcu_read_unlock(); iput(sbi->s_buddy_cache); err_freesgi: - rcu_read_lock(); - kvfree(rcu_dereference(sbi->s_group_info)); - rcu_read_unlock(); + kvfree(rcu_access_pointer(sbi->s_group_info)); return -ENOMEM; } @@ -3836,7 +3841,7 @@ int ext4_mb_init(struct super_block *sb) spin_lock_init(&lg->lg_prealloc_lock); } - if (bdev_nonrot(sb->s_bdev)) + if (!bdev_rot(sb->s_bdev)) sbi->s_mb_max_linear_groups = 0; else sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT; @@ -3889,15 +3894,14 @@ void ext4_mb_release(struct super_block *sb) struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); int count; - if (test_opt(sb, DISCARD)) { - /* - * wait the discard work to drain all of ext4_free_data - */ - flush_work(&sbi->s_discard_work); - WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); - } + /* + * wait the discard work to drain all of ext4_free_data + */ + flush_work(&sbi->s_discard_work); + WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); - if (sbi->s_group_info) { + group_info = rcu_access_pointer(sbi->s_group_info); + if (group_info) { for (i = 0; i < ngroups; i++) { cond_resched(); grinfo = ext4_get_group_info(sb, i); @@ -3915,12 +3919,9 @@ void ext4_mb_release(struct super_block *sb) num_meta_group_infos = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> EXT4_DESC_PER_BLOCK_BITS(sb); - rcu_read_lock(); - group_info = rcu_dereference(sbi->s_group_info); for (i = 0; i < num_meta_group_infos; i++) kfree(group_info[i]); kvfree(group_info); - rcu_read_unlock(); } ext4_mb_avg_fragment_size_destroy(sbi); ext4_mb_largest_free_orders_destroy(sbi); @@ -4084,7 +4085,7 @@ void ext4_exit_mballoc(void) #define EXT4_MB_BITMAP_MARKED_CHECK 0x0001 #define EXT4_MB_SYNC_UPDATE 0x0002 -static int +int ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, ext4_group_t group, ext4_grpblk_t blkoff, ext4_grpblk_t len, int flags, ext4_grpblk_t *ret_changed) @@ -4561,22 +4562,16 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, (req <= (size) || max <= (chunk_size)) /* first, try to predict filesize */ - /* XXX: should this table be tunable? */ start_off = 0; - if (size <= 16 * 1024) { - size = 16 * 1024; - } else if (size <= 32 * 1024) { - size = 32 * 1024; - } else if (size <= 64 * 1024) { - size = 64 * 1024; - } else if (size <= 128 * 1024) { - size = 128 * 1024; - } else if (size <= 256 * 1024) { - size = 256 * 1024; - } else if (size <= 512 * 1024) { - size = 512 * 1024; - } else if (size <= 1024 * 1024) { - size = 1024 * 1024; + if (size <= SZ_1M) { + /* + * For files up to 1MB, round up the preallocation size to + * the next power of two, with a minimum of 16KB. + */ + if (size <= (unsigned long)SZ_16K) + size = SZ_16K; + else + size = roundup_pow_of_two(size); } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { start_off = ((loff_t)ac->ac_o_ex.fe_logical >> (21 - bsbits)) << 21; @@ -5628,7 +5623,7 @@ void ext4_discard_preallocations(struct inode *inode) if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) return; - mb_debug(sb, "discard preallocation for inode %lu\n", + mb_debug(sb, "discard preallocation for inode %llu\n", inode->i_ino); trace_ext4_discard_preallocations(inode, atomic_read(&ei->i_prealloc_active)); @@ -7188,6 +7183,102 @@ out_unload: return error; } -#ifdef CONFIG_EXT4_KUNIT_TESTS -#include "mballoc-test.c" +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +void mb_clear_bits_test(void *bm, int cur, int len) +{ + mb_clear_bits(bm, cur, len); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bits_test); + +ext4_fsblk_t +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, + int *errp) +{ + return ext4_mb_new_blocks_simple(ar, errp); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_new_blocks_simple_test); + +int mb_find_next_zero_bit_test(void *addr, int max, int start) +{ + return mb_find_next_zero_bit(addr, max, start); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_zero_bit_test); + +int mb_find_next_bit_test(void *addr, int max, int start) +{ + return mb_find_next_bit(addr, max, start); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_bit_test); + +void mb_clear_bit_test(int bit, void *addr) +{ + mb_clear_bit(bit, addr); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bit_test); + +int mb_test_bit_test(int bit, void *addr) +{ + return mb_test_bit(bit, addr); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_test_bit_test); + +int ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, + handle_t *handle) +{ + return ext4_mb_mark_diskspace_used(ac, handle); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_diskspace_used_test); + +int mb_mark_used_test(struct ext4_buddy *e4b, struct ext4_free_extent *ex) +{ + return mb_mark_used(e4b, ex); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_mark_used_test); + +void ext4_mb_generate_buddy_test(struct super_block *sb, void *buddy, + void *bitmap, ext4_group_t group, + struct ext4_group_info *grp) +{ + ext4_mb_generate_buddy(sb, buddy, bitmap, group, grp); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_generate_buddy_test); + +int ext4_mb_load_buddy_test(struct super_block *sb, ext4_group_t group, + struct ext4_buddy *e4b) +{ + return ext4_mb_load_buddy(sb, group, e4b); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_load_buddy_test); + +void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b) +{ + ext4_mb_unload_buddy(e4b); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_unload_buddy_test); + +void mb_free_blocks_test(struct inode *inode, struct ext4_buddy *e4b, + int first, int count) +{ + mb_free_blocks(inode, e4b, first, count); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_free_blocks_test); + +void ext4_free_blocks_simple_test(struct inode *inode, ext4_fsblk_t block, + unsigned long count) +{ + return ext4_free_blocks_simple(inode, block, count); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_blocks_simple_test); + +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_wait_block_bitmap); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_init); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_desc); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_count_free_clusters); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_info); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_group_clusters_set); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_release); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_read_block_bitmap_nowait); +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_set_bits); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_fc_init_inode); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_context); #endif diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 15a049f05d04..39333ce72cbd 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -270,4 +270,34 @@ ext4_mballoc_query_range( ext4_mballoc_query_range_fn formatter, void *priv); +extern int ext4_mb_mark_context(handle_t *handle, + struct super_block *sb, bool state, + ext4_group_t group, ext4_grpblk_t blkoff, + ext4_grpblk_t len, int flags, + ext4_grpblk_t *ret_changed); +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +extern void mb_clear_bits_test(void *bm, int cur, int len); +extern ext4_fsblk_t +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, + int *errp); +extern int mb_find_next_zero_bit_test(void *addr, int max, int start); +extern int mb_find_next_bit_test(void *addr, int max, int start); +extern void mb_clear_bit_test(int bit, void *addr); +extern int mb_test_bit_test(int bit, void *addr); +extern int +ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, + handle_t *handle); +extern int mb_mark_used_test(struct ext4_buddy *e4b, + struct ext4_free_extent *ex); +extern void ext4_mb_generate_buddy_test(struct super_block *sb, + void *buddy, void *bitmap, ext4_group_t group, + struct ext4_group_info *grp); +extern int ext4_mb_load_buddy_test(struct super_block *sb, + ext4_group_t group, struct ext4_buddy *e4b); +extern void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b); +extern void mb_free_blocks_test(struct inode *inode, + struct ext4_buddy *e4b, int first, int count); +extern void ext4_free_blocks_simple_test(struct inode *inode, + ext4_fsblk_t block, unsigned long count); +#endif #endif diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 96ab95167bd6..5d60ef10fe11 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -455,7 +455,7 @@ int ext4_ext_migrate(struct inode *inode) * log, so disable fast commits for this transaction. */ ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_MIGRATE, handle); - goal = (((inode->i_ino - 1) / EXT4_INODES_PER_GROUP(inode->i_sb)) * + goal = ((((u32)inode->i_ino - 1) / EXT4_INODES_PER_GROUP(inode->i_sb)) * EXT4_INODES_PER_GROUP(inode->i_sb)) + 1; owner[0] = i_uid_read(inode); owner[1] = i_gid_read(inode); @@ -464,6 +464,7 @@ int ext4_ext_migrate(struct inode *inode) if (IS_ERR(tmp_inode)) { retval = PTR_ERR(tmp_inode); ext4_journal_stop(handle); + tmp_inode = NULL; goto out_unlock; } /* @@ -591,9 +592,9 @@ out_stop: ext4_journal_stop(handle); out_tmp_inode: unlock_new_inode(tmp_inode); - iput(tmp_inode); out_unlock: ext4_writepages_up_write(inode->i_sb, alloc_ctx); + iput(tmp_inode); return retval; } diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c index 6f57c181ff77..7ce361484b38 100644 --- a/fs/ext4/mmp.c +++ b/fs/ext4/mmp.c @@ -46,9 +46,8 @@ static int write_mmp_block_thawed(struct super_block *sb, ext4_mmp_csum_set(sb, mmp); lock_buffer(bh); - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, bh); + bh_submit(bh, REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, + bh_end_write); wait_on_buffer(bh); if (unlikely(!buffer_uptodate(bh))) return -EIO; diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c index ce1f738dff93..3329b7ad5dbd 100644 --- a/fs/ext4/move_extent.c +++ b/fs/ext4/move_extent.c @@ -224,8 +224,8 @@ static int mext_move_begin(struct mext_data *mext, struct folio *folio[2], } /* Adjust the moving length according to the length of shorter folio. */ - move_len = umin(folio_pos(folio[0]) + folio_size(folio[0]) - orig_pos, - folio_pos(folio[1]) + folio_size(folio[1]) - donor_pos); + move_len = umin(folio_next_pos(folio[0]) - orig_pos, + folio_next_pos(folio[1]) - donor_pos); move_len >>= blkbits; if (move_len < mext->orig_map.m_len) mext->orig_map.m_len = move_len; @@ -420,21 +420,21 @@ static int mext_check_validity(struct inode *orig_inode, /* origin and donor should be different inodes */ if (orig_inode == donor_inode) { - ext4_debug("ext4 move extent: The argument files should not be same inode [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: The argument files should not be same inode [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EINVAL; } /* origin and donor should belone to the same filesystem */ if (orig_inode->i_sb != donor_inode->i_sb) { - ext4_debug("ext4 move extent: The argument files should be in same FS [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: The argument files should be in same FS [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EINVAL; } /* Regular file check */ if (!S_ISREG(orig_inode->i_mode) || !S_ISREG(donor_inode->i_mode)) { - ext4_debug("ext4 move extent: The argument files should be regular file [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: The argument files should be regular file [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EINVAL; } @@ -477,26 +477,26 @@ static int mext_check_validity(struct inode *orig_inode, } if (donor_inode->i_mode & (S_ISUID|S_ISGID)) { - ext4_debug("ext4 move extent: suid or sgid is set to donor file [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: suid or sgid is set to donor file [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EINVAL; } if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode)) { - ext4_debug("ext4 move extent: donor should not be immutable or append file [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: donor should not be immutable or append file [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EPERM; } /* Ext4 move extent does not support swap files */ if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) { - ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -ETXTBSY; } if (ext4_is_quota_file(orig_inode) || ext4_is_quota_file(donor_inode)) { - ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EOPNOTSUPP; } @@ -523,7 +523,7 @@ static int mext_check_adjust_range(struct inode *orig_inode, /* Start offset should be same */ if ((orig_start & ~(PAGE_MASK >> orig_inode->i_blkbits)) != (donor_start & ~(PAGE_MASK >> orig_inode->i_blkbits))) { - ext4_debug("ext4 move extent: orig and donor's start offsets are not aligned [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: orig and donor's start offsets are not aligned [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EINVAL; } @@ -533,7 +533,7 @@ static int mext_check_adjust_range(struct inode *orig_inode, (*len > EXT_MAX_BLOCKS) || (donor_start + *len >= EXT_MAX_BLOCKS) || (orig_start + *len >= EXT_MAX_BLOCKS)) { - ext4_debug("ext4 move extent: Can't handle over [%u] blocks [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: Can't handle over [%u] blocks [ino:orig %llu, donor %llu]\n", EXT_MAX_BLOCKS, orig_inode->i_ino, donor_inode->i_ino); return -EINVAL; @@ -550,7 +550,7 @@ static int mext_check_adjust_range(struct inode *orig_inode, else if (donor_eof < donor_start + *len - 1) *len = donor_eof - donor_start; if (!*len) { - ext4_debug("ext4 move extent: len should not be 0 [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: len should not be 0 [ino:orig %llu, donor %llu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EINVAL; } diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index c4b5e252af0e..a6386c1d237f 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -144,10 +144,10 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode, bh = ext4_bread(NULL, inode, block, 0); if (IS_ERR(bh)) { __ext4_warning(inode->i_sb, func, line, - "inode #%lu: lblock %lu: comm %s: " - "error %ld reading directory block", + "inode #%llu: lblock %lu: comm %s: " + "error %pe reading directory block", inode->i_ino, (unsigned long)block, - current->comm, PTR_ERR(bh)); + current->comm, bh); return bh; } @@ -647,7 +647,7 @@ static struct stats dx_show_leaf(struct inode *dir, /* Directory is not encrypted */ (void) ext4fs_dirhash(dir, de->name, de->name_len, &h); - printk("%*.s:(U)%x.%u ", len, + printk("%.*s:(U)%x.%u ", len, name, h.hash, (unsigned) ((char *) de - base)); @@ -683,7 +683,7 @@ static struct stats dx_show_leaf(struct inode *dir, (void) ext4fs_dirhash(dir, de->name, de->name_len, &h); - printk("%*.s:(E)%x.%u ", len, name, + printk("%.*s:(E)%x.%u ", len, name, h.hash, (unsigned) ((char *) de - base)); fscrypt_fname_free_buffer( @@ -694,7 +694,7 @@ static struct stats dx_show_leaf(struct inode *dir, char *name = de->name; (void) ext4fs_dirhash(dir, de->name, de->name_len, &h); - printk("%*.s:%x.%u ", len, name, h.hash, + printk("%.*s:%x.%u ", len, name, h.hash, (unsigned) ((char *) de - base)); #endif } @@ -723,7 +723,7 @@ struct stats dx_show_entries(struct dx_hash_info *hinfo, struct inode *dir, struct stats stats; printk("%s%3u:%03u hash %8x/%8x ",levels?"":" ", i, block, hash, range); bh = ext4_bread(NULL,dir, block, 0); - if (!bh || IS_ERR(bh)) + if (IS_ERR_OR_NULL(bh)) continue; stats = levels? dx_show_entries(hinfo, dir, ((struct dx_node *) bh->b_data)->entries, levels - 1): @@ -841,7 +841,7 @@ dx_probe(struct ext4_filename *fname, struct inode *dir, indirect = root->info.indirect_levels; if (indirect >= ext4_dir_htree_level(dir->i_sb)) { ext4_warning(dir->i_sb, - "Directory (ino: %lu) htree depth %#06x exceed" + "Directory (ino: %llu) htree depth %#06x exceed" "supported value", dir->i_ino, ext4_dir_htree_level(dir->i_sb)); if (ext4_dir_htree_level(dir->i_sb) < EXT4_HTREE_LEVEL) { @@ -1467,6 +1467,8 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, /* this code is executed quadratically often */ /* do minimal checking `by hand' */ if (de->name + de->name_len <= dlimit && + (!ext4_hash_in_dirent(dir) || + (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) && ext4_match(dir, fname, de)) { /* found a match - just to be sure, do * a full check */ @@ -1793,7 +1795,7 @@ static struct dentry *ext4_lookup(struct inode *dir, struct dentry *dentry, unsi (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) && !fscrypt_has_permitted_context(dir, inode)) { ext4_warning(inode->i_sb, - "Inconsistent encryption contexts: %lu/%lu", + "Inconsistent encryption contexts: %llu/%llu", dir->i_ino, inode->i_ino); iput(inode); return ERR_PTR(-EPERM); @@ -2227,7 +2229,7 @@ static int make_indexed_dir(handle_t *handle, struct ext4_filename *fname, csum_size = sizeof(struct ext4_dir_entry_tail); blocksize = dir->i_sb->s_blocksize; - dxtrace(printk(KERN_DEBUG "Creating index: inode %lu\n", dir->i_ino)); + dxtrace(printk(KERN_DEBUG "Creating index: inode %llu\n", dir->i_ino)); BUFFER_TRACE(bh, "get_write_access"); retval = ext4_journal_get_write_access(handle, dir->i_sb, bh, EXT4_JTR_NONE); @@ -2353,10 +2355,10 @@ out_frames: * may not sleep between calling this and putting something into * the entry, as someone else might have used it while you slept. */ -static int ext4_add_entry(handle_t *handle, struct dentry *dentry, +static int __ext4_add_entry(handle_t *handle, struct inode *dir, + const struct qstr *d_name, struct inode *inode) { - struct inode *dir = d_inode(dentry->d_parent); struct buffer_head *bh = NULL; struct ext4_dir_entry_2 *de; struct super_block *sb; @@ -2373,13 +2375,10 @@ static int ext4_add_entry(handle_t *handle, struct dentry *dentry, sb = dir->i_sb; blocksize = sb->s_blocksize; - if (fscrypt_is_nokey_name(dentry)) - return -ENOKEY; - - if (!generic_ci_validate_strict_name(dir, &dentry->d_name)) + if (!generic_ci_validate_strict_name(dir, d_name)) return -EINVAL; - retval = ext4_fname_setup_filename(dir, &dentry->d_name, 0, &fname); + retval = ext4_fname_setup_filename(dir, d_name, 0, &fname); if (retval) return retval; @@ -2460,6 +2459,16 @@ out: return retval; } +static int ext4_add_entry(handle_t *handle, struct dentry *dentry, + struct inode *inode) +{ + struct inode *dir = d_inode(dentry->d_parent); + + if (fscrypt_is_nokey_name(dentry)) + return -ENOKEY; + return __ext4_add_entry(handle, dir, &dentry->d_name, inode); +} + /* * Returns 0 for success, or a negative error value */ @@ -2523,7 +2532,7 @@ again: restart = 1; } if (add_level && levels == ext4_dir_htree_level(sb)) { - ext4_warning(sb, "Directory (ino: %lu) index full, " + ext4_warning(sb, "Directory (ino: %llu) index full, " "reach max htree level :%d", dir->i_ino, levels); if (ext4_dir_htree_level(sb) < EXT4_HTREE_LEVEL) { @@ -2804,7 +2813,7 @@ static int ext4_add_nondir(handle_t *handle, * with d_instantiate(). */ static int ext4_create(struct mnt_idmap *idmap, struct inode *dir, - struct dentry *dentry, umode_t mode, bool excl) + struct dentry *dentry, umode_t mode) { handle_t *handle; struct inode *inode; @@ -3002,7 +3011,7 @@ static struct dentry *ext4_mkdir(struct mnt_idmap *idmap, struct inode *dir, credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3); retry: - inode = ext4_new_inode_start_handle(idmap, dir, S_IFDIR | mode, + inode = ext4_new_inode_start_handle(idmap, dir, mode, &dentry->d_name, 0, NULL, EXT4_HT_DIR, credits); handle = ext4_journal_current_handle(); @@ -3047,7 +3056,7 @@ out_stop: out_retry: if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) goto retry; - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; } /* @@ -3445,7 +3454,8 @@ out_retry: return err; } -int __ext4_link(struct inode *dir, struct inode *inode, struct dentry *dentry) +int __ext4_link(struct inode *dir, struct inode *inode, + const struct qstr *d_name, struct dentry *dentry) { handle_t *handle; int err, retries = 0; @@ -3461,9 +3471,8 @@ retry: inode_set_ctime_current(inode); ext4_inc_count(inode); - ihold(inode); - err = ext4_add_entry(handle, dentry, inode); + err = __ext4_add_entry(handle, dir, d_name, inode); if (!err) { err = ext4_mark_inode_dirty(handle, inode); /* this can happen only for tmpfile being @@ -3471,11 +3480,10 @@ retry: */ if (inode->i_nlink == 1) ext4_orphan_del(handle, inode); - d_instantiate(dentry, inode); - ext4_fc_track_link(handle, dentry); + if (dentry) + ext4_fc_track_link(handle, inode, dentry); } else { drop_nlink(inode); - iput(inode); } ext4_journal_stop(handle); if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) @@ -3504,9 +3512,13 @@ static int ext4_link(struct dentry *old_dentry, err = dquot_initialize(dir); if (err) return err; - return __ext4_link(dir, inode, dentry); + err = __ext4_link(dir, inode, &dentry->d_name, dentry); + if (!err) { + ihold(inode); + d_instantiate(dentry, inode); + } + return err; } - /* * Try to find buffer head where contains the parent block. * It should be the inode block if it is inlined or the 1st block diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index c0022f0bff87..b4675aa7ea96 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -4,6 +4,7 @@ #include <linux/fs.h> #include <linux/quotaops.h> #include <linux/buffer_head.h> +#include <linux/string_choices.h> #include "ext4.h" #include "ext4_jbd2.h" @@ -179,8 +180,8 @@ int ext4_orphan_add(handle_t *handle, struct inode *inode) } else brelse(iloc.bh); - ext4_debug("superblock will point to %lu\n", inode->i_ino); - ext4_debug("orphan inode %lu will point to %d\n", + ext4_debug("superblock will point to %llu\n", inode->i_ino); + ext4_debug("orphan inode %llu will point to %d\n", inode->i_ino, NEXT_ORPHAN(inode)); out: ext4_std_error(sb, err); @@ -249,7 +250,7 @@ int ext4_orphan_del(handle_t *handle, struct inode *inode) } mutex_lock(&sbi->s_orphan_lock); - ext4_debug("remove inode %lu from orphan list\n", inode->i_ino); + ext4_debug("remove inode %llu from orphan list\n", inode->i_ino); prev = ei->i_orphan.prev; list_del_init(&ei->i_orphan); @@ -284,7 +285,7 @@ int ext4_orphan_del(handle_t *handle, struct inode *inode) struct inode *i_prev = &list_entry(prev, struct ext4_inode_info, i_orphan)->vfs_inode; - ext4_debug("orphan inode %lu will point to %u\n", + ext4_debug("orphan inode %llu will point to %u\n", i_prev->i_ino, ino_next); err = ext4_reserve_inode_write(handle, i_prev, &iloc2); if (err) { @@ -328,9 +329,9 @@ static void ext4_process_orphan(struct inode *inode, if (inode->i_nlink) { if (test_opt(sb, DEBUG)) ext4_msg(sb, KERN_DEBUG, - "%s: truncating inode %lu to %lld bytes", + "%s: truncating inode %llu to %lld bytes", __func__, inode->i_ino, inode->i_size); - ext4_debug("truncating inode %lu to %lld bytes\n", + ext4_debug("truncating inode %llu to %lld bytes\n", inode->i_ino, inode->i_size); inode_lock(inode); truncate_inode_pages(inode->i_mapping, inode->i_size); @@ -349,9 +350,9 @@ static void ext4_process_orphan(struct inode *inode, } else { if (test_opt(sb, DEBUG)) ext4_msg(sb, KERN_DEBUG, - "%s: deleting unreferenced inode %lu", + "%s: deleting unreferenced inode %llu", __func__, inode->i_ino); - ext4_debug("deleting unreferenced inode %lu\n", + ext4_debug("deleting unreferenced inode %llu\n", inode->i_ino); (*nr_orphans)++; } @@ -388,7 +389,7 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); - if (!es->s_last_orphan && !oi->of_blocks) { + if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) { ext4_debug("no orphan inodes to clean up\n"); return; } @@ -486,14 +487,12 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) } } -#define PLURAL(x) (x), ((x) == 1) ? "" : "s" - if (nr_orphans) ext4_msg(sb, KERN_INFO, "%d orphan inode%s deleted", - PLURAL(nr_orphans)); + nr_orphans, str_plural(nr_orphans)); if (nr_truncates) ext4_msg(sb, KERN_INFO, "%d truncate%s cleaned up", - PLURAL(nr_truncates)); + nr_truncates, str_plural(nr_truncates)); #ifdef CONFIG_QUOTA /* Turn off quotas if they were enabled for orphan cleanup */ if (quota_update) { @@ -572,6 +571,7 @@ int ext4_init_orphan_info(struct super_block *sb) int i, j; int ret; int free; + int loaded = 0; __le32 *bdata; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); struct ext4_orphan_block_tail *ot; @@ -613,6 +613,7 @@ int ext4_init_orphan_info(struct super_block *sb) ret = -EIO; goto out_free; } + loaded++; ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh); if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) { ext4_error(sb, "orphan file block %d: bad magic", i); @@ -635,8 +636,10 @@ int ext4_init_orphan_info(struct super_block *sb) iput(inode); return 0; out_free: - for (i--; i >= 0; i--) - brelse(oi->of_binfo[i].ob_bh); + while (loaded > 0) { + loaded--; + brelse(oi->of_binfo[loaded].ob_bh); + } kvfree(oi->of_binfo); out_put: iput(inode); diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index a8c95eee91b7..0236b6b9785a 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -16,7 +16,6 @@ #include <linux/string.h> #include <linux/buffer_head.h> #include <linux/writeback.h> -#include <linux/pagevec.h> #include <linux/mpage.h> #include <linux/namei.h> #include <linux/uio.h> @@ -104,18 +103,12 @@ static void ext4_finish_bio(struct bio *bio) bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; - struct folio *io_folio = NULL; struct buffer_head *bh, *head; size_t bio_start = fi.offset; size_t bio_end = bio_start + fi.length; unsigned under_io = 0; unsigned long flags; - if (fscrypt_is_bounce_folio(folio)) { - io_folio = folio; - folio = fscrypt_pagecache_folio(folio); - } - if (bio->bi_status) { int err = blk_status_to_errno(bio->bi_status); mapping_set_error(folio->mapping, err); @@ -140,10 +133,8 @@ static void ext4_finish_bio(struct bio *bio) } } while ((bh = bh->b_this_page) != head); spin_unlock_irqrestore(&head->b_uptodate_lock, flags); - if (!under_io) { - fscrypt_free_bounce_page(&io_folio->page); + if (!under_io) folio_end_writeback(folio); - } } } @@ -169,7 +160,7 @@ static void ext4_release_io_end(ext4_io_end_t *io_end) * written. On IO failure, check if journal abort is needed. Note that * we are protected from truncate touching same part of extent tree by the * fact that truncate code waits for all DIO to finish (thus exclusion from - * direct IO is achieved) and also waits for PageWriteback bits. Thus we + * direct IO is achieved) and also waits for writeback to complete. Thus we * cannot get to ext4_ext_truncate() before all IOs overlapping that range are * completed (happens from ext4_free_ioend()). */ @@ -180,7 +171,7 @@ static int ext4_end_io_end(ext4_io_end_t *io_end) struct super_block *sb = inode->i_sb; int ret = 0; - ext4_debug("ext4_end_io_nolock: io_end 0x%p from inode %lu,list->next 0x%p," + ext4_debug("ext4_end_io_nolock: io_end 0x%p from inode %llu,list->next 0x%p," "list->prev 0x%p\n", io_end, inode->i_ino, io_end->list.next, io_end->list.prev); @@ -204,7 +195,7 @@ static int ext4_end_io_end(ext4_io_end_t *io_end) ext4_msg(sb, KERN_EMERG, "failed to convert unwritten extents to written " "extents -- potential data loss! " - "(inode %lu, error %d)", inode->i_ino, ret); + "(inode %llu, error %d)", inode->i_ino, ret); } ext4_clear_io_unwritten_flag(io_end); @@ -221,7 +212,7 @@ static void dump_completed_IO(struct inode *inode, struct list_head *head) if (list_empty(head)) return; - ext4_debug("Dump inode %lu completed io list\n", inode->i_ino); + ext4_debug("Dump inode %llu completed io list\n", inode->i_ino); list_for_each_entry(io_end, head, list) { cur = &io_end->list; before = cur->prev; @@ -229,7 +220,7 @@ static void dump_completed_IO(struct inode *inode, struct list_head *head) after = cur->next; io_end1 = container_of(after, ext4_io_end_t, list); - ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n", + ext4_debug("io 0x%p from inode %llu,prev 0x%p,next 0x%p\n", io_end, inode->i_ino, io_end0, io_end1); } #endif @@ -366,7 +357,7 @@ static void ext4_end_bio(struct bio *bio) if (bio->bi_status) { struct inode *inode = io_end->inode; - ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu " + ext4_warning(inode->i_sb, "I/O error %d writing to inode %llu " "starting block %llu)", bio->bi_status, inode->i_ino, (unsigned long long) @@ -416,6 +407,8 @@ void ext4_io_submit_init(struct ext4_io_submit *io, } static void io_submit_init_bio(struct ext4_io_submit *io, + struct inode *inode, + struct folio *folio, struct buffer_head *bh) { struct bio *bio; @@ -425,44 +418,53 @@ static void io_submit_init_bio(struct ext4_io_submit *io, * __GFP_DIRECT_RECLAIM is set, see comments for bio_alloc_bioset(). */ bio = bio_alloc(bh->b_bdev, BIO_MAX_VECS, REQ_OP_WRITE, GFP_NOIO); - fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); + fscrypt_set_bio_crypt_ctx(bio, inode, folio_pos(folio) + bh_offset(bh), + GFP_NOIO); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio->bi_end_io = ext4_end_bio; bio->bi_private = ext4_get_io_end(io->io_end); + bio->bi_write_hint = inode->i_write_hint; io->io_bio = bio; io->io_next_block = bh->b_blocknr; wbc_init_bio(io->io_wbc, bio); } +static bool io_submit_need_new_bio(struct ext4_io_submit *io, + struct inode *inode, + struct folio *folio, + struct buffer_head *bh) +{ + if (bh->b_blocknr != io->io_next_block) + return true; + if (!fscrypt_mergeable_bio(io->io_bio, inode, + folio_pos(folio) + bh_offset(bh))) + return true; + return false; +} + static void io_submit_add_bh(struct ext4_io_submit *io, struct inode *inode, struct folio *folio, - struct folio *io_folio, struct buffer_head *bh) { - if (io->io_bio && (bh->b_blocknr != io->io_next_block || - !fscrypt_mergeable_bio_bh(io->io_bio, bh))) { + if (io->io_bio && io_submit_need_new_bio(io, inode, folio, bh)) { submit_and_retry: ext4_io_submit(io); } - if (io->io_bio == NULL) { - io_submit_init_bio(io, bh); - io->io_bio->bi_write_hint = inode->i_write_hint; - } - if (!bio_add_folio(io->io_bio, io_folio, bh->b_size, bh_offset(bh))) + if (io->io_bio == NULL) + io_submit_init_bio(io, inode, folio, bh); + if (!bio_add_folio(io->io_bio, folio, bh->b_size, bh_offset(bh))) goto submit_and_retry; wbc_account_cgroup_owner(io->io_wbc, folio, bh->b_size); io->io_next_block++; } -int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, +void ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, size_t len) { - struct folio *io_folio = folio; struct inode *inode = folio->mapping->host; unsigned block_start; struct buffer_head *bh, *head; - int ret = 0; int nr_to_submit = 0; struct writeback_control *wbc = io->io_wbc; bool keep_towrite = false; @@ -524,71 +526,24 @@ int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, nr_to_submit++; } while ((bh = bh->b_this_page) != head); - /* Nothing to submit? Just unlock the folio... */ - if (!nr_to_submit) - return 0; - - bh = head = folio_buffers(folio); - - /* - * If any blocks are being written to an encrypted file, encrypt them - * into a bounce page. For simplicity, just encrypt until the last - * block which might be needed. This may cause some unneeded blocks - * (e.g. holes) to be unnecessarily encrypted, but this is rare and - * can't happen in the common case of blocksize == PAGE_SIZE. - */ - if (fscrypt_inode_uses_fs_layer_crypto(inode)) { - gfp_t gfp_flags = GFP_NOFS; - unsigned int enc_bytes = round_up(len, i_blocksize(inode)); - struct page *bounce_page; - + if (!nr_to_submit) { /* - * Since bounce page allocation uses a mempool, we can only use - * a waiting mask (i.e. request guaranteed allocation) on the - * first page of the bio. Otherwise it can deadlock. + * We have nothing to submit. Just cycle the folio through + * writeback state to properly update xarray tags. */ - if (io->io_bio) - gfp_flags = GFP_NOWAIT; - retry_encrypt: - bounce_page = fscrypt_encrypt_pagecache_blocks(folio, - enc_bytes, 0, gfp_flags); - if (IS_ERR(bounce_page)) { - ret = PTR_ERR(bounce_page); - if (ret == -ENOMEM && - (io->io_bio || wbc->sync_mode == WB_SYNC_ALL)) { - gfp_t new_gfp_flags = GFP_NOFS; - if (io->io_bio) - ext4_io_submit(io); - else - new_gfp_flags |= __GFP_NOFAIL; - memalloc_retry_wait(gfp_flags); - gfp_flags = new_gfp_flags; - goto retry_encrypt; - } - - printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret); - folio_redirty_for_writepage(wbc, folio); - do { - if (buffer_async_write(bh)) { - clear_buffer_async_write(bh); - set_buffer_dirty(bh); - } - bh = bh->b_this_page; - } while (bh != head); - - return ret; - } - io_folio = page_folio(bounce_page); + __folio_start_writeback(folio, keep_towrite); + folio_end_writeback(folio); + return; } + bh = head = folio_buffers(folio); + __folio_start_writeback(folio, keep_towrite); /* Now submit buffers to write */ do { if (!buffer_async_write(bh)) continue; - io_submit_add_bh(io, inode, folio, io_folio, bh); + io_submit_add_bh(io, inode, folio, bh); } while ((bh = bh->b_this_page) != head); - - return 0; } diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index 830f3b8a321f..c7b6cdb2e124 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -43,30 +43,19 @@ #include <linux/mpage.h> #include <linux/writeback.h> #include <linux/backing-dev.h> -#include <linux/pagevec.h> #include "ext4.h" #include <trace/events/ext4.h> -#define NUM_PREALLOC_POST_READ_CTXS 128 +#define NUM_VERITY_WORKS 128 -static struct kmem_cache *bio_post_read_ctx_cache; -static mempool_t *bio_post_read_ctx_pool; +static struct kmem_cache *ext4_verity_work_cache; +static mempool_t *ext4_verity_work_pool; -/* postprocessing steps for read bios */ -enum bio_post_read_step { - STEP_INITIAL = 0, - STEP_DECRYPT, - STEP_VERITY, - STEP_MAX, -}; - -struct bio_post_read_ctx { +struct ext4_verity_work { struct bio *bio; struct fsverity_info *vi; struct work_struct work; - unsigned int cur_step; - unsigned int enabled_steps; }; static void __read_end_io(struct bio *bio) @@ -76,40 +65,22 @@ static void __read_end_io(struct bio *bio) bio_for_each_folio_all(fi, bio) folio_end_read(fi.folio, bio->bi_status == 0); if (bio->bi_private) - mempool_free(bio->bi_private, bio_post_read_ctx_pool); + mempool_free(bio->bi_private, ext4_verity_work_pool); bio_put(bio); } -static void bio_post_read_processing(struct bio_post_read_ctx *ctx); - -static void decrypt_work(struct work_struct *work) -{ - struct bio_post_read_ctx *ctx = - container_of(work, struct bio_post_read_ctx, work); - struct bio *bio = ctx->bio; - - if (fscrypt_decrypt_bio(bio)) - bio_post_read_processing(ctx); - else - __read_end_io(bio); -} - static void verity_work(struct work_struct *work) { - struct bio_post_read_ctx *ctx = - container_of(work, struct bio_post_read_ctx, work); + struct ext4_verity_work *ctx = + container_of(work, struct ext4_verity_work, work); struct bio *bio = ctx->bio; struct fsverity_info *vi = ctx->vi; /* - * fsverity_verify_bio() may call readahead() again, and although verity - * will be disabled for that, decryption may still be needed, causing - * another bio_post_read_ctx to be allocated. So to guarantee that - * mempool_alloc() never deadlocks we must free the current ctx first. - * This is safe because verity is the last post-read step. + * Free the ext4_verity_work right away, since it's no longer needed. + * This relieves the pressure on the mempool as much as possible. */ - BUILD_BUG_ON(STEP_VERITY + 1 != STEP_MAX); - mempool_free(ctx, bio_post_read_ctx_pool); + mempool_free(ctx, ext4_verity_work_pool); bio->bi_private = NULL; fsverity_verify_bio(vi, bio); @@ -117,41 +88,6 @@ static void verity_work(struct work_struct *work) __read_end_io(bio); } -static void bio_post_read_processing(struct bio_post_read_ctx *ctx) -{ - /* - * We use different work queues for decryption and for verity because - * verity may require reading metadata pages that need decryption, and - * we shouldn't recurse to the same workqueue. - */ - switch (++ctx->cur_step) { - case STEP_DECRYPT: - if (ctx->enabled_steps & (1 << STEP_DECRYPT)) { - INIT_WORK(&ctx->work, decrypt_work); - fscrypt_enqueue_decrypt_work(&ctx->work); - return; - } - ctx->cur_step++; - fallthrough; - case STEP_VERITY: - if (IS_ENABLED(CONFIG_FS_VERITY) && - ctx->enabled_steps & (1 << STEP_VERITY)) { - INIT_WORK(&ctx->work, verity_work); - fsverity_enqueue_verify_work(&ctx->work); - return; - } - ctx->cur_step++; - fallthrough; - default: - __read_end_io(ctx->bio); - } -} - -static bool bio_post_read_required(struct bio *bio) -{ - return bio->bi_private && !bio->bi_status; -} - /* * I/O completion handler for multipage BIOs. * @@ -166,36 +102,26 @@ static bool bio_post_read_required(struct bio *bio) */ static void mpage_end_io(struct bio *bio) { - if (bio_post_read_required(bio)) { - struct bio_post_read_ctx *ctx = bio->bi_private; + if (IS_ENABLED(CONFIG_FS_VERITY) && bio->bi_private && + !bio->bi_status) { + struct ext4_verity_work *ctx = bio->bi_private; - ctx->cur_step = STEP_INITIAL; - bio_post_read_processing(ctx); + INIT_WORK(&ctx->work, verity_work); + fsverity_enqueue_verify_work(&ctx->work); return; } __read_end_io(bio); } -static void ext4_set_bio_post_read_ctx(struct bio *bio, - const struct inode *inode, - struct fsverity_info *vi) +static void ext4_set_verity_work(struct bio *bio, struct fsverity_info *vi) { - unsigned int post_read_steps = 0; - - if (fscrypt_inode_uses_fs_layer_crypto(inode)) - post_read_steps |= 1 << STEP_DECRYPT; - - if (vi) - post_read_steps |= 1 << STEP_VERITY; - - if (post_read_steps) { + if (vi) { /* Due to the mempool, this never fails. */ - struct bio_post_read_ctx *ctx = - mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); + struct ext4_verity_work *ctx = + mempool_alloc(ext4_verity_work_pool, GFP_NOFS); ctx->bio = bio; ctx->vi = vi; - ctx->enabled_steps = post_read_steps; bio->bi_private = ctx; } } @@ -215,11 +141,11 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi, sector_t last_block_in_bio = 0; const unsigned blkbits = inode->i_blkbits; const unsigned blocksize = 1 << blkbits; - sector_t next_block; sector_t block_in_file; sector_t last_block; sector_t last_block_in_file; sector_t first_block; + loff_t pos; unsigned page_block; struct block_device *bdev = inode->i_sb->s_bdev; int length; @@ -249,7 +175,8 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi, blocks_per_folio = folio_size(folio) >> blkbits; first_hole = blocks_per_folio; - block_in_file = next_block = EXT4_PG_TO_LBLK(inode, folio->index); + pos = folio_pos(folio); + block_in_file = pos >> blkbits; last_block = EXT4_PG_TO_LBLK(inode, folio->index + nr_pages); last_block_in_file = (ext4_readpage_limit(inode) + blocksize - 1) >> blkbits; @@ -342,7 +269,7 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi, * BIO off first? */ if (bio && (last_block_in_bio != first_block - 1 || - !fscrypt_mergeable_bio(bio, inode, next_block))) { + !fscrypt_mergeable_bio(bio, inode, pos))) { submit_and_realloc: blk_crypto_submit_bio(bio); bio = NULL; @@ -354,9 +281,8 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi, */ bio = bio_alloc(bdev, bio_max_segs(nr_pages), REQ_OP_READ, GFP_KERNEL); - fscrypt_set_bio_crypt_ctx(bio, inode, next_block, - GFP_KERNEL); - ext4_set_bio_post_read_ctx(bio, inode, vi); + fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_KERNEL); + ext4_set_verity_work(bio, vi); bio->bi_iter.bi_sector = first_block << (blkbits - 9); bio->bi_end_io = mpage_end_io; if (rac) @@ -430,27 +356,31 @@ void ext4_readahead(struct readahead_control *rac) ext4_mpage_readpages(inode, vi, rac, NULL); } -int __init ext4_init_post_read_processing(void) +int __init ext4_init_verity_caches(void) { - bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, SLAB_RECLAIM_ACCOUNT); + if (!IS_ENABLED(CONFIG_FS_VERITY)) + return 0; + ext4_verity_work_cache = + KMEM_CACHE(ext4_verity_work, SLAB_RECLAIM_ACCOUNT); - if (!bio_post_read_ctx_cache) + if (!ext4_verity_work_cache) goto fail; - bio_post_read_ctx_pool = - mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS, - bio_post_read_ctx_cache); - if (!bio_post_read_ctx_pool) + ext4_verity_work_pool = mempool_create_slab_pool( + NUM_VERITY_WORKS, ext4_verity_work_cache); + if (!ext4_verity_work_pool) goto fail_free_cache; return 0; fail_free_cache: - kmem_cache_destroy(bio_post_read_ctx_cache); + kmem_cache_destroy(ext4_verity_work_cache); fail: return -ENOMEM; } -void ext4_exit_post_read_processing(void) +void ext4_exit_verity_caches(void) { - mempool_destroy(bio_post_read_ctx_pool); - kmem_cache_destroy(bio_post_read_ctx_cache); + if (!IS_ENABLED(CONFIG_FS_VERITY)) + return; + mempool_destroy(ext4_verity_work_pool); + kmem_cache_destroy(ext4_verity_work_cache); } diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 43f680c750ae..bca0dc87d0b7 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -161,7 +161,7 @@ MODULE_ALIAS("ext3"); static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { if (simu_fail) { clear_buffer_uptodate(bh); @@ -176,13 +176,13 @@ static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, */ clear_buffer_verified(bh); - bh->b_end_io = end_io ? end_io : end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ | op_flags, bh); + if (!end_io) + end_io = bh_end_read; + bh_submit(bh, REQ_OP_READ | op_flags, end_io); } void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { BUG_ON(!buffer_locked(bh)); @@ -194,7 +194,7 @@ void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, } int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { BUG_ON(!buffer_locked(bh)); @@ -521,6 +521,7 @@ static bool ext4_journalled_writepage_needs_redirty(struct jbd2_inode *jinode, { struct buffer_head *bh, *head; struct journal_head *jh; + transaction_t *trans = READ_ONCE(jinode->i_transaction); bh = head = folio_buffers(folio); do { @@ -539,7 +540,7 @@ static bool ext4_journalled_writepage_needs_redirty(struct jbd2_inode *jinode, */ jh = bh2jh(bh); if (buffer_dirty(bh) || - (jh && (jh->b_transaction != jinode->i_transaction || + (jh && (jh->b_transaction != trans || jh->b_next_transaction))) return true; } while ((bh = bh->b_this_page) != head); @@ -550,15 +551,20 @@ static bool ext4_journalled_writepage_needs_redirty(struct jbd2_inode *jinode, static int ext4_journalled_submit_inode_data_buffers(struct jbd2_inode *jinode) { struct address_space *mapping = jinode->i_vfs_inode->i_mapping; + loff_t range_start, range_end; struct writeback_control wbc = { - .sync_mode = WB_SYNC_ALL, + .sync_mode = WB_SYNC_ALL, .nr_to_write = LONG_MAX, - .range_start = jinode->i_dirty_start, - .range_end = jinode->i_dirty_end, - }; + }; struct folio *folio = NULL; int error; + if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end)) + return 0; + + wbc.range_start = range_start; + wbc.range_end = range_end; + /* * writeback_iter() already checks for dirty pages and calls * folio_clear_dirty_for_io(), which we want to write protect the @@ -848,12 +854,12 @@ void __ext4_error_inode(struct inode *inode, const char *function, vaf.va = &args; if (block) printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: " - "inode #%lu: block %llu: comm %s: %pV\n", + "inode #%llu: block %llu: comm %s: %pV\n", inode->i_sb->s_id, function, line, inode->i_ino, block, current->comm, &vaf); else printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: " - "inode #%lu: comm %s: %pV\n", + "inode #%llu: comm %s: %pV\n", inode->i_sb->s_id, function, line, inode->i_ino, current->comm, &vaf); va_end(args); @@ -888,13 +894,13 @@ void __ext4_error_file(struct file *file, const char *function, vaf.va = &args; if (block) printk(KERN_CRIT - "EXT4-fs error (device %s): %s:%d: inode #%lu: " + "EXT4-fs error (device %s): %s:%d: inode #%llu: " "block %llu: comm %s: path %s: %pV\n", inode->i_sb->s_id, function, line, inode->i_ino, block, current->comm, path, &vaf); else printk(KERN_CRIT - "EXT4-fs error (device %s): %s:%d: inode #%lu: " + "EXT4-fs error (device %s): %s:%d: inode #%llu: " "comm %s: path %s: %pV\n", inode->i_sb->s_id, function, line, inode->i_ino, current->comm, path, &vaf); @@ -1035,14 +1041,14 @@ void __ext4_warning_inode(const struct inode *inode, const char *function, vaf.fmt = fmt; vaf.va = &args; printk(KERN_WARNING "EXT4-fs warning (device %s): %s:%d: " - "inode #%lu: comm %s: %pV\n", inode->i_sb->s_id, + "inode #%llu: comm %s: %pV\n", inode->i_sb->s_id, function, line, inode->i_ino, current->comm, &vaf); va_end(args); } void __ext4_grp_locked_error(const char *function, unsigned int line, struct super_block *sb, ext4_group_t grp, - unsigned long ino, ext4_fsblk_t block, + u64 ino, ext4_fsblk_t block, const char *fmt, ...) __releases(bitlock) __acquires(bitlock) @@ -1061,7 +1067,7 @@ __acquires(bitlock) printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: group %u, ", sb->s_id, function, line, grp); if (ino) - printk(KERN_CONT "inode %lu: ", ino); + printk(KERN_CONT "inode %llu: ", ino); if (block) printk(KERN_CONT "block %llu:", (unsigned long long) block); @@ -1170,7 +1176,7 @@ static void dump_orphan_list(struct super_block *sb, struct ext4_sb_info *sbi) list_for_each(l, &sbi->s_orphan) { struct inode *inode = orphan_list_entry(l); printk(KERN_ERR " " - "inode %s:%lu at %p: mode %o, nlink %d, next %d\n", + "inode %s:%llu at %p: mode %o, nlink %d, next %d\n", inode->i_sb->s_id, inode->i_ino, inode, inode->i_mode, inode->i_nlink, NEXT_ORPHAN(inode)); @@ -1254,12 +1260,10 @@ static void ext4_group_desc_free(struct ext4_sb_info *sbi) struct buffer_head **group_desc; int i; - rcu_read_lock(); - group_desc = rcu_dereference(sbi->s_group_desc); + group_desc = rcu_access_pointer(sbi->s_group_desc); for (i = 0; i < sbi->s_gdb_count; i++) brelse(group_desc[i]); kvfree(group_desc); - rcu_read_unlock(); } static void ext4_flex_groups_free(struct ext4_sb_info *sbi) @@ -1267,14 +1271,12 @@ static void ext4_flex_groups_free(struct ext4_sb_info *sbi) struct flex_groups **flex_groups; int i; - rcu_read_lock(); - flex_groups = rcu_dereference(sbi->s_flex_groups); + flex_groups = rcu_access_pointer(sbi->s_flex_groups); if (flex_groups) { for (i = 0; i < sbi->s_flex_groups_allocated; i++) kvfree(flex_groups[i]); kvfree(flex_groups); } - rcu_read_unlock(); } static void ext4_put_super(struct super_block *sb) @@ -1301,6 +1303,8 @@ static void ext4_put_super(struct super_block *sb) &sb->s_uuid); ext4_unregister_li_request(sb); + /* Drain deferred EA inode iputs while quota is still active. */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_quotas_off(sb, EXT4_MAXQUOTAS); destroy_workqueue(sbi->rsv_conversion_wq); @@ -1421,6 +1425,13 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) memset(&ei->i_dquot, 0, sizeof(ei->i_dquot)); #endif ei->jinode = NULL; + /* + * Reinitialize xattr_sem every allocation because EA inodes + * share this space with i_ea_iput_node (via union) which may + * have overwritten the semaphore when the slab object was + * previously used as an EA inode. + */ + init_rwsem(&ei->xattr_sem); INIT_LIST_HEAD(&ei->i_rsv_conversion_list); spin_lock_init(&ei->i_completed_io_lock); ei->i_sync_tid = 0; @@ -1428,6 +1439,10 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) INIT_WORK(&ei->i_rsv_conversion_work, ext4_end_io_rsv_work); ext4_fc_init_inode(&ei->vfs_inode); spin_lock_init(&ei->i_fc_lock); + ei->i_metadata_bhs = NULL; +#ifdef CONFIG_LOCKDEP + lockdep_set_subclass(&ei->i_data_sem, I_DATA_SEM_NORMAL); +#endif return &ei->vfs_inode; } @@ -1445,8 +1460,9 @@ static int ext4_drop_inode(struct inode *inode) static void ext4_free_in_core_inode(struct inode *inode) { fscrypt_free_inode(inode); + kfree(ext4_i_metadata_bhs(inode)); if (!list_empty(&(EXT4_I(inode)->i_fc_list))) { - pr_warn("%s: inode %ld still in fc list", + pr_warn("%s: inode %llu still in fc list", __func__, inode->i_ino); } kmem_cache_free(ext4_inode_cachep, EXT4_I(inode)); @@ -1456,7 +1472,7 @@ static void ext4_destroy_inode(struct inode *inode) { if (ext4_inode_orphan_tracked(inode)) { ext4_msg(inode->i_sb, KERN_ERR, - "Inode %lu (%p): inode tracked as orphan!", + "Inode %llu (%p): inode tracked as orphan!", inode->i_ino, EXT4_I(inode)); print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 4, EXT4_I(inode), sizeof(struct ext4_inode_info), @@ -1467,7 +1483,7 @@ static void ext4_destroy_inode(struct inode *inode) if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ERROR_FS) && WARN_ON_ONCE(EXT4_I(inode)->i_reserved_data_blocks)) ext4_msg(inode->i_sb, KERN_ERR, - "Inode %lu (%p): i_reserved_data_blocks (%u) not cleared!", + "Inode %llu (%p): i_reserved_data_blocks (%u) not cleared!", inode->i_ino, EXT4_I(inode), EXT4_I(inode)->i_reserved_data_blocks); } @@ -1482,7 +1498,6 @@ static void init_once(void *foo) struct ext4_inode_info *ei = foo; INIT_LIST_HEAD(&ei->i_orphan); - init_rwsem(&ei->xattr_sem); init_rwsem(&ei->i_data_sem); inode_init_once(&ei->vfs_inode); ext4_fc_init_inode(&ei->vfs_inode); @@ -1523,10 +1538,34 @@ static void destroy_inodecache(void) void ext4_clear_inode(struct inode *inode) { + struct mapping_metadata_bhs *mmb = ext4_i_metadata_bhs(inode); + ext4_fc_del(inode); - invalidate_inode_buffers(inode); + if (mmb) + mmb_invalidate(mmb); clear_inode(inode); ext4_discard_preallocations(inode); + /* + * We must remove the inode from the hash before ext4_free_inode() + * clears the bit in inode bitmap as otherwise another process reusing + * the inode will block in insert_inode_hash() waiting for inode + * eviction to complete while holding transaction handle open, but + * ext4_evict_inode() still running for that inode could block waiting + * for transaction commit if the inode is marked as IS_SYNC => deadlock. + * + * Removing the inode from the hash here is safe. There are two cases + * to consider: + * 1) The inode still has references to it (i_nlink > 0). In that case + * we are keeping the inode and once we remove the inode from the hash, + * iget() can create the new inode structure for the same inode number + * and we are fine with that as all IO on behalf of the inode is + * finished. + * 2) We are deleting the inode (i_nlink == 0). In that case inode + * number cannot be reused until ext4_free_inode() clears the bit in + * the inode bitmap, at which point all IO is done and reuse is fine + * again. + */ + remove_inode_hash(inode); ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS); dquot_drop(inode); if (EXT4_I(inode)->jinode) { @@ -1577,9 +1616,13 @@ static int ext4_nfs_commit_metadata(struct inode *inode) struct writeback_control wbc = { .sync_mode = WB_SYNC_ALL }; + int ret; trace_ext4_nfs_commit_metadata(inode); - return ext4_write_inode(inode, &wbc); + ret = ext4_write_inode(inode, &wbc); + if (!ret && inode_state_read_once(inode) & I_METADATA_WRITEBACK) + ret = ext4_sync_inode_metadata(inode, &wbc); + return ret; } #ifdef CONFIG_QUOTA @@ -1636,6 +1679,7 @@ static const struct super_operations ext4_sops = { .free_inode = ext4_free_in_core_inode, .destroy_inode = ext4_destroy_inode, .write_inode = ext4_write_inode, + .sync_inode_metadata = ext4_sync_inode_metadata, .dirty_inode = ext4_dirty_inode, .drop_inode = ext4_drop_inode, .evict_inode = ext4_evict_inode, @@ -3633,6 +3677,13 @@ int ext4_feature_set_ok(struct super_block *sb, int readonly) "extents feature\n"); return 0; } + if (ext4_has_feature_bigalloc(sb) && + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) { + ext4_msg(sb, KERN_WARNING, + "bad geometry: bigalloc file system with non-zero " + "first_data_block\n"); + return 0; + } #if !IS_ENABLED(CONFIG_QUOTA) || !IS_ENABLED(CONFIG_QFMT_V2) if (!readonly && (ext4_has_feature_quota(sb) || @@ -4440,8 +4491,9 @@ static int ext4_handle_clustersize(struct super_block *sb) sbi->s_cluster_bits = 0; } sbi->s_clusters_per_group = le32_to_cpu(es->s_clusters_per_group); - if (sbi->s_clusters_per_group > sb->s_blocksize * 8) { - ext4_msg(sb, KERN_ERR, "#clusters per group too big: %lu", + if (sbi->s_clusters_per_group > sb->s_blocksize * 8 || + sbi->s_clusters_per_group & 7) { + ext4_msg(sb, KERN_ERR, "invalid #clusters per group: %lu", sbi->s_clusters_per_group); return -EINVAL; } @@ -4509,6 +4561,7 @@ static void ext4_fast_commit_init(struct super_block *sb) sbi->s_fc_ineligible_tid = 0; mutex_init(&sbi->s_fc_lock); memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats)); + memset(&sbi->s_fc_snap_stats, 0, sizeof(sbi->s_fc_snap_stats)); sbi->s_fc_replay_state.fc_regions = NULL; sbi->s_fc_replay_state.fc_regions_size = 0; sbi->s_fc_replay_state.fc_regions_used = 0; @@ -5272,8 +5325,10 @@ static int ext4_block_group_meta_init(struct super_block *sb, int silent) return -EINVAL; } if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || - sbi->s_inodes_per_group > sb->s_blocksize * 8) { - ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n", + sbi->s_inodes_per_group > sb->s_blocksize * 8 || + sbi->s_inodes_per_group & 7 || + sbi->s_inodes_per_group % sbi->s_inodes_per_block) { + ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu", sbi->s_inodes_per_group); return -EINVAL; } @@ -5333,7 +5388,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_set_def_opts(sb, es); sbi->s_resuid = make_kuid(&init_user_ns, ext4_get_resuid(es)); - sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resuid(es)); + sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resgid(es)); sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; @@ -5403,6 +5458,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) timer_setup(&sbi->s_err_report, print_daily_error_info, 0); spin_lock_init(&sbi->s_error_lock); + mutex_init(&sbi->s_error_notify_mutex); INIT_WORK(&sbi->s_sb_upd_work, update_super_work); err = ext4_group_desc_init(sb, es, logical_sb_block, &first_not_zeroed); @@ -5460,6 +5516,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_has_feature_orphan_present(sb) || ext4_has_feature_journal_needs_recovery(sb)); + ext4_init_ea_inode_work(sbi); + if (ext4_has_feature_mmp(sb) && !sb_rdonly(sb)) { err = ext4_multi_mount_protect(sb, le64_to_cpu(es->s_mmp_block)); if (err) @@ -5710,6 +5768,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) return 0; failed_mount9: + /* Drain deferred EA inode iputs before quota shutdown */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_quotas_off(sb, EXT4_MAXQUOTAS); failed_mount8: __maybe_unused ext4_release_orphan_info(sb); @@ -5730,6 +5790,8 @@ failed_mount4: if (EXT4_SB(sb)->rsv_conversion_wq) destroy_workqueue(EXT4_SB(sb)->rsv_conversion_wq); failed_mount_wq: + /* Drain deferred EA inode iputs before freeing structures */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_xattr_destroy_cache(sbi->s_ea_inode_cache); sbi->s_ea_inode_cache = NULL; @@ -5740,6 +5802,8 @@ failed_mount_wq: ext4_journal_destroy(sbi, sbi->s_journal); } failed_mount3a: + /* Drain deferred EA inode iputs from journal replay */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_es_unregister_shrinker(sbi); failed_mount3: /* flush s_sb_upd_work before sbi destroy */ @@ -5760,7 +5824,7 @@ failed_mount: brelse(sbi->s_sbh); if (sbi->s_journal_bdev_file) { invalidate_bdev(file_bdev(sbi->s_journal_bdev_file)); - bdev_fput(sbi->s_journal_bdev_file); + fs_bdev_file_release(sbi->s_journal_bdev_file, sb); } out_fail: invalidate_bdev(sb->s_bdev); @@ -5877,6 +5941,11 @@ static struct inode *ext4_get_journal_inode(struct super_block *sb, return ERR_PTR(-EFSCORRUPTED); } +#ifdef CONFIG_LOCKDEP + lockdep_set_subclass(&EXT4_I(journal_inode)->i_data_sem, + I_DATA_SEM_JOURNAL); +#endif + ext4_debug("Journal inode found at %p: %lld bytes\n", journal_inode, journal_inode->i_size); return journal_inode; @@ -5939,13 +6008,13 @@ static struct file *ext4_get_journal_blkdev(struct super_block *sb, struct ext4_super_block *es; int errno; - bdev_file = bdev_file_open_by_dev(j_dev, + bdev_file = fs_bdev_file_open_by_dev(j_dev, BLK_OPEN_READ | BLK_OPEN_WRITE | BLK_OPEN_RESTRICT_WRITES, - sb, &fs_holder_ops); + sb, sb); if (IS_ERR(bdev_file)) { ext4_msg(sb, KERN_ERR, - "failed to open journal device unknown-block(%u,%u) %ld", - MAJOR(j_dev), MINOR(j_dev), PTR_ERR(bdev_file)); + "failed to open journal device unknown-block(%u,%u) %pe", + MAJOR(j_dev), MINOR(j_dev), bdev_file); return bdev_file; } @@ -6001,7 +6070,7 @@ static struct file *ext4_get_journal_blkdev(struct super_block *sb, out_bh: brelse(bh); out_bdev: - bdev_fput(bdev_file); + fs_bdev_file_release(bdev_file, sb); return ERR_PTR(errno); } @@ -6040,7 +6109,7 @@ static journal_t *ext4_open_dev_journal(struct super_block *sb, out_journal: ext4_journal_destroy(EXT4_SB(sb), journal); out_bdev: - bdev_fput(bdev_file); + fs_bdev_file_release(bdev_file, sb); return ERR_PTR(errno); } @@ -6283,12 +6352,10 @@ static int ext4_commit_super(struct super_block *sb) clear_buffer_write_io_error(sbh); set_buffer_uptodate(sbh); } - get_bh(sbh); /* Clear potential dirty bit if it was journalled update */ clear_buffer_dirty(sbh); - sbh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | REQ_SYNC | - (test_opt(sb, BARRIER) ? REQ_FUA : 0), sbh); + bh_submit(sbh, REQ_OP_WRITE | REQ_SYNC | + (test_opt(sb, BARRIER) ? REQ_FUA : 0), bh_end_write); wait_on_buffer(sbh); if (buffer_write_io_error(sbh)) { ext4_msg(sb, KERN_ERR, "I/O error while writing " @@ -6407,6 +6474,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait) trace_ext4_sync_fs(sb, wait); flush_workqueue(sbi->rsv_conversion_wq); + flush_delayed_work(&sbi->s_ea_inode_work); /* * Writeback quota in non-journalled quota case - journalled quota has * no dirty dquots @@ -7459,7 +7527,7 @@ static void ext4_kill_sb(struct super_block *sb) kill_block_super(sb); if (bdev_file) - bdev_fput(bdev_file); + fs_bdev_file_release(bdev_file, sb); } static struct file_system_type ext4_fs_type = { @@ -7491,7 +7559,7 @@ static int __init ext4_init_fs(void) if (err) goto out7; - err = ext4_init_post_read_processing(); + err = ext4_init_verity_caches(); if (err) goto out6; @@ -7540,7 +7608,7 @@ out3: out4: ext4_exit_pageio(); out5: - ext4_exit_post_read_processing(); + ext4_exit_verity_caches(); out6: ext4_exit_pending(); out7: @@ -7561,7 +7629,7 @@ static void __exit ext4_exit_fs(void) ext4_exit_sysfs(); ext4_exit_system_zone(); ext4_exit_pageio(); - ext4_exit_post_read_processing(); + ext4_exit_verity_caches(); ext4_exit_es(); ext4_exit_pending(); } diff --git a/fs/ext4/symlink.c b/fs/ext4/symlink.c index 645240cc0229..b612262719ed 100644 --- a/fs/ext4/symlink.c +++ b/fs/ext4/symlink.c @@ -92,7 +92,7 @@ static const char *ext4_get_link(struct dentry *dentry, struct inode *inode, if (!dentry) { bh = ext4_getblk(NULL, inode, 0, EXT4_GET_BLOCKS_CACHED_NOWAIT); - if (IS_ERR(bh) || !bh) + if (IS_ERR_OR_NULL(bh)) return ERR_PTR(-ECHILD); if (!ext4_buffer_uptodate(bh)) { brelse(bh); diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index b87d7bdab06a..923b375e017f 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -597,7 +597,10 @@ static const struct kobj_type ext4_feat_ktype = { void ext4_notify_error_sysfs(struct ext4_sb_info *sbi) { - sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); + mutex_lock(&sbi->s_error_notify_mutex); + if (sbi->s_kobj.state_in_sysfs) + sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); + mutex_unlock(&sbi->s_error_notify_mutex); } static struct kobject *ext4_root; @@ -610,8 +613,10 @@ int ext4_register_sysfs(struct super_block *sb) int err; init_completion(&sbi->s_kobj_unregister); + mutex_lock(&sbi->s_error_notify_mutex); err = kobject_init_and_add(&sbi->s_kobj, &ext4_sb_ktype, ext4_root, "%s", sb->s_id); + mutex_unlock(&sbi->s_error_notify_mutex); if (err) { kobject_put(&sbi->s_kobj); wait_for_completion(&sbi->s_kobj_unregister); @@ -644,7 +649,10 @@ void ext4_unregister_sysfs(struct super_block *sb) if (sbi->s_proc) remove_proc_subtree(sb->s_id, ext4_proc_root); + + mutex_lock(&sbi->s_error_notify_mutex); kobject_del(&sbi->s_kobj); + mutex_unlock(&sbi->s_error_notify_mutex); } int __init ext4_init_sysfs(void) diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 7bf9ba19a89d..5c310747b965 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -64,7 +64,7 @@ #ifdef EXT4_XATTR_DEBUG # define ea_idebug(inode, fmt, ...) \ - printk(KERN_DEBUG "inode %s:%lu: " fmt "\n", \ + printk(KERN_DEBUG "inode %s:%llu: " fmt "\n", \ inode->i_sb->s_id, inode->i_ino, ##__VA_ARGS__) # define ea_bdebug(bh, fmt, ...) \ printk(KERN_DEBUG "block %pg:%lu: " fmt "\n", \ @@ -114,10 +114,6 @@ const struct xattr_handler * const ext4_xattr_handlers[] = { #define EA_INODE_CACHE(inode) (((struct ext4_sb_info *) \ inode->i_sb->s_fs_info)->s_ea_inode_cache) -static int -ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, - struct inode *inode); - #ifdef CONFIG_LOCKDEP void ext4_xattr_inode_set_class(struct inode *ea_inode) { @@ -226,7 +222,7 @@ check_xattrs(struct inode *inode, struct buffer_head *bh, /* Find the end of the names list */ while (!IS_LAST_ENTRY(e)) { struct ext4_xattr_entry *next = EXT4_XATTR_NEXT(e); - if ((void *)next >= end) { + if ((void *)next + sizeof(u32) > end) { err_str = "e_name out of bounds"; goto errout; } @@ -464,6 +460,21 @@ static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino, inode_unlock(inode); } + /* + * Since this function resolves references from active xattr entries, + * the EA inode must be in active state (i_nlink=1, ref_count>0). + * i_nlink > 1, i_nlink == 0 (dangling reference), or ref_count == 0 + * (inconsistent with an active entry) all indicate on-disk corruption. + */ + if (inode->i_nlink != 1 || !ext4_xattr_inode_get_ref(inode)) { + ext4_error(parent->i_sb, + "EA inode %lu has unexpected i_nlink=%u ref_count=%llu", + ea_ino, inode->i_nlink, + ext4_xattr_inode_get_ref(inode)); + ext4_put_ea_inode(inode); + return -EFSCORRUPTED; + } + *ea_inode = inode; return 0; } @@ -567,7 +578,7 @@ ext4_xattr_inode_get(struct inode *inode, struct ext4_xattr_entry *entry, ea_inode->i_ino, true /* reusable */); } out: - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return err; } @@ -1035,7 +1046,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, ref_count = ext4_xattr_inode_get_ref(ea_inode); if ((ref_count == 0 && ref_change < 0) || (ref_count == U64_MAX && ref_change > 0)) { ext4_error_inode(ea_inode, __func__, __LINE__, 0, - "EA inode %lu ref wraparound: ref_count=%lld ref_change=%d", + "EA inode %llu ref wraparound: ref_count=%lld ref_change=%d", ea_inode->i_ino, ref_count, ref_change); brelse(iloc.bh); ret = -EFSCORRUPTED; @@ -1046,7 +1057,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, if (ref_change > 0) { if (ref_count == 1) { - WARN_ONCE(ea_inode->i_nlink, "EA inode %lu i_nlink=%u", + WARN_ONCE(ea_inode->i_nlink, "EA inode %llu i_nlink=%u", ea_inode->i_ino, ea_inode->i_nlink); set_nlink(ea_inode, 1); @@ -1055,7 +1066,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, } else { if (ref_count == 0) { WARN_ONCE(ea_inode->i_nlink != 1, - "EA inode %lu i_nlink=%u", + "EA inode %llu i_nlink=%u", ea_inode->i_ino, ea_inode->i_nlink); clear_nlink(ea_inode); @@ -1104,10 +1115,10 @@ static int ext4_xattr_inode_inc_ref_all(handle_t *handle, struct inode *parent, err = ext4_xattr_inode_inc_ref(handle, ea_inode); if (err) { ext4_warning_inode(ea_inode, "inc ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); goto cleanup; } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return 0; @@ -1133,7 +1144,7 @@ cleanup: if (err) ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return saved_err; } @@ -1160,12 +1171,11 @@ static void ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, struct buffer_head *bh, struct ext4_xattr_entry *first, bool block_csum, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits, bool skip_quota) { struct inode *ea_inode; struct ext4_xattr_entry *entry; - struct ext4_iloc iloc; + struct ext4_iloc iloc = { .bh = NULL }; bool dirty = false; unsigned int ea_ino; int err; @@ -1197,14 +1207,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err) continue; - err = ext4_expand_inode_array(ea_inode_array, ea_inode); - if (err) { - ext4_warning_inode(ea_inode, - "Expand inode array err=%d", err); - iput(ea_inode); - continue; - } - err = ext4_journal_ensure_credits_fn(handle, credits, credits, ext4_free_metadata_revoke_credits(parent->i_sb, 1), ext4_xattr_restart_fn(handle, parent, bh, block_csum, @@ -1212,6 +1214,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err < 0) { ext4_warning_inode(ea_inode, "Ensure credits err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } if (err > 0) { @@ -1221,6 +1224,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, ext4_warning_inode(ea_inode, "Re-get write access err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } } @@ -1229,6 +1233,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err) { ext4_warning_inode(ea_inode, "ea_inode dec ref err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } @@ -1245,6 +1250,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, entry->e_value_inum = 0; entry->e_value_size = 0; + ext4_put_ea_inode(ea_inode); dirty = true; } @@ -1260,6 +1266,8 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, ext4_warning_inode(parent, "handle dirty metadata err=%d", err); } + + brelse(iloc.bh); } /* @@ -1269,7 +1277,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, static void ext4_xattr_release_block(handle_t *handle, struct inode *inode, struct buffer_head *bh, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits) { struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); @@ -1311,7 +1318,6 @@ retry_ref: ext4_xattr_inode_dec_ref_all(handle, inode, bh, BFIRST(bh), true /* block_csum */, - ea_inode_array, extra_credits, true /* skip_quota */); ext4_free_blocks(handle, inode, bh, 0, 1, @@ -1503,7 +1509,7 @@ static struct inode *ext4_xattr_inode_create(handle_t *handle, if (ext4_xattr_inode_dec_ref(handle, ea_inode)) ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return ERR_PTR(err); } @@ -1548,7 +1554,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, while (ce) { ea_inode = ext4_iget(inode->i_sb, ce->e_value, - EXT4_IGET_EA_INODE); + EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT); if (IS_ERR(ea_inode)) goto next_entry; ext4_xattr_inode_set_class(ea_inode); @@ -1562,7 +1568,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, kvfree(ea_data); return ea_inode; } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); next_entry: ce = mb_cache_entry_find_next(ea_inode_cache, ce); } @@ -1613,7 +1619,7 @@ static struct inode *ext4_xattr_inode_lookup_create(handle_t *handle, ea_inode->i_ino, true /* reusable */); return ea_inode; out_err: - iput(ea_inode); + ext4_put_ea_inode(ea_inode); ext4_xattr_inode_free_quota(inode, NULL, value_len); return ERR_PTR(err); } @@ -1846,7 +1852,7 @@ update_hash: ret = 0; out: - iput(old_ea_inode); + ext4_put_ea_inode(old_ea_inode); return ret; } @@ -2008,7 +2014,7 @@ clone_block: old_ea_inode_quota = le32_to_cpu( s->here->e_value_size); } - iput(tmp_inode); + ext4_put_ea_inode(tmp_inode); s->here->e_value_inum = 0; s->here->e_value_size = 0; @@ -2073,12 +2079,13 @@ inserted: * stable so we can check the additional * reference fits. */ - ref = le32_to_cpu(BHDR(new_bh)->h_refcount) + 1; - if (ref > EXT4_XATTR_REFCOUNT_MAX) { + ref = le32_to_cpu(BHDR(new_bh)->h_refcount); + if (ref >= EXT4_XATTR_REFCOUNT_MAX) { /* * Undo everything and check mbcache * again. */ + clear_bit(MBE_REUSABLE_B, &ce->e_flags); unlock_buffer(new_bh); dquot_free_block(inode, EXT4_C2B(EXT4_SB(sb), @@ -2089,6 +2096,7 @@ inserted: new_bh = NULL; goto inserted; } + ref++; BHDR(new_bh)->h_refcount = cpu_to_le32(ref); if (ref == EXT4_XATTR_REFCOUNT_MAX) clear_bit(MBE_REUSABLE_B, &ce->e_flags); @@ -2148,7 +2156,7 @@ getblk_failed: ext4_warning_inode(ea_inode, "dec ref error=%d", error); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); ea_inode = NULL; } @@ -2180,12 +2188,8 @@ getblk_failed: /* Drop the previous xattr block. */ if (bs->bh && bs->bh != new_bh) { - struct ext4_xattr_inode_array *ea_inode_array = NULL; - ext4_xattr_release_block(handle, inode, bs->bh, - &ea_inode_array, 0 /* extra_credits */); - ext4_xattr_inode_array_free(ea_inode_array); } error = 0; @@ -2201,7 +2205,7 @@ cleanup: ext4_xattr_inode_free_quota(inode, ea_inode, i_size_read(ea_inode)); } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } if (ce) mb_cache_entry_put(ea_block_cache, ce); @@ -2283,7 +2287,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, ext4_xattr_inode_free_quota(inode, ea_inode, i_size_read(ea_inode)); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return error; } @@ -2295,7 +2299,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, header->h_magic = cpu_to_le32(0); ext4_clear_inode_state(inode, EXT4_STATE_XATTR); } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return 0; } @@ -2837,6 +2841,7 @@ retry: s_min_extra_isize) { tried_min_extra_isize++; new_extra_isize = s_min_extra_isize; + error = 0; goto retry; } goto cleanup; @@ -2854,53 +2859,13 @@ shift: cleanup: if (error && (mnt_count != le16_to_cpu(sbi->s_es->s_mnt_count))) { - ext4_warning(inode->i_sb, "Unable to expand inode %lu. Delete some EAs or run e2fsck.", + ext4_warning(inode->i_sb, "Unable to expand inode %llu. Delete some EAs or run e2fsck.", inode->i_ino); mnt_count = le16_to_cpu(sbi->s_es->s_mnt_count); } return error; } -#define EIA_INCR 16 /* must be 2^n */ -#define EIA_MASK (EIA_INCR - 1) - -/* Add the large xattr @inode into @ea_inode_array for deferred iput(). - * If @ea_inode_array is new or full it will be grown and the old - * contents copied over. - */ -static int -ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, - struct inode *inode) -{ - if (*ea_inode_array == NULL) { - /* - * Start with 15 inodes, so it fits into a power-of-two size. - */ - (*ea_inode_array) = kmalloc_flex(**ea_inode_array, inodes, - EIA_MASK, GFP_NOFS); - if (*ea_inode_array == NULL) - return -ENOMEM; - (*ea_inode_array)->count = 0; - } else if (((*ea_inode_array)->count & EIA_MASK) == EIA_MASK) { - /* expand the array once all 15 + n * 16 slots are full */ - struct ext4_xattr_inode_array *new_array = NULL; - - new_array = kmalloc_flex(**ea_inode_array, inodes, - (*ea_inode_array)->count + EIA_INCR, - GFP_NOFS); - if (new_array == NULL) - return -ENOMEM; - memcpy(new_array, *ea_inode_array, - struct_size(*ea_inode_array, inodes, - (*ea_inode_array)->count)); - kfree(*ea_inode_array); - *ea_inode_array = new_array; - } - (*ea_inode_array)->count++; - (*ea_inode_array)->inodes[(*ea_inode_array)->count - 1] = inode; - return 0; -} - /* * ext4_xattr_delete_inode() * @@ -2911,7 +2876,6 @@ ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, * references on xattr block and xattr inodes. */ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits) { struct buffer_head *bh = NULL; @@ -2950,7 +2914,6 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, ext4_xattr_inode_dec_ref_all(handle, inode, iloc.bh, IFIRST(header), false /* block_csum */, - ea_inode_array, extra_credits, false /* skip_quota */); } @@ -2984,12 +2947,12 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, continue; ext4_xattr_inode_free_quota(inode, ea_inode, le32_to_cpu(entry->e_value_size)); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } } - ext4_xattr_release_block(handle, inode, bh, ea_inode_array, + ext4_xattr_release_block(handle, inode, bh, extra_credits); /* * Update i_file_acl value in the same transaction that releases @@ -3011,16 +2974,63 @@ cleanup: return error; } -void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *ea_inode_array) +/* + * Worker function for deferred EA inode iput. Processes all inodes queued + * on s_ea_inode_to_free in a context free of xattr_sem/jbd2 handle locks. + */ +static void ext4_ea_inode_work(struct work_struct *work) { - int idx; + struct ext4_sb_info *sbi = container_of(to_delayed_work(work), + struct ext4_sb_info, + s_ea_inode_work); + struct llist_node *node = llist_del_all(&sbi->s_ea_inode_to_free); + + while (node) { + struct ext4_inode_info *ei = container_of(node, + struct ext4_inode_info, i_ea_iput_node); + node = node->next; + iput(&ei->vfs_inode); + } +} - if (ea_inode_array == NULL) +/* + * Release a VFS reference on an EA inode. Must be used instead of iput() + * in any context where xattr_sem or a jbd2 handle is held. + * + * If this is not the last reference, drops it immediately via + * iput_if_not_last() with no further action needed. + * + * If this is the last reference, the inode is linked onto a per-sb + * llist via i_ea_iput_node (embedded in ext4_inode_info, sharing space + * with the unused xattr_sem) and a delayed worker performs the final + * iput() in a clean context. + * + * Note: while an inode is on s_ea_inode_to_free, the unconsumed i_count + * reference (still 1) keeps it in the inode cache, so any concurrent + * iget() bumps i_count to >= 2 and iput_if_not_last() will succeed. + * Nobody will add the inode a second time until ext4_ea_inode_work() + * drops that reference via iput(). + */ +void ext4_put_ea_inode(struct inode *inode) +{ + if (!inode) return; + WARN_ON_ONCE(!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)); + if (iput_if_not_last(inode)) + return; + llist_add(&EXT4_I(inode)->i_ea_iput_node, + &EXT4_SB(inode->i_sb)->s_ea_inode_to_free); + /* + * Use a short delay to allow multiple EA inodes to accumulate, + * reducing workqueue wakeups when several are released together. + */ + schedule_delayed_work(&EXT4_SB(inode->i_sb)->s_ea_inode_work, 1); +} - for (idx = 0; idx < ea_inode_array->count; ++idx) - iput(ea_inode_array->inodes[idx]); - kfree(ea_inode_array); +void ext4_init_ea_inode_work(struct ext4_sb_info *sbi) +{ + init_llist_head(&sbi->s_ea_inode_to_free); + INIT_DELAYED_WORK(&sbi->s_ea_inode_work, ext4_ea_inode_work); } /* diff --git a/fs/ext4/xattr.h b/fs/ext4/xattr.h index 1fedf44d4fb6..821dc6a50e51 100644 --- a/fs/ext4/xattr.h +++ b/fs/ext4/xattr.h @@ -131,11 +131,6 @@ struct ext4_xattr_ibody_find { struct ext4_iloc iloc; }; -struct ext4_xattr_inode_array { - unsigned int count; - struct inode *inodes[] __counted_by(count); -}; - extern const struct xattr_handler ext4_xattr_user_handler; extern const struct xattr_handler ext4_xattr_trusted_handler; extern const struct xattr_handler ext4_xattr_security_handler; @@ -187,9 +182,9 @@ extern int __ext4_xattr_set_credits(struct super_block *sb, struct inode *inode, bool is_create); extern int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, - struct ext4_xattr_inode_array **array, int extra_credits); -extern void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *array); +extern void ext4_init_ea_inode_work(struct ext4_sb_info *sbi); +extern void ext4_put_ea_inode(struct inode *inode); extern int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize, struct ext4_inode *raw_inode, handle_t *handle); |