diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-08-20 13:17:11 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-08-20 13:17:11 -0700 |
| commit | c84d3e3130dfe1058cb27dc78e7ad8bd36f0545a (patch) | |
| tree | 0adeaef5227fcaf5b95e8831b5f33af00cec50ae | |
| parent | 50c44fea13ec339d0d457079b254e8c8420d6511 (diff) | |
| parent | 9091c97be34083587a75db174aab51551d8e8543 (diff) | |
Merge tag 'ext4_for_linus-7.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4
Pull ext4 updates from Ted Ts'o:
- Improve performance by allowing parallel DIO writes when we were
previously being overly conservative when checking whether it was
safe to avoid requiring an exclusive lock
- Improve the performance of ext4_mb_prefetch() used by fallocate() by
avoiding work when it is not needed
- Remove the unnecessary custom end_io function
ext4_end_buffer_io_sync()
- Improve performance when performing an overwrite to an already
uptodate folio
- Clean up how we handle deallocating EA inodes to avoid a potential
lock ordering issue when there is a failed mount while an EA inode is
still being evicted
- Use str_plural() instead of a custom macro
- Avoid soft lockups or RCU stalls if there are many busy buffers
(caused by heavy I/O) while checkpointing
- Use scoped NOFS when starting a handle in nojournal mode
- Align fields in handle structure to optimize setting and getting the
h_type and h_line_no fields
- Fix documentation of the meta_bg block group layout
- Bug fixes:
- Fix a potential out-of-bounds read in ext4_read_inline_dir()
- Fix a potential deadlock when concurrent xattr operations are
racing with each other when some of the xattrs are using the
ea_inode feature
- Fix a spurious warning with data=journal that can be triggered
when writeback races with remounting the file system read-only
- Fix a potential deadlock when EXT4_IOC_MIGRATE races with a file
system freeze operation
- Make sure all in-flight direct I/O operations are complete before
falling back to buffered I/O
- Handle IOCB_NOWAIT properly when performing a extending DAX write
- Prevent potentially sleeping on a block allocation when
IOCB_NOWAIT is set
- Fix potential races when racing an inline data write with a page
fault
- Propagate errors when adding or removing extent ranges during a
fast commit replay
- Avoid trying to expand an inode's extra size when it is being
evicted to avoid a number of corner case or deadlocks
- Avoid spurious error when retrying inode extra size expansion
- Fix corner cases where we underestimate the number of journal
credits needed
- Avoid hangs/crashes/WARNINGS caused by maliciously corrupted file
systems
- Don't issue spurious orphan clean message on RO file systems
- Avoid leaving the file system in an inconsistent state after a
crash when a WRITE_ZEROS in progress converting an unwritten
extent to a written extent
- Handle WRITE_ZEROS correctly when there are some partially dirtied
regions in the page cache
- Pass errors during zero-rage, truncate, or punch hole to the
caller if ext4_get_block() fails
- Wait for writeback to finish when triggered by zero-range or
zero-range for those devices that require stable writes
- If the reserved gid superblock field is set, set the reserved gid
instead of the reserved uid
* tag 'ext4_for_linus-7.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4: (56 commits)
ext4: fix estimate extent index blocks in ext4_ext_index_trans_blocks()
ext4: fix transaction overflow during writeback
ext4: teach ext4_meta_trans_blocks() about number of allocated extents
ext4: guard against NULL s_group_info in ext4_get_group_info
ext4: fix spurious message about orphan cleanup on RO fs
ext4: stop retrying saturated xattr cache entries
ext4: don't enable DAX on new encrypted files
ext4: protect WRITE_ZEROES written extents with orphan list
ext4: export converted block count from ext4_convert_unwritten_extents()
ext4: fix incorrect function call when initializing s_resgid
ext4: validate EA inode i_nlink in ext4_xattr_inode_iget
jbd2: align h_type and h_line_no in the handle structure on byte boundaries
ext4: enable scoped NOFS when starting a handle in nojournal mode
ext4: write back partial-zeroed edges in WRITE_ZEROES
ext4: zero out whole block for clean edges in WRITE_ZEROES
ext4: track partial-zero outcome per edge in ext4_zero_partial_blocks()
ext4: clarify return semantics of ext4_load_tail_bh()
ext4: move partial block zeroing earlier in ext4_zero_range()
ext4: check return value of ext4_get_block() in ext4_load_tail_bh()
ext4: skip tail block zeroing for inline data files
...
| -rw-r--r-- | Documentation/filesystems/ext4/group_descr.rst | 9 | ||||
| -rw-r--r-- | fs/buffer.c | 3 | ||||
| -rw-r--r-- | fs/ext4/balloc.c | 4 | ||||
| -rw-r--r-- | fs/ext4/crypto.c | 40 | ||||
| -rw-r--r-- | fs/ext4/dir.c | 20 | ||||
| -rw-r--r-- | fs/ext4/ext4.h | 40 | ||||
| -rw-r--r-- | fs/ext4/ext4_jbd2.c | 36 | ||||
| -rw-r--r-- | fs/ext4/ext4_jbd2.h | 6 | ||||
| -rw-r--r-- | fs/ext4/extents.c | 182 | ||||
| -rw-r--r-- | fs/ext4/fast_commit.c | 37 | ||||
| -rw-r--r-- | fs/ext4/file.c | 151 | ||||
| -rw-r--r-- | fs/ext4/ialloc.c | 4 | ||||
| -rw-r--r-- | fs/ext4/inline.c | 38 | ||||
| -rw-r--r-- | fs/ext4/inode.c | 207 | ||||
| -rw-r--r-- | fs/ext4/mballoc.c | 21 | ||||
| -rw-r--r-- | fs/ext4/migrate.c | 3 | ||||
| -rw-r--r-- | fs/ext4/namei.c | 2 | ||||
| -rw-r--r-- | fs/ext4/orphan.c | 17 | ||||
| -rw-r--r-- | fs/ext4/super.c | 32 | ||||
| -rw-r--r-- | fs/ext4/xattr.c | 178 | ||||
| -rw-r--r-- | fs/ext4/xattr.h | 9 | ||||
| -rw-r--r-- | fs/jbd2/checkpoint.c | 28 | ||||
| -rw-r--r-- | fs/jbd2/journal.c | 1 | ||||
| -rw-r--r-- | include/linux/fs.h | 15 | ||||
| -rw-r--r-- | include/linux/jbd2.h | 11 |
25 files changed, 734 insertions, 360 deletions
diff --git a/Documentation/filesystems/ext4/group_descr.rst b/Documentation/filesystems/ext4/group_descr.rst index 392ec44f8fb0..9a0c2d92c2ef 100644 --- a/Documentation/filesystems/ext4/group_descr.rst +++ b/Documentation/filesystems/ext4/group_descr.rst @@ -20,11 +20,10 @@ group of the flex group. If the meta_bg feature flag is set, then several block groups are grouped together into a meta group. Note that in the meta_bg case, -however, the first and last two block groups within the larger meta -group contain only group descriptors for the groups inside the meta -group. - -flex_bg and meta_bg do not appear to be mutually exclusive features. +however, the superblock and a single block group descriptor block is +placed at the beginning of the first, second, and last block groups in a +meta-block group. The flex_bg and meta_bg features are not mutually +exclusive. In ext2, ext3, and ext4 (when the 64bit feature is not enabled), the block group descriptor was only 32 bytes long and therefore ends at diff --git a/fs/buffer.c b/fs/buffer.c index 65a2893b2cbb..dd50d17b8907 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2081,6 +2081,7 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) { size_t block_start, block_end; bool partial = false; + bool uptodate = folio_test_uptodate(folio); unsigned blocksize; struct buffer_head *bh, *head; @@ -2103,6 +2104,8 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) clear_buffer_new(bh); block_start = block_end; + if (uptodate && block_start >= to) + break; bh = bh->b_this_page; } while (bh != head); diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c index 8040c731b3e4..52f4c5169f91 100644 --- a/fs/ext4/balloc.c +++ b/fs/ext4/balloc.c @@ -331,9 +331,13 @@ struct ext4_group_info *ext4_get_group_info(struct super_block *sb, if (unlikely(group >= EXT4_SB(sb)->s_groups_count)) return NULL; + if (unlikely(!EXT4_SB(sb)->s_group_info)) + return NULL; indexv = group >> (EXT4_DESC_PER_BLOCK_BITS(sb)); indexh = group & ((EXT4_DESC_PER_BLOCK(sb)) - 1); grp_info = sbi_array_rcu_deref(EXT4_SB(sb), s_group_info, indexv); + if (unlikely(!grp_info)) + return NULL; return grp_info[indexh]; } diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c index 9265cfe62c83..1a0fccb084ef 100644 --- a/fs/ext4/crypto.c +++ b/fs/ext4/crypto.c @@ -144,7 +144,13 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, if (inode->i_ino == EXT4_ROOT_INO) return -EPERM; - if (WARN_ON_ONCE(IS_DAX(inode) && i_size_read(inode))) + /* + * For new encrypted inodes, S_DAX is never set in the first place. + * + * For existing inodes, this is called only on empty directories. ext4 + * never sets S_DAX on directories. + */ + if (WARN_ON_ONCE(IS_DAX(inode))) return -EINVAL; if (ext4_test_inode_flag(inode, EXT4_INODE_DAX)) @@ -164,27 +170,24 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, if (handle) { /* + * __ext4_new_inode() should have already set the encrypt flag + * on the inode and avoided enabling inline data. + */ + if (WARN_ON_ONCE(!IS_ENCRYPTED(inode))) + return -EINVAL; + if (WARN_ON_ONCE(ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))) + return -EINVAL; + /* * Since the inode is new it is ok to pass the * XATTR_CREATE flag. This is necessary to match the * remaining journal credits check in the set_handle * function with the credits allocated for the new * inode. */ - res = ext4_xattr_set_handle(handle, inode, - EXT4_XATTR_INDEX_ENCRYPTION, - EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, - ctx, len, XATTR_CREATE); - if (!res) { - ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - ext4_clear_inode_state(inode, - EXT4_STATE_MAY_INLINE_DATA); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ - ext4_set_inode_flags(inode, false); - } - return res; + return ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, XATTR_CREATE); } res = dquot_initialize(inode); @@ -205,10 +208,7 @@ retry: ctx, len, 0); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ + /* Update inode->i_flags to set S_ENCRYPTED. */ ext4_set_inode_flags(inode, false); res = ext4_mark_inode_dirty(handle, inode); if (res) diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c index 17edd678fa87..8d7b81e6948e 100644 --- a/fs/ext4/dir.c +++ b/fs/ext4/dir.c @@ -138,6 +138,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) struct buffer_head *bh = NULL; struct fscrypt_str fstr = FSTR_INIT(NULL, 0); struct dir_private_info *info = file->private_data; + bool has_csum = ext4_has_feature_metadata_csum(sb); err = fscrypt_prepare_readdir(inode); if (err) @@ -149,7 +150,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) return err; /* Can we just clear INDEX flag to ignore htree information? */ - if (!ext4_has_feature_metadata_csum(sb)) { + if (!has_csum) { /* * We don't set the inode dirty flag since it's not * critical that it gets flushed back to the disk. @@ -235,7 +236,10 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) * dirent right now. Scan from the start of the block * to make sure. */ if (!inode_eq_iversion(inode, info->cookie)) { - for (i = 0; i < sb->s_blocksize && i < offset; ) { + for (i = 0; + i <= sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode) && + i < offset;) { de = (struct ext4_dir_entry_2 *) (bh->b_data + i); /* It's too expensive to do a full @@ -257,6 +261,17 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) info->cookie = inode_query_iversion(inode); } + if (unlikely(offset < sb->s_blocksize && + offset > sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode))) { + EXT4_ERROR_FILE(file, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, size=%lu", + "directory entry too close to block end", + offset, sb->s_blocksize); + ctx->pos = round_up(ctx->pos, sb->s_blocksize); + goto next_block; + } + while (ctx->pos < inode->i_size && offset < sb->s_blocksize) { de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); @@ -312,6 +327,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) ctx->pos += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); } +next_block: if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode)) goto done; brelse(bh); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 2fdff50526e9..724a27e8be61 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -334,7 +334,7 @@ struct ext4_io_submit { #define EXT4_MAX_BLOCK_SIZE 65536 #define EXT4_MIN_BLOCK_LOG_SIZE 10 #define EXT4_MAX_BLOCK_LOG_SIZE 16 -#define EXT4_MAX_CLUSTER_LOG_SIZE 30 +#define EXT4_MAX_CLUSTER_LOG_SIZE 28 #ifdef __KERNEL__ # define EXT4_BLOCK_SIZE(s) ((s)->s_blocksize) #else @@ -1070,8 +1070,14 @@ struct ext4_inode_info { * between readers of EAs and writers of regular file data, so * instead we synchronize on xattr_sem when reading or changing * EAs. + * + * EA inodes (EXT4_EA_INODE_FL) do not use xattr_sem; they reuse + * the space for deferred iput linkage. */ - struct rw_semaphore xattr_sem; + union { + struct rw_semaphore xattr_sem; + struct llist_node i_ea_iput_node; + }; /* * Inodes with EXT4_STATE_ORPHAN_FILE use i_orphan_idx. Otherwise @@ -1770,6 +1776,11 @@ struct ext4_sb_info { struct ext4_es_stats s_es_stats; struct mb_cache *s_ea_block_cache; struct mb_cache *s_ea_inode_cache; + + /* Deferred iput for EA inodes to avoid lock ordering issues */ + struct llist_head s_ea_inode_to_free; + struct delayed_work s_ea_inode_work; + spinlock_t s_es_lock ____cacheline_aligned_in_smp; /* Journal triggers for checksum computation */ @@ -3148,14 +3159,15 @@ int do_journal_get_write_access(handle_t *handle, struct inode *inode, struct buffer_head *bh); void ext4_set_inode_mapping_order(struct inode *inode); #define FALL_BACK_TO_NONDELALLOC 1 -#define CONVERT_INLINE_DATA 2 +#define EXT4_WRITE_DATA_INLINE 2 typedef enum { EXT4_IGET_NORMAL = 0, EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */ EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */ EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */ - EXT4_IGET_EA_INODE = 0x0008 /* Inode should contain an EA value */ + EXT4_IGET_EA_INODE = 0x0008, /* Inode should contain an EA value */ + EXT4_IGET_NOWAIT = 0x0010 /* Non-blocking lookup (skip if freeing) */ } ext4_iget_flags; extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, @@ -3196,10 +3208,13 @@ extern int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode); extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks); extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks); extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks, - int pextents); + int pextents, int alloc_extents); extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end); + +#define EXT4_PARTIAL_ZERO_START 0x1 +#define EXT4_PARTIAL_ZERO_END 0x2 extern int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, - loff_t length, bool *did_zero); + loff_t length, unsigned int *partial_zeroed); extern vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf); extern qsize_t *ext4_get_reserved_space(struct inode *inode); extern int ext4_get_projid(struct inode *inode, kprojid_t *projid); @@ -3651,7 +3666,13 @@ struct ext4_group_info { #define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \ (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state))) #define EXT4_MB_GRP_TEST_AND_SET_READ(grp) \ - (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state))) + (ext4_mb_grp_test_and_set_read((grp))) + +static inline int ext4_mb_grp_test_and_set_read(struct ext4_group_info *grp) +{ + return (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state) || + test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state)); +} #define EXT4_MAX_CONTENTION 8 #define EXT4_CONTENTION_THRESHOLD 2 @@ -3760,7 +3781,7 @@ extern int ext4_generic_write_inline_data(struct address_space *mapping, struct inode *inode, loff_t pos, unsigned len, struct folio **foliop, - void **fsdata, bool da); + bool da); extern int ext4_try_add_inline_entry(handle_t *handle, struct ext4_filename *fname, struct inode *dir, struct inode *inode); @@ -3892,7 +3913,8 @@ extern void ext4_ext_release(struct super_block *); extern long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len); extern int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, - loff_t offset, ssize_t len); + loff_t offset, ssize_t len, + ext4_lblk_t *converted); extern int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, loff_t offset, ssize_t len); extern int ext4_convert_unwritten_io_end_vec(handle_t *handle, diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c index 02b066299164..53ddedb52a6f 100644 --- a/fs/ext4/ext4_jbd2.c +++ b/fs/ext4/ext4_jbd2.c @@ -33,14 +33,22 @@ int ext4_inode_journal_mode(struct inode *inode) static handle_t *ext4_get_nojournal(void) { handle_t *handle = current->journal_info; - unsigned long ref_cnt = (unsigned long)handle; - BUG_ON(ref_cnt >= EXT4_NOJOURNAL_MAX_REF_COUNT); - - ref_cnt++; - handle = (handle_t *)ref_cnt; - - current->journal_info = handle; + BUG_ON(handle && !handle->h_invalid); + + if (!handle) { + handle = jbd2_alloc_handle(GFP_NOFS); + if (!handle) + return ERR_PTR(-ENOMEM); + handle->h_invalid = 1; + /* + * This is done by start_this_handle() if journalling + * is enabled. + */ + handle->saved_alloc_context = memalloc_nofs_save(); + current->journal_info = handle; + } + handle->h_ref++; return handle; } @@ -48,14 +56,14 @@ static handle_t *ext4_get_nojournal(void) /* Decrement the non-pointer handle value */ static void ext4_put_nojournal(handle_t *handle) { - unsigned long ref_cnt = (unsigned long)handle; + BUG_ON(handle->h_ref == 0); - BUG_ON(ref_cnt == 0); - - ref_cnt--; - handle = (handle_t *)ref_cnt; - - current->journal_info = handle; + handle->h_ref--; + if (handle->h_ref == 0) { + memalloc_nofs_restore(handle->saved_alloc_context); + jbd2_free_handle(handle); + current->journal_info = NULL; + } } /* diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h index 63d17c5201b5..2fbf48b3dfe2 100644 --- a/fs/ext4/ext4_jbd2.h +++ b/fs/ext4/ext4_jbd2.h @@ -182,15 +182,11 @@ handle_t *__ext4_journal_start_sb(struct inode *inode, struct super_block *sb, int rsv_blocks, int revoke_creds); int __ext4_journal_stop(const char *where, unsigned int line, handle_t *handle); -#define EXT4_NOJOURNAL_MAX_REF_COUNT ((unsigned long) 4096) - /* Note: Do not use this for NULL handles. This is only to determine if * a properly allocated handle is using a journal or not. */ static inline int ext4_handle_valid(handle_t *handle) { - if ((unsigned long)handle < EXT4_NOJOURNAL_MAX_REF_COUNT) - return 0; - return 1; + return (handle && !handle->h_invalid); } static inline void ext4_handle_sync(handle_t *handle) diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 15972410d460..76038b6c3655 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -2427,9 +2427,17 @@ int ext4_ext_index_trans_blocks(struct inode *inode, int extents) */ if (extents <= 1) index = (EXT4_MAX_EXTENT_DEPTH * 2) + extents; - else - index = (EXT4_MAX_EXTENT_DEPTH * 3) + - DIV_ROUND_UP(extents, ext4_ext_space_block(inode, 0)); + else { + int ext_max = ext4_ext_space_block(inode, 0); + + index = EXT4_MAX_EXTENT_DEPTH * 3; + /* + * Modified extents need not start at the beginning of the + * leaf. Already two extents may need two leaf block + * modifications... + */ + index += DIV_ROUND_UP(extents + ext_max - 1, ext_max); + } return index; } @@ -4571,6 +4579,22 @@ retry_remove_space: return err; } +/* + * Pre-allocate blocks for the range [@offset, @offset + @len). Allocated + * blocks are marked as unwritten by default. If EXT4_GET_BLOCKS_ZERO is + * set, the allocated blocks are zeroed on disk and their extents are + * converted to written state. + * + * When @new_size is nonzero, the caller intends to extend the file, and + * the file size should be updated to the end of the allocated blocks. + * + * Allocation may partially succeed due to some non-fatal issues. In that + * case, i_disksize (and i_size) is advanced up to the successfully + * processed portion of the range. + * + * Return 0 on success, or a negative error code on failure or partial + * failure. + */ static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len, loff_t new_size, int flags) { @@ -4585,6 +4609,7 @@ static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len, loff_t epos = 0, old_size = i_size_read(inode); unsigned int blkbits = inode->i_blkbits; bool alloc_zero = false; + bool orphan = false; BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)); map.m_lblk = offset >> blkbits; @@ -4659,19 +4684,49 @@ retry: if (alloc_zero && (map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) { + ext4_lblk_t converted; + + WARN_ON_ONCE(map.m_lblk + map.m_len > + EXT4_B_TO_LBLK(inode, new_size ?: old_size)); + ret = ext4_issue_zeroout(inode, map.m_lblk, map.m_pblk, map.m_len); - if (likely(!ret)) - ret = ext4_convert_unwritten_extents(NULL, + if (unlikely(ret)) + break; + + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, + credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + break; + } + + ret = ext4_convert_unwritten_extents(handle, inode, (loff_t)map.m_lblk << blkbits, - (loff_t)map.m_len << blkbits); + (loff_t)map.m_len << blkbits, + &converted); if (ret) - break; + map.m_len = converted; + + /* + * If blocks beyond i_disksize are converted, add + * the inode to the orphan list and advance the epos. + */ + if (new_size && converted) { + ret2 = ext4_orphan_add(handle, inode); + ret = ret ? ret : ret2; + orphan = true; + } + + ret3 = ext4_journal_stop(handle); + ret = ret ? ret : ret3; } map.m_lblk += map.m_len; map.m_len = len_lblk = len_lblk - map.m_len; epos = EXT4_LBLK_TO_B(inode, map.m_lblk); + if (ret) + break; } if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) @@ -4687,11 +4742,23 @@ retry: if (epos > new_size) epos = new_size; - handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); - if (IS_ERR(handle)) - return ret ? ret : PTR_ERR(handle); + handle = ext4_journal_start(inode, EXT4_HT_MISC, 2); + if (IS_ERR(handle)) { + /* + * The conversion has successfully completed. Not much to + * do with the error here so just cleanup the orphan list + * and hope for the best. + */ + if (orphan && inode->i_nlink) + ext4_orphan_del(NULL, inode); + ret2 = PTR_ERR(handle); + goto out; + } ext4_update_inode_size(inode, epos); + if (orphan && inode->i_nlink) + ext4_orphan_del(handle, inode); + ret2 = ext4_mark_inode_dirty(handle, inode); ext4_update_inode_fsync_trans(handle, inode, 1); ret3 = ext4_journal_stop(handle); @@ -4699,6 +4766,9 @@ retry: if (epos > old_size) pagecache_isize_extended(inode, old_size, epos); +out: + if (ret2) + ext4_std_error(inode->i_sb, ret2); return ret ? ret : ret2; } @@ -4715,7 +4785,7 @@ static long ext4_zero_range(struct file *file, loff_t offset, loff_t align_start, align_end, new_size = 0; loff_t end = offset + len; unsigned int blocksize = i_blocksize(inode); - bool partial_zeroed = false; + unsigned int partial_zeroed = 0; int ret, flags; trace_ext4_zero_range(inode, offset, len, mode); @@ -4734,10 +4804,16 @@ static long ext4_zero_range(struct file *file, loff_t offset, } flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; - /* Preallocate the range including the unaligned edges */ + /* + * Preallocate the range including the unaligned edges, and zero + * out partial blocks if they already contain data. + */ if (!IS_ALIGNED(offset | end, blocksize)) { ret = ext4_alloc_file_blocks(file, offset, len, new_size, flags); + if (!ret) + ret = ext4_zero_partial_blocks(inode, offset, len, + &partial_zeroed); if (ret) return ret; } @@ -4754,6 +4830,21 @@ static long ext4_zero_range(struct file *file, loff_t offset, /* Zero range excluding the unaligned edges */ align_start = round_up(offset, blocksize); align_end = round_down(end, blocksize); + + /* + * In WRITE_ZEROES mode, edges that were not partial-zeroed (clean + * unwritten or hole) must be allocated and zeroed as whole blocks. + * Expand the aligned range outward to cover them. + */ + if (mode & FALLOC_FL_WRITE_ZEROES) { + if (!IS_ALIGNED(offset, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_START)) + align_start = round_down(offset, blocksize); + if (!IS_ALIGNED(end, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_END)) + align_end = round_up(end, blocksize); + } + if (align_end > align_start) { if (mode & FALLOC_FL_WRITE_ZEROES) flags = EXT4_GET_BLOCKS_CREATE_ZERO | EXT4_EX_NOCACHE; @@ -4770,11 +4861,15 @@ static long ext4_zero_range(struct file *file, loff_t offset, if (IS_ALIGNED(offset | end, blocksize)) return ret; - /* Zero out partial block at the edges of the range */ - ret = ext4_zero_partial_blocks(inode, offset, len, &partial_zeroed); - if (ret) - return ret; - if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) { + /* + * In FALLOC_FL_WRITE_ZEROES mode, edges that have been partially + * zeroed must be written back to ensure the entire zeroed range + * is converted to the written state. In SYNC mode, writeback is + * also required to persist the zeroed data to disk. + */ + if (partial_zeroed && + ((mode & FALLOC_FL_WRITE_ZEROES) || + (file->f_flags & O_SYNC) || IS_SYNC(inode))) { ret = filemap_write_and_wait_range(inode->i_mapping, offset, end - 1); if (ret) @@ -4976,7 +5071,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * it can tell if the extent in the cache is a split extent. * But for now let's assume pextents as 2 always. */ - credits = ext4_meta_trans_blocks(inode, max_blocks, 2); + credits = ext4_meta_trans_blocks(inode, max_blocks, 2, 0); } if (credits) { @@ -5026,21 +5121,26 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * all unwritten extents within this range will be converted to * written extents. * - * This function is called from the direct IO end io call back - * function, to convert the fallocated extents after IO is completed. - * Returns 0 on success. + * This function is called from the direct/buffered I/O end io call back + * function and FALLOC_FL_WRITE_ZEROES, to convert the fallocated + * unwritten extents after data I/O is completed. + * + * Returns 0 on full success, or a negative error code on partial + * success or failure. The number of blocks converted is returned via + * @converted. */ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, - loff_t offset, ssize_t len) + loff_t offset, ssize_t len, + ext4_lblk_t *converted) { - unsigned int max_blocks; + ext4_lblk_t max_blocks, conv_blocks = 0; int ret = 0, ret2 = 0, ret3 = 0; struct ext4_map_blocks map; unsigned int blkbits = inode->i_blkbits; unsigned int credits = 0; map.m_lblk = offset >> blkbits; - max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); + map.m_len = max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); if (!handle) { /* @@ -5048,9 +5148,8 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, */ credits = ext4_chunk_trans_blocks(inode, max_blocks); } - while (ret >= 0 && ret < max_blocks) { - map.m_lblk += ret; - map.m_len = (max_blocks -= ret); + + while (max_blocks) { if (credits) { handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, credits); @@ -5067,23 +5166,34 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, ret = ext4_map_blocks(handle, inode, &map, EXT4_GET_BLOCKS_IO_CONVERT_EXT | EXT4_EX_NOCACHE); - if (ret <= 0) + if (ret <= 0) { ext4_warning(inode->i_sb, - "inode #%llu: block %u: len %u: " - "ext4_ext_map_blocks returned %d", - inode->i_ino, map.m_lblk, - map.m_len, ret); + "inode #%llu: block %u: len %u: ext4_map_blocks returned %d", + inode->i_ino, map.m_lblk, map.m_len, ret); + if (unlikely(ret == 0)) + ret = -EINVAL; + } else { + conv_blocks += map.m_len; + } + ret2 = ext4_mark_inode_dirty(handle, inode); if (credits) { ret3 = ext4_journal_stop(handle); if (unlikely(ret3)) ret2 = ret3; } - - if (ret <= 0 || ret2) + ret = ret < 0 ? ret : ret2; + if (ret) break; + + map.m_lblk += map.m_len; + map.m_len = (max_blocks -= map.m_len); } - return ret > 0 ? ret2 : ret; + /* Converted some or all blocks successfully? */ + if (converted) + *converted = conv_blocks; + + return ret; } int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end) @@ -5106,7 +5216,7 @@ int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end) list_for_each_entry(io_end_vec, &io_end->list_vec, list) { ret = ext4_convert_unwritten_extents(handle, io_end->inode, io_end_vec->offset, - io_end_vec->size); + io_end_vec->size, NULL); if (ret) break; } diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 8e2259799614..062103e42cd8 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -200,25 +200,6 @@ static inline void ext4_fc_set_snap_err(int *snap_err, int err) *snap_err = err; } -static void ext4_end_buffer_io_sync(struct bio *bio) -{ - struct buffer_head *bh; - bool uptodate = bio_endio_bh(bio, &bh); - - BUFFER_TRACE(bh, ""); - if (uptodate) { - ext4_debug("%s: Block %lld up-to-date", - __func__, bh->b_blocknr); - set_buffer_uptodate(bh); - } else { - ext4_debug("%s: Block %lld not up-to-date", - __func__, bh->b_blocknr); - clear_buffer_uptodate(bh); - } - - unlock_buffer(bh); -} - static void ext4_fc_free_inode_snap(struct inode *inode); static inline void ext4_fc_reset_inode(struct inode *inode) @@ -691,7 +672,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) lock_buffer(bh); set_buffer_dirty(bh); set_buffer_uptodate(bh); - bh_submit(bh, REQ_OP_WRITE | write_flags, ext4_end_buffer_io_sync); + bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); EXT4_SB(sb)->s_fc_bh = NULL; } @@ -2196,8 +2177,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) if (ret == 0) { /* Range is not mapped */ path = ext4_find_extent(inode, cur, path, 0); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } memset(&newex, 0, sizeof(newex)); newex.ee_block = cpu_to_le32(cur); ext4_ext_store_pblock( @@ -2209,8 +2193,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) path = ext4_ext_insert_extent(NULL, inode, path, &newex, 0); up_write((&EXT4_I(inode)->i_data_sem)); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } goto next; } @@ -2257,10 +2244,11 @@ next: } ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); + ret = 0; out: ext4_free_ext_path(path); iput(inode); - return 0; + return ret; } /* Replay DEL_RANGE tag */ @@ -2320,9 +2308,10 @@ ext4_fc_replay_del_range(struct super_block *sb, u8 *val) ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); ext4_mark_inode_dirty(NULL, inode); + ret = 0; out: iput(inode); - return 0; + return ret; } static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index f20d92255546..374b4bc25bd5 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -215,31 +215,60 @@ ext4_extending_io(struct inode *inode, loff_t offset, size_t len) return false; } -/* Is IO overwriting allocated or initialized blocks? */ -static bool ext4_overwrite_io(struct inode *inode, - loff_t pos, loff_t len, bool *unwritten) +/* + * Does an unaligned DIO write require partial block zeroing? + * + * Partial block zeroing is performed only for the head and tail blocks + * when they are partially covered by the write and the underlying extent + * is a hole or unwritten. Middle blocks (fully covered by the write) + * are written as whole blocks without zeroing. + * + * When zeroing is required, two concurrent unaligned DIO writes to the + * same partial block can race and corrupt each other's data, so the + * caller must take the exclusive i_rwsem and drain in-flight DIO. When + * zeroing is not required, shared lock is safe -- block allocation and + * unwritten conversion for middle blocks are protected by i_data_sem + * and inode_dio_begin(). + */ +static bool ext4_dio_needs_zeroing(struct inode *inode, loff_t pos, loff_t len) { struct ext4_map_blocks map; unsigned int blkbits = inode->i_blkbits; - int err, blklen; + unsigned long blockmask = inode->i_sb->s_blocksize - 1; + bool head_partial, tail_partial; + ext4_lblk_t head_lblk, tail_lblk; + int err; if (pos + len > i_size_read(inode)) - return false; + return true; - map.m_lblk = pos >> blkbits; - map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits); - blklen = map.m_len; + head_partial = (pos & blockmask) != 0; + tail_partial = ((pos + len) & blockmask) != 0; + head_lblk = pos >> blkbits; + tail_lblk = (pos + len - 1) >> blkbits; + + /* Check the head partial block. */ + if (head_partial) { + map.m_lblk = head_lblk; + map.m_len = tail_lblk - head_lblk + 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + /* If this mapping already covers the tail block, we're done. */ + if (!tail_partial || map.m_lblk + err > tail_lblk) + return false; + } - err = ext4_map_blocks(NULL, inode, &map, 0); - if (err != blklen) - return false; - /* - * 'err==len' means that all of the blocks have been preallocated, - * regardless of whether they have been initialized or not. We need to - * check m_flags to distinguish the unwritten extents. - */ - *unwritten = !(map.m_flags & EXT4_MAP_MAPPED); - return true; + /* Check the tail partial block. */ + if (tail_partial) { + map.m_lblk = tail_lblk; + map.m_len = 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + } + + return false; } static ssize_t ext4_generic_write_checks(struct kiocb *iocb, @@ -280,7 +309,7 @@ static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from) if (count <= 0) return count; - ret = file_modified(iocb->ki_filp); + ret = kiocb_modified(iocb); if (ret) return ret; @@ -311,6 +340,13 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; inode_lock(inode); + + /* + * Prevent concurrent direct I/O and buffered I/O to the same file + * range. Wait for in-flight DIO to finish before dirtying pages. + */ + inode_dio_wait(inode); + ret = ext4_write_checks(iocb, from); if (ret <= 0) goto out; @@ -402,7 +438,8 @@ static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size, error = ext4_convert_unwritten_extents_atomic(NULL, inode, pos, size); else if (!error && size && flags & IOMAP_DIO_UNWRITTEN) - error = ext4_convert_unwritten_extents(NULL, inode, pos, size); + error = ext4_convert_unwritten_extents(NULL, inode, pos, size, + NULL); if (error) return error; /* @@ -430,16 +467,28 @@ static const struct iomap_dio_ops ext4_dio_write_ops = { * condition requires an exclusive inode lock. If yes, then we restart the * whole operation by releasing the shared lock and acquiring exclusive lock. * - * - For unaligned_io we never take shared lock as it may cause data corruption - * when two unaligned IO tries to modify the same block e.g. while zeroing. + * The decision is layered, evaluated in this order: * - * - For extending writes case we don't take the shared lock, since it requires - * updating inode i_disksize and/or orphan handling with exclusive lock. + * 1. If kiocb_modified() needs to update security info (!IS_NOSEC), upgrade + * to the exclusive lock -- the security update itself requires it, + * regardless of whether the write extends the file or is aligned. * - * - shared locking will only be true mostly with overwrites, including - * initialized blocks and unwritten blocks. + * 2. If the write extends i_size or i_disksize, upgrade to the exclusive + * lock to safely update i_disksize and the orphan list, regardless of + * alignment. * - * - Otherwise we will switch to exclusive i_rwsem lock. + * 3. Otherwise, for aligned non-extending writes, shared lock is always + * sufficient regardless of extent state (written, unwritten, or hole). + * truncate/punch_hole cannot run while we hold the shared i_rwsem + * (they need it exclusively); after we release it, inode_dio_begin() + * keeps their inode_dio_wait() blocked until in-flight bios complete. + * i_data_sem serializes concurrent extent tree modifications. + * + * 4. Otherwise, the write is unaligned and non-extending. Shared lock is + * safe unless the DIO layer needs to perform partial block zeroing -- + * i.e. the head or tail partial block sits on a hole or unwritten + * extent. In that case upgrade to the exclusive lock and drain + * in-flight DIO to avoid races with concurrent partial block zeroing. */ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, bool *ilock_shared, bool *extend, @@ -450,7 +499,7 @@ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, loff_t offset; size_t count; ssize_t ret; - bool overwrite, unaligned_io, unwritten; + bool needs_zeroing = false; restart: ret = ext4_generic_write_checks(iocb, from); @@ -460,24 +509,22 @@ restart: offset = iocb->ki_pos; count = ret; - unaligned_io = ext4_unaligned_io(inode, from, offset); *extend = ext4_extending_io(inode, offset, count); - overwrite = ext4_overwrite_io(inode, offset, count, &unwritten); /* - * Determine whether we need to upgrade to an exclusive lock. This is - * required to change security info in file_modified(), for extending - * I/O, any form of non-overwrite I/O, and unaligned I/O to unwritten - * extents (as partial block zeroing may be required). + * For unaligned writes, check whether partial block zeroing will be + * needed. If so, exclusive lock is required to serialize against + * concurrent DIO that could race with the zeroing. * - * Note that unaligned writes are allowed under shared lock so long as - * they are pure overwrites. Otherwise, concurrent unaligned writes risk - * data corruption due to partial block zeroing in the dio layer, and so - * the I/O must occur exclusively. + * For aligned writes we skip this check entirely since allocation + * under shared lock is safe. */ + if (ext4_unaligned_io(inode, from, offset)) + needs_zeroing = ext4_dio_needs_zeroing(inode, offset, count); + + /* Determine whether we need to upgrade to an exclusive lock. */ if (*ilock_shared && - ((!IS_NOSEC(inode) || *extend || !overwrite || - (unaligned_io && unwritten)))) { + (!IS_NOSEC(inode) || *extend || needs_zeroing)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; @@ -491,21 +538,28 @@ restart: /* * Now that locking is settled, determine dio flags and exclusivity * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce - * behavior already. The inode lock is already held exclusive if the - * write is non-overwrite or extending, so drain all outstanding dio and - * set the force wait dio flag. + * behavior already. When holding the exclusive lock for a write that + * needs partial block zeroing or is extending the file, we must wait + * for the I/O to complete synchronously: + * + * - needs_zeroing: drain in-flight DIO whose end_io could race with + * our partial block zeroing, and force synchronous completion so we + * don't leave in-flight zeroing bios for the next writer to drain. + * + * - extend: the caller must update i_disksize after I/O completion, + * which requires the data to be on disk first. */ - if (!*ilock_shared && (unaligned_io || *extend)) { + if (!*ilock_shared && (needs_zeroing || *extend)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; } - if (unaligned_io && (!overwrite || unwritten)) + if (needs_zeroing) inode_dio_wait(inode); *dio_flags = IOMAP_DIO_FORCE_WAIT; } - ret = file_modified(file); + ret = kiocb_modified(iocb); if (ret < 0) goto out; @@ -674,6 +728,11 @@ ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) count = iov_iter_count(from); if (offset + count > EXT4_I(inode)->i_disksize) { + if (iocb->ki_flags & IOCB_NOWAIT) { + ret = -EAGAIN; + goto out; + } + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); if (IS_ERR(handle)) { ret = PTR_ERR(handle); diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index a40cb27f8116..a5831fc536db 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -997,6 +997,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, err = fscrypt_prepare_new_inode(dir, inode, &encrypt); if (err) goto out; + if (encrypt) + i_flags |= EXT4_ENCRYPT_FL; } err = dquot_initialize(inode); @@ -1306,6 +1308,8 @@ got: ei->i_extra_isize = sbi->s_want_extra_isize; ei->i_inline_off = 0; if (ext4_has_feature_inline_data(sb) && + /* Encrypted inodes cannot have inline data */ + !(ei->i_flags & EXT4_ENCRYPT_FL) && (!(ei->i_flags & (EXT4_DAX_FL|EXT4_EA_INODE_FL)) || S_ISDIR(mode))) ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); ret = inode; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 8045e4ff270c..ceee69a66482 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -22,8 +22,7 @@ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, - struct inode *inode, - void **fsdata); + struct inode *inode); static int ext4_get_inline_size(struct inode *inode) { @@ -697,7 +696,7 @@ int ext4_generic_write_inline_data(struct address_space *mapping, struct inode *inode, loff_t pos, unsigned len, struct folio **foliop, - void **fsdata, bool da) + bool da) { int ret; handle_t *handle; @@ -728,7 +727,7 @@ retry_journal: return ext4_convert_inline_data_to_extent(mapping, inode); } - ret = ext4_da_convert_inline_data_to_extent(mapping, inode, fsdata); + ret = ext4_da_convert_inline_data_to_extent(mapping, inode); if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) goto retry_journal; @@ -788,7 +787,7 @@ int ext4_try_to_write_inline_data(struct address_space *mapping, if (pos + len > ext4_get_max_inline_size(inode)) return ext4_convert_inline_data_to_extent(mapping, inode); return ext4_generic_write_inline_data(mapping, inode, pos, len, - foliop, NULL, false); + foliop, false); } int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, @@ -812,7 +811,19 @@ int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, goto out; } ext4_write_lock_xattr(inode, &no_expand); - BUG_ON(!ext4_has_inline_data(inode)); + /* + * We could have raced with ext4_page_mkwrite() converting + * the inode and clearing the inline data flag, so we just + * release resources and retry the whole write. + */ + if (unlikely(!ext4_has_inline_data(inode))) { + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + return 0; + } /* * ei->i_inline_off may have changed since @@ -883,8 +894,7 @@ out: * need to start the journal since the file's metadata isn't changed now. */ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, - struct inode *inode, - void **fsdata) + struct inode *inode) { int ret = 0, inline_size; struct folio *folio; @@ -922,7 +932,6 @@ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, folio_mark_dirty(folio); folio_mark_uptodate(folio); ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); - *fsdata = (void *)CONVERT_INLINE_DATA; out: up_read(&EXT4_I(inode)->xattr_sem); @@ -1454,6 +1463,8 @@ int ext4_read_inline_dir(struct file *file, /* for other entry, the real offset in * the buf has to be tuned accordingly. */ + if (i + ext4_dir_rec_len(1, NULL) > extra_size) + break; de = (struct ext4_dir_entry_2 *) (dir_buf + i - extra_offset); /* It's too expensive to do a full @@ -1488,10 +1499,17 @@ int ext4_read_inline_dir(struct file *file, continue; } + /* + * de lives at dir_buf + ctx->pos - extra_offset, within the + * kmalloc(inline_size) buffer. Make sure its header fits before + * ext4_check_dir_entry() dereferences de->rec_len. + */ + if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size) + goto out; de = (struct ext4_dir_entry_2 *) (dir_buf + ctx->pos - extra_offset); if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, - extra_size, ctx->pos)) + inline_size, ctx->pos)) goto out; if (le32_to_cpu(de->inode)) { if (!dir_emit(ctx, de->name, de->name_len, diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 3fc235eb65f2..bd4b778df9eb 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -176,7 +176,6 @@ void ext4_evict_inode(struct inode *inode) * (xattr block freeing), bitmap, group descriptor (inode freeing) */ int extra_credits = 6; - struct ext4_xattr_inode_array *ea_inode_array = NULL; bool freeze_protected = false; trace_ext4_evict_inode(inode); @@ -266,6 +265,7 @@ void ext4_evict_inode(struct inode *inode) if (ext4_inode_is_fast_symlink(inode)) memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data)); inode->i_size = 0; + ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); err = ext4_mark_inode_dirty(handle, inode); if (err) { ext4_warning(inode->i_sb, @@ -283,8 +283,7 @@ void ext4_evict_inode(struct inode *inode) } /* Remove xattr references. */ - err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array, - extra_credits); + err = ext4_xattr_delete_inode(handle, inode, extra_credits); if (err) { ext4_warning(inode->i_sb, "xattr delete (err %d)", err); stop_handle: @@ -292,7 +291,6 @@ stop_handle: ext4_orphan_del(NULL, inode); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); goto no_delete; } @@ -322,7 +320,6 @@ stop_handle: ext4_journal_stop(handle); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); return; no_delete: /* @@ -1184,6 +1181,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, int nr_wait = 0; int i; bool should_journal_data = ext4_should_journal_data(inode); + bool folio_uptodate = folio_test_uptodate(folio); BUG_ON(!folio_test_locked(folio)); BUG_ON(to > folio_size(folio)); @@ -1195,13 +1193,13 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, head = create_empty_buffers(folio, blocksize, 0); block = EXT4_PG_TO_LBLK(inode, folio->index); - for (bh = head, block_start = 0; bh != head || !block_start; + for (bh = head, block_start = 0; + block_start < to || (!folio_uptodate && bh != head); block++, block_start = block_end, bh = bh->b_this_page) { block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { - if (folio_test_uptodate(folio)) { + if (folio_uptodate) set_buffer_uptodate(bh); - } continue; } if (WARN_ON_ONCE(buffer_new(bh))) @@ -1222,7 +1220,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, if (should_journal_data) do_journal_get_write_access(handle, inode, bh); - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { /* * Unlike __block_write_begin() we leave * dirtying of new uptodate buffers to @@ -1239,7 +1237,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, continue; } } - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { set_buffer_uptodate(bh); continue; } @@ -1293,6 +1291,8 @@ static int ext4_write_begin(const struct kiocb *iocb, if (unlikely(ret)) return ret; + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + trace_ext4_write_begin(inode, pos, len); /* * Reserve one block more for addition to orphan list in case @@ -1307,8 +1307,10 @@ static int ext4_write_begin(const struct kiocb *iocb, foliop); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } /* @@ -1441,8 +1443,7 @@ static int ext4_write_end(const struct kiocb *iocb, trace_ext4_write_end(inode, pos, len, copied); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1551,8 +1552,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -2672,13 +2672,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * page is already under writeback and we are not doing * a data integrity writeback, skip the page */ - if (!folio_test_dirty(folio) || - (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || unlikely(folio->mapping != mapping)) { folio_unlock(folio); continue; } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } folio_wait_writeback(folio); BUG_ON(folio_test_writeback(folio)); @@ -2829,10 +2841,10 @@ static int ext4_do_writepages(struct mpage_da_data *mpd) if (ext4_should_dioread_nolock(inode)) { int bpf = ext4_journal_blocks_per_folio(inode); /* - * We may need to convert up to one extent per block in - * the folio and we may dirty the inode. + * We may need to convert up to one extent per block in the + * folio. */ - rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf); + rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); } if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) @@ -3135,11 +3147,13 @@ static int ext4_da_write_begin(const struct kiocb *iocb, if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { ret = ext4_generic_write_inline_data(mapping, inode, pos, len, - foliop, fsdata, true); + foliop, true); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } retry: @@ -3268,17 +3282,15 @@ static int ext4_da_write_end(const struct kiocb *iocb, struct folio *folio, void *fsdata) { struct inode *inode = mapping->host; - int write_mode = (int)(unsigned long)fsdata; + unsigned long write_mode = (unsigned long)fsdata; - if (write_mode == FALL_BACK_TO_NONDELALLOC) + if (write_mode & FALL_BACK_TO_NONDELALLOC) return ext4_write_end(iocb, mapping, pos, len, copied, folio, fsdata); trace_ext4_da_write_end(inode, pos, len, copied); - if (write_mode != CONVERT_INLINE_DATA && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && - ext4_has_inline_data(inode)) + if (write_mode & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -3652,6 +3664,9 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, int ret, dio_credits, m_flags = 0, retries = 0; bool force_commit = false; + if (flags & IOMAP_NOWAIT) + return -EAGAIN; + /* * Trim the mapping request to the maximum value that we can map at * once for direct I/O. @@ -3673,8 +3688,8 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, return ret; if (map->m_len < orig_mlen) { map->m_len = orig_mlen; - dio_credits = ext4_meta_trans_blocks(inode, orig_mlen, - map->m_len); + dio_credits = ext4_meta_trans_blocks(inode, map->m_len, + map->m_len, 0); } else { dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); @@ -4010,6 +4025,10 @@ void ext4_set_aops(struct inode *inode) * because it might have data in pagecache (eg, if called from ext4_zero_range, * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a * racing writeback can come later and flush the stale pagecache to disk. + * + * Return the loaded bh if it actually needs zeroing - in written, dirty + * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or + * a clean unwritten block). */ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) { @@ -4021,7 +4040,7 @@ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) int err = 0; folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + FGP_WRITEBEGIN | FGP_ACCESSED, mapping_gfp_constraint(mapping, ~__GFP_FS)); if (IS_ERR(folio)) return ERR_CAST(folio); @@ -4048,8 +4067,15 @@ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) } if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "unmapped"); - ext4_get_block(inode, iblock, bh, 0); - /* unmapped? It's a hole - nothing to do */ + err = ext4_get_block(inode, iblock, bh, 0); + if (err < 0) + goto unlock; + /* + * It's a hole or a clean unwritten block - nothing to do. + * Note that a lookup-only get_block (without + * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean + * unwritten extents. + */ if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "still unmapped"); goto unlock; @@ -4191,6 +4217,14 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) offset = from & (blocksize - 1); if (!offset || from >= end) return 0; + /* + * Inline data has no tail block to zero out. Note that a race with + * ext4_page_mkwrite() converting inline data to an extent without + * holding i_rwsem is safe, as that path zeroes the full block before + * copying in the inline data. + */ + if (ext4_has_inline_data(inode)) + return 0; /* If we are processing an encrypted inode during orphan list handling */ if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) return 0; @@ -4225,13 +4259,26 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) return 0; } +/* + * Zero out the unaligned head and tail of the [lstart, lstart+length) + * range. + * + * On return, @partial_zeroed records which edges actually got + * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if + * the head/tail block got actually partially zeroed (in written, dirty + * unwritten or delalloc state). Cleared if the head/tail block is a + * hole or a clean unwritten block, in which case there is nothing that + * needs zeroing. When the head and tail land in the same block, both + * bits are set together on a successful zeroing. + */ int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, - bool *did_zero) + unsigned int *partial_zeroed) { struct super_block *sb = inode->i_sb; unsigned partial_start, partial_end; ext4_fsblk_t start, end; loff_t byte_end = (lstart + length - 1); + bool did_zero = false; int err = 0; partial_start = lstart & (sb->s_blocksize - 1); @@ -4243,21 +4290,32 @@ int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, /* Handle partial zero within the single block */ if (start == end && (partial_start || (partial_end != sb->s_blocksize - 1))) { - err = ext4_block_zero_range(inode, lstart, length, did_zero, + err = ext4_block_zero_range(inode, lstart, length, &did_zero, NULL); + if (did_zero) + *partial_zeroed |= (EXT4_PARTIAL_ZERO_START | + EXT4_PARTIAL_ZERO_END); return err; } /* Handle partial zero out on the start of the range */ if (partial_start) { err = ext4_block_zero_range(inode, lstart, sb->s_blocksize, - did_zero, NULL); + &did_zero, NULL); if (err) return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_START; } /* Handle partial zero out on the end of the range */ - if (partial_end != sb->s_blocksize - 1) + if (partial_end != sb->s_blocksize - 1) { + did_zero = false; err = ext4_block_zero_range(inode, byte_end - partial_end, - partial_end + 1, did_zero, NULL); + partial_end + 1, &did_zero, NULL); + if (err) + return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_END; + } return err; } @@ -4406,7 +4464,7 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) loff_t end = offset + length; handle_t *handle; unsigned int credits; - bool partial_zeroed = false; + unsigned int partial_zeroed = 0; int ret; trace_ext4_punch_hole(inode, offset, length, 0); @@ -5242,6 +5300,20 @@ void ext4_set_inode_mapping_order(struct inode *inode) mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); } +static int ext4_iget_match(struct inode *inode, u64 ino, void *data) +{ + if (inode->i_ino != ino) + return 0; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { + spin_unlock(&inode->i_lock); + return -1; + } + __iget(inode); + spin_unlock(&inode->i_lock); + return 1; +} + struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ext4_iget_flags flags, const char *function, unsigned int line) @@ -5270,9 +5342,24 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, return ERR_PTR(-EFSCORRUPTED); } - inode = iget_locked(sb, ino); - if (!inode) - return ERR_PTR(-ENOMEM); + if (flags & EXT4_IGET_NOWAIT) { + inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); + if (!inode) + return ERR_PTR(-ENOENT); + + if (inode_state_read_once(inode) & I_NEW) + wait_on_new_inode(inode); + + if (unlikely(inode_unhashed(inode))) { + iput(inode); + return ERR_PTR(-ENOENT); + } + } else { + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + } + if (!(inode_state_read_once(inode) & I_NEW)) { ret = check_igot_inode(inode, flags, function, line); if (ret) { @@ -6288,17 +6375,17 @@ static int ext4_index_trans_blocks(struct inode *inode, int lblocks, } /* - * Account for index blocks, block groups bitmaps and block group - * descriptor blocks if modify datablocks and index blocks - * worse case, the indexs blocks spread over different block groups - * - * If datablocks are discontiguous, they are possible to spread over - * different block groups too. If they are contiguous, with flexbg, - * they could still across block group boundary. - * - * Also account for superblock, inode, quota and xattr blocks + * Calculate number of credits needed in a transaction to: + * * Allocate data blocks from @alloc_extents different groups - note that + * with flexbg a single physical extent can span multiple groups but + * single mballoc request only returns extent within one group. + * * Allocate metatadata (extent tree blocks, indirect blocks) to store + * pointers to @pextents data extents having @lblocks in total. + * * Modify extent tree / indirect block tree, inode, superblock, quota + * tracking, xattr blocks */ -int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) +int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, + int alloc_extents) { ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); int gdpblocks; @@ -6315,7 +6402,7 @@ int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) * Now let's see how many group bitmaps and group descriptors need * to account */ - groups = idxblocks + pextents; + groups = idxblocks + alloc_extents; gdpblocks = groups; if (groups > ngroups) groups = ngroups; @@ -6341,7 +6428,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) { int ret; - ret = ext4_meta_trans_blocks(inode, nrblocks, 1); + ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); /* Account for data blocks for journalled mode */ if (ext4_should_journal_data(inode)) ret += nrblocks; @@ -6359,7 +6446,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) */ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) { - return ext4_meta_trans_blocks(inode, nrblocks, 1); + return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); } /* @@ -6506,6 +6593,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode, return -EOVERFLOW; /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + + /* * In nojournal mode, we can immediately attempt to expand * the inode. When journaled, we first need to obtain extra * buffer credits since we may write into the EA block diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index ed1bd00e11cd..06171a11db12 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2861,8 +2861,6 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, blk_start_plug(&plug); while (nr-- > 0) { - struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, - NULL); struct ext4_group_info *grp = ext4_get_group_info(sb, group); /* @@ -2872,14 +2870,17 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, * prefetch once, so we avoid getblk() call, which can * be expensive. */ - if (gdp && grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && - EXT4_MB_GRP_NEED_INIT(grp) && - ext4_free_group_clusters(sb, gdp) > 0 ) { - bh = ext4_read_block_bitmap_nowait(sb, group, true); - if (!IS_ERR_OR_NULL(bh)) { - if (!buffer_uptodate(bh) && cnt) - (*cnt)++; - brelse(bh); + if (grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && + EXT4_MB_GRP_NEED_INIT(grp)) { + struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, NULL); + + if (gdp && ext4_free_group_clusters(sb, gdp) > 0) { + bh = ext4_read_block_bitmap_nowait(sb, group, true); + if (!IS_ERR_OR_NULL(bh)) { + if (!buffer_uptodate(bh) && cnt) + (*cnt)++; + brelse(bh); + } } } if (++group >= ngroups) diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 477d43d7e294..5d60ef10fe11 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -464,6 +464,7 @@ int ext4_ext_migrate(struct inode *inode) if (IS_ERR(tmp_inode)) { retval = PTR_ERR(tmp_inode); ext4_journal_stop(handle); + tmp_inode = NULL; goto out_unlock; } /* @@ -591,9 +592,9 @@ out_stop: ext4_journal_stop(handle); out_tmp_inode: unlock_new_inode(tmp_inode); - iput(tmp_inode); out_unlock: ext4_writepages_up_write(inode->i_sb, alloc_ctx); + iput(tmp_inode); return retval; } diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 640a03ee02c7..a6386c1d237f 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -1467,6 +1467,8 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, /* this code is executed quadratically often */ /* do minimal checking `by hand' */ if (de->name + de->name_len <= dlimit && + (!ext4_hash_in_dirent(dir) || + (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) && ext4_match(dir, fname, de)) { /* found a match - just to be sure, do * a full check */ diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index 64ea47624233..b4675aa7ea96 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -4,6 +4,7 @@ #include <linux/fs.h> #include <linux/quotaops.h> #include <linux/buffer_head.h> +#include <linux/string_choices.h> #include "ext4.h" #include "ext4_jbd2.h" @@ -388,7 +389,7 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); - if (!es->s_last_orphan && !oi->of_blocks) { + if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) { ext4_debug("no orphan inodes to clean up\n"); return; } @@ -486,14 +487,12 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) } } -#define PLURAL(x) (x), ((x) == 1) ? "" : "s" - if (nr_orphans) ext4_msg(sb, KERN_INFO, "%d orphan inode%s deleted", - PLURAL(nr_orphans)); + nr_orphans, str_plural(nr_orphans)); if (nr_truncates) ext4_msg(sb, KERN_INFO, "%d truncate%s cleaned up", - PLURAL(nr_truncates)); + nr_truncates, str_plural(nr_truncates)); #ifdef CONFIG_QUOTA /* Turn off quotas if they were enabled for orphan cleanup */ if (quota_update) { @@ -572,6 +571,7 @@ int ext4_init_orphan_info(struct super_block *sb) int i, j; int ret; int free; + int loaded = 0; __le32 *bdata; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); struct ext4_orphan_block_tail *ot; @@ -613,6 +613,7 @@ int ext4_init_orphan_info(struct super_block *sb) ret = -EIO; goto out_free; } + loaded++; ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh); if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) { ext4_error(sb, "orphan file block %d: bad magic", i); @@ -635,8 +636,10 @@ int ext4_init_orphan_info(struct super_block *sb) iput(inode); return 0; out_free: - for (i--; i >= 0; i--) - brelse(oi->of_binfo[i].ob_bh); + while (loaded > 0) { + loaded--; + brelse(oi->of_binfo[loaded].ob_bh); + } kvfree(oi->of_binfo); out_put: iput(inode); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 4b6112e5d6c5..bca0dc87d0b7 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1303,6 +1303,8 @@ static void ext4_put_super(struct super_block *sb) &sb->s_uuid); ext4_unregister_li_request(sb); + /* Drain deferred EA inode iputs while quota is still active. */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_quotas_off(sb, EXT4_MAXQUOTAS); destroy_workqueue(sbi->rsv_conversion_wq); @@ -1423,6 +1425,13 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) memset(&ei->i_dquot, 0, sizeof(ei->i_dquot)); #endif ei->jinode = NULL; + /* + * Reinitialize xattr_sem every allocation because EA inodes + * share this space with i_ea_iput_node (via union) which may + * have overwritten the semaphore when the slab object was + * previously used as an EA inode. + */ + init_rwsem(&ei->xattr_sem); INIT_LIST_HEAD(&ei->i_rsv_conversion_list); spin_lock_init(&ei->i_completed_io_lock); ei->i_sync_tid = 0; @@ -1489,7 +1498,6 @@ static void init_once(void *foo) struct ext4_inode_info *ei = foo; INIT_LIST_HEAD(&ei->i_orphan); - init_rwsem(&ei->xattr_sem); init_rwsem(&ei->i_data_sem); inode_init_once(&ei->vfs_inode); ext4_fc_init_inode(&ei->vfs_inode); @@ -4483,8 +4491,9 @@ static int ext4_handle_clustersize(struct super_block *sb) sbi->s_cluster_bits = 0; } sbi->s_clusters_per_group = le32_to_cpu(es->s_clusters_per_group); - if (sbi->s_clusters_per_group > sb->s_blocksize * 8) { - ext4_msg(sb, KERN_ERR, "#clusters per group too big: %lu", + if (sbi->s_clusters_per_group > sb->s_blocksize * 8 || + sbi->s_clusters_per_group & 7) { + ext4_msg(sb, KERN_ERR, "invalid #clusters per group: %lu", sbi->s_clusters_per_group); return -EINVAL; } @@ -5316,8 +5325,10 @@ static int ext4_block_group_meta_init(struct super_block *sb, int silent) return -EINVAL; } if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || - sbi->s_inodes_per_group > sb->s_blocksize * 8) { - ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n", + sbi->s_inodes_per_group > sb->s_blocksize * 8 || + sbi->s_inodes_per_group & 7 || + sbi->s_inodes_per_group % sbi->s_inodes_per_block) { + ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu", sbi->s_inodes_per_group); return -EINVAL; } @@ -5377,7 +5388,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_set_def_opts(sb, es); sbi->s_resuid = make_kuid(&init_user_ns, ext4_get_resuid(es)); - sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resuid(es)); + sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resgid(es)); sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; @@ -5505,6 +5516,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_has_feature_orphan_present(sb) || ext4_has_feature_journal_needs_recovery(sb)); + ext4_init_ea_inode_work(sbi); + if (ext4_has_feature_mmp(sb) && !sb_rdonly(sb)) { err = ext4_multi_mount_protect(sb, le64_to_cpu(es->s_mmp_block)); if (err) @@ -5755,6 +5768,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) return 0; failed_mount9: + /* Drain deferred EA inode iputs before quota shutdown */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_quotas_off(sb, EXT4_MAXQUOTAS); failed_mount8: __maybe_unused ext4_release_orphan_info(sb); @@ -5775,6 +5790,8 @@ failed_mount4: if (EXT4_SB(sb)->rsv_conversion_wq) destroy_workqueue(EXT4_SB(sb)->rsv_conversion_wq); failed_mount_wq: + /* Drain deferred EA inode iputs before freeing structures */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_xattr_destroy_cache(sbi->s_ea_inode_cache); sbi->s_ea_inode_cache = NULL; @@ -5785,6 +5802,8 @@ failed_mount_wq: ext4_journal_destroy(sbi, sbi->s_journal); } failed_mount3a: + /* Drain deferred EA inode iputs from journal replay */ + flush_delayed_work(&sbi->s_ea_inode_work); ext4_es_unregister_shrinker(sbi); failed_mount3: /* flush s_sb_upd_work before sbi destroy */ @@ -6455,6 +6474,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait) trace_ext4_sync_fs(sb, wait); flush_workqueue(sbi->rsv_conversion_wq); + flush_delayed_work(&sbi->s_ea_inode_work); /* * Writeback quota in non-journalled quota case - journalled quota has * no dirty dquots diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 982a1f831e22..5c310747b965 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -114,10 +114,6 @@ const struct xattr_handler * const ext4_xattr_handlers[] = { #define EA_INODE_CACHE(inode) (((struct ext4_sb_info *) \ inode->i_sb->s_fs_info)->s_ea_inode_cache) -static int -ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, - struct inode *inode); - #ifdef CONFIG_LOCKDEP void ext4_xattr_inode_set_class(struct inode *ea_inode) { @@ -464,6 +460,21 @@ static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino, inode_unlock(inode); } + /* + * Since this function resolves references from active xattr entries, + * the EA inode must be in active state (i_nlink=1, ref_count>0). + * i_nlink > 1, i_nlink == 0 (dangling reference), or ref_count == 0 + * (inconsistent with an active entry) all indicate on-disk corruption. + */ + if (inode->i_nlink != 1 || !ext4_xattr_inode_get_ref(inode)) { + ext4_error(parent->i_sb, + "EA inode %lu has unexpected i_nlink=%u ref_count=%llu", + ea_ino, inode->i_nlink, + ext4_xattr_inode_get_ref(inode)); + ext4_put_ea_inode(inode); + return -EFSCORRUPTED; + } + *ea_inode = inode; return 0; } @@ -567,7 +578,7 @@ ext4_xattr_inode_get(struct inode *inode, struct ext4_xattr_entry *entry, ea_inode->i_ino, true /* reusable */); } out: - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return err; } @@ -1104,10 +1115,10 @@ static int ext4_xattr_inode_inc_ref_all(handle_t *handle, struct inode *parent, err = ext4_xattr_inode_inc_ref(handle, ea_inode); if (err) { ext4_warning_inode(ea_inode, "inc ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); goto cleanup; } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return 0; @@ -1133,7 +1144,7 @@ cleanup: if (err) ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return saved_err; } @@ -1160,7 +1171,6 @@ static void ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, struct buffer_head *bh, struct ext4_xattr_entry *first, bool block_csum, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits, bool skip_quota) { struct inode *ea_inode; @@ -1197,14 +1207,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err) continue; - err = ext4_expand_inode_array(ea_inode_array, ea_inode); - if (err) { - ext4_warning_inode(ea_inode, - "Expand inode array err=%d", err); - iput(ea_inode); - continue; - } - err = ext4_journal_ensure_credits_fn(handle, credits, credits, ext4_free_metadata_revoke_credits(parent->i_sb, 1), ext4_xattr_restart_fn(handle, parent, bh, block_csum, @@ -1212,6 +1214,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err < 0) { ext4_warning_inode(ea_inode, "Ensure credits err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } if (err > 0) { @@ -1221,6 +1224,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, ext4_warning_inode(ea_inode, "Re-get write access err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } } @@ -1229,6 +1233,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (err) { ext4_warning_inode(ea_inode, "ea_inode dec ref err=%d", err); + ext4_put_ea_inode(ea_inode); continue; } @@ -1245,6 +1250,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, entry->e_value_inum = 0; entry->e_value_size = 0; + ext4_put_ea_inode(ea_inode); dirty = true; } @@ -1271,7 +1277,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, static void ext4_xattr_release_block(handle_t *handle, struct inode *inode, struct buffer_head *bh, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits) { struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); @@ -1313,7 +1318,6 @@ retry_ref: ext4_xattr_inode_dec_ref_all(handle, inode, bh, BFIRST(bh), true /* block_csum */, - ea_inode_array, extra_credits, true /* skip_quota */); ext4_free_blocks(handle, inode, bh, 0, 1, @@ -1505,7 +1509,7 @@ static struct inode *ext4_xattr_inode_create(handle_t *handle, if (ext4_xattr_inode_dec_ref(handle, ea_inode)) ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return ERR_PTR(err); } @@ -1550,7 +1554,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, while (ce) { ea_inode = ext4_iget(inode->i_sb, ce->e_value, - EXT4_IGET_EA_INODE); + EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT); if (IS_ERR(ea_inode)) goto next_entry; ext4_xattr_inode_set_class(ea_inode); @@ -1564,7 +1568,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, kvfree(ea_data); return ea_inode; } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); next_entry: ce = mb_cache_entry_find_next(ea_inode_cache, ce); } @@ -1615,7 +1619,7 @@ static struct inode *ext4_xattr_inode_lookup_create(handle_t *handle, ea_inode->i_ino, true /* reusable */); return ea_inode; out_err: - iput(ea_inode); + ext4_put_ea_inode(ea_inode); ext4_xattr_inode_free_quota(inode, NULL, value_len); return ERR_PTR(err); } @@ -1848,7 +1852,7 @@ update_hash: ret = 0; out: - iput(old_ea_inode); + ext4_put_ea_inode(old_ea_inode); return ret; } @@ -2010,7 +2014,7 @@ clone_block: old_ea_inode_quota = le32_to_cpu( s->here->e_value_size); } - iput(tmp_inode); + ext4_put_ea_inode(tmp_inode); s->here->e_value_inum = 0; s->here->e_value_size = 0; @@ -2075,12 +2079,13 @@ inserted: * stable so we can check the additional * reference fits. */ - ref = le32_to_cpu(BHDR(new_bh)->h_refcount) + 1; - if (ref > EXT4_XATTR_REFCOUNT_MAX) { + ref = le32_to_cpu(BHDR(new_bh)->h_refcount); + if (ref >= EXT4_XATTR_REFCOUNT_MAX) { /* * Undo everything and check mbcache * again. */ + clear_bit(MBE_REUSABLE_B, &ce->e_flags); unlock_buffer(new_bh); dquot_free_block(inode, EXT4_C2B(EXT4_SB(sb), @@ -2091,6 +2096,7 @@ inserted: new_bh = NULL; goto inserted; } + ref++; BHDR(new_bh)->h_refcount = cpu_to_le32(ref); if (ref == EXT4_XATTR_REFCOUNT_MAX) clear_bit(MBE_REUSABLE_B, &ce->e_flags); @@ -2150,7 +2156,7 @@ getblk_failed: ext4_warning_inode(ea_inode, "dec ref error=%d", error); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); ea_inode = NULL; } @@ -2182,12 +2188,8 @@ getblk_failed: /* Drop the previous xattr block. */ if (bs->bh && bs->bh != new_bh) { - struct ext4_xattr_inode_array *ea_inode_array = NULL; - ext4_xattr_release_block(handle, inode, bs->bh, - &ea_inode_array, 0 /* extra_credits */); - ext4_xattr_inode_array_free(ea_inode_array); } error = 0; @@ -2203,7 +2205,7 @@ cleanup: ext4_xattr_inode_free_quota(inode, ea_inode, i_size_read(ea_inode)); } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } if (ce) mb_cache_entry_put(ea_block_cache, ce); @@ -2285,7 +2287,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, ext4_xattr_inode_free_quota(inode, ea_inode, i_size_read(ea_inode)); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } return error; } @@ -2297,7 +2299,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, header->h_magic = cpu_to_le32(0); ext4_clear_inode_state(inode, EXT4_STATE_XATTR); } - iput(ea_inode); + ext4_put_ea_inode(ea_inode); return 0; } @@ -2839,6 +2841,7 @@ retry: s_min_extra_isize) { tried_min_extra_isize++; new_extra_isize = s_min_extra_isize; + error = 0; goto retry; } goto cleanup; @@ -2863,46 +2866,6 @@ cleanup: return error; } -#define EIA_INCR 16 /* must be 2^n */ -#define EIA_MASK (EIA_INCR - 1) - -/* Add the large xattr @inode into @ea_inode_array for deferred iput(). - * If @ea_inode_array is new or full it will be grown and the old - * contents copied over. - */ -static int -ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, - struct inode *inode) -{ - if (*ea_inode_array == NULL) { - /* - * Start with 15 inodes, so it fits into a power-of-two size. - */ - (*ea_inode_array) = kmalloc_flex(**ea_inode_array, inodes, - EIA_MASK, GFP_NOFS); - if (*ea_inode_array == NULL) - return -ENOMEM; - (*ea_inode_array)->count = 0; - } else if (((*ea_inode_array)->count & EIA_MASK) == EIA_MASK) { - /* expand the array once all 15 + n * 16 slots are full */ - struct ext4_xattr_inode_array *new_array = NULL; - - new_array = kmalloc_flex(**ea_inode_array, inodes, - (*ea_inode_array)->count + EIA_INCR, - GFP_NOFS); - if (new_array == NULL) - return -ENOMEM; - memcpy(new_array, *ea_inode_array, - struct_size(*ea_inode_array, inodes, - (*ea_inode_array)->count)); - kfree(*ea_inode_array); - *ea_inode_array = new_array; - } - (*ea_inode_array)->count++; - (*ea_inode_array)->inodes[(*ea_inode_array)->count - 1] = inode; - return 0; -} - /* * ext4_xattr_delete_inode() * @@ -2913,7 +2876,6 @@ ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array, * references on xattr block and xattr inodes. */ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, - struct ext4_xattr_inode_array **ea_inode_array, int extra_credits) { struct buffer_head *bh = NULL; @@ -2952,7 +2914,6 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, ext4_xattr_inode_dec_ref_all(handle, inode, iloc.bh, IFIRST(header), false /* block_csum */, - ea_inode_array, extra_credits, false /* skip_quota */); } @@ -2986,12 +2947,12 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, continue; ext4_xattr_inode_free_quota(inode, ea_inode, le32_to_cpu(entry->e_value_size)); - iput(ea_inode); + ext4_put_ea_inode(ea_inode); } } - ext4_xattr_release_block(handle, inode, bh, ea_inode_array, + ext4_xattr_release_block(handle, inode, bh, extra_credits); /* * Update i_file_acl value in the same transaction that releases @@ -3013,16 +2974,63 @@ cleanup: return error; } -void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *ea_inode_array) +/* + * Worker function for deferred EA inode iput. Processes all inodes queued + * on s_ea_inode_to_free in a context free of xattr_sem/jbd2 handle locks. + */ +static void ext4_ea_inode_work(struct work_struct *work) { - int idx; + struct ext4_sb_info *sbi = container_of(to_delayed_work(work), + struct ext4_sb_info, + s_ea_inode_work); + struct llist_node *node = llist_del_all(&sbi->s_ea_inode_to_free); - if (ea_inode_array == NULL) + while (node) { + struct ext4_inode_info *ei = container_of(node, + struct ext4_inode_info, i_ea_iput_node); + node = node->next; + iput(&ei->vfs_inode); + } +} + +/* + * Release a VFS reference on an EA inode. Must be used instead of iput() + * in any context where xattr_sem or a jbd2 handle is held. + * + * If this is not the last reference, drops it immediately via + * iput_if_not_last() with no further action needed. + * + * If this is the last reference, the inode is linked onto a per-sb + * llist via i_ea_iput_node (embedded in ext4_inode_info, sharing space + * with the unused xattr_sem) and a delayed worker performs the final + * iput() in a clean context. + * + * Note: while an inode is on s_ea_inode_to_free, the unconsumed i_count + * reference (still 1) keeps it in the inode cache, so any concurrent + * iget() bumps i_count to >= 2 and iput_if_not_last() will succeed. + * Nobody will add the inode a second time until ext4_ea_inode_work() + * drops that reference via iput(). + */ +void ext4_put_ea_inode(struct inode *inode) +{ + if (!inode) return; + WARN_ON_ONCE(!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)); + if (iput_if_not_last(inode)) + return; + llist_add(&EXT4_I(inode)->i_ea_iput_node, + &EXT4_SB(inode->i_sb)->s_ea_inode_to_free); + /* + * Use a short delay to allow multiple EA inodes to accumulate, + * reducing workqueue wakeups when several are released together. + */ + schedule_delayed_work(&EXT4_SB(inode->i_sb)->s_ea_inode_work, 1); +} - for (idx = 0; idx < ea_inode_array->count; ++idx) - iput(ea_inode_array->inodes[idx]); - kfree(ea_inode_array); +void ext4_init_ea_inode_work(struct ext4_sb_info *sbi) +{ + init_llist_head(&sbi->s_ea_inode_to_free); + INIT_DELAYED_WORK(&sbi->s_ea_inode_work, ext4_ea_inode_work); } /* diff --git a/fs/ext4/xattr.h b/fs/ext4/xattr.h index 1fedf44d4fb6..821dc6a50e51 100644 --- a/fs/ext4/xattr.h +++ b/fs/ext4/xattr.h @@ -131,11 +131,6 @@ struct ext4_xattr_ibody_find { struct ext4_iloc iloc; }; -struct ext4_xattr_inode_array { - unsigned int count; - struct inode *inodes[] __counted_by(count); -}; - extern const struct xattr_handler ext4_xattr_user_handler; extern const struct xattr_handler ext4_xattr_trusted_handler; extern const struct xattr_handler ext4_xattr_security_handler; @@ -187,9 +182,9 @@ extern int __ext4_xattr_set_credits(struct super_block *sb, struct inode *inode, bool is_create); extern int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, - struct ext4_xattr_inode_array **array, int extra_credits); -extern void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *array); +extern void ext4_init_ea_inode_work(struct ext4_sb_info *sbi); +extern void ext4_put_ea_inode(struct inode *inode); extern int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize, struct ext4_inode *raw_inode, handle_t *handle); diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 1508e2f54462..513273712010 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -358,15 +358,16 @@ int jbd2_cleanup_journal_tail(journal_t *journal) /* * journal_shrink_one_cp_list * - * Find all the written-back checkpoint buffers in the given list - * and try to release them. If the whole transaction is released, set - * the 'released' parameter. Return the number of released checkpointed - * buffers. + * Find written-back checkpoint buffers in the given list and try to release + * them. If 'nr_to_scan' is set, scan at most that many buffers. If the whole + * transaction is released, set the 'released' parameter. Return the number of + * released checkpointed buffers. * * Called with j_list_lock held. */ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, enum jbd2_shrink_type type, + unsigned long *nr_to_scan, bool *released) { struct journal_head *last_jh; @@ -375,13 +376,15 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, int ret; *released = false; - if (!jh) + if (!jh || (nr_to_scan && !*nr_to_scan)) return 0; last_jh = jh->b_cpprev; do { jh = next_jh; next_jh = jh->b_cpnext; + if (nr_to_scan) + (*nr_to_scan)--; if (type == JBD2_SHRINK_DESTROY) { ret = __jbd2_journal_remove_checkpoint(jh); @@ -389,7 +392,7 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, ret = jbd2_journal_try_remove_checkpoint(jh); if (ret < 0) { if (type == JBD2_SHRINK_BUSY_SKIP) - continue; + goto next; break; } } @@ -400,9 +403,10 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, break; } +next: if (need_resched()) break; - } while (jh != last_jh); + } while (jh != last_jh && (!nr_to_scan || *nr_to_scan)); return nr_freed; } @@ -424,7 +428,6 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, tid_t first_tid = 0, last_tid = 0, next_tid = 0; tid_t tid = 0; unsigned long nr_freed = 0; - unsigned long freed; bool first_set = false; again: @@ -457,10 +460,9 @@ again: next_transaction = transaction->t_cpnext; tid = transaction->t_tid; - freed = journal_shrink_one_cp_list(transaction->t_checkpoint_list, - JBD2_SHRINK_BUSY_SKIP, &released); - nr_freed += freed; - (*nr_to_scan) -= min(*nr_to_scan, freed); + nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_list, + JBD2_SHRINK_BUSY_SKIP, + nr_to_scan, &released); if (*nr_to_scan == 0) break; if (need_resched() || spin_needbreak(&journal->j_list_lock)) @@ -516,7 +518,7 @@ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, transaction = next_transaction; next_transaction = transaction->t_cpnext; journal_shrink_one_cp_list(transaction->t_checkpoint_list, - type, &released); + type, NULL, &released); /* * This function only frees up some memory if possible so we * dont have an obligation to finish processing. Bail out if diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 09efa337649e..00f5a98f3d4f 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -94,6 +94,7 @@ EXPORT_SYMBOL(jbd2_journal_init_jbd_inode); EXPORT_SYMBOL(jbd2_journal_release_jbd_inode); EXPORT_SYMBOL(jbd2_journal_begin_ordered_truncate); EXPORT_SYMBOL(jbd2_inode_cache); +EXPORT_SYMBOL(jbd2_handle_cache); #ifdef CONFIG_JBD2_DEBUG void __jbd2_debug(int level, const char *file, const char *func, diff --git a/include/linux/fs.h b/include/linux/fs.h index 8e9bc9dda0cb..072d8cd09a0b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2419,6 +2419,21 @@ static inline void super_set_sysfs_name_generic(struct super_block *sb, const ch extern void ihold(struct inode * inode); extern void iput(struct inode *); void iput_not_last(struct inode *); + +/** + * iput_if_not_last - drop an inode reference only if it is not the last one + * @inode: inode to put + * + * Returns true if the reference was dropped, false if this was the last + * reference and the caller must arrange for final iput() in a safe context. + */ +static inline bool __must_check iput_if_not_last(struct inode *inode) +{ + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); + VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); + return atomic_add_unless(&inode->i_count, -1, 1); +} + int inode_update_time(struct inode *inode, enum fs_update_time type, unsigned int flags); int generic_update_time(struct inode *inode, enum fs_update_time type, diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index b68561187e90..1b42fe47c26b 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -510,11 +510,12 @@ struct jbd2_journal_handle int h_err; /* Flags [no locking] */ - unsigned int h_sync: 1; - unsigned int h_reserved: 1; - unsigned int h_aborted: 1; - unsigned int h_type: 8; - unsigned int h_line_no: 16; + unsigned char h_sync: 1; + unsigned char h_reserved: 1; + unsigned char h_aborted: 1; + unsigned char h_invalid: 1; + unsigned char h_type; + unsigned short h_line_no; unsigned long h_start_jiffies; unsigned int h_requested_credits; |
