summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-08-20 13:17:11 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-08-20 13:17:11 -0700
commitc84d3e3130dfe1058cb27dc78e7ad8bd36f0545a (patch)
tree0adeaef5227fcaf5b95e8831b5f33af00cec50ae
parent50c44fea13ec339d0d457079b254e8c8420d6511 (diff)
parent9091c97be34083587a75db174aab51551d8e8543 (diff)
Merge tag 'ext4_for_linus-7.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4
Pull ext4 updates from Ted Ts'o: - Improve performance by allowing parallel DIO writes when we were previously being overly conservative when checking whether it was safe to avoid requiring an exclusive lock - Improve the performance of ext4_mb_prefetch() used by fallocate() by avoiding work when it is not needed - Remove the unnecessary custom end_io function ext4_end_buffer_io_sync() - Improve performance when performing an overwrite to an already uptodate folio - Clean up how we handle deallocating EA inodes to avoid a potential lock ordering issue when there is a failed mount while an EA inode is still being evicted - Use str_plural() instead of a custom macro - Avoid soft lockups or RCU stalls if there are many busy buffers (caused by heavy I/O) while checkpointing - Use scoped NOFS when starting a handle in nojournal mode - Align fields in handle structure to optimize setting and getting the h_type and h_line_no fields - Fix documentation of the meta_bg block group layout - Bug fixes: - Fix a potential out-of-bounds read in ext4_read_inline_dir() - Fix a potential deadlock when concurrent xattr operations are racing with each other when some of the xattrs are using the ea_inode feature - Fix a spurious warning with data=journal that can be triggered when writeback races with remounting the file system read-only - Fix a potential deadlock when EXT4_IOC_MIGRATE races with a file system freeze operation - Make sure all in-flight direct I/O operations are complete before falling back to buffered I/O - Handle IOCB_NOWAIT properly when performing a extending DAX write - Prevent potentially sleeping on a block allocation when IOCB_NOWAIT is set - Fix potential races when racing an inline data write with a page fault - Propagate errors when adding or removing extent ranges during a fast commit replay - Avoid trying to expand an inode's extra size when it is being evicted to avoid a number of corner case or deadlocks - Avoid spurious error when retrying inode extra size expansion - Fix corner cases where we underestimate the number of journal credits needed - Avoid hangs/crashes/WARNINGS caused by maliciously corrupted file systems - Don't issue spurious orphan clean message on RO file systems - Avoid leaving the file system in an inconsistent state after a crash when a WRITE_ZEROS in progress converting an unwritten extent to a written extent - Handle WRITE_ZEROS correctly when there are some partially dirtied regions in the page cache - Pass errors during zero-rage, truncate, or punch hole to the caller if ext4_get_block() fails - Wait for writeback to finish when triggered by zero-range or zero-range for those devices that require stable writes - If the reserved gid superblock field is set, set the reserved gid instead of the reserved uid * tag 'ext4_for_linus-7.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4: (56 commits) ext4: fix estimate extent index blocks in ext4_ext_index_trans_blocks() ext4: fix transaction overflow during writeback ext4: teach ext4_meta_trans_blocks() about number of allocated extents ext4: guard against NULL s_group_info in ext4_get_group_info ext4: fix spurious message about orphan cleanup on RO fs ext4: stop retrying saturated xattr cache entries ext4: don't enable DAX on new encrypted files ext4: protect WRITE_ZEROES written extents with orphan list ext4: export converted block count from ext4_convert_unwritten_extents() ext4: fix incorrect function call when initializing s_resgid ext4: validate EA inode i_nlink in ext4_xattr_inode_iget jbd2: align h_type and h_line_no in the handle structure on byte boundaries ext4: enable scoped NOFS when starting a handle in nojournal mode ext4: write back partial-zeroed edges in WRITE_ZEROES ext4: zero out whole block for clean edges in WRITE_ZEROES ext4: track partial-zero outcome per edge in ext4_zero_partial_blocks() ext4: clarify return semantics of ext4_load_tail_bh() ext4: move partial block zeroing earlier in ext4_zero_range() ext4: check return value of ext4_get_block() in ext4_load_tail_bh() ext4: skip tail block zeroing for inline data files ...
-rw-r--r--Documentation/filesystems/ext4/group_descr.rst9
-rw-r--r--fs/buffer.c3
-rw-r--r--fs/ext4/balloc.c4
-rw-r--r--fs/ext4/crypto.c40
-rw-r--r--fs/ext4/dir.c20
-rw-r--r--fs/ext4/ext4.h40
-rw-r--r--fs/ext4/ext4_jbd2.c36
-rw-r--r--fs/ext4/ext4_jbd2.h6
-rw-r--r--fs/ext4/extents.c182
-rw-r--r--fs/ext4/fast_commit.c37
-rw-r--r--fs/ext4/file.c151
-rw-r--r--fs/ext4/ialloc.c4
-rw-r--r--fs/ext4/inline.c38
-rw-r--r--fs/ext4/inode.c207
-rw-r--r--fs/ext4/mballoc.c21
-rw-r--r--fs/ext4/migrate.c3
-rw-r--r--fs/ext4/namei.c2
-rw-r--r--fs/ext4/orphan.c17
-rw-r--r--fs/ext4/super.c32
-rw-r--r--fs/ext4/xattr.c178
-rw-r--r--fs/ext4/xattr.h9
-rw-r--r--fs/jbd2/checkpoint.c28
-rw-r--r--fs/jbd2/journal.c1
-rw-r--r--include/linux/fs.h15
-rw-r--r--include/linux/jbd2.h11
25 files changed, 734 insertions, 360 deletions
diff --git a/Documentation/filesystems/ext4/group_descr.rst b/Documentation/filesystems/ext4/group_descr.rst
index 392ec44f8fb0..9a0c2d92c2ef 100644
--- a/Documentation/filesystems/ext4/group_descr.rst
+++ b/Documentation/filesystems/ext4/group_descr.rst
@@ -20,11 +20,10 @@ group of the flex group.
If the meta_bg feature flag is set, then several block groups are
grouped together into a meta group. Note that in the meta_bg case,
-however, the first and last two block groups within the larger meta
-group contain only group descriptors for the groups inside the meta
-group.
-
-flex_bg and meta_bg do not appear to be mutually exclusive features.
+however, the superblock and a single block group descriptor block is
+placed at the beginning of the first, second, and last block groups in a
+meta-block group. The flex_bg and meta_bg features are not mutually
+exclusive.
In ext2, ext3, and ext4 (when the 64bit feature is not enabled), the
block group descriptor was only 32 bytes long and therefore ends at
diff --git a/fs/buffer.c b/fs/buffer.c
index 65a2893b2cbb..dd50d17b8907 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -2081,6 +2081,7 @@ void block_commit_write(struct folio *folio, size_t from, size_t to)
{
size_t block_start, block_end;
bool partial = false;
+ bool uptodate = folio_test_uptodate(folio);
unsigned blocksize;
struct buffer_head *bh, *head;
@@ -2103,6 +2104,8 @@ void block_commit_write(struct folio *folio, size_t from, size_t to)
clear_buffer_new(bh);
block_start = block_end;
+ if (uptodate && block_start >= to)
+ break;
bh = bh->b_this_page;
} while (bh != head);
diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c
index 8040c731b3e4..52f4c5169f91 100644
--- a/fs/ext4/balloc.c
+++ b/fs/ext4/balloc.c
@@ -331,9 +331,13 @@ struct ext4_group_info *ext4_get_group_info(struct super_block *sb,
if (unlikely(group >= EXT4_SB(sb)->s_groups_count))
return NULL;
+ if (unlikely(!EXT4_SB(sb)->s_group_info))
+ return NULL;
indexv = group >> (EXT4_DESC_PER_BLOCK_BITS(sb));
indexh = group & ((EXT4_DESC_PER_BLOCK(sb)) - 1);
grp_info = sbi_array_rcu_deref(EXT4_SB(sb), s_group_info, indexv);
+ if (unlikely(!grp_info))
+ return NULL;
return grp_info[indexh];
}
diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c
index 9265cfe62c83..1a0fccb084ef 100644
--- a/fs/ext4/crypto.c
+++ b/fs/ext4/crypto.c
@@ -144,7 +144,13 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len,
if (inode->i_ino == EXT4_ROOT_INO)
return -EPERM;
- if (WARN_ON_ONCE(IS_DAX(inode) && i_size_read(inode)))
+ /*
+ * For new encrypted inodes, S_DAX is never set in the first place.
+ *
+ * For existing inodes, this is called only on empty directories. ext4
+ * never sets S_DAX on directories.
+ */
+ if (WARN_ON_ONCE(IS_DAX(inode)))
return -EINVAL;
if (ext4_test_inode_flag(inode, EXT4_INODE_DAX))
@@ -164,27 +170,24 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len,
if (handle) {
/*
+ * __ext4_new_inode() should have already set the encrypt flag
+ * on the inode and avoided enabling inline data.
+ */
+ if (WARN_ON_ONCE(!IS_ENCRYPTED(inode)))
+ return -EINVAL;
+ if (WARN_ON_ONCE(ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)))
+ return -EINVAL;
+ /*
* Since the inode is new it is ok to pass the
* XATTR_CREATE flag. This is necessary to match the
* remaining journal credits check in the set_handle
* function with the credits allocated for the new
* inode.
*/
- res = ext4_xattr_set_handle(handle, inode,
- EXT4_XATTR_INDEX_ENCRYPTION,
- EXT4_XATTR_NAME_ENCRYPTION_CONTEXT,
- ctx, len, XATTR_CREATE);
- if (!res) {
- ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT);
- ext4_clear_inode_state(inode,
- EXT4_STATE_MAY_INLINE_DATA);
- /*
- * Update inode->i_flags - S_ENCRYPTED will be enabled,
- * S_DAX may be disabled
- */
- ext4_set_inode_flags(inode, false);
- }
- return res;
+ return ext4_xattr_set_handle(handle, inode,
+ EXT4_XATTR_INDEX_ENCRYPTION,
+ EXT4_XATTR_NAME_ENCRYPTION_CONTEXT,
+ ctx, len, XATTR_CREATE);
}
res = dquot_initialize(inode);
@@ -205,10 +208,7 @@ retry:
ctx, len, 0);
if (!res) {
ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT);
- /*
- * Update inode->i_flags - S_ENCRYPTED will be enabled,
- * S_DAX may be disabled
- */
+ /* Update inode->i_flags to set S_ENCRYPTED. */
ext4_set_inode_flags(inode, false);
res = ext4_mark_inode_dirty(handle, inode);
if (res)
diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c
index 17edd678fa87..8d7b81e6948e 100644
--- a/fs/ext4/dir.c
+++ b/fs/ext4/dir.c
@@ -138,6 +138,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx)
struct buffer_head *bh = NULL;
struct fscrypt_str fstr = FSTR_INIT(NULL, 0);
struct dir_private_info *info = file->private_data;
+ bool has_csum = ext4_has_feature_metadata_csum(sb);
err = fscrypt_prepare_readdir(inode);
if (err)
@@ -149,7 +150,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx)
return err;
/* Can we just clear INDEX flag to ignore htree information? */
- if (!ext4_has_feature_metadata_csum(sb)) {
+ if (!has_csum) {
/*
* We don't set the inode dirty flag since it's not
* critical that it gets flushed back to the disk.
@@ -235,7 +236,10 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx)
* dirent right now. Scan from the start of the block
* to make sure. */
if (!inode_eq_iversion(inode, info->cookie)) {
- for (i = 0; i < sb->s_blocksize && i < offset; ) {
+ for (i = 0;
+ i <= sb->s_blocksize -
+ ext4_dir_rec_len(1, has_csum ? NULL : inode) &&
+ i < offset;) {
de = (struct ext4_dir_entry_2 *)
(bh->b_data + i);
/* It's too expensive to do a full
@@ -257,6 +261,17 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx)
info->cookie = inode_query_iversion(inode);
}
+ if (unlikely(offset < sb->s_blocksize &&
+ offset > sb->s_blocksize -
+ ext4_dir_rec_len(1, has_csum ? NULL : inode))) {
+ EXT4_ERROR_FILE(file, bh->b_blocknr,
+ "bad entry in directory: %s - offset=%u, size=%lu",
+ "directory entry too close to block end",
+ offset, sb->s_blocksize);
+ ctx->pos = round_up(ctx->pos, sb->s_blocksize);
+ goto next_block;
+ }
+
while (ctx->pos < inode->i_size
&& offset < sb->s_blocksize) {
de = (struct ext4_dir_entry_2 *) (bh->b_data + offset);
@@ -312,6 +327,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx)
ctx->pos += ext4_rec_len_from_disk(de->rec_len,
sb->s_blocksize);
}
+next_block:
if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode))
goto done;
brelse(bh);
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index 2fdff50526e9..724a27e8be61 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -334,7 +334,7 @@ struct ext4_io_submit {
#define EXT4_MAX_BLOCK_SIZE 65536
#define EXT4_MIN_BLOCK_LOG_SIZE 10
#define EXT4_MAX_BLOCK_LOG_SIZE 16
-#define EXT4_MAX_CLUSTER_LOG_SIZE 30
+#define EXT4_MAX_CLUSTER_LOG_SIZE 28
#ifdef __KERNEL__
# define EXT4_BLOCK_SIZE(s) ((s)->s_blocksize)
#else
@@ -1070,8 +1070,14 @@ struct ext4_inode_info {
* between readers of EAs and writers of regular file data, so
* instead we synchronize on xattr_sem when reading or changing
* EAs.
+ *
+ * EA inodes (EXT4_EA_INODE_FL) do not use xattr_sem; they reuse
+ * the space for deferred iput linkage.
*/
- struct rw_semaphore xattr_sem;
+ union {
+ struct rw_semaphore xattr_sem;
+ struct llist_node i_ea_iput_node;
+ };
/*
* Inodes with EXT4_STATE_ORPHAN_FILE use i_orphan_idx. Otherwise
@@ -1770,6 +1776,11 @@ struct ext4_sb_info {
struct ext4_es_stats s_es_stats;
struct mb_cache *s_ea_block_cache;
struct mb_cache *s_ea_inode_cache;
+
+ /* Deferred iput for EA inodes to avoid lock ordering issues */
+ struct llist_head s_ea_inode_to_free;
+ struct delayed_work s_ea_inode_work;
+
spinlock_t s_es_lock ____cacheline_aligned_in_smp;
/* Journal triggers for checksum computation */
@@ -3148,14 +3159,15 @@ int do_journal_get_write_access(handle_t *handle, struct inode *inode,
struct buffer_head *bh);
void ext4_set_inode_mapping_order(struct inode *inode);
#define FALL_BACK_TO_NONDELALLOC 1
-#define CONVERT_INLINE_DATA 2
+#define EXT4_WRITE_DATA_INLINE 2
typedef enum {
EXT4_IGET_NORMAL = 0,
EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */
EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */
EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */
- EXT4_IGET_EA_INODE = 0x0008 /* Inode should contain an EA value */
+ EXT4_IGET_EA_INODE = 0x0008, /* Inode should contain an EA value */
+ EXT4_IGET_NOWAIT = 0x0010 /* Non-blocking lookup (skip if freeing) */
} ext4_iget_flags;
extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
@@ -3196,10 +3208,13 @@ extern int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode);
extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks);
extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks);
extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
- int pextents);
+ int pextents, int alloc_extents);
extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end);
+
+#define EXT4_PARTIAL_ZERO_START 0x1
+#define EXT4_PARTIAL_ZERO_END 0x2
extern int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart,
- loff_t length, bool *did_zero);
+ loff_t length, unsigned int *partial_zeroed);
extern vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf);
extern qsize_t *ext4_get_reserved_space(struct inode *inode);
extern int ext4_get_projid(struct inode *inode, kprojid_t *projid);
@@ -3651,7 +3666,13 @@ struct ext4_group_info {
#define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \
(clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
#define EXT4_MB_GRP_TEST_AND_SET_READ(grp) \
- (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
+ (ext4_mb_grp_test_and_set_read((grp)))
+
+static inline int ext4_mb_grp_test_and_set_read(struct ext4_group_info *grp)
+{
+ return (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state) ||
+ test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state));
+}
#define EXT4_MAX_CONTENTION 8
#define EXT4_CONTENTION_THRESHOLD 2
@@ -3760,7 +3781,7 @@ extern int ext4_generic_write_inline_data(struct address_space *mapping,
struct inode *inode,
loff_t pos, unsigned len,
struct folio **foliop,
- void **fsdata, bool da);
+ bool da);
extern int ext4_try_add_inline_entry(handle_t *handle,
struct ext4_filename *fname,
struct inode *dir, struct inode *inode);
@@ -3892,7 +3913,8 @@ extern void ext4_ext_release(struct super_block *);
extern long ext4_fallocate(struct file *file, int mode, loff_t offset,
loff_t len);
extern int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
- loff_t offset, ssize_t len);
+ loff_t offset, ssize_t len,
+ ext4_lblk_t *converted);
extern int ext4_convert_unwritten_extents_atomic(handle_t *handle,
struct inode *inode, loff_t offset, ssize_t len);
extern int ext4_convert_unwritten_io_end_vec(handle_t *handle,
diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c
index 02b066299164..53ddedb52a6f 100644
--- a/fs/ext4/ext4_jbd2.c
+++ b/fs/ext4/ext4_jbd2.c
@@ -33,14 +33,22 @@ int ext4_inode_journal_mode(struct inode *inode)
static handle_t *ext4_get_nojournal(void)
{
handle_t *handle = current->journal_info;
- unsigned long ref_cnt = (unsigned long)handle;
- BUG_ON(ref_cnt >= EXT4_NOJOURNAL_MAX_REF_COUNT);
-
- ref_cnt++;
- handle = (handle_t *)ref_cnt;
-
- current->journal_info = handle;
+ BUG_ON(handle && !handle->h_invalid);
+
+ if (!handle) {
+ handle = jbd2_alloc_handle(GFP_NOFS);
+ if (!handle)
+ return ERR_PTR(-ENOMEM);
+ handle->h_invalid = 1;
+ /*
+ * This is done by start_this_handle() if journalling
+ * is enabled.
+ */
+ handle->saved_alloc_context = memalloc_nofs_save();
+ current->journal_info = handle;
+ }
+ handle->h_ref++;
return handle;
}
@@ -48,14 +56,14 @@ static handle_t *ext4_get_nojournal(void)
/* Decrement the non-pointer handle value */
static void ext4_put_nojournal(handle_t *handle)
{
- unsigned long ref_cnt = (unsigned long)handle;
+ BUG_ON(handle->h_ref == 0);
- BUG_ON(ref_cnt == 0);
-
- ref_cnt--;
- handle = (handle_t *)ref_cnt;
-
- current->journal_info = handle;
+ handle->h_ref--;
+ if (handle->h_ref == 0) {
+ memalloc_nofs_restore(handle->saved_alloc_context);
+ jbd2_free_handle(handle);
+ current->journal_info = NULL;
+ }
}
/*
diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h
index 63d17c5201b5..2fbf48b3dfe2 100644
--- a/fs/ext4/ext4_jbd2.h
+++ b/fs/ext4/ext4_jbd2.h
@@ -182,15 +182,11 @@ handle_t *__ext4_journal_start_sb(struct inode *inode, struct super_block *sb,
int rsv_blocks, int revoke_creds);
int __ext4_journal_stop(const char *where, unsigned int line, handle_t *handle);
-#define EXT4_NOJOURNAL_MAX_REF_COUNT ((unsigned long) 4096)
-
/* Note: Do not use this for NULL handles. This is only to determine if
* a properly allocated handle is using a journal or not. */
static inline int ext4_handle_valid(handle_t *handle)
{
- if ((unsigned long)handle < EXT4_NOJOURNAL_MAX_REF_COUNT)
- return 0;
- return 1;
+ return (handle && !handle->h_invalid);
}
static inline void ext4_handle_sync(handle_t *handle)
diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
index 15972410d460..76038b6c3655 100644
--- a/fs/ext4/extents.c
+++ b/fs/ext4/extents.c
@@ -2427,9 +2427,17 @@ int ext4_ext_index_trans_blocks(struct inode *inode, int extents)
*/
if (extents <= 1)
index = (EXT4_MAX_EXTENT_DEPTH * 2) + extents;
- else
- index = (EXT4_MAX_EXTENT_DEPTH * 3) +
- DIV_ROUND_UP(extents, ext4_ext_space_block(inode, 0));
+ else {
+ int ext_max = ext4_ext_space_block(inode, 0);
+
+ index = EXT4_MAX_EXTENT_DEPTH * 3;
+ /*
+ * Modified extents need not start at the beginning of the
+ * leaf. Already two extents may need two leaf block
+ * modifications...
+ */
+ index += DIV_ROUND_UP(extents + ext_max - 1, ext_max);
+ }
return index;
}
@@ -4571,6 +4579,22 @@ retry_remove_space:
return err;
}
+/*
+ * Pre-allocate blocks for the range [@offset, @offset + @len). Allocated
+ * blocks are marked as unwritten by default. If EXT4_GET_BLOCKS_ZERO is
+ * set, the allocated blocks are zeroed on disk and their extents are
+ * converted to written state.
+ *
+ * When @new_size is nonzero, the caller intends to extend the file, and
+ * the file size should be updated to the end of the allocated blocks.
+ *
+ * Allocation may partially succeed due to some non-fatal issues. In that
+ * case, i_disksize (and i_size) is advanced up to the successfully
+ * processed portion of the range.
+ *
+ * Return 0 on success, or a negative error code on failure or partial
+ * failure.
+ */
static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len,
loff_t new_size, int flags)
{
@@ -4585,6 +4609,7 @@ static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len,
loff_t epos = 0, old_size = i_size_read(inode);
unsigned int blkbits = inode->i_blkbits;
bool alloc_zero = false;
+ bool orphan = false;
BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS));
map.m_lblk = offset >> blkbits;
@@ -4659,19 +4684,49 @@ retry:
if (alloc_zero &&
(map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) {
+ ext4_lblk_t converted;
+
+ WARN_ON_ONCE(map.m_lblk + map.m_len >
+ EXT4_B_TO_LBLK(inode, new_size ?: old_size));
+
ret = ext4_issue_zeroout(inode, map.m_lblk, map.m_pblk,
map.m_len);
- if (likely(!ret))
- ret = ext4_convert_unwritten_extents(NULL,
+ if (unlikely(ret))
+ break;
+
+ handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
+ credits);
+ if (IS_ERR(handle)) {
+ ret = PTR_ERR(handle);
+ break;
+ }
+
+ ret = ext4_convert_unwritten_extents(handle,
inode, (loff_t)map.m_lblk << blkbits,
- (loff_t)map.m_len << blkbits);
+ (loff_t)map.m_len << blkbits,
+ &converted);
if (ret)
- break;
+ map.m_len = converted;
+
+ /*
+ * If blocks beyond i_disksize are converted, add
+ * the inode to the orphan list and advance the epos.
+ */
+ if (new_size && converted) {
+ ret2 = ext4_orphan_add(handle, inode);
+ ret = ret ? ret : ret2;
+ orphan = true;
+ }
+
+ ret3 = ext4_journal_stop(handle);
+ ret = ret ? ret : ret3;
}
map.m_lblk += map.m_len;
map.m_len = len_lblk = len_lblk - map.m_len;
epos = EXT4_LBLK_TO_B(inode, map.m_lblk);
+ if (ret)
+ break;
}
if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
@@ -4687,11 +4742,23 @@ retry:
if (epos > new_size)
epos = new_size;
- handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
- if (IS_ERR(handle))
- return ret ? ret : PTR_ERR(handle);
+ handle = ext4_journal_start(inode, EXT4_HT_MISC, 2);
+ if (IS_ERR(handle)) {
+ /*
+ * The conversion has successfully completed. Not much to
+ * do with the error here so just cleanup the orphan list
+ * and hope for the best.
+ */
+ if (orphan && inode->i_nlink)
+ ext4_orphan_del(NULL, inode);
+ ret2 = PTR_ERR(handle);
+ goto out;
+ }
ext4_update_inode_size(inode, epos);
+ if (orphan && inode->i_nlink)
+ ext4_orphan_del(handle, inode);
+
ret2 = ext4_mark_inode_dirty(handle, inode);
ext4_update_inode_fsync_trans(handle, inode, 1);
ret3 = ext4_journal_stop(handle);
@@ -4699,6 +4766,9 @@ retry:
if (epos > old_size)
pagecache_isize_extended(inode, old_size, epos);
+out:
+ if (ret2)
+ ext4_std_error(inode->i_sb, ret2);
return ret ? ret : ret2;
}
@@ -4715,7 +4785,7 @@ static long ext4_zero_range(struct file *file, loff_t offset,
loff_t align_start, align_end, new_size = 0;
loff_t end = offset + len;
unsigned int blocksize = i_blocksize(inode);
- bool partial_zeroed = false;
+ unsigned int partial_zeroed = 0;
int ret, flags;
trace_ext4_zero_range(inode, offset, len, mode);
@@ -4734,10 +4804,16 @@ static long ext4_zero_range(struct file *file, loff_t offset,
}
flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
- /* Preallocate the range including the unaligned edges */
+ /*
+ * Preallocate the range including the unaligned edges, and zero
+ * out partial blocks if they already contain data.
+ */
if (!IS_ALIGNED(offset | end, blocksize)) {
ret = ext4_alloc_file_blocks(file, offset, len, new_size,
flags);
+ if (!ret)
+ ret = ext4_zero_partial_blocks(inode, offset, len,
+ &partial_zeroed);
if (ret)
return ret;
}
@@ -4754,6 +4830,21 @@ static long ext4_zero_range(struct file *file, loff_t offset,
/* Zero range excluding the unaligned edges */
align_start = round_up(offset, blocksize);
align_end = round_down(end, blocksize);
+
+ /*
+ * In WRITE_ZEROES mode, edges that were not partial-zeroed (clean
+ * unwritten or hole) must be allocated and zeroed as whole blocks.
+ * Expand the aligned range outward to cover them.
+ */
+ if (mode & FALLOC_FL_WRITE_ZEROES) {
+ if (!IS_ALIGNED(offset, blocksize) &&
+ !(partial_zeroed & EXT4_PARTIAL_ZERO_START))
+ align_start = round_down(offset, blocksize);
+ if (!IS_ALIGNED(end, blocksize) &&
+ !(partial_zeroed & EXT4_PARTIAL_ZERO_END))
+ align_end = round_up(end, blocksize);
+ }
+
if (align_end > align_start) {
if (mode & FALLOC_FL_WRITE_ZEROES)
flags = EXT4_GET_BLOCKS_CREATE_ZERO | EXT4_EX_NOCACHE;
@@ -4770,11 +4861,15 @@ static long ext4_zero_range(struct file *file, loff_t offset,
if (IS_ALIGNED(offset | end, blocksize))
return ret;
- /* Zero out partial block at the edges of the range */
- ret = ext4_zero_partial_blocks(inode, offset, len, &partial_zeroed);
- if (ret)
- return ret;
- if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) {
+ /*
+ * In FALLOC_FL_WRITE_ZEROES mode, edges that have been partially
+ * zeroed must be written back to ensure the entire zeroed range
+ * is converted to the written state. In SYNC mode, writeback is
+ * also required to persist the zeroed data to disk.
+ */
+ if (partial_zeroed &&
+ ((mode & FALLOC_FL_WRITE_ZEROES) ||
+ (file->f_flags & O_SYNC) || IS_SYNC(inode))) {
ret = filemap_write_and_wait_range(inode->i_mapping, offset,
end - 1);
if (ret)
@@ -4976,7 +5071,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode,
* it can tell if the extent in the cache is a split extent.
* But for now let's assume pextents as 2 always.
*/
- credits = ext4_meta_trans_blocks(inode, max_blocks, 2);
+ credits = ext4_meta_trans_blocks(inode, max_blocks, 2, 0);
}
if (credits) {
@@ -5026,21 +5121,26 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode,
* all unwritten extents within this range will be converted to
* written extents.
*
- * This function is called from the direct IO end io call back
- * function, to convert the fallocated extents after IO is completed.
- * Returns 0 on success.
+ * This function is called from the direct/buffered I/O end io call back
+ * function and FALLOC_FL_WRITE_ZEROES, to convert the fallocated
+ * unwritten extents after data I/O is completed.
+ *
+ * Returns 0 on full success, or a negative error code on partial
+ * success or failure. The number of blocks converted is returned via
+ * @converted.
*/
int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
- loff_t offset, ssize_t len)
+ loff_t offset, ssize_t len,
+ ext4_lblk_t *converted)
{
- unsigned int max_blocks;
+ ext4_lblk_t max_blocks, conv_blocks = 0;
int ret = 0, ret2 = 0, ret3 = 0;
struct ext4_map_blocks map;
unsigned int blkbits = inode->i_blkbits;
unsigned int credits = 0;
map.m_lblk = offset >> blkbits;
- max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
+ map.m_len = max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
if (!handle) {
/*
@@ -5048,9 +5148,8 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
*/
credits = ext4_chunk_trans_blocks(inode, max_blocks);
}
- while (ret >= 0 && ret < max_blocks) {
- map.m_lblk += ret;
- map.m_len = (max_blocks -= ret);
+
+ while (max_blocks) {
if (credits) {
handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
credits);
@@ -5067,23 +5166,34 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
ret = ext4_map_blocks(handle, inode, &map,
EXT4_GET_BLOCKS_IO_CONVERT_EXT |
EXT4_EX_NOCACHE);
- if (ret <= 0)
+ if (ret <= 0) {
ext4_warning(inode->i_sb,
- "inode #%llu: block %u: len %u: "
- "ext4_ext_map_blocks returned %d",
- inode->i_ino, map.m_lblk,
- map.m_len, ret);
+ "inode #%llu: block %u: len %u: ext4_map_blocks returned %d",
+ inode->i_ino, map.m_lblk, map.m_len, ret);
+ if (unlikely(ret == 0))
+ ret = -EINVAL;
+ } else {
+ conv_blocks += map.m_len;
+ }
+
ret2 = ext4_mark_inode_dirty(handle, inode);
if (credits) {
ret3 = ext4_journal_stop(handle);
if (unlikely(ret3))
ret2 = ret3;
}
-
- if (ret <= 0 || ret2)
+ ret = ret < 0 ? ret : ret2;
+ if (ret)
break;
+
+ map.m_lblk += map.m_len;
+ map.m_len = (max_blocks -= map.m_len);
}
- return ret > 0 ? ret2 : ret;
+ /* Converted some or all blocks successfully? */
+ if (converted)
+ *converted = conv_blocks;
+
+ return ret;
}
int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end)
@@ -5106,7 +5216,7 @@ int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end)
list_for_each_entry(io_end_vec, &io_end->list_vec, list) {
ret = ext4_convert_unwritten_extents(handle, io_end->inode,
io_end_vec->offset,
- io_end_vec->size);
+ io_end_vec->size, NULL);
if (ret)
break;
}
diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c
index 8e2259799614..062103e42cd8 100644
--- a/fs/ext4/fast_commit.c
+++ b/fs/ext4/fast_commit.c
@@ -200,25 +200,6 @@ static inline void ext4_fc_set_snap_err(int *snap_err, int err)
*snap_err = err;
}
-static void ext4_end_buffer_io_sync(struct bio *bio)
-{
- struct buffer_head *bh;
- bool uptodate = bio_endio_bh(bio, &bh);
-
- BUFFER_TRACE(bh, "");
- if (uptodate) {
- ext4_debug("%s: Block %lld up-to-date",
- __func__, bh->b_blocknr);
- set_buffer_uptodate(bh);
- } else {
- ext4_debug("%s: Block %lld not up-to-date",
- __func__, bh->b_blocknr);
- clear_buffer_uptodate(bh);
- }
-
- unlock_buffer(bh);
-}
-
static void ext4_fc_free_inode_snap(struct inode *inode);
static inline void ext4_fc_reset_inode(struct inode *inode)
@@ -691,7 +672,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
lock_buffer(bh);
set_buffer_dirty(bh);
set_buffer_uptodate(bh);
- bh_submit(bh, REQ_OP_WRITE | write_flags, ext4_end_buffer_io_sync);
+ bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write);
EXT4_SB(sb)->s_fc_bh = NULL;
}
@@ -2196,8 +2177,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val)
if (ret == 0) {
/* Range is not mapped */
path = ext4_find_extent(inode, cur, path, 0);
- if (IS_ERR(path))
+ if (IS_ERR(path)) {
+ ret = PTR_ERR(path);
+ path = NULL;
goto out;
+ }
memset(&newex, 0, sizeof(newex));
newex.ee_block = cpu_to_le32(cur);
ext4_ext_store_pblock(
@@ -2209,8 +2193,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val)
path = ext4_ext_insert_extent(NULL, inode,
path, &newex, 0);
up_write((&EXT4_I(inode)->i_data_sem));
- if (IS_ERR(path))
+ if (IS_ERR(path)) {
+ ret = PTR_ERR(path);
+ path = NULL;
goto out;
+ }
goto next;
}
@@ -2257,10 +2244,11 @@ next:
}
ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
sb->s_blocksize_bits);
+ ret = 0;
out:
ext4_free_ext_path(path);
iput(inode);
- return 0;
+ return ret;
}
/* Replay DEL_RANGE tag */
@@ -2320,9 +2308,10 @@ ext4_fc_replay_del_range(struct super_block *sb, u8 *val)
ext4_ext_replay_shrink_inode(inode,
i_size_read(inode) >> sb->s_blocksize_bits);
ext4_mark_inode_dirty(NULL, inode);
+ ret = 0;
out:
iput(inode);
- return 0;
+ return ret;
}
static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
diff --git a/fs/ext4/file.c b/fs/ext4/file.c
index f20d92255546..374b4bc25bd5 100644
--- a/fs/ext4/file.c
+++ b/fs/ext4/file.c
@@ -215,31 +215,60 @@ ext4_extending_io(struct inode *inode, loff_t offset, size_t len)
return false;
}
-/* Is IO overwriting allocated or initialized blocks? */
-static bool ext4_overwrite_io(struct inode *inode,
- loff_t pos, loff_t len, bool *unwritten)
+/*
+ * Does an unaligned DIO write require partial block zeroing?
+ *
+ * Partial block zeroing is performed only for the head and tail blocks
+ * when they are partially covered by the write and the underlying extent
+ * is a hole or unwritten. Middle blocks (fully covered by the write)
+ * are written as whole blocks without zeroing.
+ *
+ * When zeroing is required, two concurrent unaligned DIO writes to the
+ * same partial block can race and corrupt each other's data, so the
+ * caller must take the exclusive i_rwsem and drain in-flight DIO. When
+ * zeroing is not required, shared lock is safe -- block allocation and
+ * unwritten conversion for middle blocks are protected by i_data_sem
+ * and inode_dio_begin().
+ */
+static bool ext4_dio_needs_zeroing(struct inode *inode, loff_t pos, loff_t len)
{
struct ext4_map_blocks map;
unsigned int blkbits = inode->i_blkbits;
- int err, blklen;
+ unsigned long blockmask = inode->i_sb->s_blocksize - 1;
+ bool head_partial, tail_partial;
+ ext4_lblk_t head_lblk, tail_lblk;
+ int err;
if (pos + len > i_size_read(inode))
- return false;
+ return true;
- map.m_lblk = pos >> blkbits;
- map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits);
- blklen = map.m_len;
+ head_partial = (pos & blockmask) != 0;
+ tail_partial = ((pos + len) & blockmask) != 0;
+ head_lblk = pos >> blkbits;
+ tail_lblk = (pos + len - 1) >> blkbits;
+
+ /* Check the head partial block. */
+ if (head_partial) {
+ map.m_lblk = head_lblk;
+ map.m_len = tail_lblk - head_lblk + 1;
+ err = ext4_map_blocks(NULL, inode, &map, 0);
+ if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED))
+ return true;
+ /* If this mapping already covers the tail block, we're done. */
+ if (!tail_partial || map.m_lblk + err > tail_lblk)
+ return false;
+ }
- err = ext4_map_blocks(NULL, inode, &map, 0);
- if (err != blklen)
- return false;
- /*
- * 'err==len' means that all of the blocks have been preallocated,
- * regardless of whether they have been initialized or not. We need to
- * check m_flags to distinguish the unwritten extents.
- */
- *unwritten = !(map.m_flags & EXT4_MAP_MAPPED);
- return true;
+ /* Check the tail partial block. */
+ if (tail_partial) {
+ map.m_lblk = tail_lblk;
+ map.m_len = 1;
+ err = ext4_map_blocks(NULL, inode, &map, 0);
+ if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED))
+ return true;
+ }
+
+ return false;
}
static ssize_t ext4_generic_write_checks(struct kiocb *iocb,
@@ -280,7 +309,7 @@ static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)
if (count <= 0)
return count;
- ret = file_modified(iocb->ki_filp);
+ ret = kiocb_modified(iocb);
if (ret)
return ret;
@@ -311,6 +340,13 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,
return -EOPNOTSUPP;
inode_lock(inode);
+
+ /*
+ * Prevent concurrent direct I/O and buffered I/O to the same file
+ * range. Wait for in-flight DIO to finish before dirtying pages.
+ */
+ inode_dio_wait(inode);
+
ret = ext4_write_checks(iocb, from);
if (ret <= 0)
goto out;
@@ -402,7 +438,8 @@ static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,
error = ext4_convert_unwritten_extents_atomic(NULL, inode, pos,
size);
else if (!error && size && flags & IOMAP_DIO_UNWRITTEN)
- error = ext4_convert_unwritten_extents(NULL, inode, pos, size);
+ error = ext4_convert_unwritten_extents(NULL, inode, pos, size,
+ NULL);
if (error)
return error;
/*
@@ -430,16 +467,28 @@ static const struct iomap_dio_ops ext4_dio_write_ops = {
* condition requires an exclusive inode lock. If yes, then we restart the
* whole operation by releasing the shared lock and acquiring exclusive lock.
*
- * - For unaligned_io we never take shared lock as it may cause data corruption
- * when two unaligned IO tries to modify the same block e.g. while zeroing.
+ * The decision is layered, evaluated in this order:
*
- * - For extending writes case we don't take the shared lock, since it requires
- * updating inode i_disksize and/or orphan handling with exclusive lock.
+ * 1. If kiocb_modified() needs to update security info (!IS_NOSEC), upgrade
+ * to the exclusive lock -- the security update itself requires it,
+ * regardless of whether the write extends the file or is aligned.
*
- * - shared locking will only be true mostly with overwrites, including
- * initialized blocks and unwritten blocks.
+ * 2. If the write extends i_size or i_disksize, upgrade to the exclusive
+ * lock to safely update i_disksize and the orphan list, regardless of
+ * alignment.
*
- * - Otherwise we will switch to exclusive i_rwsem lock.
+ * 3. Otherwise, for aligned non-extending writes, shared lock is always
+ * sufficient regardless of extent state (written, unwritten, or hole).
+ * truncate/punch_hole cannot run while we hold the shared i_rwsem
+ * (they need it exclusively); after we release it, inode_dio_begin()
+ * keeps their inode_dio_wait() blocked until in-flight bios complete.
+ * i_data_sem serializes concurrent extent tree modifications.
+ *
+ * 4. Otherwise, the write is unaligned and non-extending. Shared lock is
+ * safe unless the DIO layer needs to perform partial block zeroing --
+ * i.e. the head or tail partial block sits on a hole or unwritten
+ * extent. In that case upgrade to the exclusive lock and drain
+ * in-flight DIO to avoid races with concurrent partial block zeroing.
*/
static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,
bool *ilock_shared, bool *extend,
@@ -450,7 +499,7 @@ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,
loff_t offset;
size_t count;
ssize_t ret;
- bool overwrite, unaligned_io, unwritten;
+ bool needs_zeroing = false;
restart:
ret = ext4_generic_write_checks(iocb, from);
@@ -460,24 +509,22 @@ restart:
offset = iocb->ki_pos;
count = ret;
- unaligned_io = ext4_unaligned_io(inode, from, offset);
*extend = ext4_extending_io(inode, offset, count);
- overwrite = ext4_overwrite_io(inode, offset, count, &unwritten);
/*
- * Determine whether we need to upgrade to an exclusive lock. This is
- * required to change security info in file_modified(), for extending
- * I/O, any form of non-overwrite I/O, and unaligned I/O to unwritten
- * extents (as partial block zeroing may be required).
+ * For unaligned writes, check whether partial block zeroing will be
+ * needed. If so, exclusive lock is required to serialize against
+ * concurrent DIO that could race with the zeroing.
*
- * Note that unaligned writes are allowed under shared lock so long as
- * they are pure overwrites. Otherwise, concurrent unaligned writes risk
- * data corruption due to partial block zeroing in the dio layer, and so
- * the I/O must occur exclusively.
+ * For aligned writes we skip this check entirely since allocation
+ * under shared lock is safe.
*/
+ if (ext4_unaligned_io(inode, from, offset))
+ needs_zeroing = ext4_dio_needs_zeroing(inode, offset, count);
+
+ /* Determine whether we need to upgrade to an exclusive lock. */
if (*ilock_shared &&
- ((!IS_NOSEC(inode) || *extend || !overwrite ||
- (unaligned_io && unwritten)))) {
+ (!IS_NOSEC(inode) || *extend || needs_zeroing)) {
if (iocb->ki_flags & IOCB_NOWAIT) {
ret = -EAGAIN;
goto out;
@@ -491,21 +538,28 @@ restart:
/*
* Now that locking is settled, determine dio flags and exclusivity
* requirements. We don't use DIO_OVERWRITE_ONLY because we enforce
- * behavior already. The inode lock is already held exclusive if the
- * write is non-overwrite or extending, so drain all outstanding dio and
- * set the force wait dio flag.
+ * behavior already. When holding the exclusive lock for a write that
+ * needs partial block zeroing or is extending the file, we must wait
+ * for the I/O to complete synchronously:
+ *
+ * - needs_zeroing: drain in-flight DIO whose end_io could race with
+ * our partial block zeroing, and force synchronous completion so we
+ * don't leave in-flight zeroing bios for the next writer to drain.
+ *
+ * - extend: the caller must update i_disksize after I/O completion,
+ * which requires the data to be on disk first.
*/
- if (!*ilock_shared && (unaligned_io || *extend)) {
+ if (!*ilock_shared && (needs_zeroing || *extend)) {
if (iocb->ki_flags & IOCB_NOWAIT) {
ret = -EAGAIN;
goto out;
}
- if (unaligned_io && (!overwrite || unwritten))
+ if (needs_zeroing)
inode_dio_wait(inode);
*dio_flags = IOMAP_DIO_FORCE_WAIT;
}
- ret = file_modified(file);
+ ret = kiocb_modified(iocb);
if (ret < 0)
goto out;
@@ -674,6 +728,11 @@ ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
count = iov_iter_count(from);
if (offset + count > EXT4_I(inode)->i_disksize) {
+ if (iocb->ki_flags & IOCB_NOWAIT) {
+ ret = -EAGAIN;
+ goto out;
+ }
+
handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
if (IS_ERR(handle)) {
ret = PTR_ERR(handle);
diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c
index a40cb27f8116..a5831fc536db 100644
--- a/fs/ext4/ialloc.c
+++ b/fs/ext4/ialloc.c
@@ -997,6 +997,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap,
err = fscrypt_prepare_new_inode(dir, inode, &encrypt);
if (err)
goto out;
+ if (encrypt)
+ i_flags |= EXT4_ENCRYPT_FL;
}
err = dquot_initialize(inode);
@@ -1306,6 +1308,8 @@ got:
ei->i_extra_isize = sbi->s_want_extra_isize;
ei->i_inline_off = 0;
if (ext4_has_feature_inline_data(sb) &&
+ /* Encrypted inodes cannot have inline data */
+ !(ei->i_flags & EXT4_ENCRYPT_FL) &&
(!(ei->i_flags & (EXT4_DAX_FL|EXT4_EA_INODE_FL)) || S_ISDIR(mode)))
ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
ret = inode;
diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c
index 8045e4ff270c..ceee69a66482 100644
--- a/fs/ext4/inline.c
+++ b/fs/ext4/inline.c
@@ -22,8 +22,7 @@
static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping,
- struct inode *inode,
- void **fsdata);
+ struct inode *inode);
static int ext4_get_inline_size(struct inode *inode)
{
@@ -697,7 +696,7 @@ int ext4_generic_write_inline_data(struct address_space *mapping,
struct inode *inode,
loff_t pos, unsigned len,
struct folio **foliop,
- void **fsdata, bool da)
+ bool da)
{
int ret;
handle_t *handle;
@@ -728,7 +727,7 @@ retry_journal:
return ext4_convert_inline_data_to_extent(mapping, inode);
}
- ret = ext4_da_convert_inline_data_to_extent(mapping, inode, fsdata);
+ ret = ext4_da_convert_inline_data_to_extent(mapping, inode);
if (ret == -ENOSPC &&
ext4_should_retry_alloc(inode->i_sb, &retries))
goto retry_journal;
@@ -788,7 +787,7 @@ int ext4_try_to_write_inline_data(struct address_space *mapping,
if (pos + len > ext4_get_max_inline_size(inode))
return ext4_convert_inline_data_to_extent(mapping, inode);
return ext4_generic_write_inline_data(mapping, inode, pos, len,
- foliop, NULL, false);
+ foliop, false);
}
int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len,
@@ -812,7 +811,19 @@ int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len,
goto out;
}
ext4_write_lock_xattr(inode, &no_expand);
- BUG_ON(!ext4_has_inline_data(inode));
+ /*
+ * We could have raced with ext4_page_mkwrite() converting
+ * the inode and clearing the inline data flag, so we just
+ * release resources and retry the whole write.
+ */
+ if (unlikely(!ext4_has_inline_data(inode))) {
+ ext4_write_unlock_xattr(inode, &no_expand);
+ brelse(iloc.bh);
+ folio_unlock(folio);
+ folio_put(folio);
+ ext4_journal_stop(handle);
+ return 0;
+ }
/*
* ei->i_inline_off may have changed since
@@ -883,8 +894,7 @@ out:
* need to start the journal since the file's metadata isn't changed now.
*/
static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping,
- struct inode *inode,
- void **fsdata)
+ struct inode *inode)
{
int ret = 0, inline_size;
struct folio *folio;
@@ -922,7 +932,6 @@ static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping,
folio_mark_dirty(folio);
folio_mark_uptodate(folio);
ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
- *fsdata = (void *)CONVERT_INLINE_DATA;
out:
up_read(&EXT4_I(inode)->xattr_sem);
@@ -1454,6 +1463,8 @@ int ext4_read_inline_dir(struct file *file,
/* for other entry, the real offset in
* the buf has to be tuned accordingly.
*/
+ if (i + ext4_dir_rec_len(1, NULL) > extra_size)
+ break;
de = (struct ext4_dir_entry_2 *)
(dir_buf + i - extra_offset);
/* It's too expensive to do a full
@@ -1488,10 +1499,17 @@ int ext4_read_inline_dir(struct file *file,
continue;
}
+ /*
+ * de lives at dir_buf + ctx->pos - extra_offset, within the
+ * kmalloc(inline_size) buffer. Make sure its header fits before
+ * ext4_check_dir_entry() dereferences de->rec_len.
+ */
+ if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size)
+ goto out;
de = (struct ext4_dir_entry_2 *)
(dir_buf + ctx->pos - extra_offset);
if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf,
- extra_size, ctx->pos))
+ inline_size, ctx->pos))
goto out;
if (le32_to_cpu(de->inode)) {
if (!dir_emit(ctx, de->name, de->name_len,
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 3fc235eb65f2..bd4b778df9eb 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -176,7 +176,6 @@ void ext4_evict_inode(struct inode *inode)
* (xattr block freeing), bitmap, group descriptor (inode freeing)
*/
int extra_credits = 6;
- struct ext4_xattr_inode_array *ea_inode_array = NULL;
bool freeze_protected = false;
trace_ext4_evict_inode(inode);
@@ -266,6 +265,7 @@ void ext4_evict_inode(struct inode *inode)
if (ext4_inode_is_fast_symlink(inode))
memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
inode->i_size = 0;
+ ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND);
err = ext4_mark_inode_dirty(handle, inode);
if (err) {
ext4_warning(inode->i_sb,
@@ -283,8 +283,7 @@ void ext4_evict_inode(struct inode *inode)
}
/* Remove xattr references. */
- err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
- extra_credits);
+ err = ext4_xattr_delete_inode(handle, inode, extra_credits);
if (err) {
ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
stop_handle:
@@ -292,7 +291,6 @@ stop_handle:
ext4_orphan_del(NULL, inode);
if (freeze_protected)
sb_end_intwrite(inode->i_sb);
- ext4_xattr_inode_array_free(ea_inode_array);
goto no_delete;
}
@@ -322,7 +320,6 @@ stop_handle:
ext4_journal_stop(handle);
if (freeze_protected)
sb_end_intwrite(inode->i_sb);
- ext4_xattr_inode_array_free(ea_inode_array);
return;
no_delete:
/*
@@ -1184,6 +1181,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio,
int nr_wait = 0;
int i;
bool should_journal_data = ext4_should_journal_data(inode);
+ bool folio_uptodate = folio_test_uptodate(folio);
BUG_ON(!folio_test_locked(folio));
BUG_ON(to > folio_size(folio));
@@ -1195,13 +1193,13 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio,
head = create_empty_buffers(folio, blocksize, 0);
block = EXT4_PG_TO_LBLK(inode, folio->index);
- for (bh = head, block_start = 0; bh != head || !block_start;
+ for (bh = head, block_start = 0;
+ block_start < to || (!folio_uptodate && bh != head);
block++, block_start = block_end, bh = bh->b_this_page) {
block_end = block_start + blocksize;
if (block_end <= from || block_start >= to) {
- if (folio_test_uptodate(folio)) {
+ if (folio_uptodate)
set_buffer_uptodate(bh);
- }
continue;
}
if (WARN_ON_ONCE(buffer_new(bh)))
@@ -1222,7 +1220,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio,
if (should_journal_data)
do_journal_get_write_access(handle,
inode, bh);
- if (folio_test_uptodate(folio)) {
+ if (folio_uptodate) {
/*
* Unlike __block_write_begin() we leave
* dirtying of new uptodate buffers to
@@ -1239,7 +1237,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio,
continue;
}
}
- if (folio_test_uptodate(folio)) {
+ if (folio_uptodate) {
set_buffer_uptodate(bh);
continue;
}
@@ -1293,6 +1291,8 @@ static int ext4_write_begin(const struct kiocb *iocb,
if (unlikely(ret))
return ret;
+ *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE);
+
trace_ext4_write_begin(inode, pos, len);
/*
* Reserve one block more for addition to orphan list in case
@@ -1307,8 +1307,10 @@ static int ext4_write_begin(const struct kiocb *iocb,
foliop);
if (ret < 0)
return ret;
- if (ret == 1)
+ if (ret == 1) {
+ *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE);
return 0;
+ }
}
/*
@@ -1441,8 +1443,7 @@ static int ext4_write_end(const struct kiocb *iocb,
trace_ext4_write_end(inode, pos, len, copied);
- if (ext4_has_inline_data(inode) &&
- ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))
+ if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE)
return ext4_write_inline_data_end(inode, pos, len, copied,
folio);
@@ -1551,8 +1552,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb,
BUG_ON(!ext4_handle_valid(handle));
- if (ext4_has_inline_data(inode) &&
- ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))
+ if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE)
return ext4_write_inline_data_end(inode, pos, len, copied,
folio);
@@ -2672,13 +2672,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
* page is already under writeback and we are not doing
* a data integrity writeback, skip the page
*/
- if (!folio_test_dirty(folio) ||
- (folio_test_writeback(folio) &&
- (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
+ if ((folio_test_writeback(folio) &&
+ mpd->wbc->sync_mode == WB_SYNC_NONE) ||
unlikely(folio->mapping != mapping)) {
folio_unlock(folio);
continue;
}
+ /*
+ * If the folio is clean, skip writing it back.
+ * Cycle the folio through the writeback state
+ * though, to clear stale xarray tags.
+ */
+ if (!folio_test_dirty(folio)) {
+ if (!folio_test_writeback(folio)) {
+ __folio_start_writeback(folio, false);
+ folio_end_writeback(folio);
+ }
+ folio_unlock(folio);
+ continue;
+ }
folio_wait_writeback(folio);
BUG_ON(folio_test_writeback(folio));
@@ -2829,10 +2841,10 @@ static int ext4_do_writepages(struct mpage_da_data *mpd)
if (ext4_should_dioread_nolock(inode)) {
int bpf = ext4_journal_blocks_per_folio(inode);
/*
- * We may need to convert up to one extent per block in
- * the folio and we may dirty the inode.
+ * We may need to convert up to one extent per block in the
+ * folio.
*/
- rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf);
+ rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0);
}
if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
@@ -3135,11 +3147,13 @@ static int ext4_da_write_begin(const struct kiocb *iocb,
if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
ret = ext4_generic_write_inline_data(mapping, inode, pos, len,
- foliop, fsdata, true);
+ foliop, true);
if (ret < 0)
return ret;
- if (ret == 1)
+ if (ret == 1) {
+ *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE);
return 0;
+ }
}
retry:
@@ -3268,17 +3282,15 @@ static int ext4_da_write_end(const struct kiocb *iocb,
struct folio *folio, void *fsdata)
{
struct inode *inode = mapping->host;
- int write_mode = (int)(unsigned long)fsdata;
+ unsigned long write_mode = (unsigned long)fsdata;
- if (write_mode == FALL_BACK_TO_NONDELALLOC)
+ if (write_mode & FALL_BACK_TO_NONDELALLOC)
return ext4_write_end(iocb, mapping, pos,
len, copied, folio, fsdata);
trace_ext4_da_write_end(inode, pos, len, copied);
- if (write_mode != CONVERT_INLINE_DATA &&
- ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
- ext4_has_inline_data(inode))
+ if (write_mode & EXT4_WRITE_DATA_INLINE)
return ext4_write_inline_data_end(inode, pos, len, copied,
folio);
@@ -3652,6 +3664,9 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
int ret, dio_credits, m_flags = 0, retries = 0;
bool force_commit = false;
+ if (flags & IOMAP_NOWAIT)
+ return -EAGAIN;
+
/*
* Trim the mapping request to the maximum value that we can map at
* once for direct I/O.
@@ -3673,8 +3688,8 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
return ret;
if (map->m_len < orig_mlen) {
map->m_len = orig_mlen;
- dio_credits = ext4_meta_trans_blocks(inode, orig_mlen,
- map->m_len);
+ dio_credits = ext4_meta_trans_blocks(inode, map->m_len,
+ map->m_len, 0);
} else {
dio_credits = ext4_chunk_trans_blocks(inode,
map->m_len);
@@ -4010,6 +4025,10 @@ void ext4_set_aops(struct inode *inode)
* because it might have data in pagecache (eg, if called from ext4_zero_range,
* ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a
* racing writeback can come later and flush the stale pagecache to disk.
+ *
+ * Return the loaded bh if it actually needs zeroing - in written, dirty
+ * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or
+ * a clean unwritten block).
*/
static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from)
{
@@ -4021,7 +4040,7 @@ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from)
int err = 0;
folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT,
- FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
+ FGP_WRITEBEGIN | FGP_ACCESSED,
mapping_gfp_constraint(mapping, ~__GFP_FS));
if (IS_ERR(folio))
return ERR_CAST(folio);
@@ -4048,8 +4067,15 @@ static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from)
}
if (!buffer_mapped(bh)) {
BUFFER_TRACE(bh, "unmapped");
- ext4_get_block(inode, iblock, bh, 0);
- /* unmapped? It's a hole - nothing to do */
+ err = ext4_get_block(inode, iblock, bh, 0);
+ if (err < 0)
+ goto unlock;
+ /*
+ * It's a hole or a clean unwritten block - nothing to do.
+ * Note that a lookup-only get_block (without
+ * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean
+ * unwritten extents.
+ */
if (!buffer_mapped(bh)) {
BUFFER_TRACE(bh, "still unmapped");
goto unlock;
@@ -4191,6 +4217,14 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end)
offset = from & (blocksize - 1);
if (!offset || from >= end)
return 0;
+ /*
+ * Inline data has no tail block to zero out. Note that a race with
+ * ext4_page_mkwrite() converting inline data to an extent without
+ * holding i_rwsem is safe, as that path zeroes the full block before
+ * copying in the inline data.
+ */
+ if (ext4_has_inline_data(inode))
+ return 0;
/* If we are processing an encrypted inode during orphan list handling */
if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
return 0;
@@ -4225,13 +4259,26 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end)
return 0;
}
+/*
+ * Zero out the unaligned head and tail of the [lstart, lstart+length)
+ * range.
+ *
+ * On return, @partial_zeroed records which edges actually got
+ * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if
+ * the head/tail block got actually partially zeroed (in written, dirty
+ * unwritten or delalloc state). Cleared if the head/tail block is a
+ * hole or a clean unwritten block, in which case there is nothing that
+ * needs zeroing. When the head and tail land in the same block, both
+ * bits are set together on a successful zeroing.
+ */
int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length,
- bool *did_zero)
+ unsigned int *partial_zeroed)
{
struct super_block *sb = inode->i_sb;
unsigned partial_start, partial_end;
ext4_fsblk_t start, end;
loff_t byte_end = (lstart + length - 1);
+ bool did_zero = false;
int err = 0;
partial_start = lstart & (sb->s_blocksize - 1);
@@ -4243,21 +4290,32 @@ int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length,
/* Handle partial zero within the single block */
if (start == end &&
(partial_start || (partial_end != sb->s_blocksize - 1))) {
- err = ext4_block_zero_range(inode, lstart, length, did_zero,
+ err = ext4_block_zero_range(inode, lstart, length, &did_zero,
NULL);
+ if (did_zero)
+ *partial_zeroed |= (EXT4_PARTIAL_ZERO_START |
+ EXT4_PARTIAL_ZERO_END);
return err;
}
/* Handle partial zero out on the start of the range */
if (partial_start) {
err = ext4_block_zero_range(inode, lstart, sb->s_blocksize,
- did_zero, NULL);
+ &did_zero, NULL);
if (err)
return err;
+ if (did_zero)
+ *partial_zeroed |= EXT4_PARTIAL_ZERO_START;
}
/* Handle partial zero out on the end of the range */
- if (partial_end != sb->s_blocksize - 1)
+ if (partial_end != sb->s_blocksize - 1) {
+ did_zero = false;
err = ext4_block_zero_range(inode, byte_end - partial_end,
- partial_end + 1, did_zero, NULL);
+ partial_end + 1, &did_zero, NULL);
+ if (err)
+ return err;
+ if (did_zero)
+ *partial_zeroed |= EXT4_PARTIAL_ZERO_END;
+ }
return err;
}
@@ -4406,7 +4464,7 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
loff_t end = offset + length;
handle_t *handle;
unsigned int credits;
- bool partial_zeroed = false;
+ unsigned int partial_zeroed = 0;
int ret;
trace_ext4_punch_hole(inode, offset, length, 0);
@@ -5242,6 +5300,20 @@ void ext4_set_inode_mapping_order(struct inode *inode)
mapping_set_folio_order_range(inode->i_mapping, min_order, max_order);
}
+static int ext4_iget_match(struct inode *inode, u64 ino, void *data)
+{
+ if (inode->i_ino != ino)
+ return 0;
+ spin_lock(&inode->i_lock);
+ if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) {
+ spin_unlock(&inode->i_lock);
+ return -1;
+ }
+ __iget(inode);
+ spin_unlock(&inode->i_lock);
+ return 1;
+}
+
struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
ext4_iget_flags flags, const char *function,
unsigned int line)
@@ -5270,9 +5342,24 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
return ERR_PTR(-EFSCORRUPTED);
}
- inode = iget_locked(sb, ino);
- if (!inode)
- return ERR_PTR(-ENOMEM);
+ if (flags & EXT4_IGET_NOWAIT) {
+ inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL);
+ if (!inode)
+ return ERR_PTR(-ENOENT);
+
+ if (inode_state_read_once(inode) & I_NEW)
+ wait_on_new_inode(inode);
+
+ if (unlikely(inode_unhashed(inode))) {
+ iput(inode);
+ return ERR_PTR(-ENOENT);
+ }
+ } else {
+ inode = iget_locked(sb, ino);
+ if (!inode)
+ return ERR_PTR(-ENOMEM);
+ }
+
if (!(inode_state_read_once(inode) & I_NEW)) {
ret = check_igot_inode(inode, flags, function, line);
if (ret) {
@@ -6288,17 +6375,17 @@ static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
}
/*
- * Account for index blocks, block groups bitmaps and block group
- * descriptor blocks if modify datablocks and index blocks
- * worse case, the indexs blocks spread over different block groups
- *
- * If datablocks are discontiguous, they are possible to spread over
- * different block groups too. If they are contiguous, with flexbg,
- * they could still across block group boundary.
- *
- * Also account for superblock, inode, quota and xattr blocks
+ * Calculate number of credits needed in a transaction to:
+ * * Allocate data blocks from @alloc_extents different groups - note that
+ * with flexbg a single physical extent can span multiple groups but
+ * single mballoc request only returns extent within one group.
+ * * Allocate metatadata (extent tree blocks, indirect blocks) to store
+ * pointers to @pextents data extents having @lblocks in total.
+ * * Modify extent tree / indirect block tree, inode, superblock, quota
+ * tracking, xattr blocks
*/
-int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents)
+int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents,
+ int alloc_extents)
{
ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
int gdpblocks;
@@ -6315,7 +6402,7 @@ int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents)
* Now let's see how many group bitmaps and group descriptors need
* to account
*/
- groups = idxblocks + pextents;
+ groups = idxblocks + alloc_extents;
gdpblocks = groups;
if (groups > ngroups)
groups = ngroups;
@@ -6341,7 +6428,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks)
{
int ret;
- ret = ext4_meta_trans_blocks(inode, nrblocks, 1);
+ ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1);
/* Account for data blocks for journalled mode */
if (ext4_should_journal_data(inode))
ret += nrblocks;
@@ -6359,7 +6446,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks)
*/
int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
{
- return ext4_meta_trans_blocks(inode, nrblocks, 1);
+ return ext4_meta_trans_blocks(inode, nrblocks, 1, 1);
}
/*
@@ -6506,6 +6593,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode,
return -EOVERFLOW;
/*
+ * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize
+ * may move xattrs to external blocks and release ea_inodes via iput.
+ * When !SB_ACTIVE, iput triggers write_inode_now() which acquires
+ * s_writepages_rwsem, causing a deadlock with the caller's active
+ * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle).
+ */
+ if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE)))
+ return -EBUSY;
+
+ /*
* In nojournal mode, we can immediately attempt to expand
* the inode. When journaled, we first need to obtain extra
* buffer credits since we may write into the EA block
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
index ed1bd00e11cd..06171a11db12 100644
--- a/fs/ext4/mballoc.c
+++ b/fs/ext4/mballoc.c
@@ -2861,8 +2861,6 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group,
blk_start_plug(&plug);
while (nr-- > 0) {
- struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group,
- NULL);
struct ext4_group_info *grp = ext4_get_group_info(sb, group);
/*
@@ -2872,14 +2870,17 @@ ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group,
* prefetch once, so we avoid getblk() call, which can
* be expensive.
*/
- if (gdp && grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) &&
- EXT4_MB_GRP_NEED_INIT(grp) &&
- ext4_free_group_clusters(sb, gdp) > 0 ) {
- bh = ext4_read_block_bitmap_nowait(sb, group, true);
- if (!IS_ERR_OR_NULL(bh)) {
- if (!buffer_uptodate(bh) && cnt)
- (*cnt)++;
- brelse(bh);
+ if (grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) &&
+ EXT4_MB_GRP_NEED_INIT(grp)) {
+ struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, NULL);
+
+ if (gdp && ext4_free_group_clusters(sb, gdp) > 0) {
+ bh = ext4_read_block_bitmap_nowait(sb, group, true);
+ if (!IS_ERR_OR_NULL(bh)) {
+ if (!buffer_uptodate(bh) && cnt)
+ (*cnt)++;
+ brelse(bh);
+ }
}
}
if (++group >= ngroups)
diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c
index 477d43d7e294..5d60ef10fe11 100644
--- a/fs/ext4/migrate.c
+++ b/fs/ext4/migrate.c
@@ -464,6 +464,7 @@ int ext4_ext_migrate(struct inode *inode)
if (IS_ERR(tmp_inode)) {
retval = PTR_ERR(tmp_inode);
ext4_journal_stop(handle);
+ tmp_inode = NULL;
goto out_unlock;
}
/*
@@ -591,9 +592,9 @@ out_stop:
ext4_journal_stop(handle);
out_tmp_inode:
unlock_new_inode(tmp_inode);
- iput(tmp_inode);
out_unlock:
ext4_writepages_up_write(inode->i_sb, alloc_ctx);
+ iput(tmp_inode);
return retval;
}
diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
index 640a03ee02c7..a6386c1d237f 100644
--- a/fs/ext4/namei.c
+++ b/fs/ext4/namei.c
@@ -1467,6 +1467,8 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size,
/* this code is executed quadratically often */
/* do minimal checking `by hand' */
if (de->name + de->name_len <= dlimit &&
+ (!ext4_hash_in_dirent(dir) ||
+ (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) &&
ext4_match(dir, fname, de)) {
/* found a match - just to be sure, do
* a full check */
diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c
index 64ea47624233..b4675aa7ea96 100644
--- a/fs/ext4/orphan.c
+++ b/fs/ext4/orphan.c
@@ -4,6 +4,7 @@
#include <linux/fs.h>
#include <linux/quotaops.h>
#include <linux/buffer_head.h>
+#include <linux/string_choices.h>
#include "ext4.h"
#include "ext4_jbd2.h"
@@ -388,7 +389,7 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es)
struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info;
int inodes_per_ob = ext4_inodes_per_orphan_block(sb);
- if (!es->s_last_orphan && !oi->of_blocks) {
+ if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) {
ext4_debug("no orphan inodes to clean up\n");
return;
}
@@ -486,14 +487,12 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es)
}
}
-#define PLURAL(x) (x), ((x) == 1) ? "" : "s"
-
if (nr_orphans)
ext4_msg(sb, KERN_INFO, "%d orphan inode%s deleted",
- PLURAL(nr_orphans));
+ nr_orphans, str_plural(nr_orphans));
if (nr_truncates)
ext4_msg(sb, KERN_INFO, "%d truncate%s cleaned up",
- PLURAL(nr_truncates));
+ nr_truncates, str_plural(nr_truncates));
#ifdef CONFIG_QUOTA
/* Turn off quotas if they were enabled for orphan cleanup */
if (quota_update) {
@@ -572,6 +571,7 @@ int ext4_init_orphan_info(struct super_block *sb)
int i, j;
int ret;
int free;
+ int loaded = 0;
__le32 *bdata;
int inodes_per_ob = ext4_inodes_per_orphan_block(sb);
struct ext4_orphan_block_tail *ot;
@@ -613,6 +613,7 @@ int ext4_init_orphan_info(struct super_block *sb)
ret = -EIO;
goto out_free;
}
+ loaded++;
ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh);
if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) {
ext4_error(sb, "orphan file block %d: bad magic", i);
@@ -635,8 +636,10 @@ int ext4_init_orphan_info(struct super_block *sb)
iput(inode);
return 0;
out_free:
- for (i--; i >= 0; i--)
- brelse(oi->of_binfo[i].ob_bh);
+ while (loaded > 0) {
+ loaded--;
+ brelse(oi->of_binfo[loaded].ob_bh);
+ }
kvfree(oi->of_binfo);
out_put:
iput(inode);
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 4b6112e5d6c5..bca0dc87d0b7 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -1303,6 +1303,8 @@ static void ext4_put_super(struct super_block *sb)
&sb->s_uuid);
ext4_unregister_li_request(sb);
+ /* Drain deferred EA inode iputs while quota is still active. */
+ flush_delayed_work(&sbi->s_ea_inode_work);
ext4_quotas_off(sb, EXT4_MAXQUOTAS);
destroy_workqueue(sbi->rsv_conversion_wq);
@@ -1423,6 +1425,13 @@ static struct inode *ext4_alloc_inode(struct super_block *sb)
memset(&ei->i_dquot, 0, sizeof(ei->i_dquot));
#endif
ei->jinode = NULL;
+ /*
+ * Reinitialize xattr_sem every allocation because EA inodes
+ * share this space with i_ea_iput_node (via union) which may
+ * have overwritten the semaphore when the slab object was
+ * previously used as an EA inode.
+ */
+ init_rwsem(&ei->xattr_sem);
INIT_LIST_HEAD(&ei->i_rsv_conversion_list);
spin_lock_init(&ei->i_completed_io_lock);
ei->i_sync_tid = 0;
@@ -1489,7 +1498,6 @@ static void init_once(void *foo)
struct ext4_inode_info *ei = foo;
INIT_LIST_HEAD(&ei->i_orphan);
- init_rwsem(&ei->xattr_sem);
init_rwsem(&ei->i_data_sem);
inode_init_once(&ei->vfs_inode);
ext4_fc_init_inode(&ei->vfs_inode);
@@ -4483,8 +4491,9 @@ static int ext4_handle_clustersize(struct super_block *sb)
sbi->s_cluster_bits = 0;
}
sbi->s_clusters_per_group = le32_to_cpu(es->s_clusters_per_group);
- if (sbi->s_clusters_per_group > sb->s_blocksize * 8) {
- ext4_msg(sb, KERN_ERR, "#clusters per group too big: %lu",
+ if (sbi->s_clusters_per_group > sb->s_blocksize * 8 ||
+ sbi->s_clusters_per_group & 7) {
+ ext4_msg(sb, KERN_ERR, "invalid #clusters per group: %lu",
sbi->s_clusters_per_group);
return -EINVAL;
}
@@ -5316,8 +5325,10 @@ static int ext4_block_group_meta_init(struct super_block *sb, int silent)
return -EINVAL;
}
if (sbi->s_inodes_per_group < sbi->s_inodes_per_block ||
- sbi->s_inodes_per_group > sb->s_blocksize * 8) {
- ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n",
+ sbi->s_inodes_per_group > sb->s_blocksize * 8 ||
+ sbi->s_inodes_per_group & 7 ||
+ sbi->s_inodes_per_group % sbi->s_inodes_per_block) {
+ ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu",
sbi->s_inodes_per_group);
return -EINVAL;
}
@@ -5377,7 +5388,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb)
ext4_set_def_opts(sb, es);
sbi->s_resuid = make_kuid(&init_user_ns, ext4_get_resuid(es));
- sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resuid(es));
+ sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resgid(es));
sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ;
sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME;
sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME;
@@ -5505,6 +5516,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb)
ext4_has_feature_orphan_present(sb) ||
ext4_has_feature_journal_needs_recovery(sb));
+ ext4_init_ea_inode_work(sbi);
+
if (ext4_has_feature_mmp(sb) && !sb_rdonly(sb)) {
err = ext4_multi_mount_protect(sb, le64_to_cpu(es->s_mmp_block));
if (err)
@@ -5755,6 +5768,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb)
return 0;
failed_mount9:
+ /* Drain deferred EA inode iputs before quota shutdown */
+ flush_delayed_work(&sbi->s_ea_inode_work);
ext4_quotas_off(sb, EXT4_MAXQUOTAS);
failed_mount8: __maybe_unused
ext4_release_orphan_info(sb);
@@ -5775,6 +5790,8 @@ failed_mount4:
if (EXT4_SB(sb)->rsv_conversion_wq)
destroy_workqueue(EXT4_SB(sb)->rsv_conversion_wq);
failed_mount_wq:
+ /* Drain deferred EA inode iputs before freeing structures */
+ flush_delayed_work(&sbi->s_ea_inode_work);
ext4_xattr_destroy_cache(sbi->s_ea_inode_cache);
sbi->s_ea_inode_cache = NULL;
@@ -5785,6 +5802,8 @@ failed_mount_wq:
ext4_journal_destroy(sbi, sbi->s_journal);
}
failed_mount3a:
+ /* Drain deferred EA inode iputs from journal replay */
+ flush_delayed_work(&sbi->s_ea_inode_work);
ext4_es_unregister_shrinker(sbi);
failed_mount3:
/* flush s_sb_upd_work before sbi destroy */
@@ -6455,6 +6474,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait)
trace_ext4_sync_fs(sb, wait);
flush_workqueue(sbi->rsv_conversion_wq);
+ flush_delayed_work(&sbi->s_ea_inode_work);
/*
* Writeback quota in non-journalled quota case - journalled quota has
* no dirty dquots
diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c
index 982a1f831e22..5c310747b965 100644
--- a/fs/ext4/xattr.c
+++ b/fs/ext4/xattr.c
@@ -114,10 +114,6 @@ const struct xattr_handler * const ext4_xattr_handlers[] = {
#define EA_INODE_CACHE(inode) (((struct ext4_sb_info *) \
inode->i_sb->s_fs_info)->s_ea_inode_cache)
-static int
-ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array,
- struct inode *inode);
-
#ifdef CONFIG_LOCKDEP
void ext4_xattr_inode_set_class(struct inode *ea_inode)
{
@@ -464,6 +460,21 @@ static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino,
inode_unlock(inode);
}
+ /*
+ * Since this function resolves references from active xattr entries,
+ * the EA inode must be in active state (i_nlink=1, ref_count>0).
+ * i_nlink > 1, i_nlink == 0 (dangling reference), or ref_count == 0
+ * (inconsistent with an active entry) all indicate on-disk corruption.
+ */
+ if (inode->i_nlink != 1 || !ext4_xattr_inode_get_ref(inode)) {
+ ext4_error(parent->i_sb,
+ "EA inode %lu has unexpected i_nlink=%u ref_count=%llu",
+ ea_ino, inode->i_nlink,
+ ext4_xattr_inode_get_ref(inode));
+ ext4_put_ea_inode(inode);
+ return -EFSCORRUPTED;
+ }
+
*ea_inode = inode;
return 0;
}
@@ -567,7 +578,7 @@ ext4_xattr_inode_get(struct inode *inode, struct ext4_xattr_entry *entry,
ea_inode->i_ino, true /* reusable */);
}
out:
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
return err;
}
@@ -1104,10 +1115,10 @@ static int ext4_xattr_inode_inc_ref_all(handle_t *handle, struct inode *parent,
err = ext4_xattr_inode_inc_ref(handle, ea_inode);
if (err) {
ext4_warning_inode(ea_inode, "inc ref error %d", err);
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
goto cleanup;
}
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
}
return 0;
@@ -1133,7 +1144,7 @@ cleanup:
if (err)
ext4_warning_inode(ea_inode, "cleanup dec ref error %d",
err);
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
}
return saved_err;
}
@@ -1160,7 +1171,6 @@ static void
ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
struct buffer_head *bh,
struct ext4_xattr_entry *first, bool block_csum,
- struct ext4_xattr_inode_array **ea_inode_array,
int extra_credits, bool skip_quota)
{
struct inode *ea_inode;
@@ -1197,14 +1207,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
if (err)
continue;
- err = ext4_expand_inode_array(ea_inode_array, ea_inode);
- if (err) {
- ext4_warning_inode(ea_inode,
- "Expand inode array err=%d", err);
- iput(ea_inode);
- continue;
- }
-
err = ext4_journal_ensure_credits_fn(handle, credits, credits,
ext4_free_metadata_revoke_credits(parent->i_sb, 1),
ext4_xattr_restart_fn(handle, parent, bh, block_csum,
@@ -1212,6 +1214,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
if (err < 0) {
ext4_warning_inode(ea_inode, "Ensure credits err=%d",
err);
+ ext4_put_ea_inode(ea_inode);
continue;
}
if (err > 0) {
@@ -1221,6 +1224,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
ext4_warning_inode(ea_inode,
"Re-get write access err=%d",
err);
+ ext4_put_ea_inode(ea_inode);
continue;
}
}
@@ -1229,6 +1233,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
if (err) {
ext4_warning_inode(ea_inode, "ea_inode dec ref err=%d",
err);
+ ext4_put_ea_inode(ea_inode);
continue;
}
@@ -1245,6 +1250,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
entry->e_value_inum = 0;
entry->e_value_size = 0;
+ ext4_put_ea_inode(ea_inode);
dirty = true;
}
@@ -1271,7 +1277,6 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
static void
ext4_xattr_release_block(handle_t *handle, struct inode *inode,
struct buffer_head *bh,
- struct ext4_xattr_inode_array **ea_inode_array,
int extra_credits)
{
struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode);
@@ -1313,7 +1318,6 @@ retry_ref:
ext4_xattr_inode_dec_ref_all(handle, inode, bh,
BFIRST(bh),
true /* block_csum */,
- ea_inode_array,
extra_credits,
true /* skip_quota */);
ext4_free_blocks(handle, inode, bh, 0, 1,
@@ -1505,7 +1509,7 @@ static struct inode *ext4_xattr_inode_create(handle_t *handle,
if (ext4_xattr_inode_dec_ref(handle, ea_inode))
ext4_warning_inode(ea_inode,
"cleanup dec ref error %d", err);
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
return ERR_PTR(err);
}
@@ -1550,7 +1554,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value,
while (ce) {
ea_inode = ext4_iget(inode->i_sb, ce->e_value,
- EXT4_IGET_EA_INODE);
+ EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT);
if (IS_ERR(ea_inode))
goto next_entry;
ext4_xattr_inode_set_class(ea_inode);
@@ -1564,7 +1568,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value,
kvfree(ea_data);
return ea_inode;
}
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
next_entry:
ce = mb_cache_entry_find_next(ea_inode_cache, ce);
}
@@ -1615,7 +1619,7 @@ static struct inode *ext4_xattr_inode_lookup_create(handle_t *handle,
ea_inode->i_ino, true /* reusable */);
return ea_inode;
out_err:
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
ext4_xattr_inode_free_quota(inode, NULL, value_len);
return ERR_PTR(err);
}
@@ -1848,7 +1852,7 @@ update_hash:
ret = 0;
out:
- iput(old_ea_inode);
+ ext4_put_ea_inode(old_ea_inode);
return ret;
}
@@ -2010,7 +2014,7 @@ clone_block:
old_ea_inode_quota = le32_to_cpu(
s->here->e_value_size);
}
- iput(tmp_inode);
+ ext4_put_ea_inode(tmp_inode);
s->here->e_value_inum = 0;
s->here->e_value_size = 0;
@@ -2075,12 +2079,13 @@ inserted:
* stable so we can check the additional
* reference fits.
*/
- ref = le32_to_cpu(BHDR(new_bh)->h_refcount) + 1;
- if (ref > EXT4_XATTR_REFCOUNT_MAX) {
+ ref = le32_to_cpu(BHDR(new_bh)->h_refcount);
+ if (ref >= EXT4_XATTR_REFCOUNT_MAX) {
/*
* Undo everything and check mbcache
* again.
*/
+ clear_bit(MBE_REUSABLE_B, &ce->e_flags);
unlock_buffer(new_bh);
dquot_free_block(inode,
EXT4_C2B(EXT4_SB(sb),
@@ -2091,6 +2096,7 @@ inserted:
new_bh = NULL;
goto inserted;
}
+ ref++;
BHDR(new_bh)->h_refcount = cpu_to_le32(ref);
if (ref == EXT4_XATTR_REFCOUNT_MAX)
clear_bit(MBE_REUSABLE_B, &ce->e_flags);
@@ -2150,7 +2156,7 @@ getblk_failed:
ext4_warning_inode(ea_inode,
"dec ref error=%d",
error);
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
ea_inode = NULL;
}
@@ -2182,12 +2188,8 @@ getblk_failed:
/* Drop the previous xattr block. */
if (bs->bh && bs->bh != new_bh) {
- struct ext4_xattr_inode_array *ea_inode_array = NULL;
-
ext4_xattr_release_block(handle, inode, bs->bh,
- &ea_inode_array,
0 /* extra_credits */);
- ext4_xattr_inode_array_free(ea_inode_array);
}
error = 0;
@@ -2203,7 +2205,7 @@ cleanup:
ext4_xattr_inode_free_quota(inode, ea_inode,
i_size_read(ea_inode));
}
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
}
if (ce)
mb_cache_entry_put(ea_block_cache, ce);
@@ -2285,7 +2287,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode,
ext4_xattr_inode_free_quota(inode, ea_inode,
i_size_read(ea_inode));
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
}
return error;
}
@@ -2297,7 +2299,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode,
header->h_magic = cpu_to_le32(0);
ext4_clear_inode_state(inode, EXT4_STATE_XATTR);
}
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
return 0;
}
@@ -2839,6 +2841,7 @@ retry:
s_min_extra_isize) {
tried_min_extra_isize++;
new_extra_isize = s_min_extra_isize;
+ error = 0;
goto retry;
}
goto cleanup;
@@ -2863,46 +2866,6 @@ cleanup:
return error;
}
-#define EIA_INCR 16 /* must be 2^n */
-#define EIA_MASK (EIA_INCR - 1)
-
-/* Add the large xattr @inode into @ea_inode_array for deferred iput().
- * If @ea_inode_array is new or full it will be grown and the old
- * contents copied over.
- */
-static int
-ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array,
- struct inode *inode)
-{
- if (*ea_inode_array == NULL) {
- /*
- * Start with 15 inodes, so it fits into a power-of-two size.
- */
- (*ea_inode_array) = kmalloc_flex(**ea_inode_array, inodes,
- EIA_MASK, GFP_NOFS);
- if (*ea_inode_array == NULL)
- return -ENOMEM;
- (*ea_inode_array)->count = 0;
- } else if (((*ea_inode_array)->count & EIA_MASK) == EIA_MASK) {
- /* expand the array once all 15 + n * 16 slots are full */
- struct ext4_xattr_inode_array *new_array = NULL;
-
- new_array = kmalloc_flex(**ea_inode_array, inodes,
- (*ea_inode_array)->count + EIA_INCR,
- GFP_NOFS);
- if (new_array == NULL)
- return -ENOMEM;
- memcpy(new_array, *ea_inode_array,
- struct_size(*ea_inode_array, inodes,
- (*ea_inode_array)->count));
- kfree(*ea_inode_array);
- *ea_inode_array = new_array;
- }
- (*ea_inode_array)->count++;
- (*ea_inode_array)->inodes[(*ea_inode_array)->count - 1] = inode;
- return 0;
-}
-
/*
* ext4_xattr_delete_inode()
*
@@ -2913,7 +2876,6 @@ ext4_expand_inode_array(struct ext4_xattr_inode_array **ea_inode_array,
* references on xattr block and xattr inodes.
*/
int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode,
- struct ext4_xattr_inode_array **ea_inode_array,
int extra_credits)
{
struct buffer_head *bh = NULL;
@@ -2952,7 +2914,6 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode,
ext4_xattr_inode_dec_ref_all(handle, inode, iloc.bh,
IFIRST(header),
false /* block_csum */,
- ea_inode_array,
extra_credits,
false /* skip_quota */);
}
@@ -2986,12 +2947,12 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode,
continue;
ext4_xattr_inode_free_quota(inode, ea_inode,
le32_to_cpu(entry->e_value_size));
- iput(ea_inode);
+ ext4_put_ea_inode(ea_inode);
}
}
- ext4_xattr_release_block(handle, inode, bh, ea_inode_array,
+ ext4_xattr_release_block(handle, inode, bh,
extra_credits);
/*
* Update i_file_acl value in the same transaction that releases
@@ -3013,16 +2974,63 @@ cleanup:
return error;
}
-void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *ea_inode_array)
+/*
+ * Worker function for deferred EA inode iput. Processes all inodes queued
+ * on s_ea_inode_to_free in a context free of xattr_sem/jbd2 handle locks.
+ */
+static void ext4_ea_inode_work(struct work_struct *work)
{
- int idx;
+ struct ext4_sb_info *sbi = container_of(to_delayed_work(work),
+ struct ext4_sb_info,
+ s_ea_inode_work);
+ struct llist_node *node = llist_del_all(&sbi->s_ea_inode_to_free);
- if (ea_inode_array == NULL)
+ while (node) {
+ struct ext4_inode_info *ei = container_of(node,
+ struct ext4_inode_info, i_ea_iput_node);
+ node = node->next;
+ iput(&ei->vfs_inode);
+ }
+}
+
+/*
+ * Release a VFS reference on an EA inode. Must be used instead of iput()
+ * in any context where xattr_sem or a jbd2 handle is held.
+ *
+ * If this is not the last reference, drops it immediately via
+ * iput_if_not_last() with no further action needed.
+ *
+ * If this is the last reference, the inode is linked onto a per-sb
+ * llist via i_ea_iput_node (embedded in ext4_inode_info, sharing space
+ * with the unused xattr_sem) and a delayed worker performs the final
+ * iput() in a clean context.
+ *
+ * Note: while an inode is on s_ea_inode_to_free, the unconsumed i_count
+ * reference (still 1) keeps it in the inode cache, so any concurrent
+ * iget() bumps i_count to >= 2 and iput_if_not_last() will succeed.
+ * Nobody will add the inode a second time until ext4_ea_inode_work()
+ * drops that reference via iput().
+ */
+void ext4_put_ea_inode(struct inode *inode)
+{
+ if (!inode)
return;
+ WARN_ON_ONCE(!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL));
+ if (iput_if_not_last(inode))
+ return;
+ llist_add(&EXT4_I(inode)->i_ea_iput_node,
+ &EXT4_SB(inode->i_sb)->s_ea_inode_to_free);
+ /*
+ * Use a short delay to allow multiple EA inodes to accumulate,
+ * reducing workqueue wakeups when several are released together.
+ */
+ schedule_delayed_work(&EXT4_SB(inode->i_sb)->s_ea_inode_work, 1);
+}
- for (idx = 0; idx < ea_inode_array->count; ++idx)
- iput(ea_inode_array->inodes[idx]);
- kfree(ea_inode_array);
+void ext4_init_ea_inode_work(struct ext4_sb_info *sbi)
+{
+ init_llist_head(&sbi->s_ea_inode_to_free);
+ INIT_DELAYED_WORK(&sbi->s_ea_inode_work, ext4_ea_inode_work);
}
/*
diff --git a/fs/ext4/xattr.h b/fs/ext4/xattr.h
index 1fedf44d4fb6..821dc6a50e51 100644
--- a/fs/ext4/xattr.h
+++ b/fs/ext4/xattr.h
@@ -131,11 +131,6 @@ struct ext4_xattr_ibody_find {
struct ext4_iloc iloc;
};
-struct ext4_xattr_inode_array {
- unsigned int count;
- struct inode *inodes[] __counted_by(count);
-};
-
extern const struct xattr_handler ext4_xattr_user_handler;
extern const struct xattr_handler ext4_xattr_trusted_handler;
extern const struct xattr_handler ext4_xattr_security_handler;
@@ -187,9 +182,9 @@ extern int __ext4_xattr_set_credits(struct super_block *sb, struct inode *inode,
bool is_create);
extern int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode,
- struct ext4_xattr_inode_array **array,
int extra_credits);
-extern void ext4_xattr_inode_array_free(struct ext4_xattr_inode_array *array);
+extern void ext4_init_ea_inode_work(struct ext4_sb_info *sbi);
+extern void ext4_put_ea_inode(struct inode *inode);
extern int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize,
struct ext4_inode *raw_inode, handle_t *handle);
diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c
index 1508e2f54462..513273712010 100644
--- a/fs/jbd2/checkpoint.c
+++ b/fs/jbd2/checkpoint.c
@@ -358,15 +358,16 @@ int jbd2_cleanup_journal_tail(journal_t *journal)
/*
* journal_shrink_one_cp_list
*
- * Find all the written-back checkpoint buffers in the given list
- * and try to release them. If the whole transaction is released, set
- * the 'released' parameter. Return the number of released checkpointed
- * buffers.
+ * Find written-back checkpoint buffers in the given list and try to release
+ * them. If 'nr_to_scan' is set, scan at most that many buffers. If the whole
+ * transaction is released, set the 'released' parameter. Return the number of
+ * released checkpointed buffers.
*
* Called with j_list_lock held.
*/
static unsigned long journal_shrink_one_cp_list(struct journal_head *jh,
enum jbd2_shrink_type type,
+ unsigned long *nr_to_scan,
bool *released)
{
struct journal_head *last_jh;
@@ -375,13 +376,15 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh,
int ret;
*released = false;
- if (!jh)
+ if (!jh || (nr_to_scan && !*nr_to_scan))
return 0;
last_jh = jh->b_cpprev;
do {
jh = next_jh;
next_jh = jh->b_cpnext;
+ if (nr_to_scan)
+ (*nr_to_scan)--;
if (type == JBD2_SHRINK_DESTROY) {
ret = __jbd2_journal_remove_checkpoint(jh);
@@ -389,7 +392,7 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh,
ret = jbd2_journal_try_remove_checkpoint(jh);
if (ret < 0) {
if (type == JBD2_SHRINK_BUSY_SKIP)
- continue;
+ goto next;
break;
}
}
@@ -400,9 +403,10 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh,
break;
}
+next:
if (need_resched())
break;
- } while (jh != last_jh);
+ } while (jh != last_jh && (!nr_to_scan || *nr_to_scan));
return nr_freed;
}
@@ -424,7 +428,6 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal,
tid_t first_tid = 0, last_tid = 0, next_tid = 0;
tid_t tid = 0;
unsigned long nr_freed = 0;
- unsigned long freed;
bool first_set = false;
again:
@@ -457,10 +460,9 @@ again:
next_transaction = transaction->t_cpnext;
tid = transaction->t_tid;
- freed = journal_shrink_one_cp_list(transaction->t_checkpoint_list,
- JBD2_SHRINK_BUSY_SKIP, &released);
- nr_freed += freed;
- (*nr_to_scan) -= min(*nr_to_scan, freed);
+ nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_list,
+ JBD2_SHRINK_BUSY_SKIP,
+ nr_to_scan, &released);
if (*nr_to_scan == 0)
break;
if (need_resched() || spin_needbreak(&journal->j_list_lock))
@@ -516,7 +518,7 @@ void __jbd2_journal_clean_checkpoint_list(journal_t *journal,
transaction = next_transaction;
next_transaction = transaction->t_cpnext;
journal_shrink_one_cp_list(transaction->t_checkpoint_list,
- type, &released);
+ type, NULL, &released);
/*
* This function only frees up some memory if possible so we
* dont have an obligation to finish processing. Bail out if
diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c
index 09efa337649e..00f5a98f3d4f 100644
--- a/fs/jbd2/journal.c
+++ b/fs/jbd2/journal.c
@@ -94,6 +94,7 @@ EXPORT_SYMBOL(jbd2_journal_init_jbd_inode);
EXPORT_SYMBOL(jbd2_journal_release_jbd_inode);
EXPORT_SYMBOL(jbd2_journal_begin_ordered_truncate);
EXPORT_SYMBOL(jbd2_inode_cache);
+EXPORT_SYMBOL(jbd2_handle_cache);
#ifdef CONFIG_JBD2_DEBUG
void __jbd2_debug(int level, const char *file, const char *func,
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 8e9bc9dda0cb..072d8cd09a0b 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -2419,6 +2419,21 @@ static inline void super_set_sysfs_name_generic(struct super_block *sb, const ch
extern void ihold(struct inode * inode);
extern void iput(struct inode *);
void iput_not_last(struct inode *);
+
+/**
+ * iput_if_not_last - drop an inode reference only if it is not the last one
+ * @inode: inode to put
+ *
+ * Returns true if the reference was dropped, false if this was the last
+ * reference and the caller must arrange for final iput() in a safe context.
+ */
+static inline bool __must_check iput_if_not_last(struct inode *inode)
+{
+ VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode);
+ VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode);
+ return atomic_add_unless(&inode->i_count, -1, 1);
+}
+
int inode_update_time(struct inode *inode, enum fs_update_time type,
unsigned int flags);
int generic_update_time(struct inode *inode, enum fs_update_time type,
diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h
index b68561187e90..1b42fe47c26b 100644
--- a/include/linux/jbd2.h
+++ b/include/linux/jbd2.h
@@ -510,11 +510,12 @@ struct jbd2_journal_handle
int h_err;
/* Flags [no locking] */
- unsigned int h_sync: 1;
- unsigned int h_reserved: 1;
- unsigned int h_aborted: 1;
- unsigned int h_type: 8;
- unsigned int h_line_no: 16;
+ unsigned char h_sync: 1;
+ unsigned char h_reserved: 1;
+ unsigned char h_aborted: 1;
+ unsigned char h_invalid: 1;
+ unsigned char h_type;
+ unsigned short h_line_no;
unsigned long h_start_jiffies;
unsigned int h_requested_credits;