diff options
Diffstat (limited to 'fs/ext4/inode.c')
| -rw-r--r-- | fs/ext4/inode.c | 846 |
1 files changed, 566 insertions, 280 deletions
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 396dc3a5d16b..26f0f9714f03 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -29,7 +29,7 @@ #include <linux/string.h> #include <linux/buffer_head.h> #include <linux/writeback.h> -#include <linux/pagevec.h> +#include <linux/folio_batch.h> #include <linux/mpage.h> #include <linux/rmap.h> #include <linux/namei.h> @@ -128,6 +128,8 @@ void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, static inline int ext4_begin_ordered_truncate(struct inode *inode, loff_t new_size) { + struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode); + trace_ext4_begin_ordered_truncate(inode, new_size); /* * If jinode is zero, then we never opened the file for @@ -135,10 +137,10 @@ static inline int ext4_begin_ordered_truncate(struct inode *inode, * jbd2_journal_begin_ordered_truncate() since there's no * outstanding writes we need to flush. */ - if (!EXT4_I(inode)->jinode) + if (!jinode) return 0; return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), - EXT4_I(inode)->jinode, + jinode, new_size); } @@ -174,7 +176,6 @@ void ext4_evict_inode(struct inode *inode) * (xattr block freeing), bitmap, group descriptor (inode freeing) */ int extra_credits = 6; - struct ext4_xattr_inode_array *ea_inode_array = NULL; bool freeze_protected = false; trace_ext4_evict_inode(inode); @@ -184,8 +185,20 @@ void ext4_evict_inode(struct inode *inode) if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL) ext4_evict_ea_inode(inode); if (inode->i_nlink) { - truncate_inode_pages_final(&inode->i_data); + struct mapping_metadata_bhs *mmb; + + /* + * If there's dirty page will lead to data loss, user + * could see stale data. + */ + if (unlikely(!ext4_emergency_state(inode->i_sb) && + mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY))) + ext4_warning_inode(inode, "data will be lost"); + truncate_inode_pages_final(&inode->i_data); + mmb = ext4_i_metadata_bhs(inode); + if (mmb) + mmb_sync(mmb); goto no_delete; } @@ -252,6 +265,7 @@ void ext4_evict_inode(struct inode *inode) if (ext4_inode_is_fast_symlink(inode)) memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data)); inode->i_size = 0; + ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); err = ext4_mark_inode_dirty(handle, inode); if (err) { ext4_warning(inode->i_sb, @@ -262,15 +276,14 @@ void ext4_evict_inode(struct inode *inode) err = ext4_truncate(inode); if (err) { ext4_error_err(inode->i_sb, -err, - "couldn't truncate inode %lu (err %d)", + "couldn't truncate inode %llu (err %d)", inode->i_ino, err); goto stop_handle; } } /* Remove xattr references. */ - err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array, - extra_credits); + err = ext4_xattr_delete_inode(handle, inode, extra_credits); if (err) { ext4_warning(inode->i_sb, "xattr delete (err %d)", err); stop_handle: @@ -278,7 +291,6 @@ stop_handle: ext4_orphan_del(NULL, inode); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); goto no_delete; } @@ -308,7 +320,6 @@ stop_handle: ext4_journal_stop(handle); if (freeze_protected) sb_end_intwrite(inode->i_sb); - ext4_xattr_inode_array_free(ea_inode_array); return; no_delete: /* @@ -342,7 +353,7 @@ void ext4_da_update_reserve_space(struct inode *inode, spin_lock(&ei->i_block_reservation_lock); trace_ext4_da_update_reserve_space(inode, used, quota_claim); if (unlikely(used > ei->i_reserved_data_blocks)) { - ext4_warning(inode->i_sb, "%s: ino %lu, used %d " + ext4_warning(inode->i_sb, "%s: ino %llu, used %d " "with only %d reserved data blocks", __func__, inode->i_ino, used, ei->i_reserved_data_blocks); @@ -405,7 +416,10 @@ int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, KUNIT_STATIC_STUB_REDIRECT(ext4_issue_zeroout, inode, lblk, pblk, len); if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) - return fscrypt_zeroout_range(inode, lblk, pblk, len); + return fscrypt_zeroout_range(inode, + (loff_t)lblk << inode->i_blkbits, + pblk << (inode->i_blkbits - SECTOR_SHIFT), + (u64)len << inode->i_blkbits); ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS); if (ret > 0) @@ -475,7 +489,7 @@ static void ext4_map_blocks_es_recheck(handle_t *handle, if (es_map->m_lblk != map->m_lblk || es_map->m_flags != map->m_flags || es_map->m_pblk != map->m_pblk) { - printk("ES cache assertion failed for inode: %lu " + printk("ES cache assertion failed for inode: %llu " "es_cached ex [%d/%d/%llu/%x] != " "found ex [%d/%d/%llu/%x] retval %d flags %x\n", inode->i_ino, es_map->m_lblk, es_map->m_len, @@ -515,7 +529,7 @@ static int ext4_map_query_blocks_next_in_leaf(handle_t *handle, if (unlikely(retval != map2.m_len)) { ext4_warning(inode->i_sb, "ES len assertion failed for inode " - "%lu: retval %d != map->m_len %d", + "%llu: retval %d != map->m_len %d", inode->i_ino, retval, map2.m_len); WARN_ON(1); } @@ -563,7 +577,7 @@ int ext4_map_query_blocks(handle_t *handle, struct inode *inode, if (unlikely(retval != map->m_len)) { ext4_warning(inode->i_sb, "ES len assertion failed for inode " - "%lu: retval %d != map->m_len %d", + "%llu: retval %d != map->m_len %d", inode->i_ino, retval, map->m_len); WARN_ON(1); } @@ -630,7 +644,7 @@ int ext4_map_create_blocks(handle_t *handle, struct inode *inode, if (unlikely(retval != map->m_len)) { ext4_warning(inode->i_sb, - "ES len assertion failed for inode %lu: " + "ES len assertion failed for inode %llu: " "retval %d != map->m_len %d", inode->i_ino, retval, map->m_len); WARN_ON(1); @@ -937,7 +951,7 @@ int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, { int ret = 0; - ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n", + ext4_debug("ext4_get_block_unwritten: inode %llu, create flag %d\n", inode->i_ino, create); ret = _ext4_get_block(inode, iblock, bh_result, EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT); @@ -1167,6 +1181,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, int nr_wait = 0; int i; bool should_journal_data = ext4_should_journal_data(inode); + bool folio_uptodate = folio_test_uptodate(folio); BUG_ON(!folio_test_locked(folio)); BUG_ON(to > folio_size(folio)); @@ -1178,13 +1193,13 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, head = create_empty_buffers(folio, blocksize, 0); block = EXT4_PG_TO_LBLK(inode, folio->index); - for (bh = head, block_start = 0; bh != head || !block_start; + for (bh = head, block_start = 0; + block_start < to || (!folio_uptodate && bh != head); block++, block_start = block_end, bh = bh->b_this_page) { block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { - if (folio_test_uptodate(folio)) { + if (folio_uptodate) set_buffer_uptodate(bh); - } continue; } if (WARN_ON_ONCE(buffer_new(bh))) @@ -1205,7 +1220,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, if (should_journal_data) do_journal_get_write_access(handle, inode, bh); - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { /* * Unlike __block_write_begin() we leave * dirtying of new uptodate buffers to @@ -1222,7 +1237,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, continue; } } - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { set_buffer_uptodate(bh); continue; } @@ -1247,17 +1262,6 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, from, to); else folio_zero_new_buffers(folio, from, to); - } else if (fscrypt_inode_uses_fs_layer_crypto(inode)) { - for (i = 0; i < nr_wait; i++) { - int err2; - - err2 = fscrypt_decrypt_pagecache_blocks(folio, - blocksize, bh_offset(wait[i])); - if (err2) { - clear_buffer_uptodate(wait[i]); - err = err2; - } - } } return err; @@ -1287,6 +1291,8 @@ static int ext4_write_begin(const struct kiocb *iocb, if (unlikely(ret)) return ret; + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + trace_ext4_write_begin(inode, pos, len); /* * Reserve one block more for addition to orphan list in case @@ -1301,8 +1307,10 @@ static int ext4_write_begin(const struct kiocb *iocb, foliop); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } /* @@ -1420,9 +1428,6 @@ static int write_end_fn(handle_t *handle, struct inode *inode, /* * We need to pick up the new inode size which generic_commit_write gave us * `iocb` can be NULL - eg, when called from page_symlink(). - * - * ext4 never places buffers on inode->i_mapping->i_private_list. metadata - * buffers are managed internally. */ static int ext4_write_end(const struct kiocb *iocb, struct address_space *mapping, @@ -1438,8 +1443,7 @@ static int ext4_write_end(const struct kiocb *iocb, trace_ext4_write_end(inode, pos, len, copied); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1456,10 +1460,9 @@ static int ext4_write_end(const struct kiocb *iocb, folio_unlock(folio); folio_put(folio); - if (old_size < pos && !verity) { + if (old_size < pos && !verity) pagecache_isize_extended(inode, old_size, pos); - ext4_zero_partial_blocks(handle, inode, old_size, pos - old_size); - } + /* * Don't mark the inode dirty under folio lock. First, it unnecessarily * makes the holding time of folio lock longer. Second, it forces lock @@ -1549,7 +1552,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1574,10 +1577,8 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, folio_unlock(folio); folio_put(folio); - if (old_size < pos && !verity) { + if (old_size < pos && !verity) pagecache_isize_extended(inode, old_size, pos); - ext4_zero_partial_blocks(handle, inode, old_size, pos - old_size); - } if (size_changed) { ret2 = ext4_mark_inode_dirty(handle, inode); @@ -1659,7 +1660,7 @@ void ext4_da_release_space(struct inode *inode, int to_free) * harmless to return without any action. */ ext4_warning(inode->i_sb, "ext4_da_release_space: " - "ino %lu, to_free %d with only %d reserved " + "ino %llu, to_free %d with only %d reserved " "data blocks", inode->i_ino, to_free, ei->i_reserved_data_blocks); WARN_ON(1); @@ -1747,8 +1748,22 @@ static void mpage_release_unused_pages(struct mpage_da_data *mpd, BUG_ON(!folio_test_locked(folio)); BUG_ON(folio_test_writeback(folio)); if (invalidate) { - if (folio_mapped(folio)) + if (folio_mapped(folio)) { folio_clear_dirty_for_io(folio); + /* + * Unmap folio from page + * tables to prevent + * subsequent accesses through + * stale PTEs. This ensures + * future accesses trigger new + * page faults rather than + * reusing the invalidated + * folio. + */ + unmap_mapping_pages(folio->mapping, + folio->index, + folio_nr_pages(folio), false); + } block_invalidate_folio(folio, 0, folio_size(folio)); folio_clear_uptodate(folio); @@ -2051,11 +2066,10 @@ static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio) folio_unlock(folio); } -static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) +static void mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) { size_t len; loff_t size; - int err; WARN_ON_ONCE(folio_pos(folio) != mpd->start_pos); folio_clear_dirty_for_io(folio); @@ -2077,9 +2091,7 @@ static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) if (folio_pos(folio) + len > size && !ext4_verity_in_progress(mpd->inode)) len = size & (len - 1); - err = ext4_bio_write_folio(&mpd->io_submit, folio, len); - - return err; + ext4_bio_write_folio(&mpd->io_submit, folio, len); } #define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay)) @@ -2156,8 +2168,7 @@ static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk, * accumulated extent of buffers to map or add buffers in the page to the * extent of buffers to map. The function returns 1 if the caller can continue * by processing the next page, 0 if it should stop adding buffers to the - * extent to map because we cannot extend it anymore. It can also return value - * < 0 in case of error during IO submission. + * extent to map because we cannot extend it anymore. */ static int mpage_process_page_bufs(struct mpage_da_data *mpd, struct buffer_head *head, @@ -2165,7 +2176,6 @@ static int mpage_process_page_bufs(struct mpage_da_data *mpd, ext4_lblk_t lblk) { struct inode *inode = mpd->inode; - int err; ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1) >> inode->i_blkbits; @@ -2188,9 +2198,7 @@ static int mpage_process_page_bufs(struct mpage_da_data *mpd, } while (lblk++, (bh = bh->b_this_page) != head); /* So far everything mapped? Submit the page for IO. */ if (mpd->map.m_len == 0) { - err = mpage_submit_folio(mpd, head->b_folio); - if (err < 0) - return err; + mpage_submit_folio(mpd, head->b_folio); mpage_folio_done(mpd, head->b_folio); } if (lblk >= blocks) { @@ -2320,9 +2328,7 @@ static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd) if (err < 0 || map_bh) goto out; /* Page fully mapped - let IO run! */ - err = mpage_submit_folio(mpd, folio); - if (err < 0) - goto out; + mpage_submit_folio(mpd, folio); mpage_folio_done(mpd, folio); } folio_batch_release(&fbatch); @@ -2395,7 +2401,6 @@ static int mpage_submit_partial_folio(struct mpage_da_data *mpd) struct inode *inode = mpd->inode; struct folio *folio; loff_t pos; - int ret; folio = filemap_get_folio(inode->i_mapping, mpd->start_pos >> PAGE_SHIFT); @@ -2410,9 +2415,7 @@ static int mpage_submit_partial_folio(struct mpage_da_data *mpd) !folio_contains(folio, pos >> PAGE_SHIFT))) return -EINVAL; - ret = mpage_submit_folio(mpd, folio); - if (ret) - goto out; + mpage_submit_folio(mpd, folio); /* * Update start_pos to prevent this folio from being released in * mpage_release_unused_pages(), it will be reset to the aligned folio @@ -2421,10 +2424,9 @@ static int mpage_submit_partial_folio(struct mpage_da_data *mpd) * entire folio has finished processing. */ mpd->start_pos = pos; -out: folio_unlock(folio); folio_put(folio); - return ret; + return 0; } /* @@ -2491,7 +2493,7 @@ static int mpage_map_and_submit_extent(handle_t *handle, } ext4_msg(sb, KERN_CRIT, "Delayed block allocation failed for " - "inode %lu at logical offset %llu with" + "inode %llu at logical offset %llu with" " max blocks %u with error %d", inode->i_ino, (unsigned long long)map->m_lblk, @@ -2535,7 +2537,7 @@ update_disksize: err2 = ext4_mark_inode_dirty(handle, inode); if (err2) { ext4_error_err(inode->i_sb, -err2, - "Failed to mark inode %lu dirty", + "Failed to mark inode %llu dirty", inode->i_ino); } if (!err) @@ -2670,13 +2672,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * page is already under writeback and we are not doing * a data integrity writeback, skip the page */ - if (!folio_test_dirty(folio) || - (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || unlikely(folio->mapping != mapping)) { folio_unlock(folio); continue; } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } folio_wait_writeback(folio); BUG_ON(folio_test_writeback(folio)); @@ -2711,9 +2725,8 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * through a pin. */ if (!mpd->can_map) { - err = mpage_submit_folio(mpd, folio); - if (err < 0) - goto out; + mpage_submit_folio(mpd, folio); + err = 0; /* Pending dirtying of journalled data? */ if (folio_test_checked(folio)) { err = mpage_journal_page_buffers(handle, @@ -2828,10 +2841,10 @@ static int ext4_do_writepages(struct mpage_da_data *mpd) if (ext4_should_dioread_nolock(inode)) { int bpf = ext4_journal_blocks_per_folio(inode); /* - * We may need to convert up to one extent per block in - * the folio and we may dirty the inode. + * We may need to convert up to one extent per block in the + * folio. */ - rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf); + rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); } if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) @@ -2909,7 +2922,7 @@ retry: if (IS_ERR(handle)) { ret = PTR_ERR(handle); ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: " - "%ld pages, ino %lu; err %d", __func__, + "%ld pages, ino %llu; err %d", __func__, wbc->nr_to_write, inode->i_ino, ret); /* Release allocated io_end */ ext4_put_io_end(mpd->io_submit.io_end); @@ -3031,17 +3044,23 @@ static int ext4_writepages(struct address_space *mapping, int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode) { + loff_t range_start, range_end; struct writeback_control wbc = { .sync_mode = WB_SYNC_ALL, .nr_to_write = LONG_MAX, - .range_start = jinode->i_dirty_start, - .range_end = jinode->i_dirty_end, }; struct mpage_da_data mpd = { .inode = jinode->i_vfs_inode, .wbc = &wbc, .can_map = 0, }; + + if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end)) + return 0; + + wbc.range_start = range_start; + wbc.range_end = range_end; + return ext4_do_writepages(&mpd); } @@ -3128,11 +3147,13 @@ static int ext4_da_write_begin(const struct kiocb *iocb, if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { ret = ext4_generic_write_inline_data(mapping, inode, pos, len, - foliop, fsdata, true); + foliop, true); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } retry: @@ -3196,7 +3217,7 @@ static int ext4_da_do_write_end(struct address_space *mapping, struct inode *inode = mapping->host; loff_t old_size = inode->i_size; bool disksize_changed = false; - loff_t new_i_size, zero_len = 0; + loff_t new_i_size; handle_t *handle; if (unlikely(!folio_buffers(folio))) { @@ -3240,19 +3261,15 @@ static int ext4_da_do_write_end(struct address_space *mapping, folio_unlock(folio); folio_put(folio); - if (pos > old_size) { + if (pos > old_size) pagecache_isize_extended(inode, old_size, pos); - zero_len = pos - old_size; - } - if (!disksize_changed && !zero_len) + if (!disksize_changed) return copied; - handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); if (IS_ERR(handle)) return PTR_ERR(handle); - if (zero_len) - ext4_zero_partial_blocks(handle, inode, old_size, zero_len); ext4_mark_inode_dirty(handle, inode); ext4_journal_stop(handle); @@ -3265,17 +3282,15 @@ static int ext4_da_write_end(const struct kiocb *iocb, struct folio *folio, void *fsdata) { struct inode *inode = mapping->host; - int write_mode = (int)(unsigned long)fsdata; + unsigned long write_mode = (unsigned long)fsdata; - if (write_mode == FALL_BACK_TO_NONDELALLOC) + if (write_mode & FALL_BACK_TO_NONDELALLOC) return ext4_write_end(iocb, mapping, pos, len, copied, folio, fsdata); trace_ext4_da_write_end(inode, pos, len, copied); - if (write_mode != CONVERT_INLINE_DATA && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && - ext4_has_inline_data(inode)) + if (write_mode & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -3426,6 +3441,7 @@ static bool ext4_release_folio(struct folio *folio, gfp_t wait) static bool ext4_inode_datasync_dirty(struct inode *inode) { journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; + struct mapping_metadata_bhs *mmb; if (journal) { if (jbd2_transaction_committed(journal, @@ -3436,8 +3452,9 @@ static bool ext4_inode_datasync_dirty(struct inode *inode) return true; } + mmb = ext4_i_metadata_bhs(inode); /* Any metadata buffers to write? */ - if (!list_empty(&inode->i_mapping->i_private_list)) + if (mmb && mmb_has_buffers(mmb)) return true; return inode_state_read_once(inode) & I_DIRTY_DATASYNC; } @@ -3647,6 +3664,9 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, int ret, dio_credits, m_flags = 0, retries = 0; bool force_commit = false; + if (flags & IOMAP_NOWAIT) + return -EAGAIN; + /* * Trim the mapping request to the maximum value that we can map at * once for direct I/O. @@ -3668,8 +3688,8 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, return ret; if (map->m_len < orig_mlen) { map->m_len = orig_mlen; - dio_credits = ext4_meta_trans_blocks(inode, orig_mlen, - map->m_len); + dio_credits = ext4_meta_trans_blocks(inode, map->m_len, + map->m_len, 0); } else { dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); @@ -3745,7 +3765,7 @@ retry: } -static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned flags, struct iomap *iomap, struct iomap *srcmap) { int ret; @@ -3803,9 +3823,9 @@ static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return ret; out: /* - * When inline encryption is enabled, sometimes I/O to an encrypted file - * has to be broken up to guarantee DUN contiguity. Handle this by - * limiting the length of the mapping returned. + * Sometimes I/O to an encrypted file has to be broken up to guarantee + * DUN contiguity. Handle this by limiting the length of the mapping + * returned. */ map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); @@ -3824,8 +3844,10 @@ out: return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); + const struct iomap_ops ext4_iomap_ops = { - .iomap_begin = ext4_iomap_begin, + .iomap_next = ext4_iomap_next, }; static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, @@ -3879,8 +3901,10 @@ set_iomap: return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); + const struct iomap_ops ext4_iomap_report_ops = { - .iomap_begin = ext4_iomap_begin_report, + .iomap_next = ext4_iomap_next_report, }; /* @@ -4001,22 +4025,25 @@ void ext4_set_aops(struct inode *inode) * because it might have data in pagecache (eg, if called from ext4_zero_range, * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a * racing writeback can come later and flush the stale pagecache to disk. + * + * Return the loaded bh if it actually needs zeroing - in written, dirty + * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or + * a clean unwritten block). */ -static int __ext4_block_zero_page_range(handle_t *handle, - struct address_space *mapping, loff_t from, loff_t length) +static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) { unsigned int offset, blocksize, pos; ext4_lblk_t iblock; - struct inode *inode = mapping->host; + struct address_space *mapping = inode->i_mapping; struct buffer_head *bh; struct folio *folio; int err = 0; folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + FGP_WRITEBEGIN | FGP_ACCESSED, mapping_gfp_constraint(mapping, ~__GFP_FS)); if (IS_ERR(folio)) - return PTR_ERR(folio); + return ERR_CAST(folio); blocksize = inode->i_sb->s_blocksize; @@ -4040,8 +4067,15 @@ static int __ext4_block_zero_page_range(handle_t *handle, } if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "unmapped"); - ext4_get_block(inode, iblock, bh, 0); - /* unmapped? It's a hole - nothing to do */ + err = ext4_get_block(inode, iblock, bh, 0); + if (err < 0) + goto unlock; + /* + * It's a hole or a clean unwritten block - nothing to do. + * Note that a lookup-only get_block (without + * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean + * unwritten extents. + */ if (!buffer_mapped(bh)) { BUFFER_TRACE(bh, "still unmapped"); goto unlock; @@ -4056,59 +4090,93 @@ static int __ext4_block_zero_page_range(handle_t *handle, err = ext4_read_bh_lock(bh, 0, true); if (err) goto unlock; - if (fscrypt_inode_uses_fs_layer_crypto(inode)) { - /* We expect the key to be set. */ - BUG_ON(!fscrypt_has_encryption_key(inode)); - err = fscrypt_decrypt_pagecache_blocks(folio, - blocksize, - bh_offset(bh)); - if (err) { - clear_buffer_uptodate(bh); - goto unlock; - } - } } - if (ext4_should_journal_data(inode)) { - BUFFER_TRACE(bh, "get write access"); - err = ext4_journal_get_write_access(handle, inode->i_sb, bh, - EXT4_JTR_NONE); - if (err) - goto unlock; - } - folio_zero_range(folio, offset, length); + return bh; + +unlock: + folio_unlock(folio); + folio_put(folio); + return err ? ERR_PTR(err) : NULL; +} + +static int ext4_block_do_zero_range(struct inode *inode, loff_t from, + loff_t length, bool *did_zero, + bool *zero_written) +{ + struct buffer_head *bh; + struct folio *folio; + + bh = ext4_load_tail_bh(inode, from); + if (IS_ERR_OR_NULL(bh)) + return PTR_ERR_OR_ZERO(bh); + + folio = bh->b_folio; + folio_zero_range(folio, offset_in_folio(folio, from), length); BUFFER_TRACE(bh, "zeroed end of block"); - if (ext4_should_journal_data(inode)) { - err = ext4_dirty_journalled_data(handle, bh); - } else { - mark_buffer_dirty(bh); - /* - * Only the written block requires ordered data to prevent - * exposing stale data. - */ - if (!buffer_unwritten(bh) && !buffer_delay(bh) && - ext4_should_order_data(inode)) - err = ext4_jbd2_inode_add_write(handle, inode, from, - length); + mark_buffer_dirty(bh); + if (did_zero) + *did_zero = true; + if (zero_written && !buffer_unwritten(bh) && !buffer_delay(bh)) + *zero_written = true; + + folio_unlock(folio); + folio_put(folio); + return 0; +} + +static int ext4_block_journalled_zero_range(struct inode *inode, loff_t from, + loff_t length, bool *did_zero) +{ + struct buffer_head *bh; + struct folio *folio; + handle_t *handle; + int err; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + bh = ext4_load_tail_bh(inode, from); + if (IS_ERR_OR_NULL(bh)) { + err = PTR_ERR_OR_ZERO(bh); + goto out_handle; } + folio = bh->b_folio; -unlock: + BUFFER_TRACE(bh, "get write access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (err) + goto out; + + folio_zero_range(folio, offset_in_folio(folio, from), length); + BUFFER_TRACE(bh, "zeroed end of block"); + + err = ext4_dirty_journalled_data(handle, bh); + if (err) + goto out; + + if (did_zero) + *did_zero = true; +out: folio_unlock(folio); folio_put(folio); +out_handle: + ext4_journal_stop(handle); return err; } /* - * ext4_block_zero_page_range() zeros out a mapping of length 'length' - * starting from file offset 'from'. The range to be zero'd must - * be contained with in one block. If the specified range exceeds - * the end of the block it will be shortened to end of the block - * that corresponds to 'from' + * Zeros out a mapping of length 'length' starting from file offset + * 'from'. The range to be zero'd must be contained with in one block. + * If the specified range exceeds the end of the block it will be + * shortened to end of the block that corresponds to 'from'. */ -static int ext4_block_zero_page_range(handle_t *handle, - struct address_space *mapping, loff_t from, loff_t length) +static int ext4_block_zero_range(struct inode *inode, + loff_t from, loff_t length, bool *did_zero, + bool *zero_written) { - struct inode *inode = mapping->host; unsigned blocksize = inode->i_sb->s_blocksize; unsigned int max = blocksize - (from & (blocksize - 1)); @@ -4120,43 +4188,97 @@ static int ext4_block_zero_page_range(handle_t *handle, length = max; if (IS_DAX(inode)) { - return dax_zero_range(inode, from, length, NULL, + return dax_zero_range(inode, from, length, did_zero, &ext4_iomap_ops); + } else if (ext4_should_journal_data(inode)) { + return ext4_block_journalled_zero_range(inode, from, length, + did_zero); } - return __ext4_block_zero_page_range(handle, mapping, from, length); + return ext4_block_do_zero_range(inode, from, length, did_zero, + zero_written); } /* - * ext4_block_truncate_page() zeroes out a mapping from file offset `from' - * up to the end of the block which corresponds to `from'. - * This required during truncate. We need to physically zero the tail end - * of that block so it doesn't yield old data if the file is later grown. + * Zero out a mapping from file offset 'from' up to the end of the block + * which corresponds to 'from' or to the given 'end' inside this block. + * This required during truncate up and performing append writes. We need + * to physically zero the tail end of that block so it doesn't yield old + * data if the file is grown. */ -static int ext4_block_truncate_page(handle_t *handle, - struct address_space *mapping, loff_t from) +int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) { - unsigned length; - unsigned blocksize; - struct inode *inode = mapping->host; + unsigned int blocksize = i_blocksize(inode); + unsigned int offset; + loff_t length = end - from; + bool did_zero = false; + bool zero_written = false; + int err; + offset = from & (blocksize - 1); + if (!offset || from >= end) + return 0; + /* + * Inline data has no tail block to zero out. Note that a race with + * ext4_page_mkwrite() converting inline data to an extent without + * holding i_rwsem is safe, as that path zeroes the full block before + * copying in the inline data. + */ + if (ext4_has_inline_data(inode)) + return 0; /* If we are processing an encrypted inode during orphan list handling */ if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) return 0; - blocksize = i_blocksize(inode); - length = blocksize - (from & (blocksize - 1)); + if (length > blocksize - offset) + length = blocksize - offset; + + err = ext4_block_zero_range(inode, from, length, + &did_zero, &zero_written); + if (err) + return err; + /* + * It's necessary to order zeroed data before update i_disksize when + * truncating up or performing an append write, because there might be + * exposing stale on-disk data which may caused by concurrent post-EOF + * mmap write during folio writeback. + */ + if (ext4_should_order_data(inode) && + did_zero && zero_written && !IS_DAX(inode)) { + handle_t *handle; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + err = ext4_jbd2_inode_add_write(handle, inode, from, length); + ext4_journal_stop(handle); + if (err) + return err; + } - return ext4_block_zero_page_range(handle, mapping, from, length); + return 0; } -int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode, - loff_t lstart, loff_t length) +/* + * Zero out the unaligned head and tail of the [lstart, lstart+length) + * range. + * + * On return, @partial_zeroed records which edges actually got + * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if + * the head/tail block got actually partially zeroed (in written, dirty + * unwritten or delalloc state). Cleared if the head/tail block is a + * hole or a clean unwritten block, in which case there is nothing that + * needs zeroing. When the head and tail land in the same block, both + * bits are set together on a successful zeroing. + */ +int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, + unsigned int *partial_zeroed) { struct super_block *sb = inode->i_sb; - struct address_space *mapping = inode->i_mapping; unsigned partial_start, partial_end; ext4_fsblk_t start, end; loff_t byte_end = (lstart + length - 1); + bool did_zero = false; int err = 0; partial_start = lstart & (sb->s_blocksize - 1); @@ -4168,22 +4290,32 @@ int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode, /* Handle partial zero within the single block */ if (start == end && (partial_start || (partial_end != sb->s_blocksize - 1))) { - err = ext4_block_zero_page_range(handle, mapping, - lstart, length); + err = ext4_block_zero_range(inode, lstart, length, &did_zero, + NULL); + if (did_zero) + *partial_zeroed |= (EXT4_PARTIAL_ZERO_START | + EXT4_PARTIAL_ZERO_END); return err; } /* Handle partial zero out on the start of the range */ if (partial_start) { - err = ext4_block_zero_page_range(handle, mapping, - lstart, sb->s_blocksize); + err = ext4_block_zero_range(inode, lstart, sb->s_blocksize, + &did_zero, NULL); if (err) return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_START; } /* Handle partial zero out on the end of the range */ - if (partial_end != sb->s_blocksize - 1) - err = ext4_block_zero_page_range(handle, mapping, - byte_end - partial_end, - partial_end + 1); + if (partial_end != sb->s_blocksize - 1) { + did_zero = false; + err = ext4_block_zero_range(inode, byte_end - partial_end, + partial_end + 1, &did_zero, NULL); + if (err) + return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_END; + } return err; } @@ -4332,6 +4464,7 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) loff_t end = offset + length; handle_t *handle; unsigned int credits; + unsigned int partial_zeroed = 0; int ret; trace_ext4_punch_hole(inode, offset, length, 0); @@ -4358,17 +4491,6 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) end = max_end; length = end - offset; - /* - * Attach jinode to inode for jbd2 if we do any zeroing of partial - * block. - */ - if (!IS_ALIGNED(offset | end, sb->s_blocksize)) { - ret = ext4_inode_attach_jinode(inode); - if (ret < 0) - return ret; - } - - ret = ext4_update_disksize_before_punch(inode, offset, length); if (ret) return ret; @@ -4378,8 +4500,18 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) if (ret) return ret; + ret = ext4_zero_partial_blocks(inode, offset, length, &partial_zeroed); + if (ret) + return ret; + if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) { + ret = filemap_write_and_wait_range(inode->i_mapping, offset, + end - 1); + if (ret) + return ret; + } + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) - credits = ext4_chunk_trans_extent(inode, 2); + credits = ext4_chunk_trans_extent(inode, 0); else credits = ext4_blocks_for_truncate(inode); handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); @@ -4389,10 +4521,6 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) return ret; } - ret = ext4_zero_partial_blocks(handle, inode, offset, length); - if (ret) - goto out_handle; - /* If there are blocks to remove, do it */ start_lblk = EXT4_B_TO_LBLK(inode, offset); end_lblk = end >> inode->i_blkbits; @@ -4429,7 +4557,7 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) goto out_handle; ext4_update_inode_fsync_trans(handle, inode, 1); - if (IS_SYNC(inode)) + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) ext4_handle_sync(handle); out_handle: ext4_journal_stop(handle); @@ -4451,8 +4579,13 @@ int ext4_inode_attach_jinode(struct inode *inode) spin_unlock(&inode->i_lock); return -ENOMEM; } - ei->jinode = jinode; - jbd2_journal_init_jbd_inode(ei->jinode, inode); + jbd2_journal_init_jbd_inode(jinode, inode); + /* + * Publish ->jinode only after it is fully initialized so that + * readers never observe a partially initialized jbd2_inode. + */ + smp_wmb(); + WRITE_ONCE(ei->jinode, jinode); jinode = NULL; } spin_unlock(&inode->i_lock); @@ -4495,7 +4628,6 @@ int ext4_truncate(struct inode *inode) unsigned int credits; int err = 0, err2; handle_t *handle; - struct address_space *mapping = inode->i_mapping; /* * There is a possibility that we're either freeing the inode @@ -4525,6 +4657,11 @@ int ext4_truncate(struct inode *inode) err = ext4_inode_attach_jinode(inode); if (err) goto out_trace; + + /* Zero to the end of the block containing i_size */ + err = ext4_block_zero_eof(inode, inode->i_size, LLONG_MAX); + if (err) + goto out_trace; } if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) @@ -4538,9 +4675,6 @@ int ext4_truncate(struct inode *inode) goto out_trace; } - if (inode->i_size & (inode->i_sb->s_blocksize - 1)) - ext4_block_truncate_page(handle, mapping, inode->i_size); - /* * We add the inode to the orphan list, so that if this * truncate spans multiple transactions, and we crash, we will @@ -4922,6 +5056,57 @@ int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) return ret; } +/* + * ext4_get_inode_loc_noio() is a best-effort variant of ext4_get_inode_loc(). + * It looks up the inode table block in the buffer cache and returns -EAGAIN if + * the block is not present or not uptodate, without starting any I/O. + */ +int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc) +{ + struct super_block *sb = inode->i_sb; + struct ext4_group_desc *gdp; + struct buffer_head *bh; + ext4_fsblk_t block; + int inodes_per_block, inode_offset; + unsigned long ino = inode->i_ino; + + iloc->bh = NULL; + if (ino < EXT4_ROOT_INO || + ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)) + return -EFSCORRUPTED; + + iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); + if (!gdp) + return -EIO; + + /* Figure out the offset within the block group inode table. */ + inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; + inode_offset = ((ino - 1) % EXT4_INODES_PER_GROUP(sb)); + iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); + + block = ext4_inode_table(sb, gdp); + if (block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block) || + block >= ext4_blocks_count(EXT4_SB(sb)->s_es)) { + ext4_error(sb, + "Invalid inode table block %llu in block_group %u", + block, iloc->block_group); + return -EFSCORRUPTED; + } + block += inode_offset / inodes_per_block; + + bh = sb_find_get_block(sb, block); + if (!bh) + return -EAGAIN; + if (!ext4_buffer_uptodate(bh)) { + brelse(bh); + return -EAGAIN; + } + + iloc->bh = bh; + return 0; +} + int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, struct ext4_iloc *iloc) @@ -5115,6 +5300,20 @@ void ext4_set_inode_mapping_order(struct inode *inode) mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); } +static int ext4_iget_match(struct inode *inode, u64 ino, void *data) +{ + if (inode->i_ino != ino) + return 0; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { + spin_unlock(&inode->i_lock); + return -1; + } + __iget(inode); + spin_unlock(&inode->i_lock); + return 1; +} + struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ext4_iget_flags flags, const char *function, unsigned int line) @@ -5143,9 +5342,24 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, return ERR_PTR(-EFSCORRUPTED); } - inode = iget_locked(sb, ino); - if (!inode) - return ERR_PTR(-ENOMEM); + if (flags & EXT4_IGET_NOWAIT) { + inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); + if (!inode) + return ERR_PTR(-ENOENT); + + if (inode_state_read_once(inode) & I_NEW) + wait_on_new_inode(inode); + + if (unlikely(inode_unhashed(inode))) { + iput(inode); + return ERR_PTR(-ENOENT); + } + } else { + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + } + if (!(inode_state_read_once(inode) & I_NEW)) { ret = check_igot_inode(inode, flags, function, line); if (ret) { @@ -5401,18 +5615,36 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, inode->i_op = &ext4_encrypted_symlink_inode_operations; } else if (ext4_inode_is_fast_symlink(inode)) { inode->i_op = &ext4_fast_symlink_inode_operations; - if (inode->i_size == 0 || - inode->i_size >= sizeof(ei->i_data) || - strnlen((char *)ei->i_data, inode->i_size + 1) != - inode->i_size) { - ext4_error_inode(inode, function, line, 0, - "invalid fast symlink length %llu", - (unsigned long long)inode->i_size); - ret = -EFSCORRUPTED; - goto bad_inode; + + /* + * Orphan cleanup can see inodes with i_size == 0 + * and i_data uninitialized. Skip size checks in + * that case. This is safe because the first thing + * ext4_evict_inode() does for fast symlinks is + * clearing of i_data and i_size. + */ + if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) { + if (inode->i_nlink != 0) { + ext4_error_inode(inode, function, line, 0, + "invalid orphan symlink nlink %d", + inode->i_nlink); + ret = -EFSCORRUPTED; + goto bad_inode; + } + } else { + if (inode->i_size == 0 || + inode->i_size >= sizeof(ei->i_data) || + strnlen((char *)ei->i_data, inode->i_size + 1) != + inode->i_size) { + ext4_error_inode(inode, function, line, 0, + "invalid fast symlink length %llu", + (unsigned long long)inode->i_size); + ret = -EFSCORRUPTED; + goto bad_inode; + } + inode_set_cached_link(inode, (char *)ei->i_data, + inode->i_size); } - inode_set_cached_link(inode, (char *)ei->i_data, - inode->i_size); } else { inode->i_op = &ext4_symlink_inode_operations; } @@ -5622,6 +5854,10 @@ out_brelse: * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL * writeback. * + * For nojournal mode all the work is done in ext4_sync_inode_metadata() + * because inode content is already copied into raw inode buffer and inode + * is marked with I_METADATA_WRITEBACK. + * * Note that we are absolutely dependent upon all inode dirtiers doing the * right thing: they *must* call mark_inode_dirty() after dirtying info in * which we are interested. @@ -5647,42 +5883,54 @@ int ext4_write_inode(struct inode *inode, struct writeback_control *wbc) if (unlikely(err)) return err; - if (EXT4_SB(inode->i_sb)->s_journal) { - if (ext4_journal_current_handle()) { - ext4_debug("called recursively, non-PF_MEMALLOC!\n"); - dump_stack(); - return -EIO; - } + if (!EXT4_SB(inode->i_sb)->s_journal) + return 0; - /* - * No need to force transaction in WB_SYNC_NONE mode. Also - * ext4_sync_fs() will force the commit after everything is - * written. - */ - if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) - return 0; + if (ext4_journal_current_handle()) { + ext4_debug("called recursively, non-PF_MEMALLOC!\n"); + dump_stack(); + return -EIO; + } - err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, + /* + * No need to force transaction in WB_SYNC_NONE mode. Also + * ext4_sync_fs() will force the commit after everything is + * written. + */ + if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) + return 0; + + return ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, EXT4_I(inode)->i_sync_tid); - } else { - struct ext4_iloc iloc; +} + +int ext4_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc) +{ + struct ext4_iloc iloc; + struct mapping_metadata_bhs *mmb; + int err; - err = __ext4_get_inode_loc_noinmem(inode, &iloc); + /* We should only get here in nojournal mode */ + if (WARN_ON_ONCE(EXT4_SB(inode->i_sb)->s_journal)) + return -EFSCORRUPTED; + + err = __ext4_get_inode_loc_noinmem(inode, &iloc); + if (err) + return err; + mmb = READ_ONCE(EXT4_I(inode)->i_metadata_bhs); + if (mmb) { + err = mmb_sync(mmb); if (err) - return err; - /* - * sync(2) will flush the whole buffer cache. No need to do - * it here separately for each inode. - */ - if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) - sync_dirty_buffer(iloc.bh); - if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) { - ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO, - "IO error syncing inode"); - err = -EIO; - } - brelse(iloc.bh); + goto out; + } + sync_dirty_buffer(iloc.bh); + if (buffer_write_io_error(iloc.bh)) { + ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO, + "IO error syncing inode"); + err = -EIO; } +out: + brelse(iloc.bh); return err; } @@ -5849,6 +6097,18 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (attr->ia_size == inode->i_size) inc_ivers = false; + /* + * If file has inline data but new size exceeds inline capacity, + * convert to extent-based storage first to prevent inconsistent + * state (inline flag set but size exceeds inline capacity). + */ + if (ext4_has_inline_data(inode) && + attr->ia_size > EXT4_I(inode)->i_inline_size) { + error = ext4_convert_inline_data(inode); + if (error) + goto err_out; + } + if (shrink) { if (ext4_should_order_data(inode)) { error = ext4_begin_ordered_truncate(inode, @@ -5880,15 +6140,6 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, goto out_mmap_sem; } - handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); - if (IS_ERR(handle)) { - error = PTR_ERR(handle); - goto out_mmap_sem; - } - if (ext4_handle_valid(handle) && shrink) { - error = ext4_orphan_add(handle, inode); - orphan = 1; - } /* * Update c/mtime and tail zero the EOF folio on * truncate up. ext4_truncate() handles the shrink case @@ -5897,9 +6148,22 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (!shrink) { inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - if (oldsize & (inode->i_sb->s_blocksize - 1)) - ext4_block_truncate_page(handle, - inode->i_mapping, oldsize); + if (oldsize & (inode->i_sb->s_blocksize - 1)) { + error = ext4_block_zero_eof(inode, + oldsize, LLONG_MAX); + if (error) + goto out_mmap_sem; + } + } + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); + if (IS_ERR(handle)) { + error = PTR_ERR(handle); + goto out_mmap_sem; + } + if (ext4_handle_valid(handle) && shrink) { + error = ext4_orphan_add(handle, inode); + orphan = 1; } if (shrink) @@ -5994,11 +6258,8 @@ u32 ext4_dio_alignment(struct inode *inode) return 0; if (ext4_has_inline_data(inode)) return 0; - if (IS_ENCRYPTED(inode)) { - if (!fscrypt_dio_supported(inode)) - return 0; + if (IS_ENCRYPTED(inode)) return i_blocksize(inode); - } return 1; /* use the iomap defaults */ } @@ -6017,11 +6278,7 @@ int ext4_getattr(struct mnt_idmap *idmap, const struct path *path, stat->btime.tv_nsec = ei->i_crtime.tv_nsec; } - /* - * Return the DIO alignment restrictions if requested. We only return - * this information when requested, since on encrypted files it might - * take a fair bit of work to get if the file wasn't opened recently. - */ + /* Return the DIO alignment restrictions if requested. */ if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { u32 dio_align = ext4_dio_alignment(inode); @@ -6118,17 +6375,17 @@ static int ext4_index_trans_blocks(struct inode *inode, int lblocks, } /* - * Account for index blocks, block groups bitmaps and block group - * descriptor blocks if modify datablocks and index blocks - * worse case, the indexs blocks spread over different block groups - * - * If datablocks are discontiguous, they are possible to spread over - * different block groups too. If they are contiguous, with flexbg, - * they could still across block group boundary. - * - * Also account for superblock, inode, quota and xattr blocks + * Calculate number of credits needed in a transaction to: + * * Allocate data blocks from @alloc_extents different groups - note that + * with flexbg a single physical extent can span multiple groups but + * single mballoc request only returns extent within one group. + * * Allocate metatadata (extent tree blocks, indirect blocks) to store + * pointers to @pextents data extents having @lblocks in total. + * * Modify extent tree / indirect block tree, inode, superblock, quota + * tracking, xattr blocks */ -int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) +int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, + int alloc_extents) { ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); int gdpblocks; @@ -6145,7 +6402,7 @@ int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) * Now let's see how many group bitmaps and group descriptors need * to account */ - groups = idxblocks + pextents; + groups = idxblocks + alloc_extents; gdpblocks = groups; if (groups > ngroups) groups = ngroups; @@ -6171,7 +6428,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) { int ret; - ret = ext4_meta_trans_blocks(inode, nrblocks, 1); + ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); /* Account for data blocks for journalled mode */ if (ext4_should_journal_data(inode)) ret += nrblocks; @@ -6189,7 +6446,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) */ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) { - return ext4_meta_trans_blocks(inode, nrblocks, 1); + return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); } /* @@ -6199,9 +6456,10 @@ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) int ext4_mark_iloc_dirty(handle_t *handle, struct inode *inode, struct ext4_iloc *iloc) { + struct super_block *sb = inode->i_sb; int err = 0; - err = ext4_emergency_state(inode->i_sb); + err = ext4_emergency_state(sb); if (unlikely(err)) { put_bh(iloc->bh); return err; @@ -6214,6 +6472,24 @@ int ext4_mark_iloc_dirty(handle_t *handle, /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */ err = ext4_do_update_inode(handle, inode, iloc); put_bh(iloc->bh); + /* + * Mark that there's metadata writeout pending for the inode so that it + * gets properly flushed on fsync(2) and similar. We don't bother for + * fastcommit replay as that flushes the whole bdev afterwards anyway. + * It is faster this way and we avoid entering fs writeback paths which + * aren't fully initialized yet. + */ + if (!ext4_handle_valid(handle) && + !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) { + /* + * Inode didn't need to go through dirtying, make sure it is + * attached to wb so that writeback can handle it. + */ + spin_lock(&inode->i_lock); + inode_attach_wb(inode, NULL); + spin_unlock(&inode->i_lock); + set_inode_metadata_writeback(inode); + } return err; } @@ -6322,6 +6598,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode, return -EOVERFLOW; /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + + /* * In nojournal mode, we can immediately attempt to expand * the inode. When journaled, we first need to obtain extra * buffer credits since we may write into the EA block |