diff options
Diffstat (limited to 'fs/buffer.c')
| -rw-r--r-- | fs/buffer.c | 912 |
1 files changed, 377 insertions, 535 deletions
diff --git a/fs/buffer.c b/fs/buffer.c index 22b43642ba57..ed966fa73b1b 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -45,7 +45,7 @@ #include <linux/bitops.h> #include <linux/mpage.h> #include <linux/bit_spinlock.h> -#include <linux/pagevec.h> +#include <linux/folio_batch.h> #include <linux/sched/mm.h> #include <trace/events/block.h> #include <linux/fscrypt.h> @@ -54,10 +54,6 @@ #include "internal.h" -static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint hint, struct writeback_control *wbc); - #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) inline void touch_buffer(struct buffer_head *bh) @@ -75,9 +71,7 @@ EXPORT_SYMBOL(__lock_buffer); void unlock_buffer(struct buffer_head *bh) { - clear_bit_unlock(BH_Lock, &bh->b_state); - smp_mb__after_atomic(); - wake_up_bit(&bh->b_state, BH_Lock); + clear_and_wake_up_bit(BH_Lock, &bh->b_state); } EXPORT_SYMBOL(unlock_buffer); @@ -133,15 +127,43 @@ static void buffer_io_error(struct buffer_head *bh, char *msg) bh->b_bdev, (unsigned long long)bh->b_blocknr, msg); } -/* - * End-of-IO handler helper function which does not touch the bh after - * unlocking it. - * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but - * a race there is benign: unlock_buffer() only use the bh's address for - * hashing after unlocking the buffer, so it doesn't actually touch the bh - * itself. +/** + * bio_endio_bh - Discard the bio used to submit a buffer. + * @bio: The bio. + * @bhp: Where to return the buffer_head. + * + * Call this in your bio_end_io handler to retrieve the buffer_head + * submitted in bh_submit(). If you did not call bh_submit(), do not + * call this function; it will return garbage. + * + * This function consumes the bio refcount which will probably free the + * bio. + * + * Return: True if the I/O succeeded. */ -static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) +bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp) +{ + bool success = bio->bi_status == BLK_STS_OK; + struct buffer_head *bh = bio->bi_private; + + if (unlikely(bio_flagged(bio, BIO_QUIET))) + set_bit(BH_Quiet, &bh->b_state); + bio_put(bio); + + *bhp = bh; + return success; +} +EXPORT_SYMBOL(bio_endio_bh); + +/** + * end_buffer_read_sync - Handle buffer reads finishing + * @bh: The buffer. + * @uptodate: True if the read was successful. + * + * If a buffer is read through a mechanism that isn't bh_submit(), you + * can call this function to finish the read. + */ +void end_buffer_read_sync(struct buffer_head *bh, int uptodate) { if (uptodate) { set_buffer_uptodate(bh); @@ -151,21 +173,36 @@ static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) } unlock_buffer(bh); } +EXPORT_SYMBOL(end_buffer_read_sync); -/* - * Default synchronous end-of-IO handler.. Just mark it up-to-date and - * unlock the buffer. +/** + * bh_end_read - I/O end handler for reads + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're reading into the buffer, + * unless you need your own special I/O end handler. */ -void end_buffer_read_sync(struct buffer_head *bh, int uptodate) +void bh_end_read(struct bio *bio) { - put_bh(bh); - __end_buffer_read_notouch(bh, uptodate); + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + end_buffer_read_sync(bh, uptodate); } -EXPORT_SYMBOL(end_buffer_read_sync); +EXPORT_SYMBOL(bh_end_read); -void end_buffer_write_sync(struct buffer_head *bh, int uptodate) +/** + * bh_end_write - I/O end handler for writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're writing from the buffer, + * unless you need your own special I/O end handler. + */ +void bh_end_write(struct bio *bio) { - if (uptodate) { + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); + + if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost sync page write"); @@ -173,9 +210,8 @@ void end_buffer_write_sync(struct buffer_head *bh, int uptodate) clear_buffer_uptodate(bh); } unlock_buffer(bh); - put_bh(bh); } -EXPORT_SYMBOL(end_buffer_write_sync); +EXPORT_SYMBOL(bh_end_write); static struct buffer_head * __find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic) @@ -300,7 +336,7 @@ still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags); } -struct postprocess_bh_ctx { +struct verify_bh_ctx { struct work_struct work; struct buffer_head *bh; struct fsverity_info *vi; @@ -308,8 +344,8 @@ struct postprocess_bh_ctx { static void verify_bh(struct work_struct *work) { - struct postprocess_bh_ctx *ctx = - container_of(work, struct postprocess_bh_ctx, work); + struct verify_bh_ctx *ctx = + container_of(work, struct verify_bh_ctx, work); struct buffer_head *bh = ctx->bh; bool valid; @@ -319,70 +355,50 @@ static void verify_bh(struct work_struct *work) kfree(ctx); } -static void decrypt_bh(struct work_struct *work) -{ - struct postprocess_bh_ctx *ctx = - container_of(work, struct postprocess_bh_ctx, work); - struct buffer_head *bh = ctx->bh; - int err; - - err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size, - bh_offset(bh)); - if (err == 0 && ctx->vi) { - /* - * We use different work queues for decryption and for verity - * because verity may require reading metadata pages that need - * decryption, and we shouldn't recurse to the same workqueue. - */ - INIT_WORK(&ctx->work, verify_bh); - fsverity_enqueue_verify_work(&ctx->work); - return; - } - end_buffer_async_read(bh, err == 0); - kfree(ctx); -} - /* - * I/O completion handler for block_read_full_folio() - pages + * I/O completion handler for block_read_full_folio() - folios * which come unlocked at the end of I/O. */ -static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) +static void bh_end_async_read(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); struct inode *inode = bh->b_folio->mapping->host; - bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode); struct fsverity_info *vi = NULL; /* needed by ext4 */ if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) vi = fsverity_get_info(inode); - /* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */ - if (uptodate && (decrypt || vi)) { - struct postprocess_bh_ctx *ctx = kmalloc_obj(*ctx, GFP_ATOMIC); + /* Verify (with fsverity) if needed. */ + if (vi && uptodate) { + struct verify_bh_ctx *ctx = kmalloc_obj(*ctx, GFP_ATOMIC); if (ctx) { ctx->bh = bh; ctx->vi = vi; - if (decrypt) { - INIT_WORK(&ctx->work, decrypt_bh); - fscrypt_enqueue_decrypt_work(&ctx->work); - } else { - INIT_WORK(&ctx->work, verify_bh); - fsverity_enqueue_verify_work(&ctx->work); - } + INIT_WORK(&ctx->work, verify_bh); + fsverity_enqueue_verify_work(&ctx->work); return; } - uptodate = 0; + uptodate = false; } end_buffer_async_read(bh, uptodate); } -/* - * Completion handler for block_write_full_folio() - folios which are unlocked - * during I/O, and which have the writeback flag cleared upon I/O completion. +/** + * bh_end_async_write - I/O end handler for async folio writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're doing the equivalent of + * block_write_full_folio(). That is, the folio is unlocked, and will + * have its writeback flag cleared once all async write buffers have + * completed. */ -static void end_buffer_async_write(struct buffer_head *bh, int uptodate) +void bh_end_async_write(struct bio *bio) { + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); unsigned long flags; struct buffer_head *first; struct buffer_head *tmp; @@ -391,7 +407,7 @@ static void end_buffer_async_write(struct buffer_head *bh, int uptodate) BUG_ON(!buffer_async_write(bh)); folio = bh->b_folio; - if (uptodate) { + if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost async page write"); @@ -419,46 +435,7 @@ static void end_buffer_async_write(struct buffer_head *bh, int uptodate) still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags); } - -/* - * If a page's buffers are under async readin (end_buffer_async_read - * completion) then there is a possibility that another thread of - * control could lock one of the buffers after it has completed - * but while some of the other buffers have not completed. This - * locked buffer would confuse end_buffer_async_read() into not unlocking - * the page. So the absence of BH_Async_Read tells end_buffer_async_read() - * that this buffer is not under async I/O. - * - * The page comes unlocked when it has no locked buffer_async buffers - * left. - * - * PageLocked prevents anyone starting new async I/O reads any of - * the buffers. - * - * PageWriteback is used to prevent simultaneous writeout of the same - * page. - * - * PageLocked prevents anyone from starting writeback of a page which is - * under read I/O (PageWriteback is only ever set against a locked page). - */ -static void mark_buffer_async_read(struct buffer_head *bh) -{ - bh->b_end_io = end_buffer_async_read_io; - set_buffer_async_read(bh); -} - -static void mark_buffer_async_write_endio(struct buffer_head *bh, - bh_end_io_t *handler) -{ - bh->b_end_io = handler; - set_buffer_async_write(bh); -} - -void mark_buffer_async_write(struct buffer_head *bh) -{ - mark_buffer_async_write_endio(bh, end_buffer_async_write); -} -EXPORT_SYMBOL(mark_buffer_async_write); +EXPORT_SYMBOL(bh_end_async_write); /* @@ -468,190 +445,159 @@ EXPORT_SYMBOL(mark_buffer_async_write); * a successful fsync(). For example, ext2 indirect blocks need to be * written back and waited upon before fsync() returns. * - * The functions mark_buffer_dirty_inode(), fsync_inode_buffers(), - * inode_has_buffers() and invalidate_inode_buffers() are provided for the - * management of a list of dependent buffers at ->i_mapping->i_private_list. - * - * Locking is a little subtle: try_to_free_buffers() will remove buffers - * from their controlling inode's queue when they are being freed. But - * try_to_free_buffers() will be operating against the *blockdev* mapping - * at the time, not against the S_ISREG file which depends on those buffers. - * So the locking for i_private_list is via the i_private_lock in the address_space - * which backs the buffers. Which is different from the address_space - * against which the buffers are listed. So for a particular address_space, - * mapping->i_private_lock does *not* protect mapping->i_private_list! In fact, - * mapping->i_private_list will always be protected by the backing blockdev's - * ->i_private_lock. - * - * Which introduces a requirement: all buffers on an address_space's - * ->i_private_list must be from the same address_space: the blockdev's. - * - * address_spaces which do not place buffers at ->i_private_list via these - * utility functions are free to use i_private_lock and i_private_list for - * whatever they want. The only requirement is that list_empty(i_private_list) - * be true at clear_inode() time. - * - * FIXME: clear_inode should not call invalidate_inode_buffers(). The - * filesystems should do that. invalidate_inode_buffers() should just go - * BUG_ON(!list_empty). - * - * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should - * take an address_space, not an inode. And it should be called - * mark_buffer_dirty_fsync() to clearly define why those buffers are being - * queued up. - * - * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the - * list if it is already on a list. Because if the buffer is on a list, - * it *must* already be on the right one. If not, the filesystem is being - * silly. This will save a ton of locking. But first we have to ensure - * that buffers are taken *off* the old inode's list when they are freed - * (presumably in truncate). That requires careful auditing of all - * filesystems (do it inside bforget()). It could also be done by bringing - * b_inode back. + * The functions mmb_mark_buffer_dirty(), mmb_sync(), mmb_has_buffers() + * and mmb_invalidate() are provided for the management of a list of dependent + * buffers in mapping_metadata_bhs struct. + * + * The locking is a little subtle: The list of buffer heads is protected by + * the lock in mapping_metadata_bhs so functions coming from bdev mapping + * (such as try_to_free_buffers()) need to safely get to mapping_metadata_bhs + * using RCU, grab the lock, verify we didn't race with somebody detaching the + * bh / moving it to different inode and only then proceeding. */ -/* - * The buffer's backing address_space's i_private_lock must be held - */ -static void __remove_assoc_queue(struct buffer_head *bh) +void mmb_init(struct mapping_metadata_bhs *mmb, struct address_space *mapping) { - list_del_init(&bh->b_assoc_buffers); - WARN_ON(!bh->b_assoc_map); - bh->b_assoc_map = NULL; + spin_lock_init(&mmb->lock); + INIT_LIST_HEAD(&mmb->list); + mmb->mapping = mapping; } +EXPORT_SYMBOL(mmb_init); -int inode_has_buffers(struct inode *inode) +static void __remove_assoc_queue(struct mapping_metadata_bhs *mmb, + struct buffer_head *bh) { - return !list_empty(&inode->i_data.i_private_list); + lockdep_assert_held(&mmb->lock); + list_del_init(&bh->b_assoc_buffers); + WARN_ON(!bh->b_mmb); + bh->b_mmb = NULL; } -/* - * osync is designed to support O_SYNC io. It waits synchronously for - * all already-submitted IO to complete, but does not queue any new - * writes to the disk. - * - * To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer - * as you dirty the buffers, and then use osync_inode_buffers to wait for - * completion. Any other dirty buffers which are not yet queued for - * write will not be flushed to disk by the osync. - */ -static int osync_buffers_list(spinlock_t *lock, struct list_head *list) +static void remove_assoc_queue(struct buffer_head *bh) { - struct buffer_head *bh; - struct list_head *p; - int err = 0; + struct mapping_metadata_bhs *mmb; - spin_lock(lock); -repeat: - list_for_each_prev(p, list) { - bh = BH_ENTRY(p); - if (buffer_locked(bh)) { - get_bh(bh); - spin_unlock(lock); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) - err = -EIO; - brelse(bh); - spin_lock(lock); - goto repeat; + /* + * The locking dance is ugly here. We need to acquire the lock + * protecting the metadata bh list while possibly racing with bh + * being removed from the list or moved to a different one. We + * use RCU to pin mapping_metadata_bhs in memory to + * opportunistically acquire the lock and then recheck the bh + * didn't move under us. + */ + while (bh->b_mmb) { + rcu_read_lock(); + mmb = READ_ONCE(bh->b_mmb); + if (mmb) { + spin_lock(&mmb->lock); + if (bh->b_mmb == mmb) + __remove_assoc_queue(mmb, bh); + spin_unlock(&mmb->lock); } + rcu_read_unlock(); } - spin_unlock(lock); - return err; } -/** - * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers - * @mapping: the mapping which wants those buffers written - * - * Starts I/O against the buffers at mapping->i_private_list, and waits upon - * that I/O. - * - * Basically, this is a convenience function for fsync(). - * @mapping is a file or directory which needs those buffers to be written for - * a successful fsync(). - */ -int sync_mapping_buffers(struct address_space *mapping) +bool mmb_has_buffers(struct mapping_metadata_bhs *mmb) { - struct address_space *buffer_mapping = mapping->i_private_data; - - if (buffer_mapping == NULL || list_empty(&mapping->i_private_list)) - return 0; - - return fsync_buffers_list(&buffer_mapping->i_private_lock, - &mapping->i_private_list); + return !list_empty(&mmb->list); } -EXPORT_SYMBOL(sync_mapping_buffers); +EXPORT_SYMBOL_GPL(mmb_has_buffers); /** - * generic_buffers_fsync_noflush - generic buffer fsync implementation - * for simple filesystems with no inode lock + * mmb_sync - write out & wait upon all buffers in a list + * @mmb: the list of buffers to write * - * @file: file to synchronize - * @start: start offset in bytes - * @end: end offset in bytes (inclusive) - * @datasync: only synchronize essential metadata if true + * Starts I/O against the buffers in the given list and waits upon + * that I/O. Basically, this is a convenience function for fsync(). @mmb is + * for a file or directory which needs those buffers to be written for a + * successful fsync(). + * + * We have conflicting pressures: we want to make sure that all + * initially dirty buffers get waited on, but that any subsequently + * dirtied buffers don't. After all, we don't want fsync to last + * forever if somebody is actively writing to the file. * - * This is a generic implementation of the fsync method for simple - * filesystems which track all non-inode metadata in the buffers list - * hanging off the address_space structure. + * Do this in two main stages: first we copy dirty buffers to a + * temporary inode list, queueing the writes as we go. Then we clean + * up, waiting for those writes to complete. mark_buffer_dirty_inode() + * doesn't touch b_assoc_buffers list if b_mmb is not NULL so we are sure the + * buffer stays on our list until IO completes (at which point it can be + * reaped). */ -int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end, - bool datasync) +int mmb_sync(struct mapping_metadata_bhs *mmb) { - struct inode *inode = file->f_mapping->host; - int err; - int ret; + struct buffer_head *bh; + int err = 0; + struct blk_plug plug; + LIST_HEAD(tmp); - err = file_write_and_wait_range(file, start, end); - if (err) - return err; + if (!mmb_has_buffers(mmb)) + return 0; - ret = sync_mapping_buffers(inode->i_mapping); - if (!(inode_state_read_once(inode) & I_DIRTY_ALL)) - goto out; - if (datasync && !(inode_state_read_once(inode) & I_DIRTY_DATASYNC)) - goto out; + blk_start_plug(&plug); - err = sync_inode_metadata(inode, 1); - if (ret == 0) - ret = err; + spin_lock(&mmb->lock); + while (!list_empty(&mmb->list)) { + bh = BH_ENTRY(mmb->list.next); + WARN_ON_ONCE(bh->b_mmb != mmb); + __remove_assoc_queue(mmb, bh); + /* Avoid race with mark_buffer_dirty_inode() which does + * a lockless check and we rely on seeing the dirty bit */ + smp_mb(); + if (buffer_dirty(bh) || buffer_locked(bh)) { + list_add(&bh->b_assoc_buffers, &tmp); + bh->b_mmb = mmb; + if (buffer_dirty(bh)) { + get_bh(bh); + spin_unlock(&mmb->lock); + /* + * Ensure any pending I/O completes so that + * write_dirty_buffer() actually writes the + * current contents - it is a noop if I/O is + * still in flight on potentially older + * contents. + */ + write_dirty_buffer(bh, REQ_SYNC); -out: - /* check and advance again to catch errors after syncing out buffers */ - err = file_check_and_advance_wb_err(file); - if (ret == 0) - ret = err; - return ret; -} -EXPORT_SYMBOL(generic_buffers_fsync_noflush); + /* + * Kick off IO for the previous mapping. Note + * that we will not run the very last mapping, + * wait_on_buffer() will do that for us + * through sync_buffer(). + */ + brelse(bh); + spin_lock(&mmb->lock); + } + } + } -/** - * generic_buffers_fsync - generic buffer fsync implementation - * for simple filesystems with no inode lock - * - * @file: file to synchronize - * @start: start offset in bytes - * @end: end offset in bytes (inclusive) - * @datasync: only synchronize essential metadata if true - * - * This is a generic implementation of the fsync method for simple - * filesystems which track all non-inode metadata in the buffers list - * hanging off the address_space structure. This also makes sure that - * a device cache flush operation is called at the end. - */ -int generic_buffers_fsync(struct file *file, loff_t start, loff_t end, - bool datasync) -{ - struct inode *inode = file->f_mapping->host; - int ret; + spin_unlock(&mmb->lock); + blk_finish_plug(&plug); + spin_lock(&mmb->lock); - ret = generic_buffers_fsync_noflush(file, start, end, datasync); - if (!ret) - ret = blkdev_issue_flush(inode->i_sb->s_bdev); - return ret; + while (!list_empty(&tmp)) { + bh = BH_ENTRY(tmp.prev); + get_bh(bh); + __remove_assoc_queue(mmb, bh); + /* Avoid race with mark_buffer_dirty_inode() which does + * a lockless check and we rely on seeing the dirty bit */ + smp_mb(); + if (buffer_dirty(bh)) { + list_add(&bh->b_assoc_buffers, &mmb->list); + bh->b_mmb = mmb; + } + spin_unlock(&mmb->lock); + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + err = -EIO; + brelse(bh); + spin_lock(&mmb->lock); + } + spin_unlock(&mmb->lock); + return err; } -EXPORT_SYMBOL(generic_buffers_fsync); +EXPORT_SYMBOL(mmb_sync); /* * Called when we've recently written block `bblock', and it is known that @@ -672,26 +618,25 @@ void write_boundary_block(struct block_device *bdev, } } -void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) +void mmb_mark_buffer_dirty(struct buffer_head *bh, + struct mapping_metadata_bhs *mmb) { - struct address_space *mapping = inode->i_mapping; - struct address_space *buffer_mapping = bh->b_folio->mapping; - mark_buffer_dirty(bh); - if (!mapping->i_private_data) { - mapping->i_private_data = buffer_mapping; - } else { - BUG_ON(mapping->i_private_data != buffer_mapping); - } - if (!bh->b_assoc_map) { - spin_lock(&buffer_mapping->i_private_lock); - list_move_tail(&bh->b_assoc_buffers, - &mapping->i_private_list); - bh->b_assoc_map = mapping; - spin_unlock(&buffer_mapping->i_private_lock); + if (!bh->b_mmb) { + spin_lock(&mmb->lock); + /* + * For a corrupted filesystem with multiply claimed blocks this + * can fail. Avoid corrupting the linked list in that case. + */ + if (cmpxchg(&bh->b_mmb, NULL, mmb) != NULL) { + spin_unlock(&mmb->lock); + return; + } + list_move_tail(&bh->b_assoc_buffers, &mmb->list); + spin_unlock(&mmb->lock); } } -EXPORT_SYMBOL(mark_buffer_dirty_inode); +EXPORT_SYMBOL(mmb_mark_buffer_dirty); /** * block_dirty_folio - Mark a folio as dirty. @@ -758,153 +703,20 @@ bool block_dirty_folio(struct address_space *mapping, struct folio *folio) EXPORT_SYMBOL(block_dirty_folio); /* - * Write out and wait upon a list of buffers. - * - * We have conflicting pressures: we want to make sure that all - * initially dirty buffers get waited on, but that any subsequently - * dirtied buffers don't. After all, we don't want fsync to last - * forever if somebody is actively writing to the file. - * - * Do this in two main stages: first we copy dirty buffers to a - * temporary inode list, queueing the writes as we go. Then we clean - * up, waiting for those writes to complete. - * - * During this second stage, any subsequent updates to the file may end - * up refiling the buffer on the original inode's dirty list again, so - * there is a chance we will end up with a buffer queued for write but - * not yet completed on that list. So, as a final cleanup we go through - * the osync code to catch these locked, dirty buffers without requeuing - * any newly dirty buffers for write. - */ -static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) -{ - struct buffer_head *bh; - struct address_space *mapping; - int err = 0, err2; - struct blk_plug plug; - LIST_HEAD(tmp); - - blk_start_plug(&plug); - - spin_lock(lock); - while (!list_empty(list)) { - bh = BH_ENTRY(list->next); - mapping = bh->b_assoc_map; - __remove_assoc_queue(bh); - /* Avoid race with mark_buffer_dirty_inode() which does - * a lockless check and we rely on seeing the dirty bit */ - smp_mb(); - if (buffer_dirty(bh) || buffer_locked(bh)) { - list_add(&bh->b_assoc_buffers, &tmp); - bh->b_assoc_map = mapping; - if (buffer_dirty(bh)) { - get_bh(bh); - spin_unlock(lock); - /* - * Ensure any pending I/O completes so that - * write_dirty_buffer() actually writes the - * current contents - it is a noop if I/O is - * still in flight on potentially older - * contents. - */ - write_dirty_buffer(bh, REQ_SYNC); - - /* - * Kick off IO for the previous mapping. Note - * that we will not run the very last mapping, - * wait_on_buffer() will do that for us - * through sync_buffer(). - */ - brelse(bh); - spin_lock(lock); - } - } - } - - spin_unlock(lock); - blk_finish_plug(&plug); - spin_lock(lock); - - while (!list_empty(&tmp)) { - bh = BH_ENTRY(tmp.prev); - get_bh(bh); - mapping = bh->b_assoc_map; - __remove_assoc_queue(bh); - /* Avoid race with mark_buffer_dirty_inode() which does - * a lockless check and we rely on seeing the dirty bit */ - smp_mb(); - if (buffer_dirty(bh)) { - list_add(&bh->b_assoc_buffers, - &mapping->i_private_list); - bh->b_assoc_map = mapping; - } - spin_unlock(lock); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) - err = -EIO; - brelse(bh); - spin_lock(lock); - } - - spin_unlock(lock); - err2 = osync_buffers_list(lock, list); - if (err) - return err; - else - return err2; -} - -/* - * Invalidate any and all dirty buffers on a given inode. We are + * Invalidate any and all dirty buffers on a given buffers list. We are * probably unmounting the fs, but that doesn't mean we have already * done a sync(). Just drop the buffers from the inode list. - * - * NOTE: we take the inode's blockdev's mapping's i_private_lock. Which - * assumes that all the buffers are against the blockdev. */ -void invalidate_inode_buffers(struct inode *inode) +void mmb_invalidate(struct mapping_metadata_bhs *mmb) { - if (inode_has_buffers(inode)) { - struct address_space *mapping = &inode->i_data; - struct list_head *list = &mapping->i_private_list; - struct address_space *buffer_mapping = mapping->i_private_data; - - spin_lock(&buffer_mapping->i_private_lock); - while (!list_empty(list)) - __remove_assoc_queue(BH_ENTRY(list->next)); - spin_unlock(&buffer_mapping->i_private_lock); + if (mmb_has_buffers(mmb)) { + spin_lock(&mmb->lock); + while (!list_empty(&mmb->list)) + __remove_assoc_queue(mmb, BH_ENTRY(mmb->list.next)); + spin_unlock(&mmb->lock); } } -EXPORT_SYMBOL(invalidate_inode_buffers); - -/* - * Remove any clean buffers from the inode's buffer list. This is called - * when we're trying to free the inode itself. Those buffers can pin it. - * - * Returns true if all buffers were removed. - */ -int remove_inode_buffers(struct inode *inode) -{ - int ret = 1; - - if (inode_has_buffers(inode)) { - struct address_space *mapping = &inode->i_data; - struct list_head *list = &mapping->i_private_list; - struct address_space *buffer_mapping = mapping->i_private_data; - - spin_lock(&buffer_mapping->i_private_lock); - while (!list_empty(list)) { - struct buffer_head *bh = BH_ENTRY(list->next); - if (buffer_dirty(bh)) { - ret = 0; - break; - } - __remove_assoc_queue(bh); - } - spin_unlock(&buffer_mapping->i_private_lock); - } - return ret; -} +EXPORT_SYMBOL(mmb_invalidate); /* * Create the appropriate buffers when given a folio for data area and @@ -922,8 +734,7 @@ struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, long offset; struct mem_cgroup *memcg, *old_memcg; - /* The folio lock pins the memcg */ - memcg = folio_memcg(folio); + memcg = get_mem_cgroup_from_folio(folio); old_memcg = set_active_memcg(memcg); head = NULL; @@ -944,6 +755,7 @@ struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, } out: set_active_memcg(old_memcg); + mem_cgroup_put(memcg); return head; /* * In case anything failed, we just free everything we got. @@ -1009,7 +821,6 @@ static sector_t folio_init_buffers(struct folio *folio, do { if (!buffer_mapped(bh)) { - bh->b_end_io = NULL; bh->b_private = NULL; bh->b_bdev = bdev; bh->b_blocknr = block; @@ -1210,12 +1021,18 @@ EXPORT_SYMBOL(mark_buffer_dirty); void mark_buffer_write_io_error(struct buffer_head *bh) { + struct mapping_metadata_bhs *mmb; + set_buffer_write_io_error(bh); /* FIXME: do we need to set this in both places? */ if (bh->b_folio && bh->b_folio->mapping) mapping_set_error(bh->b_folio->mapping, -EIO); - if (bh->b_assoc_map) - mapping_set_error(bh->b_assoc_map, -EIO); + /* Protect us from mmb & inode getting freed while we work on it */ + rcu_read_lock(); + mmb = READ_ONCE(bh->b_mmb); + if (mmb) + mapping_set_error(mmb->mapping, -EIO); + rcu_read_unlock(); } EXPORT_SYMBOL(mark_buffer_write_io_error); @@ -1245,18 +1062,91 @@ EXPORT_SYMBOL(__brelse); void __bforget(struct buffer_head *bh) { clear_buffer_dirty(bh); - if (bh->b_assoc_map) { - struct address_space *buffer_mapping = bh->b_folio->mapping; - - spin_lock(&buffer_mapping->i_private_lock); - list_del_init(&bh->b_assoc_buffers); - bh->b_assoc_map = NULL; - spin_unlock(&buffer_mapping->i_private_lock); - } + remove_assoc_queue(bh); __brelse(bh); } EXPORT_SYMBOL(__bforget); +static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, + gfp_t gfp_mask) +{ + const struct address_space *mapping = folio_mapping(bh->b_folio); + + /* + * The ext4 journal (jbd2) can submit a buffer_head it directly created + * for a non-pagecache page. fscrypt doesn't care about these. + */ + if (!mapping) + return; + fscrypt_set_bio_crypt_ctx(bio, mapping->host, + folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask); +} + +static void __bh_submit(struct buffer_head *bh, blk_opf_t opf, + enum rw_hint write_hint, struct writeback_control *wbc, + bio_end_io_t end_bio) +{ + const enum req_op op = opf & REQ_OP_MASK; + struct bio *bio; + + BUG_ON(!buffer_locked(bh)); + BUG_ON(!buffer_mapped(bh)); + BUG_ON(buffer_delay(bh)); + BUG_ON(buffer_unwritten(bh)); + + /* + * Only clear out a write error when rewriting + */ + if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) + clear_buffer_write_io_error(bh); + + if (buffer_meta(bh)) + opf |= REQ_META; + if (buffer_prio(bh)) + opf |= REQ_PRIO; + + bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); + + if (folio_test_dropbehind(bh->b_folio) && op_is_write(opf)) + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); + + if (IS_ENABLED(CONFIG_FS_ENCRYPTION)) + buffer_set_crypto_ctx(bio, bh, GFP_NOIO); + + bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); + bio->bi_write_hint = write_hint; + + bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); + + bio->bi_end_io = end_bio; + bio->bi_private = bh; + + /* Take care of bh's that straddle the end of the device */ + guard_bio_eod(bio); + + if (wbc) { + wbc_init_bio(wbc, bio); + wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); + } + + blk_crypto_submit_bio(bio); +} + +/** + * bh_submit - Start I/O against a buffer head + * @bh: The buffer head to perform I/O on. + * @opf: Operation and flags for bio. + * @end_io: The routine to call when I/O has completed. + * + * If you need to do I/O on an individual bh (instead of allowing the + * page cache to do I/O on the folio that it is in), call this function. + */ +void bh_submit(struct buffer_head *bh, blk_opf_t opf, bio_end_io_t end_io) +{ + __bh_submit(bh, opf, WRITE_LIFE_NOT_SET, NULL, end_io); +} +EXPORT_SYMBOL(bh_submit); + static struct buffer_head *__bread_slow(struct buffer_head *bh) { lock_buffer(bh); @@ -1264,9 +1154,7 @@ static struct buffer_head *__bread_slow(struct buffer_head *bh) unlock_buffer(bh); return bh; } else { - get_bh(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); wait_on_buffer(bh); if (buffer_uptodate(bh)) return bh; @@ -1816,15 +1704,15 @@ static struct buffer_head *folio_create_buffers(struct folio *folio, /* * While block_write_full_folio is writing back the dirty buffers under - * the page lock, whoever dirtied the buffers may decide to clean them + * the folio lock, whoever dirtied the buffers may decide to clean them * again at any time. We handle that by only looking at the buffer * state inside lock_buffer(). * * If block_write_full_folio() is called for regular writeback - * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a - * locked buffer. This only can happen if someone has written the buffer - * directly, with submit_bh(). At the address_space level PageWriteback - * prevents this contention from occurring. + * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a folio which + * has a locked buffer. This only can happen if someone has written + * the buffer directly, with bh_submit(). At the address_space level + * the folio writeback flag prevents this contention from occurring. * * If block_write_full_folio() is called with wbc->sync_mode == * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this @@ -1910,8 +1798,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, continue; } if (test_clear_buffer_dirty(bh)) { - mark_buffer_async_write_endio(bh, - end_buffer_async_write); + set_buffer_async_write(bh); } else { unlock_buffer(bh); } @@ -1927,8 +1814,9 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, - inode->i_write_hint, wbc); + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); nr_underway++; } bh = next; @@ -1941,7 +1829,7 @@ done: /* * The folio was marked dirty, but the buffers were * clean. Someone wrote them back by hand with - * write_dirty_buffer/submit_bh. A rare case. + * write_dirty_buffer/bh_submit. A rare case. */ folio_end_writeback(folio); @@ -1965,8 +1853,7 @@ recover: if (buffer_mapped(bh) && buffer_dirty(bh) && !buffer_delay(bh)) { lock_buffer(bh); - mark_buffer_async_write_endio(bh, - end_buffer_async_write); + set_buffer_async_write(bh); } else { /* * The buffer may have been set dirty during @@ -1982,8 +1869,9 @@ recover: struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { clear_buffer_dirty(bh); - submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, - inode->i_write_hint, wbc); + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); nr_underway++; } bh = next; @@ -2196,6 +2084,7 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) { size_t block_start, block_end; bool partial = false; + bool uptodate = folio_test_uptodate(folio); unsigned blocksize; struct buffer_head *bh, *head; @@ -2218,6 +2107,8 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) clear_buffer_new(bh); block_start = block_end; + if (uptodate && block_start >= to) + break; bh = bh->b_this_page; } while (bh != head); @@ -2439,9 +2330,33 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) continue; } - mark_buffer_async_read(bh); + /* + * If a folio's buffers are under async readin + * (end_buffer_async_read completion) then there is a + * possibility that another thread of control could lock + * one of the buffers after it has completed but while + * some of the other buffers have not completed. This + * locked buffer would confuse end_buffer_async_read() + * into not unlocking the folio. So the absence of + * BH_Async_Read tells end_buffer_async_read() that this + * buffer is not under async I/O. + * + * The folio comes unlocked when it has no locked + * buffer_async buffers left. + * + * The folio lock prevents anyone starting new async + * I/O reads into any of the buffers. + * + * The writeback flag is used to prevent simultaneous + * writeout of the same folio. + * + * The folio lock prevents anyone from starting writeback + * of a folio which is under read I/O (the writeback + * flag is only ever set on a locked folio). + */ + set_buffer_async_read(bh); if (prev) - submit_bh(REQ_OP_READ, prev); + bh_submit(prev, REQ_OP_READ, bh_end_async_read); prev = bh; } while (iblock++, (bh = bh->b_this_page) != head); @@ -2455,7 +2370,7 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) * in this folio. */ if (prev) - submit_bh(REQ_OP_READ, prev); + bh_submit(prev, REQ_OP_READ, bh_end_async_read); else folio_end_read(folio, !page_error); @@ -2763,70 +2678,6 @@ sector_t generic_block_bmap(struct address_space *mapping, sector_t block, } EXPORT_SYMBOL(generic_block_bmap); -static void end_bio_bh_io_sync(struct bio *bio) -{ - struct buffer_head *bh = bio->bi_private; - - if (unlikely(bio_flagged(bio, BIO_QUIET))) - set_bit(BH_Quiet, &bh->b_state); - - bh->b_end_io(bh, !bio->bi_status); - bio_put(bio); -} - -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint write_hint, - struct writeback_control *wbc) -{ - const enum req_op op = opf & REQ_OP_MASK; - struct bio *bio; - - BUG_ON(!buffer_locked(bh)); - BUG_ON(!buffer_mapped(bh)); - BUG_ON(!bh->b_end_io); - BUG_ON(buffer_delay(bh)); - BUG_ON(buffer_unwritten(bh)); - - /* - * Only clear out a write error when rewriting - */ - if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) - clear_buffer_write_io_error(bh); - - if (buffer_meta(bh)) - opf |= REQ_META; - if (buffer_prio(bh)) - opf |= REQ_PRIO; - - bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); - - fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); - - bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); - bio->bi_write_hint = write_hint; - - bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); - - bio->bi_end_io = end_bio_bh_io_sync; - bio->bi_private = bh; - - /* Take care of bh's that straddle the end of the device */ - guard_bio_eod(bio); - - if (wbc) { - wbc_init_bio(wbc, bio); - wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); - } - - blk_crypto_submit_bio(bio); -} - -void submit_bh(blk_opf_t opf, struct buffer_head *bh) -{ - submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL); -} -EXPORT_SYMBOL(submit_bh); - void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) { lock_buffer(bh); @@ -2834,9 +2685,7 @@ void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) unlock_buffer(bh); return; } - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - submit_bh(REQ_OP_WRITE | op_flags, bh); + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); } EXPORT_SYMBOL(write_dirty_buffer); @@ -2859,9 +2708,7 @@ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) return -EIO; } - get_bh(bh); - bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | op_flags, bh); + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); wait_on_buffer(bh); if (!buffer_uptodate(bh)) return -EIO; @@ -2900,8 +2747,7 @@ drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free) do { struct buffer_head *next = bh->b_this_page; - if (bh->b_assoc_map) - __remove_assoc_queue(bh); + remove_assoc_queue(bh); bh = next; } while (bh != head); *buffers_to_free = head; @@ -3094,9 +2940,7 @@ int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait) BUG_ON(!buffer_locked(bh)); - get_bh(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ | op_flags, bh); + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); if (wait) { wait_on_buffer(bh); if (!buffer_uptodate(bh)) @@ -3138,9 +2982,7 @@ void __bh_read_batch(int nr, struct buffer_head *bhs[], continue; } - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ | op_flags, bh); + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); } } EXPORT_SYMBOL(__bh_read_batch); |