diff options
| -rw-r--r-- | fs/xfs/libxfs/xfs_attr_leaf.c | 14 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_btree_staging.c | 7 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_da_btree.c | 3 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_dquot_buf.c | 47 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_ialloc.c | 2 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_inode_buf.c | 15 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_metadir.c | 64 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_metadir.h | 8 | ||||
| -rw-r--r-- | fs/xfs/libxfs/xfs_rtgroup.c | 58 | ||||
| -rw-r--r-- | fs/xfs/xfs_bmap_util.c | 42 | ||||
| -rw-r--r-- | fs/xfs/xfs_bmap_util.h | 7 | ||||
| -rw-r--r-- | fs/xfs/xfs_buf.c | 392 | ||||
| -rw-r--r-- | fs/xfs/xfs_buf.h | 8 | ||||
| -rw-r--r-- | fs/xfs/xfs_buf_item.c | 10 | ||||
| -rw-r--r-- | fs/xfs/xfs_buf_item_recover.c | 3 | ||||
| -rw-r--r-- | fs/xfs/xfs_dquot_item_recover.c | 1 | ||||
| -rw-r--r-- | fs/xfs/xfs_file.c | 118 | ||||
| -rw-r--r-- | fs/xfs/xfs_fsops.c | 2 | ||||
| -rw-r--r-- | fs/xfs/xfs_inode.c | 4 | ||||
| -rw-r--r-- | fs/xfs/xfs_inode_item_recover.c | 1 | ||||
| -rw-r--r-- | fs/xfs/xfs_log_recover.c | 5 | ||||
| -rw-r--r-- | fs/xfs/xfs_trace.h | 2 | ||||
| -rw-r--r-- | fs/xfs/xfs_trans.c | 16 | ||||
| -rw-r--r-- | fs/xfs/xfs_trans_buf.c | 7 | ||||
| -rw-r--r-- | fs/xfs/xfs_zone_gc.c | 2 |
25 files changed, 492 insertions, 346 deletions
diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 86c5c09a5db4..b6288395f853 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -325,6 +325,13 @@ xfs_attr3_leaf_verify_entry( */ if (ent->flags & XFS_ATTR_LOCAL) { lentry = xfs_attr3_leaf_name_local(leaf, idx); + + /* Validate lentry pointer is within bounds before field access */ + if ((char *)lentry >= buf_end) + return __this_address; + if ((char *)lentry + offsetof(struct xfs_attr_leaf_name_local, nameval) > buf_end) + return __this_address; + namesize = xfs_attr_leaf_entsize_local(lentry->namelen, be16_to_cpu(lentry->valuelen)); name_end = (char *)lentry + namesize; @@ -332,6 +339,13 @@ xfs_attr3_leaf_verify_entry( return __this_address; } else { rentry = xfs_attr3_leaf_name_remote(leaf, idx); + + /* Validate rentry pointer is within bounds before field access */ + if ((char *)rentry >= buf_end) + return __this_address; + if ((char *)rentry + offsetof(struct xfs_attr_leaf_name_remote, name) > buf_end) + return __this_address; + namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); name_end = (char *)rentry + namesize; if (rentry->namelen == 0) diff --git a/fs/xfs/libxfs/xfs_btree_staging.c b/fs/xfs/libxfs/xfs_btree_staging.c index c3c7ea54895a..7314dab4bcfb 100644 --- a/fs/xfs/libxfs/xfs_btree_staging.c +++ b/fs/xfs/libxfs/xfs_btree_staging.c @@ -248,11 +248,10 @@ xfs_btree_bload_drop_buf( return 0; /* - * Mark this buffer XBF_DONE (i.e. uptodate) so that a subsequent - * xfs_buf_read will not pointlessly reread the contents from the disk. + * Mark this buffer uptodate so that a subsequent xfs_buf_read will + * not pointlessly reread the contents from the disk. */ - bp->b_flags |= XBF_DONE; - + xfs_buf_set_uptodate(bp); xfs_buf_delwri_queue_here(bp, buffers_list); xfs_buf_relse(bp); *bpp = NULL; diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 9debb95d86fa..f190c088591b 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -2354,8 +2354,7 @@ xfs_da_grow_inode_int( * If we didn't get it and the block might work if fragmented, * try without the CONTIG flag. Loop until we get it all. */ - mapp = kmalloc(sizeof(*mapp) * count, - GFP_KERNEL | __GFP_NOFAIL); + mapp = kmalloc_objs(*mapp, count, GFP_KERNEL | __GFP_NOFAIL); for (b = *bno, mapi = 0; b < *bno + count; ) { c = (int)(*bno + count - b); nmap = min(XFS_BMAP_MAX_NMAP, c); diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c index bbada0d3cc08..77954d1d924c 100644 --- a/fs/xfs/libxfs/xfs_dquot_buf.c +++ b/fs/xfs/libxfs/xfs_dquot_buf.c @@ -252,8 +252,8 @@ xfs_dquot_buf_read_verify( /* * readahead errors are silent and simply leave the buffer as !done so a real * read will then be run with the xfs_dquot_buf_ops verifier. See - * xfs_inode_buf_verify() for why we use EIO and ~XBF_DONE here rather than - * reporting the failure. + * xfs_inode_buf_verify() for why we use EIO here rather than reporting the + * failure. */ static void xfs_dquot_buf_readahead_verify( @@ -262,10 +262,8 @@ xfs_dquot_buf_readahead_verify( struct xfs_mount *mp = bp->b_mount; if (!xfs_dquot_buf_verify_crc(mp, bp, true) || - xfs_dquot_buf_verify(mp, bp, true) != NULL) { + xfs_dquot_buf_verify(mp, bp, true) != NULL) xfs_buf_ioerror(bp, -EIO); - bp->b_flags &= ~XBF_DONE; - } } /* @@ -416,6 +414,15 @@ xfs_dqinode_load( return 0; } +static int +xfs_dqinode_init( + struct xfs_metadir_update *upd, + void *priv) +{ + xfs_trans_log_inode(upd->tp, upd->ip, XFS_ILOG_CORE); + return 0; +} + /* Create a metadata directory quota inode. */ int xfs_dqinode_metadir_create( @@ -428,35 +435,9 @@ xfs_dqinode_metadir_create( .metafile_type = xfs_dqinode_metafile_type(type), .path = xfs_dqinode_path(type), }; - int error; - - error = xfs_metadir_start_create(&upd); - if (error) - return error; - error = xfs_metadir_create(&upd, S_IFREG); - if (error) - goto out_cancel; - - xfs_trans_log_inode(upd.tp, upd.ip, XFS_ILOG_CORE); - - error = xfs_metadir_commit(&upd); - if (error) - goto out_irele; - - xfs_finish_inode_setup(upd.ip); - *ipp = upd.ip; - return 0; - -out_cancel: - xfs_metadir_cancel(&upd, error); -out_irele: - /* Have to finish setting up the inode to ensure it's deleted. */ - if (upd.ip) { - xfs_finish_inode_setup(upd.ip); - xfs_irele(upd.ip); - } - return error; + return xfs_metadir_create_file(&upd, S_IFREG, xfs_dqinode_init, NULL, + ipp); } #ifndef __KERNEL__ diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index ffcdd1f691fd..58dac4d505ba 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -413,7 +413,7 @@ xfs_ialloc_inode_init( xfs_trans_ordered_buf(tp, fbuf); } } else { - fbuf->b_flags |= XBF_DONE; + xfs_buf_set_uptodate(fbuf); xfs_buf_delwri_queue(fbuf, buffer_list); xfs_buf_relse(fbuf); } diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 336ef843f2fe..e4c3f7b24e95 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -29,12 +29,14 @@ * has not had the inode cores stamped into it. Hence for readahead, the buffer * may be potentially invalid. * - * If the readahead buffer is invalid, we need to mark it with an error and - * clear the DONE status of the buffer so that a followup read will re-read it - * from disk. We don't report the error otherwise to avoid warnings during log - * recovery and we don't get unnecessary panics on debug kernels. We use EIO here - * because all we want to do is say readahead failed; there is no-one to report - * the error to, so this will distinguish it from a non-ra verifier failure. + * If the readahead buffer is invalid, we need to mark it with an error so that a + * followup read will re-read it from disk. + * + * We don't report the error otherwise to avoid warnings during log recovery and + * we don't get unnecessary panics on debug kernels. Use EIO here because all + * we want to do is say readahead failed; there is no-one to report the error + * to, so this will distinguish it from a non-ra verifier failure. + * * Changes to this readahead error behaviour also need to be reflected in * xfs_dquot_buf_readahead_verify(). */ @@ -64,7 +66,6 @@ xfs_inode_buf_verify( if (unlikely(!di_ok || XFS_TEST_ERROR(mp, XFS_ERRTAG_ITOBP_INOTOBP))) { if (readahead) { - bp->b_flags &= ~XBF_DONE; xfs_buf_ioerror(bp, -EIO); return; } diff --git a/fs/xfs/libxfs/xfs_metadir.c b/fs/xfs/libxfs/xfs_metadir.c index 74c4596ee4cf..7c6b086b73db 100644 --- a/fs/xfs/libxfs/xfs_metadir.c +++ b/fs/xfs/libxfs/xfs_metadir.c @@ -182,7 +182,7 @@ xfs_metadir_teardown( * Begin the process of creating a metadata file by allocating transactions * and taking whatever resources we're going to need. */ -int +static int xfs_metadir_start_create( struct xfs_metadir_update *upd) { @@ -236,7 +236,7 @@ out_teardown: * a negative error code. If an inode is passed back, the caller must finish * setting up the inode before releasing it. */ -int +static int xfs_metadir_create( struct xfs_metadir_update *upd, umode_t mode) @@ -425,7 +425,7 @@ xfs_metadir_commit( } /* Cancel a metadir update and unlock/drop all resources. */ -void +static void xfs_metadir_cancel( struct xfs_metadir_update *upd, int error) @@ -438,48 +438,64 @@ xfs_metadir_cancel( xfs_metadir_teardown(upd, error); } -/* Create a metadata for the last component of the path. */ int -xfs_metadir_mkdir( - struct xfs_inode *dp, - const char *path, +xfs_metadir_create_file( + struct xfs_metadir_update *upd, + umode_t mode, + xfs_metadir_createfn create, + void *priv, struct xfs_inode **ipp) { - struct xfs_metadir_update upd = { - .dp = dp, - .path = path, - .metafile_type = XFS_METAFILE_DIR, - }; int error; - if (xfs_is_shutdown(dp->i_mount)) + if (xfs_is_shutdown(upd->dp->i_mount)) return -EIO; - /* Allocate a transaction to create the last directory. */ - error = xfs_metadir_start_create(&upd); + error = xfs_metadir_start_create(upd); if (error) return error; - /* Create the subdirectory and take our reference. */ - error = xfs_metadir_create(&upd, S_IFDIR); + error = xfs_metadir_create(upd, mode); if (error) goto out_cancel; - error = xfs_metadir_commit(&upd); + if (create) { + error = create(upd, priv); + if (error) + goto out_cancel; + } + + error = xfs_metadir_commit(upd); if (error) goto out_irele; - xfs_finish_inode_setup(upd.ip); - *ipp = upd.ip; + xfs_finish_inode_setup(upd->ip); + *ipp = upd->ip; return 0; out_cancel: - xfs_metadir_cancel(&upd, error); + xfs_metadir_cancel(upd, error); out_irele: /* Have to finish setting up the inode to ensure it's deleted. */ - if (upd.ip) { - xfs_finish_inode_setup(upd.ip); - xfs_irele(upd.ip); + if (upd->ip) { + xfs_finish_inode_setup(upd->ip); + xfs_irele(upd->ip); } return error; } + +/* Create a metadata for the last component of the path. */ +int +xfs_metadir_mkdir( + struct xfs_inode *dp, + const char *path, + struct xfs_inode **ipp) +{ + struct xfs_metadir_update upd = { + .dp = dp, + .path = path, + .metafile_type = XFS_METAFILE_DIR, + }; + + return xfs_metadir_create_file(&upd, S_IFDIR, NULL, NULL, ipp); +} diff --git a/fs/xfs/libxfs/xfs_metadir.h b/fs/xfs/libxfs/xfs_metadir.h index bfecac7d3d14..e434b9d1c932 100644 --- a/fs/xfs/libxfs/xfs_metadir.h +++ b/fs/xfs/libxfs/xfs_metadir.h @@ -32,14 +32,16 @@ int xfs_metadir_load(struct xfs_trans *tp, struct xfs_inode *dp, const char *path, enum xfs_metafile_type metafile_type, struct xfs_inode **ipp); -int xfs_metadir_start_create(struct xfs_metadir_update *upd); -int xfs_metadir_create(struct xfs_metadir_update *upd, umode_t mode); +typedef int (*xfs_metadir_createfn)(struct xfs_metadir_update *upd, void *priv); + +int xfs_metadir_create_file(struct xfs_metadir_update *upd, umode_t mode, + xfs_metadir_createfn create, void *priv, + struct xfs_inode **ipp); int xfs_metadir_start_link(struct xfs_metadir_update *upd); int xfs_metadir_link(struct xfs_metadir_update *upd); int xfs_metadir_commit(struct xfs_metadir_update *upd); -void xfs_metadir_cancel(struct xfs_metadir_update *upd, int error); int xfs_metadir_mkdir(struct xfs_inode *dp, const char *path, struct xfs_inode **ipp); diff --git a/fs/xfs/libxfs/xfs_rtgroup.c b/fs/xfs/libxfs/xfs_rtgroup.c index c85d50953218..fe7222bbe449 100644 --- a/fs/xfs/libxfs/xfs_rtgroup.c +++ b/fs/xfs/libxfs/xfs_rtgroup.c @@ -517,6 +517,25 @@ xfs_rtginode_irele( *ipp = NULL; } +struct xfs_rtginode_create { + struct xfs_rtgroup *rtg; + enum xfs_rtg_inodes type; + bool init; +}; + +static int +xfs_rtginode_init( + struct xfs_metadir_update *upd, + void *priv) +{ + struct xfs_rtginode_create *rc = priv; + const struct xfs_rtginode_ops *ops = &xfs_rtginode_ops[rc->type]; + + xfs_rtginode_lockdep_setup(upd->ip, rtg_rgno(rc->rtg), rc->type); + upd->ip->i_projid = rtg_rgno(rc->rtg); + return ops->create(rc->rtg, upd->ip, upd->tp, rc->init); +} + /* Add a metadata inode for a realtime rmap btree. */ int xfs_rtginode_create( @@ -526,6 +545,11 @@ xfs_rtginode_create( { const struct xfs_rtginode_ops *ops = &xfs_rtginode_ops[type]; struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_rtginode_create rc = { + .rtg = rtg, + .type = type, + .init = init, + }; struct xfs_metadir_update upd = { .dp = mp->m_rtdirip, .metafile_type = ops->metafile_type, @@ -544,38 +568,8 @@ xfs_rtginode_create( if (!upd.path) return -ENOMEM; - error = xfs_metadir_start_create(&upd); - if (error) - goto out_path; - - error = xfs_metadir_create(&upd, S_IFREG); - if (error) - goto out_cancel; - - xfs_rtginode_lockdep_setup(upd.ip, rtg_rgno(rtg), type); - - upd.ip->i_projid = rtg_rgno(rtg); - error = ops->create(rtg, upd.ip, upd.tp, init); - if (error) - goto out_cancel; - - error = xfs_metadir_commit(&upd); - if (error) - goto out_path; - - kfree(upd.path); - xfs_finish_inode_setup(upd.ip); - rtg->rtg_inodes[type] = upd.ip; - return 0; - -out_cancel: - xfs_metadir_cancel(&upd, error); - /* Have to finish setting up the inode to ensure it's deleted. */ - if (upd.ip) { - xfs_finish_inode_setup(upd.ip); - xfs_irele(upd.ip); - } -out_path: + error = xfs_metadir_create_file(&upd, S_IFREG, xfs_rtginode_init, &rc, + &rtg->rtg_inodes[type]); kfree(upd.path); return error; } diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index c88b9ade7389..268d159339d0 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -642,11 +642,26 @@ out_unlock: return error; } +/* + * Allocate space or convert extents for a file according to @mode: + * + * XFS_ALLOC_FILE_SPACE_PREALLOC: + * Preallocate unwritten extents over holes across the range and mark the inode + * as preallocated. + * + * XFS_ALLOC_FILE_SPACE_WRITE_ZEROES: + * Allocate written extents over holes and convert unwritten extents in the + * range to written extents, initialising both to contain zeroes. + * + * This function does not update the file size; callers that extend the file + * are responsible for updating it once the extents are allocated. + */ int xfs_alloc_file_space( struct xfs_inode *ip, xfs_off_t offset, - xfs_off_t len) + xfs_off_t len, + enum xfs_alloc_file_space_mode mode) { xfs_mount_t *mp = ip->i_mount; xfs_off_t count; @@ -657,6 +672,7 @@ xfs_alloc_file_space( int rt; xfs_trans_t *tp; xfs_bmbt_irec_t imaps[1], *imapp; + uint32_t bmapi_flags, nr_exts; int error; if (xfs_is_always_cow_inode(ip)) @@ -674,6 +690,19 @@ xfs_alloc_file_space( if (len <= 0) return -EINVAL; + switch (mode) { + case XFS_ALLOC_FILE_SPACE_PREALLOC: + bmapi_flags = XFS_BMAPI_PREALLOC; + nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT; + break; + case XFS_ALLOC_FILE_SPACE_WRITE_ZEROES: + bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; + nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT; + break; + default: + return -EINVAL; + } + rt = XFS_IS_REALTIME_INODE(ip); extsz = xfs_get_extsz_hint(ip); @@ -733,8 +762,7 @@ xfs_alloc_file_space( if (error) break; - error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, - XFS_IEXT_ADD_NOSPLIT_CNT); + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts); if (error) goto error; @@ -748,7 +776,7 @@ xfs_alloc_file_space( * will eventually reach the requested range. */ error = xfs_bmapi_write(tp, ip, startoffset_fsb, - allocatesize_fsb, XFS_BMAPI_PREALLOC, 0, imapp, + allocatesize_fsb, bmapi_flags, 0, imapp, &nimaps); if (error) { if (error != -ENOSR) @@ -759,8 +787,10 @@ xfs_alloc_file_space( allocatesize_fsb -= imapp->br_blockcount; } - ip->i_diflags |= XFS_DIFLAG_PREALLOC; - xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + if (mode == XFS_ALLOC_FILE_SPACE_PREALLOC) { + ip->i_diflags |= XFS_DIFLAG_PREALLOC; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + } error = xfs_trans_commit(tp); xfs_iunlock(ip, XFS_ILOCK_EXCL); diff --git a/fs/xfs/xfs_bmap_util.h b/fs/xfs/xfs_bmap_util.h index eaaf094154b9..c7b48b2602f2 100644 --- a/fs/xfs/xfs_bmap_util.h +++ b/fs/xfs/xfs_bmap_util.h @@ -55,8 +55,13 @@ int xfs_bmap_last_extent(struct xfs_trans *tp, struct xfs_inode *ip, int *is_empty); /* preallocation and hole punch interface */ +enum xfs_alloc_file_space_mode { + XFS_ALLOC_FILE_SPACE_PREALLOC, + XFS_ALLOC_FILE_SPACE_WRITE_ZEROES, +}; + int xfs_alloc_file_space(struct xfs_inode *ip, xfs_off_t offset, - xfs_off_t len); + xfs_off_t len, enum xfs_alloc_file_space_mode mode); int xfs_free_file_space(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t len, struct xfs_zone_alloc_ctx *ac); int xfs_collapse_file_space(struct xfs_inode *, xfs_off_t offset, diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 17b9d643e1a8..ee7c2e9c0340 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -55,6 +55,29 @@ static inline bool xfs_buf_is_uncached(struct xfs_buf *bp) return bp->b_rhash_key == XFS_BUF_DADDR_NULL; } +static inline void +xfs_buf_set_flags( + struct xfs_buf *bp, + unsigned int flags) +{ + WRITE_ONCE(bp->b_flags, bp->b_flags | flags); +} + +static inline void +xfs_buf_clear_flags( + struct xfs_buf *bp, + unsigned int flags) +{ + WRITE_ONCE(bp->b_flags, bp->b_flags & ~flags); +} + +void +xfs_buf_set_uptodate( + struct xfs_buf *bp) +{ + xfs_buf_set_flags(bp, XBF_DONE); +} + /* * When we mark a buffer stale, we remove the buffer from the LRU and clear the * b_lru_ref count so that the buffer is freed immediately when the buffer @@ -69,14 +92,14 @@ xfs_buf_stale( { ASSERT(xfs_buf_islocked(bp)); - bp->b_flags |= XBF_STALE; + xfs_buf_set_flags(bp, XBF_STALE); /* * Clear the delwri status so that a delwri queue walker will not * flush this buffer to disk now that it is stale. The delwri queue has * a reference to the buffer, so this is safe to do. */ - bp->b_flags &= ~_XBF_DELWRI_Q; + xfs_buf_clear_flags(bp, _XBF_DELWRI_Q); spin_lock(&bp->b_lockref.lock); atomic_set(&bp->b_lru_ref, 0); @@ -85,6 +108,14 @@ xfs_buf_stale( spin_unlock(&bp->b_lockref.lock); } +void +xfs_buf_clear_stale( + struct xfs_buf *bp) +{ + ASSERT(bp->b_flags & XBF_STALE); + xfs_buf_clear_flags(bp, XBF_STALE); +} + static void xfs_buf_free_callback( struct callback_head *cb) @@ -159,7 +190,7 @@ xfs_buf_alloc_kmem( bp->b_addr = NULL; return -ENOMEM; } - bp->b_flags |= _XBF_KMEM; + xfs_buf_set_flags(bp, _XBF_KMEM); trace_xfs_buf_backing_kmem(bp, _RET_IP_); return 0; } @@ -282,15 +313,8 @@ xfs_buf_alloc( * specifically set by later operations on the buffer. */ flags &= ~(XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD); - - /* - * A new buffer is held and locked by the owner. This ensures that the - * buffer is owned by the caller and racing RCU lookups right after - * inserting into the hash table are safe (and will have to wait for - * the unlock to do anything non-trivial). - */ lockref_init(&bp->b_lockref); - sema_init(&bp->b_sema, 0); /* held, no waiters */ + sema_init(&bp->b_sema, 1); /* unlocked */ atomic_set(&bp->b_lru_ref, 1); init_completion(&bp->b_iowait); INIT_LIST_HEAD(&bp->b_lru); @@ -298,7 +322,7 @@ xfs_buf_alloc( INIT_LIST_HEAD(&bp->b_li_list); bp->b_target = target; bp->b_mount = target->bt_mount; - bp->b_flags = flags; + WRITE_ONCE(bp->b_flags, flags); bp->b_rhash_key = map[0].bm_bn; bp->b_length = 0; bp->b_map_count = nmaps; @@ -427,39 +451,31 @@ xfs_buf_find_lock( return -ENOENT; } ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0); - bp->b_flags &= _XBF_KMEM; + xfs_buf_clear_flags(bp, ~_XBF_KMEM); bp->b_ops = NULL; } return 0; } -static inline int +static inline struct xfs_buf * xfs_buf_lookup( struct xfs_buftarg *btp, - struct xfs_buf_map *map, - xfs_buf_flags_t flags, - struct xfs_buf **bpp) + struct xfs_buf_map *map) { struct xfs_buf *bp; - int error; rcu_read_lock(); bp = rhashtable_lookup(&btp->bt_hash, map, xfs_buf_hash_params); if (!bp || !lockref_get_not_dead(&bp->b_lockref)) { rcu_read_unlock(); - return -ENOENT; + XFS_STATS_INC(btp->bt_mount, xb_miss_locked); + return NULL; } rcu_read_unlock(); - error = xfs_buf_find_lock(bp, flags); - if (error) { - xfs_buf_rele(bp); - return error; - } - - trace_xfs_buf_find(bp, flags, _RET_IP_); - *bpp = bp; - return 0; + trace_xfs_buf_find(bp, _RET_IP_); + XFS_STATS_INC(btp->bt_mount, xb_get_locked); + return bp; } /* @@ -469,7 +485,6 @@ xfs_buf_lookup( static int xfs_buf_find_insert( struct xfs_buftarg *btp, - struct xfs_perag *pag, struct xfs_buf_map *cmap, struct xfs_buf_map *map, int nmaps, @@ -482,10 +497,13 @@ xfs_buf_find_insert( error = xfs_buf_alloc(btp, map, nmaps, flags, &new_bp); if (error) - goto out_drop_pag; + return error; /* The new buffer keeps the perag reference until it is freed. */ - new_bp->b_pag = pag; + if (!xfs_buftarg_is_mem(btp)) { + new_bp->b_pag = xfs_perag_get(btp->bt_mount, + xfs_daddr_to_agno(btp->bt_mount, cmap->bm_bn)); + } retry: rcu_read_lock(); @@ -507,11 +525,7 @@ retry: goto retry; } rcu_read_unlock(); - error = xfs_buf_find_lock(bp, flags); - if (error) - xfs_buf_rele(bp); - else - *bpp = bp; + *bpp = bp; goto out_free_buf; } rcu_read_unlock(); @@ -520,39 +534,25 @@ retry: return 0; out_free_buf: + if (new_bp->b_pag) + xfs_perag_put(new_bp->b_pag); xfs_buf_free(new_bp); -out_drop_pag: - if (pag) - xfs_perag_put(pag); return error; } -static inline struct xfs_perag * -xfs_buftarg_get_pag( - struct xfs_buftarg *btp, - const struct xfs_buf_map *map) -{ - struct xfs_mount *mp = btp->bt_mount; - - if (xfs_buftarg_is_mem(btp)) - return NULL; - return xfs_perag_get(mp, xfs_daddr_to_agno(mp, map->bm_bn)); -} - /* * Assembles a buffer covering the specified range. The code is optimised for * cache hits, as metadata intensive workloads will see 3 orders of magnitude * more hits than misses. */ -int -xfs_buf_get_map( +static int +xfs_find_get_buf( struct xfs_buftarg *btp, struct xfs_buf_map *map, int nmaps, xfs_buf_flags_t flags, struct xfs_buf **bpp) { - struct xfs_perag *pag; struct xfs_buf *bp = NULL; struct xfs_buf_map cmap = { .bm_bn = map[0].bm_bn }; int error; @@ -567,46 +567,50 @@ xfs_buf_get_map( if (error) return error; - pag = xfs_buftarg_get_pag(btp, &cmap); - - error = xfs_buf_lookup(btp, &cmap, flags, &bp); - if (error && error != -ENOENT) - goto out_put_perag; - /* cache hits always outnumber misses by at least 10:1 */ + bp = xfs_buf_lookup(btp, &cmap); if (unlikely(!bp)) { - XFS_STATS_INC(btp->bt_mount, xb_miss_locked); - if (flags & XBF_INCORE) - goto out_put_perag; - - /* xfs_buf_find_insert() consumes the perag reference. */ - error = xfs_buf_find_insert(btp, pag, &cmap, map, nmaps, - flags, &bp); + return -ENOENT; + error = xfs_buf_find_insert(btp, &cmap, map, nmaps, flags, &bp); if (error) return error; - } else { - XFS_STATS_INC(btp->bt_mount, xb_get_locked); - if (pag) - xfs_perag_put(pag); } + *bpp = bp; + return 0; +} + +int +xfs_buf_get_map( + struct xfs_buftarg *btp, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + int error; + + ASSERT(!(flags & ~(XBF_TRYLOCK | XBF_INCORE | XBF_LIVESCAN))); + ASSERT(!(flags & XBF_LIVESCAN) || (flags & XBF_INCORE)); + /* - * Clear b_error if this is a lookup from a caller that doesn't expect - * valid data to be found in the buffer. + * Zero the buffer and clear b_error as xfs_buf_get_map callers don't + * expect valid data to be found in the buffer. */ - if (!(flags & XBF_READ)) - xfs_buf_ioerror(bp, 0); + error = xfs_find_get_buf(btp, map, nmaps, flags, bpp); + if (error) + return error; + error = xfs_buf_find_lock(*bpp, flags); + if (error) { + xfs_buf_rele(*bpp); + return error; + } XFS_STATS_INC(btp->bt_mount, xb_get); - trace_xfs_buf_get(bp, flags, _RET_IP_); - *bpp = bp; + trace_xfs_buf_get(*bpp, flags, _RET_IP_); + xfs_buf_ioerror(*bpp, 0); return 0; - -out_put_perag: - if (pag) - xfs_perag_put(pag); - return error; } int @@ -615,47 +619,13 @@ _xfs_buf_read( { ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL); - bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD | XBF_DONE); - bp->b_flags |= XBF_READ; + xfs_buf_clear_flags(bp, XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD | + XBF_DONE); + xfs_buf_set_flags(bp, XBF_READ); xfs_buf_submit(bp); return xfs_buf_iowait(bp); } -/* - * Reverify a buffer found in cache without an attached ->b_ops. - * - * If the caller passed an ops structure and the buffer doesn't have ops - * assigned, set the ops and use it to verify the contents. If verification - * fails, clear XBF_DONE. We assume the buffer has no recorded errors and is - * already in XBF_DONE state on entry. - * - * Under normal operations, every in-core buffer is verified on read I/O - * completion. There are two scenarios that can lead to in-core buffers without - * an assigned ->b_ops. The first is during log recovery of buffers on a V4 - * filesystem, though these buffers are purged at the end of recovery. The - * other is online repair, which intentionally reads with a NULL buffer ops to - * run several verifiers across an in-core buffer in order to establish buffer - * type. If repair can't establish that, the buffer will be left in memory - * with NULL buffer ops. - */ -static int -xfs_buf_reverify( - struct xfs_buf *bp, - const struct xfs_buf_ops *ops) -{ - ASSERT(bp->b_flags & XBF_DONE); - ASSERT(bp->b_error == 0); - - if (!ops || bp->b_ops) - return 0; - - bp->b_ops = ops; - bp->b_ops->verify_read(bp); - if (bp->b_error) - bp->b_flags &= ~XBF_DONE; - return bp->b_error; -} - int xfs_buf_read_map( struct xfs_buftarg *target, @@ -669,31 +639,81 @@ xfs_buf_read_map( struct xfs_buf *bp; int error; - ASSERT(!(flags & (XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD))); + ASSERT(!(flags & ~XBF_TRYLOCK)); flags |= XBF_READ; *bpp = NULL; - error = xfs_buf_get_map(target, map, nmaps, flags, &bp); + error = xfs_find_get_buf(target, map, nmaps, flags, &bp); if (error) return error; + error = xfs_buf_find_lock(bp, flags); + if (error) { + xfs_buf_rele(bp); + return error; + } trace_xfs_buf_read(bp, flags, _RET_IP_); - if (!(bp->b_flags & XBF_DONE)) { + if (bp->b_flags & XBF_DONE) { + ASSERT(bp->b_error == 0); + + /* + * If the caller passed an ops structure and the buffer doesn't + * have ops assigned yet, set the ops and use them to verify the + * buffer contents. + * + * Under normal operations, every in-core buffer is verified on + * read I/O completion, but there are two scenarios that can + * lead to in-core buffers without an assigned ->b_ops: + * + * 1) During log recovery of buffers on a V4 filesystem. + * These buffers are purged at the end of recovery, though. + * 2) Oonline repair intentionally reads with a NULL buffer + * ops to run several verifiers across an in-core buffer in + * order to establish buffer type. If repair can't + * establish that, the buffer will be left in memory with + * NULL buffer ops. + */ + if (ops && !bp->b_ops) { + bp->b_ops = ops; + bp->b_ops->verify_read(bp); + /* + * If verification failed, clear XBF_DONE as we assume + * that buffers have no recorded errors when in XBF_DONE + * state. + */ + error = bp->b_error; + if (error) + xfs_buf_clear_flags(bp, XBF_DONE); + } + + /* We do not want read in the flags */ + xfs_buf_clear_flags(bp, XBF_READ); + } else { /* Initiate the buffer read and wait. */ XFS_STATS_INC(target->bt_mount, xb_get_read); bp->b_ops = ops; error = _xfs_buf_read(bp); - } else { - /* Buffer already read; all we need to do is check it. */ - error = xfs_buf_reverify(bp, ops); - - /* We do not want read in the flags */ - bp->b_flags &= ~XBF_READ; - ASSERT(bp->b_ops != NULL || ops == NULL); } + if (error) + goto out_ioerror; + + *bpp = bp; + return 0; + +out_ioerror: + /* + * Check against log shutdown for error reporting because metadata + * writeback may require a read first and we need to report errors in + * metadata writeback until the log is shut down. High level + * transaction read functions already check against mount shutdown, so + * we only need to be concerned about low level/ IO interactions here. + */ + if (!xlog_is_shutdown(target->bt_mount->m_log)) + xfs_buf_ioerror_alert(bp, fa); + /* * If we've had a read error, then the contents of the buffer are * invalid and should not be used. To ensure that a followup read tries @@ -703,30 +723,14 @@ xfs_buf_read_map( * future cache lookups will also treat it as an empty, uninitialised * buffer. */ - if (error) { - /* - * Check against log shutdown for error reporting because - * metadata writeback may require a read first and we need to - * report errors in metadata writeback until the log is shut - * down. High level transaction read functions already check - * against mount shutdown, anyway, so we only need to be - * concerned about low level IO interactions here. - */ - if (!xlog_is_shutdown(target->bt_mount->m_log)) - xfs_buf_ioerror_alert(bp, fa); - - bp->b_flags &= ~XBF_DONE; - xfs_buf_stale(bp); - xfs_buf_relse(bp); - - /* bad CRC means corrupted metadata */ - if (error == -EFSBADCRC) - error = -EFSCORRUPTED; - return error; - } + xfs_buf_clear_flags(bp, XBF_DONE); + xfs_buf_stale(bp); + xfs_buf_relse(bp); - *bpp = bp; - return 0; + /* bad CRC means corrupted metadata */ + if (error == -EFSBADCRC) + return -EFSCORRUPTED; + return error; } /* @@ -750,21 +754,36 @@ xfs_buf_readahead_map( if (xfs_buftarg_is_mem(target)) return; - if (xfs_buf_get_map(target, map, nmaps, flags | XBF_TRYLOCK, &bp)) + if (xfs_find_get_buf(target, map, nmaps, flags, &bp)) return; - trace_xfs_buf_readahead(bp, 0, _RET_IP_); - if (bp->b_flags & XBF_DONE) { - xfs_buf_reverify(bp, ops); - xfs_buf_relse(bp); - return; - } + /* + * Do a lockless fast path check for a valid uptodate buffer and avoid + * locking entirely in this case. + */ + if ((READ_ONCE(bp->b_flags) & (XBF_DONE | XBF_STALE)) == XBF_DONE) + goto out_rele; + + /* Otherwise lock the buffer to stabilize the state */ + if (!xfs_buf_trylock(bp)) + goto out_rele; + + /* Let the actual reader deal with stale buffers. */ + if (bp->b_flags & (XBF_STALE | XBF_DONE)) + goto out_unlock; + + trace_xfs_buf_readahead(bp, 0, _RET_IP_); XFS_STATS_INC(target->bt_mount, xb_get_read); bp->b_ops = ops; - bp->b_flags &= ~(XBF_WRITE | XBF_DONE); - bp->b_flags |= flags; + xfs_buf_clear_flags(bp, XBF_WRITE | XBF_DONE); + xfs_buf_set_flags(bp, flags); percpu_counter_inc(&target->bt_readahead_count); xfs_buf_submit(bp); + return; +out_unlock: + xfs_buf_unlock(bp); +out_rele: + xfs_buf_rele(bp); } /* @@ -794,7 +813,7 @@ xfs_buf_read_uncached( ASSERT(bp->b_map_count == 1); bp->b_rhash_key = XFS_BUF_DADDR_NULL; bp->b_maps[0].bm_bn = daddr; - bp->b_flags |= XBF_READ; + xfs_buf_set_flags(bp, XBF_READ); bp->b_ops = ops; xfs_buf_submit(bp); @@ -818,9 +837,11 @@ xfs_buf_get_uncached( DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks); error = xfs_buf_alloc(target, &map, 1, 0, bpp); - if (!error) - trace_xfs_buf_get_uncached(*bpp, _RET_IP_); - return error; + if (error) + return error; + xfs_buf_lock(*bpp); + trace_xfs_buf_get_uncached(*bpp, _RET_IP_); + return 0; } /* @@ -1042,7 +1063,7 @@ xfs_buf_ioend_handle_error( * We're not going to bother about retrying this during recovery. * One strike! */ - if (bp->b_flags & _XBF_LOGRECOVERY) { + if (mp->m_log && xlog_in_recovery(mp->m_log)) { xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); return false; } @@ -1086,14 +1107,14 @@ xfs_buf_ioend_handle_error( resubmit: xfs_buf_ioerror(bp, 0); - bp->b_flags |= (XBF_DONE | XBF_WRITE_FAIL); + xfs_buf_set_flags(bp, XBF_DONE | XBF_WRITE_FAIL); reinit_completion(&bp->b_iowait); xfs_buf_submit(bp); return true; out_stale: xfs_buf_stale(bp); - bp->b_flags |= XBF_DONE; - bp->b_flags &= ~XBF_WRITE; + xfs_buf_set_flags(bp, XBF_DONE); + xfs_buf_clear_flags(bp, XBF_WRITE); trace_xfs_buf_error_relse(bp, _RET_IP_); return false; } @@ -1118,7 +1139,7 @@ xfs_buf_ioend( if (!bp->b_error && bp->b_ops) bp->b_ops->verify_read(bp); if (!bp->b_error) - bp->b_flags |= XBF_DONE; + xfs_buf_set_flags(bp, XBF_DONE); if (bp->b_flags & XBF_READ_AHEAD) percpu_counter_dec(&bp->b_target->bt_readahead_count); } else { @@ -1128,8 +1149,8 @@ xfs_buf_ioend( return; } } else { - bp->b_flags &= ~XBF_WRITE_FAIL; - bp->b_flags |= XBF_DONE; + xfs_buf_clear_flags(bp, XBF_WRITE_FAIL); + xfs_buf_set_flags(bp, XBF_DONE); } /* clear the retry state */ @@ -1149,8 +1170,7 @@ xfs_buf_ioend( bp->b_iodone(bp); } - bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD | - _XBF_LOGRECOVERY); + xfs_buf_clear_flags(bp, XBF_READ | XBF_WRITE | XBF_READ_AHEAD); if (async) xfs_buf_relse(bp); } @@ -1196,8 +1216,8 @@ xfs_buf_fail( { ASSERT(xfs_buf_islocked(bp)); - bp->b_flags |= XBF_ASYNC; - bp->b_flags &= ~XBF_DONE; + xfs_buf_set_flags(bp, XBF_ASYNC); + xfs_buf_clear_flags(bp, XBF_DONE); xfs_buf_stale(bp); xfs_buf_ioerror(bp, -EIO); xfs_buf_ioend(bp); @@ -1211,9 +1231,9 @@ xfs_bwrite( ASSERT(xfs_buf_islocked(bp)); - bp->b_flags |= XBF_WRITE; - bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | - XBF_DONE); + xfs_buf_set_flags(bp, XBF_WRITE); + xfs_buf_clear_flags(bp, XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | + XBF_DONE); xfs_buf_submit(bp); error = xfs_buf_iowait(bp); @@ -1404,7 +1424,7 @@ xfs_buf_submit( return; ioerror: - bp->b_flags &= ~XBF_DONE; + xfs_buf_clear_flags(bp, XBF_DONE); xfs_buf_stale(bp); end_io: if (bp->b_flags & XBF_ASYNC) @@ -1815,7 +1835,7 @@ xfs_buf_delwri_cancel( bp = list_first_entry(list, struct xfs_buf, b_list); xfs_buf_lock(bp); - bp->b_flags &= ~_XBF_DELWRI_Q; + xfs_buf_clear_flags(bp, _XBF_DELWRI_Q); xfs_buf_list_del(bp); xfs_buf_relse(bp); } @@ -1860,7 +1880,7 @@ xfs_buf_delwri_queue( * might get readded to a delwri list after the synchronous writeout, in * which case we need just need to re-add the flag here. */ - bp->b_flags |= _XBF_DELWRI_Q; + xfs_buf_set_flags(bp, _XBF_DELWRI_Q); if (list_empty(&bp->b_list)) { xfs_buf_hold(bp); list_add_tail(&bp->b_list, list); @@ -1937,8 +1957,8 @@ xfs_buf_delwri_submit_prep( } trace_xfs_buf_delwri_split(bp, _RET_IP_); - bp->b_flags &= ~_XBF_DELWRI_Q; - bp->b_flags |= XBF_WRITE; + xfs_buf_clear_flags(bp, _XBF_DELWRI_Q); + xfs_buf_set_flags(bp, XBF_WRITE); return true; } @@ -1979,7 +1999,7 @@ xfs_buf_delwri_submit_nowait( } if (!xfs_buf_delwri_submit_prep(bp)) continue; - bp->b_flags |= XBF_ASYNC; + xfs_buf_set_flags(bp, XBF_ASYNC); xfs_buf_list_del(bp); xfs_buf_submit(bp); } @@ -2012,7 +2032,7 @@ xfs_buf_delwri_submit( xfs_buf_lock(bp); if (!xfs_buf_delwri_submit_prep(bp)) continue; - bp->b_flags &= ~XBF_ASYNC; + xfs_buf_clear_flags(bp, XBF_ASYNC); list_move_tail(&bp->b_list, &wait_list); xfs_buf_submit(bp); } diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index 79cc9c3f0254..a4729253b56f 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -34,9 +34,6 @@ struct xfs_buf; #define XBF_STALE (1u << 6) /* buffer has been staled, do not find it */ #define XBF_WRITE_FAIL (1u << 7) /* async writes have failed on this buffer */ -/* buffer type flags for write callbacks */ -#define _XBF_LOGRECOVERY (1u << 18)/* log recovery buffer */ - /* flags used only internally */ #define _XBF_KMEM (1u << 21)/* backed by heap memory */ #define _XBF_DELWRI_Q (1u << 22)/* buffer on a delwri queue */ @@ -61,7 +58,6 @@ typedef unsigned int xfs_buf_flags_t; { XBF_DONE, "DONE" }, \ { XBF_STALE, "STALE" }, \ { XBF_WRITE_FAIL, "WRITE_FAIL" }, \ - { _XBF_LOGRECOVERY, "LOG_RECOVERY" }, \ { _XBF_KMEM, "KMEM" }, \ { _XBF_DELWRI_Q, "DELWRI_Q" }, \ /* The following interface flags should never be set */ \ @@ -305,7 +301,9 @@ static inline void xfs_buf_zero(struct xfs_buf *bp, size_t boff, size_t bsize) memset(bp->b_addr + boff, 0, bsize); } -extern void xfs_buf_stale(struct xfs_buf *bp); +void xfs_buf_set_uptodate(struct xfs_buf *bp); +void xfs_buf_stale(struct xfs_buf *bp); +void xfs_buf_clear_stale(struct xfs_buf *bp); /* Delayed Write Buffer Routines */ extern void xfs_buf_delwri_cancel(struct list_head *); diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 1f055cd6732e..1a4ef34af8d5 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -1066,6 +1066,8 @@ void xfs_buf_item_done( struct xfs_buf *bp) { + struct xfs_buf_log_item *bip = bp->b_log_item; + /* * If we are forcibly shutting down, this may well be off the AIL * already. That's because we simulate the log-committed callbacks to @@ -1078,8 +1080,8 @@ xfs_buf_item_done( * Note that log recovery writes might have buffer items that are not on * the AIL even when the file system is not shut down. */ - xfs_trans_ail_delete(&bp->b_log_item->bli_item, - (bp->b_flags & _XBF_LOGRECOVERY) ? 0 : - SHUTDOWN_CORRUPT_INCORE); - xfs_buf_item_relse(bp->b_log_item); + xfs_trans_ail_delete(&bip->bli_item, + xlog_in_recovery(bip->bli_item.li_log) ? + 0 : SHUTDOWN_CORRUPT_INCORE); + xfs_buf_item_relse(bip); } diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c index 240deb3f7827..57929f115055 100644 --- a/fs/xfs/xfs_buf_item_recover.c +++ b/fs/xfs/xfs_buf_item_recover.c @@ -448,7 +448,6 @@ xlog_recover_validate_buf_type( if (bp->b_ops) { struct xfs_buf_log_item *bip; - bp->b_flags |= _XBF_LOGRECOVERY; xfs_buf_item_init(bp, mp); bip = bp->b_log_item; bip->bli_item.li_lsn = current_lsn; @@ -1122,7 +1121,6 @@ xlog_recover_buf_commit_pass2( xfs_buf_lock(rtsb_bp); xfs_buf_hold(rtsb_bp); xfs_update_rtsb(rtsb_bp, bp); - rtsb_bp->b_flags |= _XBF_LOGRECOVERY; xfs_buf_delwri_queue(rtsb_bp, buffer_list); xfs_buf_relse(rtsb_bp); } @@ -1164,7 +1162,6 @@ out_writebuf: error = xfs_bwrite(bp); } else { ASSERT(bp->b_mount == mp); - bp->b_flags |= _XBF_LOGRECOVERY; xfs_buf_delwri_queue(bp, buffer_list); } diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c index 63bc9ab7d947..b8854316ac75 100644 --- a/fs/xfs/xfs_dquot_item_recover.c +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -168,7 +168,6 @@ xlog_recover_dquot_commit_pass2( ASSERT(dq_f->qlf_size == 2); ASSERT(bp->b_mount == mp); - bp->b_flags |= _XBF_LOGRECOVERY; xfs_buf_delwri_queue(bp, buffer_list); out_release: diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 768cabf6250b..7bff07e31cbd 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -166,17 +166,23 @@ xfs_file_fsync( } /* - * If we only have a single device, and the log force about was - * a no-op we might have to flush the data device cache here. - * This can only happen for fdatasync/O_DSYNC if we were overwriting - * an already allocated file and thus do not have any metadata to - * commit. + * If the log force was a no-op, we may still need to flush the + * file data target cache here. This can happen for fdatasync/O_DSYNC + * when no metadata needed to be committed. + * + * Use the inode's actual file data target rather than assuming the + * main data device. Realtime inodes with a separate realtime device + * are flushed before the log force, so this fallback only applies + * when the file data target is the same as the log target. */ - if (!log_flushed && !XFS_IS_REALTIME_INODE(ip) && - mp->m_logdev_targp == mp->m_ddev_targp) { - err2 = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev); - if (err2 && !error) - error = err2; + if (!log_flushed) { + struct xfs_buftarg *file_targp = xfs_inode_buftarg(ip); + + if (mp->m_logdev_targp == file_targp) { + err2 = blkdev_issue_flush(file_targp->bt_bdev); + if (err2 && !error) + error = err2; + } } return error; @@ -1370,6 +1376,84 @@ xfs_falloc_force_zero( return XFS_TEST_ERROR(ip->i_mount, XFS_ERRTAG_FORCE_ZERO_RANGE); } +static int +xfs_falloc_write_zeroes( + struct file *file, + int mode, + loff_t offset, + loff_t len, + struct xfs_zone_alloc_ctx *ac) +{ + struct inode *inode = file_inode(file); + struct xfs_inode *ip = XFS_I(inode); + loff_t new_size = 0; + int error; + + /* + * XXX: There is an issue with bigrtalloc inodes where there can be blocks + * that are written after the EOF block. This breaks the promise of no + * written blocks past EOF. Return EOPNOTSUPP until it is fixed. + */ + if (xfs_is_always_cow_inode(ip) || xfs_inode_has_bigrtalloc(ip) || + !bdev_write_zeroes_unmap_sectors(xfs_inode_buftarg(ip)->bt_bdev)) + return -EOPNOTSUPP; + + error = xfs_falloc_newsize(file, mode, offset, len, &new_size); + if (error) + return error; + + /* + * + * |----------|----------|----------|----------|----------| + * ^ ^ ^ ^ ^ ^ + * | | | | | | + * | offset | | end | + * | | | | + * offset_rd offset_ru end_rd end_ru + * + * xfs_free_file_space() punches the aligned interior offset_ru -> end_rd + * to holes and byte-zeroes the in-range parts of the partial edge blocks, + * offset -> offset_ru and end_rd -> end. xfs_zero_range() only touches + * already-written blocks here; it skips holes and unwritten extents, so + * unallocated/unwritten edge blocks are left for the allocation below. + */ + error = xfs_free_file_space(ip, offset, len, ac); + if (error) + return error; + + /* + * Publish the new size while the punched range is still a hole, then + * fill it with written zeroes. Like the other fallocate modes we use + * xfs_falloc_setsize(), but it must run *before* we convert the range + * to written extents: xfs_setattr_size() zeroes [old EOF, new size) via + * xfs_zero_range(), which skips holes, so there is nothing to re-zero. + * It will also writeback partial EOF block before the on-disk size is + * logged. + * Note: extending the size before allocating means a failure below + * leaves the file larger with unallocated holes in the new range. + * That is safe as holes within i_size read back as zeroes and expose + * no stale data while the error is propagated to the caller. + */ + error = xfs_falloc_setsize(file, new_size); + if (error) + return error; + + /* + * Allocate written, zeroed extents across the range. xfs_alloc_file_space() + * rounds outward to block granularity: + * - holes (the punched interior and any unallocated edge block) are + * allocated and zeroed; + * - unwritten extents (including unwritten edge blocks) are converted to + * written and zeroed; + * - Already written edge blocks are skipped. The out-of-range bytes of + * a written edge block keep their data (offset_rd -> offset and + * end -> end_rd); their in-range bytes (offset -> offset_ru and + * end_ru -> end were already zeroed by xfs_free_file_space(). + */ + return xfs_alloc_file_space(ip, offset, len, + XFS_ALLOC_FILE_SPACE_WRITE_ZEROES); +} + /* * Punch a hole and prealloc the range. We use a hole punch rather than * unwritten extent conversion for two reasons: @@ -1408,7 +1492,8 @@ xfs_falloc_zero_range( len = round_up(offset + len, blksize) - round_down(offset, blksize); offset = round_down(offset, blksize); - error = xfs_alloc_file_space(ip, offset, len); + error = xfs_alloc_file_space(ip, offset, len, + XFS_ALLOC_FILE_SPACE_PREALLOC); } if (error) return error; @@ -1434,7 +1519,8 @@ xfs_falloc_unshare_range( if (error) return error; - error = xfs_alloc_file_space(XFS_I(inode), offset, len); + error = xfs_alloc_file_space(XFS_I(inode), offset, len, + XFS_ALLOC_FILE_SPACE_PREALLOC); if (error) return error; return xfs_falloc_setsize(file, new_size); @@ -1462,7 +1548,8 @@ xfs_falloc_allocate_range( if (error) return error; - error = xfs_alloc_file_space(XFS_I(inode), offset, len); + error = xfs_alloc_file_space(XFS_I(inode), offset, len, + XFS_ALLOC_FILE_SPACE_PREALLOC); if (error) return error; return xfs_falloc_setsize(file, new_size); @@ -1472,7 +1559,7 @@ xfs_falloc_allocate_range( (FALLOC_FL_ALLOCATE_RANGE | FALLOC_FL_KEEP_SIZE | \ FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | \ FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE | \ - FALLOC_FL_UNSHARE_RANGE) + FALLOC_FL_UNSHARE_RANGE | FALLOC_FL_WRITE_ZEROES) STATIC long __xfs_file_fallocate( @@ -1524,6 +1611,9 @@ __xfs_file_fallocate( case FALLOC_FL_ALLOCATE_RANGE: error = xfs_falloc_allocate_range(file, mode, offset, len); break; + case FALLOC_FL_WRITE_ZEROES: + error = xfs_falloc_write_zeroes(file, mode, offset, len, ac); + break; default: error = -EOPNOTSUPP; break; diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 67624a804a7f..21114bb6d4ff 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -501,7 +501,7 @@ xfs_do_force_shutdown( return; } if (mp->m_sb_bp) - mp->m_sb_bp->b_flags |= XBF_DONE; + xfs_buf_set_uptodate(mp->m_sb_bp); if (flags & SHUTDOWN_FORCE_UMOUNT) xfs_alert(mp, "User initiated shutdown received."); diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 15279d22a894..030a7c8f2c12 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1753,7 +1753,7 @@ xfs_ifree_cluster( * attachment may occur in xfs_inode_item_precommit() after we * have marked this buffer stale. If this buffer was not in * memory before xfs_ifree_cluster() started, it will not be - * marked XBF_DONE and this will cause problems later in + * marked uptodate and this will cause problems later in * xfs_inode_item_precommit() when we trip over a (stale, !done) * buffer to attached to the transaction. * @@ -1766,7 +1766,7 @@ xfs_ifree_cluster( * fail. We can acheive this by adding a write verifier to the * buffer. */ - bp->b_flags |= XBF_DONE; + xfs_buf_set_uptodate(bp); bp->b_ops = &xfs_inode_buf_ops; /* diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index 169a8fe3bf0a..1d2319ad15c5 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -586,7 +586,6 @@ out_owner_change: } ASSERT(bp->b_mount == mp); - bp->b_flags |= _XBF_LOGRECOVERY; xfs_buf_delwri_queue(bp, buffer_list); out_release: diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index fdb011e6ef60..e7e49529658b 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -3279,9 +3279,8 @@ xlog_do_recovery_pass( * checkpoints at this start LSN. * * Note: Shutting down the filesystem will result in the - * delwri submission marking all the buffers stale, - * completing them and cleaning up _XBF_LOGRECOVERY - * state without doing any IO. + * delwri submission marking all the buffers stale and + * completing them without doing any IO. */ xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); } diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index aeb89ac53bf1..f333c938fbd9 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -792,6 +792,7 @@ DEFINE_BUF_EVENT(xfs_buf_backing_folio); DEFINE_BUF_EVENT(xfs_buf_backing_kmem); DEFINE_BUF_EVENT(xfs_buf_backing_vmalloc); DEFINE_BUF_EVENT(xfs_buf_backing_fallback); +DEFINE_BUF_EVENT(xfs_buf_find); /* not really buffer traces, but the buf provides useful information */ DEFINE_BUF_EVENT(xfs_btree_corrupt); @@ -837,7 +838,6 @@ DECLARE_EVENT_CLASS(xfs_buf_flags_class, DEFINE_EVENT(xfs_buf_flags_class, name, \ TP_PROTO(struct xfs_buf *bp, unsigned flags, unsigned long caller_ip), \ TP_ARGS(bp, flags, caller_ip)) -DEFINE_BUF_FLAGS_EVENT(xfs_buf_find); DEFINE_BUF_FLAGS_EVENT(xfs_buf_get); DEFINE_BUF_FLAGS_EVENT(xfs_buf_read); DEFINE_BUF_FLAGS_EVENT(xfs_buf_readahead); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 7bfbd9f6f0df..1b36cf12d4e3 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1029,6 +1029,15 @@ xfs_trans_roll( * duplicate transaction that gets returned. */ error = __xfs_trans_commit(tp, true); + + tp = *tpp; + /* + * __xfs_trans_commit cleared the NOFS flag by calling into + * xfs_trans_free. Set it again here before doing memory + * allocations. + */ + xfs_trans_set_context(tp); + if (error) return error; @@ -1040,13 +1049,6 @@ xfs_trans_roll( * either nothing be locked across this call, or that anything that is * locked be logged in the prior and the next transactions. */ - tp = *tpp; - /* - * __xfs_trans_commit cleared the NOFS flag by calling into - * xfs_trans_free. Set it again here before doing memory - * allocations. - */ - xfs_trans_set_context(tp); error = xfs_log_regrant(tp->t_mountp, tp->t_ticket); if (error) return error; diff --git a/fs/xfs/xfs_trans_buf.c b/fs/xfs/xfs_trans_buf.c index 7e17b93fe9ad..1e025848811a 100644 --- a/fs/xfs/xfs_trans_buf.c +++ b/fs/xfs/xfs_trans_buf.c @@ -140,7 +140,7 @@ xfs_trans_get_buf_map( ASSERT(xfs_buf_islocked(bp)); if (xfs_is_shutdown(tp->t_mountp)) { xfs_buf_stale(bp); - bp->b_flags |= XBF_DONE; + xfs_buf_set_uptodate(bp); } ASSERT(bp->b_transp == tp); @@ -482,7 +482,7 @@ xfs_trans_dirty_buf( * item from the AIL and free it when the buffer is flushed * to disk. */ - bp->b_flags |= XBF_DONE; + xfs_buf_set_uptodate(bp); ASSERT(atomic_read(&bip->bli_refcount) > 0); @@ -494,8 +494,7 @@ xfs_trans_dirty_buf( */ if (bip->bli_flags & XFS_BLI_STALE) { bip->bli_flags &= ~XFS_BLI_STALE; - ASSERT(bp->b_flags & XBF_STALE); - bp->b_flags &= ~XBF_STALE; + xfs_buf_clear_stale(bp); bip->__bli_format.blf_flags &= ~XFS_BLF_CANCEL; } bip->bli_flags |= XFS_BLI_DIRTY | XFS_BLI_LOGGED; diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c index 7ab8f2218c6a..d0b85179a3d2 100644 --- a/fs/xfs/xfs_zone_gc.c +++ b/fs/xfs/xfs_zone_gc.c @@ -812,7 +812,7 @@ xfs_zone_gc_split_write( split_sectors = bio_split_rw_at(&chunk->bio, lim, &nsegs, lim->max_zone_append_sectors << SECTOR_SHIFT); - if (!split_sectors) + if (split_sectors <= 0) return NULL; /* ensure the split chunk is still block size aligned */ |